Add knowledge base: B站视频《在2000多米高空 建座"会飞的"发电站》

- 新增 knowledge-base/ 目录
- 保存央视新闻视频:在2000多米高空 建座"会飞的"发电站
- 完善 obsidian-kb skill:添加知乎 Playwright 提取器
- 更新依赖和文档
This commit is contained in:
OpenClaw Agent
2026-02-07 19:06:38 +08:00
parent 027b82eb28
commit aa206a1679
11 changed files with 1397 additions and 2 deletions
@@ -0,0 +1,17 @@
from markdownify import MarkdownConverter as _MarkdownConverter
class MarkdownConverter:
"""HTML 转 Markdown 转换器,保留完整格式"""
def __init__(self):
self.converter = _MarkdownConverter(
heading_style="ATX",
bullets="*,-",
code_blocks=True,
tables=True,
strip=["script", "style", "nav", "footer", "aside"]
)
def convert(self, html: str) -> str:
"""转换 HTML 为 Markdown"""
return self.converter.convert(html)
+30 -2
View File
@@ -51,5 +51,33 @@ class ZhihuSource(BaseSource):
def get_transcript(self, content: Content) -> Optional[str]:
"""获取文章内容"""
# TODO: 实现文章抓取
return None
try:
# 优先使用 Playwright 提取
from .zhihu_playwright_extractor import ZhihuPlaywrightExtractor
extractor = ZhihuPlaywrightExtractor()
data = extractor.extract(content.url)
if data and data.get('content'):
# 更新元数据
if data.get('title'):
content.title = data['title']
if data.get('author'):
content.author = data['author']
# 返回 Markdown 内容
return data['content']
except ImportError as e:
# Playwright 未安装
import logging
logger = logging.getLogger(__name__)
logger.warning(f"Playwright 未安装,使用 HTTP 方案: {e}")
except Exception as e:
# Playwright 提取失败
import logging
logger = logging.getLogger(__name__)
logger.error(f"Playwright 提取失败: {e}")
# 降级到原有 HTTP 方案(保留现有逻辑)
return self._fetch_with_http(content.url)
@@ -0,0 +1,385 @@
"""
知乎 Playwright 内容提取器
使用 Playwright 浏览器自动化工具抓取知乎内容
"""
import re
import sys
import os
from typing import Dict, Optional, Any
from urllib.parse import urlparse
# 添加当前目录到 Python 路径以支持绝对导入
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
try:
from .markdown_converter import MarkdownConverter
except ImportError:
from markdown_converter import MarkdownConverter
class PlaywrightBrowser:
"""Playwright 浏览器管理器"""
def __init__(self, headless: bool = True, timeout: int = 30):
self.headless = headless
self.timeout = timeout * 1000
self.browser = None
self.context = None
self.page = None
def start(self):
"""启动浏览器"""
from playwright.sync_api import sync_playwright
self.playwright = sync_playwright().start()
self.browser = self.playwright.chromium.launch(headless=self.headless)
self.context = self.browser.new_context(
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
)
self.page = self.context.new_page()
self.page.set_default_timeout(self.timeout)
def close(self):
"""关闭浏览器"""
if self.page:
self.page.close()
if self.context:
self.context.close()
if self.browser:
self.browser.close()
if hasattr(self, 'playwright'):
self.playwright.stop()
def goto(self, url: str, wait_until: str = "networkidle"):
"""导航到指定URL"""
return self.page.goto(url, wait_until=wait_until)
def wait_for_selector(self, selector: str, timeout: int = None):
"""等待选择器出现"""
timeout_ms = timeout * 1000 if timeout else self.timeout
return self.page.wait_for_selector(selector, timeout=timeout_ms)
def query_selector(self, selector: str):
"""查询选择器"""
return self.page.query_selector(selector)
def query_selector_all(self, selector: str):
"""查询所有匹配的选择器"""
return self.page.query_selector_all(selector)
def evaluate(self, script: str):
"""执行 JavaScript 代码"""
return self.page.evaluate(script)
class ZhihuPlaywrightExtractor:
"""知乎内容提取器 - 使用 Playwright"""
URL_PATTERNS = {
'answer': r'zhihu\.com/question/\d+/answer/(\d+)',
'article': r'zhuanlan\.zhihu\.com/p/(\d+)',
'question': r'zhihu\.com/question/(\d+)(?:$|/)',
}
def __init__(self, headless: bool = True, timeout: int = 30):
"""
初始化提取器
Args:
headless: 是否使用无头浏览器
timeout: 超时时间(秒)
"""
self.browser = PlaywrightBrowser(headless=headless, timeout=timeout)
self.markdown_converter = MarkdownConverter()
def identify_url_type(self, url: str) -> Optional[str]:
"""
识别 URL 类型
Returns:
'answer', 'article', 'question' 或 None
"""
for url_type, pattern in self.URL_PATTERNS.items():
if re.search(pattern, url):
return url_type
return None
def extract_id(self, url: str) -> Optional[str]:
"""
从 URL 中提取内容 ID
Returns:
内容 ID 或 None
"""
url_type = self.identify_url_type(url)
if url_type:
pattern = self.URL_PATTERNS[url_type]
match = re.search(pattern, url)
if match:
return match.group(1)
return None
def _extract_text_content(self, element) -> str:
"""提取元素的文本内容"""
if element is None:
return ''
return element.inner_text() if hasattr(element, 'inner_text') else ''
def _extract_html_content(self, element) -> str:
"""提取元素的 HTML 内容"""
if element is None:
return ''
return element.inner_html() if hasattr(element, 'inner_html') else ''
def _extract_answer_content(self) -> Dict[str, Any]:
"""提取回答内容"""
result = {
'type': 'answer',
'title': '',
'author': '',
'content': '',
'content_markdown': '',
'publish_time': '',
'vote_up': 0,
}
# 打印页面标题用于调试
page_title = self.browser.page.title()
print(f"页面标题: {page_title}")
# 尝试保存页面快照用于调试
try:
import os
debug_dir = '/home/admin/.openclaw/workspace/debug'
os.makedirs(debug_dir, exist_ok=True)
self.browser.page.screenshot(path=os.path.join(debug_dir, 'zhihu_screenshot.png'))
with open(os.path.join(debug_dir, 'zhihu_page.html'), 'w', encoding='utf-8') as f:
f.write(self.browser.page.content())
print(f"页面快照已保存到 {debug_dir}")
except Exception as e:
print(f"保存页面快照时出错: {e}")
# 尝试多个可能的选择器
selectors = {
'title': ['h1.QuestionHeader-title', 'h1', '.QuestionHeader-main'],
'author': ['.AuthorInfo-name', '.AuthorInfo .Text', '[itemprop="name"]'],
'content': ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist] article'],
'time': ['.ContentItem-time', '[itemprop="dateCreated"]'],
'vote': ['.VoteButton--up .VoteButton-label', '.VoteButton--up']
}
for selector in selectors['title']:
title_elem = self.browser.query_selector(selector)
if title_elem:
result['title'] = self._extract_text_content(title_elem)
print(f"找到标题(使用选择器 {selector}): {result['title'][:50]}...")
break
for selector in selectors['author']:
author_elem = self.browser.query_selector(selector)
if author_elem:
result['author'] = self._extract_text_content(author_elem)
print(f"找到作者(使用选择器 {selector}): {result['author']}")
break
for selector in selectors['content']:
content_elem = self.browser.query_selector(selector)
if content_elem:
html_content = self._extract_html_content(content_elem)
result['content'] = html_content
result['content_markdown'] = self.markdown_converter.convert(html_content)
print(f"找到内容(使用选择器 {selector}),长度: {len(result['content_markdown'])}")
break
for selector in selectors['time']:
time_elem = self.browser.query_selector(selector)
if time_elem:
result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem)
break
for selector in selectors['vote']:
vote_elem = self.browser.query_selector(selector)
if vote_elem:
vote_text = self._extract_text_content(vote_elem)
try:
result['vote_up'] = int(vote_text.replace(',', ''))
except (ValueError, AttributeError):
pass
break
return result
def _extract_article_content(self) -> Dict[str, Any]:
"""提取文章内容"""
result = {
'type': 'article',
'title': '',
'author': '',
'content': '',
'content_markdown': '',
'publish_time': '',
'vote_up': 0,
}
# 提取文章标题
title_elem = self.browser.query_selector('.Post-Title')
if title_elem:
result['title'] = self._extract_text_content(title_elem)
# 提取作者信息
author_elem = self.browser.query_selector('.AuthorInfo-name')
if author_elem:
result['author'] = self._extract_text_content(author_elem)
# 提取文章内容
content_elem = self.browser.query_selector('.Post-RichText')
if content_elem:
html_content = self._extract_html_content(content_elem)
result['content'] = html_content
result['content_markdown'] = self.markdown_converter.convert(html_content)
# 提取发布时间
time_elem = self.browser.query_selector('.ContentItem-time')
if time_elem:
result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem)
# 提取点赞数
vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label')
if vote_elem:
vote_text = self._extract_text_content(vote_elem)
try:
result['vote_up'] = int(vote_text.replace(',', ''))
except (ValueError, AttributeError):
pass
return result
def _extract_question_content(self) -> Dict[str, Any]:
"""提取问题内容(问题页本身)"""
result = {
'type': 'question',
'title': '',
'author': '',
'content': '',
'content_markdown': '',
'publish_time': '',
'vote_up': 0,
}
# 提取问题标题
title_elem = self.browser.query_selector('h1.QuestionHeader-title')
if title_elem:
result['title'] = self._extract_text_content(title_elem)
# 提取问题详情
detail_elem = self.browser.query_selector('.QuestionHeader-detail')
if detail_elem:
html_content = self._extract_html_content(detail_elem)
result['content'] = html_content
result['content_markdown'] = self.markdown_converter.convert(html_content)
# 问题的作者通常是提问者
author_elem = self.browser.query_selector('.AuthorInfo-name')
if author_elem:
result['author'] = self._extract_text_content(author_elem)
return result
def extract(self, url: str) -> Optional[Dict[str, Any]]:
"""
从 URL 提取知乎内容
Args:
url: 知乎 URL
Returns:
包含提取内容的字典,结构如下:
{
'type': 'answer' | 'article' | 'question',
'title': str,
'author': str,
'content': str (HTML),
'content_markdown': str (Markdown),
'publish_time': str,
'vote_up': int,
}
如果提取失败返回 None
"""
try:
# 识别 URL 类型
url_type = self.identify_url_type(url)
if not url_type:
print(f"无法识别 URL 类型: {url}")
return None
print(f"URL 类型: {url_type}")
print(f"启动浏览器...")
# 启动浏览器
self.browser.start()
print(f"访问页面: {url}")
# 访问页面
self.browser.goto(url, wait_until="domcontentloaded")
print("等待页面加载...")
import time
time.sleep(3) # 等待 JavaScript 执行
# 尝试多个可能的选择器
if url_type == 'answer':
selectors = ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist]']
print(f"尝试查找选择器: {selectors}")
result = self._extract_answer_content()
elif url_type == 'article':
selectors = ['.Post-RichText', '.Post-Title']
print(f"尝试查找选择器: {selectors}")
result = self._extract_article_content()
else: # question
selectors = ['.QuestionHeader-title', 'h1']
print(f"尝试查找选择器: {selectors}")
result = self._extract_question_content()
# 添加额外信息
result['url'] = url
result['content_id'] = self.extract_id(url)
print(f"提取成功! 标题: {result['title'][:50]}...")
return result
except Exception as e:
print(f"提取知乎内容时出错: {e}")
import traceback
traceback.print_exc()
return None
finally:
# 关闭浏览器
self.browser.close()
def __enter__(self):
"""上下文管理器入口"""
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""上下文管理器出口"""
self.browser.close()
return False
# 便捷函数
def extract_zhihu_content(url: str, headless: bool = True, timeout: int = 30) -> Optional[Dict[str, Any]]:
"""
便捷函数:提取知乎内容
Args:
url: 知乎 URL
headless: 是否使用无头浏览器
timeout: 超时时间(秒)
Returns:
提取的内容字典
"""
with ZhihuPlaywrightExtractor(headless=headless, timeout=timeout) as extractor:
return extractor.extract(url)