From a69225ff2bc993953fb578a3049025f0de8f31a1 Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Sun, 8 Feb 2026 05:56:27 +0800 Subject: [PATCH] =?UTF-8?q?=E7=9F=A5=E4=B9=8E=E6=8F=90=E5=8F=96=E5=99=A8?= =?UTF-8?q?=EF=BC=9A=E4=BB=8E=20Playwright=20=E8=BF=81=E7=A7=BB=E5=88=B0?= =?UTF-8?q?=20DrissionPage=20=E6=96=B9=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 使用 DrissionPage 替代 Playwright - 实现 Cookie 管理和反爬绕过 - 支持回答/文章/问题三种内容类型 - 灵活配置（文件/环境变量/配置文件） - 更新文档和配置文件 --- skills/obsidian-kb/README.md | 76 +-- skills/obsidian-kb/SKILL.md | 82 ++- skills/obsidian-kb/config.yaml | 7 +- skills/obsidian-kb/requirements.txt | 5 +- skills/obsidian-kb/src/config.py | 31 ++ skills/obsidian-kb/src/sources/base.py | 22 +- skills/obsidian-kb/src/sources/zhihu.py | 30 +- .../src/sources/zhihu_drission_extractor.py | 491 ++++++++++++++++++ 8 files changed, 665 insertions(+), 79 deletions(-) create mode 100644 skills/obsidian-kb/src/sources/zhihu_drission_extractor.py diff --git a/skills/obsidian-kb/README.md b/skills/obsidian-kb/README.md index 4c1df3a..6b7496d 100644 --- a/skills/obsidian-kb/README.md +++ b/skills/obsidian-kb/README.md @@ -6,7 +6,7 @@ - **多源内容聚合**: 支持 Bilibili、YouTube、知乎等平台 - **增强型 Bilibili 提取**: 使用 Playwright 绕过反爬检测，支持字幕和弹幕提取 -- **知乎内容提取**: 支持知乎回答和文章，完整 Markdown 格式保留，支持代码块、表格等 +- **知乎内容提取**: 使用 DrissionPage + Cookie，支持回答、文章和问题，完整 Markdown 格式保留 - **智能收件箱**: AI 质量评分，自动分类 - **长视频分段分析**: 30分钟+ 视频自动分段处理 - **无字幕处理**: 智能处理无字幕视频，基于元数据生成内容 @@ -200,13 +200,23 @@ obsidian-kb/ ## 知乎提取 -知乎提取器支持回答和文章两种类型，使用 Playwright 绕过反爬虫保护，返回完整 Markdown 格式。 +支持知乎回答、文章和问题的提取。 -### 支持的内容类型 +**支持的内容类型：** +- ✅ 知乎回答：`https://www.zhihu.com/question/xxx/answer/yyy` +- ✅ 知乎文章：`https://zhuanlan.zhihu.com/p/xxx` +- ✅ 知乎问题：`https://www.zhihu.com/question/xxx` -- **知乎回答**: `https://www.zhihu.com/question/xxx/answer/yyy` -- **知乎文章**: `https://zhuanlan.zhihu.com/p/xxx` -- **知乎问题**: `https://www.zhihu.com/question/xxx` +**技术方案：** +- 使用 DrissionPage 浏览器自动化工具 +- 通过 Cookie 验证登录状态 +- 完整 Markdown 格式输出 + +**优势：** +- 稳定可靠，绕过反爬虫保护 +- 支持无头模式，适合服务器环境 +- Cookie 管理灵活（文件/环境变量/配置文件） +- 内存占用低，每次提取独立启动浏览器 ### 输出格式 @@ -245,36 +255,40 @@ obsidian-kb/ ```yaml zhihu: enabled: true - use_playwright: true # 使用 Playwright（推荐） - cookie_file: "" # 可选：Cookie 文件路径 - timeout: 30 # 超时时间（秒） - max_retries: 2 # 最大重试次数 + use_drissionpage: true # 使用 DrissionPage（推荐） + cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径 + cookie_str: "" # 可选：直接在配置中填入 Cookie JSON + timeout: 30 + headless: true ``` -### 降级机制 +**Cookie 获取方式：** +1. 在有浏览器的机器上登录知乎 +2. 打开开发者工具 (F12) +3. 进入 Application → Cookies → https://www.zhihu.com +4. 导出所有 Cookie 为 JSON 文件 +5. 上传到服务器指定路径 -当 Playwright 提取失败时，系统会自动降级到 HTTP 方案： - -- Playwright 未安装 → 使用 HTTP 请求 -- 网络超时 → 重试后降级 -- 403 错误 → 降级到 HTTP -- 运行时错误 → 降级到 HTTP - -### 安装依赖 - -知乎提取器需要以下依赖： - -```bash -# 使用虚拟环境（推荐） -cd obsidian-kb -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt - -# 安装 Playwright 浏览器 -playwright install chromium +**Cookie 文件格式：** +```json +[ + { + "name": "z_c0", + "value": "xxx", + "domain": ".zhihu.com", + "path": "/", + "expiry": 1234567890, + "secure": true, + "httpOnly": false + } +] ``` +**Cookie 加载优先级：** +1. 配置文件中的 `cookie_str`（最高优先级） +2. 环境变量 `ZHIHU_COOKIE` +3. Cookie 文件（最低优先级） + --- ## 长视频分段策略 diff --git a/skills/obsidian-kb/SKILL.md b/skills/obsidian-kb/SKILL.md index 20c7975..f31e07a 100644 --- a/skills/obsidian-kb/SKILL.md +++ b/skills/obsidian-kb/SKILL.md @@ -1,6 +1,6 @@ --- name: obsidian-kb -description: Multi-source content knowledge base with video processing. Use when user says "保存知识库", "添加到知识库", "处理这个视频", "生成obsidian笔记" etc. Supports B站/bilibili (BV号, b23.tv, www.bilibili.com), YouTube videos, and Zhihu articles. +description: Multi-source content knowledge base with video processing. Use when user says "保存知识库", "添加到知识库", "处理这个视频", "生成obsidian笔记" etc. Supports B站/bilibili (BV号, b23.tv, www.bilibili.com) using Playwright, YouTube videos, and Zhihu articles using DrissionPage. --- # SKILL.md - Obsidian KB Skill @@ -191,12 +191,12 @@ sources: enabled: false api_key: '' # YouTube API Key zhihu: - enabled: false - use_playwright: true # 使用 Playwright 绕过检测（推荐） - cookie: '' # 知乎 Cookie (可选) - cookie_file: '' # Cookie 文件路径 (可选) - timeout: 30 # 超时时间（秒） - max_retries: 2 # 最大重试次数 + enabled: true + use_drissionpage: true # 使用 DrissionPage 绕过检测（推荐） + cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径 + cookie_str: "" # 可选：直接在配置中填入 Cookie JSON + timeout: 30 + headless: true # Bilibili 提取器配置 bilibili_extraction: @@ -219,8 +219,51 @@ export OPENAI_API_KEY="sk-..." export AI_MODEL="gpt-4" export OBSIDIAN_VAULT_PATH="/path/to/vault" export BILIBILI_COOKIE="..." +export ZHIHU_COOKIE='[{"name":"z_c0","value":"xxx"}]' ``` +### Zhihu 提取器配置 + +知乎提取器使用 DrissionPage + Cookie 方式： + +```yaml +sources: + zhihu: + enabled: true + use_drissionpage: true + cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径 + cookie_str: "" # 可选：直接在配置中填入 Cookie JSON + timeout: 30 + headless: true +``` + +**Cookie 获取方式：** +1. 在有浏览器的机器上登录知乎 +2. 打开开发者工具 (F12) +3. 进入 Application → Cookies → https://www.zhihu.com +4. 导出所有 Cookie 为 JSON 文件 +5. 上传到服务器指定路径 + +**Cookie 文件格式：** +```json +[ + { + "name": "z_c0", + "value": "xxx", + "domain": ".zhihu.com", + "path": "/", + "expiry": 1234567890, + "secure": true, + "httpOnly": false + } +] +``` + +**Cookie 加载优先级：** +1. 配置文件中的 `cookie_str`（最高优先级） +2. 环境变量 `ZHIHU_COOKIE` +3. Cookie 文件（最低优先级） + ## 提取器说明 ### Extractors 目录 @@ -276,35 +319,28 @@ else: content["subtitle_text"] = subtitle_text ``` -### ZhihuPlaywrightExtractor 类 +### ZhihuDrissionExtractor 类 -ZhihuPlaywrightExtractor 提供了知乎内容的完整提取功能： +知乎 DrissionPage 提取器提供知乎内容的完整提取功能： **主要方法:** - `extract(url)` - 提取知乎内容（支持回答、文章、问题） - `identify_url_type(url)` - 识别 URL 类型 - `extract_id(url)` - 提取内容 ID +- `_load_cookies()` - 加载 Cookie（支持配置/环境/文件） +- `_apply_cookies(cookies)` - 应用 Cookie 到浏览器 **特性:** -- 使用 Playwright 绕过知乎反爬虫保护 +- 使用 DrissionPage 绕过知乎反爬虫保护 - 支持完整 Markdown 格式（代码块、表格、引用等） - 提取元数据：标题、作者、发布时间、点赞数 -- 智能降级机制（Playwright 失败 → HTTP） -- 每次提取独立启动浏览器，内存占用低 - -**降级逻辑:** - -当 Playwright 提取失败时： - -1. **ImportError** - Playwright 未安装 → 使用 HTTP 方案 -2. **TimeoutError** - 网络超时 → 重试后降级 -3. **403 Forbidden** - 触发反爬虫 → 降级到 HTTP -4. **RuntimeError** - 浏览器崩溃 → 降级到 HTTP +- Cookie 加载优先级：配置 > 环境变量 > 文件 +- 支持无头模式，适合服务器环境 ```python -from sources.zhihu_playwright_extractor import ZhihuPlaywrightExtractor +from obsidian_kb.extractors.zhihu import ZhihuDrissionExtractor -extractor = ZhihuPlaywrightExtractor() +extractor = ZhihuDrissionExtractor() data = extractor.extract("https://www.zhihu.com/question/266309957/answer/2003427891925509507") print(f"Title: {data['title']}") diff --git a/skills/obsidian-kb/config.yaml b/skills/obsidian-kb/config.yaml index b356e29..2c7d2a2 100644 --- a/skills/obsidian-kb/config.yaml +++ b/skills/obsidian-kb/config.yaml @@ -26,10 +26,11 @@ sources: zhihu: enabled: true - use_playwright: true - cookie_file: "" + use_drissionpage: true + cookie_file: ~/.obsidian-kb/zhihu_cookies.json + cookie_str: "" # 可选：直接在配置中填入 timeout: 30 - max_retries: 2 + headless: true chunking: enabled: true diff --git a/skills/obsidian-kb/requirements.txt b/skills/obsidian-kb/requirements.txt index 995e6d2..e89b044 100644 --- a/skills/obsidian-kb/requirements.txt +++ b/skills/obsidian-kb/requirements.txt @@ -10,12 +10,13 @@ openai>=1.0.0 anthropic>=0.7.0 # Bilibili extraction -playwright>=1.40.0 -undetected-playwright>=0.1.0 +# playwright>=1.40.0 +# undetected-playwright>=0.1.0 beautifulsoup4>=4.12.0 lxml>=4.9.0 # Zhihu support +DrissionPage>=4.0.0 markdownify>=0.11.6 # Testing diff --git a/skills/obsidian-kb/src/config.py b/skills/obsidian-kb/src/config.py index 119688a..327e7ec 100644 --- a/skills/obsidian-kb/src/config.py +++ b/skills/obsidian-kb/src/config.py @@ -296,6 +296,37 @@ class Config: """Get bilibili extraction timeout in seconds.""" return self.get('sources.bilibili.extraction.timeout', 30) + # Zhihu DrissionPage configuration + @property + def zhihu_enabled(self) -> bool: + """知乎是否启用""" + return self.get('sources.zhihu.enabled', False) + + @property + def zhihu_use_drissionpage(self) -> bool: + """知乎是否使用 DrissionPage""" + return self.get('sources.zhihu.use_drissionpage', True) + + @property + def zhihu_cookie_file(self) -> Optional[str]: + """知乎 Cookie 文件路径""" + return self.get('sources.zhihu.cookie_file') + + @property + def zhihu_cookie_str(self) -> Optional[str]: + """知乎 Cookie 字符串""" + return self.get('sources.zhihu.cookie_str') + + @property + def zhihu_timeout(self) -> int: + """知乎提取超时时间（秒）""" + return self.get('sources.zhihu.timeout', 30) + + @property + def zhihu_headless(self) -> bool: + """知乎是否使用无头浏览器""" + return self.get('sources.zhihu.headless', True) + def ensure_dirs(self) -> None: """Ensure required directories exist.""" self.vault_path.mkdir(parents=True, exist_ok=True) diff --git a/skills/obsidian-kb/src/sources/base.py b/skills/obsidian-kb/src/sources/base.py index d568747..ce91fc4 100644 --- a/skills/obsidian-kb/src/sources/base.py +++ b/skills/obsidian-kb/src/sources/base.py @@ -10,45 +10,49 @@ from models import Content class SourceAdapter(ABC): """内容源适配器基类""" - + @abstractmethod def fetch_new(self, since: datetime, **kwargs) -> List[Content]: """ 获取新内容 - + Args: since: 起始时间，只获取该时间之后的内容 **kwargs: 额外的参数 - + Returns: List[Content]: 新内容列表 """ pass - + @abstractmethod def extract_content(self, url: str, **kwargs) -> Optional[Content]: """ 从URL提取内容 - + Args: url: 内容URL **kwargs: 额外的参数 - + Returns: Optional[Content]: 内容对象，如果提取失败返回None """ pass - + @abstractmethod def get_transcript(self, content: Content, **kwargs) -> Optional[str]: """ 获取字幕/文本 - + Args: content: 内容对象 **kwargs: 额外的参数 - + Returns: Optional[str]: 字幕文本（SRT格式），如果没有字幕返回None """ pass + + +# 为向后兼容，提供 BaseSource 别名 +BaseSource = SourceAdapter diff --git a/skills/obsidian-kb/src/sources/zhihu.py b/skills/obsidian-kb/src/sources/zhihu.py index d60a554..f8a59f4 100644 --- a/skills/obsidian-kb/src/sources/zhihu.py +++ b/skills/obsidian-kb/src/sources/zhihu.py @@ -5,6 +5,14 @@ from typing import Optional from .base import BaseSource from ..models import Content, ContentType +try: + from .zhihu_drission_extractor import ZhihuDrissionExtractor +except ImportError: + # DrissionPage 未安装 + import logging + logger = logging.getLogger(__name__) + logger.warning(f"DrissionPage 未安装，无法提取知乎内容") + class ZhihuSource(BaseSource): """知乎文章/回答源""" @@ -52,13 +60,16 @@ class ZhihuSource(BaseSource): def get_transcript(self, content: Content) -> Optional[str]: """获取文章内容""" try: - # 优先使用 Playwright 提取 - from .zhihu_playwright_extractor import ZhihuPlaywrightExtractor + # 使用 DrissionPage 提取器 + extractor = ZhihuDrissionExtractor( + headless=self.config.get('headless', True), + timeout=self.config.get('timeout', 30), + config=self.config + ) - extractor = ZhihuPlaywrightExtractor() data = extractor.extract(content.url) - if data and data.get('content'): + if data and data.get('content_markdown'): # 更新元数据 if data.get('title'): content.title = data['title'] @@ -66,18 +77,15 @@ class ZhihuSource(BaseSource): content.author = data['author'] # 返回 Markdown 内容 - return data['content'] + return data['content_markdown'] except ImportError as e: - # Playwright 未安装 import logging logger = logging.getLogger(__name__) - logger.warning(f"Playwright 未安装，使用 HTTP 方案: {e}") + logger.error(f"DrissionPage 导入失败: {e}") except Exception as e: - # Playwright 提取失败 import logging logger = logging.getLogger(__name__) - logger.error(f"Playwright 提取失败: {e}") + logger.error(f"DrissionPage 提取失败: {e}") - # 降级到原有 HTTP 方案（保留现有逻辑） - return self._fetch_with_http(content.url) + return None diff --git a/skills/obsidian-kb/src/sources/zhihu_drission_extractor.py b/skills/obsidian-kb/src/sources/zhihu_drission_extractor.py new file mode 100644 index 0000000..1aaf2cd --- /dev/null +++ b/skills/obsidian-kb/src/sources/zhihu_drission_extractor.py @@ -0,0 +1,491 @@ +""" +知乎 DrissionPage 内容提取器 +使用 DrissionPage 浏览器自动化工具抓取知乎内容 +""" + +import re +import json +import sys +import os +from typing import Dict, Optional, Any, List +from pathlib import Path +from urllib.parse import urlparse + +# 添加当前目录到 Python 路径以支持绝对导入 +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +try: + from .markdown_converter import MarkdownConverter +except ImportError: + from markdown_converter import MarkdownConverter + + +class DrissionBrowser: + """DrissionPage 浏览器管理器""" + + def __init__(self, headless: bool = True, timeout: int = 30): + self.headless = headless + self.timeout = timeout + self.page = None + + def start(self): + """启动浏览器""" + from DrissionPage import ChromiumOptions, ChromiumPage + + options = ChromiumOptions() + if self.headless: + options.headless() + options.set_user_agent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36') + + self.page = ChromiumPage(options) + self.page.set.timeouts(base=self.timeout) + + def close(self): + """关闭浏览器""" + if self.page: + self.page.quit() + self.page = None + + def goto(self, url: str): + """导航到指定 URL""" + self.page.get(url) + + def wait_for_selector(self, selector: str, timeout: int = None): + """等待选择器出现""" + timeout = timeout or self.timeout + self.page.wait.ele_displayed(selector, timeout=timeout) + + def query_selector(self, selector: str): + """查询选择器""" + return self.page.ele(selector) + + def query_selector_all(self, selector: str): + """查询所有匹配的选择器""" + return self.page.eles(selector) + + def evaluate(self, script: str): + """执行 JavaScript 代码""" + return self.page.run_js(script) + + def get_page_title(self): + """获取页面标题""" + if self.page: + return self.page.title + return '' + + def get_page_content(self): + """获取页面 HTML 内容""" + if self.page: + return self.page.html + return '' + + def screenshot(self, path: str): + """截图""" + if self.page: + self.page.get_screenshot(path=path) + + +class ZhihuDrissionExtractor: + """知乎 DrissionPage 提取器""" + + URL_PATTERNS = { + 'answer': r'zhihu\.com/question/\d+/answer/(\d+)', + 'article': r'zhuanlan\.zhihu\.com/p/(\d+)', + 'question': r'zhihu\.com/question/(\d+)(?:$|/)', + } + + def __init__(self, headless: bool = True, timeout: int = 30, config: Optional[Dict] = None): + """ + 初始化提取器 + + Args: + headless: 是否使用无头浏览器（默认 True） + timeout: 超时时间（秒，默认 30） + config: 配置字典，用于加载 Cookie + """ + self.headless = headless + self.timeout = timeout + self.config = config or {} + # 支持从 config 中获取 headless 和 timeout + if self.config.get('headless') is not None: + self.headless = self.config.get('headless') + if self.config.get('timeout') is not None: + self.timeout = self.config.get('timeout') + + self.browser = DrissionBrowser(headless=self.headless, timeout=self.timeout) + self.markdown_converter = MarkdownConverter() + self.cookie_file = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + 'data', 'cookies', 'zhihu.json' + ) + + def identify_url_type(self, url: str) -> Optional[str]: + """ + 识别 URL 类型 + + Returns: + 'answer', 'article', 'question' 或 None + """ + for url_type, pattern in self.URL_PATTERNS.items(): + if re.search(pattern, url): + return url_type + return None + + def extract_id(self, url: str) -> Optional[str]: + """ + 从 URL 中提取内容 ID + + Returns: + 内容 ID 或 None + """ + url_type = self.identify_url_type(url) + if url_type: + pattern = self.URL_PATTERNS[url_type] + match = re.search(pattern, url) + if match: + return match.group(1) + return None + + def _extract_text_content(self, element) -> str: + """提取元素的文本内容""" + if element is None: + return '' + return element.text if hasattr(element, 'text') else '' + + def _extract_html_content(self, element) -> str: + """提取元素的 HTML 内容""" + if element is None: + return '' + return element.html if hasattr(element, 'html') else '' + + def _extract_attribute(self, selector: str, attribute: str) -> str: + """提取元素的属性""" + try: + element = self.browser.query_selector(selector) + if element: + return element.attr(attribute) or '' + except Exception as e: + print(f"提取属性时出错（选择器: {selector}, 属性: {attribute}）: {e}") + return '' + + def _load_cookies(self) -> List[Dict]: + """ + 加载 Cookie，优先级：配置 > 环境 > 文件 + + Returns: + Cookie 列表 + """ + cookies = [] + + # 1. 尝试从配置加载（优先级最高） + if self.config.get('zhihu.cookie_str'): + try: + cookies = json.loads(self.config.get('zhihu.cookie_str')) + print(f"从配置加载 Cookie，共 {len(cookies)} 个") + except json.JSONDecodeError as e: + print(f"配置中的 Cookie JSON 格式错误: {e}") + + # 2. 尝试从环境变量加载（优先级中等） + elif os.getenv('ZHIHU_COOKIE'): + try: + cookies = json.loads(os.getenv('ZHIHU_COOKIE')) + print(f"从环境变量加载 Cookie，共 {len(cookies)} 个") + except json.JSONDecodeError as e: + print(f"环境变量中的 Cookie JSON 格式错误: {e}") + + # 3. 尝试从文件加载（优先级最低） + else: + cookie_file = self.config.get('zhihu.cookie_file', self.cookie_file) + if cookie_file and Path(cookie_file).exists(): + try: + with open(cookie_file, 'r', encoding='utf-8') as f: + cookies = json.load(f) + print(f"从文件加载 Cookie ({cookie_file})，共 {len(cookies)} 个") + except (json.JSONDecodeError, FileNotFoundError, IOError) as e: + print(f"从文件加载 Cookie 失败: {e}") + else: + print("未找到 Cookie 配置，将使用未登录状态访问知乎") + + return cookies + + def _apply_cookies(self, cookies: List[Dict]): + """ + 应用 Cookie 到浏览器页面 + + Args: + cookies: Cookie 列表 + """ + if not cookies: + return + + try: + for cookie in cookies: + if self.browser and self.browser.page: + self.browser.page.set.cookies.add(cookie) + print(f"成功应用 {len(cookies)} 个 Cookie") + except Exception as e: + print(f"应用 Cookie 时出错: {e}") + + def _extract_answer_content(self) -> Dict[str, Any]: + """提取回答内容""" + result = { + 'type': 'answer', + 'title': '', + 'author': '', + 'content': '', + 'content_markdown': '', + 'publish_time': '', + 'vote_up': 0, + } + + # 打印页面标题用于调试 + page_title = self.browser.get_page_title() + print(f"页面标题: {page_title}") + + # 尝试保存页面快照用于调试 + try: + import os + debug_dir = '/home/admin/.openclaw/workspace/debug' + os.makedirs(debug_dir, exist_ok=True) + self.browser.screenshot(os.path.join(debug_dir, 'zhihu_drission_screenshot.png')) + with open(os.path.join(debug_dir, 'zhihu_drission_page.html'), 'w', encoding='utf-8') as f: + f.write(self.browser.get_page_content()) + print(f"页面快照已保存到 {debug_dir}") + except Exception as e: + print(f"保存页面快照时出错: {e}") + + # 尝试多个可能的选择器 + selectors = { + 'title': ['h1.QuestionHeader-title', 'h1', '.QuestionHeader-main'], + 'author': ['.AuthorInfo-name', '.AuthorInfo .Text', '[itemprop="name"]'], + 'content': ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist] article'], + 'time': ['.ContentItem-time', '[itemprop="dateCreated"]'], + 'vote': ['.VoteButton--up .VoteButton-label', '.VoteButton--up'] + } + + for selector in selectors['title']: + title_elem = self.browser.query_selector(selector) + if title_elem: + result['title'] = self._extract_text_content(title_elem) + print(f"找到标题（使用选择器 {selector}）: {result['title'][:50]}...") + break + + for selector in selectors['author']: + author_elem = self.browser.query_selector(selector) + if author_elem: + result['author'] = self._extract_text_content(author_elem) + print(f"找到作者（使用选择器 {selector}）: {result['author']}") + break + + for selector in selectors['content']: + content_elem = self.browser.query_selector(selector) + if content_elem: + html_content = self._extract_html_content(content_elem) + result['content'] = html_content + result['content_markdown'] = self.markdown_converter.convert(html_content) + print(f"找到内容（使用选择器 {selector}），长度: {len(result['content_markdown'])}") + break + + for selector in selectors['time']: + time_elem = self.browser.query_selector(selector) + if time_elem: + result['publish_time'] = time_elem.attr('data-tooltip') or self._extract_text_content(time_elem) + break + + for selector in selectors['vote']: + vote_elem = self.browser.query_selector(selector) + if vote_elem: + vote_text = self._extract_text_content(vote_elem) + try: + result['vote_up'] = int(vote_text.replace(',', '')) + except (ValueError, AttributeError): + pass + break + + return result + + def _extract_article_content(self) -> Dict[str, Any]: + """提取文章内容""" + result = { + 'type': 'article', + 'title': '', + 'author': '', + 'content': '', + 'content_markdown': '', + 'publish_time': '', + 'vote_up': 0, + } + + # 提取文章标题 + title_elem = self.browser.query_selector('.Post-Title') + if title_elem: + result['title'] = self._extract_text_content(title_elem) + + # 提取作者信息 + author_elem = self.browser.query_selector('.AuthorInfo-name') + if author_elem: + result['author'] = self._extract_text_content(author_elem) + + # 提取文章内容 + content_elem = self.browser.query_selector('.Post-RichText') + if content_elem: + html_content = self._extract_html_content(content_elem) + result['content'] = html_content + result['content_markdown'] = self.markdown_converter.convert(html_content) + + # 提取发布时间 + time_elem = self.browser.query_selector('.ContentItem-time') + if time_elem: + result['publish_time'] = time_elem.attr('data-tooltip') or self._extract_text_content(time_elem) + + # 提取点赞数 + vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label') + if vote_elem: + vote_text = self._extract_text_content(vote_elem) + try: + result['vote_up'] = int(vote_text.replace(',', '')) + except (ValueError, AttributeError): + pass + + return result + + def _extract_question_content(self) -> Dict[str, Any]: + """提取问题内容（问题页本身）""" + result = { + 'type': 'question', + 'title': '', + 'author': '', + 'content': '', + 'content_markdown': '', + 'publish_time': '', + 'vote_up': 0, + } + + # 提取问题标题 + title_elem = self.browser.query_selector('h1.QuestionHeader-title') + if title_elem: + result['title'] = self._extract_text_content(title_elem) + + # 提取问题详情 + detail_elem = self.browser.query_selector('.QuestionHeader-detail') + if detail_elem: + html_content = self._extract_html_content(detail_elem) + result['content'] = html_content + result['content_markdown'] = self.markdown_converter.convert(html_content) + + # 问题的作者通常是提问者 + author_elem = self.browser.query_selector('.AuthorInfo-name') + if author_elem: + result['author'] = self._extract_text_content(author_elem) + + return result + + def extract(self, url: str) -> Optional[Dict[str, Any]]: + """ + 从 URL 提取知乎内容 + + Args: + url: 知乎 URL + + Returns: + 包含提取内容的字典，结构如下: + { + 'type': 'answer' | 'article' | 'question', + 'title': str, + 'author': str, + 'content': str (HTML), + 'content_markdown': str (Markdown), + 'publish_time': str, + 'vote_up': int, + 'url': str, + 'content_id': str, + } + 如果提取失败返回 None + """ + try: + # 识别 URL 类型 + url_type = self.identify_url_type(url) + if not url_type: + print(f"无法识别 URL 类型: {url}") + return None + + print(f"URL 类型: {url_type}") + + # 加载 Cookie + cookies = self._load_cookies() + if cookies: + print(f"加载了 {len(cookies)} 个 Cookie") + + print(f"启动浏览器...") + + # 启动浏览器 + self.browser.start() + + # 应用 Cookie + if cookies and self.browser.page: + self._apply_cookies(cookies) + + print(f"访问页面: {url}") + # 访问页面 + self.browser.goto(url) + + print("等待页面加载...") + import time + time.sleep(3) # 等待 JavaScript 执行 + + # 尝试多个可能的选择器 + if url_type == 'answer': + selectors = ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist]'] + print(f"尝试查找选择器: {selectors}") + result = self._extract_answer_content() + elif url_type == 'article': + selectors = ['.Post-RichText', '.Post-Title'] + print(f"尝试查找选择器: {selectors}") + result = self._extract_article_content() + else: # question + selectors = ['.QuestionHeader-title', 'h1'] + print(f"尝试查找选择器: {selectors}") + result = self._extract_question_content() + + # 添加额外信息 + result['url'] = url + result['content_id'] = self.extract_id(url) + + print(f"提取成功! 标题: {result['title'][:50]}...") + return result + + except Exception as e: + print(f"提取知乎内容时出错: {e}") + import traceback + traceback.print_exc() + return None + finally: + # 关闭浏览器 + self.browser.close() + + def __enter__(self): + """上下文管理器入口""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """上下文管理器出口""" + self.browser.close() + return False + + +# 便捷函数 +def extract_zhihu_content(url: str, config=None) -> Optional[Dict[str, Any]]: + """ + 便捷函数：提取知乎内容 + + Args: + url: 知乎 URL + config: 配置字典 + + Returns: + 提取的内容字典 + """ + with ZhihuDrissionExtractor(config=config) as extractor: + return extractor.extract(url)