From 45ba9c973fcf6de8124b8cb744f0f526565de7d9 Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Tue, 17 Mar 2026 22:37:38 +0800 Subject: [PATCH] docs(09): research phase 9 domain --- .../09-pipeline-integration/09-RESEARCH.md | 360 ++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 .planning/phases/09-pipeline-integration/09-RESEARCH.md diff --git a/.planning/phases/09-pipeline-integration/09-RESEARCH.md b/.planning/phases/09-pipeline-integration/09-RESEARCH.md new file mode 100644 index 0000000..cb50d74 --- /dev/null +++ b/.planning/phases/09-pipeline-integration/09-RESEARCH.md @@ -0,0 +1,360 @@ +# Phase 9: Pipeline 整合 - Research + +**Researched:** 2026-03-17 +**Domain:** Pipeline Integration / Zhihu Crawler Integration +**Confidence:** HIGH + +## Summary + +Phase 9 需要将 Phase 7-8 实现的 ZhihuCrawler 和 ZhihuLogin 整合进现有 Pipeline 架构，替换原有的 ZhihuPlaywrightExtractor。核心工作是修改 ZhihuAdapter 的内部实现，同时保持公共接口不变，确保对 DownloadProcessor 无感知变化。 + +**Primary recommendation:** 采用"内部替换"策略 - 修改 ZhihuAdapter 内部调用 ZhihuCrawler，保持接口稳定，评论追加到正文，Cookie 过期自动触发扫码登录。 + + +## User Constraints (from CONTEXT.md) + +### Locked Decisions + +#### 整合方式 +- **策略**: 修改现有 ZhihuAdapter 内部实现，切换到使用 ZhihuCrawler +- **接口保持**: 保持 ZhihuAdapter 的公共接口不变，对 DownloadProcessor 无感知变化 +- **实现变更**: fetch_metadata() 和 fetch_content() 内部调用 ZhihuCrawler 而非 ZhihuPlaywrightExtractor + +#### 评论数据处理 +- **默认行为**: 评论爬取默认开启 +- **存储方式**: 评论追加到正文末尾（Markdown 格式） +- **数量上限**: 20 条热门评论（使用 hot 排序，API 默认限制） +- **嵌套处理**: 不展开回复的回复，只爬第一层评论（继承自 Phase 8） + +#### Cookie 过期处理 +- **策略**: 自动触发扫码登录 +- **流程**: 检测到 Cookie 失效时，调用 ZhihuLogin 触发扫码登录，用户扫码后继续执行 +- **用户体验**: 用户会看到二维码提示，扫码后流程自动继续 + +#### 失败降级策略 +- **策略**: 直接报错，不降级 +- **不降级**: 新爬虫失败时不尝试旧版 ZhihuPlaywrightExtractor +- **错误处理**: 抛出异常，由上层 Pipeline 统一处理错误 + +### Claude's Discretion +- 评论追加到正文的具体 Markdown 格式（标题、分隔线等） +- 二维码显示的具体实现方式（终端/图片文件） +- 错误提示信息的具体文案 + +### Deferred Ideas (OUT OF SCOPE) +None — discussion stayed within phase scope + + + + +## Phase Requirements + +| ID | Description | Research Support | +|----|-------------|-----------------| +| PIPE-01 | 替换现有 ZhihuProcessor，使用新的爬虫实现 | ZhihuAdapter 内部调用 ZhihuCrawler，保持接口不变 | +| PIPE-02 | 保持与现有 Pipeline 架构的兼容性 | PlatformAdapter 接口不变，DownloadProcessor 无需修改 | +| PIPE-03 | 支持断点续传和幂等执行 | WorkflowStateManager + is_step_completed 检查机制 | + + +## Standard Stack + +### Core +| Library | Version | Purpose | Why Standard | +|---------|---------|---------|--------------| +| pydantic | v2 | 数据验证和模型定义 | 项目已使用，CrawlResult/AnswerData/ArticleData 等模型 | +| playwright | sync_api | 浏览器自动化 | ZhihuCrawler 已使用，支持代理和 Cookie | +| tenacity | ^8.0 | 重试机制 | ZhihuCrawler 使用 Retrying 对象 | +| asyncio | stdlib | 异步执行 | PlatformAdapter 接口使用 async/await | + +### Supporting +| Library | Version | Purpose | When to Use | +|---------|---------|---------|-------------| +| kb_workflow.crawler.ZhihuCrawler | internal | 问答/专栏/评论爬取 | fetch_metadata/fetch_content 调用 | +| kb_workflow.auth.ZhihuLogin | internal | QR 码登录和 Cookie 管理 | Cookie 过期时触发 | +| kb_workflow.pipeline.errors.AuthError | internal | 认证错误处理 | Cookie 失效时抛出 | + +### Key Classes + +| Class | Location | Purpose | +|-------|----------|---------| +| ZhihuCrawler | `kb_workflow/crawler/zhihu_crawler.py` | 主爬虫，crawl_answer/crawl_article/crawl_comments | +| ZhihuLogin | `kb_workflow/auth/zhihu_login.py` | login_by_qrcode()，Cookie 持久化 | +| ZhihuAdapter | `kb_workflow/pipeline/platforms/zhihu.py` | **待修改** - PlatformAdapter 实现 | +| DownloadProcessor | `kb_workflow/pipeline/processors/download.py` | 调用 Adapter，无需修改 | + +## Architecture Patterns + +### Current ZhihuAdapter Implementation + +```python +# 当前实现：使用 ZhihuPlaywrightExtractor (DOM 解析) +class ZhihuAdapter(PlatformAdapter): + def _get_extractor(self): + return ZhihuPlaywrightExtractor(headless=self.headless, cookie=self.cookie) + + async def fetch_metadata(self, content_id, content_type, source_url=None): + result = await asyncio.to_thread(self._extract_with_playwright, url) + # 返回 metadata dict + + async def fetch_content(self, content_id, content_type, source_url=None): + metadata = await self.fetch_metadata(...) + return self._format_with_frontmatter(metadata, content_markdown, content_type) +``` + +### Target Implementation Pattern + +```python +# 目标实现：使用 ZhihuCrawler (API 拦截) +class ZhihuAdapter(PlatformAdapter): + def __init__(self, ...): + self._crawler = None + + def _get_crawler(self) -> ZhihuCrawler: + if self._crawler is None: + self._crawler = ZhihuCrawler(headless=self.headless, timeout=self.timeout) + return self._crawler + + async def fetch_metadata(self, content_id, content_type, source_url=None): + crawler = self._get_crawler() + if content_type == "answer": + result = await asyncio.to_thread(crawler.crawl_answer, url, include_comments=True) + elif content_type == "zhuanlan": + result = await asyncio.to_thread(crawler.crawl_article, url) + # 转换 CrawlResult -> metadata dict + + async def fetch_content(self, content_id, content_type, source_url=None): + # 调用 fetch_metadata，格式化为 Markdown（含评论） +``` + +### Cookie Expiration Handling Pattern + +```python +# Cookie 过期处理流程 +async def fetch_metadata(self, content_id, content_type, source_url=None): + try: + result = self._crawl_with_retry(url) + if result.error and self._is_cookie_expired(result): + # 触发扫码登录 + login_result = self._trigger_qrcode_login() + if login_result.success: + # 刷新配置并重试 + get_settings.cache_clear() + result = self._crawl_with_retry(url) + return self._convert_to_metadata(result) + except Exception as e: + raise AuthError(step_name="zhihu_metadata", ...) +``` + +### Comment Append Format (Claude's Discretion) + +```markdown +## 正文内容 +[原回答/文章内容] + +--- + +## 热门评论 +1. **@用户A**: 评论内容... +2. **@用户B**: 评论内容... +3. **@用户C**: 评论内容... +``` + +### Anti-Patterns to Avoid +- **不要修改 PlatformAdapter 接口**: DownloadProcessor 依赖接口稳定 +- **不要在 ZhihuCrawler 中添加 Pipeline 逻辑**: 保持爬虫职责单一 +- **不要同步阻塞调用爬虫**: 使用 asyncio.to_thread 包装 +- **不要忽略 Cookie 过期**: 必须触发扫码登录流程 + +## Don't Hand-Roll + +| Problem | Don't Build | Use Instead | Why | +|---------|-------------|-------------|-----| +| HTML to Markdown | 自定义转换器 | MarkdownConverter (已存在) | zhihu_extractor.py 已有 | +| Cookie 解析 | 手动解析 cookie 字符串 | PlaywrightBrowser._add_cookies() | 已实现，处理格式正确 | +| 重试逻辑 | @retry 装饰器 | tenacity Retrying 对象 | 支持可配置超时，Phase 8 已用 | +| API 签名 | 逆向 x-zse-96 | 浏览器自动处理 | route.fetch() 自动包含签名 | + +**Key insight:** ZhihuCrawler 已完整实现问答/专栏/评论爬取，只需要正确调用和结果转换。 + +## Common Pitfalls + +### Pitfall 1: CrawlResult 到 Metadata Dict 转换不完整 +**What goes wrong:** ZhihuCrawler 返回 CrawlResult(AnswerData/ArticleData)，但 DownloadProcessor 期望特定格式的 metadata dict +**Why it happens:** 数据模型不完全匹配 +**How to avoid:** 实现 _convert_crawl_result_to_metadata() 转换函数，确保所有字段正确映射 +**Warning signs:** DownloadProcessor 提取 title/author 时返回空字符串 + +### Pitfall 2: 评论未正确追加到正文 +**What goes wrong:** fetch_content() 返回的内容不包含评论 +**Why it happens:** include_comments=True 未正确传递或评论格式化逻辑遗漏 +**How to avoid:** fetch_content() 必须调用 fetch_metadata(include_comments=True)，格式化时追加评论 +**Warning signs:** 输出 Markdown 文件无评论部分 + +### Pitfall 3: Cookie 过期检测不准确 +**What goes wrong:** 误判 Cookie 有效/无效，导致不必要的登录或无限重试 +**Why it happens:** 仅依赖 HTTP 状态码，未检查响应内容 +**How to avoid:** 检查 API 响应中的 error 字段和 401/403 状态码 +**Warning signs:** 登录循环或认证错误未触发 + +### Pitfall 4: asyncio.to_thread 调用错误 +**What goes wrong:** ZhihuCrawler 方法调用参数传递不正确 +**Why it happens:** to_thread 需要函数引用和参数分开传递 +**How to avoid:** `await asyncio.to_thread(crawler.crawl_answer, url, include_comments=True)` +**Warning signs:** TypeError: 'CrawlResult' object is not callable + +## Code Examples + +### ZhihuCrawler Usage (from existing code) + +```python +# Source: kb_workflow/crawler/zhihu_crawler.py +crawler = ZhihuCrawler(headless=True, timeout=30) + +# 爬取回答（含评论） +result = crawler.crawl_answer( + url="https://www.zhihu.com/question/123/answer/456", + include_comments=True +) +# result = CrawlResult(success=True, content_type="answer", data=AnswerData, comments=[...]) + +# 爬取专栏文章 +result = crawler.crawl_article(url="https://zhuanlan.zhihu.com/p/123") +# result = CrawlResult(success=True, content_type="article", data=ArticleData) +``` + +### ZhihuLogin QR Code Flow (from existing code) + +```python +# Source: kb_workflow/auth/zhihu_login.py +login = ZhihuLogin(intermediate_path=str(settings.intermediate_path), headless=False) +result = login.login_by_qrcode() + +if result.success: + # 保存 Cookie 到配置文件 + cookie_path = Path.home() / ".kb_env" + # 更新 KB_ZHIHU_COOKIE 环境变量 + # 刷新 settings 缓存 + get_settings.cache_clear() +``` + +### Metadata Conversion Pattern + +```python +def _convert_answer_to_metadata(self, result: CrawlResult) -> Dict[str, Any]: + """Convert CrawlResult[AnswerData] to metadata dict.""" + answer: AnswerData = result.data + return { + "type": "answer", + "title": answer.question_title, + "author": answer.metadata.author.name if answer.metadata.author else "", + "content": answer.content, + "content_markdown": self._html_to_markdown(answer.content), + "publish_time": self._format_timestamp(answer.metadata.created_time), + "url": f"https://www.zhihu.com/question/{answer.question_id}/answer/{answer.id}", + "question_id": str(answer.question_id), + "answer_id": str(answer.id), + "vote_up": answer.metadata.voteup_count, + "comment_count": answer.metadata.comment_count, + } +``` + +### Comment Formatting Pattern + +```python +def _format_comments_section(self, comments: List[CommentData]) -> str: + """Format comments as Markdown section.""" + if not comments: + return "" + + lines = ["\n\n---\n\n## 热门评论\n"] + for i, comment in enumerate(comments[:20], 1): # 最多 20 条 + author = f"@{comment.author.name}" if comment.author else "匿名" + lines.append(f"{i}. **{author}**: {comment.content}") + + return "\n".join(lines) +``` + +## State of the Art + +| Old Approach | Current Approach | When Changed | Impact | +|--------------|------------------|--------------|--------| +| ZhihuPlaywrightExtractor (DOM 解析) | ZhihuCrawler (API 拦截) | Phase 8 | 更可靠的数据获取，支持评论 | +| 手动 Cookie 管理 | ZhihuLogin QR 码登录 | Phase 7 | 自动化登录流程 | +| 无评论支持 | 评论爬取 + 追加到正文 | Phase 8 | 更完整的内容存档 | + +**Deprecated/outdated:** +- ZhihuPlaywrightExtractor 直接调用: 将被 ZhihuCrawler 替代 +- 手动设置 KB_ZHIHU_COOKIE: 通过 login 命令自动管理 + +## Open Questions + +1. **评论 Markdown 格式细节** + - What we know: 用户决策评论追加到正文，分隔线 + 热门评论标题 + - What's unclear: 具体的分隔线样式、评论编号格式 + - Recommendation: 使用 `---` 分隔线，`1. **@用户**: 内容` 格式 + +2. **Cookie 过期检测的精确条件** + - What we know: API 返回 401 或特定错误信息 + - What's unclear: 具体的错误码和响应结构 + - Recommendation: 检查 response.status == 401 或 JSON 中的 error 字段 + +3. **ZhihuLogin 在 headless 模式下的二维码展示** + - What we know: QR 码保存到文件，路径返回给用户 + - What's unclear: 是否需要终端 ASCII 输出 + - Recommendation: 默认保存文件并打印路径，可选终端输出 + +## Validation Architecture + +### Test Framework +| Property | Value | +|----------|-------| +| Framework | pytest | +| Config file | pytest.ini | +| Quick run command | `pytest tests/test_platforms/test_zhihu.py -x -v` | +| Full suite command | `pytest tests/ -v` | + +### Phase Requirements → Test Map +| Req ID | Behavior | Test Type | Automated Command | File Exists? | +|--------|----------|-----------|-------------------|-------------| +| PIPE-01 | ZhihuAdapter 使用 ZhihuCrawler | unit | `pytest tests/test_platforms/test_zhihu.py -x` | ✅ 需修改 | +| PIPE-02 | DownloadProcessor 兼容性 | integration | `pytest tests/test_processors/test_download.py -x` | ✅ 已存在 | +| PIPE-03 | 断点续传/幂等执行 | unit | `pytest tests/test_pipeline/test_idempotency.py -x` | ✅ 已存在 | + +### Sampling Rate +- **Per task commit:** `pytest tests/test_platforms/test_zhihu.py -x -v` +- **Per wave merge:** `pytest tests/test_platforms/ tests/test_processors/test_download.py -v` +- **Phase gate:** Full suite green before `/gsd:verify-work` + +### Wave 0 Gaps +- [ ] `tests/test_platforms/test_zhihu.py` — 更新测试使用 ZhihuCrawler mock +- [ ] 新增测试：Cookie 过期自动触发登录流程 +- [ ] 新增测试：评论追加到正文格式验证 +- [ ] 新增测试：CrawlResult 到 metadata 转换正确性 + +*(现有测试基础设施完善，主要是更新和新增 ZhihuAdapter 相关测试)* + +## Sources + +### Primary (HIGH confidence) +- `kb_workflow/crawler/zhihu_crawler.py` - ZhihuCrawler 实现 +- `kb_workflow/crawler/models.py` - CrawlResult, AnswerData, ArticleData, CommentData +- `kb_workflow/auth/zhihu_login.py` - ZhihuLogin QR 码登录 +- `kb_workflow/pipeline/platforms/zhihu.py` - ZhihuAdapter (待修改) +- `kb_workflow/pipeline/processors/download.py` - DownloadProcessor (无需修改) + +### Secondary (MEDIUM confidence) +- `tests/test_platforms/test_zhihu.py` - 现有 ZhihuAdapter 测试 +- `tests/test_crawler/` - ZhihuCrawler 测试 + +### Tertiary (LOW confidence) +- None - 所有信息均来自项目代码 + +## Metadata + +**Confidence breakdown:** +- Standard stack: HIGH - 项目已有完整实现，只需整合 +- Architecture: HIGH - PlatformAdapter 接口稳定，模式清晰 +- Pitfalls: HIGH - 基于现有代码分析，问题可预见 + +**Research date:** 2026-03-17 +**Valid until:** 30 days - 项目内部代码，稳定性高