diff --git a/.planning/phases/09-pipeline-integration/09-01-PLAN.md b/.planning/phases/09-pipeline-integration/09-01-PLAN.md new file mode 100644 index 0000000..bea7ba6 --- /dev/null +++ b/.planning/phases/09-pipeline-integration/09-01-PLAN.md @@ -0,0 +1,222 @@ +--- +phase: 09-pipeline-integration +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: [] +autonomous: true +requirements: [PIPE-01, PIPE-02, PIPE-03] +user_setup: [] + +must_haves: + truths: + - "知乎链接使用新爬虫处理器,无需手动设置 Cookie" + - "新处理器与现有 Pipeline 架构完全兼容" + - "支持断点续传和幂等执行(与其他处理器行为一致)" + artifacts: + - path: "kb_workflow/pipeline/platforms/zhihu.py" + provides: "ZhihuAdapter implementation" + min_lines: 30 + exports: ["ZhihuAdapter"] + - path: "kb_workflow/crawler/zhihu_crawler.py" + provides: "ZhihuCrawler class" + exports: ["ZhihuCrawler", "Crawl_answer", "crawl_article", "crawl_comments"] + - path: "kb_workflow/crawler/models.py" + provides: "Data models (AnswerData, ArticleData, CommentData, CrawlResult)" + exports: ["AnswerData", "ArticleData", "CommentData", "CommentAuthor", "ContentMetadata", "AuthorInfo", "CrawlResult"] + - path: "kb_workflow/auth/zhihu_login.py" + provides: "ZhihuLogin class" + exports: ["ZhihuLogin", "login_by_qrcode", "validate_cookie", "ZhihuLoginResult", "CookieValidationResult"] + - path: "tests/test_platforms/test_zhihu.py" + provides: "test coverage for ZhihuAdapter" + exports: ["ZhihuAdapter", "test_get_content_type_*"] + min_lines: 50 + exports: ["mock_zhihu_answer_result", "mock_zhihu_zhuanlan_result"] + - path: "kb_workflow/pipeline/processors/download.py" + provides: "Download processor (consumer of ZhihuAdapter)" + exports: ["DownloadProcessor"] + key_links: + - from: "ZhihuAdapter.fetch_metadata" + to: "ZhihuCrawler.crawl_answer/crawl_article" + via: "asyncio.to_thread" + pattern: "asyncio\\.to_thread.*crawl" + - from: "ZhihuAdapter.fetch_metadata" + to: "ZhihuCrawler.crawl_comments" + via: "internal _crawl_comments call" + pattern: "result\\.comments" + - from: "ZhihuAdapter._format_with_frontmatter" + to: "ZhihuAdapter._format_comments_section" + via: "content append" + pattern: "\\n---\\n## 热门评论" +--- + + +将新爬虫实现整合进现有 Pipeline 架构,替换原有知乎处理器,保持接口兼容性和,支持评论爬取和Cookie 过期自动处理。 + +Cookie 过期处理 +Cookie 过期时自动触发扫码登录,用户扫码后流程继续 +新爬虫失败时直接报错,不降级到旧版。 + + + +@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md +@/home/starmen1265/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/08-crawler/08-03-SUMMARY.md + +@.planning/phases/09-pipeline-integration/09-RESEARCH.md +@.planning/phases/09-pipeline-integration/09-CONTEXT.md + +@.planning/phases/09-pipeline-integration/09-VALIDATION.md + + + + +From kb_workflow/pipeline/platforms/base.py: +```python +class PlatformAdapter(ABC): + @abstractmethod + def get_content_type(self, url: str) -> str: + pass + + @abstractmethod + async def fetch_metadata(self, content_id: str, content_type: str, source_url: str = None) -> Dict[str, Any]: + pass + + @abstractmethod + async def fetch_content(self, content_id: str, content_type: str, source_url: str = None) -> str: + pass + + @abstractmethod + def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str: + pass +``` + +From kb_workflow/crawler/models.py +```python +class CrawlResult(BaseModel): + success: bool + content_type: str + data: Optional[Union[AnswerData, ArticleData]] + comments: Optional[List[commentData]] + error: Optional[str] + +``` + +from kb_workflow/crawler/zhihu_crawler.py +```python +class ZhihuCrawler: + def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult + ... + + def crawl_article(self, url: str) -> CrawlResult: + """ + def crawl_comments(self, content_type: str, content_id: int, include_comments: bool = True) -> CommentsResult: + """ +``` + +from kb_workflow/auth/zhihu_login.py +```python +class ZhihuLogin: + def login_by_qrcode(self) -> ZhihuLoginResult + """ + def validate_cookie(self, cookie_str: str = None) -> CookieValidationResult + """ +``` + + + + + + Task 1: Update ZhihuAdapter to use ZhihuCrawler + kb_workflow/pipeline/platforms/zhihu.py + +修改 ZhihuAdapter 类: + +内部实现",使用 ZhihuCrawler 替代 ZhihuPlaywrightExtractor。 + +保留公共接口不变 (fetch_metadata, fetch_content, get_content_type, get_output_filename). + +具体修改: +1. 移除 ZhihuPlaywrightExtractor` 导入,ZhihuCrawler" +2 添加 `_get_crawler()` 方法创建和管理 ZhihuCrawler 实例 + 添加 `_crawl_with_crawler()` 方法使用 ZhihuCrawler 进行内容爬取 + 添加 `_convert_crawl_result_to_metadata()` 方法将 CrawlResult 转换为 metadata dict + 添加 `_format_comments_section()` 方法格式化评论为 Markdown + 添加 `_is_cookie_expired()` 方法检测 Cookie 过期 + 添加 `_trigger_qrcode_login()` 方法触发扫码登录 + 移除 `_extract_with_playwright` 和 `_get_extractor` 方法(不再需要) +2 修改 `_detect_login_page` 方法,基于 CrawlResult 检测登录页面 + + + pytest tests/test_platforms/test_zhihu.py -x -v + + +Z ZhihuAdapter 使用 ZhihuCrawler 作为内容获取 +fetch_metadata 返回包含评论的 CrawlResult +fetch_content 返回 Markdown 格式内容(含评论追加) + + + + + Task 2: Add Cookie expired auto login tests + tests/test_platforms/test_zhihu.py + +添加测试用例验证 Cookie 过期时自动触发扫码登录: +测试场景: +1. Cookie 有效时,爬取正常返回数据 +2. Cookie 过期时,触发扫码登录流程 +3. 触发登录后重试成功 +需要 mock: +login_result = ZhihuLoginResult(success=True, cookie_str="new_cookie") +- mock _trigger_qrcode_login 返回登录失败 +- mock _trigger_qrcode_login 抛出异常 +测试验证正确处理各种情况 + + + pytest tests/test_platforms/test_zhihu.py -k "test_cookie" -v + + +Cookie 过期自动触发扫码登录的测试用例全部通过 + + + + + Task 3: Add comment formatting tests + tests/test_platforms/test_zhihu.py + +添加测试用例验证评论格式化: +测试场景: +1. 有评论时,格式化为 "热门评论" 部分 +2. 无评论时,返回空字符串 +使用 mock 数据验证格式正确 + + + pytest tests/test_platforms/test_zhihu.py -k "test_comment" -v + + +评论格式化测试用例全部通过 + + + + + + +运行完整的 Pipeline 测试验证整合是否正常工作: + + + +- 知乎链接使用新爬虫处理器,无需手动设置 Cookie +- 新处理器与现有 Pipeline 架构完全兼容 +- 支持断点续传和幂等执行(与其他处理器行为一致) + + + +After completion, create `.planning/phases/09-pipeline-integration/09-01-SUMMARY.md` +