docs(09): create phase 9 plans
This commit is contained in:
@@ -0,0 +1,222 @@
|
||||
---
|
||||
phase: 09-pipeline-integration
|
||||
plan: 01
|
||||
type: execute
|
||||
wave: 1
|
||||
depends_on: []
|
||||
files_modified: []
|
||||
autonomous: true
|
||||
requirements: [PIPE-01, PIPE-02, PIPE-03]
|
||||
user_setup: []
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "知乎链接使用新爬虫处理器，无需手动设置 Cookie"
|
||||
- "新处理器与现有 Pipeline 架构完全兼容"
|
||||
- "支持断点续传和幂等执行(与其他处理器行为一致)"
|
||||
artifacts:
|
||||
- path: "kb_workflow/pipeline/platforms/zhihu.py"
|
||||
provides: "ZhihuAdapter implementation"
|
||||
min_lines: 30
|
||||
exports: ["ZhihuAdapter"]
|
||||
- path: "kb_workflow/crawler/zhihu_crawler.py"
|
||||
provides: "ZhihuCrawler class"
|
||||
exports: ["ZhihuCrawler", "Crawl_answer", "crawl_article", "crawl_comments"]
|
||||
- path: "kb_workflow/crawler/models.py"
|
||||
provides: "Data models (AnswerData, ArticleData, CommentData, CrawlResult)"
|
||||
exports: ["AnswerData", "ArticleData", "CommentData", "CommentAuthor", "ContentMetadata", "AuthorInfo", "CrawlResult"]
|
||||
- path: "kb_workflow/auth/zhihu_login.py"
|
||||
provides: "ZhihuLogin class"
|
||||
exports: ["ZhihuLogin", "login_by_qrcode", "validate_cookie", "ZhihuLoginResult", "CookieValidationResult"]
|
||||
- path: "tests/test_platforms/test_zhihu.py"
|
||||
provides: "test coverage for ZhihuAdapter"
|
||||
exports: ["ZhihuAdapter", "test_get_content_type_*"]
|
||||
min_lines: 50
|
||||
exports: ["mock_zhihu_answer_result", "mock_zhihu_zhuanlan_result"]
|
||||
- path: "kb_workflow/pipeline/processors/download.py"
|
||||
provides: "Download processor (consumer of ZhihuAdapter)"
|
||||
exports: ["DownloadProcessor"]
|
||||
key_links:
|
||||
- from: "ZhihuAdapter.fetch_metadata"
|
||||
to: "ZhihuCrawler.crawl_answer/crawl_article"
|
||||
via: "asyncio.to_thread"
|
||||
pattern: "asyncio\\.to_thread.*crawl"
|
||||
- from: "ZhihuAdapter.fetch_metadata"
|
||||
to: "ZhihuCrawler.crawl_comments"
|
||||
via: "internal _crawl_comments call"
|
||||
pattern: "result\\.comments"
|
||||
- from: "ZhihuAdapter._format_with_frontmatter"
|
||||
to: "ZhihuAdapter._format_comments_section"
|
||||
via: "content append"
|
||||
pattern: "\\n---\\n## 热门评论"
|
||||
---
|
||||
|
||||
<objective>
|
||||
将新爬虫实现整合进现有 Pipeline 架构,替换原有知乎处理器,保持接口兼容性和,支持评论爬取和Cookie 过期自动处理。
|
||||
|
||||
Cookie 过期处理
|
||||
Cookie 过期时自动触发扫码登录,用户扫码后流程继续
|
||||
新爬虫失败时直接报错,不降级到旧版。
|
||||
</objective>
|
||||
|
||||
<execution_context>
|
||||
@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@/home/starmen1265/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@.planning/STATE.md
|
||||
@.planning/phases/08-crawler/08-03-SUMMARY.md
|
||||
|
||||
@.planning/phases/09-pipeline-integration/09-RESEARCH.md
|
||||
@.planning/phases/09-pipeline-integration/09-CONTEXT.md
|
||||
|
||||
@.planning/phases/09-pipeline-integration/09-VALIDATION.md
|
||||
|
||||
<interfaces>
|
||||
<!-- Key types and contracts the executor needs. Executor should use these directly - no codebase exploration needed. -->
|
||||
|
||||
From kb_workflow/pipeline/platforms/base.py:
|
||||
```python
|
||||
class PlatformAdapter(ABC):
|
||||
@abstractmethod
|
||||
def get_content_type(self, url: str) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def fetch_metadata(self, content_id: str, content_type: str, source_url: str = None) -> Dict[str, Any]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def fetch_content(self, content_id: str, content_type: str, source_url: str = None) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str:
|
||||
pass
|
||||
```
|
||||
|
||||
From kb_workflow/crawler/models.py
|
||||
```python
|
||||
class CrawlResult(BaseModel):
|
||||
success: bool
|
||||
content_type: str
|
||||
data: Optional[Union[AnswerData, ArticleData]]
|
||||
comments: Optional[List[commentData]]
|
||||
error: Optional[str]
|
||||
|
||||
```
|
||||
|
||||
from kb_workflow/crawler/zhihu_crawler.py
|
||||
```python
|
||||
class ZhihuCrawler:
|
||||
def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult
|
||||
...
|
||||
|
||||
def crawl_article(self, url: str) -> CrawlResult:
|
||||
"""
|
||||
def crawl_comments(self, content_type: str, content_id: int, include_comments: bool = True) -> CommentsResult:
|
||||
"""
|
||||
```
|
||||
|
||||
from kb_workflow/auth/zhihu_login.py
|
||||
```python
|
||||
class ZhihuLogin:
|
||||
def login_by_qrcode(self) -> ZhihuLoginResult
|
||||
"""
|
||||
def validate_cookie(self, cookie_str: str = None) -> CookieValidationResult
|
||||
"""
|
||||
```
|
||||
</interfaces>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 1: Update ZhihuAdapter to use ZhihuCrawler</name>
|
||||
<files>kb_workflow/pipeline/platforms/zhihu.py</files>
|
||||
<action>
|
||||
修改 ZhihuAdapter 类:
|
||||
|
||||
内部实现",使用 ZhihuCrawler 替代 ZhihuPlaywrightExtractor。
|
||||
|
||||
保留公共接口不变 (fetch_metadata, fetch_content, get_content_type, get_output_filename).
|
||||
|
||||
具体修改:
|
||||
1. 移除 ZhihuPlaywrightExtractor` 导入，ZhihuCrawler"
|
||||
2 添加 `_get_crawler()` 方法创建和管理 ZhihuCrawler 实例
|
||||
添加 `_crawl_with_crawler()` 方法使用 ZhihuCrawler 进行内容爬取
|
||||
添加 `_convert_crawl_result_to_metadata()` 方法将 CrawlResult 转换为 metadata dict
|
||||
添加 `_format_comments_section()` 方法格式化评论为 Markdown
|
||||
添加 `_is_cookie_expired()` 方法检测 Cookie 过期
|
||||
添加 `_trigger_qrcode_login()` 方法触发扫码登录
|
||||
移除 `_extract_with_playwright` 和 `_get_extractor` 方法（不再需要)
|
||||
2 修改 `_detect_login_page` 方法,基于 CrawlResult 检测登录页面
|
||||
</action>
|
||||
<verify>
|
||||
<automated>pytest tests/test_platforms/test_zhihu.py -x -v</automated>
|
||||
</verify>
|
||||
<done>
|
||||
Z ZhihuAdapter 使用 ZhihuCrawler 作为内容获取
|
||||
fetch_metadata 返回包含评论的 CrawlResult
|
||||
fetch_content 返回 Markdown 格式内容(含评论追加)
|
||||
</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 2: Add Cookie expired auto login tests</name>
|
||||
<files>tests/test_platforms/test_zhihu.py</files>
|
||||
<action>
|
||||
添加测试用例验证 Cookie 过期时自动触发扫码登录:
|
||||
测试场景:
|
||||
1. Cookie 有效时,爬取正常返回数据
|
||||
2. Cookie 过期时,触发扫码登录流程
|
||||
3. 触发登录后重试成功
|
||||
需要 mock:
|
||||
login_result = ZhihuLoginResult(success=True, cookie_str="new_cookie")
|
||||
- mock _trigger_qrcode_login 返回登录失败
|
||||
- mock _trigger_qrcode_login 抛出异常
|
||||
测试验证正确处理各种情况
|
||||
</action>
|
||||
<verify>
|
||||
<automated>pytest tests/test_platforms/test_zhihu.py -k "test_cookie" -v</automated>
|
||||
</verify>
|
||||
<done>
|
||||
Cookie 过期自动触发扫码登录的测试用例全部通过
|
||||
</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 3: Add comment formatting tests</name>
|
||||
<files>tests/test_platforms/test_zhihu.py</files>
|
||||
<action>
|
||||
添加测试用例验证评论格式化:
|
||||
测试场景:
|
||||
1. 有评论时,格式化为 "热门评论" 部分
|
||||
2. 无评论时,返回空字符串
|
||||
使用 mock 数据验证格式正确
|
||||
</action>
|
||||
<verify>
|
||||
<automated>pytest tests/test_platforms/test_zhihu.py -k "test_comment" -v</automated>
|
||||
</verify>
|
||||
<done>
|
||||
评论格式化测试用例全部通过
|
||||
</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
运行完整的 Pipeline 测试验证整合是否正常工作:
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- 知乎链接使用新爬虫处理器,无需手动设置 Cookie
|
||||
- 新处理器与现有 Pipeline 架构完全兼容
|
||||
- 支持断点续传和幂等执行(与其他处理器行为一致)
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/09-pipeline-integration/09-01-SUMMARY.md`
|
||||
</output>
|
||||
Reference in New Issue
Block a user