docs(09): complete phase 9 planning with 2 plans in 2 waves
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
---
|
||||
phase: 09-pipeline-integration
|
||||
plan: 02
|
||||
type: execute
|
||||
wave: 1
|
||||
depends_on: [09-01]
|
||||
files_modified: []
|
||||
autonomous: true
|
||||
requirements: []
|
||||
user_setup: []
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "用户可通过命令行测试完整 Pipeline 流程"
|
||||
- "Pipeline 正确处理知乎链接并输出内容"
|
||||
artifacts:
|
||||
- path: "tests/test_platforms/test_zhihu.py"
|
||||
provides: "测试验证 ZhihuAdapter 整合"
|
||||
min_lines: 100
|
||||
exports: ["test_cookie_expired_triggers_login", "test_comment_format", "test_crawl_with_retry"]
|
||||
- path: "kb_workflow/pipeline/platforms/zhihu.py"
|
||||
provides: "ZhihuAdapter implementation"
|
||||
min_lines: 50
|
||||
- path: "tests/test_processors/test_download.py"
|
||||
provides: "Download processor integration test"
|
||||
key_links:
|
||||
- from: "ZhihuAdapter" to: "DownloadProcessor"
|
||||
via: "platform = adapter mapping"
|
||||
pattern: "platform_adapter_map"
|
||||
- from: "DownloadProcessor" to: "content files"
|
||||
via: "content_dir / filename"
|
||||
---
|
||||
|
||||
<objective>
|
||||
将 ZhihuAdapter 从使用 ZhihuPlaywrightExtractor (DOM 解析) 切换到使用 ZhihuCrawler (API 拦截)。
|
||||
|
||||
目的: 提爬取可靠性和,支持评论,保持 Pipeline 其容性
|
||||
|
||||
输出: 更新后的 ZhihuAdapter, 更新后的测试用例
|
||||
</objective>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@.planning/STATE.md
|
||||
@.planning/phases/08-crawler/08-03-SUMMARY.md
|
||||
|
||||
@.planning/phases/09-pipeline-integration/09-RESEARCH.md
|
||||
@.planning/phases/09-pipeline-integration/09-CONTEXT.md
|
||||
@.planning/phases/09-pipeline-integration/09-VALIDation.md
|
||||
|
||||
</context>
|
||||
|
||||
<execution_context>
|
||||
@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@/home/starmen1265/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<interfaces>
|
||||
<!-- Key types and contracts the executor needs. Executor should use these directly - no codebase exploration needed. -->
|
||||
|
||||
from kb_workflow/pipeline/platforms/base.py:
|
||||
```python
|
||||
class PlatformAdapter(ABC):
|
||||
@abstractmethod
|
||||
def get_content_type(self, url: str) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def fetch_metadata(self, content_id: str, content_type: str, source_url: str = None) -> Dict[str, Any]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def fetch_content(self, content_id: str, content_type: str, source_url: str = None) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str:
|
||||
pass
|
||||
```
|
||||
From kb_workflow/crawler/models.py
|
||||
```python
|
||||
class CrawlResult(BaseModel):
|
||||
success: bool
|
||||
content_type: str
|
||||
data: Optional[Union[AnswerData, ArticleData]]
|
||||
comments: Optional[List[CommentData]]
|
||||
error: Optional[str]
|
||||
|
||||
class AnswerData(BaseModel):
|
||||
id: int
|
||||
question_id: int
|
||||
question_title: str
|
||||
content: str
|
||||
excerpt: str
|
||||
metadata: ContentMetadata
|
||||
|
||||
class ArticleData(BaseModel):
|
||||
id: int
|
||||
title: str
|
||||
content: str
|
||||
excerpt: str
|
||||
metadata: ContentMetadata
|
||||
class CommentData(BaseModel):
|
||||
id: int
|
||||
content: str
|
||||
author: Optional[CommentAuthor]
|
||||
created_time: Optional[int]
|
||||
like_count: int
|
||||
child_count: int
|
||||
is_author: bool
|
||||
class CommentsResult(BaseModel):
|
||||
content_type: str
|
||||
content_id: int
|
||||
comments: List[CommentData]
|
||||
total_count: int
|
||||
class CommentAuthor(BaseModel):
|
||||
name: str
|
||||
url_token: Optional[str]
|
||||
class ContentMetadata(BaseModel):
|
||||
author: Optional[AuthorInfo]
|
||||
voteup_count: int
|
||||
comment_count: int
|
||||
created_time: Optional[int]
|
||||
updated_time: Optional[int]
|
||||
class AuthorInfo(BaseModel):
|
||||
name: str
|
||||
url_token: Optional[str]
|
||||
headline: Optional[str]
|
||||
`` </interfaces>
|
||||
from kb_workflow/crawler/zhihu_crawler.py
|
||||
```python
|
||||
class ZhihuCrawler:
|
||||
def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult:
|
||||
...
|
||||
|
||||
def crawl_article(self, url: str) -> CrawlResult:
|
||||
...
|
||||
|
||||
def crawl_comments(self, content_type: str, content_id: int, include_comments: bool = True) -> CommentsResult:
|
||||
...
|
||||
```
|
||||
from kb_workflow/auth/zhihu_login.py
|
||||
```python
|
||||
class ZhihuLogin
|
||||
def login_by_qrcode(self) -> ZhihuLoginResult:
|
||||
...
|
||||
|
||||
def validate_cookie(self, cookie_str: str = None) -> CookieValidationResult
|
||||
"""
|
||||
</interfaces>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 1: Add Cookie expiration auto login tests</name>
|
||||
<files>tests/test_platforms/test_zhihu.py</files>
|
||||
<action>
|
||||
添加 test用例验证 Cookie 过期时自动触发扫码登录流程:
|
||||
使用 pytest fixtures mock ZhihuLogin 和 ZhihuCrawler:
|
||||
测试场景:
|
||||
1. Cookie 有效时,爬取正常返回数据
|
||||
2. Cookie 过期时,触发扫码登录流程
|
||||
3. 触发登录后重试成功
|
||||
4. 触发登录失败时,抛出异常
|
||||
验证正确处理各种情况
|
||||
</action>
|
||||
<verify>
|
||||
<automated>pytest tests/test_platforms/test_zhihu.py -k "test_cookie_expired" -v</automated>
|
||||
</verify>
|
||||
<done>
|
||||
所有 Cookie 过期自动登录测试用例通过
|
||||
</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 2: Add comment formatting tests</name>
|
||||
<files>tests/test_platforms/test_zhihu.py</files>
|
||||
<action>
|
||||
添加测试用例验证评论格式化是否正确追加到正文末尾
|
||||
使用 mock 数据验证格式正确
|
||||
测试场景:
|
||||
1. 有评论时,格式化为包含评论列表的 Markdown
|
||||
2. 无评论时,返回空字符串
|
||||
</action>
|
||||
<verify>
|
||||
<automated>pytest tests/test_platforms/test_zhihu.py -k "test_comment_format" -v</automated>
|
||||
</verify>
|
||||
<done>
|
||||
所有评论格式化测试用例通过
|
||||
</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
1. 运行单元测试: `pytest tests/test_platforms/test_zhihu.py -x -v`
|
||||
2. 运行下载处理器集成测试: `pytest tests/test_processors/test_download.py -k "zhihu" -v`
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- 所有单元测试通过
|
||||
- 下载处理器集成测试通过
|
||||
- ZhihuAdapter 与 ZhihuCrawler 正确协作
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/09-pipeline-integration/09-02-SUMMARY.md`
|
||||
</output>
|
||||
Reference in New Issue
Block a user