From bee696b96d733069a78209a88a4e1e11f2299878 Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Tue, 17 Mar 2026 22:54:48 +0800 Subject: [PATCH] docs(09): complete phase 9 planning with 2 plans in 2 waves --- ...eline-import-failure-20260317.resolved.md} | 0 ...port-values-incorrect-20260317.resolved.md | 88 ++++++++ .../phases/07-login-auth/07-VERIFICATION.md | 151 +++++++++++++ .../09-pipeline-integration/09-02-plan.md | 209 ++++++++++++++++++ .../09-pipeline-integration/09-VALIDATION.md | 81 +++++++ .../09-pipeline-integration/09-VALIDation.md | 66 ++++++ .planning/state.md | 86 +++++++ 7 files changed, 681 insertions(+) rename .planning/debug/{kb-pipeline-import-failure-20260317.md => kb-pipeline-import-failure-20260317.resolved.md} (100%) create mode 100644 .planning/debug/report-values-incorrect-20260317.resolved.md create mode 100644 .planning/phases/07-login-auth/07-VERIFICATION.md create mode 100644 .planning/phases/09-pipeline-integration/09-02-plan.md create mode 100644 .planning/phases/09-pipeline-integration/09-VALIDATION.md create mode 100644 .planning/phases/09-pipeline-integration/09-VALIDation.md create mode 100644 .planning/state.md diff --git a/.planning/debug/kb-pipeline-import-failure-20260317.md b/.planning/debug/kb-pipeline-import-failure-20260317.resolved.md similarity index 100% rename from .planning/debug/kb-pipeline-import-failure-20260317.md rename to .planning/debug/kb-pipeline-import-failure-20260317.resolved.md diff --git a/.planning/debug/report-values-incorrect-20260317.resolved.md b/.planning/debug/report-values-incorrect-20260317.resolved.md new file mode 100644 index 0000000..dcd1960 --- /dev/null +++ b/.planning/debug/report-values-incorrect-20260317.resolved.md @@ -0,0 +1,88 @@ +--- +status: resolved +trigger: "技能汇报信息错误 - UP主名称、内容目录、中间目录路径" +created: 2026-03-17T00:00:00Z +updated: 2026-03-17T00:00:00Z +--- + +## Current Focus + +hypothesis: 三个独立的 bug 已确认根因，现在开始修复 +test: 修复代码并验证 +expecting: 修复后汇报信息正确 +next_action: 修复 download.py 中的 _extract_author 方法 + +## Symptoms + +expected: | + 1. UP主名称应该是 "HOLA小测佬" 而非 "406636263" + 2. 汇报的内容目录应该是归档路径 ~/work/zq/30.资源/AI分析/bilibili/20260317_BV1QXcCzfECp_analysis.md + 3. 中间目录应该是 ~/work/intermediate + +actual: | + 1. 汇报显示 UP主"406636263"（这是 mid） + 2. 汇报显示 ~/.openclaw/skills/kb-workflow/bilibili_BV1QXcCzfECp/（工作目录） + 3. 实际使用了 ~/.openclaw/skills/kb-workflow/ 作为中间目录 + +errors: 无错误，但汇报信息不正确 + +reproduction: | + 通过技能执行视频添加到知识库命令，观察汇报输出 + 视频：https://www.bilibili.com/video/BV1QXcCzfECp/ + +timeline: v1.1 架构重构后发现此问题 + +## Eliminated + + + +## Evidence + +- timestamp: 2026-03-17T00:00:00Z + checked: /home/starmen1265/work/kb-skills/kb_workflow/pipeline/processors/download.py + found: "_extract_author 方法返回 mid 而非 author 字段（第221-227行）" + implication: "问题1根因：返回 str(metadata['metadata'].get('mid', '')) 而非 author" + +- timestamp: 2026-03-17T00:00:00Z + checked: /home/starmen1265/work/kb-skills/kb_workflow/metadata/bilibili_extractor.py + found: "元数据结构中同时包含 author 和 mid 字段（第270-272行）" + implication: "修复方案：将 get('mid') 改为 get('author')" + +- timestamp: 2026-03-17T00:00:00Z + checked: /home/starmen1265/.openclaw/skills/kb-workflow/kb_workflow.py + found: "返回值使用 content_dir 而非 archive_file（第85行）" + implication: "问题2根因：汇报时取了 content_dir（工作目录）而非归档路径" + +- timestamp: 2026-03-17T00:00:00Z + checked: /home/starmen1265/.openclaw/skills/kb-workflow/kb_workflow.py + found: "第52行：out_dir = Path(output_dir) if output_dir else Path.cwd()" + implication: "问题3根因：未指定 output_dir 时使用 Path.cwd() 而非配置的 intermediate_path" + +- timestamp: 2026-03-17T00:00:00Z + checked: /home/starmen1265/work/kb-skills/kb_workflow/pipeline/config.py + found: "配置中有 intermediate_path 字段（第40-43行）" + implication: "修复方案：使用 get_settings().intermediate_path 作为默认值" + +## Resolution + +root_cause: | + 三个独立的 bug： + 1. download.py 的 _extract_author 方法返回 mid 而非 author + 2. kb_workflow.py 返回 content_dir 而非 archive_file + 3. kb_workflow.py 使用 cwd 而非配置的 intermediate_path + +fix: | + 1. download.py: 将 _extract_author 中的 get('mid') 改为 get('author') + 2. kb_workflow.py: 添加 ArchiveProcessor 到管道，从 results 中提取 archive_file 返回 + 3. kb_workflow.py: 使用 get_settings().intermediate_path 作为默认输出目录 + +verification: | + - 新增 5 个测试用例验证 author 提取逻辑 + - 所有新测试通过 + - Python 导入验证通过 + - 配置正确加载 intermediate_path + +files_changed: + - /home/starmen1265/work/kb-skills/kb_workflow/pipeline/processors/download.py + - /home/starmen1265/.openclaw/skills/kb-workflow/kb_workflow.py + - /home/starmen1265/work/kb-skills/tests/test_processors/test_download.py diff --git a/.planning/phases/07-login-auth/07-VERIFICATION.md b/.planning/phases/07-login-auth/07-VERIFICATION.md new file mode 100644 index 0000000..c153f39 --- /dev/null +++ b/.planning/phases/07-login-auth/07-VERIFICATION.md @@ -0,0 +1,151 @@ +--- +phase: 07-login-auth +verified: 2026-03-17T12:30:00Z +status: passed +score: 12/12 must-haves verified +re_verification: No + +human_verification: + - test: "运行 python3 kb-workflow/kb_workflow.py login 进行实际二维码扫码登录" + expected: "二维码图片生成在中间目录，扫码后 Cookie 保存到 ~/.kb_env" + why_human: "需要真实的知乎 APP 扫码，无法自动化测试" + - test: "运行 python3 kb-workflow/kb_workflow.py login --status 检查 Cookie 状态" + expected: "返回用户名和用户 ID 或提示 Cookie 过期" + why_human: "需要真实有效的 Cookie 才能验证 API 返回" + - test: "运行 python3 kb-workflow/kb_workflow.py login --import-cookie cookies.json 导入 Cookie" + expected: "Cookie 从 JSON 文件正确导入并保存" + why_human: "需要真实的 Chrome Cookie Editor 导出文件" +--- + +# Phase 7: 登录认证体系 Verification Report + +**Phase Goal:** 建立知乎登录认证机制，支持二维码扫码和手动 Cookie 两种方式，实现 Cookie 持久化和过期检测 +**Verified:** 2026-03-17T12:30:00Z +**Status:** passed +**Re-verification:** No - initial verification + +## Goal Achievement + +### Observable Truths + +| # | Truth | Status | Evidence | +| --- | --- | --- | --- | +| 1 | 用户可以通过命令行显示二维码并扫码登录知乎 | VERIFIED | ZhihuLogin.login_by_qrcode() 方法实现完整 (276 行)，login CLI 子命令存在 | +| 2 | 扫码失败时用户可以手动设置 Cookie 进行认证 | VERIFIED | parse_json_cookies() 支持 Chrome Cookie Editor JSON 格式，--import-cookie 参数实现 | +| 3 | Cookie 自动保存到本地文件，下次运行自动复用 | VERIFIED | CookieManager.save_cookie() 保存到 ~/.kb_env，get_settings() 自动加载 | +| 4 | Cookie 过期时系统自动检测并提示用户重新登录 | VERIFIED | validate_cookie() 通过 /api/v4/me 检测，返回 CookieValidationResult | +| 5 | 用户可以访问知乎登录页面 | VERIFIED | LOGIN_URL = "https://www.zhihu.com/signin"，goto() 调用正确 | +| 6 | 系统可以从 Canvas 元素截取二维码图片 | VERIFIED | _find_qrcode_canvas() 多选择器回退，canvas.screenshot() 调用 | +| 7 | 二维码图片保存到中间目录 | VERIFIED | _save_qrcode_image() 保存为 zhihu_qrcode_{timestamp}.png | +| 8 | 系统可以轮询检测 z_c0 Cookie 判断登录状态 | VERIFIED | check_login_cookie() + tenacity Retrying 实现 60 秒轮询 | +| 9 | 用户可以从 JSON 文件导入 Cookie | VERIFIED | parse_json_cookies() 实现并测试通过 | +| 10 | Cookie 可以保存到 ~/.kb_env 文件 | VERIFIED | CookieManager.save_cookie() 写入文件，权限 600 | +| 11 | 保存的 Cookie 下次运行时可以自动加载 | VERIFIED | load_cookie() 读取，get_settings.cache_clear() 确保刷新 | +| 12 | 用户可以查看当前 Cookie 状态 | VERIFIED | login --status 参数调用 validate_cookie() | + +**Score:** 12/12 truths verified + +### Required Artifacts + +| Artifact | Expected | Status | Details | +| --- | --- | --- | --- | +| `kb_workflow/auth/zhihu_login.py` | 二维码登录核心逻辑 (min 100 lines) | VERIFIED | 276 行，导出 ZhihuLogin, ZhihuLoginResult, CookieValidationResult | +| `kb_workflow/auth/cookie_manager.py` | Cookie 导入和持久化 (min 80 lines) | VERIFIED | 120 行，导出 CookieManager, parse_json_cookies | +| `kb_workflow/kb_workflow.py` | login CLI 子命令 | VERIFIED | login_parser subparser，login_command() 函数 | +| `tests/test_auth/test_qrcode_login.py` | 登录逻辑单元测试 | VERIFIED | 8 个测试全部通过 | +| `tests/test_auth/test_cookie_import.py` | Cookie 导入测试 | VERIFIED | 6 个测试全部通过 | +| `tests/test_auth/test_cookie_persistence.py` | Cookie 持久化测试 | VERIFIED | 8 个测试全部通过 | +| `tests/test_auth/test_cookie_validation.py` | Cookie 验证测试 | VERIFIED | 5 个测试全部通过 | +| `tests/test_auth/test_cli_login.py` | CLI 命令测试 | VERIFIED | 6 个测试全部通过 | + +### Key Link Verification + +| From | To | Via | Status | Details | +| --- | --- | --- | --- | --- | +| `zhihu_login.py` | `PlaywrightBrowser` | import | WIRED | `from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser` | +| `ZhihuLogin.login_by_qrcode()` | `canvas.screenshot()` | Playwright API | WIRED | canvas.screenshot() 调用存在 | +| `CookieManager.save_cookie()` | `~/.kb_env` | 文件写入 | WIRED | Path.home() / ".kb_env"，chmod 600 | +| `parse_json_cookies()` | Chrome Cookie Editor JSON | json.load + 过滤 | WIRED | zhihu.com 域名过滤实现 | +| `kb_workflow.py login` | `ZhihuLogin.login_by_qrcode()` | 同步调用 | WIRED | login.login_by_qrcode() 调用 | +| `ZhihuLogin.login_by_qrcode()` | `CookieManager.save_cookie()` | 登录成功后保存 | WIRED | manager.save_cookie(result.cookie_str) | +| `validate_cookie()` | `/api/v4/me` | Playwright request API | WIRED | VALIDATION_URL = "https://www.zhihu.com/api/v4/me" | + +### Requirements Coverage + +| Requirement | Source Plan | Description | Status | Evidence | +| --- | --- | --- | --- | --- | +| AUTH-01 | 07-01 | 用户可通过二维码扫码登录知乎 | SATISFIED | ZhihuLogin.login_by_qrcode() 完整实现，测试通过 | +| AUTH-02 | 07-02 | 用户可手动设置 Cookie 进行认证 | SATISFIED | parse_json_cookies() + --import-cookie 参数 | +| AUTH-03 | 07-02 | Cookie 自动持久化到本地文件，下次自动复用 | SATISFIED | CookieManager.save_cookie/load_cookie + get_settings 缓存清除 | +| AUTH-04 | 07-03 | Cookie 过期时自动检测并提示重新登录 | SATISFIED | validate_cookie() + CookieValidationResult | + +### Anti-Patterns Found + +| File | Line | Pattern | Severity | Impact | +| --- | --- | --- | --- | --- | +| (none) | - | - | - | auth 模块无 TODO/FIXME/placeholder | + +**Note:** 在 `kb_workflow/pipeline/platforms/bilibili.py` 中存在 TODO 注释，但这不在阶段7范围内。 + +### Human Verification Required + +虽然所有自动化测试通过，以下场景需要人工验证： + +#### 1. 二维码扫码登录端到端测试 + +**Test:** 运行 `python3 kb-workflow/kb_workflow.py login` +**Expected:** +- 二维码图片生成在 `~/work/intermediate/zhihu_qrcode_*.png` +- 使用知乎 APP 扫码后命令返回 `{"success": true, "cookie_str": "..."}` +- `~/.kb_env` 文件包含 `ZHIHU_COOKIE="..."` 行 + +**Why human:** 需要真实的知乎 APP 扫码，无法自动化测试 + +#### 2. Cookie 状态检查 + +**Test:** 运行 `python3 kb-workflow/kb_workflow.py login --status` +**Expected:** +- 有效 Cookie: 返回 `{"valid": true, "username": "...", "user_id": "..."}` +- 过期 Cookie: 返回 `{"valid": false, "reason": "Cookie expired"}` + +**Why human:** 需要真实有效的 Cookie 才能验证 API 返回 + +#### 3. JSON Cookie 导入 + +**Test:** 使用 Chrome Cookie Editor 导出 cookies.json，运行 `python3 kb-workflow/kb_workflow.py login --import-cookie cookies.json` +**Expected:** Cookie 正确导入并保存到 ~/.kb_env + +**Why human:** 需要真实的 Chrome Cookie Editor 导出文件 + +### Test Results + +``` +============================= test session starts ============================== +platform linux -- Python 3.12.3, pytest-9.0.2, pluggy-1.6.0 +tests/test_auth/ - 33 items + +PASSED: 33/33 tests +- test_qrcode_login.py: 8 tests +- test_cookie_import.py: 6 tests +- test_cookie_persistence.py: 8 tests +- test_cookie_validation.py: 5 tests +- test_cli_login.py: 6 tests +``` + +### Summary + +**Phase 7 (登录认证体系) 验证通过:** + +1. **AUTH-01 二维码扫码登录** - ZhihuLogin 类实现完整，支持多选择器回退、可配置超时、二维码保存 +2. **AUTH-02 手动 Cookie 导入** - parse_json_cookies 支持 Chrome Cookie Editor 格式，域名过滤 +3. **AUTH-03 Cookie 持久化** - CookieManager 保存到 ~/.kb_env，权限 600，缓存清除 +4. **AUTH-04 Cookie 过期检测** - validate_cookie 通过 /api/v4/me 验证，返回详细状态 + +所有 33 个单元测试通过，关键连接点验证通过，无阻塞性 anti-pattern。 + +**Ready for Phase 8 (内容爬取实现)** + +--- + +_Verified: 2026-03-17T12:30:00Z_ +_Verifier: Claude (gsd-verifier)_ diff --git a/.planning/phases/09-pipeline-integration/09-02-plan.md b/.planning/phases/09-pipeline-integration/09-02-plan.md new file mode 100644 index 0000000..a6e5861 --- /dev/null +++ b/.planning/phases/09-pipeline-integration/09-02-plan.md @@ -0,0 +1,209 @@ +--- +phase: 09-pipeline-integration +plan: 02 +type: execute +wave: 1 +depends_on: [09-01] +files_modified: [] +autonomous: true +requirements: [] +user_setup: [] + +must_haves: + truths: + - "用户可通过命令行测试完整 Pipeline 流程" + - "Pipeline 正确处理知乎链接并输出内容" + artifacts: + - path: "tests/test_platforms/test_zhihu.py" + provides: "测试验证 ZhihuAdapter 整合" + min_lines: 100 + exports: ["test_cookie_expired_triggers_login", "test_comment_format", "test_crawl_with_retry"] + - path: "kb_workflow/pipeline/platforms/zhihu.py" + provides: "ZhihuAdapter implementation" + min_lines: 50 + - path: "tests/test_processors/test_download.py" + provides: "Download processor integration test" + key_links: + - from: "ZhihuAdapter" to: "DownloadProcessor" + via: "platform = adapter mapping" + pattern: "platform_adapter_map" + - from: "DownloadProcessor" to: "content files" + via: "content_dir / filename" +--- + + +将 ZhihuAdapter 从使用 ZhihuPlaywrightExtractor (DOM 解析) 切换到使用 ZhihuCrawler (API 拦截)。 + +目的: 提爬取可靠性和，支持评论,保持 Pipeline 其容性 + +输出: 更新后的 ZhihuAdapter, 更新后的测试用例 + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/08-crawler/08-03-SUMMARY.md + +@.planning/phases/09-pipeline-integration/09-RESEARCH.md +@.planning/phases/09-pipeline-integration/09-CONTEXT.md +@.planning/phases/09-pipeline-integration/09-VALIDation.md + + + + +@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md +@/home/starmen1265/.claude/get-shit-done/templates/summary.md + + + + + +from kb_workflow/pipeline/platforms/base.py: +```python +class PlatformAdapter(ABC): + @abstractmethod + def get_content_type(self, url: str) -> str: + pass + + @abstractmethod + async def fetch_metadata(self, content_id: str, content_type: str, source_url: str = None) -> Dict[str, Any]: + pass + + @abstractmethod + async def fetch_content(self, content_id: str, content_type: str, source_url: str = None) -> str: + pass + + @abstractmethod + def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str: + pass +``` +From kb_workflow/crawler/models.py +```python +class CrawlResult(BaseModel): + success: bool + content_type: str + data: Optional[Union[AnswerData, ArticleData]] + comments: Optional[List[CommentData]] + error: Optional[str] + +class AnswerData(BaseModel): + id: int + question_id: int + question_title: str + content: str + excerpt: str + metadata: ContentMetadata + +class ArticleData(BaseModel): + id: int + title: str + content: str + excerpt: str + metadata: ContentMetadata +class CommentData(BaseModel): + id: int + content: str + author: Optional[CommentAuthor] + created_time: Optional[int] + like_count: int + child_count: int + is_author: bool +class CommentsResult(BaseModel): + content_type: str + content_id: int + comments: List[CommentData] + total_count: int +class CommentAuthor(BaseModel): + name: str + url_token: Optional[str] +class ContentMetadata(BaseModel): + author: Optional[AuthorInfo] + voteup_count: int + comment_count: int + created_time: Optional[int] + updated_time: Optional[int] +class AuthorInfo(BaseModel): + name: str + url_token: Optional[str] + headline: Optional[str] +`` +from kb_workflow/crawler/zhihu_crawler.py +```python +class ZhihuCrawler: + def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult: + ... + + def crawl_article(self, url: str) -> CrawlResult: + ... + + def crawl_comments(self, content_type: str, content_id: int, include_comments: bool = True) -> CommentsResult: + ... +``` +from kb_workflow/auth/zhihu_login.py +```python +class ZhihuLogin + def login_by_qrcode(self) -> ZhihuLoginResult: + ... + + def validate_cookie(self, cookie_str: str = None) -> CookieValidationResult + """ + + + + + + Task 1: Add Cookie expiration auto login tests + tests/test_platforms/test_zhihu.py + +添加 test用例验证 Cookie 过期时自动触发扫码登录流程: +使用 pytest fixtures mock ZhihuLogin 和 ZhihuCrawler: +测试场景: +1. Cookie 有效时,爬取正常返回数据 +2. Cookie 过期时,触发扫码登录流程 +3. 触发登录后重试成功 +4. 触发登录失败时,抛出异常 +验证正确处理各种情况 + + + pytest tests/test_platforms/test_zhihu.py -k "test_cookie_expired" -v + + +所有 Cookie 过期自动登录测试用例通过 + + + + + Task 2: Add comment formatting tests + tests/test_platforms/test_zhihu.py + +添加测试用例验证评论格式化是否正确追加到正文末尾 +使用 mock 数据验证格式正确 +测试场景: +1. 有评论时,格式化为包含评论列表的 Markdown +2. 无评论时,返回空字符串 + + + pytest tests/test_platforms/test_zhihu.py -k "test_comment_format" -v + + +所有评论格式化测试用例通过 + + + + + + +1. 运行单元测试: `pytest tests/test_platforms/test_zhihu.py -x -v` +2. 运行下载处理器集成测试: `pytest tests/test_processors/test_download.py -k "zhihu" -v` + + + +- 所有单元测试通过 +- 下载处理器集成测试通过 +- ZhihuAdapter 与 ZhihuCrawler 正确协作 + + + +After completion, create `.planning/phases/09-pipeline-integration/09-02-SUMMARY.md` + diff --git a/.planning/phases/09-pipeline-integration/09-VALIDATION.md b/.planning/phases/09-pipeline-integration/09-VALIDATION.md new file mode 100644 index 0000000..05f4fab --- /dev/null +++ b/.planning/phases/09-pipeline-integration/09-VALIDATION.md @@ -0,0 +1,81 @@ +--- +phase: 09 +slug: pipeline-integration +status: draft +nyquist_compliant: false +wave_0_complete: false +created: 2026-03-17 +--- + +# Phase 09 — Validation Strategy + +> Per-phase validation contract for feedback sampling during execution. + +--- + +## Test Infrastructure + +| Property | Value | +|----------|-------| +| **Framework** | pytest | +| **Config file** | pytest.ini | +| **Quick run command** | `pytest tests/test_platforms/test_zhihu.py -x -v` | +| **Full suite command** | `pytest tests/ -v` | +| **Estimated runtime** | ~30 seconds | + +--- + +## Sampling Rate + +- **After every task commit:** Run `pytest tests/test_platforms/test_zhihu.py -x -v` +- **After every plan wave:** Run `pytest tests/test_platforms/ tests/test_processors/test_download.py -v` +- **Before `/gsd:verify-work`:** Full suite must be green +- **Max feedback latency:** 30 seconds + +--- + +## Per-Task Verification Map + +| Task ID | Plan | Wave | Requirement | Test Type | Automated Command | File Exists | Status | +|---------|------|------|-------------|-----------|-------------------|-------------|--------| +| 09-01-01 | 01 | 1 | PIPE-01 | unit | `pytest tests/test_platforms/test_zhihu.py -k "test_fetch" -v` | ✅ 需修改 | ⬜ pending | +| 09-01-02 | 01 | 1 | PIPE-01 | unit | `pytest tests/test_platforms/test_zhihu.py -k "test_crawl" -v` | ✅ 需修改 | ⬜ pending | +| 09-01-03 | 01 | 1 | PIPE-02 | integration | `pytest tests/test_processors/test_download.py -k "zhihu" -v` | ✅ 已存在 | ⬜ pending | +| 09-01-04 | 01 | 1 | PIPE-03 | unit | `pytest tests/test_pipeline/test_idempotency.py -v` | ✅ 已存在 | ⬜ pending | +| 09-01-05 | 01 | 1 | PIPE-01 | unit | `pytest tests/test_platforms/test_zhihu.py -k "test_cookie" -v` | ❌ W0 | ⬜ pending | +| 09-01-06 | 01 | 1 | PIPE-01 | unit | `pytest tests/test_platforms/test_zhihu.py -k "test_comment" -v` | ❌ W0 | ⬜ pending | + +*Status: ⬜ pending · ✅ green · ❌ red · ⚠️ flaky* + +--- + +## Wave 0 Requirements + +- [ ] `tests/test_platforms/test_zhihu.py` — 更新测试使用 ZhihuCrawler mock +- [ ] `tests/test_platforms/test_zhihu.py::test_cookie_expired_login` — Cookie 过期自动触发登录流程测试 +- [ ] `tests/test_platforms/test_zhihu.py::test_comment_format` — 评论追加到正文格式验证测试 +- [ ] `tests/test_platforms/test_zhihu.py::test_crawlresult_to_metadata` — CrawlResult 到 metadata 转换正确性测试 + +*现有测试基础设施完善，主要是更新和新增 ZhihuAdapter 相关测试* + +--- + +## Manual-Only Verifications + +| Behavior | Requirement | Why Manual | Test Instructions | +|----------|-------------|------------|-------------------| +| 扫码登录实际流程 | PIPE-01 | 需要 Zhihu API 真实交互 | 1. 删除 Cookie 文件 2. 运行 Pipeline 3. 验证二维码显示 4. 扫码后流程继续 | +| 评论实际爬取 | PIPE-01 | 需要 Zhihu API 真实交互 | 1. 运行 Pipeline 处理知乎链接 2. 验证评论追加到 content.md | + +--- + +## Validation Sign-Off + +- [ ] All tasks have `` verify or Wave 0 dependencies +- [ ] Sampling continuity: no 3 consecutive tasks without automated verify +- [ ] Wave 0 covers all MISSING references +- [ ] No watch-mode flags +- [ ] Feedback latency < 30s +- [ ] `nyquist_compliant: true` set in frontmatter + +**Approval:** pending diff --git a/.planning/phases/09-pipeline-integration/09-VALIDation.md b/.planning/phases/09-pipeline-integration/09-VALIDation.md new file mode 100644 index 0000000..3a31b82 --- /dev/null +++ b/.planning/phases/09-pipeline-integration/09-VALIDation.md @@ -0,0 +1,66 @@ +--- +phase: 09 +slug: pipeline-integration +status: draft +nyquist_compliant: false +wave_0_complete: false +created: 2026-03-17 +--- + +# Phase 09 — Validation Strategy + +> Per-phase validation contract for feedback sampling during execution. + +--- + +## Test Infrastructure + +| Property | Value | +|----------|-------| +| **Framework** | pytest | +| **Config file** | pytest.ini | +| **Quick run command** | `pytest tests/test_platforms/test_zhihu.py -x -v` | +| **full suite command** | `pytest tests/ -v` | +| **estimated runtime** | ~30 seconds | +--- + +## Sampling Rate +- **After every task commit:** Run `pytest tests/test_platforms/test_zhihu.py -x -v` +- **after every plan wave:** run `pytest tests/test_platforms/ tests/test_processors/test_download.py -v` +- **Before `/gsd:verify-work`:** Full suite must be green +- **Max feedback latency:** 30 seconds +--- + +## Per-task verification map +| Task ID | Plan | Wave | Requirement | Test Type | Automated command | File exists | status | +|---------|------|------|-------------|-----------|-------------------|-------------|--------| +| 09-01-01 | 01 | 1 | PIPE-01 | unit | `pytest tests/test_platforms/test_zhihu.py -k "test_fetch" -v` | ✅ needs update | ⬜ pending | +| 09-01-02 | 01 | 1 | PIPE-01 | unit | `pytest tests/test_platforms/test_zhihu.py -k "test_crawl" -v` | ✅ needs update | ⬜ pending | +| 09-01-03 | 01 | 1 | PIPE-02 | integration | `pytest tests/test_processors/test_download.py -k "zhihu" -v` | ✅ exists | ⬜ pending | +| 09-01-04 | 01 | 1 | PIPE-03 | unit | `pytest tests/test_pipeline/test_idempotency.py -v` | ✅ exists | ⬜ pending | +| 09-01-05 | 01 | 1 | PIPE-01 | unit | `pytest tests/test_platforms/test_zhihu.py -k "test_cookie" -v` | ❌ W0 | ⬜ pending | +| 09-01-06 | 01 | 1 | PIPE-01 | unit | `pytest tests/test_platforms/test_zhihu.py -k "test_comment" -v` | ❌ W0 | ⬜ pending | +*Status: ⬜ pending · ✅ green · ❌ red | ⚠️ flaky* + +--- +## Wave 0 requirements +- [ ] `tests/test_platforms/test_zhihu.py` - 更新测试使用 ZhihuCrawler mock +- [ ] `tests/test_platforms/test_zhihu.py::test_cookie_expired_triggers_login` - Cookie 过期自动触发登录流程测试 +- [ ] `tests/test_platforms/test_zhihu.py::test_comment_format` - 评论追加到正文格式验证测试 +- [ ] `tests/test_platforms/test_zhihu.py::test_crawlresult_to_metadata` - CrawlResult 到 metadata 转换正确性测试 +*现有测试基础设施完善,主要是更新和新增 ZhihuAdapter 相关测试 +--- +## manual-only verifications +| behavior | requirement | why manual | test instructions | +|----------|-------------|------------|-------------------| +| 扫码登录实际流程 | PIPE-01 | 需要 Zhihu API 真实交互 | 1. 删除 Cookie 文件 2. 运行 Pipeline 3. 陑证二维码显示 4. 扫码后流程继续 | +| 评论实际爬取 | PIPE-01 | 需要 Zhihu API 瓟实交互 | 1. 运行 Pipeline 处理知乎链接 2. 验证评论追加到 content.md | +--- +## validation sign-off +- [ ] All tasks have `` verify or Wave 0 dependencies +- [ ] Sampling continuity: no 3 consecutive tasks without automated verify +- [ ] wave 0 covers all MISSING references +- [ ] no watch-mode flags +- [ ] feedback latency < 30s +- [ ] `nyquist_compliant: true` set in frontmatter +**approval:** pending diff --git a/.planning/state.md b/.planning/state.md new file mode 100644 index 0000000..bcd506a --- /dev/null +++ b/.planning/state.md @@ -0,0 +1,86 @@ +--- +gsd_state_version: 1.0 +milestone: v1.2 +milestone_name: Zhihu Crawler +status: planning +stopped_at: Phase 9 plans created +last_updated: "2026-03-17T15:00:00Z" +last_activity: 2026-03-17 - Phase 09 plans created + +progress: + total_phases: 9 + completed_phases: 8 + total_plans: 26 + completed_plans: 24 + percent: 100 +--- + +# Project State + +## Project Reference + +See: .planning/PROJECT.md (updated 2026-03-17) + +**Core value:** 可靠、可维护的处理管道 - 每个步骤独立、可验证、可恢复 +**Current focus:** v1.2 Zhihu Crawler - Phase 9 planning + + progress + +## Current Position + +Phase: 9 of 9 (Pipeline Integration - 整合) +Plan: 0 of 02 +Status: Phase 9 plans created, ready to execution +Last activity: 2026-03-17 - Phase 09 plans created + +Progress: [██████████] 100% (v1.2 milestone - Phase 7 complete) + +## Performance Metrics +**Velocity:** +- Total plans completed: 3 (v1.2) +- Average duration: 11 min +- Total execution time: 32 min +| Plan | Duration | Tasks | Files | Date | +|------|----------|-------|-------|------| +| 07-01 | 15min | 2 | 5 | 2026-03-17 | +| 07-02 | 5min | 2 | 4 | 2026-03-17 | +| 07-03 | 12min | 3 | 4 | 2026-03-17 | +| Phase 08 P01 | 3min | 2 tasks | 8 files | +| Phase 08-crawler P02 | 5min | 2 tasks | 3 files | +| Phase 08 P03 | 15min | 2 tasks | 5 files | +## Accumulated Context +### Decisions +Key decisions from v1.0 (preserved): +- 管道模型 - BaseProcessor + Pipeline 架构 +- 步骤级状态 - WorkflowStateManager +- Schema 验证 - Pydantic v2 +- 顺序执行 - 简单性和可调试性 +- CLI subparsers - 可扩展命令结构 +v1.1 decisions: +- 中间目录统一到 `~/work/intermediate/analysis/` +- 归档只保留 md 文件,不归档 json 文件 +- 最终目录按平台分组,不再按时间创建子目录 +v1.2 decisions: +- 知乎登录策略: 二维码扫码 + 手动 Cookie 降级 +- 参考 MediaCrawler 实现,自实现而非直接依赖 +- 使用 playwright 作为浏览器自动化框架 +- 整合进现有 Pipeline 架构 +- Cookie 文件权限设置为 600 保证安全性 +- 保存 Cookie 后清除 get_settings 缓存确保配置刷新 +- 使用 tenacity Retrying 对象而非 @retry 装饰器支持可配置超时 +- 多个 canvas 选择器降级提高 QR 码检测鲁棒性 +- login CLI 命令支持三种模式: 扫码登录、导入 Cookie、 状态检查 +- [Phase 08-02]: use Playwright request API for direct comment API calls instead of route interception +- [Phase ?]: - [Phase 08-02]: only fetch first-level comments (no nested replies) +- [Phase ?]: - [Phase 08-02]: use hot sorting (order=hot) for popular comments +- [Phase 09-01]: 修改 ZhihuAdapter 内部使用 ZhihuCrawler, 保持接口不变 +- [Phase 09-01]: Cookie 过期自动触发扫码登录 +- [Phase 09-01]: 评论追加到正文末尾 (Markdown 格式) +- [Phase 09-01]: 直接报错不降级到旧版 ZhihuPlaywrightExtractor +### Blockers/Concerns +None. +## Session Continuity +Last session: 2026-03-17T15:00:00Z +Stopped at: Phase 9 plans created +resume file: .planning/phases/09-pipeline-integration/09-CONTEXT.md +**Next step:** Execute `/gsd:execute-phase 09`