docs(02): create phase 2 plans for platform skills
This commit is contained in:
@@ -0,0 +1,336 @@
|
||||
---
|
||||
phase: 02-platform-skills
|
||||
plan: 02
|
||||
type: execute
|
||||
wave: 1
|
||||
depends_on: [02-01]
|
||||
files_modified:
|
||||
- kb_pipeline/platforms/__init__.py
|
||||
- kb_pipeline/platforms/base.py
|
||||
- kb_pipeline/platforms/bilibili.py
|
||||
- kb_pipeline/__init__.py
|
||||
- tests/test_platforms/__init__.py
|
||||
- tests/test_platforms/conftest.py
|
||||
- tests/test_platforms/test_bilibili.py
|
||||
autonomous: true
|
||||
requirements: [PLAT-01, PLAT-02]
|
||||
user_setup: []
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "输入BV号,获取B站视频元数据(标题、作者、时长、描述等)"
|
||||
- "输入CV号,获取B站专栏元数据和正文内容"
|
||||
- "元数据保存为结构化JSON文件"
|
||||
- "内容保存为Markdown文件(YAML Frontmatter + 正文)"
|
||||
- "403错误触发降级策略(API -> Playwright)"
|
||||
- "网络错误自动重试3次"
|
||||
artifacts:
|
||||
- path: "kb_pipeline/platforms/base.py"
|
||||
provides: "平台适配器基类"
|
||||
exports: ["PlatformAdapter"]
|
||||
- path: "kb_pipeline/platforms/bilibili.py"
|
||||
provides: "B站平台适配器"
|
||||
exports: ["BilibiliAdapter"]
|
||||
min_lines: 100
|
||||
- path: "tests/test_platforms/test_bilibili.py"
|
||||
provides: "B站适配器测试"
|
||||
min_lines: 80
|
||||
key_links:
|
||||
- from: "kb_pipeline/platforms/bilibili.py"
|
||||
to: "kb_pipeline/retry.py"
|
||||
via: "retry_on_network_error decorator"
|
||||
pattern: "@retry_on_network_error"
|
||||
- from: "kb_pipeline/platforms/bilibili.py"
|
||||
to: "kb_pipeline/errors.py"
|
||||
via: "NetworkError, AuthError"
|
||||
pattern: "raise NetworkError"
|
||||
- from: "kb_pipeline/platforms/bilibili.py"
|
||||
to: "kb-bilibili-metadata/bilibili_extractor.py"
|
||||
via: "BilibiliExtractor class"
|
||||
pattern: "from kb_bilibili_metadata"
|
||||
---
|
||||
|
||||
<objective>
|
||||
创建 B站平台适配器,封装视频和专栏的元数据提取与内容获取逻辑,复用现有 BilibiliExtractor 实现。
|
||||
|
||||
Purpose: 为 B站视频和专栏提供统一的元数据提取和内容下载接口,支持错误重试和降级策略。
|
||||
Output: PlatformAdapter 基类、BilibiliAdapter 实现类
|
||||
</objective>
|
||||
|
||||
<execution_context>
|
||||
@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@/home/starmen1265/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@.planning/STATE.md
|
||||
@.planning/phases/02-platform-skills/02-CONTEXT.md
|
||||
@.planning/phases/02-platform-skills/02-RESEARCH.md
|
||||
|
||||
<interfaces>
|
||||
<!-- Phase 1 和 Plan 01 提供的接口 -->
|
||||
|
||||
From kb_pipeline/link_parser.py (Plan 01):
|
||||
```python
|
||||
@dataclass
|
||||
class ParsedLink:
|
||||
platform: str # "bilibili" | "zhihu"
|
||||
content_type: str # "video" | "article" | "answer" | "zhuanlan"
|
||||
content_id: str # BV号、CV号等
|
||||
original_url: str
|
||||
resolved_url: str
|
||||
|
||||
class LinkParser:
|
||||
def parse(self, url: str) -> Optional[ParsedLink]:
|
||||
```
|
||||
|
||||
From kb_pipeline/schemas.py:
|
||||
```python
|
||||
class StepInput(BaseModel):
|
||||
content_id: str
|
||||
platform: Literal["bilibili", "zhihu"]
|
||||
|
||||
class StepOutput(BaseModel):
|
||||
success: bool
|
||||
error_message: Optional[str]
|
||||
completed_at: datetime
|
||||
```
|
||||
|
||||
From kb_pipeline/errors.py:
|
||||
```python
|
||||
class NetworkError(KBError):
|
||||
def __init__(self, step_name, input_summary, status_code=None, cause=None)
|
||||
|
||||
class AuthError(KBError):
|
||||
def __init__(self, step_name, input_summary, auth_type=None, cause=None)
|
||||
```
|
||||
|
||||
From kb_pipeline/retry.py:
|
||||
```python
|
||||
retry_on_network_error = retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=1, min=2, max=10),
|
||||
)
|
||||
```
|
||||
|
||||
From kb_pipeline/config.py:
|
||||
```python
|
||||
class KBSettings(BaseSettings):
|
||||
base_path: str
|
||||
archive_path: str
|
||||
```
|
||||
</interfaces>
|
||||
|
||||
<existing_code>
|
||||
<!-- 现有 BilibiliExtractor 的核心方法 -->
|
||||
|
||||
From kb-bilibili-metadata/bilibili_extractor.py:
|
||||
```python
|
||||
class BilibiliExtractor:
|
||||
def extract_video_metadata(self, bvid: str) -> Dict[str, Any]:
|
||||
# 返回 {metadata: {...}, stats: {...}, description: str}
|
||||
|
||||
def extract_article_metadata(self, cvid: str) -> Dict[str, Any]:
|
||||
# 返回 {cvid, title, author, publish_time, ...}
|
||||
|
||||
async def _extract_video_by_api(self, bvid: str) -> Dict[str, Any]:
|
||||
# 使用 httpx 调用 API
|
||||
|
||||
DEFAULT_HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 ...',
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
}
|
||||
```
|
||||
</existing_code>
|
||||
</context>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto" tdd="true">
|
||||
<name>Task 1: Write interface contracts for PlatformAdapter base class</name>
|
||||
<files>kb_pipeline/platforms/__init__.py, kb_pipeline/platforms/base.py, tests/test_platforms/__init__.py, tests/test_platforms/conftest.py</files>
|
||||
<behavior>
|
||||
- Test 1: PlatformAdapter 是抽象基类,不能直接实例化
|
||||
- Test 2: 子类必须实现 get_content_type, fetch_metadata, fetch_content, get_output_filename 方法
|
||||
</behavior>
|
||||
<action>
|
||||
1. 创建 kb_pipeline/platforms/__init__.py:
|
||||
- 导出 PlatformAdapter, BilibiliAdapter
|
||||
|
||||
2. 创建 kb_pipeline/platforms/base.py:
|
||||
```python
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Any, Optional
|
||||
from pathlib import Path
|
||||
|
||||
class PlatformAdapter(ABC):
|
||||
"""平台适配器基类"""
|
||||
|
||||
@abstractmethod
|
||||
def get_content_type(self, url: str) -> str:
|
||||
"""识别内容类型 (video/article/answer/zhuanlan)"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def fetch_metadata(self, content_id: str, content_type: str) -> Dict[str, Any]:
|
||||
"""获取元数据"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def fetch_content(self, content_id: str, content_type: str) -> str:
|
||||
"""获取正文内容(Markdown格式)"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str:
|
||||
"""生成输出文件名"""
|
||||
pass
|
||||
```
|
||||
|
||||
3. 创建测试目录和 fixtures:
|
||||
- tests/test_platforms/__init__.py (空文件)
|
||||
- tests/test_platforms/conftest.py: 定义共享 fixtures (mock_api_response, tmp_content_dir)
|
||||
</action>
|
||||
<verify>
|
||||
<automated>pytest tests/test_platforms/ -x -v -k "base"</automated>
|
||||
</verify>
|
||||
<done>PlatformAdapter 基类定义完成,子类接口明确</done>
|
||||
</task>
|
||||
|
||||
<task type="auto" tdd="true">
|
||||
<name>Task 2: Implement BilibiliAdapter for video metadata</name>
|
||||
<files>kb_pipeline/platforms/bilibili.py, tests/test_platforms/test_bilibili.py</files>
|
||||
<behavior>
|
||||
- Test 1: fetch_metadata("BVxxx", "video") 返回正确的视频元数据
|
||||
- Test 2: 403错误时触发降级策略(API -> Playwright)
|
||||
- Test 3: 网络错误自动重试3次
|
||||
- Test 4: get_output_filename 返回正确格式 "bilibili_BVxxx_metadata.json"
|
||||
</behavior>
|
||||
<action>
|
||||
创建 kb_pipeline/platforms/bilibili.py:
|
||||
```python
|
||||
from kb_pipeline.platforms.base import PlatformAdapter
|
||||
from kb_pipeline.errors import NetworkError, AuthError
|
||||
from kb_pipeline.retry import retry_on_network_error
|
||||
from kb_bilibili_metadata.bilibili_extractor import BilibiliExtractor
|
||||
|
||||
class BilibiliAdapter(PlatformAdapter):
|
||||
PLATFORM = "bilibili"
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||
self.extractor = BilibiliExtractor(headless=headless, timeout=timeout)
|
||||
|
||||
def get_content_type(self, url: str) -> str:
|
||||
# 从 URL 识别 video 或 article
|
||||
|
||||
@retry_on_network_error
|
||||
async def fetch_metadata(self, content_id: str, content_type: str) -> Dict[str, Any]:
|
||||
if content_type == "video":
|
||||
return self.extractor.extract_video_metadata(content_id)
|
||||
elif content_type == "article":
|
||||
return self.extractor.extract_article_metadata(content_id)
|
||||
raise ValueError(f"Unsupported content type: {content_type}")
|
||||
|
||||
async def fetch_content(self, content_id: str, content_type: str) -> str:
|
||||
# 视频内容获取留给 Phase 3 (ASR)
|
||||
# 专栏内容需要额外实现
|
||||
pass
|
||||
|
||||
def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str:
|
||||
return f"bilibili_{content_id}_{suffix}"
|
||||
```
|
||||
|
||||
创建 tests/test_platforms/test_bilibili.py:
|
||||
- 使用 mock 模拟 BilibiliExtractor 的返回值
|
||||
- 测试视频元数据提取
|
||||
- 测试错误处理和重试机制
|
||||
</action>
|
||||
<verify>
|
||||
<automated>pytest tests/test_platforms/test_bilibili.py::test_video_metadata -x -v</automated>
|
||||
</verify>
|
||||
<done>BilibiliAdapter 能正确提取B站视频元数据,支持错误重试</done>
|
||||
</task>
|
||||
|
||||
<task type="auto" tdd="true">
|
||||
<name>Task 3: Implement BilibiliAdapter for article content</name>
|
||||
<files>kb_pipeline/platforms/bilibili.py, tests/test_platforms/test_bilibili.py</files>
|
||||
<behavior>
|
||||
- Test 1: fetch_metadata("cvxxx", "article") 返回正确的专栏元数据
|
||||
- Test 2: fetch_content("cvxxx", "article") 返回 Markdown 格式的正文
|
||||
- Test 3: Markdown 包含 YAML Frontmatter (title, author, source_url, platform, content_type, fetched_at)
|
||||
</behavior>
|
||||
<action>
|
||||
扩展 kb_pipeline/platforms/bilibili.py:
|
||||
1. 实现 fetch_content 方法:
|
||||
- 使用 BilibiliExtractor 获取专栏内容
|
||||
- 转换为 Markdown 格式
|
||||
- 添加 YAML Frontmatter
|
||||
|
||||
2. 添加 format_content_markdown 辅助函数:
|
||||
```python
|
||||
import yaml
|
||||
from datetime import datetime
|
||||
|
||||
def format_content_markdown(metadata: dict, content: str, platform: str, content_type: str) -> str:
|
||||
frontmatter = {
|
||||
"title": metadata.get("title", ""),
|
||||
"author": metadata.get("author", ""),
|
||||
"source_url": metadata.get("url", ""),
|
||||
"platform": platform,
|
||||
"content_type": content_type,
|
||||
"fetched_at": datetime.now().isoformat(),
|
||||
}
|
||||
return f"---\n{yaml.dump(frontmatter, allow_unicode=True)}---\n\n{content}"
|
||||
```
|
||||
|
||||
3. 扩展测试覆盖专栏场景
|
||||
</action>
|
||||
<verify>
|
||||
<automated>pytest tests/test_platforms/test_bilibili.py::test_article_content -x -v</automated>
|
||||
</verify>
|
||||
<done>BilibiliAdapter 能正确提取B站专栏内容和元数据</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 4: Update package exports</name>
|
||||
<files>kb_pipeline/__init__.py</files>
|
||||
<action>
|
||||
更新 kb_pipeline/__init__.py:
|
||||
```python
|
||||
from .platforms import PlatformAdapter, BilibiliAdapter
|
||||
|
||||
__all__ = [
|
||||
# ... existing exports ...
|
||||
"PlatformAdapter",
|
||||
"BilibiliAdapter",
|
||||
]
|
||||
```
|
||||
</action>
|
||||
<verify>
|
||||
<automated>python -c "from kb_pipeline import BilibiliAdapter; print('OK')"</automated>
|
||||
</verify>
|
||||
<done>BilibiliAdapter 可从 kb_pipeline 包根导入</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
- 所有测试通过: `pytest tests/test_platforms/test_bilibili.py -v`
|
||||
- 导入测试: `python -c "from kb_pipeline import BilibiliAdapter"`
|
||||
- 覆盖率: `pytest tests/test_platforms/ --cov=kb_pipeline/platforms --cov-report=term-missing`
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- [ ] PlatformAdapter 基类定义清晰,子类接口明确
|
||||
- [ ] BilibiliAdapter 正确提取B站视频元数据
|
||||
- [ ] BilibiliAdapter 正确提取B站专栏内容和元数据
|
||||
- [ ] 内容保存为 Markdown 格式,包含 YAML Frontmatter
|
||||
- [ ] 网络错误自动重试3次
|
||||
- [ ] 所有测试通过(至少 8 个测试用例)
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/02-platform-skills/02-02-SUMMARY.md`
|
||||
</output>
|
||||
Reference in New Issue
Block a user