docs(05): create phase 5 plans for archive and directory optimization
This commit is contained in:
@@ -0,0 +1,504 @@
|
||||
---
|
||||
phase: 05-archive-directory
|
||||
plan: 02
|
||||
type: execute
|
||||
wave: 2
|
||||
depends_on:
|
||||
- 05-01
|
||||
files_modified:
|
||||
- kb_pipeline/schemas.py
|
||||
- kb_pipeline/processors/__init__.py
|
||||
- kb_pipeline/processors/archive.py
|
||||
- kb_pipeline/pipeline.py
|
||||
- kb-workflow/kb_workflow.py
|
||||
autonomous: true
|
||||
requirements:
|
||||
- ARC-01
|
||||
- ARC-02
|
||||
- ARC-03
|
||||
- ARC-04
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "ArchiveProcessor is the last step in pipeline"
|
||||
- "After analysis, md file is copied to archive directory"
|
||||
- "Only md files are archived, not json files"
|
||||
- "Archive step status is recorded in workflow_state.json"
|
||||
artifacts:
|
||||
- path: "kb_pipeline/processors/archive.py"
|
||||
provides: "ArchiveProcessor implementation"
|
||||
contains: "class ArchiveProcessor"
|
||||
- path: "kb_pipeline/schemas.py"
|
||||
provides: "ArchiveInput and ArchiveOutput schemas"
|
||||
contains: "class ArchiveInput"
|
||||
- path: "kb_pipeline/pipeline.py"
|
||||
provides: "Pipeline orchestration with archive step"
|
||||
contains: "archive"
|
||||
key_links:
|
||||
- from: "kb_pipeline/pipeline.py"
|
||||
to: "kb_pipeline/processors/archive.py"
|
||||
via: "ArchiveProcessor instantiation"
|
||||
pattern: "ArchiveProcessor"
|
||||
- from: "ArchiveProcessor"
|
||||
to: "KBSettings.archive_path"
|
||||
via: "get_settings()"
|
||||
pattern: "settings.archive_path"
|
||||
---
|
||||
|
||||
<objective>
|
||||
创建 ArchiveProcessor 作为管道最后一步,自动归档分析报告到最终目录
|
||||
|
||||
Purpose: 分析完成后自动将 md 文件复制到按平台分组的归档目录
|
||||
Output: ArchiveProcessor 类、ArchiveInput/ArchiveOutput schemas、Pipeline 集成
|
||||
</objective>
|
||||
|
||||
<execution_context>
|
||||
@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@/home/starmen1265/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@.planning/STATE.md
|
||||
@.planning/phases/05-archive-directory/05-CONTEXT.md
|
||||
|
||||
<interfaces>
|
||||
<!-- Key interfaces from existing codebase -->
|
||||
|
||||
From kb_pipeline/processors/base.py:
|
||||
```python
|
||||
class BaseProcessor(ABC):
|
||||
STEP_NAME: str
|
||||
|
||||
def __init__(self, state_manager: Optional[WorkflowStateManager] = None):
|
||||
self.settings = get_settings()
|
||||
self.state_manager = state_manager
|
||||
|
||||
@abstractmethod
|
||||
async def process(self, input_data: StepInput) -> StepOutput:
|
||||
pass
|
||||
```
|
||||
|
||||
From kb_pipeline/config.py:
|
||||
```python
|
||||
class KBSettings(BaseSettings):
|
||||
archive_path: str = Field(..., description="归档目录")
|
||||
```
|
||||
|
||||
From kb_pipeline/schemas.py:
|
||||
```python
|
||||
class AnalyzeOutput(StepOutput):
|
||||
analysis_file: str = Field(..., description="Path to generated analysis report")
|
||||
# ... other fields
|
||||
```
|
||||
|
||||
From kb_pipeline/pipeline.py:
|
||||
```python
|
||||
class Pipeline:
|
||||
def __init__(self, processors: List[BaseProcessor], config: Optional[PipelineConfig] = None, output_dir: Optional[Path] = None):
|
||||
self.processors: Dict[str, BaseProcessor] = {p.STEP_NAME: p for p in processors}
|
||||
|
||||
async def run(self, url: str) -> PipelineOutput:
|
||||
for step_name in self.config.steps:
|
||||
# ... step execution
|
||||
```
|
||||
|
||||
From rich.prompt (existing pattern):
|
||||
```python
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.prompt import Confirm
|
||||
return Confirm.ask("Question?", default=False)
|
||||
except ImportError:
|
||||
response = input("Question? [y/N] ")
|
||||
return response.lower() in ('y', 'yes')
|
||||
```
|
||||
</interfaces>
|
||||
</context>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 1: Create ArchiveInput and ArchiveOutput schemas</name>
|
||||
<files>kb_pipeline/schemas.py</files>
|
||||
<action>
|
||||
Add new schema classes at the end of the schemas file:
|
||||
|
||||
```python
|
||||
# =============================================================================
|
||||
# Archive Schemas (Phase 5)
|
||||
# =============================================================================
|
||||
|
||||
class ArchiveInput(StepInput):
|
||||
"""Archive 步骤输入
|
||||
|
||||
继承 StepInput 的基础字段,添加归档特定字段。
|
||||
|
||||
Attributes:
|
||||
content_dir: 内容目录路径(中间目录)
|
||||
analysis_file: 分析报告 md 文件路径
|
||||
content_type: 内容类型
|
||||
"""
|
||||
content_dir: str = Field(..., min_length=1, description="Content directory path (intermediate)")
|
||||
analysis_file: str = Field(..., min_length=1, description="Path to analysis Markdown file")
|
||||
content_type: str = Field(..., min_length=1, description="Content type (video, article, etc.)")
|
||||
|
||||
|
||||
class ArchiveOutput(StepOutput):
|
||||
"""Archive 步骤输出
|
||||
|
||||
继承 StepOutput 的基础字段,添加归档特定字段。
|
||||
|
||||
Attributes:
|
||||
archive_file: 归档后的文件路径
|
||||
skipped: 是否跳过(已有相同文件)
|
||||
platform: 平台名称
|
||||
"""
|
||||
archive_file: str = Field(default="", description="Path to archived file")
|
||||
skipped: bool = Field(default=False, description="Whether archiving was skipped")
|
||||
platform: str = Field(default="", description="Platform name")
|
||||
```
|
||||
|
||||
Follow existing schema patterns:
|
||||
- Inherit from StepInput/StepOutput
|
||||
- Use Field() with descriptions
|
||||
- Include min_length validation for required strings
|
||||
</action>
|
||||
<verify>
|
||||
<automated>python3 -c "from kb_pipeline.schemas import ArchiveInput, ArchiveOutput; print('OK')"</automated>
|
||||
</verify>
|
||||
<done>
|
||||
- ArchiveInput class defined with content_dir, analysis_file, content_type fields
|
||||
- ArchiveOutput class defined with archive_file, skipped, platform fields
|
||||
- Both inherit from base StepInput/StepOutput
|
||||
</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 2: Implement ArchiveProcessor</name>
|
||||
<files>kb_pipeline/processors/archive.py, kb_pipeline/processors/__init__.py</files>
|
||||
<action>
|
||||
Create new file `kb_pipeline/processors/archive.py`:
|
||||
|
||||
```python
|
||||
"""ArchiveProcessor - Copy analysis reports to archive directory
|
||||
|
||||
Final step in pipeline that copies the generated analysis.md
|
||||
to the archive directory organized by platform.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from ..config import get_settings
|
||||
from ..schemas import ArchiveInput, ArchiveOutput
|
||||
from ..state import WorkflowStateManager
|
||||
from .base import BaseProcessor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ArchiveProcessor(BaseProcessor):
|
||||
"""Archive 处理器 - 归档分析报告
|
||||
|
||||
处理流程:
|
||||
1. 检查步骤是否已完成(幂等执行)
|
||||
2. 检查归档目录是否存在相同 content_id 的文件
|
||||
3. 如果存在重复,询问用户是否覆盖
|
||||
4. 复制 md 文件到归档目录
|
||||
5. 记录处理状态到 workflow_state.json
|
||||
|
||||
归档目录结构:
|
||||
{KB_ARCHIVE_PATH}/{platform}/{YYYYMMDD}_{content_id}_analysis.md
|
||||
"""
|
||||
|
||||
STEP_NAME = "archive"
|
||||
|
||||
async def process(self, input_data: ArchiveInput) -> ArchiveOutput:
|
||||
"""执行归档处理
|
||||
|
||||
Args:
|
||||
input_data: Archive 输入数据
|
||||
|
||||
Returns:
|
||||
Archive 处理结果
|
||||
"""
|
||||
try:
|
||||
# Step 1: 检查是否已完成(幂等执行)
|
||||
if self.state_manager:
|
||||
state = self.state_manager.load()
|
||||
if state and state.is_step_completed(self.STEP_NAME):
|
||||
logger.info(f"Step {self.STEP_NAME} already completed, skipping")
|
||||
return ArchiveOutput(
|
||||
success=True,
|
||||
skipped=True,
|
||||
platform=input_data.platform,
|
||||
archive_file=""
|
||||
)
|
||||
|
||||
# Step 2: 准备归档路径
|
||||
archive_base = Path(self.settings.archive_path)
|
||||
platform_dir = archive_base / input_data.platform
|
||||
platform_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 文件命名: {YYYYMMDD}_{content_id}_analysis.md
|
||||
timestamp = datetime.now().strftime("%Y%m%d")
|
||||
archive_filename = f"{timestamp}_{input_data.content_id}_analysis.md"
|
||||
archive_path = platform_dir / archive_filename
|
||||
|
||||
# Step 3: 检查重复
|
||||
existing_file = self._find_existing_archive(
|
||||
platform_dir, input_data.content_id
|
||||
)
|
||||
if existing_file:
|
||||
should_overwrite = self._ask_overwrite(existing_file)
|
||||
if not should_overwrite:
|
||||
logger.info(f"User chose not to overwrite, skipping archive")
|
||||
return ArchiveOutput(
|
||||
success=True,
|
||||
skipped=True,
|
||||
platform=input_data.platform,
|
||||
archive_file=str(existing_file)
|
||||
)
|
||||
# 删除旧文件
|
||||
existing_file.unlink()
|
||||
logger.info(f"Removed existing archive: {existing_file}")
|
||||
|
||||
# Step 4: 复制文件
|
||||
source_path = Path(input_data.analysis_file)
|
||||
if not source_path.exists():
|
||||
raise FileNotFoundError(f"Analysis file not found: {source_path}")
|
||||
|
||||
shutil.copy2(source_path, archive_path)
|
||||
logger.info(f"Archived: {source_path} -> {archive_path}")
|
||||
|
||||
# Step 5: 记录成功状态
|
||||
if self.state_manager:
|
||||
state = self.state_manager.load()
|
||||
if state:
|
||||
state.mark_step_completed(
|
||||
self.STEP_NAME,
|
||||
f"archived to {archive_path}"
|
||||
)
|
||||
self.state_manager.save(state)
|
||||
|
||||
return ArchiveOutput(
|
||||
success=True,
|
||||
skipped=False,
|
||||
platform=input_data.platform,
|
||||
archive_file=str(archive_path)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Archive failed: {e}")
|
||||
|
||||
if self.state_manager:
|
||||
state = self.state_manager.load()
|
||||
if state:
|
||||
state.mark_step_failed(self.STEP_NAME, str(e))
|
||||
self.state_manager.save(state)
|
||||
|
||||
return ArchiveOutput(
|
||||
success=False,
|
||||
error_message=str(e),
|
||||
platform=input_data.platform,
|
||||
archive_file=""
|
||||
)
|
||||
|
||||
def _find_existing_archive(self, platform_dir: Path, content_id: str) -> Optional[Path]:
|
||||
"""查找已存在的归档文件
|
||||
|
||||
使用 glob 模式 `*_{content_id}_analysis.md` 搜索
|
||||
|
||||
Args:
|
||||
platform_dir: 平台目录路径
|
||||
content_id: 内容 ID
|
||||
|
||||
Returns:
|
||||
已存在的文件路径,如果不存在返回 None
|
||||
"""
|
||||
pattern = f"*_{content_id}_analysis.md"
|
||||
matches = list(platform_dir.glob(pattern))
|
||||
return matches[0] if matches else None
|
||||
|
||||
def _ask_overwrite(self, existing_file: Path) -> bool:
|
||||
"""询问用户是否覆盖已存在的文件
|
||||
|
||||
使用 rich.prompt.Confirm 模式(与 Pipeline._ask_rerun_step 相同)
|
||||
|
||||
Args:
|
||||
existing_file: 已存在的文件路径
|
||||
|
||||
Returns:
|
||||
True 如果用户选择覆盖
|
||||
"""
|
||||
try:
|
||||
from rich.console import Console
|
||||
from rich.prompt import Confirm
|
||||
|
||||
console = Console()
|
||||
return Confirm.ask(
|
||||
f"Archive already exists: {existing_file.name}\nOverwrite?",
|
||||
default=False
|
||||
)
|
||||
except ImportError:
|
||||
response = input(
|
||||
f"Archive already exists: {existing_file.name}\nOverwrite? [y/N] "
|
||||
)
|
||||
return response.lower() in ('y', 'yes')
|
||||
```
|
||||
|
||||
Update `kb_pipeline/processors/__init__.py` to export ArchiveProcessor:
|
||||
```python
|
||||
from .archive import ArchiveProcessor
|
||||
```
|
||||
|
||||
Key points:
|
||||
- Follow BaseProcessor pattern exactly
|
||||
- Use settings.archive_path for archive base directory
|
||||
- Only copy md files (not json)
|
||||
- Glob pattern for duplicate detection
|
||||
- Use rich.prompt.Confirm with input() fallback
|
||||
</action>
|
||||
<verify>
|
||||
<automated>python3 -c "from kb_pipeline.processors import ArchiveProcessor; p = ArchiveProcessor(); assert p.STEP_NAME == 'archive'"</automated>
|
||||
</verify>
|
||||
<done>
|
||||
- ArchiveProcessor class created with STEP_NAME = "archive"
|
||||
- ArchiveInput/ArchiveOutput schemas imported
|
||||
- _find_existing_archive uses glob pattern
|
||||
- _ask_overwrite uses rich.prompt with fallback
|
||||
- Export added to __init__.py
|
||||
</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 3: Integrate ArchiveProcessor into Pipeline</name>
|
||||
<files>kb_pipeline/pipeline.py, kb-workflow/kb_workflow.py</files>
|
||||
<action>
|
||||
1. Update `kb_pipeline/pipeline.py`:
|
||||
|
||||
a. Update default steps in PipelineConfig to include "archive":
|
||||
- Find where PipelineConfig is referenced
|
||||
- The default steps are defined in PipelineConfig.steps in schemas.py
|
||||
- We need to update schemas.py default_factory
|
||||
|
||||
b. Add archive step input creation in Pipeline.run():
|
||||
```python
|
||||
elif step_name == "archive":
|
||||
# Archive needs analysis file from analyze step
|
||||
analyze_result = results.get("analyze", {})
|
||||
analysis_file = analyze_result.get("analysis_file", "")
|
||||
|
||||
step_input = ArchiveInput(
|
||||
content_id=parsed.content_id,
|
||||
platform=parsed.platform,
|
||||
content_dir=str(content_dir),
|
||||
analysis_file=analysis_file,
|
||||
content_type=parsed.content_type,
|
||||
)
|
||||
```
|
||||
|
||||
c. Import ArchiveInput at top of file:
|
||||
```python
|
||||
from kb_pipeline.schemas import (
|
||||
ParseInput, DownloadInput, ASRInput, AnalyzeInput, ArchiveInput
|
||||
)
|
||||
```
|
||||
|
||||
2. Update `kb_pipeline/schemas.py` PipelineConfig default steps:
|
||||
```python
|
||||
steps: List[str] = Field(
|
||||
default_factory=lambda: ["parse", "download", "asr", "analyze", "archive"],
|
||||
description="Steps to execute in order"
|
||||
)
|
||||
```
|
||||
|
||||
3. Update `kb-workflow/kb_workflow.py`:
|
||||
|
||||
a. Import ArchiveProcessor:
|
||||
```python
|
||||
from kb_pipeline.processors import (
|
||||
ASRProcessor, AnalyzeProcessor, ArchiveProcessor,
|
||||
DownloadProcessor, ParseProcessor
|
||||
)
|
||||
```
|
||||
|
||||
b. Add ArchiveProcessor to processors list:
|
||||
```python
|
||||
processors = [
|
||||
ParseProcessor(),
|
||||
DownloadProcessor(),
|
||||
ASRProcessor(),
|
||||
AnalyzeProcessor(),
|
||||
ArchiveProcessor(), # Add this
|
||||
]
|
||||
```
|
||||
|
||||
c. Update config steps to include "archive":
|
||||
```python
|
||||
config = PipelineConfig(
|
||||
steps=["parse", "download", "asr", "analyze", "archive"],
|
||||
# ...
|
||||
)
|
||||
```
|
||||
|
||||
Key points:
|
||||
- Archive is always the last step
|
||||
- Archive step needs analysis_file from analyze step
|
||||
- Pipeline must create ArchiveInput properly
|
||||
</action>
|
||||
<verify>
|
||||
<automated>python3 -c "from kb_pipeline.schemas import PipelineConfig; c = PipelineConfig(); assert 'archive' in c.steps"</automated>
|
||||
</verify>
|
||||
<done>
|
||||
- ArchiveProcessor added to processors list in CLI
|
||||
- PipelineConfig default steps include "archive"
|
||||
- Pipeline.run() handles ArchiveInput creation
|
||||
- ArchiveInput imported in pipeline.py
|
||||
</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
1. Verify ArchiveProcessor can be imported:
|
||||
```bash
|
||||
python3 -c "from kb_pipeline.processors import ArchiveProcessor; print('OK')"
|
||||
```
|
||||
|
||||
2. Verify schemas:
|
||||
```bash
|
||||
python3 -c "from kb_pipeline.schemas import ArchiveInput, ArchiveOutput; print('OK')"
|
||||
```
|
||||
|
||||
3. Verify PipelineConfig includes archive step:
|
||||
```bash
|
||||
python3 -c "from kb_pipeline.schemas import PipelineConfig; print(PipelineConfig().steps)"
|
||||
```
|
||||
Expected: ['parse', 'download', 'asr', 'analyze', 'archive']
|
||||
|
||||
4. Run existing tests:
|
||||
```bash
|
||||
python3 -m pytest tests/ -v --tb=short -k "not integration"
|
||||
```
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- ARC-01: ArchiveProcessor is the last step in pipeline
|
||||
- ARC-02: Analysis completion triggers automatic archiving
|
||||
- ARC-03: Only md files are copied (not json)
|
||||
- ARC-04: No automatic cleanup of intermediate files (user handles manually)
|
||||
- Archive step status recorded in workflow_state.json
|
||||
- Duplicate detection asks user for overwrite confirmation
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/05-archive-directory/05-02-SUMMARY.md`
|
||||
</output>
|
||||
Reference in New Issue
Block a user