docs(05): create phase 5 plans for archive and directory optimization
This commit is contained in:
@@ -0,0 +1,593 @@
|
||||
---
|
||||
phase: 05-archive-directory
|
||||
plan: 03
|
||||
type: execute
|
||||
wave: 3
|
||||
depends_on:
|
||||
- 05-02
|
||||
files_modified:
|
||||
- tests/test_pipeline.py
|
||||
- tests/test_processors.py
|
||||
autonomous: true
|
||||
requirements:
|
||||
- OUT-01
|
||||
- OUT-02
|
||||
- OUT-03
|
||||
- OUT-04
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "Archive files are organized by platform (bilibili/, zhihu/)"
|
||||
- "Archive files have timestamp prefix: {YYYYMMDD}_{content_id}_analysis.md"
|
||||
- "No date-based subdirectories in archive"
|
||||
- "Only md files in archive, no json files"
|
||||
artifacts:
|
||||
- path: "tests/test_pipeline.py"
|
||||
provides: "Integration test for archive step"
|
||||
contains: "test_archive"
|
||||
- path: "tests/test_processors.py"
|
||||
provides: "Unit tests for ArchiveProcessor"
|
||||
contains: "ArchiveProcessor"
|
||||
key_links:
|
||||
- from: "ArchiveProcessor"
|
||||
to: "{KB_ARCHIVE_PATH}/{platform}/"
|
||||
via: "settings.archive_path / platform"
|
||||
pattern: "platform_dir"
|
||||
---
|
||||
|
||||
<objective>
|
||||
验证最终目录结构和端到端归档流程
|
||||
|
||||
Purpose: 通过测试验证归档目录结构符合需求,确保只归档 md 文件
|
||||
Output: ArchiveProcessor 单元测试和 Pipeline 集成测试
|
||||
</objective>
|
||||
|
||||
<execution_context>
|
||||
@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@/home/starmen1265/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@.planning/STATE.md
|
||||
@.planning/phases/05-archive-directory/05-CONTEXT.md
|
||||
|
||||
<interfaces>
|
||||
<!-- Key interfaces from existing codebase -->
|
||||
|
||||
From tests/test_processors.py (existing pattern):
|
||||
```python
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_processor_success():
|
||||
# ... test implementation
|
||||
```
|
||||
|
||||
From kb_pipeline/processors/archive.py (created in 05-02):
|
||||
```python
|
||||
class ArchiveProcessor(BaseProcessor):
|
||||
STEP_NAME = "archive"
|
||||
|
||||
async def process(self, input_data: ArchiveInput) -> ArchiveOutput:
|
||||
# ...
|
||||
|
||||
def _find_existing_archive(self, platform_dir: Path, content_id: str) -> Optional[Path]:
|
||||
# ...
|
||||
|
||||
def _ask_overwrite(self, existing_file: Path) -> bool:
|
||||
# ...
|
||||
```
|
||||
|
||||
Expected archive structure:
|
||||
```
|
||||
{KB_ARCHIVE_PATH}/
|
||||
├── bilibili/
|
||||
│ ├── 20260316_BV1xxx_analysis.md
|
||||
│ └── 20260316_BV1yyy_analysis.md
|
||||
└── zhihu/
|
||||
└── 20260316_123456_analysis.md
|
||||
```
|
||||
</interfaces>
|
||||
</context>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto" tdd="true">
|
||||
<name>Task 1: Add ArchiveProcessor unit tests</name>
|
||||
<files>tests/test_processors.py</files>
|
||||
<behavior>
|
||||
- Test 1: ArchiveProcessor copies md file to correct platform directory
|
||||
- Test 2: ArchiveProcessor uses correct filename format {YYYYMMDD}_{content_id}_analysis.md
|
||||
- Test 3: ArchiveProcessor skips when step already completed
|
||||
- Test 4: ArchiveProcessor detects existing archive by content_id
|
||||
- Test 5: ArchiveProcessor handles missing analysis file with error
|
||||
</behavior>
|
||||
<action>
|
||||
Add the following tests to `tests/test_processors.py`:
|
||||
|
||||
```python
|
||||
# =============================================================================
|
||||
# ArchiveProcessor Tests
|
||||
# =============================================================================
|
||||
|
||||
import tempfile
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from kb_pipeline.processors.archive import ArchiveProcessor
|
||||
from kb_pipeline.schemas import ArchiveInput, ArchiveOutput
|
||||
from kb_pipeline.state import WorkflowStateManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def archive_test_env(tmp_path):
|
||||
"""Create test environment for archive tests"""
|
||||
# Create intermediate content directory
|
||||
content_dir = tmp_path / "intermediate" / "bilibili_BV1test"
|
||||
content_dir.mkdir(parents=True)
|
||||
|
||||
# Create analysis file
|
||||
analysis_file = content_dir / "analysis.md"
|
||||
analysis_file.write_text("# Test Analysis\n\nContent here.")
|
||||
|
||||
# Create archive directory
|
||||
archive_dir = tmp_path / "archive"
|
||||
archive_dir.mkdir(parents=True)
|
||||
|
||||
# Create state manager
|
||||
state_manager = WorkflowStateManager(content_dir)
|
||||
|
||||
return {
|
||||
"content_dir": content_dir,
|
||||
"analysis_file": analysis_file,
|
||||
"archive_dir": archive_dir,
|
||||
"state_manager": state_manager,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_processor_copies_to_platform_dir(archive_test_env, monkeypatch):
|
||||
"""Test that archive file is copied to correct platform directory"""
|
||||
# Mock settings
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.archive_path = str(archive_test_env["archive_dir"])
|
||||
|
||||
with patch.object(ArchiveProcessor, '__init__', lambda self, sm: None):
|
||||
processor = ArchiveProcessor()
|
||||
processor.settings = mock_settings
|
||||
processor.state_manager = None
|
||||
|
||||
input_data = ArchiveInput(
|
||||
content_id="BV1test",
|
||||
platform="bilibili",
|
||||
content_dir=str(archive_test_env["content_dir"]),
|
||||
analysis_file=str(archive_test_env["analysis_file"]),
|
||||
content_type="video"
|
||||
)
|
||||
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is True
|
||||
assert result.skipped is False
|
||||
assert "bilibili" in result.archive_file
|
||||
assert result.archive_file.endswith("_analysis.md")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_processor_filename_format(archive_test_env, monkeypatch):
|
||||
"""Test that archive filename follows {YYYYMMDD}_{content_id}_analysis.md format"""
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.archive_path = str(archive_test_env["archive_dir"])
|
||||
|
||||
with patch.object(ArchiveProcessor, '__init__', lambda self, sm: None):
|
||||
processor = ArchiveProcessor()
|
||||
processor.settings = mock_settings
|
||||
processor.state_manager = None
|
||||
|
||||
input_data = ArchiveInput(
|
||||
content_id="BV1test",
|
||||
platform="bilibili",
|
||||
content_dir=str(archive_test_env["content_dir"]),
|
||||
analysis_file=str(archive_test_env["analysis_file"]),
|
||||
content_type="video"
|
||||
)
|
||||
|
||||
result = await processor.process(input_data)
|
||||
|
||||
# Check filename format
|
||||
archive_path = Path(result.archive_file)
|
||||
filename = archive_path.name
|
||||
today = datetime.now().strftime("%Y%m%d")
|
||||
expected = f"{today}_BV1test_analysis.md"
|
||||
assert filename == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_processor_skips_completed(archive_test_env, monkeypatch):
|
||||
"""Test that archive step is skipped when already completed"""
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.archive_path = str(archive_test_env["archive_dir"])
|
||||
|
||||
# Create state with archive already completed
|
||||
state = archive_test_env["state_manager"].get_or_create(
|
||||
content_id="BV1test",
|
||||
platform="bilibili",
|
||||
source_url="https://test.com"
|
||||
)
|
||||
state.mark_step_completed("archive")
|
||||
archive_test_env["state_manager"].save(state)
|
||||
|
||||
with patch.object(ArchiveProcessor, '__init__', lambda self, sm: None):
|
||||
processor = ArchiveProcessor()
|
||||
processor.settings = mock_settings
|
||||
processor.state_manager = archive_test_env["state_manager"]
|
||||
|
||||
input_data = ArchiveInput(
|
||||
content_id="BV1test",
|
||||
platform="bilibili",
|
||||
content_dir=str(archive_test_env["content_dir"]),
|
||||
analysis_file=str(archive_test_env["analysis_file"]),
|
||||
content_type="video"
|
||||
)
|
||||
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is True
|
||||
assert result.skipped is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_processor_detects_existing(archive_test_env, monkeypatch):
|
||||
"""Test that archive processor detects existing archive by content_id"""
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.archive_path = str(archive_test_env["archive_dir"])
|
||||
|
||||
# Create existing archive with different timestamp
|
||||
platform_dir = archive_test_env["archive_dir"] / "bilibili"
|
||||
platform_dir.mkdir(parents=True, exist_ok=True)
|
||||
existing = platform_dir / "20260101_BV1test_analysis.md"
|
||||
existing.write_text("Old content")
|
||||
|
||||
with patch.object(ArchiveProcessor, '__init__', lambda self, sm: None):
|
||||
processor = ArchiveProcessor()
|
||||
processor.settings = mock_settings
|
||||
processor.state_manager = None
|
||||
processor._ask_overwrite = lambda f: False # User says no
|
||||
|
||||
input_data = ArchiveInput(
|
||||
content_id="BV1test",
|
||||
platform="bilibili",
|
||||
content_dir=str(archive_test_env["content_dir"]),
|
||||
analysis_file=str(archive_test_env["analysis_file"]),
|
||||
content_type="video"
|
||||
)
|
||||
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is True
|
||||
assert result.skipped is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_processor_handles_missing_file(archive_test_env, monkeypatch):
|
||||
"""Test that archive processor handles missing analysis file"""
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.archive_path = str(archive_test_env["archive_dir"])
|
||||
|
||||
with patch.object(ArchiveProcessor, '__init__', lambda self, sm: None):
|
||||
processor = ArchiveProcessor()
|
||||
processor.settings = mock_settings
|
||||
processor.state_manager = None
|
||||
|
||||
input_data = ArchiveInput(
|
||||
content_id="BV1test",
|
||||
platform="bilibili",
|
||||
content_dir=str(archive_test_env["content_dir"]),
|
||||
analysis_file="/nonexistent/analysis.md", # Missing file
|
||||
content_type="video"
|
||||
)
|
||||
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is False
|
||||
assert "not found" in result.error_message.lower()
|
||||
```
|
||||
|
||||
Run tests to verify:
|
||||
```bash
|
||||
python3 -m pytest tests/test_processors.py -v -k "archive"
|
||||
```
|
||||
</action>
|
||||
<verify>
|
||||
<automated>python3 -m pytest tests/test_processors.py -v -k "archive" --tb=short</automated>
|
||||
</verify>
|
||||
<done>
|
||||
- 5 unit tests for ArchiveProcessor pass
|
||||
- Tests cover: copy, filename format, skip completed, detect existing, missing file
|
||||
- All tests use tmp_path for isolation
|
||||
</done>
|
||||
</task>
|
||||
|
||||
<task type="auto" tdd="true">
|
||||
<name>Task 2: Add Pipeline integration test for archive step</name>
|
||||
<files>tests/test_pipeline.py</files>
|
||||
<behavior>
|
||||
- Test 1: Full pipeline with archive step copies file to archive directory
|
||||
- Test 2: Archive directory structure is {archive_path}/{platform}/
|
||||
- Test 3: Archive filename format is {YYYYMMDD}_{content_id}_analysis.md
|
||||
</behavior>
|
||||
<action>
|
||||
Add the following tests to `tests/test_pipeline.py`:
|
||||
|
||||
```python
|
||||
# =============================================================================
|
||||
# Archive Integration Tests
|
||||
# =============================================================================
|
||||
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
|
||||
from kb_pipeline.pipeline import Pipeline, INTERMEDIATE_DIR
|
||||
from kb_pipeline.schemas import PipelineConfig
|
||||
from kb_pipeline.processors import (
|
||||
ParseProcessor, DownloadProcessor, ASRProcessor,
|
||||
AnalyzeProcessor, ArchiveProcessor
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_processors_with_archive():
|
||||
"""Create mock processors including archive"""
|
||||
processors = []
|
||||
|
||||
for step_name in ["parse", "download", "asr", "analyze", "archive"]:
|
||||
processor = MagicMock()
|
||||
processor.STEP_NAME = step_name
|
||||
processor.process = AsyncMock()
|
||||
|
||||
if step_name == "parse":
|
||||
processor.process.return_value = MagicMock(
|
||||
success=True,
|
||||
platform="bilibili",
|
||||
content_type="video",
|
||||
content_id="BV1test",
|
||||
resolved_url="https://www.bilibili.com/video/BV1test",
|
||||
original_url="https://www.bilibili.com/video/BV1test"
|
||||
)
|
||||
elif step_name == "download":
|
||||
processor.process.return_value = MagicMock(
|
||||
success=True,
|
||||
metadata_file="/tmp/metadata.json",
|
||||
content_file="/tmp/content.json",
|
||||
title="Test Video",
|
||||
author="Test Author"
|
||||
)
|
||||
elif step_name == "asr":
|
||||
processor.process.return_value = MagicMock(
|
||||
success=True,
|
||||
tool="whisper",
|
||||
has_subtitle=False,
|
||||
output_files={"json": "/tmp/subtitle.json"}
|
||||
)
|
||||
elif step_name == "analyze":
|
||||
processor.process.return_value = MagicMock(
|
||||
success=True,
|
||||
analysis_file="/tmp/analysis.md",
|
||||
summary="Test summary",
|
||||
main_topics=["topic1"],
|
||||
key_insights=["insight1"],
|
||||
tags=["tag1"],
|
||||
stats={}
|
||||
)
|
||||
elif step_name == "archive":
|
||||
processor.process.return_value = MagicMock(
|
||||
success=True,
|
||||
skipped=False,
|
||||
platform="bilibili",
|
||||
archive_file="/archive/bilibili/20260316_BV1test_analysis.md"
|
||||
)
|
||||
|
||||
processors.append(processor)
|
||||
|
||||
return processors
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_includes_archive_step(mock_processors_with_archive, tmp_path):
|
||||
"""Test that pipeline includes archive as final step"""
|
||||
config = PipelineConfig(
|
||||
steps=["parse", "download", "asr", "analyze", "archive"],
|
||||
output_dir=tmp_path,
|
||||
skip_completed=False
|
||||
)
|
||||
|
||||
pipeline = Pipeline(
|
||||
processors=mock_processors_with_archive,
|
||||
config=config,
|
||||
output_dir=tmp_path
|
||||
)
|
||||
|
||||
# Mock _parse_url and _get_content_dir
|
||||
with patch.object(pipeline, '_parse_url') as mock_parse, \
|
||||
patch.object(pipeline, '_get_content_dir') as mock_dir:
|
||||
|
||||
from kb_pipeline.link_parser import ParsedLink
|
||||
mock_parse.return_value = ParsedLink(
|
||||
platform="bilibili",
|
||||
content_type="video",
|
||||
content_id="BV1test",
|
||||
original_url="https://www.bilibili.com/video/BV1test",
|
||||
resolved_url="https://www.bilibili.com/video/BV1test"
|
||||
)
|
||||
mock_dir.return_value = tmp_path / "bilibili_BV1test"
|
||||
|
||||
result = await pipeline.run("https://www.bilibili.com/video/BV1test")
|
||||
|
||||
assert result.success is True
|
||||
# Verify archive processor was called
|
||||
archive_processor = next(p for p in mock_processors_with_archive if p.STEP_NAME == "archive")
|
||||
archive_processor.process.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_uses_platform_directory(tmp_path):
|
||||
"""Test that archive files go to {archive_path}/{platform}/ directory"""
|
||||
from kb_pipeline.processors.archive import ArchiveProcessor
|
||||
from kb_pipeline.schemas import ArchiveInput
|
||||
|
||||
# Setup
|
||||
archive_dir = tmp_path / "archive"
|
||||
content_dir = tmp_path / "intermediate" / "bilibili_BV1test"
|
||||
content_dir.mkdir(parents=True)
|
||||
analysis_file = content_dir / "analysis.md"
|
||||
analysis_file.write_text("# Test")
|
||||
|
||||
# Mock settings
|
||||
with patch('kb_pipeline.processors.archive.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.archive_path = str(archive_dir)
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
processor = ArchiveProcessor()
|
||||
input_data = ArchiveInput(
|
||||
content_id="BV1test",
|
||||
platform="bilibili",
|
||||
content_dir=str(content_dir),
|
||||
analysis_file=str(analysis_file),
|
||||
content_type="video"
|
||||
)
|
||||
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is True
|
||||
# Verify platform subdirectory
|
||||
archive_path = Path(result.archive_file)
|
||||
assert archive_path.parent.name == "bilibili"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_no_date_subdirectories(tmp_path):
|
||||
"""Test that archive does not create date-based subdirectories"""
|
||||
from kb_pipeline.processors.archive import ArchiveProcessor
|
||||
from kb_pipeline.schemas import ArchiveInput
|
||||
|
||||
archive_dir = tmp_path / "archive"
|
||||
content_dir = tmp_path / "intermediate" / "bilibili_BV1test"
|
||||
content_dir.mkdir(parents=True)
|
||||
analysis_file = content_dir / "analysis.md"
|
||||
analysis_file.write_text("# Test")
|
||||
|
||||
with patch('kb_pipeline.processors.archive.get_settings') as mock_get_settings:
|
||||
mock_settings = MagicMock()
|
||||
mock_settings.archive_path = str(archive_dir)
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
processor = ArchiveProcessor()
|
||||
input_data = ArchiveInput(
|
||||
content_id="BV1test",
|
||||
platform="bilibili",
|
||||
content_dir=str(content_dir),
|
||||
analysis_file=str(analysis_file),
|
||||
content_type="video"
|
||||
)
|
||||
|
||||
result = await processor.process(input_data)
|
||||
|
||||
# Verify no date-based subdirectories
|
||||
archive_path = Path(result.archive_file)
|
||||
# Parent should be platform dir, grandparent should be archive base
|
||||
assert archive_path.parent.parent == archive_dir
|
||||
# Check that parent is just "bilibili", not a date
|
||||
assert archive_path.parent.name == "bilibili"
|
||||
```
|
||||
|
||||
Run tests:
|
||||
```bash
|
||||
python3 -m pytest tests/test_pipeline.py -v -k "archive" --tb=short
|
||||
```
|
||||
</action>
|
||||
<verify>
|
||||
<automated>python3 -m pytest tests/test_pipeline.py -v -k "archive" --tb=short</automated>
|
||||
</verify>
|
||||
<done>
|
||||
- 3 integration tests for archive step pass
|
||||
- Tests verify: archive included in pipeline, platform directory, no date subdirs
|
||||
- All tests use tmp_path for isolation
|
||||
</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 3: Run full test suite and verify</name>
|
||||
<files>tests/test_pipeline.py, tests/test_processors.py</files>
|
||||
<action>
|
||||
Run the complete test suite to verify all changes work together:
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
python3 -m pytest tests/ -v --tb=short
|
||||
|
||||
# Specifically verify archive-related tests
|
||||
python3 -m pytest tests/ -v -k "archive" --tb=short
|
||||
```
|
||||
|
||||
Verify the following:
|
||||
1. All existing tests still pass (no regression)
|
||||
2. New archive tests pass
|
||||
3. ArchiveProcessor can be imported and instantiated
|
||||
4. PipelineConfig default steps include "archive"
|
||||
|
||||
If any tests fail, fix the issues before marking complete.
|
||||
</action>
|
||||
<verify>
|
||||
<automated>python3 -m pytest tests/ -v --tb=short -x</automated>
|
||||
</verify>
|
||||
<done>
|
||||
- All tests pass (existing + new)
|
||||
- No regressions introduced
|
||||
- Archive functionality fully tested
|
||||
</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
1. Run full test suite:
|
||||
```bash
|
||||
python3 -m pytest tests/ -v --tb=short
|
||||
```
|
||||
|
||||
2. Verify archive-specific tests:
|
||||
```bash
|
||||
python3 -m pytest tests/ -v -k "archive" --tb=short
|
||||
```
|
||||
|
||||
3. Manual verification of archive structure:
|
||||
```bash
|
||||
python3 -c "
|
||||
from pathlib import Path
|
||||
from kb_pipeline.processors.archive import ArchiveProcessor
|
||||
# Archive structure should be: {archive_path}/{platform}/{timestamp}_{id}_analysis.md
|
||||
print('ArchiveProcessor loaded successfully')
|
||||
"
|
||||
```
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- OUT-01: Archive files organized by platform (bilibili/, zhihu/)
|
||||
- OUT-02: No date-based subdirectories in archive
|
||||
- OUT-03: Filename format is {YYYYMMDD}_{content_id}_analysis.md
|
||||
- OUT-04: Only md files archived (json files stay in intermediate)
|
||||
- All tests pass
|
||||
- No regressions in existing functionality
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/05-archive-directory/05-03-SUMMARY.md`
|
||||
</output>
|
||||
Reference in New Issue
Block a user