From ec268f7d17068447a9ea6b69d58ad385559a4f37 Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Fri, 13 Mar 2026 23:24:00 +0800 Subject: [PATCH] feat(04-02): add content downloading and module export for DownloadProcessor TDD GREEN phase: - DownloadProcessor downloads article/answer/zhuanlan content - Video content is skipped (handled by ASR in Phase 3) - Content files saved with correct naming convention - DownloadProcessor exported from kb_pipeline.processors - All 17 tests pass Co-Authored-By: Claude Opus 4.6 --- kb_pipeline/processors/__init__.py | 4 +- tests/test_processors/test_download.py | 160 +++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 1 deletion(-) diff --git a/kb_pipeline/processors/__init__.py b/kb_pipeline/processors/__init__.py index 8bb4bc0..bc1e5c3 100644 --- a/kb_pipeline/processors/__init__.py +++ b/kb_pipeline/processors/__init__.py @@ -3,11 +3,13 @@ This module contains processors for various content transformation tasks: - BaseProcessor: Abstract base class for all processors - ParseProcessor: URL parsing for various platforms +- DownloadProcessor: Metadata and content downloading - ASRProcessor: Audio-to-text conversion (subtitle download + Whisper.cpp) """ from .base import BaseProcessor from .parse import ParseProcessor +from .download import DownloadProcessor from .asr import ASRProcessor -__all__ = ["BaseProcessor", "ParseProcessor", "ASRProcessor"] +__all__ = ["BaseProcessor", "ParseProcessor", "DownloadProcessor", "ASRProcessor"] diff --git a/tests/test_processors/test_download.py b/tests/test_processors/test_download.py index d87542b..5fa2b7a 100644 --- a/tests/test_processors/test_download.py +++ b/tests/test_processors/test_download.py @@ -321,3 +321,163 @@ class TestDownloadProcessorMetadata: call_args = mock_state.mark_step_failed.call_args assert call_args[0][0] == "download" # step_name mock_state_manager.save.assert_called_once_with(mock_state) + + +class TestDownloadProcessorContent: + """Tests for DownloadProcessor content downloading.""" + + @pytest.fixture + def mock_bilibili_article_adapter(self): + """Create a mock BilibiliAdapter for articles.""" + adapter = MagicMock() + adapter.fetch_metadata = AsyncMock(return_value={ + "title": "Test Article Title", + "author": "Test Author", + "cvid": "CV123", + "url": "https://www.bilibili.com/read/cv123", + }) + adapter.fetch_content = AsyncMock(return_value="---\ntitle: Test Article\n---\nArticle content here") + adapter.get_output_filename = MagicMock(side_effect=lambda cid, ctype, suffix: f"bilibili_{cid}_{suffix}") + return adapter + + @pytest.fixture + def mock_bilibili_video_adapter(self): + """Create a mock BilibiliAdapter for videos.""" + adapter = MagicMock() + adapter.fetch_metadata = AsyncMock(return_value={ + "metadata": { + "title": "Test Video Title", + "mid": 12345, + "bvid": "BV1xx", + "duration": 300, + }, + "stats": {"view": 1000}, + }) + adapter.get_output_filename = MagicMock(side_effect=lambda cid, ctype, suffix: f"bilibili_{cid}_{suffix}") + return adapter + + @pytest.fixture + def mock_state_manager(self): + """Create a mock WorkflowStateManager.""" + manager = MagicMock() + manager.load = MagicMock(return_value=None) + manager.save = MagicMock() + return manager + + @pytest.mark.asyncio + async def test_content_downloads_article_content( + self, mock_bilibili_article_adapter, mock_state_manager, tmp_path + ): + """Test 1: DownloadProcessor downloads Bilibili article content""" + from kb_pipeline.processors.download import DownloadProcessor + from kb_pipeline.schemas import DownloadInput + + input_data = DownloadInput( + content_id="CV123", + platform="bilibili", + content_dir=str(tmp_path), + content_type="article" + ) + + with patch("kb_pipeline.processors.base.get_settings") as mock_settings: + mock_settings.return_value = MagicMock() + with patch.object( + DownloadProcessor, '_get_adapter', return_value=mock_bilibili_article_adapter + ): + processor = DownloadProcessor(state_manager=mock_state_manager) + result = await processor.process(input_data) + + assert result.success is True + assert result.content_file != "" # Content should be downloaded + assert "bilibili_CV123_content.md" in result.content_file + + # Verify content file was created + content_path = Path(result.content_file) + assert content_path.exists() + + # Verify fetch_content was called + mock_bilibili_article_adapter.fetch_content.assert_called_once_with("CV123", "article") + + @pytest.mark.asyncio + async def test_content_skips_video_download( + self, mock_bilibili_video_adapter, mock_state_manager, tmp_path + ): + """Test 2: DownloadProcessor skips video download (handled by ASR)""" + from kb_pipeline.processors.download import DownloadProcessor + from kb_pipeline.schemas import DownloadInput + + input_data = DownloadInput( + content_id="BV1xx", + platform="bilibili", + content_dir=str(tmp_path), + content_type="video" + ) + + with patch("kb_pipeline.processors.base.get_settings") as mock_settings: + mock_settings.return_value = MagicMock() + with patch.object( + DownloadProcessor, '_get_adapter', return_value=mock_bilibili_video_adapter + ): + processor = DownloadProcessor(state_manager=mock_state_manager) + result = await processor.process(input_data) + + assert result.success is True + assert result.content_file == "" # Video content not downloaded (handled by ASR) + assert result.duration == 300 # Duration should be extracted + + # Verify fetch_content was NOT called for video + mock_bilibili_video_adapter.fetch_content.assert_not_called() + + @pytest.mark.asyncio + async def test_content_creates_correct_output_file_paths( + self, mock_bilibili_article_adapter, mock_state_manager, tmp_path + ): + """Test 3: DownloadProcessor creates correct output file paths""" + from kb_pipeline.processors.download import DownloadProcessor + from kb_pipeline.schemas import DownloadInput + + input_data = DownloadInput( + content_id="CV123", + platform="bilibili", + content_dir=str(tmp_path), + content_type="article" + ) + + with patch("kb_pipeline.processors.base.get_settings") as mock_settings: + mock_settings.return_value = MagicMock() + with patch.object( + DownloadProcessor, '_get_adapter', return_value=mock_bilibili_article_adapter + ): + processor = DownloadProcessor(state_manager=mock_state_manager) + result = await processor.process(input_data) + + assert result.success is True + + # Check file paths follow naming convention + assert "bilibili_CV123_metadata.json" in result.metadata_file + assert "bilibili_CV123_content.md" in result.content_file + + # Check files exist + assert Path(result.metadata_file).exists() + assert Path(result.content_file).exists() + + +class TestDownloadProcessorModuleExport: + """Tests for DownloadProcessor module export.""" + + def test_download_processor_importable_from_processors(self): + """Test 4: DownloadProcessor can be imported from kb_pipeline.processors""" + from kb_pipeline.processors import DownloadProcessor + + assert DownloadProcessor is not None + assert hasattr(DownloadProcessor, 'STEP_NAME') + assert DownloadProcessor.STEP_NAME == "download" + + def test_all_processors_exported(self): + """All processors should be exported from kb_pipeline.processors""" + from kb_pipeline.processors import BaseProcessor, ParseProcessor, DownloadProcessor, ASRProcessor + + assert BaseProcessor is not None + assert ParseProcessor is not None + assert DownloadProcessor is not None + assert ASRProcessor is not None