starmen1265
c9ca1b538d
- Remove sys.path manipulation from kb_workflow.py - Update CLI imports to use kb_workflow.pipeline prefix - Update all test file imports and patch paths Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
498 lines
18 KiB
Python
498 lines
18 KiB
Python
"""Pipeline Tests
|
|
|
|
Tests for Pipeline class and PipelineInput/Output/Config schemas.
|
|
"""
|
|
|
|
import pytest
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
from pydantic import ValidationError
|
|
from unittest.mock import MagicMock, patch, AsyncMock
|
|
|
|
|
|
# =============================================================================
|
|
# Pipeline Schema Tests (Task 1)
|
|
# =============================================================================
|
|
|
|
|
|
class TestPipelineInput:
|
|
"""Test PipelineInput Schema"""
|
|
|
|
def test_schema_pipeline_input_validates_url_field(self):
|
|
"""PipelineInput must include url field (required)"""
|
|
from kb_workflow.pipeline.schemas import PipelineInput
|
|
|
|
# Missing url
|
|
with pytest.raises(ValidationError) as exc_info:
|
|
PipelineInput()
|
|
assert "url" in str(exc_info.value)
|
|
|
|
# Empty url
|
|
with pytest.raises(ValidationError):
|
|
PipelineInput(url="")
|
|
|
|
# Valid url
|
|
input_data = PipelineInput(url="https://www.bilibili.com/video/BV1xx")
|
|
assert input_data.url == "https://www.bilibili.com/video/BV1xx"
|
|
|
|
def test_schema_pipeline_input_has_optional_output_dir(self):
|
|
"""PipelineInput has optional output_dir field"""
|
|
from kb_workflow.pipeline.schemas import PipelineInput
|
|
|
|
# Without output_dir
|
|
input_data = PipelineInput(url="https://www.bilibili.com/video/BV1xx")
|
|
assert input_data.output_dir is None
|
|
|
|
# With output_dir
|
|
input_with_dir = PipelineInput(
|
|
url="https://www.bilibili.com/video/BV1xx",
|
|
output_dir=Path("/tmp/content")
|
|
)
|
|
assert input_with_dir.output_dir == Path("/tmp/content")
|
|
|
|
def test_schema_pipeline_input_has_skip_completed_default(self):
|
|
"""PipelineInput skip_completed defaults to True"""
|
|
from kb_workflow.pipeline.schemas import PipelineInput
|
|
|
|
input_data = PipelineInput(url="https://www.bilibili.com/video/BV1xx")
|
|
assert input_data.skip_completed is True
|
|
|
|
# Can be set to False
|
|
input_no_skip = PipelineInput(
|
|
url="https://www.bilibili.com/video/BV1xx",
|
|
skip_completed=False
|
|
)
|
|
assert input_no_skip.skip_completed is False
|
|
|
|
|
|
class TestPipelineOutput:
|
|
"""Test PipelineOutput Schema"""
|
|
|
|
def test_schema_pipeline_output_contains_results_dict(self):
|
|
"""PipelineOutput contains results dict"""
|
|
from kb_workflow.pipeline.schemas import PipelineOutput
|
|
|
|
output = PipelineOutput(success=True)
|
|
assert hasattr(output, "results")
|
|
assert isinstance(output.results, dict)
|
|
assert output.results == {}
|
|
|
|
def test_schema_pipeline_output_has_success_field(self):
|
|
"""PipelineOutput requires success field"""
|
|
from kb_workflow.pipeline.schemas import PipelineOutput
|
|
|
|
with pytest.raises(ValidationError) as exc_info:
|
|
PipelineOutput()
|
|
assert "success" in str(exc_info.value)
|
|
|
|
def test_schema_pipeline_output_has_optional_error_message(self):
|
|
"""PipelineOutput has optional error_message field"""
|
|
from kb_workflow.pipeline.schemas import PipelineOutput
|
|
|
|
output = PipelineOutput(success=True)
|
|
assert output.error_message is None
|
|
|
|
output_with_error = PipelineOutput(
|
|
success=False,
|
|
error_message="Pipeline failed"
|
|
)
|
|
assert output_with_error.error_message == "Pipeline failed"
|
|
|
|
def test_schema_pipeline_output_has_content_dir(self):
|
|
"""PipelineOutput has optional content_dir field"""
|
|
from kb_workflow.pipeline.schemas import PipelineOutput
|
|
|
|
output = PipelineOutput(success=True)
|
|
assert output.content_dir is None
|
|
|
|
output_with_dir = PipelineOutput(
|
|
success=True,
|
|
content_dir=Path("/tmp/content/bilibili_BV1xx")
|
|
)
|
|
assert output_with_dir.content_dir == Path("/tmp/content/bilibili_BV1xx")
|
|
|
|
|
|
class TestPipelineConfig:
|
|
"""Test PipelineConfig Schema"""
|
|
|
|
def test_schema_pipeline_config_has_default_steps_list(self):
|
|
"""PipelineConfig has default steps list"""
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
config = PipelineConfig()
|
|
assert hasattr(config, "steps")
|
|
assert isinstance(config.steps, list)
|
|
assert config.steps == ["parse", "download", "asr", "analyze", "archive"]
|
|
|
|
def test_schema_pipeline_config_allows_custom_steps(self):
|
|
"""PipelineConfig allows custom steps list"""
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
custom_config = PipelineConfig(steps=["parse", "download"])
|
|
assert custom_config.steps == ["parse", "download"]
|
|
|
|
def test_schema_pipeline_config_allows_custom_output_dir(self):
|
|
"""PipelineConfig allows custom output_dir"""
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
config = PipelineConfig(output_dir=Path("/custom/output"))
|
|
assert config.output_dir == Path("/custom/output")
|
|
|
|
# Default is None
|
|
default_config = PipelineConfig()
|
|
assert default_config.output_dir is None
|
|
|
|
def test_schema_pipeline_config_has_skip_completed_default(self):
|
|
"""PipelineConfig skip_completed defaults to True"""
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
config = PipelineConfig()
|
|
assert config.skip_completed is True
|
|
|
|
def test_schema_pipeline_config_has_max_retries_default(self):
|
|
"""PipelineConfig max_retries defaults to 3"""
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
config = PipelineConfig()
|
|
assert config.max_retries == 3
|
|
|
|
# Can be customized
|
|
custom_config = PipelineConfig(max_retries=5)
|
|
assert custom_config.max_retries == 5
|
|
|
|
|
|
# =============================================================================
|
|
# Pipeline Class Tests (Task 2)
|
|
# =============================================================================
|
|
|
|
|
|
class TestPipelineInit:
|
|
"""Test Pipeline initialization"""
|
|
|
|
def test_init_pipeline_with_processors(self):
|
|
"""Pipeline initializes with all processors"""
|
|
from kb_workflow.pipeline.pipeline import Pipeline
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
mock_processor1 = MagicMock()
|
|
mock_processor1.STEP_NAME = "parse"
|
|
|
|
mock_processor2 = MagicMock()
|
|
mock_processor2.STEP_NAME = "download"
|
|
|
|
with patch("kb_workflow.pipeline.pipeline.get_settings") as mock_settings:
|
|
mock_settings.return_value = MagicMock()
|
|
|
|
pipeline = Pipeline(processors=[mock_processor1, mock_processor2])
|
|
|
|
assert "parse" in pipeline.processors
|
|
assert "download" in pipeline.processors
|
|
assert pipeline.processors["parse"] == mock_processor1
|
|
assert pipeline.processors["download"] == mock_processor2
|
|
|
|
def test_init_pipeline_validates_processor_step_name(self):
|
|
"""Pipeline validates processor STEP_NAME matches config.steps"""
|
|
from kb_workflow.pipeline.pipeline import Pipeline
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
mock_processor = MagicMock()
|
|
mock_processor.STEP_NAME = "unknown_step"
|
|
|
|
config = PipelineConfig(steps=["parse", "download"])
|
|
|
|
with patch("kb_workflow.pipeline.pipeline.get_settings") as mock_settings:
|
|
mock_settings.return_value = MagicMock()
|
|
|
|
with pytest.raises(ValueError) as exc_info:
|
|
Pipeline(processors=[mock_processor], config=config)
|
|
|
|
assert "Unknown processor" in str(exc_info.value)
|
|
|
|
def test_init_pipeline_handles_missing_processor_gracefully(self):
|
|
"""Pipeline handles missing processor gracefully"""
|
|
from kb_workflow.pipeline.pipeline import Pipeline
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
# Empty processors list should work
|
|
config = PipelineConfig(steps=[])
|
|
|
|
with patch("kb_workflow.pipeline.pipeline.get_settings") as mock_settings:
|
|
mock_settings.return_value = MagicMock()
|
|
|
|
pipeline = Pipeline(processors=[], config=config)
|
|
assert pipeline.processors == {}
|
|
|
|
|
|
class TestPipelineParseUrl:
|
|
"""Test Pipeline parse_url method"""
|
|
|
|
def test_parse_url_creates_correct_content_directory(self):
|
|
"""Pipeline._get_content_dir creates correct content directory"""
|
|
from kb_workflow.pipeline.pipeline import Pipeline
|
|
from kb_workflow.pipeline.link_parser import ParsedLink
|
|
import tempfile
|
|
|
|
mock_processor = MagicMock()
|
|
mock_processor.STEP_NAME = "parse"
|
|
|
|
with patch("kb_workflow.pipeline.pipeline.get_settings") as mock_settings:
|
|
mock_settings.return_value = MagicMock()
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
pipeline = Pipeline(
|
|
processors=[mock_processor],
|
|
output_dir=Path(tmpdir)
|
|
)
|
|
|
|
parsed = ParsedLink(
|
|
platform="bilibili",
|
|
content_type="video",
|
|
content_id="BV1xx411c7mD",
|
|
original_url="https://www.bilibili.com/video/BV1xx411c7mD"
|
|
)
|
|
|
|
content_dir = pipeline._get_content_dir(parsed)
|
|
|
|
assert content_dir.exists()
|
|
assert content_dir.name == "bilibili_BV1xx411c7mD"
|
|
|
|
def test_parse_url_returns_none_for_empty_url(self):
|
|
"""Pipeline._parse_url returns None for empty URL"""
|
|
from kb_workflow.pipeline.pipeline import Pipeline
|
|
|
|
mock_processor = MagicMock()
|
|
mock_processor.STEP_NAME = "parse"
|
|
|
|
with patch("kb_workflow.pipeline.pipeline.get_settings") as mock_settings:
|
|
mock_settings.return_value = MagicMock()
|
|
|
|
pipeline = Pipeline(processors=[mock_processor])
|
|
result = pipeline._parse_url("")
|
|
|
|
assert result is None
|
|
|
|
def test_parse_url_returns_none_for_invalid_url(self):
|
|
"""Pipeline._parse_url returns None for unrecognized URL"""
|
|
from kb_workflow.pipeline.pipeline import Pipeline
|
|
|
|
mock_processor = MagicMock()
|
|
mock_processor.STEP_NAME = "parse"
|
|
|
|
with patch("kb_workflow.pipeline.pipeline.get_settings") as mock_settings:
|
|
mock_settings.return_value = MagicMock()
|
|
|
|
pipeline = Pipeline(processors=[mock_processor])
|
|
result = pipeline._parse_url("https://www.unknown.com/123")
|
|
|
|
assert result is None
|
|
|
|
def test_parse_url_parses_bilibili_url(self):
|
|
"""Pipeline._parse_url correctly parses Bilibili URL"""
|
|
from kb_workflow.pipeline.pipeline import Pipeline
|
|
|
|
mock_processor = MagicMock()
|
|
mock_processor.STEP_NAME = "parse"
|
|
|
|
with patch("kb_workflow.pipeline.pipeline.get_settings") as mock_settings:
|
|
mock_settings.return_value = MagicMock()
|
|
|
|
pipeline = Pipeline(processors=[mock_processor])
|
|
result = pipeline._parse_url("https://www.bilibili.com/video/BV1xx411c7mD")
|
|
|
|
assert result is not None
|
|
assert result.platform == "bilibili"
|
|
assert result.content_type == "video"
|
|
assert result.content_id == "BV1xx411c7mD"
|
|
|
|
|
|
# =============================================================================
|
|
# Archive Integration Tests
|
|
# =============================================================================
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_processors_with_archive():
|
|
"""Create mock processors including archive"""
|
|
processors = []
|
|
|
|
for step_name in ["parse", "download", "asr", "analyze", "archive"]:
|
|
processor = MagicMock()
|
|
processor.STEP_NAME = step_name
|
|
processor.process = AsyncMock()
|
|
|
|
if step_name == "parse":
|
|
processor.process.return_value = MagicMock(
|
|
success=True,
|
|
platform="bilibili",
|
|
content_type="video",
|
|
content_id="BV1test",
|
|
resolved_url="https://www.bilibili.com/video/BV1test",
|
|
original_url="https://www.bilibili.com/video/BV1test"
|
|
)
|
|
elif step_name == "download":
|
|
processor.process.return_value = MagicMock(
|
|
success=True,
|
|
metadata_file="/tmp/metadata.json",
|
|
content_file="/tmp/content.json",
|
|
title="Test Video",
|
|
author="Test Author"
|
|
)
|
|
elif step_name == "asr":
|
|
processor.process.return_value = MagicMock(
|
|
success=True,
|
|
tool="whisper",
|
|
has_subtitle=False,
|
|
output_files={"json": "/tmp/subtitle.json"}
|
|
)
|
|
elif step_name == "analyze":
|
|
processor.process.return_value = MagicMock(
|
|
success=True,
|
|
analysis_file="/tmp/analysis.md",
|
|
summary="Test summary",
|
|
main_topics=["topic1"],
|
|
key_insights=["insight1"],
|
|
tags=["tag1"],
|
|
stats={}
|
|
)
|
|
elif step_name == "archive":
|
|
processor.process.return_value = MagicMock(
|
|
success=True,
|
|
skipped=False,
|
|
platform="bilibili",
|
|
archive_file="/archive/bilibili/20260316_BV1test_analysis.md"
|
|
)
|
|
|
|
processors.append(processor)
|
|
|
|
return processors
|
|
|
|
|
|
class TestPipelineArchiveStep:
|
|
"""Test archive step integration in pipeline"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_includes_archive_step(self, mock_processors_with_archive, tmp_path):
|
|
"""Test that pipeline includes archive as final step"""
|
|
from kb_workflow.pipeline.pipeline import Pipeline
|
|
from kb_workflow.pipeline.schemas import PipelineConfig
|
|
|
|
config = PipelineConfig(
|
|
steps=["parse", "download", "asr", "analyze", "archive"],
|
|
output_dir=tmp_path,
|
|
skip_completed=False
|
|
)
|
|
|
|
with patch("kb_workflow.pipeline.pipeline.get_settings") as mock_settings:
|
|
mock_settings.return_value = MagicMock(intermediate_path=str(tmp_path))
|
|
|
|
pipeline = Pipeline(
|
|
processors=mock_processors_with_archive,
|
|
config=config,
|
|
output_dir=tmp_path
|
|
)
|
|
|
|
# Mock _parse_url and _get_content_dir
|
|
with patch.object(pipeline, '_parse_url') as mock_parse, \
|
|
patch.object(pipeline, '_get_content_dir') as mock_dir:
|
|
|
|
from kb_workflow.pipeline.link_parser import ParsedLink
|
|
mock_parse.return_value = ParsedLink(
|
|
platform="bilibili",
|
|
content_type="video",
|
|
content_id="BV1test",
|
|
original_url="https://www.bilibili.com/video/BV1test",
|
|
resolved_url="https://www.bilibili.com/video/BV1test"
|
|
)
|
|
mock_dir.return_value = tmp_path / "bilibili_BV1test"
|
|
|
|
result = await pipeline.run("https://www.bilibili.com/video/BV1test")
|
|
|
|
assert result.success is True
|
|
# Verify archive processor was called
|
|
archive_processor = next(p for p in mock_processors_with_archive if p.STEP_NAME == "archive")
|
|
archive_processor.process.assert_called_once()
|
|
|
|
|
|
class TestArchivePlatformDirectory:
|
|
"""Test that archive files go to correct platform directory"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_archive_uses_platform_directory(self, tmp_path):
|
|
"""Test that archive files go to {archive_path}/{platform}/ directory"""
|
|
from kb_workflow.pipeline.processors.archive import ArchiveProcessor
|
|
from kb_workflow.pipeline.schemas import ArchiveInput
|
|
|
|
# Setup
|
|
archive_dir = tmp_path / "archive"
|
|
content_dir = tmp_path / "intermediate" / "bilibili_BV1test"
|
|
content_dir.mkdir(parents=True)
|
|
analysis_file = content_dir / "analysis.md"
|
|
analysis_file.write_text("# Test")
|
|
|
|
# Mock settings - patch at base module level where get_settings is called in __init__
|
|
with patch('kb_workflow.pipeline.processors.base.get_settings') as mock_get_settings:
|
|
mock_settings = MagicMock()
|
|
mock_settings.archive_path = str(archive_dir)
|
|
mock_get_settings.return_value = mock_settings
|
|
|
|
processor = ArchiveProcessor()
|
|
# Mock _ask_overwrite to avoid stdin issues
|
|
processor._ask_overwrite = lambda f: True
|
|
|
|
input_data = ArchiveInput(
|
|
content_id="BV1test",
|
|
platform="bilibili",
|
|
content_dir=str(content_dir),
|
|
analysis_file=str(analysis_file),
|
|
content_type="video"
|
|
)
|
|
|
|
result = await processor.process(input_data)
|
|
|
|
assert result.success is True
|
|
# Verify platform subdirectory
|
|
archive_path = Path(result.archive_file)
|
|
assert archive_path.parent.name == "bilibili"
|
|
|
|
|
|
class TestArchiveNoDateSubdirectories:
|
|
"""Test that archive does not create date-based subdirectories"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_archive_no_date_subdirectories(self, tmp_path):
|
|
"""Test that archive does not create date-based subdirectories"""
|
|
from kb_workflow.pipeline.processors.archive import ArchiveProcessor
|
|
from kb_workflow.pipeline.schemas import ArchiveInput
|
|
|
|
archive_dir = tmp_path / "archive"
|
|
content_dir = tmp_path / "intermediate" / "bilibili_BV1test"
|
|
content_dir.mkdir(parents=True)
|
|
analysis_file = content_dir / "analysis.md"
|
|
analysis_file.write_text("# Test")
|
|
|
|
with patch('kb_workflow.pipeline.processors.base.get_settings') as mock_get_settings:
|
|
mock_settings = MagicMock()
|
|
mock_settings.archive_path = str(archive_dir)
|
|
mock_get_settings.return_value = mock_settings
|
|
|
|
processor = ArchiveProcessor()
|
|
# Mock _ask_overwrite to avoid stdin issues
|
|
processor._ask_overwrite = lambda f: True
|
|
|
|
input_data = ArchiveInput(
|
|
content_id="BV1test",
|
|
platform="bilibili",
|
|
content_dir=str(content_dir),
|
|
analysis_file=str(analysis_file),
|
|
content_type="video"
|
|
)
|
|
|
|
result = await processor.process(input_data)
|
|
|
|
# Verify no date-based subdirectories
|
|
archive_path = Path(result.archive_file)
|
|
# Parent should be platform dir, grandparent should be archive base
|
|
assert archive_path.parent.parent == archive_dir
|
|
# Check that parent is just "bilibili", not a date
|
|
assert archive_path.parent.name == "bilibili"
|