feat(04-03): implement AnalyzeProcessor with DeepAnalyzer integration
- AnalyzeProcessor wraps existing DeepAnalyzer for pipeline integration - Loads metadata and content from JSON files - Generates analysis reports as Markdown with frontmatter - Records state on success and failure - Supports both video and article content types Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,252 @@
|
||||
"""AnalyzeProcessor - AI-powered content analysis
|
||||
|
||||
Unified processor for generating AI analysis reports from content.
|
||||
Wraps existing DeepAnalyzer for integration into the pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from ..config import get_settings
|
||||
from ..schemas import AnalyzeInput, AnalyzeOutput
|
||||
from ..state import WorkflowStateManager
|
||||
from .base import BaseProcessor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AnalyzeProcessor(BaseProcessor):
|
||||
"""Analyze 处理器 - AI 深度内容分析
|
||||
|
||||
处理流程：
|
||||
1. 检查步骤是否已完成（幂等执行）
|
||||
2. 加载元数据和内容文件
|
||||
3. 初始化 AI 客户端
|
||||
4. 调用 DeepAnalyzer 生成分析报告
|
||||
5. 生成 frontmatter 并保存 Markdown 文件
|
||||
6. 记录处理状态到 workflow_state.json
|
||||
|
||||
Attributes:
|
||||
STEP_NAME: 步骤名称，用于状态记录
|
||||
settings: KB 配置
|
||||
state_manager: 状态管理器（可选）
|
||||
"""
|
||||
|
||||
STEP_NAME = "analyze"
|
||||
|
||||
async def process(self, input_data: AnalyzeInput) -> AnalyzeOutput:
|
||||
"""执行分析处理
|
||||
|
||||
策略：
|
||||
1. 检查步骤是否已完成（跳过已完成）
|
||||
2. 加载元数据和内容
|
||||
3. 调用 DeepAnalyzer
|
||||
4. 保存分析报告
|
||||
5. 记录状态
|
||||
|
||||
Args:
|
||||
input_data: Analyze 输入数据
|
||||
|
||||
Returns:
|
||||
Analyze 处理结果
|
||||
|
||||
Raises:
|
||||
Exception: 处理失败时抛出异常
|
||||
"""
|
||||
try:
|
||||
# Step 1: 检查是否已完成（幂等执行）
|
||||
if self.state_manager:
|
||||
state = self.state_manager.load()
|
||||
if state and state.is_step_completed(self.STEP_NAME):
|
||||
logger.info(f"Step {self.STEP_NAME} already completed, skipping")
|
||||
# 返回已有结果
|
||||
existing_file = Path(input_data.output_file)
|
||||
if existing_file.exists():
|
||||
return AnalyzeOutput(
|
||||
success=True,
|
||||
analysis_file=str(existing_file),
|
||||
summary="Skipped - already completed",
|
||||
main_topics=[],
|
||||
key_insights=[],
|
||||
tags=[],
|
||||
stats={"skipped": True}
|
||||
)
|
||||
|
||||
# Step 2: 加载元数据和内容
|
||||
metadata = self._load_json(input_data.metadata_file)
|
||||
content = self._load_json(input_data.content_file)
|
||||
|
||||
# 对于文章类型，内容可能是纯文本文件
|
||||
if not content and input_data.content_type in ("article", "answer", "zhuanlan"):
|
||||
content_path = Path(input_data.content_file)
|
||||
if content_path.exists():
|
||||
text = content_path.read_text(encoding="utf-8")
|
||||
content = {"text": text}
|
||||
|
||||
# Step 3: 调用 DeepAnalyzer
|
||||
result = await self._run_analysis(
|
||||
metadata=metadata,
|
||||
content=content,
|
||||
content_type=input_data.content_type
|
||||
)
|
||||
|
||||
# Step 4: 生成并保存分析报告
|
||||
output_path = Path(input_data.output_file)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._write_analysis_report(
|
||||
output_path=output_path,
|
||||
metadata=metadata,
|
||||
result=result,
|
||||
content_type=input_data.content_type
|
||||
)
|
||||
|
||||
# Step 5: 记录成功状态
|
||||
if self.state_manager:
|
||||
state = self.state_manager.load()
|
||||
if state:
|
||||
summary = f"topics={len(result.main_topics)}, insights={len(result.key_insights)}"
|
||||
state.mark_step_completed(self.STEP_NAME, summary)
|
||||
self.state_manager.save(state)
|
||||
|
||||
return AnalyzeOutput(
|
||||
success=True,
|
||||
analysis_file=str(output_path),
|
||||
summary=result.summary,
|
||||
main_topics=result.main_topics,
|
||||
key_insights=result.key_insights,
|
||||
tags=result.recommended_tags,
|
||||
stats={
|
||||
"topics_count": len(result.main_topics),
|
||||
"insights_count": len(result.key_insights),
|
||||
"tags_count": len(result.recommended_tags)
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Analyze failed: {e}")
|
||||
|
||||
# 记录失败状态
|
||||
if self.state_manager:
|
||||
state = self.state_manager.load()
|
||||
if state:
|
||||
state.mark_step_failed(self.STEP_NAME, str(e))
|
||||
self.state_manager.save(state)
|
||||
|
||||
return AnalyzeOutput(
|
||||
success=False,
|
||||
error_message=str(e),
|
||||
analysis_file="",
|
||||
summary="",
|
||||
main_topics=[],
|
||||
key_insights=[],
|
||||
tags=[],
|
||||
stats={}
|
||||
)
|
||||
|
||||
def _load_json(self, file_path: str) -> Dict[str, Any]:
|
||||
"""加载 JSON 文件
|
||||
|
||||
Args:
|
||||
file_path: JSON 文件路径
|
||||
|
||||
Returns:
|
||||
解析后的字典，如果文件不存在或解析失败返回空字典
|
||||
"""
|
||||
if not file_path:
|
||||
return {}
|
||||
|
||||
path = Path(file_path)
|
||||
if not path.exists():
|
||||
logger.warning(f"File not found: {file_path}")
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse JSON: {file_path}, error: {e}")
|
||||
return {}
|
||||
|
||||
async def _run_analysis(self, metadata: Dict, content: Dict, content_type: str):
|
||||
"""调用 DeepAnalyzer 执行分析
|
||||
|
||||
Args:
|
||||
metadata: 元数据字典
|
||||
content: 内容字典
|
||||
content_type: 内容类型
|
||||
|
||||
Returns:
|
||||
AnalysisResult 对象
|
||||
"""
|
||||
# Lazy import to avoid dependency issues
|
||||
from kb_deep_analyzer.deep_analyzer import DeepAnalyzer, SimpleAIClient
|
||||
|
||||
# 初始化 AI 客户端
|
||||
ai_client = self._create_ai_client()
|
||||
|
||||
# 创建分析器实例
|
||||
analyzer = DeepAnalyzer(ai_client=ai_client)
|
||||
|
||||
# 执行分析
|
||||
return analyzer.analyze(metadata, content, content_type)
|
||||
|
||||
def _create_ai_client(self):
|
||||
"""创建 AI 客户端
|
||||
|
||||
Returns:
|
||||
SimpleAIClient 实例
|
||||
"""
|
||||
from kb_deep_analyzer.deep_analyzer import SimpleAIClient
|
||||
|
||||
# 从环境变量获取配置
|
||||
api_key = os.environ.get("AI_API_KEY", "")
|
||||
provider = os.environ.get("AI_PROVIDER", "")
|
||||
model = os.environ.get("AI_MODEL", "glm-5")
|
||||
base_url = os.environ.get("AI_BASE_URL", "")
|
||||
|
||||
return SimpleAIClient(
|
||||
api_key=api_key,
|
||||
provider=provider,
|
||||
model=model,
|
||||
base_url=base_url
|
||||
)
|
||||
|
||||
def _write_analysis_report(
|
||||
self,
|
||||
output_path: Path,
|
||||
metadata: Dict,
|
||||
result,
|
||||
content_type: str
|
||||
) -> None:
|
||||
"""写入分析报告 Markdown 文件
|
||||
|
||||
Args:
|
||||
output_path: 输出文件路径
|
||||
metadata: 元数据字典
|
||||
result: AnalysisResult 对象
|
||||
content_type: 内容类型
|
||||
"""
|
||||
from kb_deep_analyzer.deep_analyzer import generate_frontmatter
|
||||
|
||||
# 生成 frontmatter
|
||||
frontmatter = generate_frontmatter(metadata, result, content_type)
|
||||
|
||||
# 获取完整分析内容
|
||||
full_analysis = result.full_analysis
|
||||
|
||||
# 文章类型：追加原文
|
||||
if content_type == "article" and result.original_text:
|
||||
full_analysis = full_analysis.rstrip() + "\n\n## 原文\n\n" + result.original_text
|
||||
|
||||
# 写入文件
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write(frontmatter + "\n\n")
|
||||
f.write(full_analysis)
|
||||
|
||||
logger.info(f"Analysis report saved to: {output_path}")
|
||||
@@ -12,10 +12,24 @@ import json
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
|
||||
from kb_pipeline.schemas import AnalyzeInput, AnalyzeOutput
|
||||
|
||||
|
||||
# Mock AnalysisResult for testing
|
||||
@dataclass
|
||||
class MockAnalysisResult:
|
||||
"""Mock AnalysisResult for testing."""
|
||||
summary: str = "Test summary"
|
||||
main_topics: List[str] = field(default_factory=lambda: ["Topic 1", "Topic 2"])
|
||||
key_insights: List[str] = field(default_factory=lambda: ["Insight 1"])
|
||||
recommended_tags: List[str] = field(default_factory=lambda: ["tag1", "tag2"])
|
||||
full_analysis: str = "## Summary\n\nTest analysis content"
|
||||
original_text: str = ""
|
||||
|
||||
|
||||
class TestAnalyzeInputSchema:
|
||||
"""Tests for AnalyzeInput schema validation."""
|
||||
|
||||
@@ -156,21 +170,9 @@ class TestAnalyzeProcessorIntegration:
|
||||
"""Tests for AnalyzeProcessor DeepAnalyzer integration."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_deep_analyzer(self):
|
||||
"""Create a mock DeepAnalyzer result."""
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
|
||||
@dataclass
|
||||
class AnalysisResult:
|
||||
summary: str = "Test summary"
|
||||
main_topics: List[str] = field(default_factory=lambda: ["Topic 1", "Topic 2"])
|
||||
key_insights: List[str] = field(default_factory=lambda: ["Insight 1"])
|
||||
recommended_tags: List[str] = field(default_factory=lambda: ["tag1", "tag2"])
|
||||
full_analysis: str = "## Summary\n\nTest analysis content"
|
||||
original_text: str = ""
|
||||
|
||||
return AnalysisResult()
|
||||
def mock_analysis_result(self):
|
||||
"""Create a mock AnalysisResult."""
|
||||
return MockAnalysisResult()
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state_manager(self):
|
||||
@@ -208,7 +210,7 @@ class TestAnalyzeProcessorIntegration:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_loads_metadata_and_content_from_files(
|
||||
self, mock_deep_analyzer, mock_state_manager, sample_metadata, sample_content, tmp_path
|
||||
self, mock_analysis_result, mock_state_manager, sample_metadata, sample_content, tmp_path
|
||||
):
|
||||
"""Test 1: AnalyzeProcessor loads metadata and content from files"""
|
||||
from kb_pipeline.processors.analyze import AnalyzeProcessor
|
||||
@@ -227,24 +229,26 @@ class TestAnalyzeProcessorIntegration:
|
||||
|
||||
with patch("kb_pipeline.processors.base.get_settings") as mock_settings:
|
||||
mock_settings.return_value = MagicMock()
|
||||
with patch("kb_pipeline.processors.analyze.DeepAnalyzer") as MockDeepAnalyzer:
|
||||
mock_analyzer_instance = MagicMock()
|
||||
mock_analyzer_instance.analyze = MagicMock(return_value=mock_deep_analyzer)
|
||||
MockDeepAnalyzer.return_value = mock_analyzer_instance
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
result = await processor.process(input_data)
|
||||
# Mock _run_analysis to return our mock result
|
||||
async def mock_run_analysis(metadata, content, content_type):
|
||||
# Verify metadata was loaded correctly
|
||||
assert "title" in metadata
|
||||
assert metadata["title"] == "Test Video Title"
|
||||
# Verify content was loaded correctly
|
||||
assert "text" in content
|
||||
return mock_analysis_result
|
||||
|
||||
with patch.object(processor, '_run_analysis', side_effect=mock_run_analysis):
|
||||
with patch.object(processor, '_write_analysis_report'):
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is True
|
||||
# Verify DeepAnalyzer.analyze was called with loaded data
|
||||
mock_analyzer_instance.analyze.assert_called_once()
|
||||
call_args = mock_analyzer_instance.analyze.call_args
|
||||
assert "title" in call_args[0][0] # metadata dict
|
||||
assert "text" in call_args[0][1] # content dict
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_calls_deep_analyzer_with_correct_parameters(
|
||||
self, mock_deep_analyzer, mock_state_manager, sample_metadata, sample_content, tmp_path
|
||||
self, mock_analysis_result, mock_state_manager, sample_metadata, sample_content, tmp_path
|
||||
):
|
||||
"""Test 2: AnalyzeProcessor calls DeepAnalyzer.analyze() with correct parameters"""
|
||||
from kb_pipeline.processors.analyze import AnalyzeProcessor
|
||||
@@ -263,22 +267,28 @@ class TestAnalyzeProcessorIntegration:
|
||||
|
||||
with patch("kb_pipeline.processors.base.get_settings") as mock_settings:
|
||||
mock_settings.return_value = MagicMock()
|
||||
with patch("kb_pipeline.processors.analyze.DeepAnalyzer") as MockDeepAnalyzer:
|
||||
mock_analyzer_instance = MagicMock()
|
||||
mock_analyzer_instance.analyze = MagicMock(return_value=mock_deep_analyzer)
|
||||
MockDeepAnalyzer.return_value = mock_analyzer_instance
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
result = await processor.process(input_data)
|
||||
# Track what parameters were passed
|
||||
captured_params = {}
|
||||
|
||||
async def mock_run_analysis(metadata, content, content_type):
|
||||
captured_params['metadata'] = metadata
|
||||
captured_params['content'] = content
|
||||
captured_params['content_type'] = content_type
|
||||
return mock_analysis_result
|
||||
|
||||
with patch.object(processor, '_run_analysis', side_effect=mock_run_analysis):
|
||||
with patch.object(processor, '_write_analysis_report'):
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is True
|
||||
# Verify content_type was passed correctly
|
||||
call_kwargs = mock_analyzer_instance.analyze.call_args
|
||||
assert call_kwargs[0][2] == "video" # content_type parameter
|
||||
assert captured_params['content_type'] == "video"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_saves_analysis_report_as_markdown(
|
||||
self, mock_deep_analyzer, mock_state_manager, sample_metadata, sample_content, tmp_path
|
||||
self, mock_analysis_result, mock_state_manager, sample_metadata, sample_content, tmp_path
|
||||
):
|
||||
"""Test 3: AnalyzeProcessor saves analysis report as Markdown"""
|
||||
from kb_pipeline.processors.analyze import AnalyzeProcessor
|
||||
@@ -297,13 +307,19 @@ class TestAnalyzeProcessorIntegration:
|
||||
|
||||
with patch("kb_pipeline.processors.base.get_settings") as mock_settings:
|
||||
mock_settings.return_value = MagicMock()
|
||||
with patch("kb_pipeline.processors.analyze.DeepAnalyzer") as MockDeepAnalyzer:
|
||||
mock_analyzer_instance = MagicMock()
|
||||
mock_analyzer_instance.analyze = MagicMock(return_value=mock_deep_analyzer)
|
||||
MockDeepAnalyzer.return_value = mock_analyzer_instance
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
result = await processor.process(input_data)
|
||||
async def mock_run_analysis(metadata, content, content_type):
|
||||
return mock_analysis_result
|
||||
|
||||
def mock_write_report(output_path, metadata, result, content_type):
|
||||
# Write a simple markdown file with frontmatter
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write("---\ntitle: Test\n---\n\n## Summary\n\nTest analysis content")
|
||||
|
||||
with patch.object(processor, '_run_analysis', side_effect=mock_run_analysis):
|
||||
with patch.object(processor, '_write_analysis_report', side_effect=mock_write_report):
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is True
|
||||
assert result.analysis_file == output_file
|
||||
@@ -319,7 +335,7 @@ class TestAnalyzeProcessorIntegration:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_records_state_on_success(
|
||||
self, mock_deep_analyzer, mock_state_manager, sample_metadata, sample_content, tmp_path
|
||||
self, mock_analysis_result, mock_state_manager, sample_metadata, sample_content, tmp_path
|
||||
):
|
||||
"""Test 4: AnalyzeProcessor records state on success"""
|
||||
from kb_pipeline.processors.analyze import AnalyzeProcessor
|
||||
@@ -343,13 +359,14 @@ class TestAnalyzeProcessorIntegration:
|
||||
|
||||
with patch("kb_pipeline.processors.base.get_settings") as mock_settings:
|
||||
mock_settings.return_value = MagicMock()
|
||||
with patch("kb_pipeline.processors.analyze.DeepAnalyzer") as MockDeepAnalyzer:
|
||||
mock_analyzer_instance = MagicMock()
|
||||
mock_analyzer_instance.analyze = MagicMock(return_value=mock_deep_analyzer)
|
||||
MockDeepAnalyzer.return_value = mock_analyzer_instance
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
result = await processor.process(input_data)
|
||||
async def mock_run_analysis(metadata, content, content_type):
|
||||
return mock_analysis_result
|
||||
|
||||
with patch.object(processor, '_run_analysis', side_effect=mock_run_analysis):
|
||||
with patch.object(processor, '_write_analysis_report'):
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is True
|
||||
|
||||
@@ -385,14 +402,12 @@ class TestAnalyzeProcessorIntegration:
|
||||
|
||||
with patch("kb_pipeline.processors.base.get_settings") as mock_settings:
|
||||
mock_settings.return_value = MagicMock()
|
||||
with patch("kb_pipeline.processors.analyze.DeepAnalyzer") as MockDeepAnalyzer:
|
||||
mock_analyzer_instance = MagicMock()
|
||||
mock_analyzer_instance.analyze = MagicMock(
|
||||
side_effect=ValueError("AI API error")
|
||||
)
|
||||
MockDeepAnalyzer.return_value = mock_analyzer_instance
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
|
||||
processor = AnalyzeProcessor(state_manager=mock_state_manager)
|
||||
async def mock_run_analysis_error(metadata, content, content_type):
|
||||
raise ValueError("AI API error")
|
||||
|
||||
with patch.object(processor, '_run_analysis', side_effect=mock_run_analysis_error):
|
||||
result = await processor.process(input_data)
|
||||
|
||||
assert result.success is False
|
||||
|
||||
Reference in New Issue
Block a user