openclaw-skills/skills/obsidian-kb/src/pipeline.py

"""Obsidian KB - 内容处理管道
C4.3: 实现完整处理流程
"""
import os
from datetime import datetime
from typing import Optional
from src.models import Content, ContentStatus, AnalysisResult
from src.light_scanner import LightScanner
from src.deep_analyzer import DeepAnalyzer
from src.config import get_config


class ProcessingPipeline:
    """内容处理管道 - 协调轻量扫描、深度分析、笔记生成和同步"""
    
    def __init__(self, ai_client=None, git_client=None):
        self.config = get_config()
        self.ai_client = ai_client
        self.git_client = git_client
        
        # 初始化处理器
        self.light_scanner = LightScanner(ai_client=ai_client)
        self.deep_analyzer = DeepAnalyzer(ai_client=ai_client)
    
    def light_scan(self, content: Content, content_text: str) -> Content:
        """
        轻量扫描阶段
        
        Args:
            content: Content对象
            content_text: 内容文本
            
        Returns:
            更新后的Content对象
        """
        print(f"[Pipeline] 开始轻量扫描: {content.id}")
        
        try:
            content = self.light_scanner.scan(content, content_text)
            print(f"[Pipeline] 轻量扫描完成: 摘要长度={len(content.summary)}, 关键词={content.key_points}, 质量分={content.quality_metrics.overall_score:.2f}")
            return content
        except Exception as e:
            print(f"[Pipeline] 轻量扫描失败: {e}")
            raise
    
    def deep_analyze(self, content: Content, content_text: str) -> Content:
        """
        深度分析阶段
        
        Args:
            content: Content对象
            content_text: 内容文本
            
        Returns:
            更新后的Content对象
        """
        print(f"[Pipeline] 开始深度分析: {content.id}")
        
        try:
            analysis_result = self.deep_analyzer.analyze(content, content_text)
            print(f"[Pipeline] 深度分析完成: 主题数={len(analysis_result.main_topics)}, 时间轴段数={len(analysis_result.timeline)}")
            return content
        except Exception as e:
            print(f"[Pipeline] 深度分析失败: {e}")
            content.update_status(ContentStatus.ERROR)
            raise
    
    def process_full(self, content: Content, content_text: str) -> Content:
        """
        完整处理流程:
        轻量扫描 → 入库 → 深度分析 → 生成笔记 → Git同步
        
        Args:
            content: Content对象
            content_text: 内容文本
            
        Returns:
            完全处理后的Content对象
        """
        print(f"\n{'='*60}")
        print(f"[Pipeline] 开始完整处理流程: {content.id}")
        print(f"{'='*60}\n")
        
        try:
            # Step 1: 轻量扫描
            print("\n📋 Step 1: 轻量扫描")
            content = self.light_scan(content, content_text)
            
            # Step 2: 入库（保存到数据库/文件系统）
            print("\n💾 Step 2: 保存内容")
            self._save_content(content)
            
            # Step 3: 深度分析
            print("\n🔍 Step 3: 深度分析")
            content = self.deep_analyze(content, content_text)
            
            # Step 4: 生成笔记
            print("\n📝 Step 4: 生成Obsidian笔记")
            content = self._generate_note(content)
            
            # Step 5: Git同步
            if self.config.git.enabled:
                print("\n🔄 Step 5: Git同步")
                self._sync_to_git(content)
            else:
                print("\n⏭️ Step 5: Git同步已禁用，跳过")
            
            content.update_status(ContentStatus.GENERATED)
            
            print(f"\n{'='*60}")
            print(f"[Pipeline] 处理完成: {content.id}")
            print(f"{'='*60}\n")
            
            return content
            
        except Exception as e:
            print(f"\n❌ [Pipeline] 处理流程失败: {e}")
            raise
    
    def _save_content(self, content: Content) -> None:
        """保存内容到存储"""
        # 创建存储目录
        storage_dir = os.path.expanduser("~/.obsidian-kb/storage")
        os.makedirs(storage_dir, exist_ok=True)
        
        # 保存JSON文件
        import json
        file_path = os.path.join(storage_dir, f"{content.id}.json")
        
        with open(file_path, 'w', encoding='utf-8') as f:
            json.dump(content.to_dict(), f, ensure_ascii=False, indent=2)
        
        print(f"[Pipeline] 内容已保存: {file_path}")
    
    def _generate_note(self, content: Content) -> Content:
        """生成Obsidian笔记"""
        # 构建笔记内容
        note_content = self._build_note_content(content)
        
        # 确定笔记路径
        vault_path = os.path.expanduser(self.config.obsidian.vault_path)
        notes_folder = self.config.obsidian.notes_folder
        
        note_dir = os.path.join(vault_path, notes_folder)
        os.makedirs(note_dir, exist_ok=True)
        
        # 生成文件名
        safe_title = self._sanitize_filename(content.title or content.id)
        timestamp = datetime.now().strftime("%Y%m%d")
        note_filename = f"{timestamp}_{safe_title}.md"
        note_path = os.path.join(note_dir, note_filename)
        
        # 写入文件
        with open(note_path, 'w', encoding='utf-8') as f:
            f.write(note_content)
        
        content.output_path = note_path
        content.update_status(ContentStatus.GENERATED)
        
        print(f"[Pipeline] 笔记已生成: {note_path}")
        return content
    
    def _build_note_content(self, content: Content) -> str:
        """构建Obsidian笔记内容"""
        lines = [
            f"# {content.title or '未命名内容'}",
            "",
            "## 📊 元数据",
            f"- **来源**: {content.source}",
            f"- **URL**: {content.url}",
            f"- **作者**: {content.author or '未知'}",
            f"- **类型**: {content.content_type.value}",
        ]
        
        if content.duration:
            duration_str = self._format_duration(content.duration)
            lines.append(f"- **时长**: {duration_str}")
        
        lines.extend([
            f"- **质量评分**: {content.quality_metrics.overall_score:.1f}/1.0",
            f"- **处理时间**: {datetime.now().strftime('%Y-%m-%d %H:%M')}",
            "",
            "## 📝 摘要",
            content.summary or "暂无摘要",
            "",
            "## 🏷️ 关键词",
        ])
        
        # 添加关键词标签
        if content.key_points:
            keyword_tags = ' '.join([f"#{kw.replace(' ', '_')}" for kw in content.key_points[:5]])
            lines.append(keyword_tags)
            lines.append('')
        
        # 添加时间轴
        if content.timeline:
            lines.extend([
                "",
                "## ⏱️ 时间轴",
                "",
            ])
            
            for item in content.timeline:
                lines.append(f"### {item.timestamp} - {item.title}")
                lines.append(item.description)
                lines.append('')
        
        # 添加原始内容（折叠）
        lines.extend([
            "",
            "## 📄 原始内容",
            "",
            "```",
            content.transcript[:2000] + "..." if len(content.transcript) > 2000 else content.transcript,
            "```",
            "",
            "---",
            f"*由 Obsidian KB 自动生成 - {datetime.now().strftime('%Y-%m-%d %H:%M')}*"
        ])
        
        return '\n'.join(lines)
    
    def _sync_to_git(self, content: Content) -> None:
        """同步到Git仓库"""
        if not self.git_client:
            print("[Pipeline] Git客户端未配置，跳过同步")
            return
        
        try:
            vault_path = os.path.expanduser(self.config.obsidian.vault_path)
            
            # 添加文件
            relative_path = os.path.relpath(content.output_path, vault_path)
            self.git_client.add(relative_path)
            
            # 提交
            commit_msg = self.config.git.commit_message_template.format(
                title=content.title or content.id
            )
            self.git_client.commit(commit_msg)
            
            # 推送（如果启用）
            if self.config.git.auto_push:
                self.git_client.push()
            
            print(f"[Pipeline] Git同步完成")
            
        except Exception as e:
            print(f"[Pipeline] Git同步失败: {e}")
            # Git同步失败不阻断主流程
    
    def _sanitize_filename(self, filename: str) -> str:
        """清理文件名"""
        import re
        # 替换非法字符
        filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
        # 限制长度
        if len(filename) > 50:
            filename = filename[:50]
        return filename.strip()
    
    def _format_duration(self, seconds: int) -> str:
        """格式化时长"""
        hours = seconds // 3600
        minutes = (seconds % 3600) // 60
        secs = seconds % 60
        
        if hours > 0:
            return f"{hours}小时{minutes}分钟"
        elif minutes > 0:
            return f"{minutes}分钟{secs}秒"
        else:
            return f"{secs}秒"