openclaw-skills/skills/kb-workflow/kb_pipeline.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Obsidian KB - 主流程集成

整合提取层、处理层、输出层，实现完整的数据流：
提取(BilibiliExtractor) → 处理(DeepAnalyzer/ContentChunker) → 输出(ObsidianWriter) → 同步(GitSync)
"""

import os
import sys
import logging
from typing import Dict, Any, Optional
from datetime import datetime
from pathlib import Path

# 添加项目路径
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

from src.extractors.bilibili_extractor import BilibiliExtractor, ExtractionError
from src.deep_analyzer import DeepAnalyzer
from src.content_chunker import ContentChunker, Chunk
from src.outputs.obsidian_writer import ObsidianWriter
from src.outputs.git_sync import GitSync
from src.models import Content, ContentType, ContentStatus, TimelineEntry
from src.config import get_config


# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    handlers=[
        logging.StreamHandler(sys.stdout),
        logging.FileHandler('obsidian_kb.log', encoding='utf-8')
    ]
)
logger = logging.getLogger('ObsidianKB')


class MainPipeline:
    """主流程集成器"""

    def __init__(self, config=None, enable_git_sync: bool = False):
        """
        初始化主流程

        Args:
            config: 配置对象
            enable_git_sync: 是否启用Git同步
        """
        self.config = config or get_config()
        self.enable_git_sync = enable_git_sync

        logger.info("初始化主流程...")
        logger.info(f"Obsidian vault 路径: {self.config.vault_path}")
        logger.info(f"Git 同步: {'启用' if enable_git_sync else '禁用'}")

        # 初始化各层组件
        self.extractor = BilibiliExtractor(config=self.config)
        self.analyzer = DeepAnalyzer(ai_client=None)  # 暂时不传入ai_client
        self.chunker = ContentChunker()
        self.writer = ObsidianWriter(config=self.config)

        # 初始化Git同步器（仅在需要时）
        self.git_sync = None
        if enable_git_sync and self._is_git_enabled():
            self.git_sync = GitSync(config=self.config)
            logger.info("Git 同步器已初始化")

    def _is_git_enabled(self) -> bool:
        """检查Git同步是否在配置中启用"""
        return self.config.get('git.enabled', False)

    def process_video(self, source_id: str) -> Dict[str, Any]:
        """
        处理单个视频的完整流程

        Args:
            source_id: 视频ID（支持BV号、URL）

        Returns:
            处理结果字典，包含：
                - success: bool - 是否成功
                - content: Content对象 - 处理的内容
                - message: str - 处理消息
                - error: str - 错误信息（如果失败）
        """
        result = {
            'success': False,
            'content': None,
            'message': '',
            'error': ''
        }

        try:
            logger.info("=" * 80)
            logger.info(f"开始处理视频: {source_id}")
            logger.info("=" * 80)

            # Step 1: 提取层 - 提取视频信息
            logger.info("\n[Step 1/5] 提取视频信息...")
            extractor_data = self._extract_video(source_id)

            # Step 2: 构建Content对象
            logger.info("\n[Step 2/5] 构建内容对象...")
            content = self._build_content(extractor_data)

            # Step 3: 处理层 - 深度分析
            logger.info("\n[Step 3/5] 深度分析...")
            analysis_result = self._analyze_content(extractor_data)

            # Step 4: 处理层 - 内容分片
            logger.info("\n[Step 4/5] 内容分片...")
            chunks = self._chunk_content(extractor_data)

            # 合并分析结果和分片信息到content
            self._merge_analysis_to_content(content, analysis_result, chunks)

            # Step 5: 输出层 - 写入Obsidian
            logger.info("\n[Step 5/5] 写入Obsidian...")
            output_path = self._write_obsidian_note(content)

            # Step 6: 同步层 - Git同步（可选）
            if self.enable_git_sync and self.git_sync:
                logger.info("\n[Step 6/6] Git同步...")
                self._sync_to_git(content)

            # 更新状态
            content.update_status(ContentStatus.GENERATED)
            content.output_path = output_path

            # 构建成功结果
            result['success'] = True
            result['content'] = content
            result['message'] = self._format_success_message(content, chunks)

            logger.info("\n" + "=" * 80)
            logger.info(f"✅ 处理完成: {content.title}")
            logger.info(f"   📄 笔记路径: {output_path}")
            logger.info(f"   📊 字幕条数: {len(chunks) if chunks else 0}")
            logger.info("=" * 80 + "\n")

        except ExtractionError as e:
            logger.error(f"❌ 提取失败: {e}")
            result['error'] = f"提取失败: {e}"
            result['message'] = f"提取视频信息失败: {e}"

        except Exception as e:
            logger.error(f"❌ 处理失败: {e}", exc_info=True)
            result['error'] = str(e)
            result['message'] = f"处理过程中发生错误: {e}"

        return result

    def _extract_video(self, source_id: str) -> Dict[str, Any]:
        """
        提取视频信息

        Args:
            source_id: 视频ID或URL

        Returns:
            提取器返回的数据结构

        Raises:
            ExtractionError: 提取失败
        """
        logger.info(f"  正在提取: {source_id}")

        try:
            extractor_data = self.extractor.extract(source_id)
            logger.info(f"  ✅ 提取成功: {extractor_data.get('metadata', {}).get('title', '未知标题')}")
            logger.info(f"     作者: {extractor_data.get('metadata', {}).get('author', '未知')}")
            logger.info(f"     时长: {extractor_data.get('metadata', {}).get('duration', '0')}")

            # 检查字幕
            subtitle = extractor_data.get('subtitle', '')
            if subtitle:
                subtitle_lines = subtitle.split('\n')
                logger.info(f"     字幕: {len(subtitle_lines)} 条")
            else:
                logger.warning(f"     ⚠️  无字幕")

            return extractor_data

        except ExtractionError as e:
            logger.error(f"  ❌ 提取失败: {e}")
            raise
        except Exception as e:
            logger.error(f"  ❌ 提取异常: {e}", exc_info=True)
            raise ExtractionError(f"提取过程异常: {e}", source_id=source_id)

    def _build_content(self, extractor_data: Dict[str, Any]) -> Content:
        """
        从提取器数据构建Content对象

        Args:
            extractor_data: 提取器返回的数据

        Returns:
            Content对象
        """
        metadata = extractor_data.get('metadata', {})

        content = Content(
            source='bilibili',
            source_id=metadata.get('bvid', ''),
            url=metadata.get('url', ''),
            title=metadata.get('title', ''),
            author=metadata.get('author', ''),
            description=extractor_data.get('description', ''),
            duration=self._parse_duration(metadata.get('duration', '0')),
            content_type=ContentType.VIDEO,
            transcript=extractor_data.get('subtitle', ''),
            status=ContentStatus.SCANNED
        )

        logger.info(f"  ✅ Content对象已创建: {content.id}")
        return content

    def _parse_duration(self, duration_str: str) -> Optional[int]:
        """
        解析时长字符串为秒数

        Args:
            duration_str: 时长字符串（如 "10:30"）

        Returns:
            秒数
        """
        try:
            if ':' in duration_str:
                parts = duration_str.split(':')
                if len(parts) == 2:
                    # MM:SS
                    return int(parts[0]) * 60 + int(parts[1])
                elif len(parts) == 3:
                    # HH:MM:SS
                    return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
        except:
            pass
        return None

    def _analyze_content(self, extractor_data: Dict[str, Any]) -> Dict[str, Any]:
        """
        深度分析内容

        Args:
            extractor_data: 提取器返回的数据

        Returns:
            分析结果字典
        """
        try:
            logger.info("  正在进行深度分析...")

            # 使用DeepAnalyzer进行分析
            # DeepAnalyzer接收extractor_data作为输入
            analysis_result = self.analyzer.analyze(extractor_data)

            # 将AnalysisResult转换为字典以便合并
            result_dict = {
                'main_topics': analysis_result.main_topics,
                'key_insights': analysis_result.key_insights,
                'action_items': analysis_result.action_items,
                'summary': analysis_result.summary,
                'timeline': analysis_result.timeline,
                'recommended_tags': analysis_result.recommended_tags,
                'full_analysis': analysis_result.full_analysis
            }

            logger.info(f"  ✅ 分析完成: {len(result_dict['main_topics'])} 个主题")

            return result_dict

        except Exception as e:
            logger.error(f"  ❌ 分析失败: {e}", exc_info=True)
            # 返回默认结果，不中断流程
            return {
                'main_topics': ['未分类'],
                'key_insights': [],
                'action_items': [],
                'summary': '分析失败，无法生成摘要',
                'timeline': [],
                'recommended_tags': ['未分类'],
                'full_analysis': ''
            }

    def _chunk_content(self, extractor_data: Dict[str, Any]) -> list:
        """
        内容分片

        Args:
            extractor_data: 提取器返回的数据

        Returns:
            Chunk对象列表
        """
        try:
            logger.info("  正在进行内容分片...")

            # 使用ContentChunker进行分片
            chunks = self.chunker.chunk_from_extractor(extractor_data)

            logger.info(f"  ✅ 分片完成: {len(chunks)} 段")

            return chunks

        except Exception as e:
            logger.error(f"  ❌ 分片失败: {e}", exc_info=True)
            # 返回空列表，不中断流程
            return []

    def _merge_analysis_to_content(self, content: Content, analysis_result: Dict[str, Any], chunks: list):
        """
        将分析结果和分片信息合并到Content对象

        Args:
            content: Content对象
            analysis_result: 分析结果字典
            chunks: Chunk对象列表
        """
        # 合并基本信息
        content.summary = analysis_result.get('summary', '')
        content.key_points = analysis_result.get('main_topics', [])
        content.tags = analysis_result.get('recommended_tags', [])

        # 合并时间轴
        timeline_data = analysis_result.get('timeline', [])
        if timeline_data:
            content.timeline = [
                TimelineEntry(
                    timestamp=item.get('start_time', ''),
                    title=item.get('title', ''),
                    description=item.get('summary', '')
                )
                for item in timeline_data
            ]
        else:
            # 从chunks生成时间轴
            if chunks:
                content.timeline = [
                    TimelineEntry(
                        timestamp=chunk.time_range,
                        title=f"片段 {chunk.index + 1}",
                        description=f"时长 {int(chunk.duration)}秒"
                    )
                    for chunk in chunks
                ]

        logger.info(f"  ✅ 数据合并完成: 时间轴 {len(content.timeline)} 条")

    def _write_obsidian_note(self, content: Content) -> Optional[str]:
        """
        写入Obsidian笔记

        Args:
            content: Content对象

        Returns:
            文件路径，失败返回None
        """
        try:
            logger.info(f"  正在写入笔记...")

            # 确保输出目录存在
            self.config.vault_path.mkdir(parents=True, exist_ok=True)

            # 调用ObsidianWriter写入
            output_path = self.writer.write(content, self.config)

            if output_path:
                logger.info(f"  ✅ 笔记已写入: {output_path}")
            else:
                logger.warning(f"  ⚠️  笔记写入失败")

            return output_path

        except Exception as e:
            logger.error(f"  ❌ 写入失败: {e}", exc_info=True)
            return None

    def _sync_to_git(self, content: Content):
        """
        同步到Git仓库

        Args:
            content: Content对象
        """
        if not self.git_sync:
            logger.warning("  ⚠️  Git同步器未初始化，跳过同步")
            return

        try:
            logger.info("  正在同步到Git...")

            success = self.git_sync.sync(self.config)

            if success:
                logger.info("  ✅ Git同步成功")
            else:
                logger.warning("  ⚠️  Git同步失败")

        except Exception as e:
            logger.error(f"  ❌ Git同步异常: {e}", exc_info=True)
            # Git同步失败不视为整体失败

    def _format_success_message(self, content: Content, chunks: list) -> str:
        """
        格式化成功消息

        Args:
            content: Content对象
            chunks: Chunk列表

        Returns:
            格式化的消息字符串
        """
        lines = [
            "✅ 处理完成！",
            "",
            f"📺 标题: {content.title}",
            f"👤 作者: {content.author}",
            f"📄 笔记: {content.output_path or 'N/A'}",
        ]

        if content.duration:
            lines.append(f"⏱️  时长: {self._format_duration(content.duration)}")

        if chunks:
            lines.append(f"📊 分片: {len(chunks)} 段")

        if content.tags:
            lines.append(f"🏷️  标签: {', '.join(content.tags[:5])}")

        lines.append("")

        return "\n".join(lines)

    def _format_duration(self, seconds: int) -> str:
        """格式化时长"""
        if not seconds:
            return "N/A"

        hours = seconds // 3600
        minutes = (seconds % 3600) // 60
        secs = seconds % 60

        if hours > 0:
            return f"{hours}小时{minutes}分钟"
        elif minutes > 0:
            return f"{minutes}分钟{secs}秒"
        else:
            return f"{secs}秒"


def main():
    """命令行入口"""
    import argparse

    parser = argparse.ArgumentParser(description='Obsidian KB - 主流程集成')
    parser.add_argument('source_id', help='视频ID（BV号或URL）')
    parser.add_argument('--git', action='store_true', help='启用Git同步')
    parser.add_argument('--config', help='配置文件路径')

    args = parser.parse_args()

    # 加载配置
    config = get_config(args.config) if args.config else get_config()

    # 创建主流程
    pipeline = MainPipeline(config=config, enable_git_sync=args.git)

    # 处理视频
    result = pipeline.process_video(args.source_id)

    # 输出结果
    print(result['message'])

    if not result['success']:
        print(f"\n❌ 失败: {result['error']}")
        sys.exit(1)
    else:
        sys.exit(0)


if __name__ == '__main__':
    main()