feat: Add KB knowledge base series skills

- kb-link-parser: Multi-platform URL parser (Bilibili, YouTube, Zhihu) - kb-bilibili-metadata: Bilibili video/article/up主 metadata extractor - kb-zhihu-metadata: Zhihu question/answer/user metadata extractor - kb-bilibili-content: Bilibili video download and content extraction - kb-ocr-processor: OCR text extraction from video frames - kb-deep-analyzer: AI-powered deep content analysis - kb-workflow: Workflow orchestration (inbox, archive, subscriptions, report) Also add: - ARCHITECTURE_FLOWCHART.md: System architecture diagrams - DETAILED_DESIGN.md: Complete design documentation
2026-02-10 12:18:52 +08:00
parent 2f5a269866
commit 5507b1d986
28 changed files with 7715 additions and 0 deletions
@@ -0,0 +1,489 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Obsidian KB - 主流程集成
+
+整合提取层、处理层、输出层，实现完整的数据流：
+提取(BilibiliExtractor) → 处理(DeepAnalyzer/ContentChunker) → 输出(ObsidianWriter) → 同步(GitSync)
+"""
+
+import os
+import sys
+import logging
+from typing import Dict, Any, Optional
+from datetime import datetime
+from pathlib import Path
+
+# 添加项目路径
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from src.extractors.bilibili_extractor import BilibiliExtractor, ExtractionError
+from src.deep_analyzer import DeepAnalyzer
+from src.content_chunker import ContentChunker, Chunk
+from src.outputs.obsidian_writer import ObsidianWriter
+from src.outputs.git_sync import GitSync
+from src.models import Content, ContentType, ContentStatus, TimelineEntry
+from src.config import get_config
+
+
+# 配置日志
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler('obsidian_kb.log', encoding='utf-8')
+    ]
+)
+logger = logging.getLogger('ObsidianKB')
+
+
+class MainPipeline:
+    """主流程集成器"""
+
+    def __init__(self, config=None, enable_git_sync: bool = False):
+        """
+        初始化主流程
+
+        Args:
+            config: 配置对象
+            enable_git_sync: 是否启用Git同步
+        """
+        self.config = config or get_config()
+        self.enable_git_sync = enable_git_sync
+
+        logger.info("初始化主流程...")
+        logger.info(f"Obsidian vault 路径: {self.config.vault_path}")
+        logger.info(f"Git 同步: {'启用' if enable_git_sync else '禁用'}")
+
+        # 初始化各层组件
+        self.extractor = BilibiliExtractor(config=self.config)
+        self.analyzer = DeepAnalyzer(ai_client=None)  # 暂时不传入ai_client
+        self.chunker = ContentChunker()
+        self.writer = ObsidianWriter(config=self.config)
+
+        # 初始化Git同步器（仅在需要时）
+        self.git_sync = None
+        if enable_git_sync and self._is_git_enabled():
+            self.git_sync = GitSync(config=self.config)
+            logger.info("Git 同步器已初始化")
+
+    def _is_git_enabled(self) -> bool:
+        """检查Git同步是否在配置中启用"""
+        return self.config.get('git.enabled', False)
+
+    def process_video(self, source_id: str) -> Dict[str, Any]:
+        """
+        处理单个视频的完整流程
+
+        Args:
+            source_id: 视频ID（支持BV号、URL）
+
+        Returns:
+            处理结果字典，包含：
+                - success: bool - 是否成功
+                - content: Content对象 - 处理的内容
+                - message: str - 处理消息
+                - error: str - 错误信息（如果失败）
+        """
+        result = {
+            'success': False,
+            'content': None,
+            'message': '',
+            'error': ''
+        }
+
+        try:
+            logger.info("=" * 80)
+            logger.info(f"开始处理视频: {source_id}")
+            logger.info("=" * 80)
+
+            # Step 1: 提取层 - 提取视频信息
+            logger.info("\n[Step 1/5] 提取视频信息...")
+            extractor_data = self._extract_video(source_id)
+
+            # Step 2: 构建Content对象
+            logger.info("\n[Step 2/5] 构建内容对象...")
+            content = self._build_content(extractor_data)
+
+            # Step 3: 处理层 - 深度分析
+            logger.info("\n[Step 3/5] 深度分析...")
+            analysis_result = self._analyze_content(extractor_data)
+
+            # Step 4: 处理层 - 内容分片
+            logger.info("\n[Step 4/5] 内容分片...")
+            chunks = self._chunk_content(extractor_data)
+
+            # 合并分析结果和分片信息到content
+            self._merge_analysis_to_content(content, analysis_result, chunks)
+
+            # Step 5: 输出层 - 写入Obsidian
+            logger.info("\n[Step 5/5] 写入Obsidian...")
+            output_path = self._write_obsidian_note(content)
+
+            # Step 6: 同步层 - Git同步（可选）
+            if self.enable_git_sync and self.git_sync:
+                logger.info("\n[Step 6/6] Git同步...")
+                self._sync_to_git(content)
+
+            # 更新状态
+            content.update_status(ContentStatus.GENERATED)
+            content.output_path = output_path
+
+            # 构建成功结果
+            result['success'] = True
+            result['content'] = content
+            result['message'] = self._format_success_message(content, chunks)
+
+            logger.info("\n" + "=" * 80)
+            logger.info(f"✅ 处理完成: {content.title}")
+            logger.info(f"   📄 笔记路径: {output_path}")
+            logger.info(f"   📊 字幕条数: {len(chunks) if chunks else 0}")
+            logger.info("=" * 80 + "\n")
+
+        except ExtractionError as e:
+            logger.error(f"❌ 提取失败: {e}")
+            result['error'] = f"提取失败: {e}"
+            result['message'] = f"提取视频信息失败: {e}"
+
+        except Exception as e:
+            logger.error(f"❌ 处理失败: {e}", exc_info=True)
+            result['error'] = str(e)
+            result['message'] = f"处理过程中发生错误: {e}"
+
+        return result
+
+    def _extract_video(self, source_id: str) -> Dict[str, Any]:
+        """
+        提取视频信息
+
+        Args:
+            source_id: 视频ID或URL
+
+        Returns:
+            提取器返回的数据结构
+
+        Raises:
+            ExtractionError: 提取失败
+        """
+        logger.info(f"  正在提取: {source_id}")
+
+        try:
+            extractor_data = self.extractor.extract(source_id)
+            logger.info(f"  ✅ 提取成功: {extractor_data.get('metadata', {}).get('title', '未知标题')}")
+            logger.info(f"     作者: {extractor_data.get('metadata', {}).get('author', '未知')}")
+            logger.info(f"     时长: {extractor_data.get('metadata', {}).get('duration', '0')}")
+
+            # 检查字幕
+            subtitle = extractor_data.get('subtitle', '')
+            if subtitle:
+                subtitle_lines = subtitle.split('\n')
+                logger.info(f"     字幕: {len(subtitle_lines)} 条")
+            else:
+                logger.warning(f"     ⚠️  无字幕")
+
+            return extractor_data
+
+        except ExtractionError as e:
+            logger.error(f"  ❌ 提取失败: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"  ❌ 提取异常: {e}", exc_info=True)
+            raise ExtractionError(f"提取过程异常: {e}", source_id=source_id)
+
+    def _build_content(self, extractor_data: Dict[str, Any]) -> Content:
+        """
+        从提取器数据构建Content对象
+
+        Args:
+            extractor_data: 提取器返回的数据
+
+        Returns:
+            Content对象
+        """
+        metadata = extractor_data.get('metadata', {})
+
+        content = Content(
+            source='bilibili',
+            source_id=metadata.get('bvid', ''),
+            url=metadata.get('url', ''),
+            title=metadata.get('title', ''),
+            author=metadata.get('author', ''),
+            description=extractor_data.get('description', ''),
+            duration=self._parse_duration(metadata.get('duration', '0')),
+            content_type=ContentType.VIDEO,
+            transcript=extractor_data.get('subtitle', ''),
+            status=ContentStatus.SCANNED
+        )
+
+        logger.info(f"  ✅ Content对象已创建: {content.id}")
+        return content
+
+    def _parse_duration(self, duration_str: str) -> Optional[int]:
+        """
+        解析时长字符串为秒数
+
+        Args:
+            duration_str: 时长字符串（如 "10:30"）
+
+        Returns:
+            秒数
+        """
+        try:
+            if ':' in duration_str:
+                parts = duration_str.split(':')
+                if len(parts) == 2:
+                    # MM:SS
+                    return int(parts[0]) * 60 + int(parts[1])
+                elif len(parts) == 3:
+                    # HH:MM:SS
+                    return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
+        except:
+            pass
+        return None
+
+    def _analyze_content(self, extractor_data: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        深度分析内容
+
+        Args:
+            extractor_data: 提取器返回的数据
+
+        Returns:
+            分析结果字典
+        """
+        try:
+            logger.info("  正在进行深度分析...")
+
+            # 使用DeepAnalyzer进行分析
+            # DeepAnalyzer接收extractor_data作为输入
+            analysis_result = self.analyzer.analyze(extractor_data)
+
+            # 将AnalysisResult转换为字典以便合并
+            result_dict = {
+                'main_topics': analysis_result.main_topics,
+                'key_insights': analysis_result.key_insights,
+                'action_items': analysis_result.action_items,
+                'summary': analysis_result.summary,
+                'timeline': analysis_result.timeline,
+                'recommended_tags': analysis_result.recommended_tags,
+                'full_analysis': analysis_result.full_analysis
+            }
+
+            logger.info(f"  ✅ 分析完成: {len(result_dict['main_topics'])} 个主题")
+
+            return result_dict
+
+        except Exception as e:
+            logger.error(f"  ❌ 分析失败: {e}", exc_info=True)
+            # 返回默认结果，不中断流程
+            return {
+                'main_topics': ['未分类'],
+                'key_insights': [],
+                'action_items': [],
+                'summary': '分析失败，无法生成摘要',
+                'timeline': [],
+                'recommended_tags': ['未分类'],
+                'full_analysis': ''
+            }
+
+    def _chunk_content(self, extractor_data: Dict[str, Any]) -> list:
+        """
+        内容分片
+
+        Args:
+            extractor_data: 提取器返回的数据
+
+        Returns:
+            Chunk对象列表
+        """
+        try:
+            logger.info("  正在进行内容分片...")
+
+            # 使用ContentChunker进行分片
+            chunks = self.chunker.chunk_from_extractor(extractor_data)
+
+            logger.info(f"  ✅ 分片完成: {len(chunks)} 段")
+
+            return chunks
+
+        except Exception as e:
+            logger.error(f"  ❌ 分片失败: {e}", exc_info=True)
+            # 返回空列表，不中断流程
+            return []
+
+    def _merge_analysis_to_content(self, content: Content, analysis_result: Dict[str, Any], chunks: list):
+        """
+        将分析结果和分片信息合并到Content对象
+
+        Args:
+            content: Content对象
+            analysis_result: 分析结果字典
+            chunks: Chunk对象列表
+        """
+        # 合并基本信息
+        content.summary = analysis_result.get('summary', '')
+        content.key_points = analysis_result.get('main_topics', [])
+        content.tags = analysis_result.get('recommended_tags', [])
+
+        # 合并时间轴
+        timeline_data = analysis_result.get('timeline', [])
+        if timeline_data:
+            content.timeline = [
+                TimelineEntry(
+                    timestamp=item.get('start_time', ''),
+                    title=item.get('title', ''),
+                    description=item.get('summary', '')
+                )
+                for item in timeline_data
+            ]
+        else:
+            # 从chunks生成时间轴
+            if chunks:
+                content.timeline = [
+                    TimelineEntry(
+                        timestamp=chunk.time_range,
+                        title=f"片段 {chunk.index + 1}",
+                        description=f"时长 {int(chunk.duration)}秒"
+                    )
+                    for chunk in chunks
+                ]
+
+        logger.info(f"  ✅ 数据合并完成: 时间轴 {len(content.timeline)} 条")
+
+    def _write_obsidian_note(self, content: Content) -> Optional[str]:
+        """
+        写入Obsidian笔记
+
+        Args:
+            content: Content对象
+
+        Returns:
+            文件路径，失败返回None
+        """
+        try:
+            logger.info(f"  正在写入笔记...")
+
+            # 确保输出目录存在
+            self.config.vault_path.mkdir(parents=True, exist_ok=True)
+
+            # 调用ObsidianWriter写入
+            output_path = self.writer.write(content, self.config)
+
+            if output_path:
+                logger.info(f"  ✅ 笔记已写入: {output_path}")
+            else:
+                logger.warning(f"  ⚠️  笔记写入失败")
+
+            return output_path
+
+        except Exception as e:
+            logger.error(f"  ❌ 写入失败: {e}", exc_info=True)
+            return None
+
+    def _sync_to_git(self, content: Content):
+        """
+        同步到Git仓库
+
+        Args:
+            content: Content对象
+        """
+        if not self.git_sync:
+            logger.warning("  ⚠️  Git同步器未初始化，跳过同步")
+            return
+
+        try:
+            logger.info("  正在同步到Git...")
+
+            success = self.git_sync.sync(self.config)
+
+            if success:
+                logger.info("  ✅ Git同步成功")
+            else:
+                logger.warning("  ⚠️  Git同步失败")
+
+        except Exception as e:
+            logger.error(f"  ❌ Git同步异常: {e}", exc_info=True)
+            # Git同步失败不视为整体失败
+
+    def _format_success_message(self, content: Content, chunks: list) -> str:
+        """
+        格式化成功消息
+
+        Args:
+            content: Content对象
+            chunks: Chunk列表
+
+        Returns:
+            格式化的消息字符串
+        """
+        lines = [
+            "✅ 处理完成！",
+            "",
+            f"📺 标题: {content.title}",
+            f"👤 作者: {content.author}",
+            f"📄 笔记: {content.output_path or 'N/A'}",
+        ]
+
+        if content.duration:
+            lines.append(f"⏱️  时长: {self._format_duration(content.duration)}")
+
+        if chunks:
+            lines.append(f"📊 分片: {len(chunks)} 段")
+
+        if content.tags:
+            lines.append(f"🏷️  标签: {', '.join(content.tags[:5])}")
+
+        lines.append("")
+
+        return "\n".join(lines)
+
+    def _format_duration(self, seconds: int) -> str:
+        """格式化时长"""
+        if not seconds:
+            return "N/A"
+
+        hours = seconds // 3600
+        minutes = (seconds % 3600) // 60
+        secs = seconds % 60
+
+        if hours > 0:
+            return f"{hours}小时{minutes}分钟"
+        elif minutes > 0:
+            return f"{minutes}分钟{secs}秒"
+        else:
+            return f"{secs}秒"
+
+
+def main():
+    """命令行入口"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Obsidian KB - 主流程集成')
+    parser.add_argument('source_id', help='视频ID（BV号或URL）')
+    parser.add_argument('--git', action='store_true', help='启用Git同步')
+    parser.add_argument('--config', help='配置文件路径')
+
+    args = parser.parse_args()
+
+    # 加载配置
+    config = get_config(args.config) if args.config else get_config()
+
+    # 创建主流程
+    pipeline = MainPipeline(config=config, enable_git_sync=args.git)
+
+    # 处理视频
+    result = pipeline.process_video(args.source_id)
+
+    # 输出结果
+    print(result['message'])
+
+    if not result['success']:
+        print(f"\n❌ 失败: {result['error']}")
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()