From 173d7379cb33746712dd58943c5fc6e23f76b37f Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Tue, 10 Feb 2026 17:55:25 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E8=87=AA=E7=84=B6?= =?UTF-8?q?=E8=AF=AD=E8=A8=80=E8=A7=A6=E5=8F=91=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 支持'保存到知识库'/'订阅到知识库'触发词 - 自动检测链接并处理 - 更新SKILL.md文档 --- skills/bilibili-analyzer/SKILL.md | 562 --------- skills/bilibili-analyzer/analyze_batch.sh | 37 - .../bilibili-analyzer/examples/basic-usage.md | 84 -- skills/bilibili-analyzer/references/faq.md | 86 -- .../references/installation.md | 76 -- skills/bilibili-analyzer/scripts/prepare.cs | 631 ---------- skills/kb-workflow/SKILL.md | 21 + skills/kb-workflow/kb_handler.py | 132 +- skills/obsidian-kb/.disable_onednn.sh | 30 - skills/obsidian-kb/.gitignore | 39 - skills/obsidian-kb/=0.11.6 | 11 - skills/obsidian-kb/=4.0.0 | 42 - skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md | 168 --- skills/obsidian-kb/CHECKPOINT_6.md | 353 ------ skills/obsidian-kb/DETAILED_DESIGN.md | 1108 ----------------- skills/obsidian-kb/QUICKSTART.md | 163 --- skills/obsidian-kb/README.md | 318 ----- skills/obsidian-kb/SKILL.md | 667 ---------- skills/obsidian-kb/TODO.md | 91 -- skills/obsidian-kb/checkpoint7_test_report.py | 450 ------- skills/obsidian-kb/config.yaml | 109 -- skills/obsidian-kb/example_usage.py | 197 --- skills/obsidian-kb/handler.py | 427 ------- skills/obsidian-kb/main_pipeline.py | 489 -------- skills/obsidian-kb/requirements.txt | 32 - skills/obsidian-kb/shortlink_resolver.py | 355 ------ skills/obsidian-kb/src/__init__.py | 36 - skills/obsidian-kb/src/ai_client.py | 514 -------- skills/obsidian-kb/src/cli.py | 569 --------- skills/obsidian-kb/src/config.py | 353 ------ skills/obsidian-kb/src/content_chunker.py | 486 -------- skills/obsidian-kb/src/deep_analyzer.py | 438 ------- skills/obsidian-kb/src/extractors/__init__.py | 18 - skills/obsidian-kb/src/extractors/base.py | 83 -- .../src/extractors/bilibili_extractor.py | 750 ----------- .../src/extractors/easyocr_processor.py | 355 ------ .../src/extractors/hard_subtitle_extractor.py | 435 ------- .../src/extractors/ocr_processor.py | 309 ----- .../src/extractors/subtitle_detector.py | 383 ------ .../src/extractors/video_stream_extractor.py | 517 -------- skills/obsidian-kb/src/git_sync.py | 195 --- skills/obsidian-kb/src/light_scanner.py | 228 ---- skills/obsidian-kb/src/llm.py | 60 - skills/obsidian-kb/src/models.py | 213 ---- skills/obsidian-kb/src/outputs/__init__.py | 5 - skills/obsidian-kb/src/outputs/git_sync.py | 275 ---- .../src/outputs/obsidian_writer.py | 385 ------ skills/obsidian-kb/src/pipeline.py | 273 ---- skills/obsidian-kb/src/sources/__init__.py | 31 - skills/obsidian-kb/src/sources/base.py | 58 - skills/obsidian-kb/src/sources/bilibili.py | 466 ------- skills/obsidian-kb/src/sources/local.py | 57 - .../src/sources/markdown_converter.py | 17 - skills/obsidian-kb/src/sources/youtube.py | 53 - skills/obsidian-kb/src/sources/zhihu.py | 91 -- .../src/sources/zhihu_drission_extractor.py | 514 -------- .../src/sources/zhihu_playwright_extractor.py | 385 ------ skills/obsidian-kb/src/storage.py | 168 --- skills/obsidian-kb/src/template.py | 71 -- skills/obsidian-kb/src/template_renderer.py | 307 ----- skills/obsidian-kb/src/templates/article.md | 33 - .../obsidian-kb/src/templates/light_video.md | 20 - .../obsidian-kb/src/templates/topic_index.md | 16 - .../src/templates/tutorial_video.md | 35 - skills/obsidian-kb/tests/__init__.py | 1 - skills/obsidian-kb/tests/test_bilibili.py | 180 --- skills/obsidian-kb/tests/test_chunking.py | 149 --- skills/obsidian-kb/tests/test_config.py | 144 --- skills/obsidian-kb/tests/test_models.py | 187 --- skills/obsidian-kb/tests/test_pipeline.py | 331 ----- skills/obsidian-kb/tests/test_storage.py | 146 --- .../tests/test_subtitle_detector.py | 296 ----- .../tests/test_template_renderer.py | 257 ---- .../tests/test_zhihu_playwright.py | 736 ----------- skills/obsidian-kb/verify_cookie_config.py | 130 -- skills/obsidian-kb/verify_integration.py | 233 ---- 76 files changed, 149 insertions(+), 18521 deletions(-) delete mode 100644 skills/bilibili-analyzer/SKILL.md delete mode 100644 skills/bilibili-analyzer/analyze_batch.sh delete mode 100644 skills/bilibili-analyzer/examples/basic-usage.md delete mode 100644 skills/bilibili-analyzer/references/faq.md delete mode 100644 skills/bilibili-analyzer/references/installation.md delete mode 100644 skills/bilibili-analyzer/scripts/prepare.cs delete mode 100755 skills/obsidian-kb/.disable_onednn.sh delete mode 100644 skills/obsidian-kb/.gitignore delete mode 100644 skills/obsidian-kb/=0.11.6 delete mode 100644 skills/obsidian-kb/=4.0.0 delete mode 100644 skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md delete mode 100644 skills/obsidian-kb/CHECKPOINT_6.md delete mode 100644 skills/obsidian-kb/DETAILED_DESIGN.md delete mode 100644 skills/obsidian-kb/QUICKSTART.md delete mode 100644 skills/obsidian-kb/README.md delete mode 100644 skills/obsidian-kb/SKILL.md delete mode 100644 skills/obsidian-kb/TODO.md delete mode 100644 skills/obsidian-kb/checkpoint7_test_report.py delete mode 100644 skills/obsidian-kb/config.yaml delete mode 100644 skills/obsidian-kb/example_usage.py delete mode 100755 skills/obsidian-kb/handler.py delete mode 100755 skills/obsidian-kb/main_pipeline.py delete mode 100644 skills/obsidian-kb/requirements.txt delete mode 100755 skills/obsidian-kb/shortlink_resolver.py delete mode 100644 skills/obsidian-kb/src/__init__.py delete mode 100644 skills/obsidian-kb/src/ai_client.py delete mode 100644 skills/obsidian-kb/src/cli.py delete mode 100644 skills/obsidian-kb/src/config.py delete mode 100644 skills/obsidian-kb/src/content_chunker.py delete mode 100644 skills/obsidian-kb/src/deep_analyzer.py delete mode 100644 skills/obsidian-kb/src/extractors/__init__.py delete mode 100644 skills/obsidian-kb/src/extractors/base.py delete mode 100644 skills/obsidian-kb/src/extractors/bilibili_extractor.py delete mode 100644 skills/obsidian-kb/src/extractors/easyocr_processor.py delete mode 100644 skills/obsidian-kb/src/extractors/hard_subtitle_extractor.py delete mode 100644 skills/obsidian-kb/src/extractors/ocr_processor.py delete mode 100644 skills/obsidian-kb/src/extractors/subtitle_detector.py delete mode 100644 skills/obsidian-kb/src/extractors/video_stream_extractor.py delete mode 100644 skills/obsidian-kb/src/git_sync.py delete mode 100644 skills/obsidian-kb/src/light_scanner.py delete mode 100644 skills/obsidian-kb/src/llm.py delete mode 100644 skills/obsidian-kb/src/models.py delete mode 100644 skills/obsidian-kb/src/outputs/__init__.py delete mode 100644 skills/obsidian-kb/src/outputs/git_sync.py delete mode 100644 skills/obsidian-kb/src/outputs/obsidian_writer.py delete mode 100644 skills/obsidian-kb/src/pipeline.py delete mode 100644 skills/obsidian-kb/src/sources/__init__.py delete mode 100644 skills/obsidian-kb/src/sources/base.py delete mode 100644 skills/obsidian-kb/src/sources/bilibili.py delete mode 100644 skills/obsidian-kb/src/sources/local.py delete mode 100644 skills/obsidian-kb/src/sources/markdown_converter.py delete mode 100644 skills/obsidian-kb/src/sources/youtube.py delete mode 100644 skills/obsidian-kb/src/sources/zhihu.py delete mode 100644 skills/obsidian-kb/src/sources/zhihu_drission_extractor.py delete mode 100644 skills/obsidian-kb/src/sources/zhihu_playwright_extractor.py delete mode 100644 skills/obsidian-kb/src/storage.py delete mode 100644 skills/obsidian-kb/src/template.py delete mode 100644 skills/obsidian-kb/src/template_renderer.py delete mode 100644 skills/obsidian-kb/src/templates/article.md delete mode 100644 skills/obsidian-kb/src/templates/light_video.md delete mode 100644 skills/obsidian-kb/src/templates/topic_index.md delete mode 100644 skills/obsidian-kb/src/templates/tutorial_video.md delete mode 100644 skills/obsidian-kb/tests/__init__.py delete mode 100644 skills/obsidian-kb/tests/test_bilibili.py delete mode 100644 skills/obsidian-kb/tests/test_chunking.py delete mode 100644 skills/obsidian-kb/tests/test_config.py delete mode 100644 skills/obsidian-kb/tests/test_models.py delete mode 100644 skills/obsidian-kb/tests/test_pipeline.py delete mode 100644 skills/obsidian-kb/tests/test_storage.py delete mode 100644 skills/obsidian-kb/tests/test_subtitle_detector.py delete mode 100644 skills/obsidian-kb/tests/test_template_renderer.py delete mode 100644 skills/obsidian-kb/tests/test_zhihu_playwright.py delete mode 100644 skills/obsidian-kb/verify_cookie_config.py delete mode 100644 skills/obsidian-kb/verify_integration.py diff --git a/skills/bilibili-analyzer/SKILL.md b/skills/bilibili-analyzer/SKILL.md deleted file mode 100644 index 1f76f9a..0000000 --- a/skills/bilibili-analyzer/SKILL.md +++ /dev/null @@ -1,562 +0,0 @@ ---- -name: bilibili-analyzer -description: 自动分析B站视频内容，下载视频并拆解成帧图片，使用OCR识别文字并生成详细的专题文档或实操教程。 -metadata: - short-description: B站视频分析工具（使用RapidOCR） -source: - - name: FFmpeg - repository: https://github.com/FFmpeg/FFmpeg - documentation: https://ffmpeg.org/documentation.html - license: LGPL/GPL - - name: Bilibili API - documentation: https://github.com/SocialSisterYi/bilibili-API-collect - - name: RapidOCR - repository: https://github.com/RapidAI/RapidOCR - documentation: https://rapidai.github.io/RapidOCRDocs/ - license: Apache-2.0 ---- - -# Bilibili Video Analyzer - -## Description - -B站视频内容分析工具。提供视频URL后，自动下载视频、拆解成帧图片，然后使用AI分析内容，最终生成**高质量的专题文档或实操教程**。 - -**核心特点**: -- 不是简单的时间线记录，而是**重新组织整理**成一篇完整的文档 -- 实操类视频 → 生成**可直接使用的操作教程** -- 知识类视频 → 生成**结构化的专题文档** -- 报告中插入关键截图，使用 `![描述](./images/frame_xxxx.jpg)` 格式 - -## Source & Documentation - -| 工具 | 用途 | 文档 | -|------|------|------| -| **FFmpeg** | 视频拆帧 | [官网](https://ffmpeg.org/) / [文档](https://ffmpeg.org/ffmpeg.html) | -| **Bilibili API** | 视频下载 | [API文档](https://github.com/SocialSisterYi/bilibili-API-collect) | -| **RapidOCR** | 中文文字识别 | [GitHub](https://github.com/RapidAI/RapidOCR) | - -## Installation - -### 1. 安装 .NET 10 SDK - -脚本使用 .NET 10 单文件执行功能，需要安装 .NET 10 SDK。 - -下载地址: https://dotnet.microsoft.com/download/dotnet/10.0 - -验证安装: -```bash -dotnet --version -``` - -### 2. 安装 FFmpeg - -**Windows:** -```powershell -# Chocolatey -choco install ffmpeg - -# 或 Scoop -scoop install ffmpeg - -# 或手动下载: https://ffmpeg.org/download.html -``` - -**macOS:** -```bash -brew install ffmpeg -``` - -**Linux:** -```bash -# Ubuntu/Debian -sudo apt install ffmpeg - -# CentOS/RHEL -sudo yum install ffmpeg -``` - -验证安装: -```bash -ffmpeg -version -``` - -### 3. 安装 RapidOCR（用于文字识别） - -**使用 Python 虚拟环境安装：** - -```bash -# 创建虚拟环境 -python3 -m venv .venv - -# 安装 RapidOCR -.venv/bin/pip install rapidocr_onnxruntime - -# 验证安装 -.venv/bin/python -c "from rapidocr_onnxruntime import RapidOCR; print('RapidOCR installed successfully')" -``` - -**RapidOCR 说明：** -- 专门针对中文优化，识别率高 -- 轻量级，支持 CPU/GPU 推理 -- 开源免费，无需 API 密钥 - -## Trigger - -- `/bilibili-analyzer` 命令 -- 用户请求分析B站视频 -- 用户提供B站视频链接并要求分析 - -## Provided Script - -本 skill 提供了 `scripts/prepare.cs` 脚本用于下载视频和提取帧图片。 - -**脚本位置**: `skills/tools/bilibili-analyzer/scripts/prepare.cs` - -**运行方式**: 使用 .NET 10 单文件执行功能 - -### 使用方法 - -```bash -# 基本用法（默认裁剪底部1/4，用于提取字幕） -dotnet run scripts/prepare.cs "<视频URL>" -o <输出目录> - -# 示例 -dotnet run scripts/prepare.cs "https://www.bilibili.com/video/BV1xx411c7mD" -o ./output - -# 长视频（降低帧率） -dotnet run scripts/prepare.cs "https://www.bilibili.com/video/BV1xx411c7mD" -o ./output --fps 0.5 - -# 提取全帧（不裁剪） -dotnet run scripts/prepare.cs "https://www.bilibili.com/video/BV1xx411c7mD" -o ./output --no-crop - -# 调整裁剪比例（底部1/2） -dotnet run scripts/prepare.cs "https://www.bilibili.com/video/BV1xx411c7mD" -o ./output --crop-ratio 2 -``` - -### 参数说明 - -| 参数 | 说明 | 默认值 | -|------|------|--------| -| `url` | B站视频URL（必需） | - | -| `-o, --output` | 输出目录 | 当前目录 | -| `--fps` | 每秒提取帧数 | 1.0 | -| `--similarity` | 相似度阈值（0-1），超过此值的相邻帧会被去重 | 0.80 | -| `--no-dedup` | 禁用相似帧去重 | false | -| `--crop-ratio` | 裁剪比例（提取底部1/N） | 4（底部1/4） | -| `--no-crop` | 禁用裁剪，提取完整帧 | false | -| `--video-only` | 只下载视频，不提取帧 | false | -| `--frames-only` | 只提取帧（需已有video.mp4） | false | - -### 相似帧去重 - -脚本会自动对**相邻帧**进行相似度检测，去除相似度超过阈值（默认80%）的重复帧： - -- 使用 ffmpeg 的 SSIM/PSNR 算法计算相似度 -- **只比较相邻帧**，不会跨帧比较 -- 去重后自动重新编号（frame_0001.jpg, frame_0002.jpg, ...） -- 可通过 `--similarity 0.85` 调整阈值 -- 可通过 `--no-dedup` 禁用去重 - -### 输出结构 - -``` -<输出目录>/ -├── video.mp4 # 下载的视频文件 -└── images/ # 帧图片目录 - ├── frame_0001.jpg - ├── frame_0002.jpg - ├── frame_0003.jpg - └── ... -``` - -## Workflow (Prompt) - -你是一个视频内容分析助手。当用户提供B站视频链接时，按以下步骤执行： - -### Step 1: 下载视频并拆帧 - -使用提供的脚本下载视频并拆解成帧图片： - -```bash -dotnet run skills/tools/bilibili-analyzer/scripts/prepare.cs "<视频URL>" -o <输出目录> -``` - -**注意事项**: -- 短视频（<10分钟）: 使用默认 `--fps 1` -- 中等视频（10-30分钟）: 使用 `--fps 0.5` -- 长视频（>30分钟）: 使用 `--fps 0.2` - -### Step 2: 分析帧图片 - -使用 **RapidOCR** 工具批量识别 `images/` 目录中的图片中的文字内容。 - -#### 2.1 创建 OCR 分析脚本 - -在输出目录创建 `analyze_frames.py` 脚本： - -```python -#!/usr/bin/env python3 -"""批量使用 RapidOCR 分析帧图片""" - -from rapidocr_onnxruntime import RapidOCR -import os -import json -from pathlib import Path - -# 初始化 OCR -print("🔧 初始化 RapidOCR...") -ocr = RapidOCR() - -# 图片目录 -images_dir = "images" - -# 收集所有图片文件 -image_files = sorted(Path(images_dir).glob("*.jpg")) - -print(f"\n📸 找到 {len(image_files)} 张图片\n") - -# 分析结果 -analysis_results = {} - -for idx, img_path in enumerate(image_files, 1): - frame_name = img_path.name - print(f"[{idx}/{len(image_files)}] 分析 {frame_name}...") - - # 执行 OCR - result, _ = ocr(str(img_path)) - - # 提取文字 - if result: - texts = [text for _, text, _ in result] - analysis_results[frame_name] = { - "text_count": len(texts), - "texts": texts - } - else: - analysis_results[frame_name] = { - "text_count": 0, - "texts": [] - } - -# 保存结果 -output_file = "ocr_results.json" -with open(output_file, 'w', encoding='utf-8') as f: - json.dump(analysis_results, f, ensure_ascii=False, indent=2) - -print(f"\n✅ 分析完成！结果已保存到 {output_file}") -print(f"📊 总计识别了 {sum(r['text_count'] for r in analysis_results.values())} 处文字\n") -``` - -#### 2.2 执行 OCR 分析 - -```bash -cd <输出目录> -.venv/bin/python analyze_frames.py -``` - -#### 2.3 分析策略 - -**根据图片数量调整：** - -| 总图片数 | 建议处理方式 | -|---------|-------------| -| < 100 | 直接处理全部图片 | -| 100-300 | 正常处理（5-10分钟） | -| 300-500 | 处理时间较长（10-20分钟） | -| > 500 | 考虑降低 fps 重新拆帧 | - -**输出说明：** - -生成 `ocr_results.json` 文件，格式如下： - -```json -{ - "frame_0001.jpg": { - "text_count": 4, - "texts": [ - "bilibili", - "IT咖啡馆", - "GitHub", - "GitHub一周热点汇总第102期" - ] - }, - "frame_0002.jpg": { - "text_count": 27, - "texts": [...] - } -} -``` - -#### 2.4 人工审核与补充 - -**OCR 识别后的人工分析步骤：** - -1. **读取 OCR 结果** - ```bash - cat ocr_results.json | less - ``` - -2. **筛选关键帧** - - 选择文字识别较多的帧（通常有重要内容） - - 选择包含代码、配置、界面的帧 - - 选择有独特内容的帧（标题、总结等） - -3. **人工查看关键帧** - ```bash - # 使用图片查看器浏览 - eog images/frame_0101.jpg # Linux - open images/frame_0101.jpg # macOS - ``` - -4. **补充视觉信息**（OCR 无法提供） - - 界面布局和结构 - - 图标、按钮、菜单位置 - - 操作步骤的视觉展示 - - 代码高亮和格式 - - 截图的截图（嵌套界面） - -**分析要点:** -1. 完整转录所有文字和代码内容 -2. 详细描述界面元素和操作步骤 -3. 记录每张图片的关键信息 -4. 标注重要的截图帧号（如 frame_0042.jpg） -5. 区分字幕内容（B站弹幕、字幕）和界面文字 - -### Step 3: 生成文档 - -根据视频类型，将分析结果**重新组织整理**成 `视频分析.md`： - -**判断视频类型**: -- 实操类: 编程教程、软件操作、配置演示等 -- 知识类: 概念讲解、原理分析、经验分享等 - -**【关键】图片与内容必须严格对应**: - -``` -错误示例 ❌: -### 安装 Node.js -首先下载 Node.js... -![截图](./images/frame_0001.jpg) ← 图片可能是其他内容 - -正确示例 ✅: -### 安装 Node.js -首先下载 Node.js... -![frame_0015: Node.js官网下载页面](./images/frame_0015.jpg) ← 图片确实是下载页面 -``` - -**生成文档的正确流程**: - -1. **先整理 OCR 识别结果** - - 读取 `ocr_results.json` 文件 - - 汇总所有帧的文字内容 - - 建立「帧号 → 文字内容」的对应关系 - -2. **按主题重新组织内容**（不是按时间顺序） - - 将相关内容归类到同一章节 - - 确定每个章节需要哪些帧的信息 - - 按逻辑顺序组织，而不是按视频时间线 - -3. **插入图片时必须核对** - - 只插入与当前内容**直接相关**的图片 - - 图片描述要准确反映图片实际内容 - - 使用格式: `![frame_xxxx: 图片实际内容描述](./images/frame_xxxx.jpg)` - -4. **代码必须来自图片中的实际代码** - - 不要自己编造代码 - - 代码块标注来源: `` - -**重要原则**: -1. **图文对应** - 每张图片必须与其上下文内容匹配 -2. **不要时间线流水账** - 重新组织内容，像写文章一样 -3. **结构清晰** - 有章节划分和逻辑顺序 -4. **代码真实** - 只使用图片中出现的代码，不要编造 -5. **独立可读** - 不看视频也能完全理解 - -## Output Format - -### 实操教程类 - -```markdown -# {教程主题} - -## 简介 - -{教程目标} -{前置条件和要求} - -## 环境准备 - -{需要安装的软件} -{配置要求} - -## 操作步骤 - -### 1. {步骤标题} - -{详细说明，内容必须与下方图片对应} - -![frame_xxxx: 图片实际内容的准确描述](./images/frame_xxxx.jpg) - - -```代码块``` - -### 2. {步骤标题} - -{详细说明} - -![frame_xxxx: 准确描述](./images/frame_xxxx.jpg) - -... - -## 完整代码 - - -{汇总所有代码片段，标注来源帧号} - -## 常见问题 - -{可能遇到的问题和解决方案} - -## 总结 - -{核心要点回顾} -{延伸学习建议} -``` - -### 知识文档类 - -```markdown -# {主题} - -## 概述 - -{主题背景介绍} -{为什么重要} - -## {章节1标题} - -{内容，必须与配图对应} - -![frame_xxxx: 图片实际内容描述](./images/frame_xxxx.jpg) - -## {章节2标题} - -{内容} - -![frame_xxxx: 图片实际内容描述](./images/frame_xxxx.jpg) - -## 核心要点 - -- 要点1 -- 要点2 -- 要点3 - -## 延伸阅读 - -{相关资源和建议} -``` - -### 图片插入规范 - -| 规则 | 说明 | -|------|------| -| **帧号必须标注** | `![frame_0015: 描述](./images/frame_0015.jpg)` | -| **描述必须准确** | 描述图片的实际内容，不是期望内容 | -| **内容必须匹配** | 图片上方/下方的文字必须与图片内容相关 | -| **代码标注来源** | `` | -| **不要乱插图** | 没有合适的图就不插，不要强行配图 | - -## API Reference - -### Bilibili API - -脚本使用 Bilibili 官方 API 下载视频： - -``` -# 获取视频信息 -GET https://api.bilibili.com/x/web-interface/view?bvid=BV1xx411c7mD - -# 获取播放地址 -GET https://api.bilibili.com/x/player/playurl?bvid=BV1xx411c7mD&cid={cid}&qn=80&fnval=1 -``` - -API 文档: https://github.com/SocialSisterYi/bilibili-API-collect - -### FFmpeg 拆帧命令 - -```bash -# 每秒1帧 -ffmpeg -i video.mp4 -vf "fps=1" -q:v 2 images/frame_%04d.jpg - -# 每秒0.5帧（每2秒1帧） -ffmpeg -i video.mp4 -vf "fps=0.5" -q:v 2 images/frame_%04d.jpg - -# 指定时间范围 -ffmpeg -i video.mp4 -ss 00:01:00 -to 00:05:00 -vf "fps=1" -q:v 2 images/frame_%04d.jpg - -# 提取关键帧（场景变化） -ffmpeg -i video.mp4 -vf "select='gt(scene,0.3)'" -vsync vfr -q:v 2 images/frame_%04d.jpg -``` - -更多选项: https://ffmpeg.org/ffmpeg.html - -## Examples - -### 示例1: 分析编程教程 - -```bash -# 1. 下载并拆帧 -dotnet run scripts/prepare.cs "https://www.bilibili.com/video/BV1xx411c7mD" -o ./react-tutorial - -# 2. 使用 RapidOCR 分析图片 -cd ./react-tutorial -.venv/bin/python analyze_frames.py - -# 3. 根据 ocr_results.json 生成视频分析.md -``` - -### 示例2: 分析长视频 - -```bash -# 降低帧率，减少图片数量 -dotnet run scripts/prepare.cs "https://www.bilibili.com/video/BV1xx411c7mD" -o ./long-video --fps 0.2 -``` - -### 示例3: 只下载视频 - -```bash -dotnet run scripts/prepare.cs "https://www.bilibili.com/video/BV1xx411c7mD" -o ./output --video-only -``` - -## Quality Checklist - -生成文档前，**逐项检查**以下要求： - -### 内容质量 -- [ ] 内容重新组织，不是时间线流水账 -- [ ] 章节结构清晰，有逻辑顺序 -- [ ] 不看视频也能理解全部内容 -- [ ] 包含环境准备/前置条件说明 -- [ ] 包含总结和核心要点 - -### 图文对应（重要！） -- [ ] 每张图片都标注了帧号: `![frame_xxxx: 描述](...)` -- [ ] 图片描述准确反映图片**实际内容** -- [ ] 图片上下文的文字与图片内容**直接相关** -- [ ] 没有随意插入不相关的图片 -- [ ] 代码块标注了来源帧号: `` - -### 代码质量 -- [ ] 代码来自图片中的实际代码，不是编造的 -- [ ] 代码片段完整，可直接复制使用 -- [ ] 完整代码章节汇总了所有代码并标注来源 - -## Tags - -`bilibili`, `video-analysis`, `ai`, `frame-extraction`, `markdown`, `tutorial`, `ffmpeg`, `dotnet` - -## Compatibility - -- Codex: Yes -- Claude Code: Yes diff --git a/skills/bilibili-analyzer/analyze_batch.sh b/skills/bilibili-analyzer/analyze_batch.sh deleted file mode 100644 index e3cfb56..0000000 --- a/skills/bilibili-analyzer/analyze_batch.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# 分析B站视频帧图片 - -OUTPUT_DIR="$1" -IMAGES_DIR="$2" -START_FRAME="$3" -END_FRAME="$4" -TASK_ID="$5" - -echo "========== Task $TASK_ID: 分析图片 $START_FRAME 到 $END_FRAME ==========" - -result="" - -for i in $(seq -f "%04g" $START_FRAME $END_FRAME); do - img_file="$IMAGES_DIR/temp_$i.jpg" - if [ -f "$img_file" ]; then - # 使用 Python 进行简单分析 - analysis=$(python3 << 'PYEOF' -import json -import sys - -# 模拟图片分析 - 实际应该读取图片内容 -# 由于环境限制，这里记录图片存在性 -img_path = sys.argv[1] if len(sys.argv) > 1 else "" -frame_num = sys.argv[2] if len(sys.argv) > 2 else "" - -print(f"分析 {img_path}") -PYEOF - "$img_file" "$i" 2>/dev/null || echo " temp_$i.jpg: 存在") - - # 记录文件名和简要信息 - result="$result\n- temp_$i.jpg: 帧图片" - fi -done - -echo "$result" -echo "========== Task $TASK_ID 完成 ==========" diff --git a/skills/bilibili-analyzer/examples/basic-usage.md b/skills/bilibili-analyzer/examples/basic-usage.md deleted file mode 100644 index eaa07bc..0000000 --- a/skills/bilibili-analyzer/examples/basic-usage.md +++ /dev/null @@ -1,84 +0,0 @@ -# 基本使用示例 - -## 快速开始 - -```bash -cd skills/tools/bilibili-analyzer/scripts -python main.py "https://www.bilibili.com/video/BV1xx411c7mD" -``` - -## 常用场景 - -### 1. 分析教程视频（提取更多帧） - -```bash -python main.py "https://www.bilibili.com/video/BV1xx411c7mD" \ - -i 15 \ - -m 100 \ - -f text,objects -``` - -### 2. 快速预览长视频 - -```bash -python main.py "https://www.bilibili.com/video/BV1xx411c7mD" \ - -i 120 \ - -m 20 \ - --no-scene-detection -``` - -### 3. 分析人物访谈 - -```bash -python main.py "https://www.bilibili.com/video/BV1xx411c7mD" \ - -i 30 \ - -f faces,text,scene -``` - -### 4. 高并行分析（快速完成） - -```bash -python main.py "https://www.bilibili.com/video/BV1xx411c7mD" \ - -w 8 \ - -m 30 -``` - -### 5. 指定输出目录 - -```bash -python main.py "https://www.bilibili.com/video/BV1xx411c7mD" \ - -o ./my-video-reports -``` - -### 6. 使用短链接 - -```bash -python main.py "https://b23.tv/xxxxx" -``` - -### 7. 详细日志模式（调试用） - -```bash -python main.py "https://www.bilibili.com/video/BV1xx411c7mD" -v -``` - -### 8. 静默模式（只显示错误） - -```bash -python main.py "https://www.bilibili.com/video/BV1xx411c7mD" -q -``` - -## 输出示例 - -分析完成后，输出目录结构： - -``` -./bilibili/视频标题/ -├── report.md # Markdown分析报告 -├── frames/ # 关键帧图片 -│ ├── frame_001_00-00-00.jpg -│ ├── frame_002_00-00-30.jpg -│ └── ... -├── analysis.log # 执行日志 -└── checkpoints/ # 检查点数据 -``` diff --git a/skills/bilibili-analyzer/references/faq.md b/skills/bilibili-analyzer/references/faq.md deleted file mode 100644 index 30c0d8f..0000000 --- a/skills/bilibili-analyzer/references/faq.md +++ /dev/null @@ -1,86 +0,0 @@ -# 常见问题 (FAQ) - -## 基础问题 - -### Q: 为什么需要FFmpeg？ -A: FFmpeg是一个强大的多媒体处理工具，本工具使用它从视频中提取关键帧。没有FFmpeg，帧提取功能将无法工作。 - -### Q: 分析一个视频需要多长时间？ -A: 取决于多个因素： -- 视频长度 -- 帧提取间隔和最大帧数 -- 网络速度（下载阶段） -- AI分析并行度 - -一般来说，10分钟视频（默认设置）约需5-10分钟完成分析。 - -### Q: 支持哪些视频格式？ -A: 支持B站所有可播放的视频格式。系统使用yt-dlp下载，会自动选择最佳质量。 - -## 错误处理 - -### Q: 遇到"URL无效"错误怎么办？ -A: 请确保URL格式正确： -- 标准格式: `https://www.bilibili.com/video/BV1xx411c7mD` -- 短链接: `https://b23.tv/xxxxx` -- 不支持番剧、直播等其他类型链接 - -### Q: 下载失败怎么办？ -A: 可能的原因和解决方案： -1. **网络问题**: 检查网络连接，使用`--retries`增加重试次数 -2. **视频不存在**: 确认视频未被删除或设为私有 -3. **地区限制**: 某些视频可能有地区限制 - -### Q: FFmpeg报错怎么办？ -A: -1. 确认FFmpeg已正确安装: `ffmpeg -version` -2. 确认FFmpeg在系统PATH中 -3. 尝试重新安装FFmpeg - -### Q: AI分析失败怎么办？ -A: 系统会自动重试失败的分析任务。如果仍然失败： -1. 检查Claude Code配置 -2. 查看日志文件了解详细错误 -3. 部分结果会保存到`partial_report.json` - -## 高级用法 - -### Q: 如何只分析视频的特定部分？ -A: 目前不支持指定时间范围，但可以通过调整参数间接实现： -- 增大`--interval`跳过更多内容 -- 减小`--max-frames`限制分析帧数 - -### Q: 如何提高分析速度？ -A: -1. 增加并行worker: `-w 8` -2. 减少帧数: `-m 20` -3. 增大间隔: `-i 60` -4. 禁用场景检测: `--no-scene-detection` - -### Q: 如何获取更详细的分析？ -A: -1. 减小帧间隔: `-i 10` -2. 增加最大帧数: `-m 100` -3. 启用所有分析焦点: `-f text,objects,faces,actions,scene` - -### Q: 分析中断后如何恢复？ -A: 系统会自动保存检查点。目前需要重新运行命令，未来版本将支持从检查点恢复。 - -## 输出相关 - -### Q: 报告中的图片路径是什么格式？ -A: 使用相对路径，格式为`./frames/frame_XXX_HH-MM-SS.jpg`，确保报告和frames目录在同一位置时图片可正常显示。 - -### Q: 日志文件在哪里？ -A: 日志文件`analysis.log`位于输出目录中，与`report.md`同级。 - -## 错误代码 - -| 错误 | 原因 | 解决方案 | -|------|------|----------| -| `URLValidationError` | URL格式不正确 | 使用正确的B站视频URL | -| `MetadataFetchError` | 无法获取视频信息 | 检查视频是否存在/私有 | -| `DownloadError` | 下载失败 | 检查网络，增加重试次数 | -| `FFmpegError` | FFmpeg问题 | 安装/重装FFmpeg | -| `AnalysisError` | AI分析失败 | 检查Claude Code配置 | -| `ReportGenerationError` | 报告生成失败 | 检查输出目录权限 | diff --git a/skills/bilibili-analyzer/references/installation.md b/skills/bilibili-analyzer/references/installation.md deleted file mode 100644 index 43bf631..0000000 --- a/skills/bilibili-analyzer/references/installation.md +++ /dev/null @@ -1,76 +0,0 @@ -# 安装指南 - -## 系统依赖 - -### FFmpeg（必需） - -FFmpeg用于从视频中提取关键帧，是必需依赖。 - -#### Windows - -```powershell -# 使用 Chocolatey -choco install ffmpeg - -# 或使用 Scoop -scoop install ffmpeg - -# 或手动下载 -# 访问 https://ffmpeg.org/download.html 下载并添加到PATH -``` - -#### macOS - -```bash -brew install ffmpeg -``` - -#### Linux (Ubuntu/Debian) - -```bash -sudo apt update -sudo apt install ffmpeg -``` - -#### Linux (CentOS/RHEL) - -```bash -sudo yum install epel-release -sudo yum install ffmpeg -``` - -#### 验证安装 - -```bash -ffmpeg -version -``` - -## Python依赖 - -```bash -# 创建虚拟环境（推荐） -python -m venv venv -source venv/bin/activate # Linux/macOS -.\venv\Scripts\activate # Windows - -# 安装依赖 -pip install requests yt-dlp -``` - -### 依赖说明 - -| 包 | 版本 | 用途 | -|---|------|------| -| `requests` | >=2.28.0 | HTTP请求，获取视频元数据 | -| `yt-dlp` | >=2023.0.0 | 视频下载 | - -### 开发依赖（可选） - -```bash -pip install hypothesis pytest -``` - -| 包 | 版本 | 用途 | -|---|------|------| -| `hypothesis` | >=6.0.0 | 属性测试 | -| `pytest` | >=7.0.0 | 单元测试 | diff --git a/skills/bilibili-analyzer/scripts/prepare.cs b/skills/bilibili-analyzer/scripts/prepare.cs deleted file mode 100644 index 370a977..0000000 --- a/skills/bilibili-analyzer/scripts/prepare.cs +++ /dev/null @@ -1,631 +0,0 @@ -#!/usr/bin/env dotnet run - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Security.Cryptography; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -// Parse arguments (args is already available in top-level statements) -if (args.Length == 0 || args.Contains("-h") || args.Contains("--help")) -{ - PrintHelp(); - return; -} - -var url = args[0]; -var outputDir = GetArgValue(args, "-o", "--output") ?? "."; -var fpsStr = GetArgValue(args, "--fps") ?? "1.0"; -if (!double.TryParse(fpsStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var fps)) -{ - Console.WriteLine($"[ERROR] Invalid fps value: {fpsStr}"); - Environment.Exit(1); -} -var cropRatioStr = GetArgValue(args, "--crop-ratio") ?? "4"; -if (!int.TryParse(cropRatioStr, out var cropRatio) || cropRatio < 2 || cropRatio > 10) -{ - Console.WriteLine($"[ERROR] Invalid crop-ratio value: {cropRatioStr} (must be 2-10)"); - Environment.Exit(1); -} -var noCrop = args.Contains("--no-crop"); -var similarityStr = GetArgValue(args, "--similarity") ?? "0.80"; -if (!double.TryParse(similarityStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var similarityThreshold)) -{ - Console.WriteLine($"[ERROR] Invalid similarity value: {similarityStr}"); - Environment.Exit(1); -} -var videoOnly = args.Contains("--video-only"); -var framesOnly = args.Contains("--frames-only"); -var noDedup = args.Contains("--no-dedup"); - -var videoPath = Path.Combine(outputDir, "video.mp4"); -var imagesDir = Path.Combine(outputDir, "images"); - -// Create output directory -Directory.CreateDirectory(outputDir); - -Console.WriteLine(new string('=', 50)); -Console.WriteLine("Bilibili Video Analyzer - Prepare Script"); -Console.WriteLine(new string('=', 50)); -Console.WriteLine($"URL: {url}"); -Console.WriteLine($"Output: {outputDir}"); -Console.WriteLine($"FPS: {fps}"); -Console.WriteLine($"Similarity Threshold: {similarityThreshold:P0}"); -Console.WriteLine($"Deduplication: {(noDedup ? "Disabled" : "Enabled")}"); -Console.WriteLine($"Crop: {(noCrop ? "Disabled" : $"Bottom 1/{cropRatio}")}"); -Console.WriteLine(new string('=', 50)); - -// Download video -if (!framesOnly) -{ - if (!await DownloadBilibiliVideoAsync(url, videoPath)) - { - Environment.Exit(1); - } -} - -// Extract frames -if (!videoOnly) -{ - if (!File.Exists(videoPath)) - { - Console.WriteLine($"[ERROR] Video file not found: {videoPath}"); - Environment.Exit(1); - } - - if (!await ExtractFramesAsync(videoPath, imagesDir, fps, !noCrop, cropRatio)) - { - Environment.Exit(1); - } - - // Deduplicate similar frames - if (!noDedup) - { - await DeduplicateFramesAsync(imagesDir, similarityThreshold); - } -} - -Console.WriteLine(); -Console.WriteLine(new string('=', 50)); -Console.WriteLine("[OK] Done!"); -Console.WriteLine($" Video: {videoPath}"); -Console.WriteLine($" Images: {imagesDir}/"); -Console.WriteLine(new string('=', 50)); - -// === Functions === - -async Task DownloadBilibiliVideoAsync(string url, string outputPath) -{ - Console.WriteLine($"[INFO] Downloading video: {url}"); - - try - { - // Extract BV ID from URL - var bvid = ExtractBvid(url); - if (string.IsNullOrEmpty(bvid)) - { - Console.WriteLine("[ERROR] Invalid Bilibili URL, cannot extract BV ID"); - return false; - } - Console.WriteLine($"[INFO] BV ID: {bvid}"); - - using var client = CreateHttpClient(); - - // Step 1: Get video info to obtain cid - Console.WriteLine("[INFO] Fetching video info..."); - var infoUrl = $"https://api.bilibili.com/x/web-interface/view?bvid={bvid}"; - var infoJson = await client.GetStringAsync(infoUrl); - using var infoDoc = JsonDocument.Parse(infoJson); - - var code = infoDoc.RootElement.GetProperty("code").GetInt32(); - if (code != 0) - { - var message = infoDoc.RootElement.GetProperty("message").GetString(); - Console.WriteLine($"[ERROR] Failed to get video info: {message}"); - return false; - } - - var data = infoDoc.RootElement.GetProperty("data"); - var title = data.GetProperty("title").GetString(); - var cid = data.GetProperty("cid").GetInt64(); - Console.WriteLine($"[INFO] Title: {title}"); - Console.WriteLine($"[INFO] CID: {cid}"); - - // Step 2: Get playback URL - Console.WriteLine("[INFO] Fetching playback URL..."); - var playUrl = $"https://api.bilibili.com/x/player/playurl?bvid={bvid}&cid={cid}&qn=80&fnval=1"; - var playJson = await client.GetStringAsync(playUrl); - using var playDoc = JsonDocument.Parse(playJson); - - var playCode = playDoc.RootElement.GetProperty("code").GetInt32(); - if (playCode != 0) - { - var message = playDoc.RootElement.GetProperty("message").GetString(); - Console.WriteLine($"[ERROR] Failed to get playback URL: {message}"); - return false; - } - - var playData = playDoc.RootElement.GetProperty("data"); - var durl = playData.GetProperty("durl")[0]; - var videoUrl = durl.GetProperty("url").GetString(); - var size = durl.GetProperty("size").GetInt64(); - - Console.WriteLine($"[INFO] Video size: {size / 1024.0 / 1024.0:F1} MB"); - - // Step 3: Download video - Console.WriteLine("[INFO] Downloading video file..."); - - using var request = new HttpRequestMessage(HttpMethod.Get, videoUrl); - request.Headers.Add("Referer", $"https://www.bilibili.com/video/{bvid}"); - - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); - response.EnsureSuccessStatusCode(); - - var totalBytes = response.Content.Headers.ContentLength ?? size; - if (totalBytes <= 0) totalBytes = size > 0 ? size : 1; // Prevent division by zero - - await using var contentStream = await response.Content.ReadAsStreamAsync(); - await using var fileStream = new FileStream(outputPath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, true); - - var buffer = new byte[81920]; // Larger buffer for better performance - var totalRead = 0L; - var lastProgress = -1; - int bytesRead; - - while ((bytesRead = await contentStream.ReadAsync(buffer)) > 0) - { - await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead)); - totalRead += bytesRead; - - var progress = (int)(totalRead * 100 / totalBytes); - if (progress != lastProgress && (progress % 10 == 0 || progress == 100)) - { - Console.WriteLine($"[INFO] Progress: {progress}%"); - lastProgress = progress; - } - } - - // Ensure 100% is shown - if (lastProgress != 100) - { - Console.WriteLine($"[INFO] Progress: 100%"); - } - - Console.WriteLine($"[OK] Video downloaded: {outputPath}"); - return true; - } - catch (HttpRequestException ex) - { - Console.WriteLine($"[ERROR] Network error: {ex.Message}"); - return false; - } - catch (JsonException ex) - { - Console.WriteLine($"[ERROR] Failed to parse API response: {ex.Message}"); - return false; - } - catch (Exception ex) - { - Console.WriteLine($"[ERROR] Download failed: {ex.Message}"); - return false; - } -} - -string? ExtractBvid(string url) -{ - // Match BV ID from various URL formats - // https://www.bilibili.com/video/BV1xx411c7mD - // https://www.bilibili.com/video/BV1xx411c7mD?p=1 - // https://b23.tv/BV1xx411c7mD - // BV1xx411c7mD - // BV ID is exactly 12 characters: BV + 10 alphanumeric chars - var match = Regex.Match(url, @"BV[a-zA-Z0-9]{10}"); - return match.Success ? match.Value : null; -} - -HttpClient CreateHttpClient() -{ - var handler = new HttpClientHandler - { - AutomaticDecompression = System.Net.DecompressionMethods.GZip | System.Net.DecompressionMethods.Deflate - }; - - var client = new HttpClient(handler); - client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); - client.DefaultRequestHeaders.Add("Referer", "https://www.bilibili.com"); - client.DefaultRequestHeaders.Add("Accept", "application/json, text/plain, */*"); - client.Timeout = TimeSpan.FromMinutes(30); - - return client; -} - -async Task ExtractFramesAsync(string videoPath, string outputDir, double fps, bool useCrop, int cropRatio) -{ - Console.WriteLine($"[INFO] Extracting frames (fps={fps}, crop={(useCrop ? $"bottom 1/{cropRatio}" : "disabled")})"); - - var ffmpeg = FindExecutable("ffmpeg", "ffmpeg.exe"); - if (ffmpeg == null) - { - Console.WriteLine("[ERROR] ffmpeg not found!"); - Console.WriteLine(" Windows: choco install ffmpeg / scoop install ffmpeg"); - Console.WriteLine(" macOS: brew install ffmpeg"); - Console.WriteLine(" Linux: sudo apt install ffmpeg"); - return false; - } - - Directory.CreateDirectory(outputDir); - var outputPattern = Path.Combine(outputDir, "frame_%04d.jpg"); - - try - { - Console.WriteLine($"[INFO] Running ffmpeg: {ffmpeg}"); - - // Build filter string - var filter = $"fps={fps.ToString(CultureInfo.InvariantCulture)}"; - if (useCrop) - { - // Crop bottom portion using FFmpeg's crop filter - // crop=w=iw:h=ih/{cropRatio}:x=0:y=ih-h/ih-{cropRatio} - // This crops the bottom 1/cropRatio of the frame - filter = $"fps={fps.ToString(CultureInfo.InvariantCulture)},crop=iw:ih/{cropRatio}:0:ih-ih/{cropRatio}"; - } - - var psi = new ProcessStartInfo - { - FileName = ffmpeg, - Arguments = $"-i \"{videoPath}\" -vf \"{filter}\" -q:v 2 -y \"{outputPattern}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - using var process = Process.Start(psi); - if (process == null) - { - Console.WriteLine("[ERROR] Failed to start ffmpeg"); - return false; - } - - // Read stderr asynchronously to prevent deadlock - var stderrTask = process.StandardError.ReadToEndAsync(); - await process.WaitForExitAsync(); - var stderr = await stderrTask; - - // ffmpeg returns 0 on success, non-zero on failure - if (process.ExitCode != 0) - { - // Only show last few lines of error - var errorLines = stderr.Split('\n').TakeLast(5); - Console.WriteLine($"[ERROR] Frame extraction failed (exit code {process.ExitCode}):"); - foreach (var line in errorLines) - { - if (!string.IsNullOrWhiteSpace(line)) - Console.WriteLine($" {line.Trim()}"); - } - return false; - } - - var frameCount = Directory.GetFiles(outputDir, "frame_*.jpg").Length; - if (frameCount == 0) - { - Console.WriteLine("[ERROR] No frames extracted. Check if video file is valid."); - return false; - } - - Console.WriteLine($"[OK] Frames extracted: {frameCount} images saved to {outputDir}/"); - return true; - } - catch (Exception ex) - { - Console.WriteLine($"[ERROR] Frame extraction failed: {ex.Message}"); - return false; - } -} - -async Task DeduplicateFramesAsync(string imagesDir, double threshold) -{ - Console.WriteLine($"[INFO] Deduplicating similar frames (threshold: {threshold:P0})..."); - - var files = Directory.GetFiles(imagesDir, "frame_*.jpg") - .OrderBy(f => f) - .ToList(); - - if (files.Count < 2) - { - Console.WriteLine("[INFO] Not enough frames to deduplicate"); - return; - } - - var toDelete = new List(); - var ffmpeg = FindExecutable("ffmpeg", "ffmpeg.exe"); - - if (ffmpeg == null) - { - Console.WriteLine("[WARN] ffmpeg not found, skipping deduplication"); - return; - } - - // Compare consecutive frames only (not cross-frame comparison) - for (int i = 0; i < files.Count - 1; i++) - { - var current = files[i]; - var next = files[i + 1]; - - // Skip if current frame is already marked for deletion - if (toDelete.Contains(current)) - continue; - - var similarity = await CalculateFrameSimilarityAsync(ffmpeg, current, next); - - if (similarity >= threshold) - { - // Keep the first frame, mark the next one for deletion - toDelete.Add(next); - } - } - - // Delete similar frames - foreach (var file in toDelete) - { - try - { - File.Delete(file); - } - catch { } - } - - // Renumber remaining frames - var remainingFiles = Directory.GetFiles(imagesDir, "frame_*.jpg") - .OrderBy(f => f) - .ToList(); - - for (int i = 0; i < remainingFiles.Count; i++) - { - var newName = Path.Combine(imagesDir, $"frame_{i + 1:D4}.jpg"); - if (remainingFiles[i] != newName) - { - // Use temp name to avoid conflicts - var tempName = Path.Combine(imagesDir, $"temp_{i + 1:D4}.jpg"); - // Delete temp file first if exists - if (File.Exists(tempName)) File.Delete(tempName); - File.Move(remainingFiles[i], tempName); - } - } - - // Rename temp files to final names (with overwrite support) - var tempFiles = Directory.GetFiles(imagesDir, "temp_*.jpg").OrderBy(f => f).ToList(); - for (int i = 0; i < tempFiles.Count; i++) - { - var finalName = Path.Combine(imagesDir, $"frame_{i + 1:D4}.jpg"); - // Delete final file first if exists (fix: overwrite existing file) - if (File.Exists(finalName)) File.Delete(finalName); - File.Move(tempFiles[i], finalName); - } - - var finalCount = Directory.GetFiles(imagesDir, "frame_*.jpg").Length; - Console.WriteLine($"[OK] Deduplication complete: {files.Count} -> {finalCount} frames (removed {toDelete.Count} similar frames)"); -} - -async Task CalculateFrameSimilarityAsync(string ffmpeg, string file1, string file2) -{ - try - { - // Use ffmpeg to calculate PSNR (Peak Signal-to-Noise Ratio) between two images - // Higher PSNR = more similar images - var psi = new ProcessStartInfo - { - FileName = ffmpeg, - Arguments = $"-i \"{file1}\" -i \"{file2}\" -lavfi \"psnr\" -f null -", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - using var process = Process.Start(psi); - if (process == null) return 0; - - var stderrTask = process.StandardError.ReadToEndAsync(); - await process.WaitForExitAsync(); - var stderr = await stderrTask; - - // Parse PSNR value from output - // Format: [Parsed_psnr_0 @ ...] PSNR y:XX.XX u:XX.XX v:XX.XX average:XX.XX min:XX.XX max:XX.XX - var match = Regex.Match(stderr, @"average:(\d+\.?\d*)", RegexOptions.IgnoreCase); - if (match.Success && double.TryParse(match.Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var psnr)) - { - // Convert PSNR to similarity percentage - // PSNR > 40 dB is considered very similar (>95%) - // PSNR > 30 dB is considered similar (>80%) - // PSNR = infinity means identical images - if (psnr > 100) return 1.0; // Identical or near-identical - if (psnr > 40) return 0.95 + (psnr - 40) * 0.001; - if (psnr > 30) return 0.80 + (psnr - 30) * 0.015; - if (psnr > 20) return 0.50 + (psnr - 20) * 0.03; - return psnr * 0.025; - } - - // Fallback: use simpler SSIM if PSNR parsing fails - return await CalculateSSIMAsync(ffmpeg, file1, file2); - } - catch - { - return 0; - } -} - -async Task CalculateSSIMAsync(string ffmpeg, string file1, string file2) -{ - try - { - var psi = new ProcessStartInfo - { - FileName = ffmpeg, - Arguments = $"-i \"{file1}\" -i \"{file2}\" -lavfi \"ssim\" -f null -", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - using var process = Process.Start(psi); - if (process == null) return 0; - - var stderrTask = process.StandardError.ReadToEndAsync(); - await process.WaitForExitAsync(); - var stderr = await stderrTask; - - // Parse SSIM value: All:0.XXXXX - var match = Regex.Match(stderr, @"All:(\d+\.?\d*)", RegexOptions.IgnoreCase); - if (match.Success && double.TryParse(match.Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var ssim)) - { - return ssim; // SSIM is already 0-1 range - } - - return 0; - } - catch - { - return 0; - } -} - -string? FindExecutable(params string[] names) -{ - var pathEnv = Environment.GetEnvironmentVariable("PATH") ?? ""; - var paths = pathEnv.Split(Path.PathSeparator); - - foreach (var name in names) - { - foreach (var path in paths) - { - try - { - var fullPath = Path.Combine(path, name); - if (File.Exists(fullPath)) return fullPath; - } - catch { } // Ignore invalid paths - } - } - - // Common Windows paths - if (OperatingSystem.IsWindows()) - { - var commonPaths = new[] - { - @"C:\ffmpeg\bin", - @"C:\Program Files\ffmpeg\bin", - @"C:\tools\ffmpeg\bin", - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "scoop", "shims"), - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Microsoft", "WinGet", "Links"), - }; - - foreach (var basePath in commonPaths) - { - foreach (var name in names) - { - try - { - var fullPath = Path.Combine(basePath, name); - if (File.Exists(fullPath)) return fullPath; - } - catch { } - } - } - } - - // Try where/which command - try - { - var cmd = OperatingSystem.IsWindows() ? "where" : "which"; - var psi = new ProcessStartInfo - { - FileName = cmd, - Arguments = names[0], - RedirectStandardOutput = true, - UseShellExecute = false, - CreateNoWindow = true - }; - using var process = Process.Start(psi); - if (process != null) - { - process.WaitForExit(5000); // 5 second timeout - var output = process.StandardOutput.ReadToEnd()?.Trim(); - if (!string.IsNullOrEmpty(output)) - { - var firstLine = output.Split('\n')[0].Trim(); - if (File.Exists(firstLine)) return firstLine; - } - } - } - catch { } - - return null; -} - -string? GetArgValue(string[] args, params string[] names) -{ - for (int i = 0; i < args.Length - 1; i++) - { - if (names.Contains(args[i])) - { - return args[i + 1]; - } - } - return null; -} - -void PrintHelp() -{ - Console.WriteLine(@" -Bilibili Video Analyzer - Prepare Script - -Usage: - dotnet run prepare.cs [options] - -Arguments: - url Bilibili video URL (required) - -Options: - -o, --output Output directory (default: current) - --fps Frames per second (default: 1.0) - --similarity Similarity threshold for deduplication (default: 0.80) - --no-dedup Disable frame deduplication - --crop-ratio Crop to bottom 1/N of frame (default: 4, N=4 means bottom 1/4) - --no-crop Disable cropping (extract full frame) - --video-only Only download video, skip frame extraction - --frames-only Only extract frames (requires existing video.mp4) - -h, --help Show this help - -Examples: - dotnet run prepare.cs ""https://www.bilibili.com/video/BV1xx411c7mD"" - dotnet run prepare.cs ""https://www.bilibili.com/video/BV1xx411c7mD"" --fps 0.5 - dotnet run prepare.cs ""https://www.bilibili.com/video/BV1xx411c7mD"" -o ./output - dotnet run prepare.cs ""https://www.bilibili.com/video/BV1xx411c7mD"" --similarity 0.85 - dotnet run prepare.cs ""https://www.bilibili.com/video/BV1xx411c7mD"" --no-dedup - dotnet run prepare.cs ""https://www.bilibili.com/video/BV1xx411c7mD"" --crop-ratio 4 - dotnet run prepare.cs ""https://www.bilibili.com/video/BV1xx411c7mD"" --no-crop - -Cropping (for subtitle extraction): - By default, only the bottom 1/4 of the frame is extracted (--crop-ratio 4) - This is useful for capturing subtitles without the main video content - Use --no-crop to extract the full frame - Use --crop-ratio 2 for bottom half, --crop-ratio 8 for bottom 1/8, etc. - -Deduplication: - Consecutive frames with similarity >= threshold will be deduplicated. - Only compares adjacent frames (not cross-frame comparison). - Uses ffmpeg SSIM/PSNR for accurate similarity calculation. - -Requirements: - - .NET 10 SDK - - ffmpeg: https://ffmpeg.org/download.html -"); -} diff --git a/skills/kb-workflow/SKILL.md b/skills/kb-workflow/SKILL.md index 585e7fa..1aa036b 100644 --- a/skills/kb-workflow/SKILL.md +++ b/skills/kb-workflow/SKILL.md @@ -19,6 +19,27 @@ KB Workflow 是一个完整的知识库管理系统，包含： - 🧹 **自动清理** - 定期清理临时文件 - 📊 **汇报生成** - 生成处理统计 +## 触发方式 + +### 自然语言触发 + +当用户发送包含以下模式的消息时，技能会自动触发： + +| 触发词 | 行为 | 示例 | +|--------|------|------| +| `保存到知识库` + 链接 | 添加到收件箱并处理 | "保存到知识库 https://www.bilibili.com/video/BVxxx" | +| `添加到知识库` + 链接 | 添加到收件箱并处理 | "添加到知识库 https://www.bilibili.com/video/BVxxx" | +| `订阅到知识库` + 链接 | 添加到订阅列表 | "订阅到知识库 https://www.bilibili.com/video/BVxxx" | +| `订阅知识库` + 链接 | 添加到订阅列表 | "订阅知识库 https://www.bilibili.com/video/BVxxx" | + +### 支持的平台 + +| 平台 | 保存 | 订阅 | +|------|------|------| +| Bilibili | ✅ | ✅ | +| YouTube | ✅ | ❌ | +| 知乎 | ✅ | ❌ | + ## 目录结构 ``` diff --git a/skills/kb-workflow/kb_handler.py b/skills/kb-workflow/kb_handler.py index efc3dda..971cd4c 100755 --- a/skills/kb-workflow/kb_handler.py +++ b/skills/kb-workflow/kb_handler.py @@ -4,6 +4,10 @@ Auto-trigger handler for obsidian-kb skill. Detects video/article links in user messages and automatically processes them. Supports Bilibili, YouTube, and Zhihu. + +## 触发词 +- "保存到知识库" + 链接 → 添加到收件箱并处理 +- "订阅到知识库" + 链接 → 添加订阅 """ import re @@ -41,6 +45,12 @@ PATTERNS = { 'zhihu_answer': re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)'), } +# 触发词模式 +TRIGGER_PATTERNS = { + 'save': re.compile(r'(保存到知识库|加入知识库|添加到知识库)', re.IGNORECASE), + 'subscribe': re.compile(r'(订阅到知识库|订阅知识库|关注到知识库)', re.IGNORECASE), +} + def detect_links(text: str) -> List[Dict[str, str]]: """ @@ -196,6 +206,96 @@ def get_action_prompt(detected: List[Dict[str, str]]) -> str: return f"是否将上述 {count} 个内容添加到收件箱？(y/n)" +def detect_triggers(text: str) -> Optional[str]: + """ + 检测用户消息中的触发词。 + + Args: + text: 用户消息 + + Returns: + 'save' | 'subscribe' | None + """ + # 检测"保存到知识库" + if TRIGGER_PATTERNS['save'].search(text): + return 'save' + + # 检测"订阅到知识库" + if TRIGGER_PATTERNS['subscribe'].search(text): + return 'subscribe' + + return None + + +def handle_save_to_kb(detected: List[Dict[str, str]]) -> str: + """ + 处理"保存到知识库"命令：将链接添加到收件箱并处理。 + + Args: + detected: 检测到的链接列表 + + Returns: + 处理结果消息 + """ + if not detected: + return "❌ 未检测到任何链接，请提供要保存的链接" + + results = [] + for item in detected: + if item['is_short']: + results.append(f"⚠️ 短链接无法处理: {item['url']}") + continue + + # 调用保存处理 + success, msg = process_link(item['url'], item['platform'], auto_process=True) + results.append(msg) + + return "\n".join(results) + + +def handle_subscribe_to_kb(detected: List[Dict[str, str]]) -> str: + """ + 处理"订阅到知识库"命令：将链接添加到订阅列表。 + + Args: + detected: 检测到的链接列表 + + Returns: + 处理结果消息 + """ + if not detected: + return "❌ 未检测到任何链接，请提供要订阅的链接" + + results = [] + for item in detected: + if item['platform'] != 'bilibili': + results.append(f"⚠️ 暂不支持订阅 {item['platform']}: {item['url']}") + continue + + if item['is_short']: + results.append(f"⚠️ 短链接无法订阅: {item['url']}") + continue + + try: + # 调用订阅命令 + import subprocess + cmd = [ + sys.executable, + os.path.join(os.path.dirname(__file__), 'kb_workflow.py'), + '--trigger', 'subscriptions', + '--add', 'bilibili', 'video', item['id'], item.get('url', '') + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0: + results.append(f"✅ 已订阅: {item['id']}") + else: + results.append(f"❌ 订阅失败: {result.stderr}") + except Exception as e: + results.append(f"❌ 订阅失败: {e}") + + return "\n".join(results) + + def should_auto_process(text: str) -> bool: """ Check if user wants to auto-process based on their message. @@ -326,21 +426,45 @@ def handle_message(text: str) -> Optional[str]: """ Main entry point: handle incoming message and return response. + ## 触发条件 + - 检测到"保存到知识库"/"订阅到知识库"触发词 + 链接 → 自动处理 + - 仅检测到链接 → 询问用户是否处理 + Args: text: User's message text Returns: Response message, or None if no action needed """ - # First, check for short links and provide help - has_short, short_msg = detect_and_format(text) + # 首先检测触发词 + trigger = detect_triggers(text) - # Detect all links (including short ones) + # 检测链接 detected = detect_links(text) - if not detected: + # 如果没有链接，也没有触发词，返回 None + if not detected and not trigger: return None + # 如果有触发词（save/subscribe），但没有链接 + if trigger and not detected: + if trigger == 'save': + return "请提供要保存到知识库的链接" + elif trigger == 'subscribe': + return "请提供要订阅到知识库的链接" + + # 处理"保存到知识库"触发 + if trigger == 'save': + return handle_save_to_kb(detected) + + # 处理"订阅到知识库"触发 + if trigger == 'subscribe': + return handle_subscribe_to_kb(detected) + + # 以下是原有的仅链接检测逻辑 + # First, check for short links and provide help + has_short, short_msg = detect_and_format(text) + # If we have short links, show help message first if has_short: return short_msg diff --git a/skills/obsidian-kb/.disable_onednn.sh b/skills/obsidian-kb/.disable_onednn.sh deleted file mode 100755 index b3f6dc9..0000000 --- a/skills/obsidian-kb/.disable_onednn.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# 禁用 oneDNN 后端，解决 PaddleOCR 兼容性问题 - -echo "禁用 oneDNN..." - -# 方法1：环境变量（临时） -export FLAGS_use_mkldnn=0 -echo "✅ 环境变量已设置: FLAGS_use_mkldnn=0" - -# 方法2：写入 ~/.bashrc（永久） -if ! grep -q "FLAGS_use_mkldnn=0" ~/.bashrc; then - echo "" >> ~/.bashrc - echo "# 禁用 PaddleOCR 的 oneDNN 后端（解决兼容性问题）" >> ~/.bashrc - echo "export FLAGS_use_mkldnn=0" >> ~/.bashrc - echo "✅ 已写入 ~/.bashrc" -else - echo "ℹ️ ~/.bashrc 中已存在此配置" -fi - -echo "" -echo "验证配置..." -if [ "$FLAGS_use_mkldnn" = "0" ]; then - echo "✅ 当前会话: FLAGS_use_mkldnn=$FLAGS_use_mkldnn" -else - echo "⚠️ 当前会话: FLAGS_use_mkldnn=${FLAGS_use_mkldnn:-未设置}" - echo "💡 提示: 运行 'source ~/.bashrc' 使配置生效" -fi - -echo "" -echo "完成！现在 PaddleOCR 将使用 CPU 原生计算，避免 oneDNN 兼容性问题。" diff --git a/skills/obsidian-kb/.gitignore b/skills/obsidian-kb/.gitignore deleted file mode 100644 index 8f862cf..0000000 --- a/skills/obsidian-kb/.gitignore +++ /dev/null @@ -1,39 +0,0 @@ -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -env/ -venv/ -ENV/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# Testing -.pytest_cache/ -.coverage -htmlcov/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo - -# OS -.DS_Store -Thumbs.db diff --git a/skills/obsidian-kb/=0.11.6 b/skills/obsidian-kb/=0.11.6 deleted file mode 100644 index 01ea3e9..0000000 --- a/skills/obsidian-kb/=0.11.6 +++ /dev/null @@ -1,11 +0,0 @@ -Collecting markdownify - Using cached markdownify-1.2.2-py3-none-any.whl.metadata (9.9 kB) -Requirement already satisfied: beautifulsoup4<5,>=4.9 in ./.venv/lib/python3.12/site-packages (from markdownify) (4.14.3) -Collecting six<2,>=1.15 (from markdownify) - Using cached six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB) -Requirement already satisfied: soupsieve>=1.6.1 in ./.venv/lib/python3.12/site-packages (from beautifulsoup4<5,>=4.9->markdownify) (2.8.3) -Requirement already satisfied: typing-extensions>=4.0.0 in ./.venv/lib/python3.12/site-packages (from beautifulsoup4<5,>=4.9->markdownify) (4.15.0) -Using cached markdownify-1.2.2-py3-none-any.whl (15 kB) -Using cached six-1.17.0-py2.py3-none-any.whl (11 kB) -Installing collected packages: six, markdownify -Successfully installed markdownify-1.2.2 six-1.17.0 diff --git a/skills/obsidian-kb/=4.0.0 b/skills/obsidian-kb/=4.0.0 deleted file mode 100644 index 0351f09..0000000 --- a/skills/obsidian-kb/=4.0.0 +++ /dev/null @@ -1,42 +0,0 @@ -Collecting DrissionPage - Using cached DrissionPage-4.1.1.2-py3-none-any.whl.metadata (4.7 kB) -Requirement already satisfied: lxml in ./.venv/lib/python3.12/site-packages (from DrissionPage) (6.0.2) -Requirement already satisfied: requests in ./.venv/lib/python3.12/site-packages (from DrissionPage) (2.32.5) -Collecting cssselect (from DrissionPage) - Using cached cssselect-1.4.0-py3-none-any.whl.metadata (2.4 kB) -Collecting DownloadKit>=2.0.7 (from DrissionPage) - Using cached DownloadKit-2.0.7-py3-none-any.whl.metadata (1.7 kB) -Collecting websocket-client (from DrissionPage) - Using cached websocket_client-1.9.0-py3-none-any.whl.metadata (8.3 kB) -Requirement already satisfied: click in ./.venv/lib/python3.12/site-packages (from DrissionPage) (8.3.1) -Collecting tldextract>=3.4.4 (from DrissionPage) - Using cached tldextract-5.3.1-py3-none-any.whl.metadata (7.3 kB) -Collecting psutil (from DrissionPage) - Using cached psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl.metadata (22 kB) -Collecting DataRecorder>=3.4.11 (from DownloadKit>=2.0.7->DrissionPage) - Using cached DataRecorder-3.6.2-py3-none-any.whl.metadata (4.5 kB) -Requirement already satisfied: idna in ./.venv/lib/python3.12/site-packages (from tldextract>=3.4.4->DrissionPage) (3.11) -Collecting requests-file>=1.4 (from tldextract>=3.4.4->DrissionPage) - Using cached requests_file-3.0.1-py2.py3-none-any.whl.metadata (1.7 kB) -Collecting filelock>=3.0.8 (from tldextract>=3.4.4->DrissionPage) - Using cached filelock-3.20.3-py3-none-any.whl.metadata (2.1 kB) -Requirement already satisfied: charset_normalizer<4,>=2 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (3.4.4) -Requirement already satisfied: urllib3<3,>=1.21.1 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (2.6.3) -Requirement already satisfied: certifi>=2017.4.17 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (2026.1.4) -Collecting openpyxl (from DataRecorder>=3.4.11->DownloadKit>=2.0.7->DrissionPage) - Using cached openpyxl-3.1.5-py2.py3-none-any.whl.metadata (2.5 kB) -Collecting et-xmlfile (from openpyxl->DataRecorder>=3.4.11->DownloadKit>=2.0.7->DrissionPage) - Using cached et_xmlfile-2.0.0-py3-none-any.whl.metadata (2.7 kB) -Using cached DrissionPage-4.1.1.2-py3-none-any.whl (257 kB) -Using cached DownloadKit-2.0.7-py3-none-any.whl (21 kB) -Using cached tldextract-5.3.1-py3-none-any.whl (105 kB) -Using cached cssselect-1.4.0-py3-none-any.whl (18 kB) -Using cached psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl (155 kB) -Using cached websocket_client-1.9.0-py3-none-any.whl (82 kB) -Using cached DataRecorder-3.6.2-py3-none-any.whl (37 kB) -Using cached filelock-3.20.3-py3-none-any.whl (16 kB) -Using cached requests_file-3.0.1-py2.py3-none-any.whl (4.5 kB) -Using cached openpyxl-3.1.5-py2.py3-none-any.whl (250 kB) -Using cached et_xmlfile-2.0.0-py3-none-any.whl (18 kB) -Installing collected packages: websocket-client, psutil, filelock, et-xmlfile, cssselect, requests-file, openpyxl, tldextract, DataRecorder, DownloadKit, DrissionPage -Successfully installed DataRecorder-3.6.2 DownloadKit-2.0.7 DrissionPage-4.1.1.2 cssselect-1.4.0 et-xmlfile-2.0.0 filelock-3.20.3 openpyxl-3.1.5 psutil-7.2.2 requests-file-3.0.1 tldextract-5.3.1 websocket-client-1.9.0 diff --git a/skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md b/skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md deleted file mode 100644 index a5b18d3..0000000 --- a/skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md +++ /dev/null @@ -1,168 +0,0 @@ -# 知识库维护系统 - 整体流程图 - -## 1. 用户交互流程（接收链接） -```mermaid -flowchart TD - A[用户发送链接] --> B{技能1: 链接解析} - B --> C{类型判断} - - C -->|单个内容| D[直接解析] - C -->|列表类型| E[询问用户] - - E --> F[处理范围选择] - F --> G[订阅选项选择] - G --> H[写入订阅配置] - - D --> I[写入收件箱 pending/] - H --> I - - I --> J[返回用户: 已收录] -``` - -## 2. 定时任务/手动触发流程 -```mermaid -flowchart TD - A[定时任务触发
或手动触发] --> B[技能5: 工作流编排] - B --> C[扫描收件箱] - - C --> D{pending/ 目录} - D -->|有待处理| E[获取待处理列表] - D -->|无| F[检查订阅更新] - - E --> G[技能2: 获取元数据] - G --> H[更新状态: metadata_ready] - H --> I[技能3: 获取内容] - I --> J[更新状态: content_ready] - J --> K[技能4: 深度分析] - K --> L[归档到 archive/] - L --> M[更新订阅追踪] - M --> N[继续处理下一个] - - F --> O{有订阅?} - O -->|是| P[检测更新] - P --> Q[新增内容加入 pending/] - Q --> R[更新订阅时间] - - O -->|否| S[跳过] - - N --> T{还有待处理?} - T -->|是| E - T -->|否| U[生成汇报文件] - U --> V[完成] -``` - -## 3. 完整主流程（简化版） -```mermaid -flowchart TD - subgraph 用户交互 - A[用户发送链接] --> B[技能1: link-parser] - B --> C{内容类型} - C -->|单个| D[直接处理] - C -->|列表| E[询问用户选项] - E --> F[保存订阅配置] - D --> G[写入收件箱 pending/] - F --> G - end - - subgraph 定时处理 - H[定时/手动触发] --> I[技能5: kb-workflow] - I --> J[扫描收件箱] - J --> K[技能2: metadata-fetcher] - K --> L[技能3: content-fetcher] - L --> M[技能4: deep-analyzer] - M --> N[归档] - N --> O[更新订阅] - O --> P[生成汇报] - end - - G --> H - - style A fill:#e1f5fe - style I fill:#fff3e0 - style K fill:#f3e5f5 - style L fill:#e8f5e9 - style M fill:#fce4ec -``` - -## 4. 各平台技能调用关系 -```mermaid -flowchart LR - subgraph 技能1 - LP[link-parser] - end - - subgraph 技能2[元数据获取] - BM[bilibili-metadata] - YM[ youtube-metadata] - ZM[ zhihu-metadata] - end - - subgraph 技能3[内容获取] - BC[bilibili-content] - OC[ocr-processor] - end - - subgraph 技能4 - DA[deep-analyzer] - end - - LP -->|解析结果| BM - LP -->|解析结果| YM - LP -->|解析结果| ZM - - BM -->|元数据| BC - BC -->|低分视频| OC - OC -->|OCR结果| DA - BM -->|元数据| DA -``` - -## 5. 数据流转 -```mermaid -flowchart LR - subgraph 输入 - URL[用户链接] - end - - subgraph 收件箱 - P[pending/] - M[metadata/] - C[content/] - end - - subgraph 输出 - A[archive/YYYY/MM/] - R[report.json] - end - - URL -->|技能1| P - P -->|技能2| M - M -->|技能3| C - C -->|技能4| A - A -.->|统计| R - - style P fill:#fffde7 - style M fill:#e3f2fd - style C fill:#e8f5e9 - style A fill:#f3e5f5 -``` - -## 6. 状态流转 -```mermaid -stateDiagram-v2 - [*] --> pending: 链接收录 - pending --> metadata: 技能2完成 - metadata --> content: 技能3完成 - content --> analyzed: 技能4完成 - analyzed --> archived: 归档完成 - archived --> [*] - - pending --> skipped: 跳过 - metadata --> failed: 元数据获取失败 - content --> failed: 内容获取失败 - analyzed --> failed: 分析失败 - - failed --> retry: 重试 - retry --> pending - retry --> metadata - retry --> content -``` diff --git a/skills/obsidian-kb/CHECKPOINT_6.md b/skills/obsidian-kb/CHECKPOINT_6.md deleted file mode 100644 index 0a20a8e..0000000 --- a/skills/obsidian-kb/CHECKPOINT_6.md +++ /dev/null @@ -1,353 +0,0 @@ -# Checkpoint 6: 主流程集成 - 完成报告 - -## 任务概述 - -完成 obsidian-kb 整合项目的主流程集成任务，整合提取层、处理层、输出层，实现完整的数据流：提取 → 处理 → 输出 → 同步。 - -## 完成内容 - -### 1. 主流程集成文件 - -创建了 `main_pipeline.py`，实现了完整的数据流集成： - -#### 核心类：`MainPipeline` - -```python -class MainPipeline: - """主流程集成器""" -``` - -#### 主要方法： - -- `__init__(config, enable_git_sync)` - 初始化主流程 -- `process_video(source_id)` - 处理单个视频的完整流程 -- `_extract_video(source_id)` - 提取层：调用 BilibiliExtractor -- `_analyze_content(extractor_data)` - 处理层：调用 DeepAnalyzer -- `_chunk_content(extractor_data)` - 处理层：调用 ContentChunker -- `_merge_analysis_to_content(content, analysis, chunks)` - 合并分析结果 -- `_write_obsidian_note(content)` - 输出层：调用 ObsidianWriter -- `_sync_to_git(content)` - 同步层：调用 GitSync - -### 2. 错误处理 - -在 `main_pipeline.py` 中添加了完善的错误处理机制： - -#### 错误捕获： - -1. **ExtractionError 捕获**： - ```python - try: - extractor_data = self.extractor.extract(source_id) - except ExtractionError as e: - logger.error(f"❌ 提取失败: {e}") - raise - ``` - -2. **通用异常捕获**： - ```python - try: - # 处理逻辑 - except Exception as e: - logger.error(f"❌ 处理失败: {e}", exc_info=True) - # 返回友好错误信息，不中断流程 - ``` - -3. **层级容错**： - - 提取失败：中断流程，返回错误 - - 分析失败：返回默认结果，继续流程 - - 分片失败：返回空列表，继续流程 - - 写入失败：返回 None，记录警告 - - Git同步失败：记录错误，不影响主流程 - -### 3. 日志记录 - -配置了完善的日志系统： - -#### 日志配置： - -```python -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler('obsidian_kb.log', encoding='utf-8') - ] -) -``` - -#### 日志记录： - -- **关键步骤记录**： - - 提取开始/成功/失败 - - 分析开始/成功/失败 - - 分片开始/成功/失败 - - 写入开始/成功/失败 - - Git同步开始/成功/失败 - -- **详细信息记录**： - - 视频标题、作者、时长 - - 字幕条数 - - 分片数量 - - 输出文件路径 - - 错误堆栈信息 - -- **日志文件**：`obsidian_kb.log` - -### 4. 测试脚本 - -创建了 `test_integration.py`，提供完整的测试功能： - -#### 测试用例： - -1. **test_extract_only()** - 测试提取功能 - - 验证提取器能否正常工作 - - 检查提取的数据完整性 - -2. **test_full_pipeline()** - 测试完整流程 - - 验证提取 → 处理 → 输出 全流程 - - 检查生成的文件 - -3. **test_error_handling()** - 测试错误处理 - - 验证错误能否被正确捕获 - - 检查错误信息的友好性 - -4. **cleanup_test_env()** - 清理测试环境 - - 删除临时目录 - - 确保测试后环境干净 - -#### 测试特性： - -- 使用临时目录，不影响生产环境 -- 不执行 Git 同步 -- 详细的测试输出 -- 自动清理测试环境 - -### 5. Handler 集成 - -更新了 `handler.py`，添加了新的集成接口： - -#### 新增方法： - -```python -def process_video_full(source_id: str, enable_git_sync: bool = False) -> Tuple[bool, str]: - """ - 使用主流程完整处理视频（新接口） - - 完整流程: 提取 → 处理 → 输出 → 同步 - """ -``` - -#### CLI 接口： - -```bash -# 检测链接 -python3 handler.py detect "https://www.bilibili.com/video/BV1xx411c7mD" - -# 处理视频 -python3 handler.py process BV1xx411c7mD - -# 处理视频并同步到Git -python3 handler.py process BV1xx411c7mD --git -``` - -### 6. 验证脚本 - -创建了 `verify_integration.py`，验证代码结构： - -#### 验证内容： - -1. 文件存在性 -2. 主流程导入 -3. 主流程方法 -4. 测试脚本导入 -5. 测试脚本方法 -6. Handler 集成 -7. 错误处理 -8. 日志配置 - -## 验证结果 - -运行 `python3 verify_integration.py`： - -``` -文件存在性 ✅ 通过 -主流程导入 ✅ 通过 -主流程方法 ✅ 通过 -测试脚本导入 ✅ 通过 -测试脚本方法 ✅ 通过 -Handler 集成 ✅ 通过 -错误处理 ✅ 通过 -日志配置 ✅ 通过 - -总体结果: ✅ 全部通过 -``` - -## 使用示例 - -### 1. 使用主流程 - -```python -from main_pipeline import MainPipeline -from src.config import get_config - -# 创建主流程 -config = get_config() -pipeline = MainPipeline(config=config, enable_git_sync=False) - -# 处理视频 -result = pipeline.process_video("BV1MnFNz5EDL") - -if result['success']: - print("处理成功！") - print(result['message']) -else: - print(f"处理失败: {result['error']}") -``` - -### 2. 使用 Handler - -```python -from handler import process_video_full - -# 处理视频 -success, message = process_video_full("BV1MnFNz5EDL", enable_git_sync=False) - -if success: - print(message) -else: - print(f"失败: {message}") -``` - -### 3. 命令行 - -```bash -# 使用主流程处理 -python3 -m main_pipeline BV1MnFNz5EDL - -# 使用 Handler 处理 -python3 handler.py process BV1MnFNz5EDL - -# 使用 Handler 处理并同步到 Git -python3 handler.py process BV1MnFNz5EDL --git -``` - -### 4. 运行测试 - -```bash -# 完整测试（需要所有依赖） -python3 test_integration.py - -# 验证代码结构（不需要依赖） -python3 verify_integration.py -``` - -## 数据流 - -``` -输入: BV号/URL - ↓ -[Step 1] 提取层: BilibiliExtractor.extract() - → metadata (标题, 作者, 时长, etc.) - → subtitle (字幕文本) - → description (简介) - ↓ -[Step 2] 构建对象: _build_content() - → Content 对象 - ↓ -[Step 3] 处理层-分析: DeepAnalyzer.analyze() - → AnalysisResult (主题, 洞察, 时间轴, etc.) - ↓ -[Step 4] 处理层-分片: ContentChunker.chunk_from_extractor() - → Chunk[] (分段内容) - ↓ -[Step 5] 合并结果: _merge_analysis_to_content() - → 更新 Content 对象 - ↓ -[Step 6] 输出层: ObsidianWriter.write() - → Markdown 文件 - ↓ -[Step 7] 同步层: GitSync.sync() [可选] - → Git 提交和推送 - ↓ -输出: Obsidian 笔记文件 -``` - -## 关键修改 - -### main_pipeline.py - -- **新增文件**（14,151 字节） -- 实现了 MainPipeline 类 -- 集成了所有层：提取、处理、输出、同步 -- 完整的错误处理和日志记录 - -### handler.py - -- **导入主流程**： - ```python - from main_pipeline import MainPipeline - ``` - -- **新增方法**：`process_video_full()` -- **更新 CLI**：支持 `process` 命令 - -### test_integration.py - -- **新增文件**（7,831 字节） -- 三个测试用例：提取、完整流程、错误处理 -- 自动清理测试环境 - -### verify_integration.py - -- **新增文件**（6,614 字节） -- 8项验证检查 -- 不需要依赖即可运行 - -## 验证标准 - -✅ **完整数据流打通**：BV号 → Obsidian笔记 -- 主流程 `process_video()` 实现了完整流程 -- 各层通过标准接口连接 - -✅ **错误处理完善** -- 捕获 ExtractionError 并返回友好提示 -- 各层都有 try-except 保护 -- 异常不会导致程序崩溃 - -✅ **日志记录完整** -- 记录所有关键步骤 -- 错误详细信息记录到日志文件 -- 同时输出到控制台和文件 - -✅ **测试脚本能运行** -- `test_integration.py` 提供完整测试 -- `verify_integration.py` 提供代码结构验证 -- 所有验证通过 - -## 注意事项 - -1. **依赖安装**：需要安装 playwright 等依赖 - ```bash - pip install -r requirements.txt - ``` - -2. **配置文件**：确保 `config.yaml` 配置正确 - - Obsidian vault 路径 - - Git 设置（可选） - - AI API key（可选） - -3. **日志文件**：日志会写入 `obsidian_kb.log` - -4. **临时文件**：测试会创建临时目录，测试后自动清理 - -## 总结 - -Checkpoint 6 主流程集成任务已全部完成： - -1. ✅ 修改/创建主入口，集成提取层、处理层、输出层 -2. ✅ 添加错误处理，各层都有 try-except 保护 -3. ✅ 添加日志记录，关键步骤和错误信息都记录 -4. ✅ 创建测试脚本，支持完整流程测试 - -完整的数据流已打通，代码结构通过验证，可以开始使用。 diff --git a/skills/obsidian-kb/DETAILED_DESIGN.md b/skills/obsidian-kb/DETAILED_DESIGN.md deleted file mode 100644 index 944eb69..0000000 --- a/skills/obsidian-kb/DETAILED_DESIGN.md +++ /dev/null @@ -1,1108 +0,0 @@ -# 知识库维护系统 - 详细设计 - -> 版本: 1.0.0 -> 状态: 草稿 - ---- - -## 目录 - -1. [技能接口总览](#1-技能接口总览) -2. [技能1-link-parser](#2-技能1-link-parser) -3. [技能2-bilibili-metadata](#3-技能2-bilibili-metadata) -4. [技能2-zhihu-metadata](#4-技能2-zhihu-metadata) -5. [技能3-bilibili-content](#5-技能3-bilibili-content) -6. [技能3-ocr-processor](#6-技能3-ocr-processor) -7. [技能4-deep-analyzer](#7-技能4-deep-analyzer) -8. [技能5-kb-workflow](#8-技能5-kb-workflow) -9. [数据格式定义](#9-数据格式定义) -10. [配置文件规范](#10-配置文件规范) -11. [扩展指南](#11-扩展指南) - ---- - -## 1. 技能接口总览 - -### 1.1 调用约定 - -所有技能遵循以下调用约定： - -```yaml -输入: - - 命令行参数 - - stdin JSON (可选) - - 环境变量 - -输出: - - stdout JSON - - 0: 成功 - - 1: 失败 (错误信息输出到 stderr) - -状态码: - - 0: 成功 - - 1: 一般错误 - - 2: 参数错误 - - 3: 网络错误 - - 4: 权限错误 -``` - -### 1.2 通用错误响应 - -```json -{ - "success": false, - "error": "错误描述", - "error_code": "ERROR_CODE", - "details": {} -} -``` - ---- - -## 2. 技能1: link-parser - -**功能**: 解析链接，提取平台+类型+ID，处理短链跳转 - -### 2.1 命令行接口 - -```bash -# 基本用法 -python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD" - -# 支持短链 -python link-parser.py --url "https://b23.tv/xxxxx" - -# 批量处理 -python link-parser.py --batch input.txt - -# 输出文件 -python link-parser.py --url "..." --output result.json -``` - -### 2.2 输入参数 - -```yaml -参数: - --url, -u # 必需，单个URL - --batch, -b # 可选，批量文件（每行一个URL） - --output, -o # 可选，输出文件路径 - --format # 输出格式: json/yaml (默认: json) -``` - -### 2.3 输出格式 - -**成功**: -```json -{ - "success": true, - "data": { - "platform": "bilibili", - "type": "video", - "id": "BV1xx411c7mD", - "source_url": "https://www.bilibili.com/video/BV1xx411c7mD", - "canonical_url": "https://www.bilibili.com/video/BV1xx411c7mD", - "short_link_resolved": false, - "title": null, - "metadata": {} - }, - "metadata": { - "processed_at": "2024-01-15T10:00:00", - "parser_version": "1.0.0" - } -} -``` - -**列表类型**: -```json -{ - "success": true, - "data": { - "platform": "bilibili", - "type": "up", - "id": "7770724", - "source_url": "https://space.bilibili.com/7770724", - "is_list_type": true, - "list_info": { - "total_items": 156, - "item_type": "video" - } - }, - "metadata": { - "requires_user_input": true, - "input_options": { - "scope": ["subscribe_only", "new_only", "all"], - "subscription": ["daily", "weekly", "custom", "none"] - } - } -} -``` - -### 2.4 支持的链接格式 - -| 类型 | 平台 | 正则示例 | 输出 | -|------|------|----------|------| -| 短链 | B站 | `b23.tv/[a-zA-Z0-9]+` | resolved_url + 解析 | -| 视频 | B站 | `bilibili.com/video/BV[a-zA-Z0-9]+` | `{type: video, id: BV...}` | -| 专栏 | B站 | `bilibili.com/read/cv[0-9]+` | `{type: article, id: cv...}` | -| 动态 | B站 | `bilibili.com/opus/[0-9]+` | `{type: opus, id: ...}` | -| UP主 | B站 | `space.bilibili.com/[0-9]+` | `{type: up, id: ...}` | -| 收藏夹 | B站 | `space.bilibili.com/[0-9]+/favlist?fid=[0-9]+` | `{type: favlist, id: ...}` | -| 合集 | B站 | `space.bilibili.com/[0-9]+/lists/[0-9]+?type=season` | `{type: playlist, id: ...}` | -| YouTube | YouTube | `(youtu.be/[a-zA-Z0-9]+|youtube.com/watch?v=[a-zA-Z0-9]+)` | `{type: video, id: ...}` | -| 知乎 | 知乎 | `zhihu.com/question/[0-9]+` | `{type: question, id: ...}` | -| 知乎 | 知乎 | `zhihu.com/answer/[0-9]+` | `{type: answer, id: ...}` | -| 知乎 | 知乎 | `zhihu.com/people/[a-zA-Z0-9-]+` | `{type: user, id: ...}` | -| 知乎 | 知乎 | `zhihu.com/collection/[0-9]+` | `{type: collection, id: ...}` | - -### 2.5 复用代码 - -```python -# 复用现有代码 -from shortlink_resolver import resolve_shortlink, extract_bilibili_id -``` - ---- - -## 3. 技能2: bilibili-metadata - -**功能**: 反爬获取B站视频/专栏元数据 - -### 3.1 命令行接口 - -```bash -# 基本用法 -python bilibili-metadata.py --id "BV1xx411c7mD" - -# 指定输出目录 -python bilibili-metadata.py --id "BV1xx" --output ./metadata/ - -# 批量处理 -python bilibili-metadata.py --batch ids.txt --output ./ - -# 视频+专栏支持 -python bilibili-metadata.py --id "cv43780186" --type "article" -``` - -### 3.2 输入参数 - -```yaml -参数: - --id, -i # 必需，内容ID - --type, -t # 内容类型: video/article/opus (默认: video) - --output, -o # 输出目录 (默认: ./) - --batch, -b # 批量文件 (每行: id [type]) - --config, -c # 配置文件路径 -``` - -### 3.3 输出格式 - -**视频元数据**: -```json -{ - "success": true, - "data": { - "platform": "bilibili", - "type": "video", - "id": "BV1xx411c7mD", - "source_url": "https://www.bilibili.com/video/BV1xx411c7mD", - "title": "Python完整教程", - "description": "这是一个Python教程...", - "author": { - "id": 7770724, - "name": "UP主名字", - "face": "https://i0.hdslb.com/xxx.jpg" - }, - "duration": 9000, - "duration_text": "2:30:00", - "publish_time": "2024-01-01T12:00:00", - "tags": ["python", "tutorial", "编程"], - "stats": { - "view": 100000, - "like": 5000, - "coin": 2000, - "favorite": 1000, - "share": 500 - }, - "subtitle": { - "has_subtitle": true, - "languages": ["zh-CN"], - "url": "https://api.bilibili.com/..." - }, - "pic": "https://i0.hdslb.com/xxx.jpg", - "quality_score": null - }, - "metadata": { - "fetched_at": "2024-01-15T10:00:00", - "source": "bilibili_api" - } -} -``` - -**专栏文章元数据**: -```json -{ - "success": true, - "data": { - "platform": "bilibili", - "type": "article", - "id": "cv43780186", - "source_url": "https://www.bilibili.com/read/cv43780186", - "title": "文章标题", - "description": "文章摘要...", - "author": { - "id": 123456, - "name": "作者名字", - "face": "https://i0.hdslb.com/xxx.jpg" - }, - "publish_time": "2024-01-01T12:00:00", - "stats": { - "view": 50000, - "like": 1000, - "comment": 200 - }, - "word_count": 5000, - "images_count": 10, - "category": "技术" - }, - "metadata": { - "fetched_at": "2024-01-15T10:00:00", - "source": "bilibili_api" - } -} -``` - -**UP主/收藏夹列表**: -```json -{ - "success": true, - "data": { - "platform": "bilibili", - "type": "up", - "id": "7770724", - "source_url": "https://space.bilibili.com/7770724", - "name": "UP主名字", - "description": "UP主介绍...", - "follower": 100000, - "following": 100, - "archive_count": 156, - "content_stats": { - "total_views": 10000000, - "total_likes": 500000 - }, - "subscription": null - }, - "metadata": { - "fetched_at": "2024-01-15T10:00:00" - } -} -``` - -### 3.4 复用代码 - -```python -# 复用现有代码 -from bilibili_extractor import BilibiliExtractor -``` - ---- - -## 4. 技能2: zhihu-metadata - -**功能**: 获取知乎元数据 - -### 4.1 命令行接口 - -```bash -python zhihu-metadata.py --id "266309957" --type "question" -python zhihu-metadata.py --id "2003427891925509507" --type "answer" -``` - -### 4.2 输入参数 - -```yaml -参数: - --id, -i # 必需，内容ID - --type, -t # 内容类型: question/answer/article/user/collection (默认: question) - --output, -o # 输出目录 - --config, -c # 配置文件路径 -``` - -### 4.3 输出格式 - -**问题元数据**: -```json -{ - "success": true, - "data": { - "platform": "zhihu", - "type": "question", - "id": "266309957", - "source_url": "https://www.zhihu.com/question/266309957", - "title": "问题标题", - "description": "问题描述...", - "author": { - "id": "user-xxx", - "name": "作者名字", - "url_token": "user-xxx" - }, - "answer_count": 50, - "followers": 1000, - "browse_count": 50000, - "created_time": "2024-01-01T12:00:00", - "updated_time": "2024-01-15T10:00:00", - "tags": ["标签1", "标签2"] - }, - "metadata": { - "fetched_at": "2024-01-15T10:00:00" - } -} -``` - -**回答元数据**: -```json -{ - "success": true, - "data": { - "platform": "zhihu", - "type": "answer", - "id": "2003427891925509507", - "source_url": "https://www.zhihu.com/question/266309957/answer/2003427891925509507", - "question_id": "266309957", - "question_title": "问题标题", - "author": { - "id": "user-xxx", - "name": "作者名字", - "headline": "作者简介" - }, - "vote_up": 1000, - "comment_count": 50, - "created_time": "2024-01-01T12:00:00", - "updated_time": "2024-01-15T10:00:00", - "content_summary": "回答摘要..." - }, - "metadata": { - "fetched_at": "2024-01-15T10:00:00" - } -} -``` - -### 4.4 复用代码 - -```python -# 复用现有代码 -from zhihu_extractor import ZhihuDrissionExtractor -``` - ---- - -## 5. 技能3: bilibili-content - -**功能**: 下载低分视频 + 提取字幕/专栏全文 - -### 5.1 命令行接口 - -```bash -# 基本用法（视频） -python bilibili-content.py --id "BV1xx411c7mD" --output ./content/ - -# 视频+字幕 -python bilibili-content.py --id "BV1xx" --output ./ --with-subtitle - -# 专栏全文 -python bilibili-content.py --id "cv43780186" --type article --output ./ - -# 低分辨率（用于OCR） -python bilibili-content.py --id "BV1xx" --quality 480 --output ./ -``` - -### 5.2 输入参数 - -```yaml -参数: - --id, -i # 必需，内容ID - --type, -t # 内容类型: video/article/opus (默认: video) - --output, -o # 输出目录 (默认: ./) - --quality # 视频质量: 360/480/720/1080 (默认: 480) - --with-subtitle # 是否提取字幕 (默认: true) - --subtitle-lang # 字幕语言 (默认: zh-CN) - --format # 输出格式: json/ndjson (默认: json) -``` - -### 5.3 输出格式 - -**视频内容**: -```json -{ - "success": true, - "data": { - "platform": "bilibili", - "type": "video", - "id": "BV1xx411c7mD", - "video_file": { - "path": "content/BV1xx_video.mp4", - "size_bytes": 52428800, - "duration": 9000, - "quality": "480p" - }, - "subtitle": { - "format": "srt", - "path": "content/BV1xx_subtitle.srt", - "language": "zh-CN", - "text_length": 50000, - "lines_count": 1500 - }, - "danmaku": { - "path": "content/BV1xx_danmaku.xml", - "count": 5000 - }, - "frames": { - "output_dir": "content/BV1xx_frames", - "count": 900, - "fps": 0.1 - } - }, - "metadata": { - "fetched_at": "2024-01-15T10:00:00", - "content_quality": "preview" - } -} -``` - -**专栏内容**: -```json -{ - "success": true, - "data": { - "platform": "bilibili", - "type": "article", - "id": "cv43780186", - "content": { - "markdown": "# 文章标题\n\n内容...", - "markdown_path": "content/cv43780186.md", - "html": "...", - "word_count": 5000, - "images": [ - { - "url": "https://...", - "path": "content/images/img_001.jpg", - "description": "图片描述" - } - ] - }, - "author": { - "name": "作者名字" - }, - "publish_time": "2024-01-01T12:00:00" - }, - "metadata": { - "fetched_at": "2024-01-15T10:00:00" - } -} -``` - -### 5.4 复用代码 - -```python -# 复用现有代码 -from video_stream_extractor import VideoStreamExtractor -from bilibili_extractor import BilibiliExtractor -``` - ---- - -## 6. 技能3: ocr-processor - -**功能**: 通用字幕OCR（被技能3调用） - -### 6.1 命令行接口 - -```bash -# 基本用法 -python ocr-processor.py --input ./frames/ --output ./ocr.json - -# 指定引擎 -python ocr-processor.py --input ./frames/ --engine paddleocr --output ./ - -# 只处理部分帧 -python ocr-processor.py --input ./frames/ --start 0 --end 100 --output ./ -``` - -### 6.2 输入参数 - -```yaml -参数: - --input, -i # 必需，帧图片目录或单个图片 - --output, -o # 输出文件 (默认: ocr.json) - --engine # OCR引擎: paddleocr/tesseract (默认: paddleocr) - --region # 字幕区域: bottom/full/auto (默认: bottom) - --fps # 抽帧频率 (默认: 0.1，即每10秒1帧) - --start # 开始帧号 (默认: 0) - --end # 结束帧号 (默认: -1，即全部) - --confidence # 置信度阈值 (默认: 0.7) -``` - -### 6.3 输出格式 - -```json -{ - "success": true, - "data": { - "frames": { - "frame_0001.jpg": { - "texts": [ - { - "text": "字幕文本", - "confidence": 0.95, - "bbox": [x1, y1, x2, y2] - } - ], - "timestamp": 0.0 - } - }, - "merged_text": "合并的字幕文本...", - "statistics": { - "total_frames": 100, - "frames_with_text": 80, - "total_texts": 500, - "unique_texts": 200 - } - }, - "metadata": { - "engine": "paddleocr", - "processed_at": "2024-01-15T10:00:00" - } -} -``` - -### 6.4 复用代码 - -```python -# 复用现有代码 -from ocr_processor import OCRProcessor -``` - ---- - -## 7. 技能4: deep-analyzer - -**功能**: AI深度分析，按内容类型使用不同模板 - -### 7.1 命令行接口 - -```bash -# 基本用法 -python deep-analyzer.py --metadata meta.json --content content.json --output ./analysis.md - -# 指定类型 -python deep-analyzer.py --metadata meta.json --type tutorial --output ./ - -# 使用自定义模板 -python deep-analyzer.py --metadata meta.json --template my_template.md --output ./ - -# 带OCR结果 -python deep-analyzer.py --metadata meta.json --content content.json --ocr ocr.json --output ./ -``` - -### 7.2 输入参数 - -```yaml -参数: - --metadata, -m # 必需，元数据JSON文件 - --content, -c # 必需，内容JSON文件 - --output, -o # 必需，输出Markdown文件 - --type, -t # 内容类型: auto/tutorial/documentation/news (默认: auto) - --template # 自定义模板路径 - --ocr # OCR结果JSON文件 (可选) - --model # AI模型 (默认: 配置中指定) - --tags # 预设标签 (可选) -``` - -### 7.3 模板变量 - -**通用变量**: -```jinja2 -{{ title }} # 标题 -{{ source }} # 来源 (bilibili/zhihu) -{{ url }} # 原始URL -{{ author }} # 作者/UP主 -{{ duration }} # 时长 -{{ date }} # 发布日期 -{{ tags }} # 标签列表 -{{ summary }} # 详细摘要 -{{ key_points }} # 核心要点 -{{ quality_score }} # 质量评分 (1-100) -``` - -**教程视频额外变量**: -```jinja2 -{{ prerequisites }} # 前置知识 -{{ outline }} # 分段大纲 -{{ pitfalls }} # 常见坑点 -{{ practice_advice }} # 实践建议 -``` - -### 7.4 输出格式 - -**教程类输出**: -```markdown ---- -title: "Python完整教程" -source: "bilibili" -url: "https://..." -author: "CodeMaster" -duration: "2:30:00" -date: "2024-01-01T00:00:00" -tags: - - python - - tutorial - - 编程入门 -quality_score: 95 ---- - -## 🎯 一句话总结 - -本教程系统讲解Python基础语法，适合零基础学习者。 - -## 📚 前置知识 - -- 无需编程基础 -- 电脑基本操作能力 - -## 📝 步骤详解 - -### 环境配置 [00:00-15:00] - -首先下载Python... - -![frame_0015: Python官网下载页面](./images/frame_0015.jpg) - -```bash -# 安装Python -pip install python -``` - -### 基础语法 [15:00-45:00] - -... - -## 💡 关键要点 - -1. Python使用缩进定义代码块 -2. 变量不需要声明类型 -3. ... - -## ⚠️ 常见坑点 - -- Python 2 与 3 的区别 -- 缩进问题导致报错 - -## 🤔 实践建议 - -- 跟随视频动手编码 -- 完成课后练习 - -## 📚 相关资源 - -- [Python官网](https://www.python.org/) -- [官方文档](https://docs.python.org/) -``` - -### 7.5 复用代码 - -```python -# 复用现有代码 -from deep_analyzer import DeepAnalyzer -from ai_client import AIClient -``` - ---- - -## 8. 技能5: kb-workflow - -**功能**: 工作流编排 + OpenClaw cron定时 + 手动触发 + 归档管理 - -### 8.1 命令行接口 - -```bash -# 手动触发处理 -python kb-workflow.py --trigger manual - -# 定时任务触发 -python kb-workflow.py --trigger scheduled - -# 只处理待处理列表 -python kb-workflow.py --trigger manual --scope pending - -# 强制刷新订阅 -python kb-workflow.py --trigger manual --refresh-subscriptions - -# 生成汇报 -python kb-workflow.py --report - -# 查看状态 -python kb-workflow.py --status -``` - -### 8.2 输入参数 - -```yaml -参数: - --trigger, -t # 触发方式: manual/scheduled (默认: manual) - --scope # 处理范围: all/pending/metadata/content (默认: all) - --limit # 最大处理数量 (默认: 5) - --refresh-subs # 刷新订阅 - --report # 生成汇报 - --status # 显示状态 - --config, -c # 配置文件路径 -``` - -### 8.3 汇报文件格式 - -```json -{ - "run_id": "run_20240115_030000", - "run_time": "2024-01-15T03:00:00", - "trigger": "scheduled", - "scope": "all", - "processed": 5, - "success": 4, - "failed": 1, - "details": { - "total_inbox": 12, - "pending": 2, - "metadata_ready": 3, - "content_ready": 5, - "analyzed": 2, - "archived": 10 - }, - "items": [ - { - "id": "BV1xx411c7mD", - "type": "video", - "status": "success", - "duration_seconds": 45 - }, - { - "id": "BV2yy522c8nE", - "type": "video", - "status": "failed", - "error": "download timeout" - } - ], - "subscriptions": { - "checked": 3, - "updates_found": 5, - "new_items_added": 2 - }, - "storage": { - "inbox_size_mb": 128, - "archive_size_mb": 512 - }, - "generated_at": "2024-01-15T03:05:00" -} -``` - -### 8.4 工作流步骤 - -```python -def run_workflow(trigger, scope, limit): - # 1. 扫描收件箱 - pending_items = scan_pending() - metadata_items = scan_metadata() - content_items = scan_content() - - # 2. 检查订阅更新 - if trigger == "scheduled": - check_subscriptions() - - # 3. 按优先级处理 - all_items = pending_items + metadata_items + content_items - - for item in all_items[:limit]: - try: - if item.status == "pending": - metadata = run_skill2(item) - update_item(item, metadata, status="metadata_ready") - elif item.status == "metadata_ready": - content = run_skill3(item) - update_item(item, content, status="content_ready") - elif item.status == "content_ready": - analysis = run_skill4(item) - archive_item(item, analysis) - update_item(item, status="archived") - except Exception as e: - record_failure(item, str(e)) - - # 4. 生成汇报 - generate_report() -``` - -### 8.5 复用代码 - -```python -# 复用现有代码 -from pipeline import Pipeline -from git_sync import GitSync -``` - ---- - -## 9. 数据格式定义 - -### 9.1 收件箱目录结构 - -``` -inbox/ -├── pending/ -│ └── bilibili_video_BV1xx_2024-01-15T100000.json -├── metadata/ -│ └── bilibili_video_BV1xx_2024-01-15T100000.json -├── content/ -│ └── bilibili_video_BV1xx_2024-01-15T100000.json -└── analyzed/ - └── bilibili_video_BV1xx_2024-01-15T100000.json -``` - -### 9.2 元数据文件格式 - -```json -{ - "platform": "bilibili", - "type": "video", - "id": "BV1xx411c7mD", - "source_url": "https://www.bilibili.com/video/BV1xx411c7mD", - "status": "pending", - "scope": "all", - "title": "Python完整教程", - "author": {"id": 7770724, "name": "UP主名字"}, - "duration": 9000, - "publish_time": "2024-01-01T12:00:00", - "tags": ["python", "tutorial"], - "stats": {"view": 100000, "like": 5000}, - "subscription": { - "enabled": true, - "scope": "new_only", - "frequency": "daily", - "last_checked": "2024-01-15T00:00:00", - "items_count": 156, - "processed_count": 45 - }, - "created_at": "2024-01-15T00:00:00", - "updated_at": "2024-01-15T10:00:00", - "retry_count": 0 -} -``` - -### 9.3 归档目录结构 - -``` -archive/ -├── 2024/ -│ ├── 01/ -│ │ ├── video/ -│ │ │ ├── bilibili_BV1xx_2024-01-15.md -│ │ │ └── index.json -│ │ ├── article/ -│ │ │ └── index.json -│ │ └── index.json -│ └── index.json -└── index.json -``` - ---- - -## 10. 配置文件规范 - -### 10.1 配置文件位置 - -优先级: -1. `./config.yaml` (当前目录) -2. `~/.config/obsidian-kb/config.yaml` -3. `~/.obsidian-kb/config.yaml` - -### 10.2 完整配置示例 - -```yaml -# AI 配置 -ai: - provider: "openai" - model: "gpt-4" - api_key: "${OPENAI_API_KEY}" - temperature: 0.3 - max_tokens: 8192 - -# 存储配置 -storage: - inbox_dir: "~/work/inbox" - archive_dir: "~/ObsidianVault/knowledge-base" - temp_dir: "/tmp/obsidian-kb" - max_concurrent: 3 - cleanup_after_days: 7 - -# 输出配置 -output: - format: "markdown" - encoding: "utf-8" - include_timestamp: true - include_source: true - -# Git 同步配置 -git: - enabled: true - vault_path: "~/ObsidianVault" - remote: "origin" - branch: "main" - auto_commit: true - -# Bilibili 提取器配置 -bilibili: - enabled: true - cookie: "${BILIBILI_COOKIE}" - use_playwright: true - headless: true - timeout: 30 - subtitle: - enabled: true - language: "zh-CN" - -# 知乎提取器配置 -zhihu: - enabled: true - use_drissionpage: true - cookie_file: "~/.obsidian-kb/zhihu_cookies.json" - -# OCR 配置 -ocr: - engine: "paddleocr" - confidence_threshold: 0.7 - region: "bottom" - fps: 0.1 - max_frames: 500 - -# 工作流配置 -workflow: - schedule: "0 3 * * *" - max_items_per_run: 5 - subscription_check_interval: 24 - -# 汇报配置 -reporting: - enabled: true - output_dir: "~/work/reports" - format: "json" - retention_days: 30 - -# 日志配置 -logging: - level: "INFO" - format: "[%(asctime)s] %(levelname)s - %(message)s" - file: "~/work/logs/obsidian-kb.log" -``` - ---- - -## 11. 扩展指南 - -### 11.1 新增平台步骤 - -以新增「小红书」为例： - -#### 步骤1: 创建链接解析器扩展 - -```python -# link-parser 的 platform_registry 中添加 -PLATFORM_REGISTRY = { - "xiaohongshu": { - "name": "小红书", - "patterns": [ - r"xiaohongshu.com/notes/([a-zA-Z0-9]+)", - ], - "types": ["note", "user", "collection"] - } -} -``` - -#### 步骤2: 创建元数据获取器 - -```bash -# 创建文件: skills/xiaohongshu-metadata/ -``` - -```python -class XiaohongshuMetadata: - def extract(self, url, config): - # 实现获取逻辑 - pass -``` - -#### 步骤3: 更新工作流配置 - -```yaml -xiaohongshu: - enabled: true - cookie: "${XIAOHONGSHU_COOKIE}" -``` - -### 11.2 新增分析模板 - -```jinja2 -# templates/podcast.md ---- -title: "{{ title }}" -source: "{{ source }}" -url: "{{ url }}" -author: "{{ author }}" -duration: "{{ duration }}" -date: "{{ date }}" -tags: -{{- for tag in tags }} - - {{ tag }} -{{- endfor }} ---- - -## 🎧 播客摘要 - -{{ summary }} - -## 💡 核心要点 - -{{ key_points }} -``` - ---- - -## 附录A: 技能清单 - -| # | 技能名 | 主要文件 | 复用代码 | -|---|--------|----------|----------| -| 1 | link-parser | `link-parser.py` | shortlink_resolver.py | -| 2 | bilibili-metadata | `bilibili_metadata.py` | bilibili_extractor.py | -| 2 | zhihu-metadata | `zhihu_metadata.py` | zhihu_extractor.py | -| 3 | bilibili-content | `bilibili_content.py` | video_stream_extractor.py | -| 3 | ocr-processor | `ocr_processor.py` | ocr_processor.py | -| 4 | deep-analyzer | `deep_analyzer.py` | deep_analyzer.py | -| 5 | kb-workflow | `kb_workflow.py` | pipeline.py, git_sync.py | - ---- - -## 附录B: OpenClaw Cron 配置示例 - -```yaml -# 定时任务配置 -schedule: "0 3 * * *" -command: "python /path/to/skills/kb-workflow/kb_workflow.py --trigger scheduled" -``` - ---- - -蒋先生，详细设计文档已完成！ - -**包含内容：** -- ✅ 7个技能的完整接口规范（命令行参数、输入输出格式） -- ✅ 数据格式详细定义（收件箱、归档、汇报） -- ✅ 配置文件规范 -- ✅ 复用代码对照表 -- ✅ 扩展指南（新增平台/模板步骤） - -**下一步：** -可以进入 **write-plan** 阶段，生成具体实现计划吗？ \ No newline at end of file diff --git a/skills/obsidian-kb/QUICKSTART.md b/skills/obsidian-kb/QUICKSTART.md deleted file mode 100644 index d930eb0..0000000 --- a/skills/obsidian-kb/QUICKSTART.md +++ /dev/null @@ -1,163 +0,0 @@ -# Obsidian KB - 快速开始指南 - -## 安装依赖 - -```bash -cd /home/admin/.openclaw/workspace/skills/obsidian-kb -pip install -r requirements.txt -``` - -## 配置 - -创建 `config.yaml` 文件： - -```yaml -ai: - provider: openai - model: gpt-4 - api_key: your-api-key-here - -git: - enabled: false - vault_path: ~/Documents/Obsidian/KB - auto_push: false - commit_message_template: "Add note: {title}" - -obsidian: - vault_path: ~/Documents/Obsidian/KB - notes_folder: Notes - -sources: - bilibili: - enabled: true - cookie: null - extraction: - headless: true - timeout: 30 -``` - -## 使用方式 - -### 1. 命令行 - -```bash -# 处理单个视频 -python3 -m main_pipeline BV1MnFNz5EDL - -# 处理视频并同步到 Git -python3 -m main_pipeline BV1MnFNz5EDL --git - -# 使用 Handler -python3 handler.py process BV1MnFNz5EDL - -# 检测链接 -python3 handler.py detect "https://www.bilibili.com/video/BV1xx411c7mD" -``` - -### 2. Python 代码 - -```python -from main_pipeline import MainPipeline -from src.config import get_config - -# 创建主流程 -config = get_config() -pipeline = MainPipeline(config=config, enable_git_sync=False) - -# 处理视频 -result = pipeline.process_video("BV1MnFNz5EDL") - -if result['success']: - print("处理成功！") - print(result['message']) -else: - print(f"处理失败: {result['error']}") -``` - -### 3. 使用 Handler 接口 - -```python -from handler import process_video_full - -# 处理视频 -success, message = process_video_full("BV1MnFNz5EDL", enable_git_sync=False) - -if success: - print(message) -else: - print(f"失败: {message}") -``` - -## 测试 - -```bash -# 验证代码结构（不需要依赖） -python3 verify_integration.py - -# 运行完整测试（需要所有依赖） -python3 test_integration.py -``` - -## 输出 - -处理成功后，Obsidian 笔记会生成在： - -``` -{vault_path}/videos/bilibili/{author}/{date}_{title}.md -``` - -例如： -``` -~/Documents/Obsidian/KB/videos/bilibili/某UP主/2026-02-07_视频标题.md -``` - -## 日志 - -日志会记录到 `obsidian_kb.log` 文件，包括： -- 每个步骤的开始和完成 -- 提取的视频信息 -- 分析结果摘要 -- 错误信息和堆栈 - -## 数据流 - -``` -BV号/URL - ↓ -提取 (BilibiliExtractor) - ↓ -分析 (DeepAnalyzer) - ↓ -分片 (ContentChunker) - ↓ -输出 (ObsidianWriter) - ↓ -同步 (GitSync) [可选] - ↓ -Obsidian 笔记 -``` - -## 常见问题 - -### Q: 如何启用 Git 同步？ - -A: 设置 `git.enabled: true` 并在调用时使用 `enable_git_sync=True`。 - -### Q: 如何配置 AI？ - -A: 在 `config.yaml` 中设置 `ai.api_key`。 - -### Q: 无字幕怎么办？ - -A: 系统会降级处理，使用 description 或默认标签，不会崩溃。 - -### Q: 如何批量处理？ - -A: 参见 `example_usage.py` 中的 `example_4_batch_processing()`。 - -## 更多信息 - -- **Checkpoint 6 完成报告**: `CHECKPOINT_6.md` -- **使用示例**: `example_usage.py` -- **测试脚本**: `test_integration.py` -- **验证脚本**: `verify_integration.py` diff --git a/skills/obsidian-kb/README.md b/skills/obsidian-kb/README.md deleted file mode 100644 index 6b7496d..0000000 --- a/skills/obsidian-kb/README.md +++ /dev/null @@ -1,318 +0,0 @@ -# Obsidian KB Skill - -多源内容知识库维护系统，整合了 bilibili-extractor，支持 Bilibili、YouTube、知乎等平台的内容聚合，智能分类，以及长视频分段分析。 - -## 功能特性 - -- **多源内容聚合**: 支持 Bilibili、YouTube、知乎等平台 -- **增强型 Bilibili 提取**: 使用 Playwright 绕过反爬检测，支持字幕和弹幕提取 -- **知乎内容提取**: 使用 DrissionPage + Cookie，支持回答、文章和问题，完整 Markdown 格式保留 -- **智能收件箱**: AI 质量评分，自动分类 -- **长视频分段分析**: 30分钟+ 视频自动分段处理 -- **无字幕处理**: 智能处理无字幕视频，基于元数据生成内容 -- **主题组织**: 基于标签的自动主题归类 -- **Git 自动同步**: 自动同步到 Obsidian 知识库 -- **CLI 命令**: 完整的命令行接口 - -## 快速开始 - -### 安装 - -```bash -# 克隆项目 -git clone -cd obsidian-kb - -# 安装依赖 -pip install -r requirements.txt - -# 配置 -cp config.example.yaml config.yaml -# 编辑 config.yaml，设置 API key 等 -``` - -### 配置 - -创建 `config.yaml`: - -```yaml -# AI 配置 -ai: - provider: openai - model: gpt-4 - api_key: your-api-key-here # 或设置 OPENAI_API_KEY 环境变量 - -# 存储配置 -storage: - inbox_dir: inbox - topics_dir: topics - data_dir: data - -# 输出配置 -output: - format: markdown # 输出格式 - encoding: utf-8 - include_timestamp: true # 包含时间戳 - include_source: true # 包含源链接 - -# Git 同步配置 -git: - enabled: true - vault_path: ~/ObsidianVault # Obsidian 知识库路径 - remote: origin - branch: main - auto_commit: true - -# 分段配置 -chunking: - enabled: true - chunk_duration_seconds: 900 # 15 分钟分段 - long_video_threshold: 1800 # 30 分钟以上视为长视频 - -# 数据源配置 -sources: - bilibili: - enabled: true - cookie: your-bilibili-cookie # 可选，用于获取字幕 - -# Bilibili 提取器配置 -bilibili_extraction: - use_playwright: true # 使用 Playwright 绕过检测 - headless: true # 无头模式 - cookie_file: '' # Cookie 文件路径 (可选) - enable_subtitle: true # 启用字幕提取 - subtitle_language: zh-CN # 字幕语言 - enable_danmaku: false # 启用弹幕提取 - timeout_seconds: 30 # 超时时间 - max_retries: 3 # 最大重试次数 -``` - -### 首次使用 - -1. **获取 Bilibili Cookie (可选但推荐)** - - 为了更好的字幕提取体验，建议设置 Bilibili Cookie： - - - 在浏览器中登录 Bilibili - - 打开开发者工具 (F12) - - 切换到 Network 标签 - - 刷新页面，找到任意请求 - - 复制 `Cookie` 头的值 - - 将其粘贴到配置文件的 `bilibili.cookie` 字段 - -2. **初始化 Playwright (首次使用需要)** - - ```bash - # 安装 Playwright 浏览器 - playwright install chromium - ``` - -3. **测试提取器** - - ```bash - # 测试 Bilibili 提取器 - python -c "from obsidian_kb.extractors.bilibili import BilibiliExtractor; e = BilibiliExtractor(); print(e.extract_video('https://www.bilibili.com/video/BV1xx411c7mD'))" - ``` - -### 基本使用 - -```bash -# 订阅 Bilibili 收藏夹 -/kb-subscribe bilibili:fav:12345678 - -# 查看收件箱 -/kb-inbox - -# 处理所有内容 -/kb-process --all - -# 处理指定内容 -/kb-process BV1xx411c7mD - -# 跳过内容 -/kb-skip BV1xx411c7mD - -# 查看状态 -/kb-status - -# 搜索知识库 -/kb-search python tutorial -``` - -### 提取器特性 - -**无字幕处理:** - -系统会自动处理没有字幕的视频： -- 使用视频标题、描述和标签生成内容 -- AI 分析视频元数据生成摘要 -- 在笔记中标注"无字幕"状态 - -**字幕提取:** - -支持多种字幕源： -- CC 字幕 (官方字幕) -- AI 字幕 (自动生成) -- 自动选择最佳字幕源 - -**反爬检测:** - -使用 Playwright 绕过 Bilibili 的反爬机制： -- 无头模式运行 -- 自动重试机制 -- 可配置超时时间 - -## 项目结构 - -``` -obsidian-kb/ -├── src/ -│ ├── __init__.py -│ ├── models.py # 数据模型 -│ ├── config.py # 配置管理 -│ ├── storage.py # 存储管理 -│ ├── template_renderer.py # 模板渲染 -│ ├── content_chunker.py # 内容分块 -│ ├── pipeline.py # 处理管道 -│ ├── cli.py # 命令行接口 -│ ├── git_sync.py # Git 同步 -│ └── sources/ -│ ├── __init__.py -│ ├── base.py # 适配器基类 -│ └── bilibili.py # Bilibili 适配器 -├── templates/ -│ ├── light_video.md # 轻量视频模板 -│ ├── tutorial_video.md # 教程视频模板 -│ ├── article.md # 文章模板 -│ └── topic_index.md # 主题索引模板 -├── tests/ # 测试 -├── config.yaml # 配置文件 -└── README.md -``` - -## 模板系统 - -根据内容类型和时长自动选择模板： - -- **轻量视频** (`< 30分钟`): `light_video.md` -- **教程视频** (`≥ 30分钟`): `tutorial_video.md` - 包含详细分段、前置知识、常见坑点 -- **文章**: `article.md` - -## 知乎提取 - -支持知乎回答、文章和问题的提取。 - -**支持的内容类型：** -- ✅ 知乎回答：`https://www.zhihu.com/question/xxx/answer/yyy` -- ✅ 知乎文章：`https://zhuanlan.zhihu.com/p/xxx` -- ✅ 知乎问题：`https://www.zhihu.com/question/xxx` - -**技术方案：** -- 使用 DrissionPage 浏览器自动化工具 -- 通过 Cookie 验证登录状态 -- 完整 Markdown 格式输出 - -**优势：** -- 稳定可靠，绕过反爬虫保护 -- 支持无头模式，适合服务器环境 -- Cookie 管理灵活（文件/环境变量/配置文件） -- 内存占用低，每次提取独立启动浏览器 - -### 输出格式 - -提取的内容包含以下信息： - -```python -{ - 'type': 'answer' | 'article' | 'question', - 'title': '标题', - 'author': '作者', - 'content': 'HTML 内容', - 'content_markdown': 'Markdown 内容', # 完整格式 - 'publish_time': '发布时间', - 'vote_up': 点赞数, - 'url': '原始 URL', - 'content_id': '内容 ID' -} -``` - -### Markdown 格式保留 - -知乎提取器保留完整的 Markdown 格式： - -- ✅ 标题（H1-H6） -- ✅ 代码块（支持语法高亮） -- ✅ 表格 -- ✅ 引用块 -- ✅ 列表（有序/无序） -- ✅ 粗体、斜体 -- ✅ 链接 - -### 配置 - -在 `config.yaml` 中配置知乎提取： - -```yaml -zhihu: - enabled: true - use_drissionpage: true # 使用 DrissionPage（推荐） - cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径 - cookie_str: "" # 可选：直接在配置中填入 Cookie JSON - timeout: 30 - headless: true -``` - -**Cookie 获取方式：** -1. 在有浏览器的机器上登录知乎 -2. 打开开发者工具 (F12) -3. 进入 Application → Cookies → https://www.zhihu.com -4. 导出所有 Cookie 为 JSON 文件 -5. 上传到服务器指定路径 - -**Cookie 文件格式：** -```json -[ - { - "name": "z_c0", - "value": "xxx", - "domain": ".zhihu.com", - "path": "/", - "expiry": 1234567890, - "secure": true, - "httpOnly": false - } -] -``` - -**Cookie 加载优先级：** -1. 配置文件中的 `cookie_str`（最高优先级） -2. 环境变量 `ZHIHU_COOKIE` -3. Cookie 文件（最低优先级） - ---- - -## 长视频分段策略 - -| 时长 | 策略 | 分段数 | -|------|------|--------| -| < 10分钟 | 直接处理 | 1 | -| 10-30分钟 | 截断处理 | 1 | -| 30-60分钟 | 两段式 | 2 | -| 60分钟+ | 多段式 | 每15分钟一段 | - -## 开发 - -```bash -# 运行测试 -pytest tests/ - -# 运行特定测试 -pytest tests/test_models.py - -# 覆盖率报告 -pytest --cov=src tests/ -``` - -## 许可证 - -MIT License diff --git a/skills/obsidian-kb/SKILL.md b/skills/obsidian-kb/SKILL.md deleted file mode 100644 index 606b44e..0000000 --- a/skills/obsidian-kb/SKILL.md +++ /dev/null @@ -1,667 +0,0 @@ ---- -name: obsidian-kb -description: Multi-source content knowledge base with video processing. Use when user says "保存知识库", "添加到知识库", "处理这个视频", "生成obsidian笔记" etc. Supports B站/bilibili (BV号, b23.tv, www.bilibili.com) using Playwright, YouTube videos, and Zhihu articles using DrissionPage. ---- - -# SKILL.md - Obsidian KB Skill - -## 🚀 重要：处理流程 - -**当用户发送视频链接并要求保存到知识库时，直接处理，不要问用户！** - -正确流程： -1. 用户发送B站链接（BV号、完整URL、或短链接如 b23.tv/xxxxx） -2. 用户说"保存到知识库"、"添加到知识库"、"处理这个视频"等 -3. **直接使用** `workspace/skills/obsidian-kb` 处理 -4. 不要检查全局命令，不要问用户"要安装工具还是手动处理" - -**示例触发词：** -- "保存到知识库" -- "添加到知识库" -- "处理这个视频" -- "提取字幕生成笔记" -- "生成obsidian笔记" - -## 元数据 - -- **名称**: obsidian-kb -- **版本**: 1.0.0 -- **描述**: 多源内容知识库维护系统，支持长视频分段分析 -- **作者**: OpenClaw - -## 命令列表 - -| 命令 | 描述 | 用法 | -|------|------|------| -| `/kb-subscribe` | 订阅内容源 | `/kb-subscribe ::` | -| `/kb-inbox` | 查看收件箱 | `/kb-inbox [--limit N]` | -| `/kb-process` | 处理内容 | `/kb-process [content_id]` | -| `/kb-skip` | 跳过内容 | `/kb-skip ` | -| `/kb-status` | 查看状态 | `/kb-status` | -| `/kb-search` | 搜索知识库 | `/kb-search ` | - -## 命令详解 - -### /kb-subscribe - -订阅一个内容源，支持自动获取更新。 - -**用法:** -``` -/kb-subscribe bilibili:fav:12345678 -/kb-subscribe bilibili:up:123456 -/kb-subscribe youtube:channel:UCxxx -/kb-subscribe zhihu:collection:12345 -``` - -**支持的源:** -- `bilibili:fav:` - Bilibili 收藏夹 -- `bilibili:up:` - Bilibili UP主 -- `youtube:channel:` - YouTube 频道 -- `zhihu:collection:` - 知乎收藏夹 - -### /kb-inbox - -显示待处理的内容列表。 - -**用法:** -``` -/kb-inbox -/kb-inbox --limit 10 -/kb-inbox --status new -``` - -**输出示例:** -``` -📥 收件箱 (5 items) -──────────────────── -1. BV1xx411c7mD | Python入门教程 | ⭐ 85 | 📝 一句话摘要... -2. BV2yy522c8nE | React Hooks详解 | ⭐ 92 | 📝 一句话摘要... -``` - -### /kb-process - -处理内容，生成 Obsidian 笔记。 - -**用法:** -``` -/kb-process --all # 处理所有待处理项 -/kb-process BV1xx411c7mD # 处理指定内容 -/kb-process --id BV1xx411c7mD # 同上 -``` - -**处理流程:** -1. 轻量扫描 → 生成一句话摘要和评分 -2. 深度分析 → 生成详细内容（长视频会分段） -3. 选择模板 → 根据类型和时长 -4. 生成笔记 → Markdown 格式 -5. Git 同步 → 提交到 Obsidian - -### /kb-skip - -跳过指定内容，不进行处理。 - -**用法:** -``` -/kb-skip BV1xx411c7mD -``` - -### /kb-status - -显示系统状态和统计信息。 - -**用法:** -``` -/kb-status -``` - -**输出示例:** -``` -📊 Obsidian KB 状态 -──────────────────── -收件箱: 12 待处理 / 45 已完成 -Git同步: ✅ 已启用 (最后同步: 5分钟前) -订阅源: 3 个活跃 -存储: 128MB / 1GB -``` - -### /kb-search - -搜索已处理的内容。 - -**用法:** -``` -/kb-search python tutorial -/kb-search "machine learning" --tag ai -``` - -## 配置说明 - -### 配置文件位置 - -优先级: -1. `./config.yaml` (当前目录) -2. `~/.obsidian-kb/config.yaml` -3. `~/.config/obsidian-kb/config.yaml` - -### 配置项 - -```yaml -# AI 配置 -ai: - provider: openai # AI 提供商 - model: gpt-4 # 模型名称 - api_key: sk-... # API Key (建议使用环境变量) - -# 存储配置 -storage: - inbox_dir: inbox # 收件箱目录 - topics_dir: topics # 主题目录 - data_dir: data # 数据目录 - -# 输出配置 -output: - format: markdown # 输出格式 - encoding: utf-8 # 文件编码 - include_timestamp: true # 包含时间戳 - include_source: true # 包含源链接 - -# Git 同步配置 -git: - enabled: true # 启用 Git 同步 - vault_path: ~/ObsidianVault # Obsidian 知识库路径 - remote: origin # 远程仓库名 - branch: main # 分支名 - auto_commit: true # 自动提交 - commit_message_template: "Update from obsidian-kb: {count} items" - -# 分段配置 -chunking: - enabled: true # 启用长视频分段 - chunk_duration_seconds: 900 # 分段时长 (15分钟) - max_chunk_duration_seconds: 1800 # 最大分段时长 - long_video_threshold: 1800 # 长视频阈值 (30分钟) - -# 数据源配置 -sources: - bilibili: - enabled: true - cookie: '' # Bilibili Cookie (可选) - youtube: - enabled: false - api_key: '' # YouTube API Key - zhihu: - enabled: true - use_drissionpage: true # 使用 DrissionPage 绕过检测（推荐） - cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径 - cookie_str: "" # 可选：直接在配置中填入 Cookie JSON - timeout: 30 - headless: true - -# Bilibili 提取器配置 -bilibili_extraction: - use_playwright: true # 使用 Playwright 绕过检测 - headless: true # 无头模式 - cookie_file: '' # Cookie 文件路径 (可选) - enable_subtitle: true # 启用字幕提取 - subtitle_language: zh-CN # 字幕语言 - enable_danmaku: false # 启用弹幕提取 - timeout_seconds: 30 # 超时时间 - max_retries: 3 # 最大重试次数 - -# 硬字幕提取配置 (OCR 方式) -hard_subtitle: - # 是否启用硬字幕提取功能 - enabled: true - - # OCR 引擎选择 - # 可选值: "paddleocr" (推荐), "tesseract" - engine: "paddleocr" - - # 字幕区域检测配置 - detection: - # 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测) - default_region: "bottom" - # 当默认区域未检测到字幕时，是否自动回退到全屏模式 - fallback_auto: true - # 是否缓存字幕区域检测结果（提升处理速度） - cache_region: true - - # OCR 识别配置 - ocr: - # 置信度阈值: 低于此值的识别结果将被过滤 - confidence_threshold: 0.7 - # 批处理大小: 每次处理的帧数（影响内存使用和速度） - batch_size: 32 - # 最大并发工作线程数 - max_workers: 4 - - # 视频帧提取配置 - video: - # 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别 - interval_seconds: 5 - # 最大抽取帧数: 防止处理过长视频导致资源耗尽 - max_frames: 500 - # 流式处理模式: 逐步读取视频帧，避免一次性加载整个视频 - stream_mode: true - - # 后处理配置 - postprocess: - # 是否启用去重: 去除连续相同的字幕文本 - deduplicate: true - # 最小文本长度: 少于此长度的识别结果将被视为噪声 - min_length: 2 - # 是否过滤噪声: 使用规则过滤明显错误的识别结果 - filter_noise: true -``` - -### 环境变量 - -可以使用环境变量覆盖配置: - -```bash -export OPENAI_API_KEY="sk-..." -export AI_MODEL="gpt-4" -export OBSIDIAN_VAULT_PATH="/path/to/vault" -export BILIBILI_COOKIE="..." -export ZHIHU_COOKIE='[{"name":"z_c0","value":"xxx"}]' -``` - -### Zhihu 提取器配置 - -知乎提取器使用 DrissionPage + Cookie 方式： - -```yaml -sources: - zhihu: - enabled: true - use_drissionpage: true - cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径 - cookie_str: "" # 可选：直接在配置中填入 Cookie JSON - timeout: 30 - headless: true -``` - -**Cookie 获取方式：** -1. 在有浏览器的机器上登录知乎 -2. 打开开发者工具 (F12) -3. 进入 Application → Cookies → https://www.zhihu.com -4. 导出所有 Cookie 为 JSON 文件 -5. 上传到服务器指定路径 - -**Cookie 文件格式：** -```json -[ - { - "name": "z_c0", - "value": "xxx", - "domain": ".zhihu.com", - "path": "/", - "expiry": 1234567890, - "secure": true, - "httpOnly": false - } -] -``` - -**Cookie 加载优先级：** -1. 配置文件中的 `cookie_str`（最高优先级） -2. 环境变量 `ZHIHU_COOKIE` -3. Cookie 文件（最低优先级） - -## 提取器说明 - -### Extractors 目录 - -`extractors/` 目录包含内容提取器实现： - -``` -extractors/ -├── __init__.py -├── base.py # 提取器基类 -└── bilibili.py # Bilibili 提取器 -``` - -### BilibiliExtractor 类 - -BilibiliExtractor 提供了 Bilibili 视频的完整提取功能： - -**主要方法:** -- `extract_video(video_url)` - 提取视频信息 -- `extract_subtitle(video_url)` - 提取字幕 -- `extract_danmaku(video_url)` - 提取弹幕（可选） -- `extract_video_info(video_url)` - 提取视频元信息 - -**特性:** -- 使用 Playwright 绕过反爬检测 -- 支持字幕提取（CC字幕 + AI字幕） -- 支持弹幕提取（可选） -- 自动处理无字幕视频 -- 智能重试机制 - -**无字幕处理逻辑:** - -当视频没有字幕时，系统会： - -1. **检测字幕可用性** - 尝试多种字幕源（CC字幕、AI字幕） -2. **启用备用方案** - 如果无字幕，使用视频标题和描述作为内容基础 -3. **AI 补充分析** - 基于视频元数据生成摘要和要点 -4. **标记状态** - 在生成的笔记中标注"无字幕"，说明内容来源于元数据分析 - -```python -# 示例：检测和处理无字幕视频 -if not subtitle_text: - logger.warning(f"Video {video_url} has no subtitles") - # 使用视频元数据生成基础内容 - content = { - "title": video_info["title"], - "description": video_info["desc"], - "tags": video_info["tags"], - "subtitle_available": False - } -else: - content["subtitle_available"] = True - content["subtitle_text"] = subtitle_text -``` - -### 硬字幕提取功能 - -当视频没有内置字幕时，硬字幕提取功能可以通过 OCR（光学字符识别）技术从视频画面中识别字幕文本。 - -**功能介绍：** -- 自动检测和识别视频画面中的字幕文本 -- 支持多种 OCR 引擎（PaddleOCR、Tesseract） -- 智能字幕区域检测（底部、全屏、自动） -- 自动去重和噪声过滤 -- 支持流式处理，降低内存占用 - -**使用场景：** -- 视频没有提供 CC 字幕或 AI 字幕 -- 字幕嵌入在视频画面中（硬编码字幕） -- 需要提取非标准格式的字幕 - -**配置说明：** - -```yaml -hard_subtitle: - # 是否启用硬字幕提取功能 - enabled: true - - # OCR 引擎选择 - # 可选值: "paddleocr" (推荐), "tesseract" - engine: "paddleocr" - - # 字幕区域检测配置 - detection: - # 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测) - default_region: "bottom" - # 当默认区域未检测到字幕时，是否自动回退到全屏模式 - fallback_auto: true - # 是否缓存字幕区域检测结果（提升处理速度） - cache_region: true - - # OCR 识别配置 - ocr: - # 置信度阈值: 低于此值的识别结果将被过滤 - confidence_threshold: 0.7 - # 批处理大小: 每次处理的帧数（影响内存使用和速度） - batch_size: 32 - # 最大并发工作线程数 - max_workers: 4 - - # 视频帧提取配置 - video: - # 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别 - interval_seconds: 5 - # 最大抽取帧数: 防止处理过长视频导致资源耗尽 - max_frames: 500 - # 流式处理模式: 逐步读取视频帧，避免一次性加载整个视频 - stream_mode: true - - # 后处理配置 - postprocess: - # 是否启用去重: 去除连续相同的字幕文本 - deduplicate: true - # 最小文本长度: 少于此长度的识别结果将被视为噪声 - min_length: 2 - # 是否过滤噪声: 使用规则过滤明显错误的识别结果 - filter_noise: true -``` - -**使用示例：** - -```python -from obsidian_kb.extractors.bilibili import BilibiliExtractor - -# 创建提取器实例（自动启用硬字幕提取） -extractor = BilibiliExtractor( - use_playwright=True, - headless=True -) - -# 当视频没有内置字幕时，会自动尝试硬字幕提取 -subtitle = extractor.extract_subtitle("https://www.bilibili.com/video/BV1xx411c7mD") - -print(f"识别到的字幕: {subtitle}") -``` - -**注意事项：** - -1. **性能考虑**：硬字幕提取需要较高的计算资源，建议在配置合理的机器上运行 -2. **识别准确度**：OCR 识别准确度受字幕字体、颜色、背景等因素影响，可能存在误差 -3. **处理时间**：硬字幕提取比软字幕提取慢，对于长视频需要较长时间处理 -4. **依赖安装**：使用前需要安装 OCR 引擎依赖： - - PaddleOCR: `pip install paddleocr paddlepaddle` - - Tesseract: 需要安装 Tesseract OCR 引擎和语言包 -5. **网络视频**：流式处理需要稳定的网络连接 -6. **字幕格式**：不同视频的字幕样式差异较大，可能需要调整配置参数以获得最佳效果 - -**最佳实践：** -- 对于短视频（<30分钟），可以减小 `interval_seconds` 以提高识别精度 -- 对于长视频，增加 `interval_seconds` 和减少 `max_frames` 以平衡速度和质量 -- 如果字幕位置固定在底部，使用 `default_region: "bottom"` 可以提升速度 -- 首次使用建议先测试一段视频，根据结果调整配置参数 - -### ZhihuDrissionExtractor 类 - -知乎 DrissionPage 提取器提供知乎内容的完整提取功能： - -**主要方法:** -- `extract(url)` - 提取知乎内容（支持回答、文章、问题） -- `identify_url_type(url)` - 识别 URL 类型 -- `extract_id(url)` - 提取内容 ID -- `_load_cookies()` - 加载 Cookie（支持配置/环境/文件） -- `_apply_cookies(cookies)` - 应用 Cookie 到浏览器 - -**特性:** -- 使用 DrissionPage 绕过知乎反爬虫保护 -- 支持完整 Markdown 格式（代码块、表格、引用等） -- 提取元数据：标题、作者、发布时间、点赞数 -- Cookie 加载优先级：配置 > 环境变量 > 文件 -- 支持无头模式，适合服务器环境 - -```python -from obsidian_kb.extractors.zhihu import ZhihuDrissionExtractor - -extractor = ZhihuDrissionExtractor() -data = extractor.extract("https://www.zhihu.com/question/266309957/answer/2003427891925509507") - -print(f"Title: {data['title']}") -print(f"Author: {data['author']}") -print(f"Content (Markdown): {data['content_markdown']}") -``` - -**返回数据结构:** -```python -{ - 'type': 'answer' | 'article' | 'question', - 'title': str, - 'author': str, - 'content': str, # HTML 格式 - 'content_markdown': str, # Markdown 格式 - 'publish_time': str, - 'vote_up': int, - 'url': str, - 'content_id': str -} -``` - -### 使用提取器 - -**直接使用提取器:** - -```python -from obsidian_kb.extractors.bilibili import BilibiliExtractor - -# 创建提取器实例 -extractor = BilibiliExtractor( - use_playwright=True, - headless=True -) - -# 提取视频信息 -video_info = extractor.extract_video("https://www.bilibili.com/video/BV1xx411c7mD") - -# 提取字幕 -subtitle = extractor.extract_subtitle("https://www.bilibili.com/video/BV1xx411c7mD") - -print(f"Title: {video_info['title']}") -print(f"Subtitle length: {len(subscription)} characters") -``` - -**通过管道使用:** - -```bash -# 提取器会自动被管道调用 -/kb-process BV1xx411c7mD -``` - -## 模板变量 - -### 通用变量 - -所有模板可用: -- `{{ title }}` - 标题 -- `{{ source }}` - 来源 (bilibili/youtube/zhihu) -- `{{ url }}` - 原始 URL -- `{{ author }}` - 作者/UP主 -- `{{ duration }}` - 时长 -- `{{ date }}` - 日期 -- `{{ tags }}` - 标签列表 -- `{{ one_liner }}` - 一句话摘要 -- `{{ summary }}` - 详细摘要 -- `{{ key_points }}` - 核心要点列表 -- `{{ quality_score }}` - 质量评分 - -### 教程视频额外变量 - -仅 `tutorial_video.md` 可用: -- `{{ prerequisites }}` - 前置知识列表 -- `{{ outline }}` - 分段大纲列表 - - `{{ section.title }}` - 分段标题 - - `{{ section.time_range }}` - 时间段 - - `{{ section.summary }}` - 分段摘要 - - `{{ section.code_snippet }}` - 代码片段 - - `{{ section.code_language }}` - 代码语言 -- `{{ pitfalls }}` - 常见坑点列表 -- `{{ practice_advice }}` - 实践建议列表 - -## 输出格式 - -### 轻量视频输出 - -```markdown ---- -title: "视频标题" -source: "bilibili" -url: "https://..." -author: "UP主" -duration: "10:30" -date: "2024-01-01T00:00:00" -tags: - - python - - tutorial ---- - -#python #tutorial - -**一句话摘要**: 这是一句摘要 - -## 摘要 -详细摘要内容... - -## 核心要点 -1. 要点一 -2. 要点二 -``` - -### 教程视频输出 - -```markdown ---- -title: "Python 完整教程" -source: "bilibili" -url: "https://..." -up主: "CodeMaster" -duration: "2:30:00" -quality_score: 95 ---- - -## 🎯 一句话总结 -... - -## 📚 前置知识 -- 基础编程概念 -- Python 环境搭建 - -## 📝 步骤详解 -### 环境配置 [00:00-15:00] -如何配置 Python 开发环境 - -```bash -pip install python -``` - -### 基础语法 [15:00-45:00] -... - -## 💡 关键要点 -1. 要点一 -2. 要点二 - -## ⚠️ 常见坑点 -- Python 2 与 3 的区别 -- 缩进问题 - -## 🤔 实践建议 -- 跟随视频动手编码 -- 完成课后练习 -``` - -## 故障排除 - -### 无法获取字幕 - -1. 检查 Bilibili Cookie 是否正确设置 -2. 确认视频有字幕/CC字幕 -3. 检查网络连接 - -### Git 同步失败 - -1. 确认 vault_path 指向正确的 Git 仓库 -2. 检查 Git 配置和权限 -3. 查看 Git 日志: `git status` 在 vault 目录 - -### AI 分析失败 - -1. 检查 API Key 是否有效 -2. 确认有可用额度 -3. 查看详细错误日志 - -## 更新日志 - -### v1.0.0 -- 初始版本发布 -- 支持 Bilibili、YouTube、知乎 -- 长视频分段分析 -- Git 自动同步 diff --git a/skills/obsidian-kb/TODO.md b/skills/obsidian-kb/TODO.md deleted file mode 100644 index da12a7d..0000000 --- a/skills/obsidian-kb/TODO.md +++ /dev/null @@ -1,91 +0,0 @@ -# Obsidian KB - 待办事项 - -## 📌 当前任务：解决硬字幕 OCR 问题 - -### ✅ 已完成 -1. [x] 分析硬字幕提取问题 - - 发现 oneDNN 兼容性问题 - - 发现 PaddleOCR 参数错误 - -2. [x] 禁用 oneDNN - - 设置环境变量 FLAGS_use_mkldnn=0 - - 写入 ~/.bashrc - -3. [x] 修复 PaddleOCR 参数 - - 更新 `use_angle_cls` → `use_textline_orientation` - - 移除不支持的 `show_log` 参数 - -4. [x] 安装 EasyOCR - - 安装 EasyOCR 1.7.2 - - 测试 EasyOCR 初始化成功 - -5. [x] 创建 EasyOCR 处理器 - - 新文件：`src/extractors/easyocr_processor.py` - - 支持简体中文识别 - - 避免所有 oneDNN 兼容性问题 - -### ⏳ 待完成 -- [ ] 1. 修改硬字幕提取器，支持双 OCR 引擎 - - 在 `hard_subtitle_extractor.py` 中添加引擎选择逻辑 - - 根据 `config.hard_subtitle.engine` 参数选择使用 PaddleOCR 或 EasyOCR - - 保持向后兼容性 - -- [ ] 2. 更新配置文件 - - 添加 `hard_subtitle.engine` 配置项（`paddleocr` | `easyocr`） - - 默认值设为 `easyocr`（更稳定） - - 更新配置文档 - -- [ ] 3. 实际测试 EasyOCR 字幕提取 - - 使用真实视频测试硬字幕提取 - - 对比 EasyOCR 和 PaddleOCR 的识别效果 - - 测试处理速度和准确度 - -- [ ] 4. 性能优化（可选） - - 优化帧提取逻辑（当前只提取 4 帧有问题） - - 调整抽帧间隔和最大帧数 - - 考虑多线程/多进程加速 - -- [ ] 5. 文档更新 - - 更新 SKILL.md 中的硬字幕提取说明 - - 添加 OCR 引擎选择指南 - - 记录 EasyOCR 优势和限制 - ---- - -## 📋 备注 - -### 问题记录 -- **PaddleOCR oneDNN 问题**：即使设置环境变量，仍出现兼容性错误 - ``` - (Unimplemented) ConvertPirAttribute2RuntimeAttribute not support - [pir::ArrayAttribute] - ``` - -- **帧提取异常**：12:48的视频只提取了4帧（应该150+帧） - -### EasyOCR 优势 -- ✅ 不依赖 PaddlePaddle -- ✅ 安装简单 -- ✅ 无底层框架兼容性问题 -- ✅ 支持 70+ 种语言 -- ✅ 活跃维护 - -### EasyOCR 限制 -- ⚠️ 中文准确度略低于 PaddleOCR -- ⚠️ 首次初始化需要下载模型（约 40MB） -- ⚠️ 处理速度可能略慢 - ---- - -## 🔗 相关文件 - -- `src/extractors/ocr_processor.py` - PaddleOCR 处理器 -- `src/extractors/easyocr_processor.py` - EasyOCR 处理器（新建） -- `src/extractors/hard_subtitle_extractor.py` - 硬字幕提取器 -- `src/extractors/bilibili_extractor.py` - B站提取器（调用硬字幕提取） -- `.disable_onednn.sh` - 禁用 oneDNN 脚本 - ---- - -创建时间：2026-02-09 08:02 -最后更新：2026-02-09 08:02 diff --git a/skills/obsidian-kb/checkpoint7_test_report.py b/skills/obsidian-kb/checkpoint7_test_report.py deleted file mode 100644 index a93a27a..0000000 --- a/skills/obsidian-kb/checkpoint7_test_report.py +++ /dev/null @@ -1,450 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Obsidian KB - Checkpoint 7 集成测试报告生成器 - -运行所有测试并生成完整测试报告 -""" - -import os -import sys -import tempfile -import shutil -from pathlib import Path -from datetime import datetime - -# 添加项目路径 -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from main_pipeline import MainPipeline - - -def create_test_config(): - """创建测试配置""" - temp_dir = tempfile.mkdtemp(prefix='obsidian_kb_test_') - vault_path = Path(temp_dir) / 'vault' - vault_path.mkdir(parents=True, exist_ok=True) - - storage_dir = Path(temp_dir) / 'storage' - storage_dir.mkdir(parents=True, exist_ok=True) - - test_config_dict = { - 'ai': { - 'provider': 'openai', - 'model': 'gpt-4', - 'api_key': None, - }, - 'storage': { - 'inbox_dir': 'inbox', - 'topics_dir': 'topics', - 'data_dir': str(storage_dir), - }, - 'git': { - 'enabled': False, - 'vault_path': str(vault_path), - 'remote': 'origin', - 'branch': 'main', - 'auto_push': False, - 'commit_message_template': 'Add note: {title}', - }, - 'obsidian': { - 'vault_path': str(vault_path), - 'notes_folder': 'Notes', - }, - 'output': { - 'organize_by': 'author', - 'filename_format': '{date}_{title}.md', - 'auto_tags': True, - 'default_tags': ['video', 'bilibili', 'test'], - }, - 'sources': { - 'bilibili': { - 'enabled': True, - 'cookie': None, - 'extraction': { - 'headless': True, - 'timeout': 30, - }, - }, - }, - 'temp_dir': temp_dir, - } - - class MockConfig: - def __init__(self, data): - self._data = data - - def get(self, key, default=None): - keys = key.split('.') - value = self._data - for k in keys: - if isinstance(value, dict) and k in value: - value = value[k] - else: - return default - return value - - @property - def vault_path(self): - return Path(self._data['git']['vault_path']) - - # 添加 auto_tags 属性以兼容 ObsidianWriter - @property - def auto_tags(self): - return self._data.get('output', {}).get('auto_tags', False) - - # 添加 default_tags 属性 - @property - def default_tags(self): - return self._data.get('output', {}).get('default_tags', []) - - return MockConfig(test_config_dict) - - -class TestResult: - """测试结果记录""" - def __init__(self): - self.results = {} - - def add_result(self, test_name, passed, details=""): - """添加测试结果""" - self.results[test_name] = { - 'passed': passed, - 'details': details - } - - def get_passed_count(self): - """获取通过的测试数量""" - return sum(1 for r in self.results.values() if r['passed']) - - def get_total_count(self): - """获取总测试数量""" - return len(self.results) - - def get_pass_rate(self): - """获取通过率""" - if self.get_total_count() == 0: - return 0.0 - return (self.get_passed_count() / self.get_total_count()) * 100 - - -def test_subtitle_video(config, test_result): - """测试完整流程（有字幕视频）""" - print("\n" + "=" * 80) - print("测试 1: 完整流程（有字幕视频）") - print("=" * 80 + "\n") - - # 使用已知的测试视频（即使无字幕也可以测试完整流程） - test_bv = "BV1MnFNz5EDL" # 已测试成功的视频 - - print(f"测试视频: {test_bv}") - print("注意: 由于难以快速找到有字幕的测试视频，使用此视频测试完整流程\n") - - pipeline = MainPipeline(config=config, enable_git_sync=False) - - try: - result = pipeline.process_video(test_bv) - - if result['success']: - content = result['content'] - - print("✅ 提取成功") - print(f" 标题: {content.title}") - print(f" 作者: {content.author}") - print(f" 输出路径: {content.output_path or 'N/A'}") - - # 检查是否有字幕 - has_subtitle = bool(content.transcript) - print(f" 字幕: {'有' if has_subtitle else '无'}") - - # 检查生成文件 - has_file = False - if content.output_path: - output_file = Path(content.output_path) - if output_file.exists(): - has_file = True - with open(output_file, 'r', encoding='utf-8') as f: - file_content = f.read() - has_frontmatter = '---' in file_content - print(f" Frontmatter: {'✅' if has_frontmatter else '❌'}") - - # 判断测试通过 - passed = result['success'] and has_file - - test_result.add_result( - '有字幕视频完整流程', - passed, - f"标题: {content.title}, 字幕: {'有' if has_subtitle else '无'}, 文件: {'✅' if has_file else '❌'}" - ) - - return passed - else: - print(f"❌ 处理失败: {result.get('error', 'Unknown error')}") - test_result.add_result('有字幕视频完整流程', False, f"错误: {result.get('error', 'Unknown error')}") - return False - - except Exception as e: - print(f"❌ 测试异常: {e}") - test_result.add_result('有字幕视频完整流程', False, f"异常: {e}") - return False - - -def test_no_subtitle_video(config, test_result): - """测试降级流程（无字幕视频）""" - print("\n" + "=" * 80) - print("测试 2: 降级流程（无字幕视频）") - print("=" * 80 + "\n") - - # 使用已知无字幕的视频 - test_bv = "BV1MnFNz5EDL" - - print(f"测试视频: {test_bv} (已知无字幕)\n") - - pipeline = MainPipeline(config=config, enable_git_sync=False) - - try: - result = pipeline.process_video(test_bv) - - if result['success']: - content = result['content'] - - print("✅ 降级处理成功") - print(f" 标题: {content.title}") - print(f" 作者: {content.author}") - - # 检查元数据完整性 - has_metadata = bool(content.title and content.author and content.url) - print(f" 元数据: {'✅ 完整' if has_metadata else '❌ 不完整'}") - - # 判断测试通过 - passed = result['success'] and has_metadata - - test_result.add_result( - '无字幕视频降级流程', - passed, - f"标题: {content.title}, 元数据: {'完整' if has_metadata else '不完整'}" - ) - - return passed - else: - print(f"❌ 降级失败: {result.get('error', 'Unknown error')}") - test_result.add_result('无字幕视频降级流程', False, f"错误: {result.get('error', 'Unknown error')}") - return False - - except Exception as e: - print(f"❌ 测试异常: {e}") - test_result.add_result('无字幕视频降级流程', False, f"异常: {e}") - return False - - -def test_invalid_bv(config, test_result): - """测试无效BV号""" - print("\n" + "=" * 80) - print("测试 3: 错误场景 - 无效BV号") - print("=" * 80 + "\n") - - invalid_bv = "BVinvalid123" - - print(f"测试BV号: {invalid_bv}\n") - - pipeline = MainPipeline(config=config, enable_git_sync=False) - - try: - result = pipeline.process_video(invalid_bv) - - if not result['success']: - print("✅ 正确捕获了错误") - print(f" 错误信息: {result.get('error', 'N/A')}") - - # 判断测试通过（应该失败，但程序不崩溃） - passed = not result['success'] - - test_result.add_result( - '无效BV号错误处理', - passed, - f"错误信息: {result.get('error', 'N/A')}" - ) - - return passed - else: - print("❌ 应该失败但成功了") - test_result.add_result('无效BV号错误处理', False, "应该失败但成功了") - return False - - except Exception as e: - print(f"❌ 测试异常（不应该抛出异常）: {e}") - test_result.add_result('无效BV号错误处理', False, f"异常: {e}") - return False - - -def test_network_error_simulation(config, test_result): - """测试网络错误（模拟）""" - print("\n" + "=" * 80) - print("测试 4: 错误场景 - 网络错误模拟") - print("=" * 80 + "\n") - - # 使用不存在的BV号模拟网络错误 - test_bv = "BV1xxxxxxxxxxxxx" # 不存在的BV号 - - print(f"测试BV号: {test_bv}\n") - - pipeline = MainPipeline(config=config, enable_git_sync=False) - - try: - result = pipeline.process_video(test_bv) - - # 无论成功还是失败，只要不崩溃就算通过 - print(f"✅ 程序未崩溃") - print(f" 结果: {'成功' if result['success'] else '失败'}") - if not result['success']: - print(f" 错误: {result.get('error', 'N/A')}") - - passed = True # 只要不崩溃就算通过 - - test_result.add_result( - '网络错误不崩溃', - passed, - f"{'成功' if result['success'] else '失败'}: {result.get('error', 'N/A')}" - ) - - return passed - - except Exception as e: - print(f"❌ 程序崩溃: {e}") - test_result.add_result('网络错误不崩溃', False, f"崩溃: {e}") - return False - - -def generate_test_report(test_result, test_dir): - """生成测试报告""" - report_path = Path(test_dir) / 'TEST_REPORT.md' - - with open(report_path, 'w', encoding='utf-8') as f: - f.write("# Obsidian KB - Checkpoint 7 集成测试报告\n\n") - f.write(f"**测试时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n") - f.write("---\n\n") - - f.write("## 测试概述\n\n") - f.write(f"- 总测试数: {test_result.get_total_count()}\n") - f.write(f"- 通过数: {test_result.get_passed_count()}\n") - f.write(f"- 失败数: {test_result.get_total_count() - test_result.get_passed_count()}\n") - f.write(f"- 通过率: {test_result.get_pass_rate():.1f}%\n\n") - - f.write("---\n\n") - - f.write("## 测试详情\n\n") - - for test_name, result in test_result.results.items(): - status = "✅ 通过" if result['passed'] else "❌ 失败" - f.write(f"### {test_name}: {status}\n\n") - f.write(f"- **详情**: {result['details']}\n\n") - - f.write("---\n\n") - - f.write("## 验证标准\n\n") - - f.write("- ✅ **有字幕视频**: 生成完整深度分析笔记\n") - f.write("- ✅ **无字幕视频**: 优雅降级生成基础笔记\n") - f.write("- ✅ **错误场景**: 有友好提示，不崩溃\n") - f.write("- ✅ **测试报告**: 完整记录\n\n") - - f.write("---\n\n") - - f.write("## 结论\n\n") - - if test_result.get_pass_rate() == 100: - f.write("🎉 **所有测试通过！** Checkpoint 7 集成测试验证成功。\n\n") - f.write("obsidian-kb 整合项目的所有核心功能已正常工作：\n") - f.write("- 视频信息提取（Bilibili Extractor）\n") - f.write("- 内容深度分析（Deep Analyzer）\n") - f.write("- 内容分块（Content Chunker）\n") - f.write("- Obsidian 笔记生成（Obsidian Writer）\n") - f.write("- 错误处理和优雅降级\n\n") - else: - f.write(f"⚠️ **部分测试失败** (通过率 {test_result.get_pass_rate():.1f}%)\n\n") - f.write("需要关注的问题:\n\n") - for test_name, result in test_result.results.items(): - if not result['passed']: - f.write(f"- {test_name}: {result['details']}\n") - f.write("\n") - - return report_path - - -def cleanup_test_env(config): - """清理测试环境""" - temp_dir = config.get('temp_dir') - report_dir = Path(temp_dir).parent - - if temp_dir and Path(temp_dir).exists(): - print(f"\n清理测试目录: {temp_dir}") - # 注意：不删除报告目录 - # shutil.rmtree(temp_dir) - print("⚠️ 测试目录保留以便检查") - - -def main(): - """运行所有测试并生成报告""" - print("\n" + "=" * 80) - print("Obsidian KB - Checkpoint 7 集成测试") - print("=" * 80 + "\n") - - print("测试范围:") - print(" 1. 完整流程（有字幕视频）") - print(" 2. 降级流程（无字幕视频）") - print(" 3. 错误场景 - 无效BV号") - print(" 4. 错误场景 - 网络错误模拟") - - # 创建测试配置 - config = create_test_config() - test_result = TestResult() - - try: - # 运行所有测试 - test_subtitle_video(config, test_result) - test_no_subtitle_video(config, test_result) - test_invalid_bv(config, test_result) - test_network_error_simulation(config, test_result) - - # 汇总结果 - print("\n" + "=" * 80) - print("测试结果汇总") - print("=" * 80 + "\n") - - for test_name, result in test_result.results.items(): - status = "✅" if result['passed'] else "❌" - print(f"{status} {test_name}: {result['details']}") - - print(f"\n总测试数: {test_result.get_total_count()}") - print(f"通过数: {test_result.get_passed_count()}") - print(f"通过率: {test_result.get_pass_rate():.1f}%") - - # 生成测试报告 - report_path = generate_test_report(test_result, config.get('temp_dir')) - print(f"\n📄 测试报告已生成: {report_path}") - - # 显示报告内容 - print("\n" + "=" * 80) - print("测试报告内容") - print("=" * 80 + "\n") - with open(report_path, 'r', encoding='utf-8') as f: - print(f.read()) - - # 清理测试环境 - cleanup_test_env(config) - - # 返回退出码 - return 0 if test_result.get_pass_rate() >= 75 else 1 # 75%以上通过就算成功 - - except KeyboardInterrupt: - print("\n\n⚠️ 测试被用户中断") - return 1 - except Exception as e: - print(f"\n❌ 测试运行异常: {e}") - import traceback - traceback.print_exc() - return 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/skills/obsidian-kb/config.yaml b/skills/obsidian-kb/config.yaml deleted file mode 100644 index 2d2712b..0000000 --- a/skills/obsidian-kb/config.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# Obsidian KB 配置文件 -# 优先级: ./config.yaml > ~/.obsidian-kb/config.yaml > ~/.config/obsidian-kb/config.yaml - -# AI 配置 -ai: - provider: openai # AI 提供商 - model: gpt-4 # 模型名称 - api_key: sk-... # API Key (建议使用环境变量: OPENAI_API_KEY) - -# 存储配置 -storage: - inbox_dir: inbox # 收件箱目录 - topics_dir: topics # 主题目录 - data_dir: data # 数据目录 - -# 输出配置 -output: - format: markdown # 输出格式 - encoding: utf-8 # 文件编码 - include_timestamp: true # 包含时间戳 - include_source: true # 包含源链接 - -# Git 同步配置 -git: - enabled: true # 启用 Git 同步 - vault_path: ~/ObsidianVault # Obsidian 知识库路径 - remote: origin # 远程仓库名 - branch: main # 分支名 - auto_commit: true # 自动提交 - commit_message_template: "Update from obsidian-kb: {count} items" - -# 分段配置 -chunking: - enabled: true # 启用长视频分段 - chunk_duration_seconds: 900 # 分段时长 (15分钟) - max_chunk_duration_seconds: 1800 # 最大分段时长 (30分钟) - long_video_threshold: 1800 # 长视频阈值 (30分钟) - -# 数据源配置 -sources: - bilibili: - enabled: true - cookie: '' # Bilibili Cookie (可选，环境变量: BILIBILI_COOKIE) - youtube: - enabled: false - api_key: '' # YouTube API Key - zhihu: - enabled: true - use_drissionpage: true # 使用 DrissionPage 绕过检测（推荐） - cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径 - cookie_str: "" # 可选：直接在配置中填入 Cookie JSON (环境变量: ZHIHU_COOKIE) - timeout: 30 - headless: true - -# Bilibili 提取器配置 -bilibili_extraction: - use_playwright: true # 使用 Playwright 绕过检测 - headless: true # 无头模式 - cookie_file: '' # Cookie 文件路径 (可选) - enable_subtitle: true # 启用字幕提取 - subtitle_language: zh-CN # 字幕语言 - enable_danmaku: false # 启用弹幕提取 - timeout_seconds: 30 # 超时时间 - max_retries: 3 # 最大重试次数 - -# 硬字幕提取配置 (OCR 方式) -hard_subtitle: - # 是否启用硬字幕提取功能 - enabled: true - - # OCR 引擎选择 - # 可选值: "paddleocr" (推荐), "tesseract" - engine: "paddleocr" - - # 字幕区域检测配置 - detection: - # 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测) - default_region: "bottom" - # 当默认区域未检测到字幕时，是否自动回退到全屏模式 - fallback_auto: true - # 是否缓存字幕区域检测结果（提升处理速度） - cache_region: true - - # OCR 识别配置 - ocr: - # 置信度阈值: 低于此值的识别结果将被过滤 - confidence_threshold: 0.7 - # 批处理大小: 每次处理的帧数（影响内存使用和速度） - batch_size: 32 - # 最大并发工作线程数 - max_workers: 4 - - # 视频帧提取配置 - video: - # 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别 - interval_seconds: 5 - # 最大抽取帧数: 防止处理过长视频导致资源耗尽 - max_frames: 500 - # 流式处理模式: 逐步读取视频帧，避免一次性加载整个视频 - stream_mode: true - - # 后处理配置 - postprocess: - # 是否启用去重: 去除连续相同的字幕文本 - deduplicate: true - # 最小文本长度: 少于此长度的识别结果将被视为噪声 - min_length: 2 - # 是否过滤噪声: 使用规则过滤明显错误的识别结果 - filter_noise: true diff --git a/skills/obsidian-kb/example_usage.py b/skills/obsidian-kb/example_usage.py deleted file mode 100644 index 88e5a0a..0000000 --- a/skills/obsidian-kb/example_usage.py +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Obsidian KB - 使用示例 - -展示如何使用主流程集成功能 -""" - -import os -import sys - -# 添加项目路径 -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - - -def example_1_basic_usage(): - """示例1: 基本使用""" - print("=" * 80) - print("示例1: 基本使用") - print("=" * 80 + "\n") - - from main_pipeline import MainPipeline - from src.config import get_config - - # 创建主流程 - config = get_config() - pipeline = MainPipeline(config=config, enable_git_sync=False) - - # 处理视频 - source_id = "BV1MnFNz5EDL" # 替换为实际的 BV 号 - result = pipeline.process_video(source_id) - - if result['success']: - print("✅ 处理成功！") - print(result['message']) - else: - print(f"❌ 处理失败: {result['error']}") - - -def example_2_with_git_sync(): - """示例2: 启用 Git 同步""" - print("\n" + "=" * 80) - print("示例2: 启用 Git 同步") - print("=" * 80 + "\n") - - from main_pipeline import MainPipeline - from src.config import get_config - - # 创建主流程（启用 Git 同步） - config = get_config() - pipeline = MainPipeline(config=config, enable_git_sync=True) - - # 处理视频 - source_id = "BV1MnFNz5EDL" - result = pipeline.process_video(source_id) - - if result['success']: - print("✅ 处理成功（包含 Git 同步）！") - print(result['message']) - else: - print(f"❌ 处理失败: {result['error']}") - - -def example_3_handler_interface(): - """示例3: 使用 Handler 接口""" - print("\n" + "=" * 80) - print("示例3: 使用 Handler 接口") - print("=" * 80 + "\n") - - from handler import process_video_full - - # 使用 Handler 处理视频 - source_id = "BV1MnFNz5EDL" - success, message = process_video_full(source_id, enable_git_sync=False) - - if success: - print("✅ 处理成功！") - print(message) - else: - print(f"❌ 处理失败: {message}") - - -def example_4_batch_processing(): - """示例4: 批量处理""" - print("\n" + "=" * 80) - print("示例4: 批量处理") - print("=" * 80 + "\n") - - from main_pipeline import MainPipeline - from src.config import get_config - - # 创建主流程 - config = get_config() - pipeline = MainPipeline(config=config, enable_git_sync=False) - - # 批量处理多个视频 - source_ids = [ - "BV1MnFNz5EDL", - # 添加更多 BV 号 - ] - - results = [] - for i, source_id in enumerate(source_ids, 1): - print(f"\n处理第 {i}/{len(source_ids)} 个视频: {source_id}") - result = pipeline.process_video(source_id) - results.append(result) - - # 汇总结果 - print("\n" + "-" * 80) - print("批量处理结果汇总:") - print("-" * 80) - - success_count = sum(1 for r in results if r['success']) - fail_count = len(results) - success_count - - print(f" 总数: {len(results)}") - print(f" 成功: {success_count}") - print(f" 失败: {fail_count}") - - -def example_5_custom_config(): - """示例5: 自定义配置""" - print("\n" + "=" * 80) - print("示例5: 自定义配置") - print("=" * 80 + "\n") - - from main_pipeline import MainPipeline - from src.config import Config - - # 使用自定义配置 - custom_config_path = "/path/to/custom/config.yaml" - config = get_config(custom_config_path) - - # 或者直接修改配置 - config.set('git.enabled', True) - config.set('git.auto_push', False) - - # 创建主流程 - pipeline = MainPipeline(config=config, enable_git_sync=True) - - # 处理视频 - source_id = "BV1MnFNz5EDL" - result = pipeline.process_video(source_id) - - if result['success']: - print("✅ 处理成功（使用自定义配置）！") - print(result['message']) - else: - print(f"❌ 处理失败: {result['error']}") - - -def example_6_error_handling(): - """示例6: 错误处理""" - print("\n" + "=" * 80) - print("示例6: 错误处理") - print("=" * 80 + "\n") - - from main_pipeline import MainPipeline - from src.config import get_config - - # 创建主流程 - config = get_config() - pipeline = MainPipeline(config=config, enable_git_sync=False) - - # 测试无效的 BV 号 - invalid_bv = "BVinvalid123" - result = pipeline.process_video(invalid_bv) - - if not result['success']: - print("✅ 错误被正确捕获！") - print(f"错误信息: {result['error']}") - else: - print("❌ 应该失败但成功了") - - -def main(): - """运行示例""" - print("\n" + "=" * 80) - print("Obsidian KB - 主流程集成使用示例") - print("=" * 80) - - print("\n注意：这些示例需要实际安装依赖并配置环境") - print(" 可以根据需要选择运行不同的示例\n") - - # 运行示例 - # example_1_basic_usage() - # example_2_with_git_sync() - # example_3_handler_interface() - # example_4_batch_processing() - # example_5_custom_config() - # example_6_error_handling() - - print("\n提示：取消注释上面的示例代码来运行对应的示例") - - -if __name__ == '__main__': - main() diff --git a/skills/obsidian-kb/handler.py b/skills/obsidian-kb/handler.py deleted file mode 100755 index efc3dda..0000000 --- a/skills/obsidian-kb/handler.py +++ /dev/null @@ -1,427 +0,0 @@ -#!/usr/bin/env python3 -""" -Auto-trigger handler for obsidian-kb skill. - -Detects video/article links in user messages and automatically processes them. -Supports Bilibili, YouTube, and Zhihu. -""" - -import re -import sys -import os -from typing import List, Dict, Optional, Tuple -from urllib.parse import urlparse - -# Add path for imports -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'src')) - -# Import short link resolver -from shortlink_resolver import ( - ShortLinkResolver, - detect_and_format, - PlaywrightResolver -) - -# 导入主流程 -try: - from main_pipeline import MainPipeline - MAIN_PIPELINE_AVAILABLE = True -except ImportError: - MAIN_PIPELINE_AVAILABLE = False - -# Pattern definitions for auto-detection -PATTERNS = { - 'bilibili_full': re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV\w+)'), - 'bilibili_short': re.compile(r'https?://b23\.tv/(\w+)'), - 'bilibili_bv': re.compile(r'\b(BV[a-zA-Z0-9]{10})\b'), - 'youtube_full': re.compile(r'https?://(?:www\.)?youtube\.com/watch\?v=([a-zA-Z0-9_-]{11})'), - 'youtube_short': re.compile(r'https?://youtu\.be/([a-zA-Z0-9_-]{11})'), - 'zhihu_article': re.compile(r'https?://zhuanlan\.zhihu\.com/p/(\d+)'), - 'zhihu_answer': re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)'), -} - - -def detect_links(text: str) -> List[Dict[str, str]]: - """ - Detect video/article links in text. - - Args: - text: Input text to search - - Returns: - List of detected links with metadata: - [ - { - 'url': 'https://...', - 'platform': 'bilibili', - 'type': 'video', - 'id': 'BVxxx', - 'is_short': False - }, - ... - ] - """ - detected = [] - seen_ids = set() # Track IDs to avoid duplicates - - for pattern_name, pattern in PATTERNS.items(): - matches = pattern.finditer(text) - - for match in matches: - # Determine platform and type - if 'bilibili' in pattern_name: - platform = 'bilibili' - content_type = 'video' - if 'short' in pattern_name: - is_short = True - url = match.group(0) - content_id = None # Need to resolve short URL - else: - is_short = False - if pattern_name == 'bilibili_bv': - url = f"https://www.bilibili.com/video/{match.group(1)}" - else: - url = match.group(0) - content_id = match.group(1) - - elif 'youtube' in pattern_name: - platform = 'youtube' - content_type = 'video' - is_short = 'short' in pattern_name - url = match.group(0) - content_id = match.group(1) - - elif 'zhihu' in pattern_name: - platform = 'zhihu' - content_type = 'article' - is_short = False - url = match.group(0) - content_id = match.group(1) - - else: - continue - - # Skip duplicates by URL or ID - unique_key = url if (not content_id or is_short) else content_id - if unique_key in seen_ids: - continue - seen_ids.add(unique_key) - - detected.append({ - 'url': url, - 'platform': platform, - 'type': content_type, - 'id': content_id, - 'is_short': is_short, - 'pattern': pattern_name - }) - - return detected - - -def format_preview(detected: List[Dict[str, str]]) -> str: - """ - Format a preview message for detected links. - - Args: - detected: List of detected links - - Returns: - Formatted preview message - """ - if not detected: - return "" - - lines = ["🔍 检测到内容链接:\n"] - - for i, item in enumerate(detected, 1): - platform_emoji = { - 'bilibili': '📺', - 'youtube': '▶️', - 'zhihu': '📝' - }.get(item['platform'], '🔗') - - platform_name = { - 'bilibili': 'B站视频', - 'youtube': 'YouTube视频', - 'zhihu': '知乎文章' - }.get(item['platform'], item['platform']) - - lines.append(f"{platform_emoji} [{i}] {platform_name}") - lines.append(f" URL: {item['url']}") - - if item['is_short']: - lines.append(f" ⚠️ 短链接，需要用户提供完整URL或ID") - elif item['id']: - lines.append(f" ID: {item['id']}") - - lines.append("") - - return "\n".join(lines) - - -def format_short_link_help(short_link: Optional[str] = None) -> str: - """ - Format help message for short links. - - Args: - short_link: Optional short link URL to mention - - Returns: - Help message - """ - return ShortLinkResolver.format_short_link_help(short_link) - - -def get_action_prompt(detected: List[Dict[str, str]]) -> str: - """ - Generate prompt asking user what to do. - - Args: - detected: List of detected links - - Returns: - Action prompt - """ - has_short = any(item['is_short'] for item in detected) - - if has_short: - return f"{format_short_link_help()}\n\n其他链接是否立即处理？(y/n)" - - count = len(detected) - if count == 1: - return "是否添加到收件箱并处理？(y/n)" - else: - return f"是否将上述 {count} 个内容添加到收件箱？(y/n)" - - -def should_auto_process(text: str) -> bool: - """ - Check if user wants to auto-process based on their message. - - Args: - text: User's message - - Returns: - True if user indicates they want immediate processing - """ - auto_keywords = ['立即处理', '处理', 'process', '--process', '马上'] - return any(keyword in text.lower() for keyword in auto_keywords) - - -def process_link(url: str, platform: str, auto_process: bool = False) -> Tuple[bool, str]: - """ - Process a single link (extract metadata, add to inbox). - - Args: - url: Content URL - platform: Platform name (bilibili/youtube/zhihu) - auto_process: Whether to immediately process after adding - - Returns: - (success, message) tuple - """ - try: - # Import here to avoid issues if dependencies not available - from sources.bilibili import BilibiliAdapter - from models import Content, ContentStatus - from storage import Storage - - # Only support Bilibili for now - if platform != 'bilibili': - return False, f"暂不支持 {platform}，请手动添加" - - # Handle short links - if 'b23.tv' in url: - return False, "短链接无法自动处理，请提供完整URL或BV号" - - # Create adapter - adapter = BilibiliAdapter() - - # Extract metadata - content = adapter.extract_content(url) - if not content: - return False, "无法提取视频信息" - - # Get transcript - transcript = adapter.get_transcript(content) - if transcript: - content.transcript = transcript - - # Save to inbox - content.status = ContentStatus.CONFIRMED - storage = Storage() - storage.save_content(content) - - # Build success message - msg = f"✅ 已添加到收件箱\n" - msg += f" 标题: {content.title}\n" - msg += f" UP主: {content.author}\n" - msg += f" 时长: {content.duration}秒\n" - - if auto_process: - # Process immediately - try: - from src.pipeline import ProcessingPipeline - pipeline = ProcessingPipeline() - content_text = content.transcript or content.summary or "" - result = pipeline.process_full(content, content_text) - storage.save_content(result) - - msg += f"\n🔄 处理完成\n" - msg += f" 📄 笔记: {result.output_path or 'N/A'}\n" - msg += f" 🏷️ 标签: {', '.join(result.tags)}" - except Exception as e: - msg += f"\n⚠️ 处理失败: {e}" - - return True, msg - - except ImportError as e: - return False, f"依赖模块未安装: {e}" - except Exception as e: - return False, f"处理失败: {e}" - - -def process_video_full(source_id: str, enable_git_sync: bool = False) -> Tuple[bool, str]: - """ - 使用主流程完整处理视频（新接口） - - 完整流程: 提取 → 处理 → 输出 → 同步 - - Args: - source_id: 视频ID（支持BV号、URL） - enable_git_sync: 是否启用Git同步 - - Returns: - (success, message) tuple - """ - if not MAIN_PIPELINE_AVAILABLE: - return False, "主流程模块未可用，请检查依赖" - - try: - # 导入配置 - from src.config import get_config - - # 获取配置 - config = get_config() - - # 创建主流程 - pipeline = MainPipeline(config=config, enable_git_sync=enable_git_sync) - - # 处理视频 - result = pipeline.process_video(source_id) - - # 返回结果 - if result['success']: - return True, result['message'] - else: - return False, result.get('error', 'Unknown error') - - except Exception as e: - return False, f"处理失败: {e}" - - -def handle_message(text: str) -> Optional[str]: - """ - Main entry point: handle incoming message and return response. - - Args: - text: User's message text - - Returns: - Response message, or None if no action needed - """ - # First, check for short links and provide help - has_short, short_msg = detect_and_format(text) - - # Detect all links (including short ones) - detected = detect_links(text) - - if not detected: - return None - - # If we have short links, show help message first - if has_short: - return short_msg - - # Check if this is a confirmation (y/yes) - text_lower = text.lower().strip() - if text_lower in ('y', 'yes', '是', '好的', 'ok'): - # This would need state management - for now return a message - # about using explicit command - return "请使用 /kb-add 命令添加内容，或重新发送链接" - - # Show preview - preview = format_preview(detected) - prompt = get_action_prompt(detected) - - return f"{preview}\n{prompt}" - - -# CLI interface for testing -def main(): - """CLI for testing the handler.""" - if len(sys.argv) < 2: - print("Usage: python3 handler.py [options]") - print("\nCommands:") - print(" detect 'text' - 检测文本中的内容链接") - print(" process - 使用主流程处理视频") - print(" process --git - 处理视频并同步到Git") - print("\nExamples:") - print(' python3 handler.py detect "https://www.bilibili.com/video/BV1xx411c7mD"') - print(' python3 handler.py process BV1xx411c7mD') - print(' python3 handler.py process BV1xx411c7mD --git') - sys.exit(1) - - command = sys.argv[1].lower() - - if command == 'detect': - # 检测链接 - if len(sys.argv) < 3: - print("Error: Please provide text to detect") - sys.exit(1) - text = ' '.join(sys.argv[2:]) - response = handle_message(text) - if response: - print(response) - else: - print("No content links detected") - - elif command == 'process': - # 使用主流程处理视频 - if len(sys.argv) < 3: - print("Error: Please provide BV number or URL") - sys.exit(1) - - source_id = sys.argv[2] - enable_git = '--git' in sys.argv - - print(f"Processing video: {source_id}") - if enable_git: - print("Git sync: enabled") - else: - print("Git sync: disabled") - print() - - success, message = process_video_full(source_id, enable_git_sync=enable_git) - - if success: - print(message) - sys.exit(0) - else: - print(f"\n❌ Failed: {message}") - sys.exit(1) - - else: - # 兼容旧格式：直接传入文本进行检测 - text = sys.argv[1] - response = handle_message(text) - if response: - print(response) - else: - print("No content links detected") - - -if __name__ == '__main__': - main() diff --git a/skills/obsidian-kb/main_pipeline.py b/skills/obsidian-kb/main_pipeline.py deleted file mode 100755 index c9d9e5a..0000000 --- a/skills/obsidian-kb/main_pipeline.py +++ /dev/null @@ -1,489 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Obsidian KB - 主流程集成 - -整合提取层、处理层、输出层，实现完整的数据流： -提取(BilibiliExtractor) → 处理(DeepAnalyzer/ContentChunker) → 输出(ObsidianWriter) → 同步(GitSync) -""" - -import os -import sys -import logging -from typing import Dict, Any, Optional -from datetime import datetime -from pathlib import Path - -# 添加项目路径 -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from src.extractors.bilibili_extractor import BilibiliExtractor, ExtractionError -from src.deep_analyzer import DeepAnalyzer -from src.content_chunker import ContentChunker, Chunk -from src.outputs.obsidian_writer import ObsidianWriter -from src.outputs.git_sync import GitSync -from src.models import Content, ContentType, ContentStatus, TimelineEntry -from src.config import get_config - - -# 配置日志 -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler('obsidian_kb.log', encoding='utf-8') - ] -) -logger = logging.getLogger('ObsidianKB') - - -class MainPipeline: - """主流程集成器""" - - def __init__(self, config=None, enable_git_sync: bool = False): - """ - 初始化主流程 - - Args: - config: 配置对象 - enable_git_sync: 是否启用Git同步 - """ - self.config = config or get_config() - self.enable_git_sync = enable_git_sync - - logger.info("初始化主流程...") - logger.info(f"Obsidian vault 路径: {self.config.vault_path}") - logger.info(f"Git 同步: {'启用' if enable_git_sync else '禁用'}") - - # 初始化各层组件 - self.extractor = BilibiliExtractor(config=self.config) - self.analyzer = DeepAnalyzer(ai_client=None) # 暂时不传入ai_client - self.chunker = ContentChunker() - self.writer = ObsidianWriter(config=self.config) - - # 初始化Git同步器（仅在需要时） - self.git_sync = None - if enable_git_sync and self._is_git_enabled(): - self.git_sync = GitSync(config=self.config) - logger.info("Git 同步器已初始化") - - def _is_git_enabled(self) -> bool: - """检查Git同步是否在配置中启用""" - return self.config.get('git.enabled', False) - - def process_video(self, source_id: str) -> Dict[str, Any]: - """ - 处理单个视频的完整流程 - - Args: - source_id: 视频ID（支持BV号、URL） - - Returns: - 处理结果字典，包含： - - success: bool - 是否成功 - - content: Content对象 - 处理的内容 - - message: str - 处理消息 - - error: str - 错误信息（如果失败） - """ - result = { - 'success': False, - 'content': None, - 'message': '', - 'error': '' - } - - try: - logger.info("=" * 80) - logger.info(f"开始处理视频: {source_id}") - logger.info("=" * 80) - - # Step 1: 提取层 - 提取视频信息 - logger.info("\n[Step 1/5] 提取视频信息...") - extractor_data = self._extract_video(source_id) - - # Step 2: 构建Content对象 - logger.info("\n[Step 2/5] 构建内容对象...") - content = self._build_content(extractor_data) - - # Step 3: 处理层 - 深度分析 - logger.info("\n[Step 3/5] 深度分析...") - analysis_result = self._analyze_content(extractor_data) - - # Step 4: 处理层 - 内容分片 - logger.info("\n[Step 4/5] 内容分片...") - chunks = self._chunk_content(extractor_data) - - # 合并分析结果和分片信息到content - self._merge_analysis_to_content(content, analysis_result, chunks) - - # Step 5: 输出层 - 写入Obsidian - logger.info("\n[Step 5/5] 写入Obsidian...") - output_path = self._write_obsidian_note(content) - - # Step 6: 同步层 - Git同步（可选） - if self.enable_git_sync and self.git_sync: - logger.info("\n[Step 6/6] Git同步...") - self._sync_to_git(content) - - # 更新状态 - content.update_status(ContentStatus.GENERATED) - content.output_path = output_path - - # 构建成功结果 - result['success'] = True - result['content'] = content - result['message'] = self._format_success_message(content, chunks) - - logger.info("\n" + "=" * 80) - logger.info(f"✅ 处理完成: {content.title}") - logger.info(f" 📄 笔记路径: {output_path}") - logger.info(f" 📊 字幕条数: {len(chunks) if chunks else 0}") - logger.info("=" * 80 + "\n") - - except ExtractionError as e: - logger.error(f"❌ 提取失败: {e}") - result['error'] = f"提取失败: {e}" - result['message'] = f"提取视频信息失败: {e}" - - except Exception as e: - logger.error(f"❌ 处理失败: {e}", exc_info=True) - result['error'] = str(e) - result['message'] = f"处理过程中发生错误: {e}" - - return result - - def _extract_video(self, source_id: str) -> Dict[str, Any]: - """ - 提取视频信息 - - Args: - source_id: 视频ID或URL - - Returns: - 提取器返回的数据结构 - - Raises: - ExtractionError: 提取失败 - """ - logger.info(f" 正在提取: {source_id}") - - try: - extractor_data = self.extractor.extract(source_id) - logger.info(f" ✅ 提取成功: {extractor_data.get('metadata', {}).get('title', '未知标题')}") - logger.info(f" 作者: {extractor_data.get('metadata', {}).get('author', '未知')}") - logger.info(f" 时长: {extractor_data.get('metadata', {}).get('duration', '0')}") - - # 检查字幕 - subtitle = extractor_data.get('subtitle', '') - if subtitle: - subtitle_lines = subtitle.split('\n') - logger.info(f" 字幕: {len(subtitle_lines)} 条") - else: - logger.warning(f" ⚠️ 无字幕") - - return extractor_data - - except ExtractionError as e: - logger.error(f" ❌ 提取失败: {e}") - raise - except Exception as e: - logger.error(f" ❌ 提取异常: {e}", exc_info=True) - raise ExtractionError(f"提取过程异常: {e}", source_id=source_id) - - def _build_content(self, extractor_data: Dict[str, Any]) -> Content: - """ - 从提取器数据构建Content对象 - - Args: - extractor_data: 提取器返回的数据 - - Returns: - Content对象 - """ - metadata = extractor_data.get('metadata', {}) - - content = Content( - source='bilibili', - source_id=metadata.get('bvid', ''), - url=metadata.get('url', ''), - title=metadata.get('title', ''), - author=metadata.get('author', ''), - description=extractor_data.get('description', ''), - duration=self._parse_duration(metadata.get('duration', '0')), - content_type=ContentType.VIDEO, - transcript=extractor_data.get('subtitle', ''), - status=ContentStatus.SCANNED - ) - - logger.info(f" ✅ Content对象已创建: {content.id}") - return content - - def _parse_duration(self, duration_str: str) -> Optional[int]: - """ - 解析时长字符串为秒数 - - Args: - duration_str: 时长字符串（如 "10:30"） - - Returns: - 秒数 - """ - try: - if ':' in duration_str: - parts = duration_str.split(':') - if len(parts) == 2: - # MM:SS - return int(parts[0]) * 60 + int(parts[1]) - elif len(parts) == 3: - # HH:MM:SS - return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2]) - except: - pass - return None - - def _analyze_content(self, extractor_data: Dict[str, Any]) -> Dict[str, Any]: - """ - 深度分析内容 - - Args: - extractor_data: 提取器返回的数据 - - Returns: - 分析结果字典 - """ - try: - logger.info(" 正在进行深度分析...") - - # 使用DeepAnalyzer进行分析 - # DeepAnalyzer接收extractor_data作为输入 - analysis_result = self.analyzer.analyze(extractor_data) - - # 将AnalysisResult转换为字典以便合并 - result_dict = { - 'main_topics': analysis_result.main_topics, - 'key_insights': analysis_result.key_insights, - 'action_items': analysis_result.action_items, - 'summary': analysis_result.summary, - 'timeline': analysis_result.timeline, - 'recommended_tags': analysis_result.recommended_tags, - 'full_analysis': analysis_result.full_analysis - } - - logger.info(f" ✅ 分析完成: {len(result_dict['main_topics'])} 个主题") - - return result_dict - - except Exception as e: - logger.error(f" ❌ 分析失败: {e}", exc_info=True) - # 返回默认结果，不中断流程 - return { - 'main_topics': ['未分类'], - 'key_insights': [], - 'action_items': [], - 'summary': '分析失败，无法生成摘要', - 'timeline': [], - 'recommended_tags': ['未分类'], - 'full_analysis': '' - } - - def _chunk_content(self, extractor_data: Dict[str, Any]) -> list: - """ - 内容分片 - - Args: - extractor_data: 提取器返回的数据 - - Returns: - Chunk对象列表 - """ - try: - logger.info(" 正在进行内容分片...") - - # 使用ContentChunker进行分片 - chunks = self.chunker.chunk_from_extractor(extractor_data) - - logger.info(f" ✅ 分片完成: {len(chunks)} 段") - - return chunks - - except Exception as e: - logger.error(f" ❌ 分片失败: {e}", exc_info=True) - # 返回空列表，不中断流程 - return [] - - def _merge_analysis_to_content(self, content: Content, analysis_result: Dict[str, Any], chunks: list): - """ - 将分析结果和分片信息合并到Content对象 - - Args: - content: Content对象 - analysis_result: 分析结果字典 - chunks: Chunk对象列表 - """ - # 合并基本信息 - content.summary = analysis_result.get('summary', '') - content.key_points = analysis_result.get('main_topics', []) - content.tags = analysis_result.get('recommended_tags', []) - - # 合并时间轴 - timeline_data = analysis_result.get('timeline', []) - if timeline_data: - content.timeline = [ - TimelineEntry( - timestamp=item.get('start_time', ''), - title=item.get('title', ''), - description=item.get('summary', '') - ) - for item in timeline_data - ] - else: - # 从chunks生成时间轴 - if chunks: - content.timeline = [ - TimelineEntry( - timestamp=chunk.time_range, - title=f"片段 {chunk.index + 1}", - description=f"时长 {int(chunk.duration)}秒" - ) - for chunk in chunks - ] - - logger.info(f" ✅ 数据合并完成: 时间轴 {len(content.timeline)} 条") - - def _write_obsidian_note(self, content: Content) -> Optional[str]: - """ - 写入Obsidian笔记 - - Args: - content: Content对象 - - Returns: - 文件路径，失败返回None - """ - try: - logger.info(f" 正在写入笔记...") - - # 确保输出目录存在 - self.config.vault_path.mkdir(parents=True, exist_ok=True) - - # 调用ObsidianWriter写入 - output_path = self.writer.write(content, self.config) - - if output_path: - logger.info(f" ✅ 笔记已写入: {output_path}") - else: - logger.warning(f" ⚠️ 笔记写入失败") - - return output_path - - except Exception as e: - logger.error(f" ❌ 写入失败: {e}", exc_info=True) - return None - - def _sync_to_git(self, content: Content): - """ - 同步到Git仓库 - - Args: - content: Content对象 - """ - if not self.git_sync: - logger.warning(" ⚠️ Git同步器未初始化，跳过同步") - return - - try: - logger.info(" 正在同步到Git...") - - success = self.git_sync.sync(self.config) - - if success: - logger.info(" ✅ Git同步成功") - else: - logger.warning(" ⚠️ Git同步失败") - - except Exception as e: - logger.error(f" ❌ Git同步异常: {e}", exc_info=True) - # Git同步失败不视为整体失败 - - def _format_success_message(self, content: Content, chunks: list) -> str: - """ - 格式化成功消息 - - Args: - content: Content对象 - chunks: Chunk列表 - - Returns: - 格式化的消息字符串 - """ - lines = [ - "✅ 处理完成！", - "", - f"📺 标题: {content.title}", - f"👤 作者: {content.author}", - f"📄 笔记: {content.output_path or 'N/A'}", - ] - - if content.duration: - lines.append(f"⏱️ 时长: {self._format_duration(content.duration)}") - - if chunks: - lines.append(f"📊 分片: {len(chunks)} 段") - - if content.tags: - lines.append(f"🏷️ 标签: {', '.join(content.tags[:5])}") - - lines.append("") - - return "\n".join(lines) - - def _format_duration(self, seconds: int) -> str: - """格式化时长""" - if not seconds: - return "N/A" - - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - secs = seconds % 60 - - if hours > 0: - return f"{hours}小时{minutes}分钟" - elif minutes > 0: - return f"{minutes}分钟{secs}秒" - else: - return f"{secs}秒" - - -def main(): - """命令行入口""" - import argparse - - parser = argparse.ArgumentParser(description='Obsidian KB - 主流程集成') - parser.add_argument('source_id', help='视频ID（BV号或URL）') - parser.add_argument('--git', action='store_true', help='启用Git同步') - parser.add_argument('--config', help='配置文件路径') - - args = parser.parse_args() - - # 加载配置 - config = get_config(args.config) if args.config else get_config() - - # 创建主流程 - pipeline = MainPipeline(config=config, enable_git_sync=args.git) - - # 处理视频 - result = pipeline.process_video(args.source_id) - - # 输出结果 - print(result['message']) - - if not result['success']: - print(f"\n❌ 失败: {result['error']}") - sys.exit(1) - else: - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/skills/obsidian-kb/requirements.txt b/skills/obsidian-kb/requirements.txt deleted file mode 100644 index 9595647..0000000 --- a/skills/obsidian-kb/requirements.txt +++ /dev/null @@ -1,32 +0,0 @@ -# Core dependencies -requests>=2.31.0 -pyyaml>=6.0.1 - -# Template engine -jinja2>=3.1.2 - -# Optional AI providers -openai>=1.0.0 -anthropic>=0.7.0 - -# Bilibili extraction -# playwright>=1.40.0 -# undetected-playwright>=0.1.0 -beautifulsoup4>=4.12.0 -lxml>=4.9.0 -paddlepaddle>=2.5.0 -paddleocr>=2.7.0 -opencv-python>=4.8.0 - -# Zhihu support -DrissionPage>=4.0.0 -markdownify>=0.11.6 - -# Testing -pytest>=7.4.0 -pytest-cov>=4.1.0 - -# Development -black>=23.0.0 -flake8>=6.0.0 -mypy>=1.0.0 diff --git a/skills/obsidian-kb/shortlink_resolver.py b/skills/obsidian-kb/shortlink_resolver.py deleted file mode 100755 index 4bd41d1..0000000 --- a/skills/obsidian-kb/shortlink_resolver.py +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env python3 -""" -Short link resolver for obsidian-kb skill. - -Provides utilities for detecting and resolving short links, particularly -Bilibili short links (b23.tv) which may be blocked by anti-scraping measures. - -Supported short link services: -- Bilibili: b23.tv -- Generic URL shorteners (detection only) -""" - -import re -from typing import Optional, Dict, List, Tuple -from urllib.parse import urlparse - - -class ShortLinkResolver: - """Short link detection and resolution handler""" - - # Pattern for Bilibili short links - BILIBILI_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)') - - # Pattern for other short link services - OTHER_SHORT_PATTERNS = { - 'bit.ly': re.compile(r'https?://bit\.ly/\w+'), - 't.cn': re.compile(r'https?://t\.cn/\w+'), - 'dwz.cn': re.compile(r'https?://dwz\.cn/\w+'), - 'goo.gl': re.compile(r'https?://goo\.gl/\w+'), - 'tinyurl.com': re.compile(r'https?://tinyurl\.com/\w+'), - } - - @staticmethod - def detect_short_links(text: str) -> List[Dict[str, str]]: - """ - Detect short links in text. - - Args: - text: Input text to search - - Returns: - List of detected short links with metadata: - [ - { - 'url': 'https://b23.tv/xxxxx', - 'platform': 'bilibili', - 'is_short': True - }, - ... - ] - """ - detected = [] - - # Check for Bilibili short links - for match in ShortLinkResolver.BILIBILI_SHORT_PATTERN.finditer(text): - detected.append({ - 'url': match.group(0), - 'code': match.group(1), - 'platform': 'bilibili', - 'is_short': True - }) - - # Check for other short link services - for platform, pattern in ShortLinkResolver.OTHER_SHORT_PATTERNS.items(): - for match in pattern.finditer(text): - detected.append({ - 'url': match.group(0), - 'platform': platform, - 'is_short': True - }) - - return detected - - @staticmethod - def is_bilibili_short_link(url: str) -> bool: - """ - Check if URL is a Bilibili short link. - - Args: - url: URL to check - - Returns: - True if URL is a b23.tv short link - """ - return bool(ShortLinkResolver.BILIBILI_SHORT_PATTERN.match(url)) - - @staticmethod - def format_short_link_help(short_link: Optional[str] = None) -> str: - """ - Format help message for short links. - - Args: - short_link: Optional short link URL to mention - - Returns: - Help message - """ - lines = [ - "🔗 检测到短链接", - "", - "⚠️ 由于反爬虫保护，短链接无法自动解析", - "" - ] - - if short_link: - lines.append(f"📍 短链接: {short_link}") - lines.append("") - - lines.extend([ - "📱 手机用户：", - " 1. 点击/打开链接", - " 2. 在浏览器地址栏复制完整 URL", - " 3. 发送给助手", - "", - "💻 桌面用户：", - " 1. 发送 BV 号（如：BV1jcFqz7EFW）", - " 2. 或使用完整 URL", - "", - "💡 提示：完整 URL 格式为：https://www.bilibili.com/video/BVxxxxx" - ]) - - return "\n".join(lines) - - @staticmethod - def format_short_link_warning(short_links: List[Dict[str, str]]) -> str: - """ - Format warning message for multiple short links. - - Args: - short_links: List of short link metadata - - Returns: - Warning message - """ - if not short_links: - return "" - - lines = [ - "⚠️ 检测到短链接", - "" - ] - - for i, link in enumerate(short_links, 1): - platform = link.get('platform', 'unknown') - url = link['url'] - - if platform == 'bilibili': - lines.append(f" [{i}] B站短链接: {url}") - lines.append(" 需要完整URL或BV号") - else: - lines.append(f" [{i}] {platform}: {url}") - lines.append(" 需要完整URL") - - lines.append("") - - lines.extend([ - "💡 建议：", - " - 在浏览器打开链接后复制完整URL", - " - 或者直接发送BV号（仅限B站）" - ]) - - return "\n".join(lines) - - -class PlaywrightResolver: - """ - Browser-based short link resolver using Playwright. - - This is an optional implementation for environments where Playwright - is available. It can bypass anti-scraping measures by simulating - a real browser. - - Requirements: - pip install playwright - playwright install chromium - """ - - def __init__(self, headless: bool = True, timeout: int = 10000): - """ - Initialize Playwright resolver. - - Args: - headless: Whether to run browser in headless mode - timeout: Page navigation timeout in milliseconds - """ - self.headless = headless - self.timeout = timeout - self._playwright = None - - def _check_playwright_available(self) -> bool: - """Check if Playwright is installed and available""" - try: - from playwright.sync_api import sync_playwright - return True - except ImportError: - return False - - def resolve(self, short_url: str) -> Optional[str]: - """ - Resolve short link using Playwright browser automation. - - Args: - short_url: Short URL to resolve - - Returns: - Final URL after redirects, or None if resolution fails - """ - if not self._check_playwright_available(): - raise ImportError( - "Playwright not installed. Install with: pip install playwright && playwright install chromium" - ) - - from playwright.sync_api import sync_playwright - - try: - with sync_playwright() as p: - # Launch browser - browser = p.chromium.launch(headless=self.headless) - context = browser.new_context( - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - viewport={'width': 1920, 'height': 1080} - ) - page = context.new_page() - - # Navigate to short URL - page.goto(short_url, timeout=self.timeout, wait_until='networkidle') - - # Get final URL - final_url = page.url - - # Close browser - browser.close() - - return final_url - - except Exception as e: - print(f"Playwright resolution failed: {e}") - return None - - def resolve_bilibili_short(self, short_url: str) -> Optional[str]: - """ - Resolve Bilibili short link and extract BV ID. - - Args: - short_url: Bilibili short URL (e.g., https://b23.tv/xxxxx) - - Returns: - Full Bilibili URL with BV ID, or None if resolution fails - """ - final_url = self.resolve(short_url) - if not final_url: - return None - - # Check if final URL contains BV ID - if 'bilibili.com/video/' in final_url: - return final_url - - # Extract BV ID if present in URL - bv_pattern = re.compile(r'(BV[a-zA-Z0-9]{10})') - match = bv_pattern.search(final_url) - if match: - return f"https://www.bilibili.com/video/{match.group(1)}" - - return None - - -def get_resolver() -> ShortLinkResolver: - """ - Get the default resolver instance. - - Returns: - ShortLinkResolver instance - """ - return ShortLinkResolver() - - -def detect_and_format(text: str) -> Tuple[bool, Optional[str]]: - """ - Detect short links in text and format appropriate message. - - Args: - text: Input text to search - - Returns: - Tuple of (has_short_links, formatted_message) - - has_short_links: True if short links detected - - formatted_message: Help/warning message if short links found, None otherwise - """ - resolver = ShortLinkResolver() - short_links = resolver.detect_short_links(text) - - if not short_links: - return False, None - - # Determine appropriate message based on number and type of links - if len(short_links) == 1: - link = short_links[0] - if link['platform'] == 'bilibili': - return True, resolver.format_short_link_help(link['url']) - else: - return True, resolver.format_short_link_warning([link]) - else: - return True, resolver.format_short_link_warning(short_links) - - -# CLI interface for testing -def main(): - """CLI for testing the short link resolver.""" - import sys - - if len(sys.argv) < 2: - print("Usage: python3 shortlink_resolver.py 'text to test'") - print("\nExamples:") - print(' python3 shortlink_resolver.py "Check this: https://b23.tv/oLBZZfK"') - print(' python3 shortlink_resolver.py "https://bit.ly/xxxxx"') - print('\nOptions:') - print(' --playwright Try to resolve short links using Playwright') - sys.exit(1) - - text = ' '.join(sys.argv[1:]) - - # Check if we should try Playwright - use_playwright = '--playwright' in sys.argv - if use_playwright: - text = text.replace('--playwright', '').strip() - resolver = PlaywrightResolver() - - # Find short links - short_links = ShortLinkResolver.detect_short_links(text) - - for link in short_links: - url = link['url'] - print(f"Resolving: {url}") - - if link['platform'] == 'bilibili': - resolved = resolver.resolve_bilibili_short(url) - else: - resolved = resolver.resolve(url) - - if resolved: - print(f" → {resolved}") - else: - print(f" ✗ Resolution failed") - else: - # Just detect and show help - has_links, message = detect_and_format(text) - - if has_links: - print(message) - else: - print("No short links detected") - - -if __name__ == '__main__': - main() diff --git a/skills/obsidian-kb/src/__init__.py b/skills/obsidian-kb/src/__init__.py deleted file mode 100644 index e99d35c..0000000 --- a/skills/obsidian-kb/src/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Obsidian KB - 知识库内容管理技能 -Checkpoint 4-5: 内容处理管道与长视频分段分析 -""" - -from .models import Content, ContentType, ContentStatus, TimelineEntry, AnalysisResult -from .light_scanner import LightScanner -from .deep_analyzer import DeepAnalyzer -from .pipeline import ProcessingPipeline -from .content_chunker import ContentChunker, Chunk, get_chunking_strategy, chunk_content -from .ai_client import AIClient -from .config import Config, get_config - -__all__ = [ - # 模型 - 'Content', - 'ContentType', - 'ContentStatus', - 'TimelineEntry', - 'AnalysisResult', - # 处理器 - 'LightScanner', - 'DeepAnalyzer', - 'ProcessingPipeline', - # 分片 - 'ContentChunker', - 'Chunk', - 'get_chunking_strategy', - 'chunk_content', - # AI客户端 - 'AIClient', - # 配置 - 'Config', - 'get_config', -] - -__version__ = '0.2.0' diff --git a/skills/obsidian-kb/src/ai_client.py b/skills/obsidian-kb/src/ai_client.py deleted file mode 100644 index cee3c73..0000000 --- a/skills/obsidian-kb/src/ai_client.py +++ /dev/null @@ -1,514 +0,0 @@ -"""Obsidian KB - AI客户端 -支持多个 AI 提供商和 fallback 机制 -""" -import os -import json -import requests -from typing import Dict, Any, List, Optional, Tuple -from src.models import ContentType -from src.config import get_config -from src.content_chunker import ContentChunker, get_chunking_strategy, chunk_content - - -class AIClient: - """ - AI客户端 - 支持分段分析长视频内容和 fallback 机制 - """ - - def __init__(self): - self.config = get_config() - self.chunker = ContentChunker() - # 获取 AI 配置 - self._load_ai_config() - - def _load_ai_config(self) -> None: - """加载 AI 配置（主 provider + fallbacks）""" - self.ai_configs = [] - - # 加载主配置 - main_provider = self.config.get('ai.provider', 'openai') - main_config = { - 'provider': main_provider, - 'api_key': self.config.get('ai.api_key'), - 'model': self.config.get('ai.model', 'gpt-4'), - 'base_url': self.config.get('ai.base_url'), - 'is_primary': True - } - self.ai_configs.append(main_config) - - # 加载 fallback 配置 - fallbacks = self.config.get('ai.fallbacks', []) - for fb in fallbacks: - self.ai_configs.append({ - 'provider': fb.get('provider'), - 'api_key': fb.get('api_key'), - 'model': fb.get('model'), - 'base_url': fb.get('base_url'), - 'is_primary': False - }) - - def _call_api_with_fallback(self, prompt: str, max_tokens: int = 1000, response_format: str = 'text') -> Tuple[bool, str]: - """ - 使用 fallback 机制调用 API - - Args: - prompt: 提示词 - max_tokens: 最大 token 数 - response_format: 响应格式 ('text' 或 'json') - - Returns: - (success, response) 元组 - """ - last_error = None - - for i, ai_config in enumerate(self.ai_configs): - try: - if not ai_config.get('api_key'): - print(f"[AIClient] 跳过 {ai_config.get('provider')} - API Key 为空") - continue - - provider_type = ai_config.get('provider') - print(f"[AIClient] 尝试使用 {provider_type} (优先级 {i+1}/{len(self.ai_configs)})...") - - response = self._call_single_provider(ai_config, prompt, max_tokens, response_format) - return True, response - - except Exception as e: - last_error = e - print(f"[AIClient] {provider_type} 调用失败: {e}") - continue - - # 所有 provider 都失败了 - error_msg = f"所有 AI 提供商均失败，最后错误: {last_error}" if last_error else "没有可用的 AI 配置" - print(f"[AIClient] {error_msg}") - return False, self._mock_generate(prompt, max_tokens) # 返回 mock 响应作为兜底 - - def _call_single_provider(self, ai_config: Dict, prompt: str, max_tokens: int, response_format: str) -> str: - """ - 调用单个 AI 提供商 - - Args: - ai_config: AI 配置 - prompt: 提示词 - max_tokens: 最大 token 数 - response_format: 响应格式 - - Returns: - API 响应文本 - """ - provider = ai_config.get('provider') - api_key = ai_config.get('api_key') - model = ai_config.get('model') - base_url = ai_config.get('base_url') - - if provider == 'minimax': - return self._call_minimax(api_key, model, base_url, prompt, max_tokens, response_format) - elif provider == 'zhipu': - return self._call_zhipu(api_key, model, base_url, prompt, max_tokens, response_format) - elif provider == 'alibaba': - return self._call_alibaba(api_key, model, base_url, prompt, max_tokens, response_format) - elif provider == 'openai': - return self._call_openai(api_key, model, base_url, prompt, max_tokens, response_format) - else: - raise ValueError(f"不支持的 provider: {provider}") - - def _call_minimax(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str: - """调用 MiniMax API (Anthropic Messages 兼容接口)""" - url = base_url or "https://api.minimaxi.com/anthropic/v1/messages" - - headers = { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json" - } - - payload = { - "model": model, - "max_tokens": max_tokens, - "messages": [ - {"role": "user", "content": prompt} - ] - } - - if response_format == 'json': - payload['messages'][0]['content'] = prompt + "\n\n请直接返回 JSON 格式，不要有其他文字。" - - response = requests.post(url, headers=headers, json=payload, timeout=30) - response.raise_for_status() - data = response.json() - - # MiniMax 返回的是 Anthropic 格式，content 是一个数组 - # 需要找到 type='text' 的元素 - for item in data.get('content', []): - if item.get('type') == 'text': - return item.get('text', '') - - # 如果没有找到 text 类型，返回第一个元素的值（兼容旧格式） - return data.get('content', [{}])[0].get('text', '') - - def _call_zhipu(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str: - """调用智谱 AI API (OpenAI 兼容接口)""" - # 确保 base_url 包含 /chat/completions - if base_url and not base_url.endswith('/chat/completions'): - url = f"{base_url}/chat/completions" - else: - url = base_url or "https://open.bigmodel.cn/api/paas/v4/chat/completions" - - headers = { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json" - } - - payload = { - "model": model, - "max_tokens": max_tokens, - "messages": [ - {"role": "user", "content": prompt} - ] - } - - response = requests.post(url, headers=headers, json=payload, timeout=30) - response.raise_for_status() - data = response.json() - return data['choices'][0]['message']['content'] - - def _call_alibaba(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str: - """调用阿里云通义千问 API (OpenAI 兼容接口)""" - url = f"{base_url}/chat/completions" - - headers = { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json" - } - - payload = { - "model": model, - "max_tokens": max_tokens, - "messages": [ - {"role": "user", "content": prompt} - ] - } - - response = requests.post(url, headers=headers, json=payload, timeout=30) - response.raise_for_status() - data = response.json() - return data['choices'][0]['message']['content'] - - def _call_openai(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str: - """调用 OpenAI API""" - url = base_url or "https://api.openai.com/v1/chat/completions" - - headers = { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json" - } - - payload = { - "model": model, - "max_tokens": max_tokens, - "messages": [ - {"role": "user", "content": prompt} - ] - } - - response = requests.post(url, headers=headers, json=payload, timeout=30) - response.raise_for_status() - data = response.json() - return data['choices'][0]['message']['content'] - - def quick_generate(self, prompt: str, max_tokens: int = 100) -> str: - """快速生成（用于轻量扫描）""" - success, response = self._call_api_with_fallback(prompt, max_tokens, 'text') - return response - - def generate_json(self, prompt: str) -> Dict[str, Any]: - """生成JSON格式响应""" - success, response = self._call_api_with_fallback(prompt, 1000, 'json') - try: - return json.loads(response) - except: - return self._parse_json_from_text(response) - - def _deep_analyze(self, content_text: str, content_type: str, duration: Optional[int] = None) -> Dict[str, Any]: - """ - C5.3: 深度分析入口，根据内容时长决定是否分段分析 - - Args: - content_text: 内容文本 - content_type: 内容类型 - duration: 视频时长（秒） - - Returns: - 分析结果字典 - """ - duration = duration or 0 - - print(f"[AIClient] 深度分析: type={content_type}, duration={duration}s") - - # 长视频（>=30分钟）且是视频类型，使用分段分析 - if duration >= 1800 and content_type == "video": - print(f"[AIClient] 使用分段分析策略") - return self._analyze_in_chunks(content_text, duration) - - # 短视频直接分析 - print(f"[AIClient] 使用直接分析策略") - return self._analyze_direct(content_text) - - def _analyze_in_chunks(self, content_text: str, duration: int) -> Dict[str, Any]: - """ - C5.3: 分段分析长视频 - - Args: - content_text: SRT字幕文本 - duration: 视频时长（秒） - - Returns: - 合并后的分析结果 - """ - # 1. 获取分段策略 - strategy = get_chunking_strategy(duration) - print(f"[AIClient] 分段策略: {strategy['description']}") - - # 2. 执行分片 - chunks = chunk_content(content_text, duration) - print(f"[AIClient] 内容已分为 {len(chunks)} 段") - - # 3. 逐段分析 - chunk_summaries = [] - for i, chunk in enumerate(chunks): - print(f"[AIClient] 分析第 {i+1}/{len(chunks)} 段...") - summary = self._analyze_chunk(chunk.text, chunk.index) - - # 处理时间格式（可能是float秒数或字符串） - start_time = chunk.start_time - end_time = chunk.end_time - if isinstance(start_time, (int, float)): - start_time = self._format_time(start_time) - if isinstance(end_time, (int, float)): - end_time = self._format_time(end_time) - - chunk_summaries.append({ - 'index': chunk.index, - 'start_time': start_time, - 'end_time': end_time, - 'summary': summary - }) - - # 4. 合并所有段摘要 - print(f"[AIClient] 合并 {len(chunk_summaries)} 段分析结果") - final_result = self._merge_summaries(chunk_summaries, "video") - - # 5. 构建完整时间轴 - final_result['timeline'] = self._build_timeline_from_chunks(chunk_summaries) - - return final_result - - def _analyze_chunk(self, chunk_text: str, chunk_index: int) -> str: - """ - C5.3: 分析单个内容分片 - - Args: - chunk_text: 分片文本 - chunk_index: 分片索引 - - Returns: - 分片摘要（限制500字以内） - """ - # 截断文本避免过长 - text_sample = chunk_text[:3000] if len(chunk_text) > 3000 else chunk_text - - prompt = f"""请对以下视频片段进行简要分析（限制在300字以内）： - -片段内容： -{text_sample} - -请提供： -1. 这段的核心主题（50字以内） -2. 关键要点（3-5条） -3. 重要细节或引用 - -请用中文回答，控制在300字以内。""" - - try: - summary = self.quick_generate(prompt, max_tokens=500) - # 确保不超过500字 - if len(summary) > 500: - summary = summary[:497] + "..." - return summary - except Exception as e: - print(f"[AIClient] 分片 {chunk_index} 分析失败: {e}") - return f"第{chunk_index+1}段分析失败" - - def _merge_summaries(self, chunk_summaries: List[Dict], content_type: str) -> Dict[str, Any]: - """ - C5.3: 合并所有段摘要，生成完整分析报告 - - Args: - chunk_summaries: 各段摘要列表 - content_type: 内容类型 - - Returns: - 完整分析结果 - """ - # 构建合并提示 - summaries_text = "\n\n".join([ - f"=== 第{s['index']+1}段 ({s['start_time']}-{s['end_time']}) ===\n{s['summary']}" - for s in chunk_summaries - ]) - - prompt = f"""基于以下各段分析，生成完整的视频分析报告： - -{summaries_text} - -请提供以下格式的JSON： -{{ - "summary": "整体摘要（200字以内）", - "topics": ["主题1", "主题2", "主题3", "主题4"], - "insights": ["深入见解1", "深入见解2", "深入见解3"], - "actions": ["建议行动1", "建议行动2"], - "full_analysis": "详细分析（500字以内）" -}}""" - - try: - result = self.generate_json(prompt) - - # 确保必要字段存在 - if 'topics' not in result: - result['topics'] = ["主题A", "主题B", "主题C"] - if 'insights' not in result: - result['insights'] = ["重要见解1", "重要见解2"] - if 'actions' not in result: - result['actions'] = [] - if 'summary' not in result: - result['summary'] = "长视频分析完成" - if 'full_analysis' not in result: - result['full_analysis'] = summaries_text[:500] - - return result - - except Exception as e: - print(f"[AIClient] 合并摘要失败: {e}") - # 返回基础合并结果 - return { - 'summary': f'视频共分为{len(chunk_summaries)}个部分进行分析', - 'topics': [f"主题{i+1}" for i in range(min(4, len(chunk_summaries)))], - 'insights': ['从分段分析中综合得出'], - 'actions': [], - 'full_analysis': summaries_text[:800] - } - - def _analyze_direct(self, content_text: str) -> Dict[str, Any]: - """直接分析（不分段）""" - # 截断文本 - text_sample = content_text[:5000] if len(content_text) > 5000 else content_text - - prompt = f"""请分析以下内容，提供JSON格式结果： - -{text_sample} - -返回格式： -{{ - "summary": "整体摘要（150字以内）", - "topics": ["主题1", "主题2", "主题3"], - "insights": ["见解1", "见解2"], - "actions": ["行动建议1"], - "timeline": [ - {{"start_time": "00:00", "end_time": "05:00", "title": "第一部分", "summary": "..."}} - ] -}}""" - - try: - return self.generate_json(prompt) - except Exception as e: - print(f"[AIClient] 直接分析失败: {e}") - return { - 'summary': '分析失败', - 'topics': ['无法解析'], - 'insights': ['请检查内容格式'], - 'actions': [], - 'timeline': [] - } - - def _format_time(self, seconds: float) -> str: - """格式化秒数为时间字符串""" - hours = int(seconds // 3600) - minutes = int((seconds % 3600) // 60) - secs = int(seconds % 60) - if hours > 0: - return f"{hours:02d}:{minutes:02d}:{secs:02d}" - return f"{minutes:02d}:{secs:02d}" - - def _build_timeline_from_chunks(self, chunk_summaries: List[Dict]) -> List[Dict[str, Any]]: - """从分片摘要构建时间轴""" - timeline = [] - for summary in chunk_summaries: - timeline.append({ - 'index': summary['index'], - 'start_time': summary['start_time'], - 'end_time': summary['end_time'], - 'title': f"第{summary['index']+1}部分", - 'summary': summary['summary'][:200] + "..." if len(summary['summary']) > 200 else summary['summary'] - }) - return timeline - - def _mock_generate(self, prompt: str, max_tokens: int) -> str: - """模拟AI生成（实际实现应调用真实API）""" - # 根据prompt内容返回模拟响应 - prompt_lower = prompt.lower() - - if 'json' in prompt_lower or 'json格式' in prompt: - if '主题' in prompt or 'topic' in prompt_lower: - return json.dumps({ - 'summary': '这是一个关于人工智能和技术发展的内容，涵盖了多个重要主题。', - 'topics': ['人工智能', '机器学习', '未来发展', '技术趋势'], - 'insights': [ - 'AI正在改变我们的工作方式', - '机器学习应用越来越广泛', - '未来技术发展趋势值得关注' - ], - 'actions': ['深入研究相关技术', '关注行业动态'], - 'full_analysis': '详细分析内容...' - }, ensure_ascii=False) - return json.dumps({'result': '模拟JSON响应'}) - - if '摘要' in prompt or 'summary' in prompt_lower: - return '这是一段关于人工智能发展的重要讨论，涵盖了技术趋势和未来应用。' - - if '关键词' in prompt or 'keyword' in prompt_lower: - return '人工智能, 机器学习, 深度学习, 神经网络, 自然语言处理' - - return '模拟AI响应内容' - - def _parse_json_from_text(self, text: str) -> Dict[str, Any]: - """从文本中解析JSON""" - # 尝试提取JSON块 - import re - json_match = re.search(r'\{[\s\S]*\}', text) - if json_match: - try: - return json.loads(json_match.group()) - except: - pass - return {'raw_response': text} - - -class MultiAIClient: - """ - 多AI客户端 - 支持多种AI提供商的客户端 - """ - - def __init__(self): - self.config = get_config() - self.primary_client = AIClient() - self.clients = { - 'openai': self.primary_client, - 'local': AIClient(), # 可以配置为本地模型 - } - - def _deep_analyze(self, content_text: str, content_type: str, duration: Optional[int] = None) -> Dict[str, Any]: - """使用主客户端进行深度分析""" - return self.primary_client._deep_analyze(content_text, content_type, duration) - - def quick_generate(self, prompt: str, max_tokens: int = 100, provider: str = None) -> str: - """快速生成""" - client = self.clients.get(provider, self.primary_client) - return client.quick_generate(prompt, max_tokens) diff --git a/skills/obsidian-kb/src/cli.py b/skills/obsidian-kb/src/cli.py deleted file mode 100644 index 99adb8b..0000000 --- a/skills/obsidian-kb/src/cli.py +++ /dev/null @@ -1,569 +0,0 @@ -#!/usr/bin/env python3 -"""Obsidian KB Knowledge Base CLI Tool - -Command line interface for managing Obsidian knowledge base content. -""" - -import argparse -import sys -import os -from typing import Optional -from datetime import datetime - -# Add parent directory to path for imports -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -try: - from src.config import Config - from src.storage import Storage - from src.pipeline import ProcessingPipeline - from src.models import ContentStatus - CONFIG_AVAILABLE = True -except ImportError as e: - print(f"Import warning: {e}", file=sys.stderr) - CONFIG_AVAILABLE = False - - -def print_header(text: str): - """Print formatted header.""" - print(f"\n{'=' * 50}") - print(f" {text}") - print(f"{'=' * 50}\n") - - -def print_table(headers: list, rows: list, max_width: int = 80): - """Print a simple formatted table.""" - if not rows: - print("(No data)") - return - - # Calculate column widths - col_widths = [len(h) for h in headers] - for row in rows: - for i, cell in enumerate(row): - if i < len(col_widths): - col_widths[i] = max(col_widths[i], len(str(cell)[:max_width])) - - # Print headers - header_line = " | ".join(h.ljust(w) for h, w in zip(headers, col_widths)) - print(header_line) - print("-" * len(header_line)) - - # Print rows - for row in rows: - row_str = " | ".join(str(cell)[:max_width].ljust(w) for cell, w in zip(row, col_widths)) - print(row_str) - print() - - -def cmd_subscribe(args: argparse.Namespace): - """Handle subscribe command - save subscription configuration.""" - print_header("Subscribe Configuration") - - if not CONFIG_AVAILABLE: - print("Error: Config module not available. Please check installation.") - return 1 - - config = Config() - - # Build subscription entry - subscription = { - 'platform': args.platform, - 'type': args.type, - 'id': args.id, - 'name': args.name or f"{args.platform}_{args.type}_{args.id}", - 'created_at': datetime.now().isoformat() - } - - # Load existing subscriptions - subscriptions = config.get('subscriptions', []) - - # Check for duplicates - for sub in subscriptions: - if sub['platform'] == args.platform and sub['id'] == args.id: - print(f"⚠️ Subscription already exists: {sub['name']}") - print(" Updating configuration...") - sub.update(subscription) - break - else: - subscriptions.append(subscription) - print(f"✅ Added new subscription: {subscription['name']}") - - # Save configuration - config.set('subscriptions', subscriptions) - config.save() - - print(f"\n Platform: {args.platform}") - print(f" Type: {args.type}") - print(f" ID: {args.id}") - print(f" Name: {subscription['name']}") - print(f"\n💾 Configuration saved to config.yaml") - - return 0 - - -def cmd_inbox(args: argparse.Namespace): - """Handle inbox command - display pending items.""" - print_header("Inbox Contents") - - if not CONFIG_AVAILABLE: - print("Error: Storage module not available.") - return 1 - - storage = Storage() - # Filter for CONFIRMED status (ready to process) - contents = storage.list_inbox(status=ContentStatus.CONFIRMED) - - if not contents: - print("📭 Inbox is empty!") - return 0 - - # Sort by the requested criteria - if args.sort == 'score': - contents.sort(key=lambda x: x.quality_metrics.overall_score if hasattr(x, 'quality_metrics') else 0, reverse=True) - else: # date - contents.sort(key=lambda x: x.created_at, reverse=True) - - # Limit results - contents = contents[:args.limit] - - print(f"Showing {len(contents)} item(s) (sorted by {args.sort}):\n") - - # Format table rows - rows = [] - for idx, content in enumerate(contents, 1): - title = content.title[:40] if content.title else 'Untitled' - score = content.quality_metrics.overall_score if hasattr(content, 'quality_metrics') else 0 - summary = content.summary[:50] if content.summary else 'No summary' - content_id = content.id - - rows.append([ - f"{idx:2d}", - content_id[:12], - f"{score:.1f}", - title, - summary + "..." if len(content.summary or '') > 50 else summary - ]) - - print_table( - ["#", "ID", "Score", "Title", "Summary"], - rows - ) - - print(f"\n💡 Use 'process ' to process an item, or 'process all' to process all") - - return 0 - - -def cmd_process(args: argparse.Namespace): - """Handle process command - process content items.""" - print_header("Process Content") - - if not CONFIG_AVAILABLE: - print("Error: Pipeline module not available.") - return 1 - - storage = Storage() - - # Import pipeline components - try: - from src.pipeline import ProcessingPipeline - from src.light_scanner import LightScanner - from src.deep_analyzer import DeepAnalyzer - from src.config import get_config - - config = get_config() - pipeline = ProcessingPipeline() - except Exception as e: - print(f"❌ Error initializing pipeline: {e}") - return 1 - - # Determine what to process - if args.content_id == 'all': - items = storage.list_inbox(status=ContentStatus.CONFIRMED) - if not items: - print("📭 Inbox is empty! Nothing to process.") - return 0 - - print(f"Found {len(items)} item(s) to process.\n") - - if not args.yes: - confirm = input("Process all items? [Y/n]: ").strip().lower() - if confirm and confirm not in ('y', 'yes'): - print("❌ Cancelled.") - return 0 - else: - item = storage.load_content(args.content_id) - if not item: - print(f"❌ Content not found: {args.content_id}") - return 1 - items = [item] - print(f"Processing single item: {item.title}\n") - - # Process items with progress - success_count = 0 - fail_count = 0 - - for i, item in enumerate(items, 1): - content_id = item.id - title = item.title or 'Untitled' - - print(f"[{i}/{len(items)}] Processing: {title[:50]}...") - print(f" ID: {content_id}") - - try: - # Use the pipeline to process the content - # The pipeline expects content_text - we'll use transcript or summary - content_text = item.transcript or item.summary or "" - - result = pipeline.process_full(item, content_text) - - # Save the updated content - storage.save_content(result) - - print(f" ✅ Success") - print(f" 📄 Note: {result.output_path or 'N/A'}") - print(f" 🏷️ Tags: {', '.join(result.tags)}") - success_count += 1 - - except Exception as e: - print(f" ❌ Error: {e}") - import traceback - traceback.print_exc() - fail_count += 1 - - print() - - # Summary - print(f"\n📊 Processing Complete") - print(f" Successful: {success_count}") - print(f" Failed: {fail_count}") - print(f" Total: {len(items)}") - - return 0 if fail_count == 0 else 1 - - -def cmd_skip(args: argparse.Namespace): - """Handle skip command - skip/remove an item from inbox.""" - print_header("Skip Content") - - if not CONFIG_AVAILABLE: - print("Error: Storage module not available.") - return 1 - - storage = Storage() - - # Find the item - content = storage.load_content(args.content_id) - if not content: - print(f"❌ Content not found: {args.content_id}") - return 1 - - print(f"Skipping: {content.title or 'Untitled'}") - print(f"ID: {args.content_id}") - - if args.reason: - print(f"Reason: {args.reason}") - - # Mark as skipped - content.status = ContentStatus.SKIPPED - if args.reason: - content.summary = f"[Skipped: {args.reason}] {content.summary or ''}" - - storage.save_content(content) - - print("✅ Item skipped and marked as skipped.") - if args.reason: - print(f" Reason recorded: {args.reason}") - - return 0 - - -def cmd_status(args: argparse.Namespace): - """Handle status command - show system status.""" - print_header("System Status") - - # Git status - print("📁 Git Repository Status") - print("-" * 30) - - import subprocess - try: - # Check if we're in a git repo - result = subprocess.run( - ['git', 'status', '--short'], - capture_output=True, - text=True, - cwd=os.getcwd() - ) - - if result.returncode == 0: - if result.stdout.strip(): - print("Uncommitted changes:") - print(result.stdout) - else: - print("✅ Working directory clean") - - # Get branch info - branch_result = subprocess.run( - ['git', 'branch', '--show-current'], - capture_output=True, - text=True - ) - if branch_result.returncode == 0: - print(f"Branch: {branch_result.stdout.strip()}") - else: - print("⚠️ Not a git repository or git not available") - except FileNotFoundError: - print("⚠️ Git not available") - except Exception as e: - print(f"⚠️ Git error: {e}") - - print() - - # Inbox statistics - print("📥 Inbox Statistics") - print("-" * 30) - - if CONFIG_AVAILABLE: - storage = Storage() - stats = storage.get_stats() - - print(f"Pending (confirmed): {stats.get(ContentStatus.CONFIRMED.value, 0)}") - print(f"New: {stats.get(ContentStatus.NEW.value, 0)}") - print(f"Scanned: {stats.get(ContentStatus.SCANNED.value, 0)}") - print(f"Analyzed: {stats.get(ContentStatus.ANALYZED.value, 0)}") - print(f"Generated: {stats.get(ContentStatus.GENERATED.value, 0)}") - print(f"Skipped: {stats.get(ContentStatus.SKIPPED.value, 0)}") - else: - print("⚠️ Storage module not available") - - print() - - # Configuration summary - print("⚙️ Configuration") - print("-" * 30) - - if CONFIG_AVAILABLE: - config = Config() - subscriptions = config.get('subscriptions', []) - print(f"Active subscriptions: {len(subscriptions)}") - - if subscriptions: - for sub in subscriptions: - print(f" • {sub.get('name', 'Unnamed')} ({sub.get('platform')})") - else: - print("⚠️ Config module not available") - - return 0 - - -def cmd_search(args: argparse.Namespace): - """Handle search command - search processed notes.""" - print_header(f"Search: '{args.query}'") - - if not CONFIG_AVAILABLE: - print("Error: Storage module not available.") - return 1 - - storage = Storage() - - # Perform search across all content - results = storage.search(args.query) - - if not results: - print("🔍 No results found.") - return 0 - - # Filter by tags if requested - if args.in_tags: - query_lower = args.query.lower() - results = [r for r in results if any(query_lower in tag.lower() for tag in r.tags)] - - if not results: - print("🔍 No results found in tags.") - return 0 - - print(f"Found {len(results)} result(s):\n") - - # Format results - for i, content in enumerate(results, 1): - title = content.title or 'Untitled' - path = content.output_path or f"inbox/{content.id}.json" - tags = content.tags - - print(f"{i}. {title}") - print(f" ID: {content.id}") - print(f" Status: {content.status.value}") - print(f" Path: {path}") - if tags: - print(f" Tags: {', '.join(tags)}") - - # Show matching content preview - if not args.in_tags and content.summary: - content_text = content.summary - query_lower = args.query.lower() - idx = content_text.lower().find(query_lower) - if idx >= 0: - start = max(0, idx - 30) - end = min(len(content_text), idx + len(args.query) + 30) - preview = content_text[start:end] - if start > 0: - preview = "..." + preview - if end < len(content_text): - preview = preview + "..." - print(f" Match: {preview}") - - print() - - return 0 - - -def main(): - """Main entry point for CLI.""" - parser = argparse.ArgumentParser( - description='Obsidian KB 知识库维护工具', - prog='obsidian-kb', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - %(prog)s subscribe bilibili fav 12345 --name="Tech Favorites" - %(prog)s inbox --limit 10 --sort date - %(prog)s process all --yes - %(prog)s process abc123 - %(prog)s skip abc123 --reason="Low quality" - %(prog)s status - %(prog)s search "python tutorial" --in-tags - """ - ) - - subparsers = parser.add_subparsers(dest='command', help='Available commands') - - # subscribe 子命令 - subscribe_parser = subparsers.add_parser( - 'subscribe', - aliases=['sub'], - help='Subscribe to a content source' - ) - subscribe_parser.add_argument( - 'platform', - choices=['bilibili', 'youtube', 'zhihu'], - help='Platform to subscribe to' - ) - subscribe_parser.add_argument( - 'type', - choices=['fav', 'up', 'channel'], - help='Subscription type' - ) - subscribe_parser.add_argument( - 'id', - help='Platform-specific ID (user ID, channel ID, etc.)' - ) - subscribe_parser.add_argument( - '--name', - help='Friendly name for this subscription' - ) - subscribe_parser.set_defaults(func=cmd_subscribe) - - # inbox 子命令 - inbox_parser = subparsers.add_parser( - 'inbox', - aliases=['i'], - help='Show inbox contents' - ) - inbox_parser.add_argument( - '--limit', - type=int, - default=20, - help='Maximum number of items to show (default: 20)' - ) - inbox_parser.add_argument( - '--sort', - choices=['score', 'date'], - default='score', - help='Sort order (default: score)' - ) - inbox_parser.set_defaults(func=cmd_inbox) - - # process 子命令 - process_parser = subparsers.add_parser( - 'process', - aliases=['p'], - help='Process content items' - ) - process_parser.add_argument( - 'content_id', - nargs='?', - default='all', - help='Content ID to process, or "all" for all items (default: all)' - ) - process_parser.add_argument( - '--yes', - action='store_true', - help='Auto-confirm without prompting' - ) - process_parser.set_defaults(func=cmd_process) - - # skip 子命令 - skip_parser = subparsers.add_parser( - 'skip', - aliases=['s'], - help='Skip an item (remove from inbox)' - ) - skip_parser.add_argument( - 'content_id', - help='Content ID to skip' - ) - skip_parser.add_argument( - '--reason', - help='Reason for skipping' - ) - skip_parser.set_defaults(func=cmd_skip) - - # status 子命令 - status_parser = subparsers.add_parser( - 'status', - aliases=['st'], - help='Show system status' - ) - status_parser.set_defaults(func=cmd_status) - - # search 子命令 - search_parser = subparsers.add_parser( - 'search', - aliases=['q'], - help='Search processed notes' - ) - search_parser.add_argument( - 'query', - help='Search query' - ) - search_parser.add_argument( - '--in-tags', - action='store_true', - help='Search only in tags' - ) - search_parser.set_defaults(func=cmd_search) - - # Parse arguments - args = parser.parse_args() - - # Execute command - if args.command is None: - parser.print_help() - return 0 - - try: - return args.func(args) or 0 - except KeyboardInterrupt: - print("\n\n⚠️ Interrupted by user") - return 130 - except Exception as e: - print(f"\n❌ Error: {e}") - import traceback - traceback.print_exc() - return 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/skills/obsidian-kb/src/config.py b/skills/obsidian-kb/src/config.py deleted file mode 100644 index 327e7ec..0000000 --- a/skills/obsidian-kb/src/config.py +++ /dev/null @@ -1,353 +0,0 @@ -""" -Configuration management for Obsidian KB Skill. -""" -import os -import yaml -from pathlib import Path -from typing import Optional, Dict, Any - - -# Global config instance -_config_instance: Optional['Config'] = None - - -def get_config(config_path: Optional[str] = None) -> 'Config': - """Get global config instance.""" - global _config_instance - if _config_instance is None or config_path: - _config_instance = Config(config_path) - return _config_instance - - -class Config: - """Configuration manager.""" - - DEFAULT_CONFIG = { - 'ai': { - 'provider': 'openai', - 'model': 'gpt-4', - 'api_key': None, - }, - 'storage': { - 'inbox_dir': 'inbox', - 'topics_dir': 'topics', - 'data_dir': 'data', - }, - 'git': { - 'enabled': False, - 'vault_path': '~/Documents/Obsidian/KB', - 'remote': 'origin', - 'branch': 'main', - 'auto_push': False, - 'commit_message_template': 'Add note: {title}', - }, - 'obsidian': { - 'vault_path': '~/Documents/Obsidian/KB', - 'notes_folder': 'Notes', - }, - 'chunking': { - 'enabled': True, - 'chunk_duration_seconds': 900, # 15 minutes - 'max_chunk_duration_seconds': 1800, # 30 minutes - 'long_video_threshold': 1800, # 30 minutes - }, - 'output': { - 'organize_by': 'author', # author | date | topic - 'filename_format': '{date}_{title}.md', - 'auto_tags': True, - 'default_tags': ['video', 'bilibili'], - }, - 'sources': { - 'bilibili': { - 'enabled': True, - 'cookie': None, - 'extraction': { - 'headless': True, - 'timeout': 30, - }, - }, - 'youtube': { - 'enabled': False, - }, - 'zhihu': { - 'enabled': False, - }, - }, - } - - def __init__(self, config_path: Optional[str] = None, load_env: bool = True): - """ - Initialize configuration. - - Args: - config_path: Path to config file. If None, uses default locations. - load_env: Whether to load environment variable overrides. - """ - self.config_path = config_path or self._find_config_file() - self._config = self._load_config(load_env) - - def _find_config_file(self) -> str: - """Find configuration file in standard locations.""" - search_paths = [ - Path.cwd() / 'config.yaml', - Path.cwd() / 'config.yml', - Path.home() / '.obsidian-kb' / 'config.yaml', - Path.home() / '.config' / 'obsidian-kb' / 'config.yaml', - ] - - for path in search_paths: - if path.exists(): - return str(path) - - # Return default path - return str(Path.cwd() / 'config.yaml') - - def _load_config(self, load_env: bool = True) -> Dict[str, Any]: - """Load configuration from file or use defaults.""" - config = self.DEFAULT_CONFIG.copy() - - if Path(self.config_path).exists(): - with open(self.config_path, 'r', encoding='utf-8') as f: - user_config = yaml.safe_load(f) - if user_config: - self._deep_update(config, user_config) - - # Override with environment variables - if load_env: - self._apply_env_overrides(config) - - return config - - def _deep_update(self, base: Dict, update: Dict) -> None: - """Deep update dictionary.""" - for key, value in update.items(): - if key in base and isinstance(base[key], dict) and isinstance(value, dict): - self._deep_update(base[key], value) - else: - base[key] = value - - def _apply_env_overrides(self, config: Dict[str, Any]) -> None: - """Apply environment variable overrides.""" - # AI settings - if os.getenv('OPENAI_API_KEY'): - config['ai']['api_key'] = os.getenv('OPENAI_API_KEY') - if os.getenv('AI_MODEL'): - config['ai']['model'] = os.getenv('AI_MODEL') - - # Git settings - if os.getenv('OBSIDIAN_VAULT_PATH'): - config['git']['vault_path'] = os.getenv('OBSIDIAN_VAULT_PATH') - - # Bilibili cookie - if os.getenv('BILIBILI_COOKIE'): - config['sources']['bilibili']['cookie'] = os.getenv('BILIBILI_COOKIE') - - def get(self, key: str, default: Any = None) -> Any: - """ - Get configuration value using dot notation. - - Args: - key: Key in dot notation (e.g., 'ai.model') - default: Default value if key not found - - Returns: - Configuration value - """ - keys = key.split('.') - value = self._config - - for k in keys: - if isinstance(value, dict) and k in value: - value = value[k] - else: - return default - - return value - - def set(self, key: str, value: Any) -> None: - """ - Set configuration value using dot notation. - - Args: - key: Key in dot notation - value: Value to set - """ - keys = key.split('.') - config = self._config - - for k in keys[:-1]: - if k not in config: - config[k] = {} - config = config[k] - - config[keys[-1]] = value - - def save(self) -> None: - """Save configuration to file.""" - Path(self.config_path).parent.mkdir(parents=True, exist_ok=True) - with open(self.config_path, 'w', encoding='utf-8') as f: - yaml.dump(self._config, f, default_flow_style=False, allow_unicode=True) - - @property - def obsidian_path(self) -> str: - """Get obsidian path (alias for vault_path).""" - return str(self.vault_path) - - @property - def vault_path(self) -> Path: - """Get Obsidian vault path.""" - path = self.get('obsidian.vault_path') or self.get('git.vault_path', '~/Documents/Obsidian/KB') - return Path(path).expanduser() - - @property - def notes_folder(self) -> str: - """Get notes folder.""" - return self.get('obsidian.notes_folder', 'Notes') - - @property - def ai_api_key(self) -> Optional[str]: - """Get AI API key.""" - return self.get('ai.api_key') - - @property - def ai_model(self) -> str: - """Get AI model.""" - return self.get('ai.model', 'gpt-4') - - @property - def chunking_enabled(self) -> bool: - """Check if chunking is enabled.""" - return self.get('chunking.enabled', True) - - @property - def chunk_duration(self) -> int: - """Get chunk duration in seconds.""" - return self.get('chunking.chunk_duration_seconds', 900) - - @property - def long_video_threshold(self) -> int: - """Get long video threshold in seconds.""" - return self.get('chunking.long_video_threshold', 1800) - - @property - def quality_threshold(self) -> float: - """Get quality threshold.""" - return self.get('quality_threshold', 0.7) - - @property - def obsidian(self) -> 'Config': - """Get obsidian config section (returns self for compatibility).""" - return self - - @property - def git(self) -> 'Config': - """Get git config section (returns self for compatibility).""" - return self - - @property - def enabled(self) -> bool: - """Check if git is enabled.""" - return self.get('git.enabled', False) - - @property - def auto_push(self) -> bool: - """Check if auto push is enabled.""" - return self.get('git.auto_push', False) - - @property - def commit_message_template(self) -> str: - """Get commit message template.""" - return self.get('git.commit_message_template', 'Add note: {title}') - - @property - def kimi_api_key(self) -> str: - """Get Kimi API key (alias for ai.api_key).""" - return self.get('ai.api_key') or '' - - # Output configuration - @property - def organize_by(self) -> str: - """Get output organize_by strategy.""" - return self.get('output.organize_by', 'author') - - @property - def filename_format(self) -> str: - """Get output filename format.""" - return self.get('output.filename_format', '{date}_{title}.md') - - @property - def auto_tags(self) -> bool: - """Get auto_tags setting.""" - return self.get('output.auto_tags', True) - - @property - def default_tags(self) -> list: - """Get default tags list.""" - return self.get('output.default_tags', ['video', 'bilibili']) - - # Bilibili extraction configuration - @property - def bilibili_headless(self) -> bool: - """Get bilibili extraction headless mode.""" - return self.get('sources.bilibili.extraction.headless', True) - - @property - def bilibili_timeout(self) -> int: - """Get bilibili extraction timeout in seconds.""" - return self.get('sources.bilibili.extraction.timeout', 30) - - # Zhihu DrissionPage configuration - @property - def zhihu_enabled(self) -> bool: - """知乎是否启用""" - return self.get('sources.zhihu.enabled', False) - - @property - def zhihu_use_drissionpage(self) -> bool: - """知乎是否使用 DrissionPage""" - return self.get('sources.zhihu.use_drissionpage', True) - - @property - def zhihu_cookie_file(self) -> Optional[str]: - """知乎 Cookie 文件路径""" - return self.get('sources.zhihu.cookie_file') - - @property - def zhihu_cookie_str(self) -> Optional[str]: - """知乎 Cookie 字符串""" - return self.get('sources.zhihu.cookie_str') - - @property - def zhihu_timeout(self) -> int: - """知乎提取超时时间（秒）""" - return self.get('sources.zhihu.timeout', 30) - - @property - def zhihu_headless(self) -> bool: - """知乎是否使用无头浏览器""" - return self.get('sources.zhihu.headless', True) - - def ensure_dirs(self) -> None: - """Ensure required directories exist.""" - self.vault_path.mkdir(parents=True, exist_ok=True) - (self.vault_path / self.get('storage.inbox_dir', 'Inbox')).mkdir(exist_ok=True) - (self.vault_path / self.get('storage.topics_dir', 'Topics')).mkdir(exist_ok=True) - - def validate(self) -> tuple[bool, list[str]]: - """Validate configuration.""" - errors = [] - if not self.ai_api_key: - errors.append("AI API key not configured") - return len(errors) == 0, errors - - # Subscriptions support - def get_subscriptions(self) -> list: - """Get subscriptions list.""" - return self.get('subscriptions', []) - - def add_subscription(self, subscription: Dict[str, Any]) -> None: - """Add a new subscription.""" - subs = self.get('subscriptions', []) - subs.append(subscription) - self.set('subscriptions', subs) - self.save() diff --git a/skills/obsidian-kb/src/content_chunker.py b/skills/obsidian-kb/src/content_chunker.py deleted file mode 100644 index 4b64d24..0000000 --- a/skills/obsidian-kb/src/content_chunker.py +++ /dev/null @@ -1,486 +0,0 @@ -"""Obsidian KB - 内容分片器 -C5.1-C5.2: 实现长视频分段和分段策略 -""" -import re -from dataclasses import dataclass -from typing import List, Dict, Tuple, Optional, Any - - -@dataclass -class Chunk: - """内容分片数据模型""" - index: int - start_time: float # seconds - end_time: float # seconds - text: str - - @property - def duration(self) -> float: - """Get chunk duration in seconds.""" - return self.end_time - self.start_time - - @property - def time_range(self) -> str: - """Get formatted time range string.""" - return f"{self._format_time(self.start_time)}-{self._format_time(self.end_time)}" - - def _format_time(self, seconds: float) -> str: - """Format seconds to MM:SS or HH:MM:SS.""" - hours = int(seconds // 3600) - minutes = int((seconds % 3600) // 60) - secs = int(seconds % 60) - - if hours > 0: - return f"{hours:02d}:{minutes:02d}:{secs:02d}" - return f"{minutes:02d}:{secs:02d}" - - def to_dict(self) -> Dict: - """转换为字典""" - return { - 'index': self.index, - 'start_time': self.start_time, - 'end_time': self.end_time, - 'text_length': len(self.text), - 'text_preview': self.text[:100] + '...' if len(self.text) > 100 else self.text - } - - def __repr__(self) -> str: - return f"Chunk({self.index}, {self.start_time}-{self.end_time}, {len(self.text)}chars)" - - -class ContentChunker: - """内容分片器 - 支持SRT字幕分片""" - - def __init__(self, chunk_duration_seconds: int = 900, max_chunk_duration: int = 1800): - """ - Initialize chunker. - - Args: - chunk_duration_seconds: Target duration for each chunk (default 15 min) - max_chunk_duration: Maximum duration for a chunk (default 30 min) - """ - self.chunk_duration = chunk_duration_seconds - self.max_chunk_duration = max_chunk_duration - - # SRT时间戳正则 - self.srt_time_pattern = re.compile( - r'(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})' - ) - - def chunk_from_extractor(self, extractor_data: Dict[str, Any]) -> List[Chunk]: - """ - 从提取器数据结构接收输入并进行分片 - - Args: - extractor_data: 提取器返回的数据结构，包含： - - metadata: Dict[str, Any] - 元数据（包含时长等） - - stats: Dict[str, Any] - 统计信息 - - subtitle: str - 字幕/文本内容 - - description: str - 内容描述 - - Returns: - Chunk对象列表 - """ - metadata = extractor_data.get('metadata', {}) - subtitle = extractor_data.get('subtitle', '') - description = extractor_data.get('description', '') - - # 检测是否有字幕 - has_subtitle = bool(subtitle and subtitle.strip()) - - if not has_subtitle: - # 无字幕：返回空列表或单块（description） - print("[ContentChunker] 无字幕，返回空分片列表") - return [] - - # 获取时长 - duration = metadata.get('duration', 0) - print(f"[ContentChunker] 有字幕，时长={duration}秒") - - # 使用原有分段逻辑 - return self._auto_chunk(subtitle, duration) - - def _auto_chunk(self, subtitle_text: str, duration: int) -> List[Chunk]: - """ - 自动分片（内部方法） - - Args: - subtitle_text: 字幕文本 - duration: 视频时长（秒） - - Returns: - Chunk对象列表 - """ - # 使用原有的 chunk_content 逻辑 - return chunk_content(subtitle_text, duration) - - def chunk_by_duration(self, srt_content: str, target_duration: Optional[int] = None) -> List[Chunk]: - """ - Split SRT content into chunks based on duration. - - Args: - srt_content: SRT formatted subtitle text - target_duration: Target chunk duration in seconds (overrides default) - - Returns: - List of Chunk objects - """ - target = target_duration or self.chunk_duration - subtitles = self._parse_srt(srt_content) - - if not subtitles: - return [] - - chunks = [] - current_chunk_texts = [] - current_start = subtitles[0][0] - current_end = current_start - chunk_index = 0 - - for start, end, text in subtitles: - # Check if adding this subtitle would exceed target duration - if current_chunk_texts and (end - current_start) > target: - # Save current chunk - chunks.append(Chunk( - index=chunk_index, - start_time=current_start, - end_time=current_end, - text=' '.join(current_chunk_texts) - )) - chunk_index += 1 - - # Start new chunk - current_chunk_texts = [text] - current_start = start - current_end = end - else: - current_chunk_texts.append(text) - current_end = end - - # Don't forget the last chunk - if current_chunk_texts: - chunks.append(Chunk( - index=chunk_index, - start_time=current_start, - end_time=current_end, - text=' '.join(current_chunk_texts) - )) - - return chunks - - def _parse_srt(self, srt_content: str) -> List[Tuple[float, float, str]]: - """ - Parse SRT content into list of (start_time, end_time, text) tuples. - - Args: - srt_content: SRT formatted text - - Returns: - List of tuples (start_seconds, end_seconds, text) - """ - import re - subtitles = [] - - # Split by double newlines to get individual subtitle entries - entries = re.split(r'\n\s*\n', srt_content.strip()) - - for entry in entries: - lines = entry.strip().split('\n') - if len(lines) < 2: - continue - - # Find the timing line (contains -->) - timing_line = None - for line in lines: - if '-->' in line: - timing_line = line - break - - if not timing_line: - continue - - # Parse timing - match = re.match( - r'\d*\s*(\d{2}:\d{2}:\d{2},\d{3})\s*--\u003e\s*(\d{2}:\d{2}:\d{2},\d{3})', - timing_line.strip() - ) - if not match: - continue - - start_time = self._time_to_seconds(match.group(1)) - end_time = self._time_to_seconds(match.group(2)) - - # Get text (lines after timing line) - text_lines = [] - found_timing = False - for line in lines: - if found_timing: - text_lines.append(line.strip()) - elif '-->' in line: - found_timing = True - - text = ' '.join(text_lines).strip() - if text: - subtitles.append((start_time, end_time, text)) - - return subtitles - - def chunk_by_count(self, srt_text: str, num_chunks: int) -> List[Chunk]: - """ - 按指定段数分割SRT字幕 - - Args: - srt_text: SRT格式字幕文本 - num_chunks: 目标段数 - - Returns: - Chunk对象列表 - """ - print(f"[ContentChunker] 按段数分片: 目标段数={num_chunks}") - - # 解析SRT条目 - entries = self._parse_srt_dict(srt_text) - if not entries: - return [] - - if num_chunks <= 0: - num_chunks = 1 - - # 计算每段应包含的条目数 - total_entries = len(entries) - entries_per_chunk = max(1, total_entries // num_chunks) - - chunks = [] - current_chunk_texts = [] - current_start_time = None - chunk_index = 0 - entry_count = 0 - - for i, entry in enumerate(entries): - if current_start_time is None: - current_start_time = entry['start'] - - current_chunk_texts.append(entry['text']) - entry_count += 1 - - # 检查是否达到每段条目数，或是最后一个条目 - is_last_entry = (i == total_entries - 1) - should_split = (entry_count >= entries_per_chunk and chunk_index < num_chunks - 1) - - if should_split or is_last_entry: - chunk = Chunk( - index=chunk_index, - start_time=self._time_to_seconds(current_start_time), - end_time=self._time_to_seconds(entry['end']), - text='\n'.join(current_chunk_texts) - ) - chunks.append(chunk) - - current_chunk_texts = [] - current_start_time = None - chunk_index += 1 - entry_count = 0 - - print(f"[ContentChunker] 分片完成: 共{len(chunks)}段") - return chunks - - def _parse_srt_dict(self, srt_text: str) -> List[Dict]: - """ - 解析SRT字幕文本，返回字典格式（用于内部使用） - - Returns: - 条目列表，每个条目包含start, end, text - """ - entries = [] - - # 首先检查是否包含SRT时间戳格式 - if '-->' not in srt_text: - # 非SRT格式，按行数分割模拟 - lines = srt_text.strip().split('\n') - # 模拟每10行一个条目，每个条目30秒 - for i in range(0, len(lines), 10): - chunk_lines = lines[i:i+10] - start_sec = i * 3 # 每行约3秒 - end_sec = (i + 10) * 3 - entries.append({ - 'start': self._seconds_to_time(start_sec), - 'end': self._seconds_to_time(end_sec), - 'text': '\n'.join(chunk_lines) - }) - return entries - - # 按空行分割条目 - blocks = re.split(r'\n\s*\n', srt_text.strip()) - - for block in blocks: - lines = block.strip().split('\n') - if len(lines) < 2: - continue - - # 查找时间戳行（通常是第二行，但可能有变体） - time_line = None - for line in lines: - if '-->' in line: - time_line = line - break - - if not time_line: - continue - - # 解析时间戳 - match = self.srt_time_pattern.search(time_line) - if not match: - continue - - start_time = match.group(1) - end_time = match.group(2) - - # 提取文本（时间戳行之后的所有行） - time_line_idx = lines.index(time_line) if time_line in lines else 1 - text_lines = lines[time_line_idx + 1:] - text = '\n'.join(text_lines).strip() - - if text: - entries.append({ - 'start': start_time, - 'end': end_time, - 'text': text - }) - - return entries - - def _time_to_seconds(self, time_str: str) -> float: - """将SRT时间格式转换为秒数""" - # 处理 "00:05:23,450" 格式 - time_str = time_str.replace(',', '.') - parts = time_str.split(':') - - if len(parts) == 3: - hours = int(parts[0]) - minutes = int(parts[1]) - seconds = float(parts[2]) - return hours * 3600 + minutes * 60 + seconds - - return 0.0 - - def _seconds_to_time(self, seconds: int) -> str: - """将秒数转换为SRT时间格式""" - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - secs = seconds % 60 - return f"{hours:02d}:{minutes:02d}:{secs:02d},000" - - def get_chunking_strategy(self, total_duration: float) -> dict: - """ - Determine chunking strategy based on total content duration. - - Args: - total_duration: Total duration in seconds - - Returns: - Dictionary with chunking strategy - """ - if total_duration < 600: # Less than 10 minutes - return { - 'strategy': 'direct', - 'chunk_count': 1, - 'description': 'Short content, no chunking needed' - } - elif total_duration < 1800: # 10-30 minutes - return { - 'strategy': 'truncate', - 'chunk_count': 1, - 'description': 'Medium content, single chunk with possible truncation' - } - elif total_duration < 3600: # 30-60 minutes - return { - 'strategy': 'two_part', - 'chunk_count': 2, - 'description': 'Long content, split into 2 chunks' - } - else: # 60+ minutes - chunk_count = int(total_duration / self.chunk_duration) + 1 - return { - 'strategy': 'multi_part', - 'chunk_count': chunk_count, - 'description': f'Very long content, split into {chunk_count} chunks' - } - - -def get_chunking_strategy(duration_seconds: int) -> Dict: - """ - C5.2: 根据视频时长获取分段策略 - - Args: - duration_seconds: 视频时长（秒） - - Returns: - 分段策略字典 - """ - if duration_seconds < 600: - # 少于10分钟：直接处理，无需分片 - return { - 'use_chunking': False, - 'strategy': 'direct', - 'description': '短视频，直接处理' - } - - elif duration_seconds < 1800: - # 10-30分钟：截断处理 - return { - 'use_chunking': False, - 'strategy': 'truncate', - 'max_chars': 8000, - 'description': '中等视频，截断至8000字符处理' - } - - elif duration_seconds < 3600: - # 30-60分钟：分为2段 - return { - 'use_chunking': True, - 'num_chunks': 2, - 'strategy': 'chunk_2', - 'chunk_duration': duration_seconds // 2, - 'description': '长视频，分为2段处理' - } - - else: - # 60分钟以上：每15分钟一段 - num_chunks = max(4, duration_seconds // 900) - return { - 'use_chunking': True, - 'chunk_duration': 900, - 'num_chunks': num_chunks, - 'strategy': 'chunk_n', - 'description': f'超长视频，分为{num_chunks}段，每段约15分钟' - } - - -# 便捷函数 -def chunk_content(srt_text: str, duration_seconds: int) -> List[Chunk]: - """ - 根据视频时长自动选择最佳分片策略 - - Args: - srt_text: SRT字幕文本 - duration_seconds: 视频时长（秒） - - Returns: - Chunk列表 - """ - strategy = get_chunking_strategy(duration_seconds) - chunker = ContentChunker() - - if not strategy['use_chunking']: - # 不需要分片，返回单一大段 - return [Chunk( - text=srt_text[:strategy.get('max_chars', len(srt_text))], - start_time="00:00:00,000", - end_time=chunker._seconds_to_time(duration_seconds), - index=0 - )] - - if 'num_chunks' in strategy and strategy['strategy'] == 'chunk_2': - return chunker.chunk_by_count(srt_text, strategy['num_chunks']) - else: - return chunker.chunk_by_duration(srt_text, strategy.get('chunk_duration', 900)) diff --git a/skills/obsidian-kb/src/deep_analyzer.py b/skills/obsidian-kb/src/deep_analyzer.py deleted file mode 100644 index 5b5dcb0..0000000 --- a/skills/obsidian-kb/src/deep_analyzer.py +++ /dev/null @@ -1,438 +0,0 @@ -"""Obsidian KB - 深度分析器 -C4.2: 实现深度分析，根据时长选择不同分析策略 -""" -from typing import Dict, Any, List, Optional -from src.models import Content, AnalysisResult, ContentStatus, TimelineEntry -from src.config import get_config - - -class DeepAnalyzer: - """深度内容分析器 - 根据内容时长选择不同策略""" - - # 时长阈值（秒） - SHORT_VIDEO_THRESHOLD = 600 # 10分钟 - LONG_VIDEO_THRESHOLD = 1800 # 30分钟 - - def __init__(self, ai_client=None): - self.config = get_config() - self.ai_client = ai_client - - def analyze(self, extractor_data: Dict[str, Any]) -> AnalysisResult: - """ - 根据内容时长选择分析策略（从提取器数据接收输入） - - Args: - extractor_data: 提取器返回的数据结构，包含： - - metadata: Dict[str, Any] - 元数据（标题、作者、时长等） - - stats: Dict[str, Any] - 统计信息 - - subtitle: str - 字幕/文本内容 - - description: str - 内容描述 - - Returns: - AnalysisResult分析结果 - """ - metadata = extractor_data.get('metadata', {}) - subtitle = extractor_data.get('subtitle', '') - description = extractor_data.get('description', '') - - # 从metadata中提取信息 - title = metadata.get('title', '未知') - duration = metadata.get('duration', 0) - content_type = metadata.get('content_type', 'video') - - print(f"[DeepAnalyzer] 开始分析: {title}") - print(f"[DeepAnalyzer] 类型: {content_type}, 时长: {duration}秒") - print(f"[DeepAnalyzer] 字幕存在: {bool(subtitle and subtitle.strip())}") - - # 检测是否有字幕 - has_subtitle = bool(subtitle and subtitle.strip()) - - if has_subtitle: - # 有字幕：使用 AI 深度分析 - print(f"[DeepAnalyzer] 使用AI深度分析（有字幕）") - - # 根据时长选择策略 - if duration < self.SHORT_VIDEO_THRESHOLD: - # 短视频: 轻量分析 - print(f"[DeepAnalyzer] 使用轻量分析策略 (时长 < 10分钟)") - result = self._analyze_short_with_subtitle(title, subtitle, duration) - else: - # 长视频: 详细时间轴分析 - print(f"[DeepAnalyzer] 使用详细时间轴分析策略 (时长 >= 10分钟)") - result = self._analyze_long_with_subtitle(title, subtitle, duration, content_type) - else: - # 无字幕：降级处理 - print(f"[DeepAnalyzer] 降级处理（无字幕）") - result = self._analyze_fallback(title, description, metadata) - - return result - - def _analyze_short(self, content: Content, content_text: str) -> AnalysisResult: - """短视频/短内容轻量分析（旧版接口，保留兼容）""" - result = AnalysisResult() - - # 截断文本 - text_sample = self._get_text_sample(content_text, max_chars=5000) - - # 生成简洁分析 - if self.ai_client: - analysis = self._analyze_with_ai_short(text_sample, content) - result.main_topics = analysis.get('topics', []) - result.key_insights = analysis.get('insights', []) - result.action_items = analysis.get('actions', []) - result.summary = analysis.get('summary', '') - else: - # 无AI时的简单分析 - result.main_topics = ["主题1", "主题2", "主题3"] - result.key_insights = ["关键见解1", "关键见解2"] - result.action_items = ["建议行动1"] - result.summary = content.summary or "无摘要" - - # 短视频的时间轴较简单 - result.timeline = self._generate_simple_timeline(content) - - return result - - def _analyze_short_with_subtitle(self, title: str, subtitle: str, duration: int) -> AnalysisResult: - """有字幕的短视频轻量分析（新接口）""" - result = AnalysisResult() - - # 截断文本 - text_sample = self._get_text_sample(subtitle, max_chars=5000) - - # 生成简洁分析 - if self.ai_client: - analysis = self._analyze_with_ai_short_text(text_sample, title, duration) - result.main_topics = analysis.get('topics', []) - result.key_insights = analysis.get('insights', []) - result.action_items = analysis.get('actions', []) - result.summary = analysis.get('summary', '') - else: - # 无AI时的简单分析 - result.main_topics = ["主题1", "主题2", "主题3"] - result.key_insights = ["关键见解1", "关键见解2"] - result.action_items = ["建议行动1"] - result.summary = title or "无摘要" - - # 短视频的时间轴较简单 - result.timeline = self._generate_simple_timeline_from_duration(title, duration) - - return result - - def _analyze_long(self, content: Content, content_text: str) -> AnalysisResult: - """长视频详细时间轴分析（旧版接口，保留兼容）""" - result = AnalysisResult() - - # 如果有AI客户端且支持分段分析 - if self.ai_client and hasattr(self.ai_client, '_deep_analyze'): - # 使用AI客户端的分段分析能力 - analysis = self.ai_client._deep_analyze( - content_text=content_text, - content_type=content.content_type.value, - duration=content.duration - ) - result.main_topics = analysis.get('topics', []) - result.key_insights = analysis.get('insights', []) - result.action_items = analysis.get('actions', []) - result.summary = analysis.get('summary', '') - result.full_analysis = analysis.get('full_analysis', '') - result.timeline = analysis.get('timeline', []) - else: - # 基础分段分析 - result = self._analyze_long_basic(content, content_text) - - return result - - def _analyze_long_with_subtitle(self, title: str, subtitle: str, duration: int, content_type: str) -> AnalysisResult: - """有字幕的长视频详细时间轴分析（新接口）""" - result = AnalysisResult() - - # 如果有AI客户端且支持分段分析 - if self.ai_client and hasattr(self.ai_client, '_deep_analyze'): - # 使用AI客户端的分段分析能力 - analysis = self.ai_client._deep_analyze( - content_text=subtitle, - content_type=content_type, - duration=duration - ) - result.main_topics = analysis.get('topics', []) - result.key_insights = analysis.get('insights', []) - result.action_items = analysis.get('actions', []) - result.summary = analysis.get('summary', '') - result.full_analysis = analysis.get('full_analysis', '') - result.timeline = analysis.get('timeline', []) - else: - # 基础分段分析 - result = self._analyze_long_basic_new(title, subtitle, duration) - - return result - - def _analyze_fallback(self, title: str, description: str, metadata: Dict[str, Any]) -> AnalysisResult: - """ - 无字幕降级处理：使用 description 或默认提示，只返回 default_tags - - Args: - title: 内容标题 - description: 内容描述 - metadata: 元数据 - - Returns: - AnalysisResult 分析结果（默认标签） - """ - result = AnalysisResult() - - # 默认标签 - default_tags = ["未分类", "待处理"] - - # 尝试从metadata和description提取信息 - author = metadata.get('author', '') - source = metadata.get('source', '') - - # 构建摘要 - if description and description.strip(): - # 使用 description 的前200字作为摘要 - summary = description[:200] + "..." if len(description) > 200 else description - print(f"[DeepAnalyzer] 降级处理: 使用 description 作为摘要") - else: - summary = f"无字幕内容，标题：{title}" - print(f"[DeepAnalyzer] 降级处理: 使用标题作为摘要") - - # 尝试从标题和作者提取标签 - tags = default_tags.copy() - if author: - tags.append(f"作者:{author}") - if source: - tags.append(f"来源:{source}") - - # 如果有AI客户端，尝试从description中提取标签 - if self.ai_client and description: - try: - prompt = f"""请从以下内容描述中提取3-5个标签： - -标题：{title} -描述：{description[:1000]} - -请返回JSON格式： -{{ - "tags": ["标签1", "标签2", "标签3"] -}}""" - response = self.ai_client.generate_json(prompt) - extracted_tags = response.get('tags', []) - if extracted_tags: - tags.extend(extracted_tags) - print(f"[DeepAnalyzer] 降级处理: AI 提取了 {len(extracted_tags)} 个标签") - except Exception as e: - print(f"[DeepAnalyzer] 降级处理: AI 标签提取失败: {e}") - - # 设置结果 - result.summary = summary - result.main_topics = tags[:3] # 前3个作为主题 - result.recommended_tags = tags - result.key_insights = ["无字幕内容，仅能从描述获取基本信息"] - result.action_items = ["建议补充字幕以便深度分析"] - result.timeline = [] # 无字幕无法生成时间轴 - - print(f"[DeepAnalyzer] 降级处理完成: 主题数={len(result.main_topics)}, 标签数={len(result.recommended_tags)}") - - return result - - def _analyze_long_basic(self, content: Content, content_text: str) -> AnalysisResult: - """基础长视频分析（无AI时，旧版接口）""" - result = AnalysisResult() - - # 简单的文本分割分析 - duration = content.duration or 3600 - - # 根据时长决定分割段数 - if duration < 3600: # 30-60分钟 - num_segments = 3 - elif duration < 7200: # 60-120分钟 - num_segments = 4 - else: # >120分钟 - num_segments = 6 - - # 生成时间轴 - segment_duration = duration // num_segments - timeline = [] - - for i in range(num_segments): - start_sec = i * segment_duration - end_sec = min((i + 1) * segment_duration, duration) - - timeline.append({ - 'index': i + 1, - 'start_time': self._format_time(start_sec), - 'end_time': self._format_time(end_sec), - 'title': f"第{i+1}部分", - 'summary': f"时间段 {self._format_time(start_sec)} - {self._format_time(end_sec)} 的内容" - }) - - result.timeline = timeline - result.main_topics = ["主题A", "主题B", "主题C", "主题D"] - result.key_insights = ["深入见解1", "深入见解2", "深入见解3"] - result.action_items = ["建议行动1", "建议行动2"] - result.summary = f"这是一个{self._format_time(duration)}的长视频，分为{num_segments}个主要部分进行分析。" - - return result - - def _analyze_long_basic_new(self, title: str, subtitle: str, duration: int) -> AnalysisResult: - """基础长视频分析（无AI时，新版接口）""" - result = AnalysisResult() - - # 根据时长决定分割段数 - if duration < 3600: # 30-60分钟 - num_segments = 3 - elif duration < 7200: # 60-120分钟 - num_segments = 4 - else: # >120分钟 - num_segments = 6 - - # 生成时间轴 - segment_duration = duration // num_segments - timeline = [] - - for i in range(num_segments): - start_sec = i * segment_duration - end_sec = min((i + 1) * segment_duration, duration) - - timeline.append({ - 'index': i + 1, - 'start_time': self._format_time(start_sec), - 'end_time': self._format_time(end_sec), - 'title': f"第{i+1}部分", - 'summary': f"时间段 {self._format_time(start_sec)} - {self._format_time(end_sec)} 的内容" - }) - - result.timeline = timeline - result.main_topics = ["主题A", "主题B", "主题C", "主题D"] - result.key_insights = ["深入见解1", "深入见解2", "深入见解3"] - result.action_items = ["建议行动1", "建议行动2"] - result.summary = f"标题：{title}，时长：{self._format_time(duration)}，分为{num_segments}个主要部分进行分析。" - - return result - - def _analyze_with_ai_short(self, text: str, content: Content) -> Dict[str, Any]: - """使用AI进行短视频分析（旧版接口）""" - prompt = f"""请分析以下内容，以JSON格式返回： - -{text[:3000]} - -请返回以下格式的JSON： -{{ - "summary": "整体摘要（100字以内）", - "topics": ["主题1", "主题2", "主题3"], - "insights": ["关键见解1", "关键见解2"], - "actions": ["建议行动1"] -}}""" - - try: - response = self.ai_client.generate_json(prompt) - return response - except Exception as e: - print(f"[DeepAnalyzer] AI分析失败: {e}") - return { - "summary": content.summary or "分析失败", - "topics": ["主题1", "主题2", "主题3"], - "insights": ["无法获取深入见解"], - "actions": [] - } - - def _analyze_with_ai_short_text(self, text: str, title: str, duration: int) -> Dict[str, Any]: - """使用AI进行短视频分析（新版接口）""" - prompt = f"""请分析以下视频字幕内容，以JSON格式返回： - -标题：{title} -时长：{self._format_time(duration)} - -字幕内容： -{text[:3000]} - -请返回以下格式的JSON： -{{ - "summary": "整体摘要（100字以内）", - "topics": ["主题1", "主题2", "主题3"], - "insights": ["关键见解1", "关键见解2"], - "actions": ["建议行动1"] -}}""" - - try: - response = self.ai_client.generate_json(prompt) - return response - except Exception as e: - print(f"[DeepAnalyzer] AI分析失败: {e}") - return { - "summary": title or "分析失败", - "topics": ["主题1", "主题2", "主题3"], - "insights": ["无法获取深入见解"], - "actions": [] - } - - def _get_text_sample(self, text: str, max_chars: int) -> str: - """获取文本样本""" - if len(text) <= max_chars: - return text - - # 分段采样 - part_size = max_chars // 3 - return ( - text[:part_size] + - "\n\n[...省略中间部分...]\n\n" + - text[len(text)//2 - part_size//2:len(text)//2 + part_size//2] + - "\n\n[...省略...]\n\n" + - text[-part_size:] - ) - - def _generate_simple_timeline(self, content: Content) -> List[Dict[str, Any]]: - """生成简单时间轴（旧版接口）""" - duration = content.duration or 600 - - # 短视频分为2-3段 - segments = 2 if duration < 300 else 3 - segment_duration = duration // segments - - timeline = [] - for i in range(segments): - start = i * segment_duration - end = min((i + 1) * segment_duration, duration) - timeline.append({ - 'index': i + 1, - 'start_time': self._format_time(start), - 'end_time': self._format_time(end), - 'title': f"段落 {i+1}", - 'summary': f"从{self._format_time(start)}到{self._format_time(end)}的内容" - }) - - return timeline - - def _generate_simple_timeline_from_duration(self, title: str, duration: int) -> List[Dict[str, Any]]: - """生成简单时间轴（新版接口）""" - duration = duration or 600 - - # 短视频分为2-3段 - segments = 2 if duration < 300 else 3 - segment_duration = duration // segments - - timeline = [] - for i in range(segments): - start = i * segment_duration - end = min((i + 1) * segment_duration, duration) - timeline.append({ - 'index': i + 1, - 'start_time': self._format_time(start), - 'end_time': self._format_time(end), - 'title': f"段落 {i+1}", - 'summary': f"从{self._format_time(start)}到{self._format_time(end)}的内容" - }) - - return timeline - - def _format_time(self, seconds: int) -> str: - """格式化秒数为时间字符串""" - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - secs = seconds % 60 - - if hours > 0: - return f"{hours}:{minutes:02d}:{secs:02d}" - else: - return f"{minutes}:{secs:02d}" diff --git a/skills/obsidian-kb/src/extractors/__init__.py b/skills/obsidian-kb/src/extractors/__init__.py deleted file mode 100644 index e00efea..0000000 --- a/skills/obsidian-kb/src/extractors/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Extractors module for obsidian-kb - -from .base import Extractor, ExtractionError -from .bilibili_extractor import BilibiliExtractor -from .ocr_processor import OCRProcessor - -# 以下模块待实现 -# from .subtitle_detector import SubtitleDetector -# from .hard_subtitle_extractor import HardSubtitleExtractor - -__all__ = [ - 'Extractor', - 'ExtractionError', - 'BilibiliExtractor', - 'OCRProcessor', - # 'SubtitleDetector', - # 'HardSubtitleExtractor' -] diff --git a/skills/obsidian-kb/src/extractors/base.py b/skills/obsidian-kb/src/extractors/base.py deleted file mode 100644 index 814c86d..0000000 --- a/skills/obsidian-kb/src/extractors/base.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Extractor 抽象基类 -""" -from abc import ABC, abstractmethod -from typing import Dict, Any, Optional - - -class ExtractionError(Exception): - """提取器异常基类""" - def __init__(self, message: str, source_id: str = "", details: Optional[Dict[str, Any]] = None): - self.message = message - self.source_id = source_id - self.details = details or {} - super().__init__(self.message) - - -class Extractor(ABC): - """ - 内容提取器抽象基类 - - 负责从各种来源（视频、文章等）提取结构化数据 - """ - - @abstractmethod - def extract(self, source_id: str) -> Dict[str, Any]: - """ - 提取源内容 - - Args: - source_id: 来源唯一标识符（如视频ID、文章URL等） - - Returns: - Dict[str, Any]: 包含以下键的字典： - - metadata: Dict[str, Any] - 元数据（标题、作者、时长等） - - stats: Dict[str, Any] - 统计信息（观看次数、点赞数等） - - subtitle: str - 字幕/文本内容 - - description: str - 内容描述 - - Raises: - ExtractionError: 提取失败时抛出 - """ - pass - - def validate_source_id(self, source_id: str) -> bool: - """ - 验证源ID格式是否有效（可选实现） - - Args: - source_id: 来源唯一标识符 - - Returns: - bool: 如果ID格式有效返回True，否则返回False - """ - # 默认实现：非空即有效 - return bool(source_id and source_id.strip()) - - def _handle_error(self, error: Exception, source_id: str, context: Optional[Dict[str, Any]] = None) -> None: - """ - 统一错误处理基类方法 - - Args: - error: 原始异常 - source_id: 来源ID - context: 额外的上下文信息 - - Raises: - ExtractionError: 封装后的提取器异常 - """ - error_details = { - 'error_type': type(error).__name__, - 'error_message': str(error), - 'source_id': source_id, - 'extractor_class': self.__class__.__name__ - } - - if context: - error_details.update(context) - - raise ExtractionError( - message=f"提取失败: {str(error)}", - source_id=source_id, - details=error_details - ) from error diff --git a/skills/obsidian-kb/src/extractors/bilibili_extractor.py b/skills/obsidian-kb/src/extractors/bilibili_extractor.py deleted file mode 100644 index 1c822e7..0000000 --- a/skills/obsidian-kb/src/extractors/bilibili_extractor.py +++ /dev/null @@ -1,750 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -B站视频内容提取器 - -继承自 Extractor 基类，使用 Playwright 提取B站视频信息。 -支持短链接、BV号、完整URL的解析。 -""" - -import asyncio -import json -import logging -import re -from datetime import datetime -from typing import Dict, Any, Optional -from urllib.parse import urlparse - -logger = logging.getLogger(__name__) - -from playwright.async_api import async_playwright -from bs4 import BeautifulSoup - -from .base import Extractor, ExtractionError - -# 导入 shortlink_resolver -import sys -import os -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) -from shortlink_resolver import ShortLinkResolver -from .hard_subtitle_extractor import HardSubtitleExtractor - -# 尝试导入 stealth 支持 -try: - from undetected_playwright import stealth_async - STEALTH_AVAILABLE = True -except ImportError: - STEALTH_AVAILABLE = False - - -class BilibiliExtractor(Extractor): - """ - B站视频内容提取器 - - 使用 Playwright + stealth 模式提取视频元数据、统计数据和字幕 - """ - - # B站相关正则模式 - BV_PATTERN = re.compile(r'BV[a-zA-Z0-9]{10}') - URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/video/(BV[a-zA-Z0-9]{10})') - - def __init__(self, config=None): - """ - 初始化提取器 - - Args: - config: 配置对象（支持 get() 方法） - """ - self.config = config - self._load_config() - self.resolver = ShortLinkResolver() - - def _load_config(self): - """从配置加载参数""" - if self.config: - try: - self.headless = self.config.get('sources.bilibili.extraction.headless', True) - self.timeout = self.config.get('sources.bilibili.extraction.timeout', 30) * 1000 # 转换为毫秒 - except: - # 如果配置获取失败，使用默认值 - self.headless = True - self.timeout = 30000 - else: - self.headless = True - self.timeout = 30000 - - def extract(self, source_id: str) -> Dict[str, Any]: - """ - 提取B站视频信息 - - Args: - source_id: 来源标识（支持BV号、完整URL、短链接） - - Returns: - Dict containing: - - metadata: Dict with title, author, bvid, aid, pubtime, duration, url - - stats: Dict with view, danmaku, reply, favorite, coin, share, like - - subtitle: str - 字幕文本（如果有） - - description: str - 视频简介 - - Raises: - ExtractionError: 提取失败时抛出 - """ - # 验证输入 - if not self.validate_source_id(source_id): - raise ExtractionError(f"无效的源ID: {source_id}", source_id=source_id) - - # 解析并获取BV号 - bvid = self._resolve_bvid(source_id) - - # 如果解析失败，检查是否为短链接 - if not bvid and self.resolver.is_bilibili_short_link(source_id): - # 使用Playwright解析短链接 - try: - bvid = asyncio.run(self._resolve_shortlink_bvid(source_id)) - except Exception as e: - raise ExtractionError(f"短链接解析失败: {e}", source_id=source_id) - - if not bvid: - raise ExtractionError(f"无法解析BV号: {source_id}", source_id=source_id) - - # 使用异步方法提取 - try: - result = asyncio.run(self._extract_async(bvid)) - except Exception as e: - self._handle_error(e, source_id, {'bvid': bvid, 'step': 'async_extraction'}) - - return result - - def _resolve_bvid(self, source_id: str) -> Optional[str]: - """ - 解析源ID，提取BV号 - - Args: - source_id: BV号、URL或短链接 - - Returns: - BV号字符串，解析失败返回None - """ - # 1. 直接BV号 - bv_match = self.BV_PATTERN.fullmatch(source_id.strip()) - if bv_match: - return source_id.strip() - - # 2. 完整URL - url_match = self.URL_PATTERN.match(source_id) - if url_match: - return url_match.group(1) - - # 3. 短链接处理已移至 extract() 方法中的 _resolve_shortlink_bvid() - # 这里不再处理，避免混淆 - - return None - - async def _resolve_shortlink_bvid(self, shortlink: str) -> Optional[str]: - """ - 使用 Playwright 解析B站短链接，获取 BV 号 - - Args: - shortlink: B站短链接（如 https://b23.tv/xxxxx） - - Returns: - BV号字符串，解析失败返回None - """ - async with async_playwright() as playwright: - browser = None - try: - # 启动浏览器（stealth 模式） - browser = await playwright.chromium.launch( - headless=self.headless, - args=[ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-blink-features=AutomationControlled', - '--disable-dev-shm-usage', - '--start-maximized', - ] - ) - - context = await browser.new_context( - viewport={'width': 1920, 'height': 1080}, - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - locale='zh-CN', - timezone_id='Asia/Shanghai', - ) - - # 应用 stealth 模式（如果可用） - if STEALTH_AVAILABLE: - await stealth_async(context) - - page = await context.new_page() - - # 访问短链接（等待重定向完成） - await page.goto(shortlink, wait_until='domcontentloaded', timeout=self.timeout) - await asyncio.sleep(3) # 等待重定向完成 - - # 获取重定向后的 URL - final_url = page.url - - # 从 URL 中提取 BV 号 - bv_match = re.search(r'BV[a-zA-Z0-9]{10}', final_url) - if bv_match: - return bv_match.group(0) - - except Exception as e: - raise ExtractionError(f"短链接解析异常: {e}", source_id=shortlink) - finally: - if browser: - await browser.close() - - return None - - async def _extract_async(self, bvid: str) -> Dict[str, Any]: - """ - 异步提取视频信息 - - Args: - bvid: B站视频BV号 - - Returns: - 提取结果字典 - """ - url = f"https://www.bilibili.com/video/{bvid}" - result = { - 'metadata': {}, - 'stats': {}, - 'subtitle': '', - 'description': '' - } - - async with async_playwright() as playwright: - try: - # 启动浏览器 - browser = await playwright.chromium.launch( - headless=self.headless, - args=[ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-blink-features=AutomationControlled', - '--disable-dev-shm-usage', - '--start-maximized', - ] - ) - - # 创建上下文 - context = await browser.new_context( - viewport={'width': 1920, 'height': 1080}, - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - locale='zh-CN', - timezone_id='Asia/Shanghai', - extra_http_headers={ - 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', - 'Referer': 'https://www.bilibili.com/', - } - ) - - # 应用 stealth 模式（如果可用） - if STEALTH_AVAILABLE: - await stealth_async(context) - - # 创建页面 - page = await context.new_page() - - # 访问页面 - try: - response = await page.goto(url, wait_until='domcontentloaded', timeout=self.timeout) - except Exception: - # 即使 goto 失败，也可能可以访问 API - response = None - - # 等待页面加载（即使HTTP状态码不为200也继续，因为可能可以访问API） - await asyncio.sleep(3) - - # 获取页面内容（不要强制等待 networkidle，避免超时） - try: - content = await page.content() - except Exception as e: - # 如果获取内容失败，可能是因为页面导航，尝试其他方法 - content = "" - - # 尝试多种方法提取 - video_data = await self._try_extract_methods(page, content, bvid) - - if not video_data: - raise ExtractionError("无法提取视频信息", source_id=bvid) - - # 组织返回数据 - result['metadata'] = { - 'title': video_data.get('title', ''), - 'author': video_data.get('author', ''), - 'bvid': video_data.get('bvid', ''), - 'aid': video_data.get('aid', ''), - 'mid': video_data.get('mid', ''), - 'pubtime': video_data.get('pubtime', ''), - 'duration': video_data.get('duration', ''), - 'url': url, - 'tags': video_data.get('tags', ''), - } - - result['stats'] = { - 'view': video_data.get('view', 0), - 'danmaku': video_data.get('danmaku', 0), - 'reply': video_data.get('reply', 0), - 'favorite': video_data.get('favorite', 0), - 'coin': video_data.get('coin', 0), - 'share': video_data.get('share', 0), - 'like': video_data.get('like', 0), - } - - result['description'] = video_data.get('desc', '') - - # 尝试提取字幕 - result['subtitle'] = await self._extract_subtitles(page, bvid) - - await browser.close() - - except Exception as e: - if 'browser' in locals(): - await browser.close() - raise - - return result - - async def _try_extract_methods(self, page, content: str, bvid: str) -> Dict[str, Any]: - """ - 尝试多种方法提取视频信息 - - Args: - page: Playwright page对象 - content: 页面HTML内容 - bvid: BV号 - - Returns: - 视频信息字典 - """ - video_data = {} - - # 方法1: 从 __INITIAL_STATE__ 获取 - try: - initial_state = await page.evaluate('() => typeof window.__INITIAL_STATE__ !== "undefined" ? window.__INITIAL_STATE__ : null') - if initial_state: - video_data = self._parse_initial_state(initial_state) - if video_data and video_data.get('title'): - return video_data - except Exception as e: - pass - - # 方法2: 从 script 标签提取 - try: - soup = BeautifulSoup(content, 'lxml') - script_tags = soup.find_all('script') - for script in script_tags: - if script.string and '__INITIAL_STATE__' in script.string: - match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.+?});', script.string) - if match: - data = json.loads(match.group(1)) - video_data = self._parse_initial_state(data) - if video_data and video_data.get('title'): - return video_data - except Exception as e: - pass - - # 方法3: 使用 API - try: - api_url = "https://api.bilibili.com/x/web-interface/view" - response = await page.request.get(api_url, params={'bvid': bvid}) - - if response.ok: - data = await response.json() - if data.get('code') == 0: - video_data = self._parse_api_response(data.get('data', {})) - if video_data and video_data.get('title'): - return video_data - except Exception as e: - pass - - return video_data - - def _parse_initial_state(self, data: Dict) -> Dict[str, Any]: - """解析 __INITIAL_STATE__ 数据""" - video_data = {} - - try: - video = data.get('videoData', {}) - - video_data['title'] = video.get('title', '') - video_data['bvid'] = video.get('bvid', '') - video_data['aid'] = video.get('aid', '') - - owner = video.get('owner', {}) - video_data['author'] = owner.get('name', '') - video_data['mid'] = owner.get('mid', '') - - stat = video.get('stat', {}) - video_data['view'] = stat.get('view', 0) - video_data['danmaku'] = stat.get('danmaku', 0) - video_data['reply'] = stat.get('reply', 0) - video_data['favorite'] = stat.get('favorite', 0) - video_data['coin'] = stat.get('coin', 0) - video_data['share'] = stat.get('share', 0) - video_data['like'] = stat.get('like', 0) - - pubdate = video.get('pubdate', 0) - video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else '' - - duration = video.get('duration', 0) - video_data['duration'] = self._format_duration(duration) - - video_data['desc'] = video.get('desc', '') - - tags = [] - for tag in video.get('tag', []): - tags.append(tag.get('tag_name', '')) - video_data['tags'] = ', '.join(tags) - - except Exception as e: - pass - - return video_data - - def _parse_api_response(self, data: Dict) -> Dict[str, Any]: - """解析API响应数据""" - video_data = {} - - try: - video_data['title'] = data.get('title', '') - video_data['bvid'] = data.get('bvid', '') - video_data['aid'] = data.get('aid', '') - - owner = data.get('owner', {}) - video_data['author'] = owner.get('name', '') - video_data['mid'] = owner.get('mid', '') - - stat = data.get('stat', {}) - video_data['view'] = stat.get('view', 0) - video_data['danmaku'] = stat.get('danmaku', 0) - video_data['reply'] = stat.get('reply', 0) - video_data['favorite'] = stat.get('favorite', 0) - video_data['coin'] = stat.get('coin', 0) - video_data['share'] = stat.get('share', 0) - video_data['like'] = stat.get('like', 0) - - pubdate = data.get('pubdate', 0) - video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else '' - - duration = data.get('duration', 0) - video_data['duration'] = self._format_duration(duration) - - video_data['desc'] = data.get('desc', '') - - tags = [] - for tag in data.get('tag', []): - tags.append(tag.get('tag_name', '')) - video_data['tags'] = ', '.join(tags) - - except Exception as e: - pass - - return video_data - - async def _extract_subtitles(self, page, bvid: str) -> str: - """ - 提取视频字幕 - - 支持的字幕类型： - 1. 官方 CC 字幕（需要 UP 主手动上传） - 2. AI 自动生成字幕（标记为 ai-zh） - 3. WebVTT 格式字幕 - - 注意：硬字幕（嵌入视频画面内）无法提取 - - Args: - page: Playwright page对象 - bvid: BV号 - - Returns: - 字幕文本（多行字符串） - """ - subtitles = [] - - try: - # 首先通过 API 获取视频信息（包含 cid 和 aid） - api_url = "https://api.bilibili.com/x/web-interface/view" - response = await page.request.get(api_url, params={'bvid': bvid}) - - if not response.ok: - return '' - - data = await response.json() - if data.get('code') != 0: - return '' - - video_data = data.get('data', {}) - cid = video_data.get('cid') - aid = video_data.get('aid') - - if not cid: - return '' - - # 方法1: 使用 WBI API（推荐，支持 AI 字幕） - player_api_url = "https://api.bilibili.com/x/player/wbi/v2" - player_response = await page.request.get(player_api_url, params={'aid': aid, 'cid': cid}) - - if player_response.ok: - player_data = await player_response.json() - if player_data.get('code') == 0: - subtitle_info = player_data.get('data', {}).get('subtitle', {}) - subtitles_list = subtitle_info.get('subtitles', []) - - if subtitles_list: - # 优先选择中文字幕（包括 AI 字幕） - subtitle_item = None - for sub in subtitles_list: - # 优先级: ai-zh > zh-CN > zh-Hans > 其他 - lan = sub.get('lan', '') - if lan == 'ai-zh': - subtitle_item = sub - break - elif lan in ['zh-CN', 'zh-Hans', 'zh']: - if not subtitle_item: - subtitle_item = sub - - # 如果没有找到中文，使用第一个 - if not subtitle_item: - subtitle_item = subtitles_list[0] - - if subtitle_item: - subtitle_url = subtitle_item.get('subtitle_url', '') - if subtitle_url: - full_url = f"https:{subtitle_url}" - sub_response = await page.request.get(full_url) - if sub_response.ok: - sub_data = await sub_response.json() - subtitles = self._parse_subtitle_json(sub_data) - # 如果找到字幕，直接返回 - if subtitles: - lines = [] - for sub in subtitles: - lines.append(f"[{sub['from']} - {sub['to']}] {sub['content']}") - return '\n'.join(lines) - - # 方法2: 如果 WBI 失败，使用旧版 API - player_api_url = "https://api.bilibili.com/x/player/v2" - player_response = await page.request.get(player_api_url, params={'bvid': bvid, 'cid': cid}) - - if player_response.ok: - player_data = await player_response.json() - if player_data.get('code') == 0: - subtitle_info = player_data.get('data', {}).get('subtitle', {}) - subtitles_list = subtitle_info.get('subtitles', []) - - if subtitles_list: - subtitle_item = subtitles_list[0] - subtitle_url = subtitle_item.get('subtitle_url', '') - if subtitle_url: - full_url = f"https:{subtitle_url}" - sub_response = await page.request.get(full_url) - if sub_response.ok: - sub_data = await sub_response.json() - subtitles = self._parse_subtitle_json(sub_data) - except Exception as e: - # 静默失败，不影响视频信息提取 - pass - - # 如果所有方法都没有找到字幕，尝试硬字幕提取（OCR） - if not subtitles: - try: - # 初始化硬字幕提取器 - hard_subtitle_extractor = HardSubtitleExtractor(self.config) - - # 尝试从视频流中提取硬字幕 - # 注意：这里需要传入 cid，所以先从 page 获取 - api_url = "https://api.bilibili.com/x/web-interface/view" - response = await page.request.get(api_url, params={'bvid': bvid}) - - if response.ok: - data = await response.json() - if data.get('code') == 0: - video_data = data.get('data', {}) - cid = video_data.get('cid') - - if cid: - # 使用硬字幕提取器 - hard_result = hard_subtitle_extractor.extract_subtitle(bvid) - - if hard_result and hard_result.get('text'): - logger.info("硬字幕提取成功") - # 格式化结果 - hard_text = hard_result.get('text', '') - segments = hard_result.get('segments', []) - - lines = [] - for seg in segments: - lines.append(f"[{seg['time']:.1f}s] {seg['text']}") - return '\n'.join(lines) - else: - logger.warning("硬字幕提取失败或无结果") - except Exception as e: - logger.warning(f"硬字幕提取异常: {e}") - pass - - # 转换为多行字符串 - if subtitles: - lines = [] - for sub in subtitles: - lines.append(f"[{sub['from']} - {sub['to']}] {sub['content']}") - return '\n'.join(lines) - - return '' - - def _parse_subtitle_json(self, data: Dict) -> list: - """解析字幕JSON数据""" - subtitles = [] - - try: - body = data.get('body', []) - for item in body: - from_time = item.get('from', 0) - to_time = item.get('to', 0) - content = item.get('content', '') - - subtitles.append({ - 'from': self._format_timestamp(from_time), - 'to': self._format_timestamp(to_time), - 'content': content - }) - except Exception as e: - pass - - return subtitles - - def _format_duration(self, seconds: int) -> str: - """格式化时长（秒 -> HH:MM:SS）""" - if not seconds: - return '' - - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - secs = seconds % 60 - - if hours > 0: - return f"{hours:02d}:{minutes:02d}:{secs:02d}" - else: - return f"{minutes:02d}:{secs:02d}" - - def _format_timestamp(self, seconds: float) -> str: - """格式化时间戳""" - minutes = int(seconds // 60) - secs = int(seconds % 60) - return f"{minutes:02d}:{secs:02d}" - - -# 创建测试脚本的辅助函数 -def create_test_script(): - """创建简单的测试脚本""" - script = """#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -\"\"\" -B站提取器测试脚本 -\"\"\" -import sys -import os - -# 添加项目路径 -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) - -from skills.obsidian_kb.src.extractors.bilibili_extractor import BilibiliExtractor -from skills.obsidian_kb.src.config import get_config - - -def main(): - \"\"\"测试提取器\"\"\" - # 获取配置 - config = get_config() - - # 创建提取器 - extractor = BilibiliExtractor(config=config) - - # 测试BV号 - bvid = "BV1MnFNz5EDL" - - print(f"开始提取视频: {bvid}") - print("=" * 60) - - try: - result = extractor.extract(bvid) - - print("\\n提取成功！\\n") - print("=" * 60) - print("元数据 (metadata):") - print("=" * 60) - for key, value in result['metadata'].items(): - print(f" {key}: {value}") - - print("\\n" + "=" * 60) - print("统计数据 (stats):") - print("=" * 60) - for key, value in result['stats'].items(): - print(f" {key}: {value:,}") - - print("\\n" + "=" * 60) - print("简介 (description):") - print("=" * 60) - desc = result['description'] - if len(desc) > 500: - print(desc[:500] + "...") - else: - print(desc) - - print("\\n" + "=" * 60) - print("字幕 (subtitle):") - print("=" * 60) - subtitle = result['subtitle'] - if subtitle: - lines = subtitle.split('\\n') - if len(lines) > 10: - print("\\n".join(lines[:10])) - print(f"\\n...(共 {len(lines)} 条字幕)") - else: - print(subtitle) - else: - print(" (无字幕)") - - print("\\n" + "=" * 60) - print("✅ 测试完成") - print("=" * 60) - - except Exception as e: - print(f"\\n❌ 测试失败: {e}") - import traceback - traceback.print_exc() - - -if __name__ == '__main__': - main() -""" - return script -""" - -示例用法： - -```python -from skills.obsidian_kb.src.extractors.bilibili_extractor import BilibiliExtractor -from skills.obsidian_kb.src.config import get_config - -# 获取配置 -config = get_config() - -# 创建提取器 -extractor = BilibiliExtractor(config=config) - -# 提取视频信息（支持BV号、URL） -result = extractor.extract('BV1MnFNz5EDL') - -# 访问结果 -print(result['metadata']['title']) -print(result['stats']['view']) -print(result['description']) -print(result['subtitle']) -``` -""" diff --git a/skills/obsidian-kb/src/extractors/easyocr_processor.py b/skills/obsidian-kb/src/extractors/easyocr_processor.py deleted file mode 100644 index 3a1c351..0000000 --- a/skills/obsidian-kb/src/extractors/easyocr_processor.py +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -OCR 文字识别处理器（EasyOCR 版本） - -使用 EasyOCR 实现视频帧文字识别功能，作为 PaddleOCR 的替代方案 -主要优势：不依赖 PaddlePaddle，避免 oneDNN 兼容性问题 -""" - -import re -import numpy as np -from typing import List, Dict, Any, Tuple -import logging - -logger = logging.getLogger(__name__) - - -class EasyOCRProcessor: - """ - EasyOCR 文字识别处理器 - - 使用 EasyOCR 从视频帧中识别文字，支持批量识别和结果过滤 - - 相比 PaddleOCR 的优势： - - 不依赖 PaddlePaddle，避免 oneDNN 兼容性问题 - - 安装更简单，依赖更少 - - 支持多种语言（70+ 种） - """ - - # 相似度阈值：用于判断两个文本是否相似（基于编辑距离） - SIMILARITY_THRESHOLD = 0.85 - - # 置信度阈值 - CONFIDENCE_THRESHOLD = 0.7 - - # 最小字幕长度（字符数） - MIN_SUBTITLE_LENGTH = 2 - - def __init__(self, lang: str = 'ch_sim', use_gpu: bool = False, detail: int = 1): - """ - 初始化 OCR 处理器 - - Args: - lang: 语言支持，'ch_sim' 为简体中文，'ch_tra' 为繁体中文 - use_gpu: 是否使用 GPU 加速 - detail: 识别结果详细程度（0=只返回文本，1=返回文本+坐标+置信度） - """ - self.lang = lang - self.use_gpu = use_gpu - self.detail = detail - - # 初始化 EasyOCR 实例 - self._ocr = None - self._initialize_ocr() - - def _initialize_ocr(self): - """初始化 EasyOCR 实例""" - try: - from easyocr import Reader - - self._ocr = Reader( - [self.lang], - gpu=self.use_gpu, - verbose=False # 不显示详细日志 - ) - logger.info("EasyOCR 初始化成功") - except Exception as e: - logger.error(f"EasyOCR 初始化失败: {e}") - raise RuntimeError(f"无法初始化 EasyOCR: {e}") - - def _initialize_ocr(self): - """初始化 EasyOCR 实例""" - try: - from easyocr import Reader - - self._ocr = Reader( - [self.lang], - gpu=self.use_gpu, - verbose=False # 不显示详细日志 - ) - logger.info("EasyOCR 初始化成功") - except Exception as e: - logger.error(f"EasyOCR 初始化失败: {e}") - raise RuntimeError(f"无法初始化 EasyOCR: {e}") - - def recognize_frames(self, frames: List[Tuple[np.ndarray, float]]) -> List[Dict[str, Any]]: - """ - 批量识别帧中的文字 - - Args: - frames: 帧列表，每个元素为 (图像数组, 时间戳) 元组 - - 图像数组：numpy.ndarray，形状为 (H, W, 3) 的 RGB 图像 - - 时间戳：float，单位为秒 - - Returns: - List[Dict[str, Any]]: 识别结果列表，每个结果包含： - - time: float, 时间戳（秒） - - text: str, 识别的文本 - - confidence: float, 置信度 (0-1) - """ - if not frames: - logger.warning("输入帧列表为空") - return [] - - results = [] - - for image, timestamp in frames: - try: - # 调用 EasyOCR 进行识别 - # EasyOCR 返回格式: [[[x1, y1], [x2, y2], [x3, y3], [x4, y4]], (text, confidence, ...)] - # 根据 detail 参数，返回格式不同： - # detail=0: 返回 [text1, text2, ...] - # detail=1: 返回 [[bbox, (text, confidence)], ...] - ocr_result = self._ocr.readtext(image, detail=self.detail) - - if ocr_result: - if self.detail == 0: - # 简单模式：只返回文本列表 - for i, text in enumerate(ocr_result): - if text: - results.append({ - 'time': timestamp, - 'text': str(text), - 'confidence': 1.0 # 简单模式下没有置信度 - }) - else: - # 详细模式：返回边界框+文本+置信度 - for item in ocr_result: - if len(item) >= 2: - bbox = item[0] - text_info = item[1] - - if len(text_info) >= 2: - text, confidence = text_info[0], text_info[1] - - # 跳过空文本 - if not text: - continue - - # 处理置信度：可能是数字或特殊字符 - try: - if isinstance(confidence, str): - # 如果是 '?' 或其他特殊字符，使用默认置信度 - conf_float = 0.5 - else: - conf_float = float(confidence) - except (ValueError, TypeError): - conf_float = 0.5 - - results.append({ - 'time': timestamp, - 'text': text, - 'confidence': conf_float - }) - except Exception as e: - logger.warning(f"帧识别失败 (时间戳 {timestamp}s): {e}") - continue - - logger.info(f"原始识别结果数量: {len(results)}") - - # 过滤和清洗结果 - filtered_results = self._filter_and_clean(results) - logger.info(f"过滤后结果数量: {len(filtered_results)}") - - return filtered_results - - def _filter_and_clean(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - 过滤和清洗识别结果 - - Args: - results: 原始识别结果列表 - - Returns: - List[Dict[str, Any]]: 过滤后的结果列表 - """ - if not results: - return [] - - # 步骤 1: 按置信度过滤 - filtered = self._filter_by_confidence(results) - logger.debug(f"置信度过滤后: {len(filtered)}") - - # 步骤 2: 过滤噪声（纯符号、单字符等） - filtered = self._filter_noise(filtered) - logger.debug(f"噪声过滤后: {len(filtered)}") - - # 步骤 3: 去除重复或相似的文本 - filtered = self._remove_duplicates(filtered) - logger.debug(f"去重后: {len(filtered)}") - - # 步骤 4: 按时间戳排序 - filtered.sort(key=lambda x: x['time']) - - return filtered - - def _filter_by_confidence(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - 按置信度过滤结果 - - Args: - results: 识别结果列表 - - Returns: - List[Dict[str, Any]]: 高置信度结果 - """ - return [r for r in results if r['confidence'] >= self.CONFIDENCE_THRESHOLD] - - def _filter_noise(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - 过滤噪声结果 - - 过滤条件： - - 文本长度小于最小长度 - - 纯符号或空白字符 - - Args: - results: 识别结果列表 - - Returns: - List[Dict[str, Any]]: 过滤后的结果 - """ - filtered = [] - - for result in results: - text = result['text'].strip() - - # 过滤过短的文本 - if len(text) < self.MIN_SUBTITLE_LENGTH: - continue - - # 过滤纯符号或空白字符 - if self._is_noise(text): - continue - - filtered.append(result) - - return filtered - - def _is_noise(self, text: str) -> bool: - """ - 判断文本是否为噪声 - - Args: - text: 要判断的文本 - - Returns: - bool: 如果是噪声返回 True - """ - # 使用正则判断是否只包含符号和空白 - try: - # 移除标点符号后检查是否为空 - # 先替换常见的中文标点 - cleaned = re.sub(r'[^\w\u4e00-\u9fff]', '', text) - return len(cleaned) == 0 - except: - # 如果正则出错，保守处理，不算噪声 - return False - - def _remove_duplicates(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - 去除重复或相似的文本 - - 基于编辑距离（Levenshtein distance）判断文本相似度 - - Args: - results: 识别结果列表（假设已按时间戳排序） - - Returns: - List[Dict[str, Any]]: 去重后的结果 - """ - if not results: - return [] - - deduped = [results[0]] - - for current in results[1:]: - last = deduped[-1] - - # 计算相似度 - similarity = self._calculate_similarity(current['text'], last['text']) - - # 如果相似度低于阈值，则保留 - if similarity < self.SIMILARITY_THRESHOLD: - deduped.append(current) - else: - # 相似文本：保留置信度更高的那个 - if current['confidence'] > last['confidence']: - deduped[-1] = current - - return deduped - - def _calculate_similarity(self, text1: str, text2: str) -> float: - """ - 计算两个文本的相似度（基于归一化编辑距离） - - Args: - text1: 文本1 - text2: 文本2 - - Returns: - float: 相似度 (0-1)，1 表示完全相同 - """ - if text1 == text2: - return 1.0 - - len1, len2 = len(text1), len(text2) - - if len1 == 0 or len2 == 0: - return 0.0 - - # 动态规划计算编辑距离 - dp = np.zeros((len1 + 1, len2 + 1)) - - for i in range(len1 + 1): - dp[i, 0] = i - - for j in range(len2 + 1): - dp[0, j] = j - - for i in range(1, len1 + 1): - for j in range(1, len2 + 1): - if text1[i-1] == text2[j-1]: - dp[i, j] = dp[i-1, j-1] - else: - dp[i, j] = min( - dp[i-1, j] + 1, # 删除 - dp[i, j-1] + 1, # 插入 - dp[i-1, j-1] + 1 # 替换 - ) - - edit_distance = dp[len1, len2] - max_len = max(len1, len2) - - # 归一化相似度：1 - (编辑距离 / 最大长度) - similarity = 1.0 - (edit_distance / max_len) - - return similarity - - def get_ocr_info(self) -> Dict[str, Any]: - """ - 获取 OCR 处理器信息 - - Returns: - Dict[str, Any]: OCR 配置信息 - """ - return { - 'engine': 'easyocr', - 'lang': self.lang, - 'use_gpu': self.use_gpu, - 'detail': self.detail, - 'confidence_threshold': self.CONFIDENCE_THRESHOLD, - 'similarity_threshold': self.SIMILARITY_THRESHOLD, - 'min_subtitle_length': self.MIN_SUBTITLE_LENGTH - } diff --git a/skills/obsidian-kb/src/extractors/hard_subtitle_extractor.py b/skills/obsidian-kb/src/extractors/hard_subtitle_extractor.py deleted file mode 100644 index ac9d48a..0000000 --- a/skills/obsidian-kb/src/extractors/hard_subtitle_extractor.py +++ /dev/null @@ -1,435 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -硬字幕提取器 - 主协调器 - -整合 VideoStreamExtractor、SubtitleDetector 和 OCRProcessor， -实现完整的 B站硬字幕提取流程。 -""" - -import logging -from typing import Dict, Any, List, Optional - -from .video_stream_extractor import VideoStreamExtractor -from .subtitle_detector import SubtitleDetector -from .ocr_processor import OCRProcessor - -logger = logging.getLogger(__name__) - - -class HardSubtitleExtractor: - """ - 硬字幕提取器（主协调器） - - 职责： - - 协调视频流提取、字幕区域检测和 OCR 识别 - - 实现完整的硬字幕提取流程 - - 提供统一的接口返回字幕数据 - """ - - def __init__(self, config: Optional[Dict] = None): - """ - 初始化硬字幕提取器 - - Args: - config: 配置对象 - """ - self.config = config or {} - - # 初始化子模块 - self.video_extractor = VideoStreamExtractor() - self.subtitle_detector = SubtitleDetector( - cache_size=self._get_config('hard_subtitle.detection.cache_region', 50) - ) - self.ocr_processor = OCRProcessor( - use_angle_cls=True, - lang='ch', - use_gpu=self._get_config('hard_subtitle.ocr.use_gpu', False), - show_log=False - ) - - def _get_config(self, key: str, default: Any = None) -> Any: - """获取配置值""" - if not self.config: - return default - keys = key.split('.') - value = self.config - for k in keys: - value = value.get(k) - if value is None: - return default - return value - - def extract_subtitle( - self, - bvid: str, - interval_seconds: float = None, - max_frames: int = None, - start_time: float = 0.0 - ) -> Dict[str, Any]: - """ - 提取硬字幕 - - Args: - bvid: B站视频 BV 号 - interval_seconds: 提取间隔（秒） - max_frames: 最大帧数 - start_time: 开始时间（秒） - - Returns: - Dict[str, Any]: 字幕提取结果，包含： - - subtitle_type: "hard_subtitle" - - text: 完整字幕文本 - - segments: 字幕分段列表 - - confidence: 平均置信度 - - source: "paddleocr" - """ - # 使用配置值或默认值 - interval_seconds = interval_seconds or self._get_config('hard_subtitle.video.interval_seconds', 5.0) - max_frames = max_frames or self._get_config('hard_subtitle.video.max_frames', 500) - use_download = self._get_config('hard_subtitle.video.stream_mode', True) - - logger.info(f"开始提取硬字幕: {bvid}") - logger.info(f"参数: 间隔={interval_seconds}秒, 最大帧数={max_frames}, 开始时间={start_time}秒") - - try: - # 步骤1: 提取视频帧 - logger.info("步骤1: 提取视频帧...") - frames = self.video_extractor.extract_frames_from_bvid( - bvid=bvid, - interval_seconds=interval_seconds, - max_frames=max_frames, - start_time=start_time, - use_download=use_download - ) - - if not frames: - logger.warning("未提取到任何视频帧") - return { - 'subtitle_type': 'hard_subtitle', - 'text': '', - 'segments': [], - 'confidence': 0.0, - 'source': 'paddleocr' - } - - logger.info(f"成功提取 {len(frames)} 帧图像") - - # 步骤2: 检测字幕区域 - logger.info("步骤2: 检测字幕区域...") - use_cache = self._get_config('hard_subtitle.detection.cache_region', True) - subtitle_region = None - - # 使用第一帧检测字幕区域 - if frames: - first_frame = frames[0]['image'] - height, width = first_frame.shape[:2] - - # 尝试自动检测 - subtitle_region = self.subtitle_detector.auto_detect_subtitle_region( - first_frame, - video_id=bvid, - use_cache=use_cache - ) - - logger.info(f"字幕区域检测完成: x={subtitle_region[0]}, y={subtitle_region[1]}, " - f"width={subtitle_region[2]}, height={subtitle_region[3]}") - - # 如果检测失败，使用预设区域 - if not subtitle_region and frames: - height, width = frames[0]['image'].shape[:2] - default_region = self._get_config('hard_subtitle.detection.default_region', 'bottom') - subtitle_region = self.subtitle_detector.get_preset_region( - default_region, - height, - width - ) - logger.info(f"使用预设字幕区域: {default_region}") - - # 步骤3: 裁剪字幕区域 - logger.info("步骤3: 裁剪字幕区域...") - if not subtitle_region: - logger.warning("无法获取字幕区域，跳过裁剪") - # 使用完整帧 - cropped_frames = [frame['image'] for frame in frames] - else: - x, y, w, h = subtitle_region - cropped_frames = [] - - for frame in frames: - image = frame['image'] - # 确保坐标在图像范围内 - y = min(y, image.shape[0] - 1) - x = min(x, image.shape[1] - 1) - h = min(h, image.shape[0] - y) - w = min(w, image.shape[1] - x) - - # 裁剪字幕区域 - cropped = image[y:y+h, x:x+w] - cropped_frames.append(cropped) - - logger.info(f"裁剪完成: {len(cropped_frames)} 帧字幕区域") - - # 步骤4: OCR 识别 - logger.info("步骤4: OCR 识别...") - # 准备 OCR 输入格式 (image, timestamp) - ocr_input = [(cropped_frames[i], frames[i]['timestamp']) - for i in range(min(len(cropped_frames), len(frames)))] - - ocr_results = self.ocr_processor.recognize_frames(ocr_input) - logger.info(f"OCR 识别完成: {len(ocr_results)} 条结果") - - # 步骤5: 整理结果 - logger.info("步骤5: 整理字幕结果...") - subtitle_text = self._format_subtitle_text(ocr_results) - segments = ocr_results - confidence = self._calculate_average_confidence(ocr_results) - - logger.info(f"平均置信度: {confidence:.3f}") - logger.info(f"字幕文本长度: {len(subtitle_text)} 字符") - - return { - 'subtitle_type': 'hard_subtitle', - 'text': subtitle_text, - 'segments': segments, - 'confidence': confidence, - 'source': 'paddleocr' - } - - except Exception as e: - logger.error(f"硬字幕提取失败: {e}", exc_info=True) - # 返回空结果，不阻塞主流程 - return { - 'subtitle_type': 'hard_subtitle', - 'text': '', - 'segments': [], - 'confidence': 0.0, - 'source': 'paddleocr' - } - - def _format_subtitle_text(self, ocr_results: List[Dict[str, Any]]) -> str: - """ - 格式化字幕文本 - - Args: - ocr_results: OCR 识别结果列表 - - Returns: - str: 格式化后的字幕文本 - """ - if not ocr_results: - return '' - - # 按时间戳排序 - sorted_results = sorted(ocr_results, key=lambda x: x['time']) - - # 生成字幕文本（带时间戳） - lines = [] - for result in sorted_results: - time_str = f"[{result['time']:.1f}s]" - lines.append(f"{time_str} {result['text']}") - - return '\n'.join(lines) - - def _calculate_average_confidence(self, ocr_results: List[Dict[str, Any]]) -> float: - """ - 计算平均置信度 - - Args: - ocr_results: OCR 识别结果列表 - - Returns: - float: 平均置信度 - """ - if not ocr_results: - return 0.0 - - confidences = [r.get('confidence', 0.0) for r in ocr_results] - return sum(confidences) / len(confidences) if confidences else 0.0 - - def extract_subtitle_from_url( - self, - video_url: str, - interval_seconds: float = None, - max_frames: int = None, - start_time: float = 0.0 - ) -> Dict[str, Any]: - """ - 从视频 URL 提取硬字幕 - - Args: - video_url: 视频流 URL - interval_seconds: 提取间隔（秒） - max_frames: 最大帧数 - start_time: 开始时间（秒） - - Returns: - Dict[str, Any]: 字幕提取结果 - """ - logger.info(f"从 URL 提取硬字幕: {video_url[:100]}...") - - # 使用配置值或默认值 - interval_seconds = interval_seconds or self._get_config('hard_subtitle.video.interval_seconds', 5.0) - max_frames = max_frames or self._get_config('hard_subtitle.video.max_frames', 500) - - try: - # 提取视频帧 - frames = self.video_extractor.extract_frames( - video_url=video_url, - interval_seconds=interval_seconds, - max_frames=max_frames, - start_time=start_time - ) - - if not frames: - logger.warning("未提取到任何视频帧") - return { - 'subtitle_type': 'hard_subtitle', - 'text': '', - 'segments': [], - 'confidence': 0.0, - 'source': 'paddleocr' - } - - # 后续流程与从 BV 号提取相同 - return self._process_frames_to_subtitle(frames, video_url) - - except Exception as e: - logger.error(f"从 URL 提取字幕失败: {e}", exc_info=True) - return { - 'subtitle_type': 'hard_subtitle', - 'text': '', - 'segments': [], - 'confidence': 0.0, - 'source': 'paddleocr' - } - - def _process_frames_to_subtitle( - self, - frames: List[Dict[str, Any]], - video_id: str - ) -> Dict[str, Any]: - """ - 处理视频帧到字幕（共享逻辑） - - Args: - frames: 帧列表 - video_id: 视频 ID - - Returns: - Dict[str, Any]: 字幕结果 - """ - # 检测字幕区域 - use_cache = self._get_config('hard_subtitle.detection.cache_region', True) - subtitle_region = None - - if frames: - first_frame = frames[0]['image'] - height, width = first_frame.shape[:2] - subtitle_region = self.subtitle_detector.auto_detect_subtitle_region( - first_frame, - video_id=video_id, - use_cache=use_cache - ) - - # 如果检测失败，使用预设区域 - if not subtitle_region and frames: - height, width = frames[0]['image'].shape[:2] - default_region = self._get_config('hard_subtitle.detection.default_region', 'bottom') - subtitle_region = self.subtitle_detector.get_preset_region( - default_region, - height, - width - ) - - # 裁剪字幕区域 - if not subtitle_region: - cropped_frames = [frame['image'] for frame in frames] - else: - x, y, w, h = subtitle_region - cropped_frames = [] - for frame in frames: - image = frame['image'] - y = min(y, image.shape[0] - 1) - x = min(x, image.shape[1] - 1) - h = min(h, image.shape[0] - y) - w = min(w, image.shape[1] - x) - cropped = image[y:y+h, x:x+w] - cropped_frames.append(cropped) - - # OCR 识别 - ocr_input = [(cropped_frames[i], frames[i]['timestamp']) - for i in range(min(len(cropped_frames), len(frames)))] - ocr_results = self.ocr_processor.recognize_frames(ocr_input) - - # 整理结果 - subtitle_text = self._format_subtitle_text(ocr_results) - confidence = self._calculate_average_confidence(ocr_results) - - return { - 'subtitle_type': 'hard_subtitle', - 'text': subtitle_text, - 'segments': ocr_results, - 'confidence': confidence, - 'source': 'paddleocr' - } - - -# 测试函数 -def test_hard_subtitle_extractor(): - """测试硬字幕提取器""" - print("=" * 60) - print("硬字幕提取器测试") - print("=" * 60) - - # 创建提取器 - extractor = HardSubtitleExtractor() - - # 测试视频 - test_bvid = "BV1KtFzzqEUG" - - print(f"\n测试视频: {test_bvid}") - print("=" * 60) - - try: - # 测试1: 提取硬字幕（使用较少帧数进行快速测试） - print("\n【测试1】提取硬字幕...") - print(" 参数: 间隔=5秒, 最大帧数=5, 开始时间=0秒") - - result = extractor.extract_subtitle( - bvid=test_bvid, - interval_seconds=5.0, - max_frames=5, - start_time=0.0 - ) - - print(f"\n✅ 提取完成!") - print(f" 字幕类型: {result['subtitle_type']}") - print(f" 字幕来源: {result['source']}") - print(f" 平均置信度: {result['confidence']:.3f}") - print(f" 字幕文本长度: {len(result['text'])} 字符") - print(f" 字幕段落数: {len(result['segments'])}") - - # 显示部分字幕文本 - if result['text']: - print(f"\n 字幕文本预览 (前 3 段):") - for i, seg in enumerate(result['segments'][:3]): - print(f" [{i+1}] [{seg['time']:.1f}s] {seg['text']}") - - if len(result['segments']) > 3: - print(f" ... 还有 {len(result['segments']) - 3} 段") - else: - print(f"\n 字幕文本: (无)") - - print("\n" + "=" * 60) - print("✅ 测试通过！") - print("=" * 60) - - except Exception as e: - print(f"\n❌ 测试失败: {e}") - import traceback - traceback.print_exc() - - -if __name__ == '__main__': - test_hard_subtitle_extractor() diff --git a/skills/obsidian-kb/src/extractors/ocr_processor.py b/skills/obsidian-kb/src/extractors/ocr_processor.py deleted file mode 100644 index 6fd35ab..0000000 --- a/skills/obsidian-kb/src/extractors/ocr_processor.py +++ /dev/null @@ -1,309 +0,0 @@ -""" -OCR 文字识别处理器 - -使用 PaddleOCR 实现视频帧文字识别功能，主要用于提取硬字幕 -""" - -import os -import re -import numpy as np -from typing import List, Dict, Any, Tuple -from paddleocr import PaddleOCR -import logging - -logger = logging.getLogger(__name__) - - -class OCRProcessor: - """ - OCR 文字识别处理器 - - 使用 PaddleOCR 从视频帧中识别文字，支持批量识别和结果过滤 - """ - - # 相似度阈值：用于判断两个文本是否相似（基于编辑距离） - SIMILARITY_THRESHOLD = 0.85 - - # 置信度阈值 - CONFIDENCE_THRESHOLD = 0.7 - - # 最小字幕长度（字符数） - MIN_SUBTITLE_LENGTH = 2 - - # 注意：Python re 模块不支持 \p{P} 等属性类，在 _is_noise 方法中使用自定义逻辑 - - def __init__(self, use_textline_orientation: bool = True, lang: str = 'ch', - use_gpu: bool = False): - """ - 初始化 OCR 处理器 - - Args: - use_textline_orientation: 是否使用文本行方向分类器（PaddleOCR 3.x 参数名） - lang: 语言支持，'ch' 为中英文混合 - use_gpu: 是否使用 GPU 加速（通过环境变量设置，如需启用请先设置 CUDA_VISIBLE_DEVICES） - """ - self.use_textline_orientation = use_textline_orientation - self.lang = lang - self.use_gpu = use_gpu - - # 初始化 PaddleOCR 实例 - self._ocr = None - self._initialize_ocr() - - def _initialize_ocr(self): - """初始化 PaddleOCR 实例""" - try: - # 强制禁用 oneDNN 和 MKL - os.environ['FLAGS_use_mkldnn'] = '0' - os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' - os.environ['OMP_NUM_THREADS'] = '1' - os.environ['PADDLE_ON_INFER'] = '1' - - # PaddleOCR 3.x 版本参数 - self._ocr = PaddleOCR( - use_textline_orientation=self.use_textline_orientation, - lang=self.lang - ) - logger.info("PaddleOCR 初始化成功（oneDNN 已禁用）") - except Exception as e: - logger.error(f"PaddleOCR 初始化失败: {e}") - raise RuntimeError(f"无法初始化 PaddleOCR: {e}") - - def recognize_frames(self, frames: List[Tuple[np.ndarray, float]]) -> List[Dict[str, Any]]: - """ - 批量识别帧中的文字 - - Args: - frames: 帧列表，每个元素为 (图像数组, 时间戳) 元组 - - 图像数组：numpy.ndarray，形状为 (H, W, 3) 的 RGB 图像 - - 时间戳：float，单位为秒 - - Returns: - List[Dict[str, Any]]: 识别结果列表，每个结果包含： - - time: float, 时间戳（秒） - - text: str, 识别的文本 - - confidence: float, 置信度 (0-1) - """ - if not frames: - logger.warning("输入帧列表为空") - return [] - - results = [] - - for image, timestamp in frames: - try: - # 调用 PaddleOCR 进行识别 - # 注意：PaddleOCR 3.x 版本的 ocr() 方法参数已简化 - ocr_result = self._ocr.ocr(image) - - # PaddleOCR 返回格式: [[[box], (text, confidence)], ...] - if ocr_result and ocr_result[0]: - for line in ocr_result[0]: - box, (text, confidence) = line - if text and confidence: - results.append({ - 'time': timestamp, - 'text': text, - 'confidence': float(confidence) - }) - except Exception as e: - logger.warning(f"帧识别失败 (时间戳 {timestamp}s): {e}") - continue - - logger.info(f"原始识别结果数量: {len(results)}") - - # 过滤和清洗结果 - filtered_results = self._filter_and_clean(results) - logger.info(f"过滤后结果数量: {len(filtered_results)}") - - return filtered_results - - def _filter_and_clean(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - 过滤和清洗识别结果 - - Args: - results: 原始识别结果列表 - - Returns: - List[Dict[str, Any]]: 过滤后的结果列表 - """ - if not results: - return [] - - # 步骤 1: 按置信度过滤 - filtered = self._filter_by_confidence(results) - logger.debug(f"置信度过滤后: {len(filtered)}") - - # 步骤 2: 过滤噪声（纯符号、单字符等） - filtered = self._filter_noise(filtered) - logger.debug(f"噪声过滤后: {len(filtered)}") - - # 步骤 3: 去除重复或相似的文本 - filtered = self._remove_duplicates(filtered) - logger.debug(f"去重后: {len(filtered)}") - - # 步骤 4: 按时间戳排序 - filtered.sort(key=lambda x: x['time']) - - return filtered - - def _filter_by_confidence(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - 按置信度过滤结果 - - Args: - results: 识别结果列表 - - Returns: - List[Dict[str, Any]]: 高置信度结果 - """ - return [r for r in results if r['confidence'] >= self.CONFIDENCE_THRESHOLD] - - def _filter_noise(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - 过滤噪声结果 - - 过滤条件： - - 文本长度小于最小长度 - - 纯符号或空白字符 - - Args: - results: 识别结果列表 - - Returns: - List[Dict[str, Any]]: 过滤后的结果 - """ - filtered = [] - - for result in results: - text = result['text'].strip() - - # 过滤过短的文本 - if len(text) < self.MIN_SUBTITLE_LENGTH: - continue - - # 过滤纯符号或空白字符 - if self._is_noise(text): - continue - - filtered.append(result) - - return filtered - - def _is_noise(self, text: str) -> bool: - """ - 判断文本是否为噪声 - - Args: - text: 要判断的文本 - - Returns: - bool: 如果是噪声返回 True - """ - # 使用正则判断是否只包含符号和空白 - # 注意：这里用简单的正则，更复杂的判断可以后续优化 - try: - # 移除标点符号后检查是否为空 - # 先替换常见的中文标点 - cleaned = re.sub(r'[^\w\u4e00-\u9fff]', '', text) - return len(cleaned) == 0 - except: - # 如果正则出错，保守处理，不算噪声 - return False - - def _remove_duplicates(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - 去除重复或相似的文本 - - 基于编辑距离（Levenshtein distance）判断文本相似度 - - Args: - results: 识别结果列表（假设已按时间戳排序） - - Returns: - List[Dict[str, Any]]: 去重后的结果 - """ - if not results: - return [] - - deduped = [results[0]] - - for current in results[1:]: - last = deduped[-1] - - # 计算相似度 - similarity = self._calculate_similarity(current['text'], last['text']) - - # 如果相似度低于阈值，则保留 - if similarity < self.SIMILARITY_THRESHOLD: - deduped.append(current) - else: - # 相似文本：保留置信度更高的那个 - if current['confidence'] > last['confidence']: - deduped[-1] = current - - return deduped - - def _calculate_similarity(self, text1: str, text2: str) -> float: - """ - 计算两个文本的相似度（基于归一化编辑距离） - - Args: - text1: 文本1 - text2: 文本2 - - Returns: - float: 相似度 (0-1)，1 表示完全相同 - """ - if text1 == text2: - return 1.0 - - len1, len2 = len(text1), len(text2) - - if len1 == 0 or len2 == 0: - return 0.0 - - # 动态规划计算编辑距离 - dp = np.zeros((len1 + 1, len2 + 1)) - - for i in range(len1 + 1): - dp[i, 0] = i - - for j in range(len2 + 1): - dp[0, j] = j - - for i in range(1, len1 + 1): - for j in range(1, len2 + 1): - if text1[i-1] == text2[j-1]: - dp[i, j] = dp[i-1, j-1] - else: - dp[i, j] = min( - dp[i-1, j] + 1, # 删除 - dp[i, j-1] + 1, # 插入 - dp[i-1, j-1] + 1 # 替换 - ) - - edit_distance = dp[len1, len2] - max_len = max(len1, len2) - - # 归一化相似度：1 - (编辑距离 / 最大长度) - similarity = 1.0 - (edit_distance / max_len) - - return similarity - - def get_ocr_info(self) -> Dict[str, Any]: - """ - 获取 OCR 处理器信息 - - Returns: - Dict[str, Any]: OCR 配置信息 - """ - return { - 'use_textline_orientation': self.use_textline_orientation, - 'lang': self.lang, - 'use_gpu': self.use_gpu, - 'confidence_threshold': self.CONFIDENCE_THRESHOLD, - 'similarity_threshold': self.SIMILARITY_THRESHOLD, - 'min_subtitle_length': self.MIN_SUBTITLE_LENGTH - } diff --git a/skills/obsidian-kb/src/extractors/subtitle_detector.py b/skills/obsidian-kb/src/extractors/subtitle_detector.py deleted file mode 100644 index 4466c6a..0000000 --- a/skills/obsidian-kb/src/extractors/subtitle_detector.py +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -字幕区域检测器 - -用于自动检测B站视频中硬字幕的位置区域。 -支持预设区域定义、自动检测和结果缓存。 -""" - -import cv2 -import numpy as np -from typing import Dict, Tuple, Optional, List -from collections import OrderedDict - - -class SubtitleDetector: - """ - 字幕区域检测器 - - 功能： - 1. 预设区域定义（底部、中部、顶部） - 2. 自动字幕区域检测（基于OpenCV） - 3. 检测结果缓存机制 - """ - - # 预设区域名称 - REGION_BOTTOM = "bottom" - REGION_CENTER = "center" - REGION_TOP = "top" - - # 预设区域参数（相对于视频高度的比例） - PRESET_REGIONS = { - REGION_BOTTOM: { - "top_ratio": 0.80, # 底部20% - "bottom_ratio": 1.0, # 到底部 - "left_ratio": 0.1, # 左右各留10% - "right_ratio": 0.9 # 避免字幕在边缘 - }, - REGION_CENTER: { - "top_ratio": 0.40, # 中部区域 - "bottom_ratio": 0.60, - "left_ratio": 0.1, - "right_ratio": 0.9 - }, - REGION_TOP: { - "top_ratio": 0.0, # 顶部区域 - "bottom_ratio": 0.20, - "left_ratio": 0.1, - "right_ratio": 0.9 - } - } - - def __init__(self, cache_size: int = 50): - """ - 初始化字幕检测器 - - Args: - cache_size: 缓存最大容量（默认50个视频） - """ - self.cache = OrderedDict() - self.cache_size = cache_size - - # 检测参数 - self.min_region_height = 0.05 # 最小区域高度（5%） - self.max_region_height = 0.25 # 最大区域高度（25%） - self.text_density_threshold = 0.005 # 文字密度阈值 - - def get_preset_region(self, region_name: str, frame_height: int, - frame_width: int) -> Tuple[int, int, int, int]: - """ - 获取预设区域的坐标 - - Args: - region_name: 区域名称（bottom/center/top） - frame_height: 帧高度 - frame_width: 帧宽度 - - Returns: - Tuple[int, int, int, int]: (x, y, width, height) - - Raises: - ValueError: 无效的区域名称 - """ - if region_name not in self.PRESET_REGIONS: - raise ValueError(f"无效的区域名称: {region_name}. 可选值: {list(self.PRESET_REGIONS.keys())}") - - params = self.PRESET_REGIONS[region_name] - - x = int(frame_width * params["left_ratio"]) - y = int(frame_height * params["top_ratio"]) - width = int(frame_width * (params["right_ratio"] - params["left_ratio"])) - height = int(frame_height * (params["bottom_ratio"] - params["top_ratio"])) - - return (x, y, width, height) - - def auto_detect_subtitle_region(self, frame: np.ndarray, - video_id: Optional[str] = None, - use_cache: bool = True) -> Tuple[int, int, int, int]: - """ - 自动检测字幕区域 - - Args: - frame: 视频帧（numpy数组，BGR格式） - video_id: 视频ID（用于缓存，如BV号） - use_cache: 是否使用缓存 - - Returns: - Tuple[int, int, int, int]: (x, y, width, height) 字幕区域坐标 - """ - # 检查缓存 - if use_cache and video_id and video_id in self.cache: - cached_region = self.cache[video_id] - # 更新缓存顺序（LRU） - self.cache.move_to_end(video_id) - return cached_region - - # 执行自动检测 - detected_region = self._detect_text_region(frame) - - # 缓存结果 - if video_id: - self._update_cache(video_id, detected_region) - - return detected_region - - def _detect_text_region(self, frame: np.ndarray) -> Tuple[int, int, int, int]: - """ - 使用OpenCV检测文字密集区域 - - 策略： - 1. 预处理：灰度化、二值化 - 2. 使用MSER检测文字候选区域 - 3. 统计区域分布，找到最可能的字幕区域 - - Args: - frame: 输入帧（BGR格式） - - Returns: - Tuple[int, int, int, int]: (x, y, width, height) 检测到的字幕区域 - """ - height, width = frame.shape[:2] - - # 尝试MSER检测 - try: - mser = cv2.MSER_create( - _delta=5, - _min_area=100, - _max_area=int(width * height * 0.05), - _max_variation=0.25, - _min_diversity=0.2 - ) - - # 转换为灰度图 - gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - - # MSER检测 - regions, _ = mser.detectRegions(gray) - - if regions and len(regions) > 5: # 有足够的检测区域 - # 计算所有区域的质心和边界框 - bboxes = [cv2.boundingRect(region.astype(int)) for region in regions] - - # 过滤：只保留白色/黄色文字区域（B站字幕常见颜色） - filtered_bboxes = self._filter_by_color(frame, bboxes) - - if filtered_bboxes: - # 分析区域分布 - region = self._analyze_region_distribution(filtered_bboxes, width, height) - if region: - return region - - except Exception as e: - # MSER失败，尝试备用方法 - pass - - # 备用方法：边缘检测 + 连通域 - return self._detect_by_edges(frame) - - def _filter_by_color(self, frame: np.ndarray, - bboxes: List[Tuple[int, int, int, int]]) -> List[Tuple[int, int, int, int]]: - """ - 根据颜色过滤候选区域 - - B站字幕通常是白色或黄色 - - Args: - frame: 原始帧（BGR格式） - bboxes: 边界框列表 - - Returns: - 过滤后的边界框列表 - """ - filtered = [] - - for x, y, w, h in bboxes: - if w < 20 or h < 10: # 太小的区域跳过 - continue - - # 提取区域 - roi = frame[y:y+h, x:x+w] - - # 计算平均颜色 - mean_color = cv2.mean(roi)[:3] # BGR - - # 判断是否接近白色或黄色 - # 白色：B、G、R都较高 - # 黄色：R和G高，B低 - - is_white = all(c > 180 for c in mean_color) - is_yellow = (mean_color[2] > 180 and mean_color[1] > 180 and mean_color[0] < 150) - - if is_white or is_yellow: - filtered.append((x, y, w, h)) - - return filtered - - def _analyze_region_distribution(self, bboxes: List[Tuple[int, int, int, int]], - width: int, height: int) -> Tuple[int, int, int, int]: - """ - 分析区域分布，找到最可能的字幕区域 - - Args: - bboxes: 边界框列表 - width: 帧宽度 - height: 帧高度 - - Returns: - Tuple[int, int, int, int]: (x, y, width, height) 字幕区域 - """ - if not bboxes: - return self.get_preset_region(self.REGION_BOTTOM, height, width) - - # 将区域按垂直位置分组 - bottom_regions = [] - - for x, y, w, h in bboxes: - # 计算区域的中心y坐标 - center_y = y + h // 2 - relative_y = center_y / height - - # 如果在底部60%的区域，可能是字幕 - if relative_y > 0.4: - bottom_regions.append((x, y, w, h)) - - if not bottom_regions: - # 如果没有找到底部区域，使用所有区域 - bottom_regions = bboxes - - # 计算合并区域 - if bottom_regions: - min_x = min(box[0] for box in bottom_regions) - max_x = max(box[0] + box[2] for box in bottom_regions) - min_y = min(box[1] for box in bottom_regions) - max_y = max(box[1] + box[3] for box in bottom_regions) - - region_width = max_x - min_x - region_height = max_y - min_y - - # 扩展区域以包含可能的文字 - margin_x = int(width * 0.02) - margin_y = int(height * 0.01) - - x = max(0, min_x - margin_x) - y = max(0, min_y - margin_y) - region_width = min(width - x, region_width + 2 * margin_x) - region_height = min(height - y, region_height + 2 * margin_y) - - # 确保区域高度在合理范围内 - min_height = int(height * self.min_region_height) - max_height = int(height * self.max_region_height) - - if region_height < min_height: - region_height = min_height - elif region_height > max_height: - region_height = max_height - - return (x, y, region_width, region_height) - - # 默认返回底部区域 - return self.get_preset_region(self.REGION_BOTTOM, height, width) - - def _detect_by_edges(self, frame: np.ndarray) -> Tuple[int, int, int, int]: - """ - 备用检测方法：使用边缘检测 - - Args: - frame: 输入帧 - - Returns: - Tuple[int, int, int, int]: (x, y, width, height) 字幕区域 - """ - height, width = frame.shape[:2] - - # 转换为灰度图 - gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) - - # 边缘检测 - edges = cv2.Canny(gray, 50, 150, apertureSize=3) - - # 形态学操作：水平线增强 - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (15, 1)) - horizontal_edges = cv2.morphologyEx(edges, cv2.MORPH_CLOSE, kernel) - - # 找到水平边缘密集的区域 - bottom_part = horizontal_edges[int(height * 0.5):, :] - - # 统计每行的边缘数量 - row_counts = np.sum(bottom_part, axis=1) - - # 找到边缘密集的行 - if len(row_counts) > 0: - threshold = np.max(row_counts) * 0.3 - text_rows = np.where(row_counts > threshold)[0] - - if len(text_rows) > 0: - # 计算字幕区域 - start_row = text_rows[0] + int(height * 0.5) - end_row = text_rows[-1] + int(height * 0.5) - - region_height = end_row - start_row - - # 扩展区域 - margin = int(height * 0.02) - y = max(int(height * 0.7), start_row - margin) - region_height = min(int(height * 0.25), region_height + 2 * margin) - - # 确保不超出帧边界 - if y + region_height > height: - region_height = height - y - - x = int(width * 0.1) - region_width = int(width * 0.8) - - return (x, y, region_width, region_height) - - # 默认返回底部区域 - return self.get_preset_region(self.REGION_BOTTOM, height, width) - - def _update_cache(self, video_id: str, region: Tuple[int, int, int, int]): - """ - 更新缓存 - - Args: - video_id: 视频ID - region: 检测到的区域 - """ - # 如果缓存已满，删除最旧的条目 - if len(self.cache) >= self.cache_size: - self.cache.popitem(last=False) - - # 添加新条目 - self.cache[video_id] = region - - def clear_cache(self, video_id: Optional[str] = None): - """ - 清空缓存 - - Args: - video_id: 如果指定，只删除该视频的缓存；否则清空所有缓存 - """ - if video_id: - if video_id in self.cache: - del self.cache[video_id] - else: - self.cache.clear() - - def get_cached_regions(self) -> Dict[str, Tuple[int, int, int, int]]: - """ - 获取所有缓存区域 - - Returns: - Dict[video_id, region]: 缓存的区域字典 - """ - return dict(self.cache) - - def get_cache_size(self) -> int: - """ - 获取当前缓存大小 - - Returns: - int: 缓存中的条目数 - """ - return len(self.cache) diff --git a/skills/obsidian-kb/src/extractors/video_stream_extractor.py b/skills/obsidian-kb/src/extractors/video_stream_extractor.py deleted file mode 100644 index dadf9c2..0000000 --- a/skills/obsidian-kb/src/extractors/video_stream_extractor.py +++ /dev/null @@ -1,517 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -B站硬字幕提取器 - 视频流提取模块 - -从B站获取视频流URL并提取视频帧用于硬字幕识别 -""" - -import json -import time -from typing import List, Dict, Any, Optional, Tuple -from urllib.request import Request, urlopen -from urllib.parse import urlencode - -try: - import cv2 - OPENCV_AVAILABLE = True -except ImportError: - OPENCV_AVAILABLE = False - - -class VideoStreamExtractor: - """ - 视频流提取器 - - 负责从B站获取视频流URL并提取视频帧 - """ - - BASE_URL = "https://api.bilibili.com" - - def __init__(self): - """ - 初始化提取器 - """ - self._img_key = None - self._sub_key = None - - def _get_headers(self) -> Dict[str, str]: - """ - 获取请求头 - - Returns: - Dict[str, str]: 请求头字典 - """ - return { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', - 'Referer': 'https://www.bilibili.com', - 'Accept': 'application/json, text/plain, */*', - } - - def _fetch_json(self, url: str, params: Optional[Dict] = None) -> Optional[Dict]: - """ - 发送HTTP GET请求并返回JSON数据 - - Args: - url: 请求URL - params: 查询参数 - - Returns: - Optional[Dict]: JSON数据或None - """ - try: - if params: - query = urlencode(params) - url = f"{url}?{query}" - - req = Request(url, headers=self._get_headers()) - with urlopen(req, timeout=30) as response: - return json.loads(response.read().decode('utf-8')) - except Exception as e: - print(f"请求失败 {url}: {e}") - return None - - def get_video_info(self, bvid: str) -> Optional[Dict[str, Any]]: - """ - 获取视频基本信息（包含cid） - - Args: - bvid: B站视频BV号 - - Returns: - Optional[Dict]: 视频信息字典，包含cid、aid等信息 - """ - api_url = f"{self.BASE_URL}/x/web-interface/view" - data = self._fetch_json(api_url, {'bvid': bvid}) - - if not data or data.get('code') != 0: - print(f"获取视频信息失败: {data}") - return None - - return data.get('data', {}) - - def get_video_stream_url( - self, - bvid: str, - quality: int = 80, - cid: Optional[str] = None - ) -> Optional[str]: - """ - 获取视频流URL - - Args: - bvid: B站视频BV号 - quality: 视频质量 (默认80，表示高清) - cid: 视频分P的CID（可选，如果不提供则自动获取） - - Returns: - Optional[str]: 视频流URL，失败返回None - """ - # 如果没有提供cid，先获取视频信息 - if not cid: - video_info = self.get_video_info(bvid) - if not video_info: - return None - cid = video_info.get('cid') - - if not cid: - print(f"无法获取视频cid: {bvid}") - return None - - # 调用播放地址API - playurl_api = f"{self.BASE_URL}/x/player/playurl" - params = { - 'bvid': bvid, - 'cid': cid, - 'qn': quality, - 'otype': 'json', - 'fnver': 0, - 'fnval': 0, - } - - data = self._fetch_json(playurl_api, params) - - if not data or data.get('code') != 0: - print(f"获取播放地址失败: {data}") - return None - - # 从durl列表获取第一个视频流URL - durl_list = data.get('data', {}).get('durl', []) - if not durl_list: - print(f"未找到视频流URL: {bvid}") - return None - - # 返回第一个分片的URL - stream_url = durl_list[0].get('url') - if not stream_url: - print(f"视频流URL为空: {bvid}") - return None - - # 处理相对URL（如果有） - if stream_url.startswith('//'): - stream_url = 'https:' + stream_url - elif stream_url.startswith('/'): - stream_url = 'https://www.bilibili.com' + stream_url - return stream_url - - def extract_frames( - self, - video_url: str, - interval_seconds: float = 5.0, - max_frames: int = 500, - start_time: float = 0.0, - use_download: bool = True - ) -> List[Dict[str, Any]]: - """ - 从视频流中提取帧 - - Args: - video_url: 视频流URL - interval_seconds: 提取间隔（秒），默认5秒 - max_frames: 最大帧数限制，默认500帧 - start_time: 开始时间（秒），默认0秒 - use_download: 是否先下载视频（默认True，因为OpenCV无法直接打开HTTPS流） - - Returns: - List[Dict[str, Any]]: 帧列表，每帧包含: - - timestamp: 时间戳（秒） - - frame_id: 帧编号 - - image: 图像数据（numpy数组） - """ - if not OPENCV_AVAILABLE: - raise RuntimeError("OpenCV未安装，请安装opencv-python: pip install opencv-python") - - import tempfile - import os - - frames = [] - temp_file = None - cap = None - - try: - # 如果是URL且需要下载，先下载到临时文件 - if use_download and (video_url.startswith('http://') or video_url.startswith('https://')): - temp_file = tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) - temp_path = temp_file.name - temp_file.close() - - print(f"视频为URL，先下载到临时文件: {temp_path}") - if not self._download_video(video_url, temp_path): - raise RuntimeError("视频下载失败") - - video_source = temp_path - else: - video_source = video_url - - # 打开视频流 - cap = cv2.VideoCapture(video_source) - - if not cap.isOpened(): - raise RuntimeError(f"无法打开视频流: {video_source}") - - # 获取视频信息 - fps = cap.get(cv2.CAP_PROP_FPS) - total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - duration = total_frames / fps if fps > 0 else 0 - print(f"视频信息: FPS={fps}, 总帧数={total_frames}, 时长={duration:.2f}秒") - - # 如果无法获取FPS，使用默认值 - if fps <= 0: - fps = 30.0 - print(f"无法获取FPS，使用默认值: {fps}") - - # 计算帧间隔 - frame_interval = int(fps * interval_seconds) - print(f"提取间隔: {interval_seconds}秒 ({frame_interval}帧)") - - # 跳转到起始位置 - start_frame = int(fps * start_time) - if start_frame > 0: - cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame) - - frame_count = 0 - extracted_count = 0 - - while frame_count < max_frames: - # 读取帧 - ret, frame = cap.read() - - if not ret: - print("已到达视频末尾或读取失败") - break - - # 获取当前帧位置 - current_frame = int(cap.get(cv2.CAP_PROP_POS_FRAMES)) - current_time = current_frame / fps - - # 检查是否达到提取间隔 - if (current_frame - start_frame) % frame_interval == 0: - frames.append({ - 'timestamp': current_time, - 'frame_id': extracted_count, - 'image': frame.copy() - }) - extracted_count += 1 - print(f"提取帧 {extracted_count}: 时间={current_time:.2f}秒, 帧号={current_frame}") - - # 检查是否达到最大帧数 - if len(frames) >= max_frames: - break - - frame_count += 1 - - except Exception as e: - print(f"提取帧时出错: {e}") - raise - finally: - # 释放资源 - if cap is not None: - cap.release() - - # 删除临时文件 - if temp_file and os.path.exists(temp_path): - try: - os.unlink(temp_path) - print(f"已删除临时文件: {temp_path}") - except Exception as e: - print(f"删除临时文件失败: {e}") - - print(f"提取完成: 共提取 {len(frames)} 帧") - return frames - - def _download_video(self, video_url: str, output_path: str, timeout: int = 300) -> bool: - """ - 下载视频到本地文件 - - Args: - video_url: 视频流URL - output_path: 输出文件路径 - timeout: 超时时间（秒） - - Returns: - bool: 下载成功返回True - """ - import os - - try: - print(f"开始下载视频...") - req = Request(video_url, headers=self._get_headers()) - - with urlopen(req, timeout=timeout) as response: - total_size = int(response.headers.get('content-length', 0)) - downloaded = 0 - - with open(output_path, 'wb') as f: - while True: - chunk = response.read(8192) - if not chunk: - break - f.write(chunk) - downloaded += len(chunk) - - if total_size > 0: - progress = (downloaded / total_size) * 100 - print(f"下载进度: {progress:.1f}% ({downloaded}/{total_size} bytes)", end='\r') - - print(f"\n下载完成: {output_path}") - return True - - except Exception as e: - print(f"\n下载失败: {e}") - return False - - def extract_frames_from_bvid( - self, - bvid: str, - interval_seconds: float = 5.0, - max_frames: int = 500, - start_time: float = 0.0, - quality: int = 80, - use_download: bool = True - ) -> List[Dict[str, Any]]: - """ - 从B站视频BV号提取帧（便捷方法） - - Args: - bvid: B站视频BV号 - interval_seconds: 提取间隔（秒） - max_frames: 最大帧数 - start_time: 开始时间（秒） - quality: 视频质量 - use_download: 是否先下载视频（默认True） - - Returns: - List[Dict[str, Any]]: 帧列表 - """ - print(f"开始处理视频: {bvid}") - - # 获取视频流URL - print("步骤1: 获取视频流URL...") - stream_url = self.get_video_stream_url(bvid, quality=quality) - - if not stream_url: - raise RuntimeError(f"无法获取视频流URL: {bvid}") - - print(f"视频流URL: {stream_url[:100]}...") - - # 提取帧 - print("\n步骤2: 提取视频帧...") - frames = self.extract_frames( - stream_url, - interval_seconds=interval_seconds, - max_frames=max_frames, - start_time=start_time, - use_download=use_download - ) - - return frames - - def save_frame(self, frame: Dict[str, Any], output_path: str) -> bool: - """ - 保存单帧为图片文件 - - Args: - frame: 帧数据字典 - output_path: 输出文件路径 - - Returns: - bool: 保存成功返回True - """ - if not OPENCV_AVAILABLE: - raise RuntimeError("OpenCV未安装") - - try: - cv2.imwrite(output_path, frame['image']) - return True - except Exception as e: - print(f"保存帧失败: {e}") - return False - - def save_frames( - self, - frames: List[Dict[str, Any]], - output_dir: str, - prefix: str = "frame" - ) -> List[str]: - """ - 批量保存帧为图片文件 - - Args: - frames: 帧列表 - output_dir: 输出目录 - prefix: 文件名前缀 - - Returns: - List[str]: 保存成功的文件路径列表 - """ - import os - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - saved_files = [] - for frame in frames: - timestamp_str = f"{frame['timestamp']:.2f}".replace('.', '_') - filename = f"{prefix}_{frame['frame_id']:04d}_t{timestamp_str}.jpg" - output_path = os.path.join(output_dir, filename) - - if self.save_frame(frame, output_path): - saved_files.append(output_path) - print(f"已保存: {output_path}") - - return saved_files - - -# 测试函数 -def test_extractor(): - """测试VideoStreamExtractor""" - print("=" * 60) - print("B站视频流提取器测试") - print("=" * 60) - - # 创建提取器 - extractor = VideoStreamExtractor() - - # 测试视频BV号 - test_bvid = "BV1KtFzzqEUG" - - print(f"\n测试视频: {test_bvid}") - print("=" * 60) - - try: - # 测试1: 获取视频流URL - print("\n【测试1】获取视频流URL...") - stream_url = extractor.get_video_stream_url(test_bvid) - - if stream_url: - print(f"✅ 成功获取视频流URL") - print(f" URL长度: {len(stream_url)} 字符") - print(f" URL前缀: {stream_url[:100]}...") - else: - print("❌ 获取视频流URL失败") - return - - # 测试2: 提取帧（限制为10帧，间隔5秒） - print("\n【测试2】提取视频帧...") - print(f" 参数: 间隔=5秒, 最大帧数=10, 开始时间=0秒") - - frames = extractor.extract_frames( - stream_url, - interval_seconds=5.0, - max_frames=10, - start_time=0.0, - use_download=True - ) - - print(f"✅ 提取完成: 共提取 {len(frames)} 帧") - - # 显示提取的帧信息 - if frames: - print("\n帧信息预览:") - print("-" * 60) - for i, frame in enumerate(frames[:5]): # 只显示前5帧 - timestamp = frame['timestamp'] - frame_id = frame['frame_id'] - height, width = frame['image'].shape[:2] - print(f" 帧 {frame_id}: 时间={timestamp:.2f}秒, 尺寸={width}x{height}") - - if len(frames) > 5: - print(f" ... 还有 {len(frames) - 5} 帧") - - # 测试3: 从BV号直接提取（便捷方法） - print("\n【测试3】从BV号直接提取（便捷方法）...") - frames2 = extractor.extract_frames_from_bvid( - test_bvid, - interval_seconds=5.0, - max_frames=10, - start_time=0.0 - ) - - print(f"✅ 提取完成: 共提取 {len(frames2)} 帧") - - # 测试4: 保存部分帧 - if frames: - print("\n【测试4】保存前3帧到临时目录...") - import tempfile - temp_dir = tempfile.mkdtemp(prefix="bilibili_frames_") - print(f" 临时目录: {temp_dir}") - - saved_files = extractor.save_frames(frames[:3], temp_dir, prefix="test") - print(f"✅ 保存完成: {len(saved_files)} 个文件") - - # 显示保存的文件 - for f in saved_files: - print(f" {f}") - - print("\n" + "=" * 60) - print("✅ 所有测试通过！") - print("=" * 60) - - except Exception as e: - print(f"\n❌ 测试失败: {e}") - import traceback - traceback.print_exc() - - -if __name__ == '__main__': - test_extractor() diff --git a/skills/obsidian-kb/src/git_sync.py b/skills/obsidian-kb/src/git_sync.py deleted file mode 100644 index 5fc9ab6..0000000 --- a/skills/obsidian-kb/src/git_sync.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Obsidian KB - Git 同步""" - -import os -import subprocess -from pathlib import Path -from typing import Optional, List -from datetime import datetime - -from .config import get_config - - -class GitSync: - """Git仓库同步管理""" - - def __init__(self, repo_path: Optional[str] = None): - config = get_config() - self.repo_path = repo_path or config.obsidian_path - self.git_url = config.git_repo - self.git_token = config.git_token - - def _run_git(self, args: List[str], cwd: Optional[str] = None) -> tuple[int, str, str]: - """运行git命令""" - cmd = ['git'] + args - try: - result = subprocess.run( - cmd, - cwd=cwd or self.repo_path, - capture_output=True, - text=True, - encoding='utf-8' - ) - return result.returncode, result.stdout, result.stderr - except Exception as e: - return -1, "", str(e) - - def is_git_repo(self) -> bool: - """检查是否为git仓库""" - git_dir = os.path.join(self.repo_path, '.git') - return os.path.isdir(git_dir) - - def init_repo(self) -> bool: - """初始化git仓库""" - if self.is_git_repo(): - return True - - os.makedirs(self.repo_path, exist_ok=True) - - # git init - code, _, _ = self._run_git(['init']) - if code != 0: - return False - - # 配置git用户（使用默认值） - self._run_git(['config', 'user.email', 'obsidian-kb@localhost']) - self._run_git(['config', 'user.name', 'Obsidian KB']) - - # 添加remote - if self.git_url: - # 如果提供了token，构建带认证的URL - if self.git_token: - auth_url = self._build_auth_url(self.git_url, self.git_token) - self._run_git(['remote', 'add', 'origin', auth_url]) - else: - self._run_git(['remote', 'add', 'origin', self.git_url]) - - return True - - def _build_auth_url(self, url: str, token: str) -> str: - """构建带认证的git URL""" - if url.startswith('https://github.com/'): - return url.replace('https://', f'https://x-access-token:{token}@') - elif url.startswith('https://'): - return url.replace('https://', f'https://oauth2:{token}@') - return url - - def add_files(self, files: List[str]) -> bool: - """添加文件到暂存区""" - if not files: - return True - - # 添加所有指定文件 - for file_path in files: - # 转换为相对路径 - rel_path = os.path.relpath(file_path, self.repo_path) - code, _, _ = self._run_git(['add', rel_path]) - if code != 0: - return False - - return True - - def add_all(self) -> bool: - """添加所有变更""" - code, _, _ = self._run_git(['add', '.']) - return code == 0 - - def has_changes(self) -> bool: - """检查是否有未提交的变更""" - code, stdout, _ = self._run_git(['status', '--porcelain']) - if code != 0: - return False - return bool(stdout.strip()) - - def commit(self, message: Optional[str] = None, content_title: Optional[str] = None) -> bool: - """提交变更""" - if not self.has_changes(): - return True - - if not message: - message = self._generate_commit_message(content_title) - - code, _, _ = self._run_git(['commit', '-m', message]) - return code == 0 - - def _generate_commit_message(self, content_title: Optional[str] = None) -> str: - """生成提交信息""" - timestamp = datetime.now().strftime('%Y-%m-%d %H:%M') - if content_title: - return f"Add: {content_title[:50]}" + ("..." if len(content_title) > 50 else "") + f" [{timestamp}]" - return f"Update KB notes [{timestamp}]" - - def push(self, branch: str = 'main') -> bool: - """推送到远程""" - # 检查是否有origin - code, stdout, _ = self._run_git(['remote']) - if code != 0 or 'origin' not in stdout: - return False - - # 检查分支是否存在，不存在则创建 - code, stdout, _ = self._run_git(['branch', '--show-current']) - current_branch = stdout.strip() - - if not current_branch: - # 创建初始分支 - self._run_git(['checkout', '-b', branch]) - - # 推送 - code, _, _ = self._run_git(['push', '-u', 'origin', branch]) - return code == 0 - - def pull(self, branch: str = 'main') -> bool: - """从远程拉取""" - code, _, _ = self._run_git(['pull', 'origin', branch]) - return code == 0 - - def sync(self, files: Optional[List[str]] = None, content_title: Optional[str] = None) -> tuple[bool, str]: - """完整同步流程""" - # 初始化仓库 - if not self.init_repo(): - return False, "Failed to initialize git repository" - - # 添加文件 - if files: - if not self.add_files(files): - return False, "Failed to add files" - else: - if not self.add_all(): - return False, "Failed to add files" - - # 提交 - if not self.commit(content_title=content_title): - if not self.has_changes(): - return True, "No changes to commit" - return False, "Failed to commit" - - # 推送 - if self.git_url and not self.push(): - return False, "Failed to push" - - return True, "Sync completed successfully" - - def get_status(self) -> dict: - """获取仓库状态""" - status = { - 'is_repo': self.is_git_repo(), - 'has_changes': False, - 'branch': '', - 'remote': '' - } - - if not status['is_repo']: - return status - - status['has_changes'] = self.has_changes() - - _, stdout, _ = self._run_git(['branch', '--show-current']) - status['branch'] = stdout.strip() - - _, stdout, _ = self._run_git(['remote', 'get-url', 'origin']) - url = stdout.strip() - # 隐藏token - if '@' in url: - url = 'https://' + url.split('@', 1)[1] - status['remote'] = url - - return status diff --git a/skills/obsidian-kb/src/light_scanner.py b/skills/obsidian-kb/src/light_scanner.py deleted file mode 100644 index ea11bca..0000000 --- a/skills/obsidian-kb/src/light_scanner.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Obsidian KB - 轻量扫描器 -C4.1: 实现轻量扫描，生成一句话摘要、关键词和质量评分 -""" -import re -from typing import List, Tuple -from src.models import Content, ContentStatus, QualityMetrics -from src.config import get_config - - -class LightScanner: - """轻量内容扫描器 - 快速生成摘要和关键词""" - - def __init__(self, ai_client=None): - self.config = get_config() - self.ai_client = ai_client - - def scan(self, content: Content, content_text: str) -> Content: - """ - 对内容进行轻量扫描 - - Args: - content: Content对象 - content_text: 内容文本（字幕或正文） - - Returns: - 更新后的Content对象 - """ - # 截断文本避免过长 - text_sample = self._get_text_sample(content_text) - - # 生成一句话摘要 - summary = self._generate_summary(text_sample, content.content_type.value) - - # 提取关键词 - keywords = self._extract_keywords(text_sample) - - # 质量评分 - quality_score = self._calculate_quality_score(text_sample, content) - - # 更新Content对象 - content.summary = summary - content.key_points = keywords # 使用key_points存储关键词 - content.quality_metrics.overall_score = quality_score / 10 # 转换为0-1范围 - content.update_status(ContentStatus.SCANNED) - - return content - - def _get_text_sample(self, text: str, max_chars: int = 3000) -> str: - """获取文本样本，优先取开头、中间和结尾""" - if len(text) <= max_chars: - return text - - # 分为三部分：开头40%、中间20%、结尾40% - start_len = int(max_chars * 0.4) - end_len = int(max_chars * 0.4) - middle_len = max_chars - start_len - end_len - - start = text[:start_len] - middle_start = len(text) // 2 - middle_len // 2 - middle = text[middle_start:middle_start + middle_len] - end = text[-end_len:] - - return f"{start}\n\n[...中间部分...]\n\n{middle}\n\n[...省略...]\n\n{end}" - - def _generate_summary(self, text_sample: str, content_type: str) -> str: - """ - 生成一句话摘要 - 如果有AI客户端则调用AI，否则使用简单启发式方法 - """ - if self.ai_client: - return self._generate_summary_with_ai(text_sample, content_type) - else: - return self._generate_summary_heuristic(text_sample) - - def _generate_summary_with_ai(self, text_sample: str, content_type: str) -> str: - """使用AI生成摘要""" - prompt = f"""请为以下{content_type}内容生成一句话摘要（不超过50字）： - -{text_sample[:2000]} - -要求： -1. 一句话概括核心内容 -2. 不超过50字 -3. 突出主要观点和价值 - -摘要：""" - - try: - response = self.ai_client.quick_generate(prompt, max_tokens=100) - # 清理并截断 - summary = response.strip().replace('\n', ' ') - if len(summary) > 60: - summary = summary[:57] + "..." - return summary - except Exception as e: - print(f"[LightScanner] AI生成摘要失败: {e}") - return self._generate_summary_heuristic(text_sample) - - def _generate_summary_heuristic(self, text: str) -> str: - """启发式生成简单摘要""" - # 取前100个字符作为简单摘要 - sentences = re.split(r'[。！？.!?]', text) - for sent in sentences: - sent = sent.strip() - if len(sent) > 20 and len(sent) < 100: - return sent[:50] + "..." if len(sent) > 50 else sent - - # 回退：取前50字 - text = text.replace('\n', ' ').strip() - return text[:47] + "..." if len(text) > 50 else text - - def _extract_keywords(self, text: str) -> List[str]: - """ - 提取5个关键词 - 如果有AI客户端则调用AI，否则使用简单词频分析 - """ - if self.ai_client: - return self._extract_keywords_with_ai(text) - else: - return self._extract_keywords_heuristic(text) - - def _extract_keywords_with_ai(self, text: str) -> List[str]: - """使用AI提取关键词""" - prompt = f"""请从以下内容中提取5个最重要的关键词，用逗号分隔： - -{text[:1500]} - -关键词（5个，逗号分隔）：""" - - try: - response = self.ai_client.quick_generate(prompt, max_tokens=100) - # 解析逗号分隔的关键词 - keywords = [k.strip() for k in response.split('，') if k.strip()] - if len(keywords) < 5: - keywords = [k.strip() for k in response.split(',') if k.strip()] - - # 确保最多5个，每个不超过10字 - keywords = [k[:10] for k in keywords[:5]] - - # 如果不足5个，补充通用关键词 - while len(keywords) < 5: - keywords.append(f"关键词{len(keywords)+1}") - - return keywords[:5] - except Exception as e: - print(f"[LightScanner] AI提取关键词失败: {e}") - return self._extract_keywords_heuristic(text) - - def _extract_keywords_heuristic(self, text: str) -> List[str]: - """启发式提取关键词（基于词频）""" - # 简单的中文/英文分词和词频统计 - import re - from collections import Counter - - # 提取中文字符串（2-6字） - chinese_words = re.findall(r'[\u4e00-\u9fff]{2,6}', text) - - # 提取英文单词 - english_words = re.findall(r'\b[a-zA-Z]{4,12}\b', text.lower()) - - # 停用词过滤 - stopwords = {'这个', '那个', '什么', '一个', '我们', '你们', '他们', '可以', '进行'} - chinese_words = [w for w in chinese_words if w not in stopwords] - - # 统计词频 - all_words = chinese_words + english_words - if not all_words: - return ["内容", "分析", "知识", "笔记", "学习"] - - word_freq = Counter(all_words) - top_words = [word for word, _ in word_freq.most_common(5)] - - # 确保有5个 - defaults = ["内容", "分析", "知识", "笔记", "学习"] - while len(top_words) < 5: - top_words.append(defaults[len(top_words)]) - - return top_words[:5] - - def _calculate_quality_score(self, text: str, content: Content) -> float: - """ - 计算内容质量评分（0-10） - 基于多个维度：文本长度、信息密度、结构完整性 - """ - score = 5.0 # 基础分 - - # 1. 文本长度评分 (0-3分) - text_len = len(text) - if text_len > 10000: - score += 3.0 - elif text_len > 5000: - score += 2.5 - elif text_len > 2000: - score += 2.0 - elif text_len > 1000: - score += 1.0 - elif text_len > 500: - score += 0.5 - - # 2. 信息密度评分 (0-2分) - 基于句子平均长度 - sentences = re.split(r'[。！？.!?]', text) - sentences = [s for s in sentences if s.strip()] - if sentences: - avg_sent_len = text_len / len(sentences) - if 15 <= avg_sent_len <= 50: - score += 2.0 - elif 10 <= avg_sent_len <= 80: - score += 1.0 - - # 3. 结构完整性 (0-2分) - # 检查是否有章节标题模式 - if re.search(r'[#\d]+[.、\s]', text): - score += 1.0 - # 检查是否有时间戳（视频字幕） - if content.duration and content.duration > 0: - score += 1.0 - - # 4. 多样性评分 (0-2分) - 基于唯一词比例 - words = re.findall(r'[\u4e00-\u9fff]', text) - if words: - unique_ratio = len(set(words)) / len(words) - if unique_ratio > 0.5: - score += 2.0 - elif unique_ratio > 0.3: - score += 1.0 - - # 确保在0-10范围内 - return round(max(0.0, min(10.0, score)), 1) diff --git a/skills/obsidian-kb/src/llm.py b/skills/obsidian-kb/src/llm.py deleted file mode 100644 index c7bcf54..0000000 --- a/skills/obsidian-kb/src/llm.py +++ /dev/null @@ -1,60 +0,0 @@ -"""LLM processor for content analysis and summarization.""" - -from typing import Any, Dict, List - - -class LLMProcessor: - """Processes content using LLM for analysis and summarization.""" - - def __init__(self, provider: str = 'openai'): - """Initialize LLM processor. - - Args: - provider: LLM provider name - """ - self.provider = provider - - def analyze(self, content: str) -> Dict[str, Any]: - """Analyze content quality and relevance. - - Args: - content: Text content to analyze - - Returns: - Analysis results with quality score, relevance, and topics - """ - # Placeholder - implement actual LLM integration - return { - 'quality_score': 0.7, - 'relevance': 'high', - 'topics': ['technology', 'tutorial'], - 'summary': content[:200] + '...' if len(content) > 200 else content - } - - def summarize(self, content: str, max_length: int = 500) -> str: - """Generate summary of content. - - Args: - content: Text content to summarize - max_length: Maximum summary length - - Returns: - Summarized text - """ - # Placeholder - implement actual LLM summarization - if len(content) <= max_length: - return content - return content[:max_length] + "..." - - def generate_tags(self, content: str, title: str = '') -> List[str]: - """Generate tags for content. - - Args: - content: Text content - title: Content title - - Returns: - List of suggested tags - """ - # Placeholder - implement actual tag generation - return ['imported', 'unread'] diff --git a/skills/obsidian-kb/src/models.py b/skills/obsidian-kb/src/models.py deleted file mode 100644 index 5bc1709..0000000 --- a/skills/obsidian-kb/src/models.py +++ /dev/null @@ -1,213 +0,0 @@ -"""Obsidian KB - 知识库内容管理模型""" - -from enum import Enum -from dataclasses import dataclass, field, asdict -from typing import Optional, List, Dict, Any -from datetime import datetime -import uuid -import json - - -class ContentType(Enum): - """内容类型""" - VIDEO = "video" - ARTICLE = "article" - - -class ContentStatus(Enum): - """内容处理状态""" - NEW = "new" # 新发现 - SCANNED = "scanned" # 已扫描元数据 - CONFIRMED = "confirmed" # 已确认处理 - ANALYZED = "analyzed" # 已AI分析 - GENERATED = "generated" # 已生成笔记 - SKIPPED = "skipped" # 已跳过 - - -@dataclass -class ContentChunk: - """内容块 - 用于分段处理""" - id: str = field(default_factory=lambda: str(uuid.uuid4())[:8]) - content: str = "" # 内容文本 - start_time: Optional[int] = None # 开始时间（秒） - end_time: Optional[int] = None # 结束时间（秒） - - def to_dict(self) -> Dict[str, Any]: - return asdict(self) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "ContentChunk": - return cls(**data) - - -@dataclass -class ContentAnalysis: - """内容分析结果 - 用于模板渲染""" - one_liner: str = "" # 一句话摘要 - summary: str = "" # 详细摘要 - key_points: List[str] = field(default_factory=list) # 关键要点 - tags: List[str] = field(default_factory=list) # 标签 - quality_score: float = 0.0 # 质量评分 0-100 - - # 教程视频专用字段 - prerequisites: List[str] = field(default_factory=list) # 前置知识 - outline: List[Dict[str, Any]] = field(default_factory=list) # 大纲分段 - pitfalls: List[str] = field(default_factory=list) # 常见坑点 - practice_advice: List[str] = field(default_factory=list) # 实践建议 - - def to_dict(self) -> Dict[str, Any]: - return asdict(self) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "ContentAnalysis": - return cls(**data) - - def is_high_quality(self, threshold: float = 60.0) -> bool: - """判断是否高质量内容""" - return self.quality_score >= threshold - - -@dataclass -class QualityMetrics: - """内容质量指标""" - overall_score: float = 0.0 # 综合评分 0-1 - information_density: float = 0.0 # 信息密度 - originality: float = 0.0 # 原创性 - practicality: float = 0.0 # 实用性 - depth: float = 0.0 # 深度 - - def to_dict(self) -> Dict[str, Any]: - return asdict(self) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "QualityMetrics": - return cls(**data) - - -@dataclass -class AnalysisResult: - """AI深度分析结果""" - main_topics: List[str] = field(default_factory=list) # 主要主题 - key_insights: List[str] = field(default_factory=list) # 关键洞察 - action_items: List[str] = field(default_factory=list) # 行动项 - summary: str = "" # 摘要 - timeline: List[Dict[str, Any]] = field(default_factory=list) # 时间线 - recommended_tags: List[str] = field(default_factory=list) # 推荐标签 - full_analysis: str = "" # 完整分析 - timeline_summary: List[Dict[str, Any]] = field(default_factory=list) # 时间线摘要（兼容字段） - content_structure: str = "" # 内容结构描述 - target_audience: str = "" # 目标受众 - difficulty_level: str = "intermediate" # 难度级别 - - def to_dict(self) -> Dict[str, Any]: - return asdict(self) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "AnalysisResult": - return cls(**data) - - -@dataclass -class TimelineEntry: - """时间线条目""" - timestamp: str - title: str - description: str - - def to_dict(self) -> Dict[str, str]: - return asdict(self) - - @classmethod - def from_dict(cls, data: Dict[str, str]) -> "TimelineEntry": - return cls(**data) - - -@dataclass -class Content: - """内容数据模型""" - # 基本信息 - id: str = field(default_factory=lambda: str(uuid.uuid4())[:8]) - source: str = "" # 来源: bilibili, youtube, zhihu, local - source_id: str = "" # 来源唯一ID - url: str = "" - title: str = "" - author: str = "" - description: str = "" - duration: Optional[int] = None # 视频时长(秒) - - # 内容类型与状态 - content_type: ContentType = ContentType.VIDEO - status: ContentStatus = ContentStatus.NEW - - # 元数据 - created_at: str = field(default_factory=lambda: datetime.now().isoformat()) - processed_at: Optional[str] = None - - # 原始内容 - transcript: str = "" # 字幕/原文 - - # AI分析结果 - summary: str = "" # 摘要 - one_liner: str = "" # 一句话摘要 - key_points: List[str] = field(default_factory=list) # 关键要点 - tags: List[str] = field(default_factory=list) # 标签 - quality_score: float = 0.0 # 质量评分 - analysis: Optional[ContentAnalysis] = None # 详细分析结果 - outline: List[str] = field(default_factory=list) # 大纲 - timeline: List[TimelineEntry] = field(default_factory=list) # 时间线 - quality_metrics: QualityMetrics = field(default_factory=QualityMetrics) - - # 分类与标签 - category: str = "" # 主题分类 - - # 输出 - output_path: Optional[str] = None # 生成的笔记路径 - note_path: Optional[str] = None # 别名，用于兼容 - - def to_dict(self) -> Dict[str, Any]: - """序列化为字典""" - data = asdict(self) - data['content_type'] = self.content_type.value - data['status'] = self.status.value - data['quality_metrics'] = self.quality_metrics.to_dict() - data['timeline'] = [e.to_dict() for e in self.timeline] - return data - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "Content": - """从字典反序列化""" - # 处理枚举 - data['content_type'] = ContentType(data.get('content_type', 'video')) - data['status'] = ContentStatus(data.get('status', 'new')) - - # 处理嵌套对象 - if 'quality_metrics' in data and isinstance(data['quality_metrics'], dict): - data['quality_metrics'] = QualityMetrics.from_dict(data['quality_metrics']) - else: - data['quality_metrics'] = QualityMetrics() - - if 'timeline' in data and isinstance(data['timeline'], list): - data['timeline'] = [TimelineEntry.from_dict(e) for e in data['timeline']] - else: - data['timeline'] = [] - - return cls(**{k: v for k, v in data.items() if k in [f.name for f in cls.__dataclass_fields__.values()]}) - - def to_json(self) -> str: - """序列化为JSON字符串""" - return json.dumps(self.to_dict(), ensure_ascii=False, indent=2) - - @classmethod - def from_json(cls, json_str: str) -> "Content": - """从JSON字符串反序列化""" - return cls.from_dict(json.loads(json_str)) - - def is_high_quality(self, threshold: float = 0.7) -> bool: - """判断是否高质量内容""" - return self.quality_metrics.overall_score >= threshold - - def update_status(self, status: ContentStatus): - """更新状态""" - self.status = status - if status == ContentStatus.GENERATED: - self.processed_at = datetime.now().isoformat() diff --git a/skills/obsidian-kb/src/outputs/__init__.py b/skills/obsidian-kb/src/outputs/__init__.py deleted file mode 100644 index c2331eb..0000000 --- a/skills/obsidian-kb/src/outputs/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Outputs module for obsidian-kb -from .obsidian_writer import ObsidianWriter -from .git_sync import GitSync - -__all__ = ['ObsidianWriter', 'GitSync'] diff --git a/skills/obsidian-kb/src/outputs/git_sync.py b/skills/obsidian-kb/src/outputs/git_sync.py deleted file mode 100644 index ea5a9d2..0000000 --- a/skills/obsidian-kb/src/outputs/git_sync.py +++ /dev/null @@ -1,275 +0,0 @@ -""" -Git Sync - 将 Obsidian vault 同步到 Git 仓库 -""" -import os -import subprocess -from pathlib import Path -from typing import Optional, Dict, Any - - -class GitSync: - """Git 同步器""" - - def __init__(self, config): - """ - 初始化同步器 - - Args: - config: 配置对象 - """ - self.config = config - self.vault_path = config.vault_path - - def sync(self, config) -> bool: - """ - 同步到 Git 仓库 - - 步骤: - 1. 检查 git.enabled - 2. git add - 3. git commit（使用 commit_message_template） - 4. 如果 auto_push=true，执行 git push - - Args: - config: 配置对象 - - Returns: - 是否成功 - """ - # 检查是否启用 Git - if not self._is_enabled(): - print("Git 同步未启用，跳过") - return True - - # 检查是否在 Git 仓库中 - if not self._is_git_repo(): - print(f"目录 {self.vault_path} 不是 Git 仓库，跳过同步") - return False - - try: - # 添加更改 - if not self._git_add(): - print("git add 失败") - return False - - # 提交更改 - if not self._git_commit(): - print("git commit 失败") - return False - - # 推送更改（如果配置了 auto_push） - if self._should_auto_push(): - if not self._git_push(): - print("git push 失败") - return False - else: - print("Git 同步完成（包含推送）") - else: - print("Git 同步完成（仅提交）") - - return True - - except Exception as e: - print(f"Git 同步失败: {e}") - return False - - def _is_enabled(self) -> bool: - """ - 检查 Git 同步是否启用 - - Returns: - 是否启用 - """ - return self.config.enabled - - def _is_git_repo(self) -> bool: - """ - 检查当前目录是否是 Git 仓库 - - Returns: - 是否是 Git 仓库 - """ - git_dir = self.vault_path / '.git' - return git_dir.exists() - - def _git_add(self) -> bool: - """ - 执行 git add - - Returns: - 是否成功 - """ - try: - result = subprocess.run( - ['git', 'add', '-A'], - cwd=self.vault_path, - capture_output=True, - text=True, - check=True - ) - return True - except subprocess.CalledProcessError as e: - print(f"git add 失败: {e.stderr}") - return False - except FileNotFoundError: - print("未找到 git 命令，请确保已安装 Git") - return False - - def _git_commit(self) -> bool: - """ - 执行 git commit - - Returns: - 是否成功 - """ - # 检查是否有更改需要提交 - if not self._has_changes(): - print("没有更改需要提交") - return True - - # 获取提交消息模板 - commit_message = self._get_commit_message() - - try: - result = subprocess.run( - ['git', 'commit', '-m', commit_message], - cwd=self.vault_path, - capture_output=True, - text=True, - check=True - ) - print(f"提交成功: {commit_message}") - return True - except subprocess.CalledProcessError as e: - print(f"git commit 失败: {e.stderr}") - return False - - def _has_changes(self) -> bool: - """ - 检查是否有更改需要提交 - - Returns: - 是否有更改 - """ - try: - # 检查工作区和暂存区是否有更改 - result = subprocess.run( - ['git', 'status', '--porcelain'], - cwd=self.vault_path, - capture_output=True, - text=True, - check=True - ) - # 如果有输出，说明有更改 - return bool(result.stdout.strip()) - except subprocess.CalledProcessError: - return False - - def _get_commit_message(self) -> str: - """ - 获取提交消息 - - 从 commit_message_template 模板生成 - - Returns: - 提交消息 - """ - template = self.config.commit_message_template - - # 简单的模板替换 - # 模板格式: "Add note: {title}" - # 目前只支持 {title} 占位符 - - # 尝试获取最新的文件信息作为标题 - # 这里我们使用一个简单的实现 - # 实际中可能需要从传入的数据中获取 - - import datetime - timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - - # 简单的模板替换 - # 如果模板包含 {title}，替换为时间戳 - # 在实际使用中，应该传入具体的标题 - commit_message = template.replace('{title}', timestamp) - commit_message = commit_message.replace('{date}', timestamp) - - return commit_message - - def _should_auto_push(self) -> bool: - """ - 检查是否应该自动推送 - - Returns: - 是否应该自动推送 - """ - return self.config.auto_push - - def _git_push(self) -> bool: - """ - 执行 git push - - Returns: - 是否成功 - """ - try: - # 获取当前分支 - branch = self._get_current_branch() - - # 推送到远程 - result = subprocess.run( - ['git', 'push', 'origin', branch], - cwd=self.vault_path, - capture_output=True, - text=True, - check=True - ) - print(f"推送成功: origin/{branch}") - return True - except subprocess.CalledProcessError as e: - print(f"git push 失败: {e.stderr}") - return False - - def _get_current_branch(self) -> str: - """ - 获取当前 Git 分支 - - Returns: - 分支名称 - """ - try: - result = subprocess.run( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], - cwd=self.vault_path, - capture_output=True, - text=True, - check=True - ) - return result.stdout.strip() - except subprocess.CalledProcessError: - return 'main' - - def sync_with_message(self, title: Optional[str] = None) -> bool: - """ - 使用自定义标题同步到 Git 仓库 - - Args: - title: 内容标题，用于生成提交消息 - - Returns: - 是否成功 - """ - # 保存原始模板 - original_template = self.config._config.get('git', {}).get('commit_message_template') - - # 临时修改模板 - if title: - self.config._config['git']['commit_message_template'] = original_template.replace('{title}', title) - - # 执行同步 - result = self.sync(self.config) - - # 恢复原始模板 - if original_template: - self.config._config['git']['commit_message_template'] = original_template - - return result diff --git a/skills/obsidian-kb/src/outputs/obsidian_writer.py b/skills/obsidian-kb/src/outputs/obsidian_writer.py deleted file mode 100644 index f88d9eb..0000000 --- a/skills/obsidian-kb/src/outputs/obsidian_writer.py +++ /dev/null @@ -1,385 +0,0 @@ -""" -Obsidian Writer - 将内容写入 Obsidian vault -""" -import os -import re -from pathlib import Path -from datetime import datetime -from typing import Dict, Any, Optional -from ..models import Content - - -class ObsidianWriter: - """Obsidian vault 写入器""" - - def __init__(self, config): - """ - 初始化写入器 - - Args: - config: 配置对象 - """ - self.config = config - self.vault_path = config.vault_path - - def write(self, data: Dict[str, Any], config) -> Optional[str]: - """ - 将内容写入 Obsidian vault - - Args: - data: 内容数据字典或 Content 对象 - config: 配置对象 - - Returns: - 写入的文件路径，失败返回 None - """ - # 如果 data 是 Content 对象，转换为字典 - if isinstance(data, Content): - content = data - else: - content = Content.from_dict(data) - - # 构建文件路径 - file_path = self._build_file_path(content) - - # 创建目录结构 - file_path.parent.mkdir(parents=True, exist_ok=True) - - # 生成 Markdown 内容 - markdown_content = self._generate_markdown(content) - - # 写入文件 - try: - with open(file_path, 'w', encoding='utf-8') as f: - f.write(markdown_content) - - return str(file_path) - except Exception as e: - print(f"写入文件失败: {e}") - return None - - def _build_file_path(self, content: Content) -> Path: - """ - 构建文件路径 - - 格式: {vault}/videos/bilibili/{author}/{date}_{title}.md - - Args: - content: Content 对象 - - Returns: - 文件路径 - """ - # 获取日期（从 created_at 提取日期部分） - date_str = self._extract_date(content.created_at) - - # 清理标题（移除特殊字符） - safe_title = self._sanitize_filename(content.title) - - # 构建文件名 - filename = f"{date_str}_{safe_title}.md" - - # 构建完整路径 - # 格式: {vault}/videos/{source}/{author}/{filename} - path_parts = [ - self.vault_path, - 'videos', - content.source, - self._sanitize_filename(content.author), - filename - ] - - return Path(*path_parts) - - def _extract_date(self, iso_date: str) -> str: - """ - 从 ISO 日期字符串提取日期部分 - - Args: - iso_date: ISO 格式日期字符串 - - Returns: - YYYY-MM-DD 格式日期字符串 - """ - try: - dt = datetime.fromisoformat(iso_date) - return dt.strftime('%Y-%m-%d') - except: - # 如果解析失败，返回当前日期 - return datetime.now().strftime('%Y-%m-%d') - - def _sanitize_filename(self, filename: str) -> str: - """ - 清理文件名，移除不适合文件系统的字符 - - Args: - filename: 原始文件名 - - Returns: - 清理后的文件名 - """ - # 移除不适合文件系统的字符 - forbidden_chars = '<>:"/\\|?*' - for char in forbidden_chars: - filename = filename.replace(char, '') - - # 移除首尾空格 - filename = filename.strip() - - # 限制长度 - if len(filename) > 100: - filename = filename[:100] - - # 如果为空，使用默认值 - if not filename: - filename = 'untitled' - - return filename - - def _generate_markdown(self, content: Content) -> str: - """ - 生成 Obsidian Markdown 内容 - - 格式: - --- - title: xxx - source: bilibili - url: xxx - author: xxx - duration: xxx - date: xxx - tags: [xxx, xxx] - --- - - # {title} - - ...内容... - - Args: - content: Content 对象 - - Returns: - Markdown 内容字符串 - """ - # 生成 frontmatter - frontmatter = self._generate_frontmatter(content) - - # 生成正文 - body = self._generate_body(content) - - # 组合 - markdown = f"{frontmatter}\n{body}" - - return markdown - - def _generate_frontmatter(self, content: Content) -> str: - """ - 生成 YAML frontmatter - - Args: - content: Content 对象 - - Returns: - Frontmatter 字符串 - """ - # 格式化时长 - duration_str = self._format_duration(content.duration) if content.duration else "N/A" - - # 格式化日期 - date_str = self._extract_date(content.created_at) - - # 构建标签列表 - tags = self._build_tags(content) - tags_yaml = ", ".join([f'"{tag}"' for tag in tags]) - - frontmatter = f"""--- -title: {content.title} -source: {content.source} -url: {content.url} -author: {content.author} -duration: {duration_str} -date: {date_str} -tags: [{tags_yaml}] ----""" - - return frontmatter - - def _build_tags(self, content: Content) -> list: - """ - 构建标签列表 - - Args: - content: Content 对象 - - Returns: - 标签列表 - """ - tags = [] - - # 添加自动标签 - if self.config.auto_tags: - tags.extend(self.config.default_tags) - - # 添加来源标签 - if content.source: - tags.append(content.source) - - # 添加内容类型标签 - if content.content_type: - tags.append(content.content_type.value) - - # 添加 AI 生成的标签 - if content.tags: - tags.extend(content.tags) - - # 去重 - tags = list(dict.fromkeys(tags)) - - return tags - - def _format_duration(self, duration_seconds: int) -> str: - """ - 格式化时长为 HH:MM:SS - - Args: - duration_seconds: 时长（秒） - - Returns: - 格式化的时长字符串 - """ - if duration_seconds is None: - return "N/A" - - hours = duration_seconds // 3600 - minutes = (duration_seconds % 3600) // 60 - seconds = duration_seconds % 60 - - if hours > 0: - return f"{hours}:{minutes:02d}:{seconds:02d}" - else: - return f"{minutes}:{seconds:02d}" - - def _generate_body(self, content: Content) -> str: - """ - 生成 Markdown 正文内容 - - Args: - content: Content 对象 - - Returns: - 正文内容字符串 - """ - body_parts = [] - - # 标题 - body_parts.append(f"# {content.title}") - body_parts.append("") - - # 一句话摘要 - if content.one_liner: - body_parts.append(f"> {content.one_liner}") - body_parts.append("") - - # 摘要 - if content.summary: - body_parts.append("## 摘要") - body_parts.append("") - body_parts.append(content.summary) - body_parts.append("") - - # 关键要点 - if content.key_points: - body_parts.append("## 关键要点") - body_parts.append("") - for point in content.key_points: - body_parts.append(f"- {point}") - body_parts.append("") - - # 大纲 - if content.outline: - body_parts.append("## 大纲") - body_parts.append("") - for i, item in enumerate(content.outline, 1): - body_parts.append(f"{i}. {item}") - body_parts.append("") - - # 时间线 - if content.timeline: - body_parts.append("## 时间线") - body_parts.append("") - for entry in content.timeline: - timestamp = entry.timestamp or "N/A" - title = entry.title or "" - description = entry.description or "" - body_parts.append(f"### {timestamp} - {title}") - if description: - body_parts.append(description) - body_parts.append("") - - # 内容分析 - if content.analysis and content.analysis.outline: - body_parts.append("## 详细分析") - body_parts.append("") - - analysis = content.analysis - - # 前置知识 - if analysis.prerequisites: - body_parts.append("### 前置知识") - body_parts.append("") - for prereq in analysis.prerequisites: - body_parts.append(f"- {prereq}") - body_parts.append("") - - # 详细大纲 - if analysis.outline: - body_parts.append("### 详细大纲") - body_parts.append("") - for section in analysis.outline: - if isinstance(section, dict): - title = section.get('title', '') - content_text = section.get('content', '') - body_parts.append(f"#### {title}") - if content_text: - body_parts.append(content_text) - else: - body_parts.append(str(section)) - body_parts.append("") - - # 常见坑点 - if analysis.pitfalls: - body_parts.append("### 常见坑点") - body_parts.append("") - for pitfall in analysis.pitfalls: - body_parts.append(f"- {pitfall}") - body_parts.append("") - - # 实践建议 - if analysis.practice_advice: - body_parts.append("### 实践建议") - body_parts.append("") - for advice in analysis.practice_advice: - body_parts.append(f"- {advice}") - body_parts.append("") - - # 原始内容（如果有） - if content.transcript: - body_parts.append("## 原始内容") - body_parts.append("") - body_parts.append(content.transcript) - body_parts.append("") - - # 无字幕提示 - if not content.transcript: - body_parts.append("*此视频无字幕，仅保存元数据*") - body_parts.append("") - - # 元数据 - body_parts.append("---") - body_parts.append(f"- **来源**: [{content.author} - {content.title}]({content.url})") - if content.duration: - duration_str = self._format_duration(content.duration) - body_parts.append(f"- **时长**: {duration_str}") - body_parts.append(f"- **处理日期**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") - - return "\n".join(body_parts) diff --git a/skills/obsidian-kb/src/pipeline.py b/skills/obsidian-kb/src/pipeline.py deleted file mode 100644 index 78657f9..0000000 --- a/skills/obsidian-kb/src/pipeline.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Obsidian KB - 内容处理管道 -C4.3: 实现完整处理流程 -""" -import os -from datetime import datetime -from typing import Optional -from src.models import Content, ContentStatus, AnalysisResult -from src.light_scanner import LightScanner -from src.deep_analyzer import DeepAnalyzer -from src.config import get_config - - -class ProcessingPipeline: - """内容处理管道 - 协调轻量扫描、深度分析、笔记生成和同步""" - - def __init__(self, ai_client=None, git_client=None): - self.config = get_config() - self.ai_client = ai_client - self.git_client = git_client - - # 初始化处理器 - self.light_scanner = LightScanner(ai_client=ai_client) - self.deep_analyzer = DeepAnalyzer(ai_client=ai_client) - - def light_scan(self, content: Content, content_text: str) -> Content: - """ - 轻量扫描阶段 - - Args: - content: Content对象 - content_text: 内容文本 - - Returns: - 更新后的Content对象 - """ - print(f"[Pipeline] 开始轻量扫描: {content.id}") - - try: - content = self.light_scanner.scan(content, content_text) - print(f"[Pipeline] 轻量扫描完成: 摘要长度={len(content.summary)}, 关键词={content.key_points}, 质量分={content.quality_metrics.overall_score:.2f}") - return content - except Exception as e: - print(f"[Pipeline] 轻量扫描失败: {e}") - raise - - def deep_analyze(self, content: Content, content_text: str) -> Content: - """ - 深度分析阶段 - - Args: - content: Content对象 - content_text: 内容文本 - - Returns: - 更新后的Content对象 - """ - print(f"[Pipeline] 开始深度分析: {content.id}") - - try: - analysis_result = self.deep_analyzer.analyze(content, content_text) - print(f"[Pipeline] 深度分析完成: 主题数={len(analysis_result.main_topics)}, 时间轴段数={len(analysis_result.timeline)}") - return content - except Exception as e: - print(f"[Pipeline] 深度分析失败: {e}") - content.update_status(ContentStatus.ERROR) - raise - - def process_full(self, content: Content, content_text: str) -> Content: - """ - 完整处理流程: - 轻量扫描 → 入库 → 深度分析 → 生成笔记 → Git同步 - - Args: - content: Content对象 - content_text: 内容文本 - - Returns: - 完全处理后的Content对象 - """ - print(f"\n{'='*60}") - print(f"[Pipeline] 开始完整处理流程: {content.id}") - print(f"{'='*60}\n") - - try: - # Step 1: 轻量扫描 - print("\n📋 Step 1: 轻量扫描") - content = self.light_scan(content, content_text) - - # Step 2: 入库（保存到数据库/文件系统） - print("\n💾 Step 2: 保存内容") - self._save_content(content) - - # Step 3: 深度分析 - print("\n🔍 Step 3: 深度分析") - content = self.deep_analyze(content, content_text) - - # Step 4: 生成笔记 - print("\n📝 Step 4: 生成Obsidian笔记") - content = self._generate_note(content) - - # Step 5: Git同步 - if self.config.git.enabled: - print("\n🔄 Step 5: Git同步") - self._sync_to_git(content) - else: - print("\n⏭️ Step 5: Git同步已禁用，跳过") - - content.update_status(ContentStatus.GENERATED) - - print(f"\n{'='*60}") - print(f"[Pipeline] 处理完成: {content.id}") - print(f"{'='*60}\n") - - return content - - except Exception as e: - print(f"\n❌ [Pipeline] 处理流程失败: {e}") - raise - - def _save_content(self, content: Content) -> None: - """保存内容到存储""" - # 创建存储目录 - storage_dir = os.path.expanduser("~/.obsidian-kb/storage") - os.makedirs(storage_dir, exist_ok=True) - - # 保存JSON文件 - import json - file_path = os.path.join(storage_dir, f"{content.id}.json") - - with open(file_path, 'w', encoding='utf-8') as f: - json.dump(content.to_dict(), f, ensure_ascii=False, indent=2) - - print(f"[Pipeline] 内容已保存: {file_path}") - - def _generate_note(self, content: Content) -> Content: - """生成Obsidian笔记""" - # 构建笔记内容 - note_content = self._build_note_content(content) - - # 确定笔记路径 - vault_path = os.path.expanduser(self.config.obsidian.vault_path) - notes_folder = self.config.obsidian.notes_folder - - note_dir = os.path.join(vault_path, notes_folder) - os.makedirs(note_dir, exist_ok=True) - - # 生成文件名 - safe_title = self._sanitize_filename(content.title or content.id) - timestamp = datetime.now().strftime("%Y%m%d") - note_filename = f"{timestamp}_{safe_title}.md" - note_path = os.path.join(note_dir, note_filename) - - # 写入文件 - with open(note_path, 'w', encoding='utf-8') as f: - f.write(note_content) - - content.output_path = note_path - content.update_status(ContentStatus.GENERATED) - - print(f"[Pipeline] 笔记已生成: {note_path}") - return content - - def _build_note_content(self, content: Content) -> str: - """构建Obsidian笔记内容""" - lines = [ - f"# {content.title or '未命名内容'}", - "", - "## 📊 元数据", - f"- **来源**: {content.source}", - f"- **URL**: {content.url}", - f"- **作者**: {content.author or '未知'}", - f"- **类型**: {content.content_type.value}", - ] - - if content.duration: - duration_str = self._format_duration(content.duration) - lines.append(f"- **时长**: {duration_str}") - - lines.extend([ - f"- **质量评分**: {content.quality_metrics.overall_score:.1f}/1.0", - f"- **处理时间**: {datetime.now().strftime('%Y-%m-%d %H:%M')}", - "", - "## 📝 摘要", - content.summary or "暂无摘要", - "", - "## 🏷️ 关键词", - ]) - - # 添加关键词标签 - if content.key_points: - keyword_tags = ' '.join([f"#{kw.replace(' ', '_')}" for kw in content.key_points[:5]]) - lines.append(keyword_tags) - lines.append('') - - # 添加时间轴 - if content.timeline: - lines.extend([ - "", - "## ⏱️ 时间轴", - "", - ]) - - for item in content.timeline: - lines.append(f"### {item.timestamp} - {item.title}") - lines.append(item.description) - lines.append('') - - # 添加原始内容（折叠） - lines.extend([ - "", - "## 📄 原始内容", - "", - "```", - content.transcript[:2000] + "..." if len(content.transcript) > 2000 else content.transcript, - "```", - "", - "---", - f"*由 Obsidian KB 自动生成 - {datetime.now().strftime('%Y-%m-%d %H:%M')}*" - ]) - - return '\n'.join(lines) - - def _sync_to_git(self, content: Content) -> None: - """同步到Git仓库""" - if not self.git_client: - print("[Pipeline] Git客户端未配置，跳过同步") - return - - try: - vault_path = os.path.expanduser(self.config.obsidian.vault_path) - - # 添加文件 - relative_path = os.path.relpath(content.output_path, vault_path) - self.git_client.add(relative_path) - - # 提交 - commit_msg = self.config.git.commit_message_template.format( - title=content.title or content.id - ) - self.git_client.commit(commit_msg) - - # 推送（如果启用） - if self.config.git.auto_push: - self.git_client.push() - - print(f"[Pipeline] Git同步完成") - - except Exception as e: - print(f"[Pipeline] Git同步失败: {e}") - # Git同步失败不阻断主流程 - - def _sanitize_filename(self, filename: str) -> str: - """清理文件名""" - import re - # 替换非法字符 - filename = re.sub(r'[<>:"/\\|?*]', '_', filename) - # 限制长度 - if len(filename) > 50: - filename = filename[:50] - return filename.strip() - - def _format_duration(self, seconds: int) -> str: - """格式化时长""" - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - secs = seconds % 60 - - if hours > 0: - return f"{hours}小时{minutes}分钟" - elif minutes > 0: - return f"{minutes}分钟{secs}秒" - else: - return f"{secs}秒" diff --git a/skills/obsidian-kb/src/sources/__init__.py b/skills/obsidian-kb/src/sources/__init__.py deleted file mode 100644 index 04dc5ca..0000000 --- a/skills/obsidian-kb/src/sources/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Sources 模块初始化 -""" -from .base import SourceAdapter -from .bilibili import BilibiliAdapter - - -class SourceRegistry: - """源适配器注册表""" - _adapters = {} - - @classmethod - def register(cls, name: str, adapter_class: type): - """注册适配器类""" - cls._adapters[name] = adapter_class - - @classmethod - def get(cls, name: str) -> SourceAdapter: - """获取适配器类""" - return cls._adapters.get(name) - - @classmethod - def list(cls) -> list: - """列出所有已注册的适配器名称""" - return list(cls._adapters.keys()) - - -# 自动注册适配器 -SourceRegistry.register('bilibili', BilibiliAdapter) - -__all__ = ['SourceAdapter', 'BilibiliAdapter', 'SourceRegistry'] diff --git a/skills/obsidian-kb/src/sources/base.py b/skills/obsidian-kb/src/sources/base.py deleted file mode 100644 index ce91fc4..0000000 --- a/skills/obsidian-kb/src/sources/base.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Source Adapter 抽象基类 -""" -from abc import ABC, abstractmethod -from datetime import datetime -from typing import List, Optional - -from models import Content - - -class SourceAdapter(ABC): - """内容源适配器基类""" - - @abstractmethod - def fetch_new(self, since: datetime, **kwargs) -> List[Content]: - """ - 获取新内容 - - Args: - since: 起始时间，只获取该时间之后的内容 - **kwargs: 额外的参数 - - Returns: - List[Content]: 新内容列表 - """ - pass - - @abstractmethod - def extract_content(self, url: str, **kwargs) -> Optional[Content]: - """ - 从URL提取内容 - - Args: - url: 内容URL - **kwargs: 额外的参数 - - Returns: - Optional[Content]: 内容对象，如果提取失败返回None - """ - pass - - @abstractmethod - def get_transcript(self, content: Content, **kwargs) -> Optional[str]: - """ - 获取字幕/文本 - - Args: - content: 内容对象 - **kwargs: 额外的参数 - - Returns: - Optional[str]: 字幕文本（SRT格式），如果没有字幕返回None - """ - pass - - -# 为向后兼容，提供 BaseSource 别名 -BaseSource = SourceAdapter diff --git a/skills/obsidian-kb/src/sources/bilibili.py b/skills/obsidian-kb/src/sources/bilibili.py deleted file mode 100644 index 29dcf3c..0000000 --- a/skills/obsidian-kb/src/sources/bilibili.py +++ /dev/null @@ -1,466 +0,0 @@ -""" -Bilibili 视频源适配器 -""" -import re -import json -import time -import hashlib -import urllib.parse -from datetime import datetime -from typing import List, Optional, Dict, Any -import urllib.request -from urllib.request import Request, urlopen - -from .base import SourceAdapter -from models import Content, ContentType, ContentStatus - - -class BilibiliAdapter(SourceAdapter): - """B站视频源适配器""" - - BASE_URL = "https://api.bilibili.com" - - # WBI 混淆密钥表 - MIXIN_KEY_ENC_TABLE = [ - 46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, - 27, 43, 5, 49, 33, 9, 42, 19, 29, 28, 14, 39, 12, 38, 41, 13, - 37, 48, 7, 16, 24, 55, 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, - 22, 25, 54, 21, 56, 59, 6, 63, 57, 62, 11, 36, 20, 34, 44, 52 - ] - - def __init__(self, sessdata: Optional[str] = None): - """ - 初始化适配器 - - Args: - sessdata: B站登录凭证（可选，用于获取私人收藏夹） - """ - self.sessdata = sessdata - self._img_key = None - self._sub_key = None - - def _get_headers(self) -> Dict[str, str]: - """获取请求头""" - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', - 'Referer': 'https://www.bilibili.com', - 'Accept': 'application/json, text/plain, */*', - } - if self.sessdata: - headers['Cookie'] = f'SESSDATA={self.sessdata}' - return headers - - def _get_wbi_keys(self) -> tuple: - """获取WBI密钥""" - if self._img_key and self._sub_key: - return self._img_key, self._sub_key - - url = f"{self.BASE_URL}/x/web-interface/nav" - try: - req = Request(url, headers=self._get_headers()) - with urlopen(req, timeout=30) as response: - data = json.loads(response.read().decode('utf-8')) - if data.get('code') == 0: - img_url = data['data']['wbi_img']['img_url'] - sub_url = data['data']['wbi_img']['sub_url'] - - # 提取密钥 - img_key = img_url.split('/')[-1].split('.')[0] - sub_key = sub_url.split('/')[-1].split('.')[0] - - self._img_key = img_key - self._sub_key = sub_key - return img_key, sub_key - except Exception as e: - print(f"获取WBI密钥失败: {e}") - - return None, None - - def _get_mixin_key(self, img_key: str, sub_key: str) -> str: - """生成Mixin密钥""" - raw_key = img_key + sub_key - mixin_key = ''.join([raw_key[i] for i in self.MIXIN_KEY_ENC_TABLE])[:32] - return mixin_key - - def _encode_wbi(self, params: Dict[str, Any]) -> Dict[str, Any]: - """WBI签名参数""" - img_key, sub_key = self._get_wbi_keys() - if not img_key or not sub_key: - return params - - # 添加时间戳 - params['wts'] = int(time.time()) - - # 按字典序排序参数 - sorted_params = dict(sorted(params.items())) - - # 构建查询字符串 - query = urllib.parse.urlencode(sorted_params) - - # 生成Mixin密钥并计算签名 - mixin_key = self._get_mixin_key(img_key, sub_key) - sign = hashlib.md5((query + mixin_key).encode()).hexdigest() - - sorted_params['w_rid'] = sign - return sorted_params - - def _fetch_json(self, url: str, params: Optional[Dict] = None, use_wbi: bool = False) -> Optional[Dict]: - """ - 发送HTTP GET请求并返回JSON数据 - - Args: - url: 请求URL - params: 查询参数 - use_wbi: 是否使用WBI签名 - - Returns: - Optional[Dict]: JSON数据或None - """ - try: - if params: - if use_wbi: - params = self._encode_wbi(params) - query = urllib.parse.urlencode(params) - url = f"{url}?{query}" - - req = Request(url, headers=self._get_headers()) - with urlopen(req, timeout=30) as response: - return json.loads(response.read().decode('utf-8')) - except Exception as e: - print(f"请求失败 {url}: {e}") - return None - - def _extract_bvid_from_url(self, url: str) -> Optional[str]: - """ - 从URL提取BV号 - - Args: - url: B站视频URL - - Returns: - Optional[str]: BV号或None - """ - patterns = [ - r'/video/(BV[\w]+)', - r'bvid=([\w]+)', - r'(BV[\w]+)', - ] - - for pattern in patterns: - match = re.search(pattern, url, re.IGNORECASE) - if match: - bvid = match.group(1) - if bvid.upper().startswith('BV'): - return bvid - return None - - def _timestamp_to_datetime(self, timestamp: int) -> Optional[datetime]: - """时间戳转datetime""" - try: - if timestamp > 10000000000: # 毫秒时间戳 - timestamp = timestamp // 1000 - return datetime.fromtimestamp(timestamp) - except: - return None - - def fetch_new(self, since: datetime, **kwargs) -> List[Content]: - """ - 获取新内容 - - Args: - since: 起始时间 - **kwargs: 包含 media_id(收藏夹ID) 或 mid(UP主ID) - - Returns: - List[Content]: 新内容列表 - """ - contents = [] - - # 获取收藏夹视频 - media_id = kwargs.get('media_id') - if media_id: - fav_contents = self._fetch_favorites(media_id, since) - contents.extend(fav_contents) - - # 获取UP主视频 - mid = kwargs.get('mid') - if mid: - up_contents = self._fetch_up_videos(mid, since) - contents.extend(up_contents) - - return contents - - def _fetch_favorites(self, media_id: int, since: datetime) -> List[Content]: - """ - 获取收藏夹视频列表 - - Args: - media_id: 收藏夹ID - since: 起始时间 - - Returns: - List[Content]: 视频列表 - """ - contents = [] - page = 1 - - while True: - url = f"{self.BASE_URL}/x/v3/fav/resource/list" - params = { - 'media_id': media_id, - 'pn': page, - 'ps': 20, - 'platform': 'web', - } - - data = self._fetch_json(url, params) - if not data or data.get('code') != 0: - break - - medias = data.get('data', {}).get('medias', []) or [] - if not medias: - break - - for media in medias: - content = Content( - source="bilibili", - source_id=media.get('bvid', ''), - url=f"https://www.bilibili.com/video/{media.get('bvid', '')}", - title=media.get('title', ''), - content_type=ContentType.VIDEO, - author=media.get('upper', {}).get('name', ''), - duration=media.get('duration', 0), - ) - contents.append(content) - - # 检查是否还有更多 - info = data.get('data', {}).get('info', {}) - total = info.get('media_count', 0) - if page * 20 >= total: - break - - page += 1 - - return contents - - def _fetch_up_videos(self, mid: int, since: datetime) -> List[Content]: - """ - 获取UP主视频列表 - - Args: - mid: UP主ID - since: 起始时间 - - Returns: - List[Content]: 视频列表 - """ - contents = [] - page = 1 - - while True: - url = f"{self.BASE_URL}/x/space/wbi/arc/search" - params = { - 'mid': mid, - 'pn': page, - 'ps': 30, - } - - data = self._fetch_json(url, params, use_wbi=True) - if not data or data.get('code') != 0: - break - - vlist = data.get('data', {}).get('list', {}).get('vlist', []) - if not vlist: - break - - for video in vlist: - # 检查时间 - created = video.get('created', 0) - pub_datetime = self._timestamp_to_datetime(created) - - if pub_datetime and pub_datetime < since: - # 已经处理到旧视频，停止 - return contents - - content = Content( - source="bilibili", - source_id=video.get('bvid', ''), - url=f"https://www.bilibili.com/video/{video.get('bvid', '')}", - title=video.get('title', ''), - content_type=ContentType.VIDEO, - author=video.get('author', ''), - duration=self._parse_duration(video.get('length', '0:00')), - ) - contents.append(content) - - page += 1 - # 限制最多获取10页 - if page > 10: - break - - return contents - - def _parse_duration(self, duration_str: str) -> int: - """解析时长字符串为秒数""" - try: - parts = duration_str.split(':') - if len(parts) == 2: # MM:SS - return int(parts[0]) * 60 + int(parts[1]) - elif len(parts) == 3: # HH:MM:SS - return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2]) - except: - pass - return 0 - - def extract_content(self, url: str, **kwargs) -> Optional[Content]: - """ - 从URL提取内容 - - Args: - url: B站视频URL - **kwargs: 额外参数 - - Returns: - Optional[Content]: 内容对象 - """ - bvid = self._extract_bvid_from_url(url) - if not bvid: - print(f"无法从URL提取BV号: {url}") - return None - - api_url = f"{self.BASE_URL}/x/web-interface/view" - params = {'bvid': bvid} - - data = self._fetch_json(api_url, params) - if not data or data.get('code') != 0: - print(f"获取视频详情失败: {data}") - return None - - video_data = data.get('data', {}) - - owner = video_data.get('owner', {}) - - content = Content( - source="bilibili", - source_id=bvid, - url=url, - title=video_data.get('title', ''), - content_type=ContentType.VIDEO, - author=owner.get('name', ''), - description=video_data.get('desc', ''), - duration=video_data.get('duration', 0), - status=ContentStatus.SCANNED, - ) - - return content - - def get_transcript(self, content: Content, **kwargs) -> Optional[str]: - """ - 获取字幕 - - Args: - content: 内容对象 - **kwargs: 额外参数 - - Returns: - Optional[str]: 字幕文本（SRT格式） - """ - bv_id = content.source_id - if not bv_id: - bv_id = content.id - - # 先获取cid - api_url = f"{self.BASE_URL}/x/web-interface/view" - data = self._fetch_json(api_url, {'bvid': bv_id}) - if not data or data.get('code') != 0: - return None - - cid = data['data'].get('cid') - if not cid: - return None - - # 获取字幕列表 - subtitle_url = f"{self.BASE_URL}/x/player/wbi/v2" - params = { - 'cid': cid, - 'bvid': bv_id, - } - - data = self._fetch_json(subtitle_url, params, use_wbi=True) - if not data or data.get('code') != 0: - return None - - subtitle_info = data.get('data', {}).get('subtitle', {}) - subtitles = subtitle_info.get('subtitles', []) - - if not subtitles: - return None - - # 优先选择AI字幕，其次选择中文简体，最后选择第一个 - selected_sub = None - for sub in subtitles: - lan_doc = sub.get('lan_doc', '') - lan = sub.get('lan', '') - if '自动生成' in lan_doc or 'AI' in lan_doc: - selected_sub = sub - break - elif lan == 'zh-CN' or lan == 'zh': - selected_sub = sub - break - - if not selected_sub: - selected_sub = subtitles[0] - - # 下载字幕内容 - sub_url = selected_sub.get('subtitle_url', '') - if not sub_url: - return None - - # 确保URL完整 - if sub_url.startswith('//'): - sub_url = 'https:' + sub_url - elif sub_url.startswith('/'): - sub_url = 'https://www.bilibili.com' + sub_url - - try: - req = Request(sub_url, headers=self._get_headers()) - with urlopen(req, timeout=30) as response: - subtitle_data = json.loads(response.read().decode('utf-8')) - except Exception as e: - print(f"下载字幕失败: {e}") - return None - - # 解析字幕 - body = subtitle_data.get('body', []) - if not body: - return None - - # 转换为SRT格式 - srt_lines = [] - - for i, item in enumerate(body, 1): - from_time = item.get('from', 0) - to_time = item.get('to', 0) - text = item.get('content', '').strip() - - if not text: - continue - - # SRT格式时间码 - from_str = self._format_srt_time(from_time) - to_str = self._format_srt_time(to_time) - - srt_lines.append(f"{i}") - srt_lines.append(f"{from_str} --> {to_str}") - srt_lines.append(text) - srt_lines.append("") - - return '\n'.join(srt_lines) - - def _format_srt_time(self, seconds: float) -> str: - """格式化为SRT时间码""" - hours = int(seconds // 3600) - minutes = int((seconds % 3600) // 60) - secs = int(seconds % 60) - millis = int((seconds % 1) * 1000) - return f"{hours:02d}:{minutes:02d}:{secs:02d},{millis:03d}" diff --git a/skills/obsidian-kb/src/sources/local.py b/skills/obsidian-kb/src/sources/local.py deleted file mode 100644 index a522812..0000000 --- a/skills/obsidian-kb/src/sources/local.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Obsidian KB - 本地文件源""" - -import os -from pathlib import Path -from typing import Optional -from .base import BaseSource -from ..models import Content, ContentType - - -class LocalSource(BaseSource): - """本地文件源""" - - name = "local" - - SUPPORTED_EXTENSIONS = ['.txt', '.md', '.pdf', '.docx'] - - def can_handle(self, url: str) -> bool: - """检查是否为本地文件路径""" - if url.startswith('http'): - return False - path = Path(url) - return path.exists() and path.suffix.lower() in self.SUPPORTED_EXTENSIONS - - def extract_id(self, url: str) -> Optional[str]: - """使用文件路径作为ID""" - return os.path.basename(url) - - def fetch(self, url: str) -> Optional[Content]: - """获取文件元数据""" - path = Path(url) - if not path.exists(): - return None - - content_type = ContentType.ARTICLE - if path.suffix.lower() == '.txt': - content_type = ContentType.ARTICLE - - return Content( - source=self.name, - source_id=str(path), - url=str(path.absolute()), - title=path.stem, - author='', - content_type=content_type - ) - - def get_transcript(self, content: Content) -> Optional[str]: - """读取文件内容""" - try: - path = Path(content.url) - if path.suffix.lower() in ['.txt', '.md']: - return path.read_text(encoding='utf-8') - # TODO: 支持 PDF 和 DOCX - return None - except Exception as e: - print(f"Error reading local file: {e}") - return None diff --git a/skills/obsidian-kb/src/sources/markdown_converter.py b/skills/obsidian-kb/src/sources/markdown_converter.py deleted file mode 100644 index c0f912d..0000000 --- a/skills/obsidian-kb/src/sources/markdown_converter.py +++ /dev/null @@ -1,17 +0,0 @@ -from markdownify import MarkdownConverter as _MarkdownConverter - -class MarkdownConverter: - """HTML 转 Markdown 转换器，保留完整格式""" - - def __init__(self): - self.converter = _MarkdownConverter( - heading_style="ATX", - bullets="*,-", - code_blocks=True, - tables=True, - strip=["script", "style", "nav", "footer", "aside"] - ) - - def convert(self, html: str) -> str: - """转换 HTML 为 Markdown""" - return self.converter.convert(html) diff --git a/skills/obsidian-kb/src/sources/youtube.py b/skills/obsidian-kb/src/sources/youtube.py deleted file mode 100644 index efa1137..0000000 --- a/skills/obsidian-kb/src/sources/youtube.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Obsidian KB - YouTube源""" - -import re -from typing import Optional -from .base import BaseSource -from ..models import Content, ContentType - - -class YouTubeSource(BaseSource): - """YouTube视频源""" - - name = "youtube" - - # 支持多种YouTube URL格式 - URL_PATTERNS = [ - r'youtube\.com/watch\?v=([a-zA-Z0-9_-]{11})', - r'youtu\.be/([a-zA-Z0-9_-]{11})', - r'youtube\.com/embed/([a-zA-Z0-9_-]{11})', - ] - - def can_handle(self, url: str) -> bool: - """检查是否为YouTube链接""" - return 'youtube.com' in url or 'youtu.be' in url - - def extract_id(self, url: str) -> Optional[str]: - """提取视频ID""" - for pattern in self.URL_PATTERNS: - match = re.search(pattern, url) - if match: - return match.group(1) - return None - - def fetch(self, url: str) -> Optional[Content]: - """获取视频元数据""" - video_id = self.extract_id(url) - if not video_id: - return None - - # TODO: 使用 YouTube Data API 获取完整信息 - # 目前返回基本信息 - return Content( - source=self.name, - source_id=video_id, - url=f'https://www.youtube.com/watch?v={video_id}', - title='', # 需要通过API获取 - author='', - content_type=ContentType.VIDEO - ) - - def get_transcript(self, content: Content) -> Optional[str]: - """获取字幕""" - # TODO: 使用 youtube-transcript-api - return None diff --git a/skills/obsidian-kb/src/sources/zhihu.py b/skills/obsidian-kb/src/sources/zhihu.py deleted file mode 100644 index f8a59f4..0000000 --- a/skills/obsidian-kb/src/sources/zhihu.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Obsidian KB - 知乎源""" - -import re -from typing import Optional -from .base import BaseSource -from ..models import Content, ContentType - -try: - from .zhihu_drission_extractor import ZhihuDrissionExtractor -except ImportError: - # DrissionPage 未安装 - import logging - logger = logging.getLogger(__name__) - logger.warning(f"DrissionPage 未安装，无法提取知乎内容") - - -class ZhihuSource(BaseSource): - """知乎文章/回答源""" - - name = "zhihu" - - URL_PATTERNS = [ - r'zhihu\.com/question/\d+/answer/(\d+)', - r'zhuanlan\.zhihu\.com/p/(\d+)', - r'zhihu\.com/question/(\d+)', - ] - - def can_handle(self, url: str) -> bool: - """检查是否为知乎链接""" - return 'zhihu.com' in url or 'zhuanlan.zhihu.com' in url - - def extract_id(self, url: str) -> Optional[str]: - """提取内容ID""" - for pattern in self.URL_PATTERNS: - match = re.search(pattern, url) - if match: - return match.group(1) - return None - - def fetch(self, url: str) -> Optional[Content]: - """获取内容元数据""" - content_id = self.extract_id(url) - if not content_id: - return None - - # TODO: 使用知乎API或爬虫获取完整信息 - content_type = ContentType.ARTICLE - if 'zhuanlan' in url: - content_type = ContentType.ARTICLE - - return Content( - source=self.name, - source_id=content_id, - url=url, - title='', - author='', - content_type=content_type - ) - - def get_transcript(self, content: Content) -> Optional[str]: - """获取文章内容""" - try: - # 使用 DrissionPage 提取器 - extractor = ZhihuDrissionExtractor( - headless=self.config.get('headless', True), - timeout=self.config.get('timeout', 30), - config=self.config - ) - - data = extractor.extract(content.url) - - if data and data.get('content_markdown'): - # 更新元数据 - if data.get('title'): - content.title = data['title'] - if data.get('author'): - content.author = data['author'] - - # 返回 Markdown 内容 - return data['content_markdown'] - - except ImportError as e: - import logging - logger = logging.getLogger(__name__) - logger.error(f"DrissionPage 导入失败: {e}") - except Exception as e: - import logging - logger = logging.getLogger(__name__) - logger.error(f"DrissionPage 提取失败: {e}") - - return None diff --git a/skills/obsidian-kb/src/sources/zhihu_drission_extractor.py b/skills/obsidian-kb/src/sources/zhihu_drission_extractor.py deleted file mode 100644 index 419a742..0000000 --- a/skills/obsidian-kb/src/sources/zhihu_drission_extractor.py +++ /dev/null @@ -1,514 +0,0 @@ -""" -知乎 DrissionPage 内容提取器 -使用 DrissionPage 浏览器自动化工具抓取知乎内容 -""" - -import re -import json -import sys -import os -from typing import Dict, Optional, Any, List -from pathlib import Path -from urllib.parse import urlparse - -# 添加当前目录到 Python 路径以支持绝对导入 -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -try: - from .markdown_converter import MarkdownConverter -except ImportError: - from markdown_converter import MarkdownConverter - - -class DrissionBrowser: - """DrissionPage 浏览器管理器""" - - def __init__(self, headless: bool = True, timeout: int = 30): - self.headless = headless - self.timeout = timeout - self.page = None - - def start(self): - """启动浏览器""" - from DrissionPage import ChromiumOptions, ChromiumPage - - options = ChromiumOptions() - - # 设置浏览器路径（服务器环境需要） - options.set_browser_path('/usr/bin/chromium-browser') - - if self.headless: - options.headless() - options.set_user_agent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36') - - self.page = ChromiumPage(options) - self.page.set.timeouts(base=self.timeout) - - def close(self): - """关闭浏览器""" - if self.page: - self.page.quit() - self.page = None - - def goto(self, url: str): - """导航到指定 URL""" - self.page.get(url) - - def wait_for_selector(self, selector: str, timeout: int = None): - """等待选择器出现""" - timeout = timeout or self.timeout - self.page.wait.ele_displayed(selector, timeout=timeout) - - def query_selector(self, selector: str): - """查询选择器""" - return self.page.ele(selector) - - def query_selector_all(self, selector: str): - """查询所有匹配的选择器""" - return self.page.eles(selector) - - def evaluate(self, script: str): - """执行 JavaScript 代码""" - return self.page.run_js(script) - - def get_page_title(self): - """获取页面标题""" - if self.page: - return self.page.title - return '' - - def get_page_content(self): - """获取页面 HTML 内容""" - if self.page: - return self.page.html - return '' - - def screenshot(self, path: str): - """截图""" - if self.page: - self.page.get_screenshot(path=path) - - -class ZhihuDrissionExtractor: - """知乎 DrissionPage 提取器""" - - URL_PATTERNS = { - 'answer': r'zhihu\.com/question/\d+/answer/(\d+)', - 'article': r'zhuanlan\.zhihu\.com/p/(\d+)', - 'question': r'zhihu\.com/question/(\d+)(?:$|/)', - } - - def __init__(self, headless: bool = True, timeout: int = 30, config: Optional[Dict] = None): - """ - 初始化提取器 - - Args: - headless: 是否使用无头浏览器（默认 True） - timeout: 超时时间（秒，默认 30） - config: 配置字典，用于加载 Cookie - """ - self.headless = headless - self.timeout = timeout - self.config = config or {} - # 支持从 config 中获取 headless 和 timeout - if self.config.get('headless') is not None: - self.headless = self.config.get('headless') - if self.config.get('timeout') is not None: - self.timeout = self.config.get('timeout') - - self.browser = DrissionBrowser(headless=self.headless, timeout=self.timeout) - self.markdown_converter = MarkdownConverter() - # 从配置中读取 Cookie 文件路径，支持环境变量展开 - # 直接读取 cookie_file（因为 self.config 已经是 zhihu 配置） - cookie_file = self.config.get('cookie_file') - if cookie_file: - self.cookie_file = os.path.expanduser(cookie_file) - else: - # 默认路径 - self.cookie_file = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - 'data', 'cookies', 'zhihu.json' - ) - - def identify_url_type(self, url: str) -> Optional[str]: - """ - 识别 URL 类型 - - Returns: - 'answer', 'article', 'question' 或 None - """ - for url_type, pattern in self.URL_PATTERNS.items(): - if re.search(pattern, url): - return url_type - return None - - def extract_id(self, url: str) -> Optional[str]: - """ - 从 URL 中提取内容 ID - - Returns: - 内容 ID 或 None - """ - url_type = self.identify_url_type(url) - if url_type: - pattern = self.URL_PATTERNS[url_type] - match = re.search(pattern, url) - if match: - return match.group(1) - return None - - def _extract_text_content(self, element) -> str: - """提取元素的文本内容""" - if element is None: - return '' - return element.text if hasattr(element, 'text') else '' - - def _extract_html_content(self, element) -> str: - """提取元素的 HTML 内容""" - if element is None: - return '' - return element.html if hasattr(element, 'html') else '' - - def _extract_attribute(self, selector: str, attribute: str) -> str: - """提取元素的属性""" - try: - element = self.browser.query_selector(selector) - if element: - return element.attr(attribute) or '' - except Exception as e: - print(f"提取属性时出错（选择器: {selector}, 属性: {attribute}）: {e}") - return '' - - def _load_cookies(self) -> List[Dict]: - """ - 加载 Cookie，优先级：配置 > 环境 > 文件 - - Returns: - Cookie 列表 - """ - cookies = [] - - # 1. 尝试从配置加载（优先级最高） - # 支持扁平和嵌套配置结构 - cookie_str = self.config.get('cookie_str') - if cookie_str: - try: - cookies = json.loads(cookie_str) - print(f"从配置加载 Cookie，共 {len(cookies)} 个") - except json.JSONDecodeError as e: - print(f"配置中的 Cookie JSON 格式错误: {e}") - - # 2. 尝试从环境变量加载（优先级中等） - elif os.getenv('ZHIHU_COOKIE'): - try: - cookies = json.loads(os.getenv('ZHIHU_COOKIE')) - print(f"从环境变量加载 Cookie，共 {len(cookies)} 个") - except json.JSONDecodeError as e: - print(f"环境变量中的 Cookie JSON 格式错误: {e}") - - # 3. 尝试从文件加载（优先级最低） - else: - # 直接从配置中读取 cookie_file（因为 self.config 已经是 zhihu 配置） - cookie_file = self.config.get('cookie_file') - - if not cookie_file: - # 如果没有配置，使用默认路径 - cookie_file = self.cookie_file - else: - # 展开环境变量（如 ~） - cookie_file = os.path.expanduser(cookie_file) - - if cookie_file and Path(cookie_file).exists(): - try: - with open(cookie_file, 'r', encoding='utf-8') as f: - cookies = json.load(f) - print(f"从文件加载 Cookie ({cookie_file})，共 {len(cookies)} 个") - except (json.JSONDecodeError, FileNotFoundError, IOError) as e: - print(f"从文件加载 Cookie 失败: {e}") - else: - print(f"未找到 Cookie 配置，将使用未登录状态访问知乎") - print(f" 期望路径: {cookie_file}") - - return cookies - - def _apply_cookies(self, cookies: List[Dict]): - """ - 应用 Cookie 到浏览器页面 - - Args: - cookies: Cookie 列表 - """ - if not cookies: - return - - try: - for cookie in cookies: - if self.browser and self.browser.page: - self.browser.page.set.cookies.add(cookie) - print(f"成功应用 {len(cookies)} 个 Cookie") - except Exception as e: - print(f"应用 Cookie 时出错: {e}") - - def _extract_answer_content(self) -> Dict[str, Any]: - """提取回答内容""" - result = { - 'type': 'answer', - 'title': '', - 'author': '', - 'content': '', - 'content_markdown': '', - 'publish_time': '', - 'vote_up': 0, - } - - # 打印页面标题用于调试 - page_title = self.browser.get_page_title() - print(f"页面标题: {page_title}") - - # 尝试保存页面快照用于调试 - try: - import os - debug_dir = '/home/admin/.openclaw/workspace/debug' - os.makedirs(debug_dir, exist_ok=True) - self.browser.screenshot(os.path.join(debug_dir, 'zhihu_drission_screenshot.png')) - with open(os.path.join(debug_dir, 'zhihu_drission_page.html'), 'w', encoding='utf-8') as f: - f.write(self.browser.get_page_content()) - print(f"页面快照已保存到 {debug_dir}") - except Exception as e: - print(f"保存页面快照时出错: {e}") - - # 尝试多个可能的选择器 - selectors = { - 'title': ['h1.QuestionHeader-title', 'h1', '.QuestionHeader-main'], - 'author': ['.AuthorInfo-name', '.AuthorInfo .Text', '[itemprop="name"]'], - 'content': ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist] article'], - 'time': ['.ContentItem-time', '[itemprop="dateCreated"]'], - 'vote': ['.VoteButton--up .VoteButton-label', '.VoteButton--up'] - } - - for selector in selectors['title']: - title_elem = self.browser.query_selector(selector) - if title_elem: - result['title'] = self._extract_text_content(title_elem) - print(f"找到标题（使用选择器 {selector}）: {result['title'][:50]}...") - break - - for selector in selectors['author']: - author_elem = self.browser.query_selector(selector) - if author_elem: - result['author'] = self._extract_text_content(author_elem) - print(f"找到作者（使用选择器 {selector}）: {result['author']}") - break - - for selector in selectors['content']: - content_elem = self.browser.query_selector(selector) - if content_elem: - html_content = self._extract_html_content(content_elem) - result['content'] = html_content - result['content_markdown'] = self.markdown_converter.convert(html_content) - print(f"找到内容（使用选择器 {selector}），长度: {len(result['content_markdown'])}") - break - - for selector in selectors['time']: - time_elem = self.browser.query_selector(selector) - if time_elem: - result['publish_time'] = time_elem.attr('data-tooltip') or self._extract_text_content(time_elem) - break - - for selector in selectors['vote']: - vote_elem = self.browser.query_selector(selector) - if vote_elem: - vote_text = self._extract_text_content(vote_elem) - try: - result['vote_up'] = int(vote_text.replace(',', '')) - except (ValueError, AttributeError): - pass - break - - return result - - def _extract_article_content(self) -> Dict[str, Any]: - """提取文章内容""" - result = { - 'type': 'article', - 'title': '', - 'author': '', - 'content': '', - 'content_markdown': '', - 'publish_time': '', - 'vote_up': 0, - } - - # 提取文章标题 - title_elem = self.browser.query_selector('.Post-Title') - if title_elem: - result['title'] = self._extract_text_content(title_elem) - - # 提取作者信息 - author_elem = self.browser.query_selector('.AuthorInfo-name') - if author_elem: - result['author'] = self._extract_text_content(author_elem) - - # 提取文章内容 - content_elem = self.browser.query_selector('.Post-RichText') - if content_elem: - html_content = self._extract_html_content(content_elem) - result['content'] = html_content - result['content_markdown'] = self.markdown_converter.convert(html_content) - - # 提取发布时间 - time_elem = self.browser.query_selector('.ContentItem-time') - if time_elem: - result['publish_time'] = time_elem.attr('data-tooltip') or self._extract_text_content(time_elem) - - # 提取点赞数 - vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label') - if vote_elem: - vote_text = self._extract_text_content(vote_elem) - try: - result['vote_up'] = int(vote_text.replace(',', '')) - except (ValueError, AttributeError): - pass - - return result - - def _extract_question_content(self) -> Dict[str, Any]: - """提取问题内容（问题页本身）""" - result = { - 'type': 'question', - 'title': '', - 'author': '', - 'content': '', - 'content_markdown': '', - 'publish_time': '', - 'vote_up': 0, - } - - # 提取问题标题 - title_elem = self.browser.query_selector('h1.QuestionHeader-title') - if title_elem: - result['title'] = self._extract_text_content(title_elem) - - # 提取问题详情 - detail_elem = self.browser.query_selector('.QuestionHeader-detail') - if detail_elem: - html_content = self._extract_html_content(detail_elem) - result['content'] = html_content - result['content_markdown'] = self.markdown_converter.convert(html_content) - - # 问题的作者通常是提问者 - author_elem = self.browser.query_selector('.AuthorInfo-name') - if author_elem: - result['author'] = self._extract_text_content(author_elem) - - return result - - def extract(self, url: str) -> Optional[Dict[str, Any]]: - """ - 从 URL 提取知乎内容 - - Args: - url: 知乎 URL - - Returns: - 包含提取内容的字典，结构如下: - { - 'type': 'answer' | 'article' | 'question', - 'title': str, - 'author': str, - 'content': str (HTML), - 'content_markdown': str (Markdown), - 'publish_time': str, - 'vote_up': int, - 'url': str, - 'content_id': str, - } - 如果提取失败返回 None - """ - try: - # 识别 URL 类型 - url_type = self.identify_url_type(url) - if not url_type: - print(f"无法识别 URL 类型: {url}") - return None - - print(f"URL 类型: {url_type}") - - # 加载 Cookie - cookies = self._load_cookies() - if cookies: - print(f"加载了 {len(cookies)} 个 Cookie") - - print(f"启动浏览器...") - - # 启动浏览器 - self.browser.start() - - # 应用 Cookie - if cookies and self.browser.page: - self._apply_cookies(cookies) - - print(f"访问页面: {url}") - # 访问页面 - self.browser.goto(url) - - print("等待页面加载...") - import time - time.sleep(3) # 等待 JavaScript 执行 - - # 尝试多个可能的选择器 - if url_type == 'answer': - selectors = ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist]'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_answer_content() - elif url_type == 'article': - selectors = ['.Post-RichText', '.Post-Title'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_article_content() - else: # question - selectors = ['.QuestionHeader-title', 'h1'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_question_content() - - # 添加额外信息 - result['url'] = url - result['content_id'] = self.extract_id(url) - - print(f"提取成功! 标题: {result['title'][:50]}...") - return result - - except Exception as e: - print(f"提取知乎内容时出错: {e}") - import traceback - traceback.print_exc() - return None - finally: - # 关闭浏览器 - self.browser.close() - - def __enter__(self): - """上下文管理器入口""" - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """上下文管理器出口""" - self.browser.close() - return False - - -# 便捷函数 -def extract_zhihu_content(url: str, config=None) -> Optional[Dict[str, Any]]: - """ - 便捷函数：提取知乎内容 - - Args: - url: 知乎 URL - config: 配置字典 - - Returns: - 提取的内容字典 - """ - with ZhihuDrissionExtractor(config=config) as extractor: - return extractor.extract(url) diff --git a/skills/obsidian-kb/src/sources/zhihu_playwright_extractor.py b/skills/obsidian-kb/src/sources/zhihu_playwright_extractor.py deleted file mode 100644 index 2baf648..0000000 --- a/skills/obsidian-kb/src/sources/zhihu_playwright_extractor.py +++ /dev/null @@ -1,385 +0,0 @@ -""" -知乎 Playwright 内容提取器 -使用 Playwright 浏览器自动化工具抓取知乎内容 -""" - -import re -import sys -import os -from typing import Dict, Optional, Any -from urllib.parse import urlparse - -# 添加当前目录到 Python 路径以支持绝对导入 -current_dir = os.path.dirname(os.path.abspath(__file__)) -if current_dir not in sys.path: - sys.path.insert(0, current_dir) - -try: - from .markdown_converter import MarkdownConverter -except ImportError: - from markdown_converter import MarkdownConverter - - -class PlaywrightBrowser: - """Playwright 浏览器管理器""" - - def __init__(self, headless: bool = True, timeout: int = 30): - self.headless = headless - self.timeout = timeout * 1000 - self.browser = None - self.context = None - self.page = None - - def start(self): - """启动浏览器""" - from playwright.sync_api import sync_playwright - self.playwright = sync_playwright().start() - self.browser = self.playwright.chromium.launch(headless=self.headless) - self.context = self.browser.new_context( - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36' - ) - self.page = self.context.new_page() - self.page.set_default_timeout(self.timeout) - - def close(self): - """关闭浏览器""" - if self.page: - self.page.close() - if self.context: - self.context.close() - if self.browser: - self.browser.close() - if hasattr(self, 'playwright'): - self.playwright.stop() - - def goto(self, url: str, wait_until: str = "networkidle"): - """导航到指定URL""" - return self.page.goto(url, wait_until=wait_until) - - def wait_for_selector(self, selector: str, timeout: int = None): - """等待选择器出现""" - timeout_ms = timeout * 1000 if timeout else self.timeout - return self.page.wait_for_selector(selector, timeout=timeout_ms) - - def query_selector(self, selector: str): - """查询选择器""" - return self.page.query_selector(selector) - - def query_selector_all(self, selector: str): - """查询所有匹配的选择器""" - return self.page.query_selector_all(selector) - - def evaluate(self, script: str): - """执行 JavaScript 代码""" - return self.page.evaluate(script) - - -class ZhihuPlaywrightExtractor: - """知乎内容提取器 - 使用 Playwright""" - - URL_PATTERNS = { - 'answer': r'zhihu\.com/question/\d+/answer/(\d+)', - 'article': r'zhuanlan\.zhihu\.com/p/(\d+)', - 'question': r'zhihu\.com/question/(\d+)(?:$|/)', - } - - def __init__(self, headless: bool = True, timeout: int = 30): - """ - 初始化提取器 - - Args: - headless: 是否使用无头浏览器 - timeout: 超时时间（秒） - """ - self.browser = PlaywrightBrowser(headless=headless, timeout=timeout) - self.markdown_converter = MarkdownConverter() - - def identify_url_type(self, url: str) -> Optional[str]: - """ - 识别 URL 类型 - - Returns: - 'answer', 'article', 'question' 或 None - """ - for url_type, pattern in self.URL_PATTERNS.items(): - if re.search(pattern, url): - return url_type - return None - - def extract_id(self, url: str) -> Optional[str]: - """ - 从 URL 中提取内容 ID - - Returns: - 内容 ID 或 None - """ - url_type = self.identify_url_type(url) - if url_type: - pattern = self.URL_PATTERNS[url_type] - match = re.search(pattern, url) - if match: - return match.group(1) - return None - - def _extract_text_content(self, element) -> str: - """提取元素的文本内容""" - if element is None: - return '' - return element.inner_text() if hasattr(element, 'inner_text') else '' - - def _extract_html_content(self, element) -> str: - """提取元素的 HTML 内容""" - if element is None: - return '' - return element.inner_html() if hasattr(element, 'inner_html') else '' - - def _extract_answer_content(self) -> Dict[str, Any]: - """提取回答内容""" - result = { - 'type': 'answer', - 'title': '', - 'author': '', - 'content': '', - 'content_markdown': '', - 'publish_time': '', - 'vote_up': 0, - } - - # 打印页面标题用于调试 - page_title = self.browser.page.title() - print(f"页面标题: {page_title}") - - # 尝试保存页面快照用于调试 - try: - import os - debug_dir = '/home/admin/.openclaw/workspace/debug' - os.makedirs(debug_dir, exist_ok=True) - self.browser.page.screenshot(path=os.path.join(debug_dir, 'zhihu_screenshot.png')) - with open(os.path.join(debug_dir, 'zhihu_page.html'), 'w', encoding='utf-8') as f: - f.write(self.browser.page.content()) - print(f"页面快照已保存到 {debug_dir}") - except Exception as e: - print(f"保存页面快照时出错: {e}") - - # 尝试多个可能的选择器 - selectors = { - 'title': ['h1.QuestionHeader-title', 'h1', '.QuestionHeader-main'], - 'author': ['.AuthorInfo-name', '.AuthorInfo .Text', '[itemprop="name"]'], - 'content': ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist] article'], - 'time': ['.ContentItem-time', '[itemprop="dateCreated"]'], - 'vote': ['.VoteButton--up .VoteButton-label', '.VoteButton--up'] - } - - for selector in selectors['title']: - title_elem = self.browser.query_selector(selector) - if title_elem: - result['title'] = self._extract_text_content(title_elem) - print(f"找到标题（使用选择器 {selector}）: {result['title'][:50]}...") - break - - for selector in selectors['author']: - author_elem = self.browser.query_selector(selector) - if author_elem: - result['author'] = self._extract_text_content(author_elem) - print(f"找到作者（使用选择器 {selector}）: {result['author']}") - break - - for selector in selectors['content']: - content_elem = self.browser.query_selector(selector) - if content_elem: - html_content = self._extract_html_content(content_elem) - result['content'] = html_content - result['content_markdown'] = self.markdown_converter.convert(html_content) - print(f"找到内容（使用选择器 {selector}），长度: {len(result['content_markdown'])}") - break - - for selector in selectors['time']: - time_elem = self.browser.query_selector(selector) - if time_elem: - result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem) - break - - for selector in selectors['vote']: - vote_elem = self.browser.query_selector(selector) - if vote_elem: - vote_text = self._extract_text_content(vote_elem) - try: - result['vote_up'] = int(vote_text.replace(',', '')) - except (ValueError, AttributeError): - pass - break - - return result - - def _extract_article_content(self) -> Dict[str, Any]: - """提取文章内容""" - result = { - 'type': 'article', - 'title': '', - 'author': '', - 'content': '', - 'content_markdown': '', - 'publish_time': '', - 'vote_up': 0, - } - - # 提取文章标题 - title_elem = self.browser.query_selector('.Post-Title') - if title_elem: - result['title'] = self._extract_text_content(title_elem) - - # 提取作者信息 - author_elem = self.browser.query_selector('.AuthorInfo-name') - if author_elem: - result['author'] = self._extract_text_content(author_elem) - - # 提取文章内容 - content_elem = self.browser.query_selector('.Post-RichText') - if content_elem: - html_content = self._extract_html_content(content_elem) - result['content'] = html_content - result['content_markdown'] = self.markdown_converter.convert(html_content) - - # 提取发布时间 - time_elem = self.browser.query_selector('.ContentItem-time') - if time_elem: - result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem) - - # 提取点赞数 - vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label') - if vote_elem: - vote_text = self._extract_text_content(vote_elem) - try: - result['vote_up'] = int(vote_text.replace(',', '')) - except (ValueError, AttributeError): - pass - - return result - - def _extract_question_content(self) -> Dict[str, Any]: - """提取问题内容（问题页本身）""" - result = { - 'type': 'question', - 'title': '', - 'author': '', - 'content': '', - 'content_markdown': '', - 'publish_time': '', - 'vote_up': 0, - } - - # 提取问题标题 - title_elem = self.browser.query_selector('h1.QuestionHeader-title') - if title_elem: - result['title'] = self._extract_text_content(title_elem) - - # 提取问题详情 - detail_elem = self.browser.query_selector('.QuestionHeader-detail') - if detail_elem: - html_content = self._extract_html_content(detail_elem) - result['content'] = html_content - result['content_markdown'] = self.markdown_converter.convert(html_content) - - # 问题的作者通常是提问者 - author_elem = self.browser.query_selector('.AuthorInfo-name') - if author_elem: - result['author'] = self._extract_text_content(author_elem) - - return result - - def extract(self, url: str) -> Optional[Dict[str, Any]]: - """ - 从 URL 提取知乎内容 - - Args: - url: 知乎 URL - - Returns: - 包含提取内容的字典，结构如下: - { - 'type': 'answer' | 'article' | 'question', - 'title': str, - 'author': str, - 'content': str (HTML), - 'content_markdown': str (Markdown), - 'publish_time': str, - 'vote_up': int, - } - 如果提取失败返回 None - """ - try: - # 识别 URL 类型 - url_type = self.identify_url_type(url) - if not url_type: - print(f"无法识别 URL 类型: {url}") - return None - - print(f"URL 类型: {url_type}") - print(f"启动浏览器...") - - # 启动浏览器 - self.browser.start() - - print(f"访问页面: {url}") - # 访问页面 - self.browser.goto(url, wait_until="domcontentloaded") - - print("等待页面加载...") - import time - time.sleep(3) # 等待 JavaScript 执行 - - # 尝试多个可能的选择器 - if url_type == 'answer': - selectors = ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist]'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_answer_content() - elif url_type == 'article': - selectors = ['.Post-RichText', '.Post-Title'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_article_content() - else: # question - selectors = ['.QuestionHeader-title', 'h1'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_question_content() - - # 添加额外信息 - result['url'] = url - result['content_id'] = self.extract_id(url) - - print(f"提取成功! 标题: {result['title'][:50]}...") - return result - - except Exception as e: - print(f"提取知乎内容时出错: {e}") - import traceback - traceback.print_exc() - return None - finally: - # 关闭浏览器 - self.browser.close() - - def __enter__(self): - """上下文管理器入口""" - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """上下文管理器出口""" - self.browser.close() - return False - - -# 便捷函数 -def extract_zhihu_content(url: str, headless: bool = True, timeout: int = 30) -> Optional[Dict[str, Any]]: - """ - 便捷函数：提取知乎内容 - - Args: - url: 知乎 URL - headless: 是否使用无头浏览器 - timeout: 超时时间（秒） - - Returns: - 提取的内容字典 - """ - with ZhihuPlaywrightExtractor(headless=headless, timeout=timeout) as extractor: - return extractor.extract(url) diff --git a/skills/obsidian-kb/src/storage.py b/skills/obsidian-kb/src/storage.py deleted file mode 100644 index 4ec146b..0000000 --- a/skills/obsidian-kb/src/storage.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Obsidian KB - 存储管理""" - -import os -import json -from pathlib import Path -from typing import List, Optional, Dict, Any -from datetime import datetime - -from models import Content, ContentStatus -from config import get_config - - -class Storage: - """JSON文件存储管理""" - - def __init__(self, base_path: Optional[str] = None): - config = get_config() - self.base_path = base_path or config.obsidian_path - self.inbox_path = os.path.join(self.base_path, 'Inbox') - self.topics_path = os.path.join(self.base_path, 'Topics') - self._ensure_dirs() - - def _ensure_dirs(self) -> None: - """确保目录存在""" - os.makedirs(self.inbox_path, exist_ok=True) - os.makedirs(self.topics_path, exist_ok=True) - - def _content_file_path(self, content_id: str) -> str: - """获取内容文件路径""" - return os.path.join(self.inbox_path, f'{content_id}.json') - - def _topic_index_path(self, topic: str) -> str: - """获取主题索引路径""" - topic_dir = os.path.join(self.topics_path, topic) - os.makedirs(topic_dir, exist_ok=True) - return os.path.join(topic_dir, 'index.json') - - # ===== Inbox 管理 ===== - - def save_content(self, content: Content) -> None: - """保存内容到Inbox""" - file_path = self._content_file_path(content.id) - with open(file_path, 'w', encoding='utf-8') as f: - json.dump(content.to_dict(), f, ensure_ascii=False, indent=2) - - def load_content(self, content_id: str) -> Optional[Content]: - """加载指定内容""" - file_path = self._content_file_path(content_id) - if not os.path.exists(file_path): - return None - with open(file_path, 'r', encoding='utf-8') as f: - data = json.load(f) - return Content.from_dict(data) - - def delete_content(self, content_id: str) -> bool: - """删除内容""" - file_path = self._content_file_path(content_id) - if os.path.exists(file_path): - os.remove(file_path) - return True - return False - - def list_inbox(self, status: Optional[ContentStatus] = None) -> List[Content]: - """列出Inbox中的内容""" - contents = [] - if not os.path.exists(self.inbox_path): - return contents - - for filename in os.listdir(self.inbox_path): - if filename.endswith('.json'): - file_path = os.path.join(self.inbox_path, filename) - with open(file_path, 'r', encoding='utf-8') as f: - data = json.load(f) - content = Content.from_dict(data) - if status is None or content.status == status: - contents.append(content) - - # 按创建时间排序 - contents.sort(key=lambda x: x.created_at, reverse=True) - return contents - - def get_pending_count(self) -> int: - """获取待处理内容数量""" - return len(self.list_inbox(ContentStatus.CONFIRMED)) - - def get_stats(self) -> Dict[str, int]: - """获取统计信息""" - all_contents = self.list_inbox() - stats = {status.value: 0 for status in ContentStatus} - for content in all_contents: - stats[content.status.value] += 1 - return stats - - # ===== Topics 管理 ===== - - def add_to_topic(self, content: Content, topic: str) -> None: - """将内容添加到主题""" - index_path = self._topic_index_path(topic) - - # 读取现有索引 - entries = [] - if os.path.exists(index_path): - with open(index_path, 'r', encoding='utf-8') as f: - entries = json.load(f) - - # 添加新条目 - entry = { - 'id': content.id, - 'title': content.title, - 'url': content.url, - 'added_at': datetime.now().isoformat(), - 'output_path': content.output_path - } - - # 避免重复 - entries = [e for e in entries if e['id'] != content.id] - entries.append(entry) - - # 保存索引 - with open(index_path, 'w', encoding='utf-8') as f: - json.dump(entries, f, ensure_ascii=False, indent=2) - - def get_topic_contents(self, topic: str) -> List[Dict[str, Any]]: - """获取主题下的所有内容""" - index_path = self._topic_index_path(topic) - if not os.path.exists(index_path): - return [] - with open(index_path, 'r', encoding='utf-8') as f: - return json.load(f) - - def list_topics(self) -> List[str]: - """列出所有主题""" - if not os.path.exists(self.topics_path): - return [] - return [d for d in os.listdir(self.topics_path) - if os.path.isdir(os.path.join(self.topics_path, d))] - - def search(self, query: str) -> List[Content]: - """搜索内容""" - results = [] - all_contents = self.list_inbox() - query_lower = query.lower() - - for content in all_contents: - if (query_lower in content.title.lower() or - query_lower in content.summary.lower() or - query_lower in content.author.lower() or - any(query_lower in tag.lower() for tag in content.tags)): - results.append(content) - - return results - - def move_to_archive(self, content_id: str) -> bool: - """将内容移动到归档""" - content = self.load_content(content_id) - if not content: - return False - - archive_path = os.path.join(self.base_path, 'Archive') - os.makedirs(archive_path, exist_ok=True) - - source_file = self._content_file_path(content_id) - target_file = os.path.join(archive_path, f'{content_id}.json') - - if os.path.exists(source_file): - os.rename(source_file, target_file) - return True - return False diff --git a/skills/obsidian-kb/src/template.py b/skills/obsidian-kb/src/template.py deleted file mode 100644 index 2c71873..0000000 --- a/skills/obsidian-kb/src/template.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Template engine for note generation.""" - -from typing import Any, Dict -from pathlib import Path - - -class TemplateEngine: - """Manages templates for Obsidian note generation.""" - - def __init__(self, templates_dir: str = None): - """Initialize template engine. - - Args: - templates_dir: Directory containing template files - """ - if templates_dir is None: - base_dir = Path(__file__).parent.parent - templates_dir = base_dir / 'src' / 'templates' - - self.templates_dir = Path(templates_dir) - self.templates_dir.mkdir(parents=True, exist_ok=True) - - def render(self, template_name: str, context: Dict[str, Any]) -> str: - """Render a template with context. - - Args: - template_name: Name of template to use - context: Template context variables - - Returns: - Rendered template string - """ - template_path = self.templates_dir / f"{template_name}.md" - - if template_path.exists(): - template = template_path.read_text(encoding='utf-8') - else: - # Default template - template = self._default_template() - - # Simple variable substitution - result = template - for key, value in context.items(): - placeholder = f"{{{{{key}}}}}" - result = result.replace(placeholder, str(value)) - - return result - - def _default_template(self) -> str: - """Get default note template.""" - return """# {{{title}}} - -{{tags}} - -## Summary - -{{summary}} - -## Original Content - -**Source:** [{{platform}}]({{url}}) -**ID:** {{content_id}} - ---- - -{{content}} - -## Notes - - -""" diff --git a/skills/obsidian-kb/src/template_renderer.py b/skills/obsidian-kb/src/template_renderer.py deleted file mode 100644 index 5661b78..0000000 --- a/skills/obsidian-kb/src/template_renderer.py +++ /dev/null @@ -1,307 +0,0 @@ -""" -Template Renderer Module -Handles rendering of content into Markdown templates using Jinja2. -""" -import os -from datetime import datetime -from typing import Dict, Any, Optional -from pathlib import Path - -# Try to import jinja2, fallback to simple string formatting -try: - from jinja2 import Template, Environment, FileSystemLoader - JINJA2_AVAILABLE = True -except ImportError: - JINJA2_AVAILABLE = False - Template = None - Environment = None - FileSystemLoader = None - - -class TemplateRenderer: - """ - Renders content objects into Markdown templates. - Supports both Jinja2 and simple string formatting. - """ - - def __init__(self, templates_dir: Optional[str] = None): - """ - Initialize the template renderer. - - Args: - templates_dir: Path to templates directory. If None, uses default. - """ - if templates_dir is None: - # Default to templates directory next to this file - self.templates_dir = Path(__file__).parent / "templates" - else: - self.templates_dir = Path(templates_dir) - - self.templates_dir.mkdir(parents=True, exist_ok=True) - - # Initialize Jinja2 environment if available - if JINJA2_AVAILABLE: - self.env = Environment( - loader=FileSystemLoader(str(self.templates_dir)), - trim_blocks=True, - lstrip_blocks=True - ) - else: - self.env = None - - def _get_template_name(self, content: Any) -> str: - """ - Determine which template to use based on content type and duration. - - Args: - content: Content object with content_type and duration attributes - - Returns: - Template filename - """ - content_type = getattr(content, 'content_type', None) - duration = getattr(content, 'duration', 0) - - # Handle enum types - if hasattr(content_type, 'value'): - content_type = content_type.value - - # Convert duration to seconds if it's a string - if isinstance(duration, str): - try: - parts = duration.split(':') - if len(parts) == 2: # MM:SS - duration = int(parts[0]) * 60 + int(parts[1]) - elif len(parts) == 3: # HH:MM:SS - duration = int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2]) - else: - duration = 0 - except (ValueError, IndexError): - duration = 0 - - # Select template based on type and duration - if content_type == 'article': - return 'article.md' - elif content_type == 'video': - # Long videos (>>30 min) use tutorial template - if duration and duration > 1800: # 30 minutes - return 'tutorial_video.md' - else: - return 'light_video.md' - elif duration and duration > 1800: - return 'tutorial_video.md' - else: - return 'light_video.md' - - def _prepare_context(self, content: Any, extra_context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - """ - Prepare the context dictionary for template rendering. - - Args: - content: Content object - extra_context: Additional context to merge - - Returns: - Context dictionary - """ - # Get analysis data if available - analysis_data = {} - if hasattr(content, 'analysis') and content.analysis: - if isinstance(content.analysis, dict): - analysis_data = content.analysis - else: - # It's a ContentAnalysis object - analysis_data = { - 'one_liner': getattr(content.analysis, 'one_liner', ''), - 'summary': getattr(content.analysis, 'summary', ''), - 'key_points': getattr(content.analysis, 'key_points', []), - 'tags': getattr(content.analysis, 'tags', []), - 'quality_score': getattr(content.analysis, 'quality_score', 0), - 'prerequisites': getattr(content.analysis, 'prerequisites', []), - 'outline': getattr(content.analysis, 'outline', []), - 'pitfalls': getattr(content.analysis, 'pitfalls', []), - 'practice_advice': getattr(content.analysis, 'practice_advice', []), - } - - # Base context from content object - context = { - 'title': getattr(content, 'title', 'Untitled'), - 'source': getattr(content, 'source', 'unknown'), - 'url': getattr(content, 'url', ''), - 'author': getattr(content, 'author', 'Unknown'), - 'duration': getattr(content, 'duration', ''), - 'date': getattr(content, 'date', datetime.now().isoformat()), - 'tags': getattr(content, 'tags', ['untagged']), - # First check analysis_data, then fall back to content attributes - 'one_liner': analysis_data.get('one_liner', getattr(content, 'one_liner', '')), - 'summary': analysis_data.get('summary', getattr(content, 'summary', '')), - 'key_points': analysis_data.get('key_points', getattr(content, 'key_points', [])), - 'quality_score': analysis_data.get('quality_score', getattr(content, 'quality_score', 0)), - 'prerequisites': analysis_data.get('prerequisites', []), - 'outline': analysis_data.get('outline', []), - 'pitfalls': analysis_data.get('pitfalls', []), - 'practice_advice': analysis_data.get('practice_advice', []), - } - - # Add transcript-related data - if hasattr(content, 'transcript'): - context['transcript'] = content.transcript - - # Merge extra context - if extra_context: - context.update(extra_context) - - # Ensure tags is a list - if isinstance(context['tags'], str): - context['tags'] = [tag.strip() for tag in context['tags'].split(',')] - if not context['tags']: - context['tags'] = ['untagged'] - - return context - - def render(self, content: Any, extra_context: Optional[Dict[str, Any]] = None) -> str: - """ - Render content into Markdown using appropriate template. - - Args: - content: Content object to render - extra_context: Additional context for template - - Returns: - Rendered Markdown string - """ - template_name = self._get_template_name(content) - context = self._prepare_context(content, extra_context) - - template_path = self.templates_dir / template_name - - # Check if template exists - if not template_path.exists(): - raise FileNotFoundError(f"Template not found: {template_path}") - - # Read template content - template_content = template_path.read_text(encoding='utf-8') - - # Render using Jinja2 if available - if JINJA2_AVAILABLE and self.env: - template = self.env.from_string(template_content) - return template.render(**context) - else: - # Fallback to simple string formatting - return self._simple_render(template_content, context) - - def _simple_render(self, template_content: str, context: Dict[str, Any]) -> str: - """ - Simple string-based template rendering fallback. - Supports basic {{ variable }} and {% for %} loops. - - Args: - template_content: Template string - context: Context dictionary - - Returns: - Rendered string - """ - result = template_content - - # Handle simple for loops (very basic implementation) - import re - - # Pattern for {% for item in items %}...{% endfor %} - for_pattern = r'{%-?\s*for\s+(\w+)\s+in\s+(\w+)\s*-?%}(.*?){%-?\s*endfor\s*-?%}' - - def replace_for(match): - var_name = match.group(1) - list_name = match.group(2) - inner_template = match.group(3) - - items = context.get(list_name, []) - if not items: - return '' - - output = [] - for item in items: - item_str = inner_template - # Handle {{ var.attribute }} or {{ var }} - if isinstance(item, dict): - for key, value in item.items(): - placeholder1 = f'{{{{{var_name}.{key}}}}}' - placeholder2 = f'{{{{ {var_name}.{key} }}}}' - str_val = str(value) if value is not None else '' - item_str = item_str.replace(placeholder1, str_val) - item_str = item_str.replace(placeholder2, str_val) - else: - placeholder1 = f'{{{{{var_name}}}}}' - placeholder2 = f'{{{{ {var_name} }}}}' - str_val = str(item) if item is not None else '' - item_str = item_str.replace(placeholder1, str_val) - item_str = item_str.replace(placeholder2, str_val) - output.append(item_str) - - return ''.join(output) - - result = re.sub(for_pattern, replace_for, result, flags=re.DOTALL) - - # Handle simple if statements (basic) - if_pattern = r'{%-?\s*if\s+(\w+)\s*-?%}(.*?){%-?\s*endif\s*-?%}' - - def replace_if(match): - var_name = match.group(1) - inner_content = match.group(2) - if context.get(var_name): - return inner_content - return '' - - result = re.sub(if_pattern, replace_if, result, flags=re.DOTALL) - - # Handle variable substitution {{ variable }} or {{variable}} - # Sort keys by length (descending) to avoid partial replacements - for var_name in sorted(context.keys(), key=len, reverse=True): - value = context[var_name] - if isinstance(value, list): - str_val = ', '.join(str(v) for v in value) - else: - str_val = str(value) if value is not None else '' - - # Replace both {{ var }} and {{var}} - result = result.replace(f'{{{{ {var_name} }}}}', str_val) - result = result.replace(f'{{{{{var_name}}}}}', str_val) - - return result - - def render_to_file(self, content: Any, output_path: str, extra_context: Optional[Dict[str, Any]] = None) -> str: - """ - Render content and save to file. - - Args: - content: Content object to render - output_path: Path to output file - extra_context: Additional context for template - - Returns: - Path to rendered file - """ - rendered = self.render(content, extra_context) - - output_file = Path(output_path) - output_file.parent.mkdir(parents=True, exist_ok=True) - output_file.write_text(rendered, encoding='utf-8') - - return str(output_file) - - -# Convenience function -def render_content(content: Any, templates_dir: Optional[str] = None, extra_context: Optional[Dict[str, Any]] = None) -> str: - """ - Convenience function to render content without creating a renderer instance. - - Args: - content: Content object to render - templates_dir: Optional templates directory path - extra_context: Additional context for template - - Returns: - Rendered Markdown string - """ - renderer = TemplateRenderer(templates_dir) - return renderer.render(content, extra_context) diff --git a/skills/obsidian-kb/src/templates/article.md b/skills/obsidian-kb/src/templates/article.md deleted file mode 100644 index 157c274..0000000 --- a/skills/obsidian-kb/src/templates/article.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: "{{ title }}" -source: "{{ source }}" -url: "{{ url }}" -author: "{{ author }}" -date: "{{ date }}" -tags:{% for tag in tags %} - - {{ tag }}{% endfor %} -quality_score: {{ quality_score }} -reading_time: {{ reading_time }} ---- - -# {{ title }} - -**原文链接**: [{{ title }}]({{ url }}) - -## 🎯 一句话总结 -{{ one_liner }} - -## 📖 文章摘要 -{{ summary }} - -## 💡 核心观点{% for point in key_points %} -1. {{ point }}{% endfor %} - -## 📝 详细内容 -{{ content }} - -## 🔗 相关链接{% for link in related_links %} -- [{{ link.title }}]({{ link.url }}){% endfor %} - -## 🏷️ 标签 -{% for tag in tags %}#{{ tag }} {% endfor %} diff --git a/skills/obsidian-kb/src/templates/light_video.md b/skills/obsidian-kb/src/templates/light_video.md deleted file mode 100644 index 59b3148..0000000 --- a/skills/obsidian-kb/src/templates/light_video.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: "{{ title }}" -source: "{{ source }}" -url: "{{ url }}" -author: "{{ author }}" -duration: "{{ duration }}" -date: "{{ date }}" -tags:{% for tag in tags %} - - {{ tag }}{% endfor %} ---- - -#{{ tags[0] }} #{{ tags[1] }} - -**一句话摘要**: {{ one_liner }} - -## 摘要 -{{ summary }} - -## 核心要点{% for point in key_points %} -1. {{ point }}{% endfor %} diff --git a/skills/obsidian-kb/src/templates/topic_index.md b/skills/obsidian-kb/src/templates/topic_index.md deleted file mode 100644 index 78c5303..0000000 --- a/skills/obsidian-kb/src/templates/topic_index.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -title: {{topic_name}} -date: {{date}} -tags: [topic-index] ---- - -# {{topic_name}} - -## 简介 -{{description}} - -## 相关内容 -{{content_list}} - -## 学习路径 -{{learning_path}} diff --git a/skills/obsidian-kb/src/templates/tutorial_video.md b/skills/obsidian-kb/src/templates/tutorial_video.md deleted file mode 100644 index 7f589df..0000000 --- a/skills/obsidian-kb/src/templates/tutorial_video.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: "{{ title }}" -source: "{{ source }}" -url: "{{ url }}" -up主: "{{ author }}" -duration: "{{ duration }}" -date: "{{ date }}" -tags:{% for tag in tags %} - - {{ tag }}{% endfor %} -quality_score: {{ quality_score }} ---- - -## 🎯 一句话总结 -{{ one_liner }} - -## 📚 前置知识{% for pre in prerequisites %} -- {{ pre }}{% endfor %} - -## 📝 步骤详解{% for section in outline %} -### {{ section.title }} [{{ section.time_range }}] -{{ section.summary }} -{% if section.code_snippet %} -```{{ section.code_language }} -{{ section.code_snippet }} -``` -{% endif %}{% endfor %} - -## 💡 关键要点{% for point in key_points %} -1. {{ point }}{% endfor %} - -## ⚠️ 常见坑点{% for pitfall in pitfalls %} -- {{ pitfall }}{% endfor %} - -## 🤔 实践建议{% for advice in practice_advice %} -- {{ advice }}{% endfor %} diff --git a/skills/obsidian-kb/tests/__init__.py b/skills/obsidian-kb/tests/__init__.py deleted file mode 100644 index 6774067..0000000 --- a/skills/obsidian-kb/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests for Obsidian KB Skill.""" diff --git a/skills/obsidian-kb/tests/test_bilibili.py b/skills/obsidian-kb/tests/test_bilibili.py deleted file mode 100644 index 95ff829..0000000 --- a/skills/obsidian-kb/tests/test_bilibili.py +++ /dev/null @@ -1,180 +0,0 @@ -""" -Bilibili Adapter 测试 -""" -import sys -import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) - -from datetime import datetime, timedelta -from sources import SourceRegistry, BilibiliAdapter -from sources.bilibili import BilibiliAdapter as BiliAdapter - - -def test_registry(): - """测试注册表""" - print("=" * 50) - print("测试1: SourceRegistry 注册表") - print("=" * 50) - - adapters = SourceRegistry.list() - print(f"已注册适配器: {adapters}") - - adapter_class = SourceRegistry.get('bilibili') - assert adapter_class is not None, "bilibili适配器未注册" - print(f"✓ bilibili适配器类: {adapter_class}") - print() - - -def test_adapter_init(): - """测试适配器初始化""" - print("=" * 50) - print("测试2: 适配器初始化") - print("=" * 50) - - adapter = BiliAdapter() - assert adapter is not None - print("✓ BilibiliAdapter实例化成功") - print() - - -def test_extract_bvid(): - """测试BV号提取""" - print("=" * 50) - print("测试3: BV号提取") - print("=" * 50) - - adapter = BiliAdapter() - - test_urls = [ - ("https://www.bilibili.com/video/BV1GJ411x7h7", "BV1GJ411x7h7"), - ("https://b23.tv/BV1GJ411x7h7", "BV1GJ411x7h7"), - ("BV1GJ411x7h7", "BV1GJ411x7h7"), - ("https://www.bilibili.com/video/bv1gj411x7h7", "bv1gj411x7h7"), - ] - - for url, expected in test_urls: - result = adapter._extract_bvid_from_url(url) - if result and result.upper() == expected.upper(): - print(f"✓ {url} -> {result}") - else: - print(f"✗ {url} -> {result} (期望: {expected})") - print() - - -def test_fetch_content(): - """测试获取视频内容（真实API调用）""" - print("=" * 50) - print("测试4: 从URL提取视频内容") - print("=" * 50) - - adapter = BiliAdapter() - - # 使用一个公开的视频链接测试 - test_url = "https://www.bilibili.com/video/BV1GJ411x7h7" - - try: - content = adapter.extract_content(test_url) - if content: - print(f"✓ 成功提取视频信息:") - print(f" - ID: {content.id}") - print(f" - 标题: {content.title[:50]}...") - print(f" - 作者: {content.author}") - print(f" - 时长: {content.duration}秒") - print(f" - URL: {content.url}") - else: - print(f"✗ 无法提取视频内容") - except Exception as e: - print(f"✗ 提取失败: {e}") - print() - - -def test_get_transcript(): - """测试获取字幕（真实API调用）""" - print("=" * 50) - print("测试5: 获取视频字幕") - print("=" * 50) - - adapter = BiliAdapter() - - # 找一个有字幕的视频 - # BV1GJ411x7h7 是经典测试视频 - test_url = "https://www.bilibili.com/video/BV1GJ411x7h7" - - try: - content = adapter.extract_content(test_url) - if content: - transcript = adapter.get_transcript(content) - if transcript: - lines = transcript.split('\n') - print(f"✓ 成功获取字幕:") - print(f" - 总行数: {len(lines)}") - print(f" - 前200字符:\n{transcript[:200]}...") - else: - print(f"ℹ 该视频无字幕或字幕获取失败") - else: - print(f"✗ 无法获取视频内容") - except Exception as e: - print(f"✗ 获取字幕失败: {e}") - print() - - -def test_parse_duration(): - """测试时长解析""" - print("=" * 50) - print("测试6: 时长解析") - print("=" * 50) - - adapter = BiliAdapter() - - test_cases = [ - ("5:30", 330), - ("1:23:45", 5025), - ("0:45", 45), - ("10:00", 600), - ] - - for duration_str, expected in test_cases: - result = adapter._parse_duration(duration_str) - if result == expected: - print(f"✓ {duration_str} -> {result}秒") - else: - print(f"✗ {duration_str} -> {result}秒 (期望: {expected})") - print() - - -def test_timestamp_conversion(): - """测试时间戳转换""" - print("=" * 50) - print("测试7: 时间戳转换") - print("=" * 50) - - adapter = BiliAdapter() - - # 测试秒级时间戳 - ts = 1704067200 # 2024-01-01 00:00:00 - dt = adapter._timestamp_to_datetime(ts) - print(f"✓ 秒级时间戳 {ts} -> {dt}") - - # 测试毫秒级时间戳 - ts_ms = 1704067200000 - dt = adapter._timestamp_to_datetime(ts_ms) - print(f"✓ 毫秒级时间戳 {ts_ms} -> {dt}") - print() - - -if __name__ == "__main__": - print("\n" + "=" * 60) - print("Bilibili Adapter 测试套件") - print("=" * 60 + "\n") - - test_registry() - test_adapter_init() - test_extract_bvid() - test_parse_duration() - test_timestamp_conversion() - test_fetch_content() - test_get_transcript() - - print("\n" + "=" * 60) - print("测试完成") - print("=" * 60) diff --git a/skills/obsidian-kb/tests/test_chunking.py b/skills/obsidian-kb/tests/test_chunking.py deleted file mode 100644 index 97cc4eb..0000000 --- a/skills/obsidian-kb/tests/test_chunking.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -Unit tests for content chunking module. -""" -import unittest - -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) - -from content_chunker import ContentChunker, Chunk - - -class TestContentChunker(unittest.TestCase): - """Test ContentChunker class.""" - - def setUp(self): - """Set up test fixtures.""" - self.chunker = ContentChunker(chunk_duration_seconds=900) - - def test_chunker_initialization(self): - """Test chunker initialization.""" - self.assertEqual(self.chunker.chunk_duration, 900) - self.assertEqual(self.chunker.max_chunk_duration, 1800) - - def test_parse_srt(self): - """Test SRT parsing.""" - srt_content = """1 -00:00:01,000 --> 00:00:05,000 -First subtitle - -2 -00:00:06,000 --> 00:00:10,000 -Second subtitle - -3 -00:00:11,000 --> 00:00:15,000 -Third subtitle""" - - subtitles = self.chunker._parse_srt(srt_content) - - self.assertEqual(len(subtitles), 3) - self.assertEqual(subtitles[0], (1.0, 5.0, "First subtitle")) - self.assertEqual(subtitles[1], (6.0, 10.0, "Second subtitle")) - self.assertEqual(subtitles[2], (11.0, 15.0, "Third subtitle")) - - def test_time_to_seconds(self): - """Test time conversion.""" - self.assertEqual(self.chunker._time_to_seconds("00:00:30,000"), 30.0) - self.assertEqual(self.chunker._time_to_seconds("00:01:30,000"), 90.0) - self.assertEqual(self.chunker._time_to_seconds("01:00:00,000"), 3600.0) - self.assertEqual(self.chunker._time_to_seconds("01:30:45,500"), 5445.5) - - def test_chunk_by_duration_short_content(self): - """Test chunking short content (single chunk).""" - srt_content = """1 -00:00:00,000 --> 00:00:10,000 -This is the first line - -2 -00:00:10,000 --> 00:00:20,000 -This is the second line""" - - chunks = self.chunker.chunk_by_duration(srt_content, target_duration=900) - - self.assertEqual(len(chunks), 1) - self.assertEqual(chunks[0].index, 0) - self.assertEqual(chunks[0].start_time, 0.0) - self.assertEqual(chunks[0].end_time, 20.0) - - def test_chunk_by_duration_long_content(self): - """Test chunking long content (multiple chunks).""" - # Create SRT content that spans 25 minutes - srt_lines = [] - for i in range(25): - start = i * 60 - end = (i + 1) * 60 - srt_lines.append(f"{i+1}") - srt_lines.append(f"00:{i:02d}:00,000 --> 00:{i+1:02d}:00,000") - srt_lines.append(f"Minute {i+1} content") - srt_lines.append("") - - srt_content = "\n".join(srt_lines) - - # Use 10-minute chunks - chunks = self.chunker.chunk_by_duration(srt_content, target_duration=600) - - self.assertGreater(len(chunks), 1) - # First chunk should start at 0 - self.assertEqual(chunks[0].start_time, 0.0) - - def test_chunk_time_range(self): - """Test chunk time range formatting.""" - chunk = Chunk(index=0, start_time=125.0, end_time=245.0, text="Test") - - self.assertEqual(chunk.time_range, "02:05-04:05") - self.assertEqual(chunk.duration, 120.0) - - def test_chunk_time_range_with_hours(self): - """Test chunk time range with hours.""" - chunk = Chunk(index=0, start_time=3665.0, end_time=7265.0, text="Test") - - self.assertEqual(chunk.time_range, "01:01:05-02:01:05") - - def test_get_chunking_strategy_short(self): - """Test chunking strategy for short content.""" - strategy = self.chunker.get_chunking_strategy(300) # 5 minutes - - self.assertEqual(strategy['strategy'], 'direct') - self.assertEqual(strategy['chunk_count'], 1) - - def test_get_chunking_strategy_medium(self): - """Test chunking strategy for medium content.""" - strategy = self.chunker.get_chunking_strategy(1200) # 20 minutes - - self.assertEqual(strategy['strategy'], 'truncate') - self.assertEqual(strategy['chunk_count'], 1) - - def test_get_chunking_strategy_long(self): - """Test chunking strategy for long content.""" - strategy = self.chunker.get_chunking_strategy(2400) # 40 minutes - - self.assertEqual(strategy['strategy'], 'two_part') - self.assertEqual(strategy['chunk_count'], 2) - - def test_get_chunking_strategy_very_long(self): - """Test chunking strategy for very long content.""" - strategy = self.chunker.get_chunking_strategy(7200) # 2 hours - - self.assertEqual(strategy['strategy'], 'multi_part') - self.assertGreater(strategy['chunk_count'], 2) - - -class TestChunk(unittest.TestCase): - """Test Chunk dataclass.""" - - def test_chunk_properties(self): - chunk = Chunk( - index=1, - start_time=100.0, - end_time=250.0, - text="Test content" - ) - - self.assertEqual(chunk.index, 1) - self.assertEqual(chunk.duration, 150.0) - - -if __name__ == '__main__': - unittest.main() diff --git a/skills/obsidian-kb/tests/test_config.py b/skills/obsidian-kb/tests/test_config.py deleted file mode 100644 index 233b9fb..0000000 --- a/skills/obsidian-kb/tests/test_config.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Unit tests for config module. -""" -import unittest -import tempfile -import os -from pathlib import Path - -import sys -sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) - -from config import Config - - -class TestConfig(unittest.TestCase): - """Test Config class.""" - - def setUp(self): - """Set up test fixtures.""" - self.temp_dir = tempfile.mkdtemp() - self.config_path = Path(self.temp_dir) / "config.yaml" - # Save and clean up environment variables - self.env_backup = {} - for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']: - if key in os.environ: - self.env_backup[key] = os.environ[key] - del os.environ[key] - - def tearDown(self): - """Restore environment variables after each test.""" - # Restore original environment variables - for key, value in self.env_backup.items(): - os.environ[key] = value - # Clean up any that weren't backed up - for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']: - if key in os.environ and key not in self.env_backup: - del os.environ[key] - - def test_default_config(self): - """Test default configuration.""" - config = Config(config_path=str(self.config_path), load_env=False) - - self.assertEqual(config.get('ai.provider'), 'openai') - self.assertEqual(config.get('ai.model'), 'gpt-4') - self.assertTrue(config.get('chunking.enabled')) - self.assertEqual(config.get('chunking.chunk_duration_seconds'), 900) - - def test_get_with_default(self): - """Test get with default value.""" - # Ensure clean environment - for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']: - if key in os.environ: - del os.environ[key] - - config = Config(config_path=str(self.config_path), load_env=False) - - # Existing key - self.assertEqual(config.get('ai.model'), 'gpt-4') - # Non-existing key with default - self.assertEqual(config.get('nonexistent.key', 'default'), 'default') - # Non-existing key without default - self.assertIsNone(config.get('nonexistent.key')) - - def test_set_value(self): - """Test setting configuration value.""" - config = Config(config_path=str(self.config_path), load_env=False) - - config.set('ai.model', 'gpt-3.5-turbo') - self.assertEqual(config.get('ai.model'), 'gpt-3.5-turbo') - - config.set('custom.new_key', 'new_value') - self.assertEqual(config.get('custom.new_key'), 'new_value') - - def test_save_and_load(self): - """Test saving and loading configuration.""" - # Create and save config - config1 = Config(config_path=str(self.config_path), load_env=False) - config1.set('ai.model', 'gpt-3.5-turbo') - config1.set('custom.test', 'value') - config1.save() - - # Load config - config2 = Config(config_path=str(self.config_path), load_env=False) - - self.assertEqual(config2.get('ai.model'), 'gpt-3.5-turbo') - self.assertEqual(config2.get('custom.test'), 'value') - - def test_properties(self): - """Test config properties.""" - # Ensure clean environment - for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']: - if key in os.environ: - del os.environ[key] - - config = Config(config_path=str(self.config_path), load_env=False) - - self.assertIsNone(config.ai_api_key) - self.assertEqual(config.ai_model, 'gpt-4') - self.assertTrue(config.chunking_enabled) - self.assertEqual(config.chunk_duration, 900) - self.assertEqual(config.long_video_threshold, 1800) - - def test_vault_path(self): - """Test vault path property.""" - config = Config(config_path=str(self.config_path), load_env=False) - - path = config.vault_path - self.assertTrue(isinstance(path, Path)) - - def test_env_override_api_key(self): - """Test environment variable override for API key.""" - # Clean up environment first - for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']: - if key in os.environ: - del os.environ[key] - - os.environ['OPENAI_API_KEY'] = 'test_api_key_123' - - config = Config(config_path=str(self.config_path)) - - self.assertEqual(config.ai_api_key, 'test_api_key_123') - - # Clean up - del os.environ['OPENAI_API_KEY'] - - def test_env_override_model(self): - """Test environment variable override for model.""" - # Clean up environment first - for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']: - if key in os.environ: - del os.environ[key] - - os.environ['AI_MODEL'] = 'custom-model' - - config = Config(config_path=str(self.config_path)) - - self.assertEqual(config.ai_model, 'custom-model') - - # Clean up - del os.environ['AI_MODEL'] - - -if __name__ == '__main__': - unittest.main() diff --git a/skills/obsidian-kb/tests/test_models.py b/skills/obsidian-kb/tests/test_models.py deleted file mode 100644 index 07aeb46..0000000 --- a/skills/obsidian-kb/tests/test_models.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Unit tests for models module. -""" -import unittest -import json -from datetime import datetime - -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) - -from models import Content, ContentType, ContentStatus, ContentAnalysis -from content_chunker import Chunk - - -class TestContentType(unittest.TestCase): - """Test ContentType enum.""" - - def test_content_type_values(self): - self.assertEqual(ContentType.VIDEO.value, 'video') - self.assertEqual(ContentType.ARTICLE.value, 'article') - self.assertEqual(ContentType.AUDIO.value, 'audio') - - -class TestContentStatus(unittest.TestCase): - """Test ContentStatus enum.""" - - def test_content_status_values(self): - self.assertEqual(ContentStatus.NEW.value, 'new') - self.assertEqual(ContentStatus.SCANNING.value, 'scanning') - self.assertEqual(ContentStatus.SCANNED.value, 'scanned') - self.assertEqual(ContentStatus.ANALYZING.value, 'analyzing') - self.assertEqual(ContentStatus.GENERATED.value, 'completed') - self.assertEqual(ContentStatus.FAILED.value, 'failed') - self.assertEqual(ContentStatus.SKIPPED.value, 'skipped') - - -class TestContent(unittest.TestCase): - """Test Content model.""" - - def test_content_creation(self): - """Test basic content creation.""" - content = Content( - id="test_001", - source="bilibili", - title="Test Video", - url="https://bilibili.com/video/test" - ) - self.assertEqual(content.id, "test_001") - self.assertEqual(content.source, "bilibili") - self.assertEqual(content.title, "Test Video") - self.assertEqual(content.url, "https://bilibili.com/video/test") - self.assertEqual(content.content_type, ContentType.VIDEO) - self.assertEqual(content.status, ContentStatus.NEW) - - def test_content_with_all_fields(self): - """Test content creation with all fields.""" - analysis = ContentAnalysis( - one_liner="Test summary", - summary="Detailed summary", - key_points=["Point 1", "Point 2"], - tags=["test", "video"], - quality_score=85 - ) - - content = Content( - id="test_002", - source="youtube", - title="Full Test Video", - url="https://youtube.com/watch?v=test", - author="Test Author", - duration=3600, - content_type=ContentType.VIDEO, - status=ContentStatus.GENERATED, - tags=["python", "tutorial"], - transcript="Test transcript", - analysis=analysis - ) - - self.assertEqual(content.author, "Test Author") - self.assertEqual(content.duration, 3600) - self.assertEqual(content.tags, ["python", "tutorial"]) - self.assertEqual(content.analysis.quality_score, 85) - self.assertEqual(content.analysis.key_points, ["Point 1", "Point 2"]) - - def test_content_to_dict(self): - """Test serialization to dictionary.""" - content = Content( - id="test_003", - source="zhihu", - title="Test Article", - url="https://zhihu.com/question/test" - ) - - data = content.to_dict() - - self.assertEqual(data['id'], "test_003") - self.assertEqual(data['source'], "zhihu") - self.assertEqual(data['content_type'], 'video') - self.assertEqual(data['status'], 'new') - - def test_content_from_dict(self): - """Test deserialization from dictionary.""" - data = { - 'id': 'test_004', - 'source': 'bilibili', - 'title': 'From Dict', - 'url': 'https://example.com', - 'content_type': 'article', - 'status': 'scanned', - 'tags': ['test', 'article'], - 'duration': 600 - } - - content = Content.from_dict(data) - - self.assertEqual(content.id, 'test_004') - self.assertEqual(content.content_type, ContentType.ARTICLE) - self.assertEqual(content.status, ContentStatus.SCANNED) - self.assertEqual(content.duration, 600) - - def test_content_json_roundtrip(self): - """Test JSON serialization roundtrip.""" - original = Content( - id="test_005", - source="bilibili", - title="JSON Test", - url="https://example.com", - tags=["test", "json"] - ) - - json_str = original.to_json() - restored = Content.from_json(json_str) - - self.assertEqual(restored.id, original.id) - self.assertEqual(restored.title, original.title) - self.assertEqual(restored.tags, original.tags) - self.assertEqual(restored.content_type, original.content_type) - - -class TestChunk(unittest.TestCase): - """Test Chunk model.""" - - def test_chunk_creation(self): - chunk = Chunk( - index=0, - start_time=0.0, - end_time=300.0, - text="Test transcript" - ) - - self.assertEqual(chunk.index, 0) - self.assertEqual(chunk.start_time, 0.0) - self.assertEqual(chunk.end_time, 300.0) - - -class TestContentAnalysis(unittest.TestCase): - """Test ContentAnalysis model.""" - - def test_analysis_defaults(self): - analysis = ContentAnalysis() - - self.assertEqual(analysis.one_liner, "") - self.assertEqual(analysis.quality_score, 0) - self.assertEqual(analysis.key_points, []) - self.assertEqual(analysis.tags, []) - - def test_analysis_full(self): - analysis = ContentAnalysis( - one_liner="Quick summary", - summary="Full summary", - key_points=["Point A", "Point B", "Point C"], - tags=["python", "tutorial", "advanced"], - quality_score=90, - prerequisites=["Basic Python"], - outline=[{"title": "Part 1", "summary": "Intro"}], - pitfalls=["Don't forget X"], - practice_advice=["Try Y first"] - ) - - self.assertEqual(analysis.quality_score, 90) - self.assertEqual(len(analysis.key_points), 3) - self.assertEqual(len(analysis.outline), 1) - - -if __name__ == '__main__': - unittest.main() diff --git a/skills/obsidian-kb/tests/test_pipeline.py b/skills/obsidian-kb/tests/test_pipeline.py deleted file mode 100644 index b2437fe..0000000 --- a/skills/obsidian-kb/tests/test_pipeline.py +++ /dev/null @@ -1,331 +0,0 @@ -""" -Integration tests for the complete Obsidian KB pipeline. -Tests the full workflow: subscribe → fetch → analyze → generate → sync -""" -import unittest -import tempfile -import json -from pathlib import Path -from datetime import datetime - -import sys -sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) - -from models import Content, ContentType, ContentStatus, ContentAnalysis -from storage import Storage -from template_renderer import TemplateRenderer -from config import Config -from content_chunker import ContentChunker - - -class TestPipeline(unittest.TestCase): - """Integration tests for the complete pipeline.""" - - def setUp(self): - """Set up test environment.""" - self.temp_dir = tempfile.mkdtemp() - self.data_dir = Path(self.temp_dir) / "data" - self.vault_dir = Path(self.temp_dir) / "vault" - self.vault_dir.mkdir(parents=True, exist_ok=True) - - # Initialize components - self.storage = Storage(base_path=str(self.data_dir)) - self.config = Config(config_path=str(self.temp_dir / "config.yaml")) - self.config.set('git.vault_path', str(self.vault_dir)) - - templates_dir = Path(__file__).parent.parent / 'src' / 'templates' - self.renderer = TemplateRenderer(str(templates_dir)) - self.chunker = ContentChunker() - - def test_full_pipeline_short_video(self): - """Test complete pipeline with short video.""" - # 1. Create content (simulating subscription/fetch) - content = Content( - id="bvid_BV1test001", - source="bilibili", - title="Python Tips in 5 Minutes", - url="https://bilibili.com/video/BV1test001", - author="CodeMaster", - duration=300, # 5 minutes - content_type=ContentType.VIDEO, - status=ContentStatus.NEW, - tags=["python", "programming"] - ) - - # 2. Add to inbox - self.storage.add_to_inbox(content) - inbox = self.storage.get_inbox() - self.assertEqual(len(inbox), 1) - - # 3. Simulate light scanning - content.status = ContentStatus.SCANNED - content.analysis = ContentAnalysis( - one_liner="Quick Python productivity tips", - summary="This video covers 5 essential Python tips for beginners.", - key_points=["Use list comprehensions", "Leverage built-in functions"], - tags=["python", "productivity"], - quality_score=85 - ) - self.storage.update_inbox_item(content) - - # 4. Generate note - output_path = self.vault_dir / "inbox" / "python_tips.md" - result_path = self.renderer.render_to_file(content, str(output_path)) - - # 5. Verify output - self.assertTrue(Path(result_path).exists()) - note_content = Path(result_path).read_text(encoding='utf-8') - - # Verify content - self.assertIn("Python Tips in 5 Minutes", note_content) - self.assertIn("Quick Python productivity tips", note_content) - self.assertIn("#python", note_content) - - # 6. Update status to completed - content.status = ContentStatus.GENERATED - content.note_path = str(result_path) - self.storage.update_inbox_item(content) - - # 7. Verify final state - completed_item = self.storage.get_item_by_id(content.id) - self.assertEqual(completed_item.status, ContentStatus.GENERATED) - self.assertIsNotNone(completed_item.note_path) - - def test_pipeline_long_video_with_chunking(self): - """Test pipeline with long video requiring chunking.""" - # 1. Create long video content - content = Content( - id="bvid_BV2long001", - source="bilibili", - title="Complete Python Course", - url="https://bilibili.com/video/BV2long001", - author="PythonAcademy", - duration=7200, # 2 hours - content_type=ContentType.VIDEO, - status=ContentStatus.NEW, - tags=["python", "tutorial", "course"] - ) - - # 2. Add to inbox - self.storage.add_to_inbox(content) - - # 3. Determine chunking strategy - strategy = self.chunker.get_chunking_strategy(content.duration) - self.assertEqual(strategy['strategy'], 'multi_part') - self.assertGreater(strategy['chunk_count'], 2) - - # 4. Simulate chunking analysis - # Create mock chunks - chunks = [ - {"index": 0, "start": 0, "end": 1800, "summary": "Introduction to Python"}, - {"index": 1, "start": 1800, "end": 3600, "summary": "Data structures"}, - {"index": 2, "start": 3600, "end": 5400, "summary": "Functions and classes"}, - {"index": 3, "start": 5400, "end": 7200, "summary": "Advanced topics"}, - ] - - # 5. Simulate deep analysis with chunks - content.status = ContentStatus.GENERATED - content.analysis = ContentAnalysis( - one_liner="Comprehensive Python course from basics to advanced", - summary="A 2-hour course covering Python fundamentals and advanced concepts.", - key_points=["Python basics", "Data structures", "OOP", "Advanced patterns"], - tags=["python", "course", "programming"], - quality_score=95, - prerequisites=["Basic programming knowledge"], - outline=[ - {"title": "Introduction", "time_range": "00:00-30:00", "summary": "Python basics"}, - {"title": "Data Structures", "time_range": "30:00-60:00", "summary": "Lists, dicts, sets"}, - {"title": "OOP", "time_range": "60:00-90:00", "summary": "Classes and objects"}, - {"title": "Advanced", "time_range": "90:00-120:00", "summary": "Decorators, generators"}, - ], - pitfalls=["Python 2 vs 3 confusion", "Mutable default arguments"], - practice_advice=["Code along with examples", "Build projects"] - ) - - # 6. Generate tutorial note - output_path = self.vault_dir / "topics" / "programming" / "python_course.md" - result_path = self.renderer.render_to_file(content, str(output_path)) - - # 7. Verify tutorial template was used - note_content = Path(result_path).read_text(encoding='utf-8') - - self.assertIn("Complete Python Course", note_content) - self.assertIn("Comprehensive Python course", note_content) - self.assertIn("前置知识", note_content) # Prerequisites section - self.assertIn("步骤详解", note_content) # Steps section - self.assertIn("常见坑点", note_content) # Pitfalls section - self.assertIn("实践建议", note_content) # Practice advice section - - def test_pipeline_article(self): - """Test pipeline with article content.""" - # 1. Create article content - content = Content( - id="zhihu_article_001", - source="zhihu", - title="Understanding Neural Networks", - url="https://zhihu.com/question/123456", - author="AIResearcher", - content_type=ContentType.ARTICLE, - status=ContentStatus.NEW, - tags=["ai", "machine-learning"] - ) - - # 2. Add to inbox - self.storage.add_to_inbox(content) - - # 3. Simulate analysis - content.status = ContentStatus.GENERATED - content.analysis = ContentAnalysis( - one_liner="Comprehensive guide to neural network fundamentals", - summary="Deep dive into how neural networks work.", - key_points=["Perceptrons", "Backpropagation", "Activation functions"], - tags=["ai", "neural-networks", "deep-learning"], - quality_score=90 - ) - - # 4. Generate note - output_path = self.vault_dir / "inbox" / "neural_networks.md" - result_path = self.renderer.render_to_file(content, str(output_path)) - - # 5. Verify article template - note_content = Path(result_path).read_text(encoding='utf-8') - - self.assertIn("Understanding Neural Networks", note_content) - self.assertIn("原文链接", note_content) # Original link section - - def test_inbox_to_topic_workflow(self): - """Test moving content from inbox to topics.""" - # 1. Add content to inbox - content = Content( - id="bvid_BV3topic01", - source="bilibili", - title="React Tutorial", - url="https://bilibili.com/video/BV3topic01", - author="FrontendDev", - duration=1800, - tags=["react", "javascript", "frontend"] - ) - content.analysis = ContentAnalysis( - one_liner="React fundamentals", - quality_score=88 - ) - - self.storage.add_to_inbox(content) - - # 2. Process and complete - content.status = ContentStatus.GENERATED - self.storage.update_inbox_item(content) - - # 3. Move to topic (simulate) - topic_path = self.vault_dir / "topics" / "frontend" / "react_tutorial.md" - topic_path.parent.mkdir(parents=True, exist_ok=True) - - result_path = self.renderer.render_to_file(content, str(topic_path)) - content.note_path = str(result_path) - self.storage.update_inbox_item(content) - - # 4. Verify - self.assertTrue(Path(result_path).exists()) - self.assertIn("topics/frontend", content.note_path) - - def test_skip_content(self): - """Test skipping content.""" - # 1. Add content - content = Content( - id="bvid_BV4skip001", - source="bilibili", - title="Low Quality Video", - url="https://bilibili.com/video/BV4skip001" - ) - content.analysis = ContentAnalysis(quality_score=30) # Low quality - - self.storage.add_to_inbox(content) - - # 2. Skip it - content.status = ContentStatus.SKIPPED - self.storage.update_inbox_item(content) - - # 3. Verify not in pending - pending = self.storage.get_pending_items() - skipped_ids = [c.id for c in self.storage.get_inbox() if c.status == ContentStatus.SKIPPED] - - self.assertIn(content.id, skipped_ids) - self.assertNotIn(content, pending) - - def test_batch_processing(self): - """Test processing multiple items.""" - # Add multiple items - contents = [ - Content(id=f"batch_{i}", source="bilibili", title=f"Video {i}", url=f"https://example.com/{i}") - for i in range(5) - ] - - for content in contents: - self.storage.add_to_inbox(content) - - # Process all pending - pending = self.storage.get_pending_items() - self.assertEqual(len(pending), 5) - - for content in pending: - content.status = ContentStatus.GENERATED - content.analysis = ContentAnalysis( - one_liner=f"Summary for {content.title}", - quality_score=80 - ) - self.storage.update_inbox_item(content) - - # Verify all processed - final_pending = self.storage.get_pending_items() - self.assertEqual(len(final_pending), 0) - - -class TestEndToEnd(unittest.TestCase): - """End-to-end integration tests.""" - - def test_config_storage_renderer_integration(self): - """Test integration of config, storage, and renderer.""" - with tempfile.TemporaryDirectory() as tmpdir: - # Setup - data_dir = Path(tmpdir) / "data" - vault_dir = Path(tmpdir) / "vault" - config_path = Path(tmpdir) / "config.yaml" - - config = Config(config_path=str(config_path)) - config.set('git.vault_path', str(vault_dir)) - config.set('chunking.chunk_duration_seconds', 600) - config.save() - - storage = Storage(data_dir=str(data_dir)) - - templates_dir = Path(__file__).parent.parent / 'src' / 'templates' - renderer = TemplateRenderer(str(templates_dir)) - - # Create and process content - content = Content( - id="e2e_test_001", - source="bilibili", - title="E2E Test Video", - url="https://example.com/e2e", - duration=1200 - ) - content.analysis = ContentAnalysis( - one_liner="End to end test", - key_points=["Point 1", "Point 2"] - ) - - # Full workflow - storage.add_to_inbox(content) - output_path = Path(config.vault_path) / "e2e_output.md" - renderer.render_to_file(content, str(output_path)) - content.status = ContentStatus.GENERATED - content.note_path = str(output_path) - storage.update_inbox_item(content) - - # Verify - self.assertTrue(output_path.exists()) - self.assertEqual(storage.get_item_by_id(content.id).status, ContentStatus.GENERATED) - - -if __name__ == '__main__': - unittest.main() diff --git a/skills/obsidian-kb/tests/test_storage.py b/skills/obsidian-kb/tests/test_storage.py deleted file mode 100644 index 1a9476c..0000000 --- a/skills/obsidian-kb/tests/test_storage.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -Unit tests for storage module. -""" -import unittest -import tempfile -from pathlib import Path - -import sys -sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) - -from storage import Storage -from models import Content, ContentType, ContentStatus - - -class TestStorage(unittest.TestCase): - """Test Storage class.""" - - def setUp(self): - """Set up test fixtures.""" - self.temp_dir = tempfile.mkdtemp() - self.storage = Storage(base_path=self.temp_dir) - - def test_initialization(self): - """Test storage initialization.""" - self.assertTrue(Path(self.temp_dir).exists()) - self.assertTrue(Path(self.temp_dir / "topics").exists()) - - def test_get_empty_inbox(self): - """Test getting empty inbox.""" - inbox = self.storage.get_inbox() - self.assertEqual(inbox, []) - - def test_add_to_inbox(self): - """Test adding content to inbox.""" - content = Content( - id="test_001", - source="bilibili", - title="Test Video", - url="https://example.com" - ) - - self.storage.add_to_inbox(content) - inbox = self.storage.get_inbox() - - self.assertEqual(len(inbox), 1) - self.assertEqual(inbox[0].id, "test_001") - - def test_add_duplicate(self): - """Test adding duplicate content.""" - content = Content( - id="test_002", - source="bilibili", - title="Test", - url="https://example.com" - ) - - self.storage.add_to_inbox(content) - self.storage.add_to_inbox(content) # Duplicate - - inbox = self.storage.get_inbox() - self.assertEqual(len(inbox), 1) - - def test_update_inbox_item(self): - """Test updating inbox item.""" - content = Content( - id="test_003", - source="bilibili", - title="Original Title", - url="https://example.com" - ) - - self.storage.add_to_inbox(content) - - # Update content - content.title = "Updated Title" - content.status = ContentStatus.COMPLETED - self.storage.update_inbox_item(content) - - # Retrieve and verify - inbox = self.storage.get_inbox() - self.assertEqual(inbox[0].title, "Updated Title") - self.assertEqual(inbox[0].status, ContentStatus.COMPLETED) - - def test_remove_from_inbox(self): - """Test removing content from inbox.""" - content = Content( - id="test_004", - source="bilibili", - title="To Remove", - url="https://example.com" - ) - - self.storage.add_to_inbox(content) - self.storage.remove_from_inbox("test_004") - - inbox = self.storage.get_inbox() - self.assertEqual(len(inbox), 0) - - def test_get_pending_items(self): - """Test getting pending items.""" - # Add new item - new_content = Content( - id="test_005", - source="bilibili", - title="New", - url="https://example.com", - status=ContentStatus.NEW - ) - self.storage.add_to_inbox(new_content) - - # Add completed item - completed_content = Content( - id="test_006", - source="bilibili", - title="Completed", - url="https://example.com", - status=ContentStatus.COMPLETED - ) - self.storage.add_to_inbox(completed_content) - - pending = self.storage.get_pending_items() - - self.assertEqual(len(pending), 1) - self.assertEqual(pending[0].id, "test_005") - - def test_get_item_by_id(self): - """Test getting item by ID.""" - content = Content( - id="test_007", - source="bilibili", - title="Find Me", - url="https://example.com" - ) - - self.storage.add_to_inbox(content) - - found = self.storage.get_item_by_id("test_007") - not_found = self.storage.get_item_by_id("nonexistent") - - self.assertIsNotNone(found) - self.assertEqual(found.title, "Find Me") - self.assertIsNone(not_found) - - -if __name__ == '__main__': - unittest.main() diff --git a/skills/obsidian-kb/tests/test_subtitle_detector.py b/skills/obsidian-kb/tests/test_subtitle_detector.py deleted file mode 100644 index 821b36e..0000000 --- a/skills/obsidian-kb/tests/test_subtitle_detector.py +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -字幕检测器测试脚本 - -验证： -1. 预设区域定义 -2. 自动字幕区域检测 -3. 缓存机制 -""" - -import sys -import os -import cv2 -import numpy as np - -# 添加项目路径 -project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, project_root) - -from src.extractors.subtitle_detector import SubtitleDetector - - -def create_test_frame_with_text(width=1280, height=720): - """ - 创建包含模拟字幕的测试帧 - - Args: - width: 帧宽度 - height: 帧高度 - - Returns: - numpy.ndarray: 测试帧 - """ - # 创建黑色背景 - frame = np.zeros((height, width, 3), dtype=np.uint8) - - # 添加一些噪声背景 - noise = np.random.randint(0, 50, (height, width, 3), dtype=np.uint8) - frame = cv2.add(frame, noise) - - # 在底部添加白色文字（模拟B站字幕） - font = cv2.FONT_HERSHEY_SIMPLEX - font_scale = 1.0 - thickness = 2 - - # 添加多行字幕 - texts = [ - "这是测试字幕内容", - "B站硬字幕提取测试", - "00:01:23 第一行字幕", - "00:01:25 第二行字幕" - ] - - y_start = int(height * 0.85) - line_height = 35 - - for i, text in enumerate(texts): - y = y_start + i * line_height - x = int(width * 0.15) - - # 绘制文字阴影（黑色） - cv2.putText(frame, text, (x + 2, y + 2), font, font_scale, (0, 0, 0), thickness + 2) - # 绘制白色文字 - cv2.putText(frame, text, (x, y), font, font_scale, (255, 255, 255), thickness) - - return frame - - -def test_preset_regions(): - """测试1: 预设区域定义""" - print("\n" + "="*60) - print("测试1: 预设区域定义") - print("="*60) - - detector = SubtitleDetector() - frame_width, frame_height = 1280, 720 - - # 测试所有预设区域 - for region_name in [detector.REGION_BOTTOM, detector.REGION_CENTER, detector.REGION_TOP]: - try: - x, y, width, height = detector.get_preset_region(region_name, frame_height, frame_width) - print(f"\n✓ 区域 '{region_name}':") - print(f" 坐标: x={x}, y={y}") - print(f" 尺寸: width={width}, height={height}") - print(f" 相对位置: top={y/frame_height:.2f}, bottom={(y+height)/frame_height:.2f}") - except Exception as e: - print(f"✗ 区域 '{region_name}' 失败: {e}") - - # 测试无效区域名称 - try: - detector.get_preset_region("invalid", frame_height, frame_width) - print("\n✗ 错误: 应该抛出异常但没有") - except ValueError as e: - print(f"\n✓ 无效区域名称正确抛出异常: {e}") - - print("\n测试1完成!") - - -def test_auto_detection(): - """测试2: 自动字幕区域检测""" - print("\n" + "="*60) - print("测试2: 自动字幕区域检测") - print("="*60) - - detector = SubtitleDetector() - - # 创建测试帧（带字幕） - print("\n创建测试帧...") - test_frame = create_test_frame_with_text() - height, width = test_frame.shape[:2] - print(f"帧尺寸: {width}x{height}") - - # 保存测试帧用于可视化 - test_frame_path = "/tmp/test_subtitle_frame.png" - cv2.imwrite(test_frame_path, test_frame) - print(f"✓ 测试帧已保存到: {test_frame_path}") - - # 执行自动检测 - print("\n执行自动检测...") - try: - x, y, w, h = detector.auto_detect_subtitle_region(test_frame, video_id=None, use_cache=False) - print(f"✓ 检测到字幕区域:") - print(f" 坐标: x={x}, y={y}") - print(f" 尺寸: width={w}, height={h}") - print(f" 相对位置: top={y/height:.2f}, bottom={(y+h)/height:.2f}") - - # 在测试帧上绘制检测框 - frame_with_box = test_frame.copy() - cv2.rectangle(frame_with_box, (x, y), (x+w, y+h), (0, 255, 0), 2) - - # 保存结果 - result_path = "/tmp/test_subtitle_detected.png" - cv2.imwrite(result_path, frame_with_box) - print(f"✓ 检测结果已保存到: {result_path}") - - # 验证检测区域是否合理（应该在底部） - region_bottom = (y + h) / height - if region_bottom > 0.7: # 底部30% - print(f"✓ 检测区域位置合理（底部）") - else: - print(f"! 警告: 检测区域可能不在底部 (bottom={region_bottom:.2f})") - - except Exception as e: - print(f"✗ 自动检测失败: {e}") - import traceback - traceback.print_exc() - - print("\n测试2完成!") - - -def test_cache_mechanism(): - """测试3: 缓存机制""" - print("\n" + "="*60) - print("测试3: 缓存机制") - print("="*60) - - detector = SubtitleDetector(cache_size=3) # 小缓存便于测试 - - # 创建测试帧 - test_frame = create_test_frame_with_text() - - # 测试1: 首次检测并缓存 - print("\n1. 首次检测并缓存...") - video_id1 = "BV1xx411c7mD" - region1 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id1, use_cache=True) - print(f"✓ 检测完成，区域: {region1}") - print(f"✓ 缓存大小: {detector.get_cache_size()}") - - # 测试2: 再次检测同一视频（应使用缓存） - print("\n2. 再次检测同一视频（应使用缓存）...") - region2 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id1, use_cache=True) - print(f"✓ 缓存结果: {region2}") - if region1 == region2: - print(f"✓ 缓存命中，结果一致") - else: - print(f"✗ 警告: 缓存结果不一致") - - # 测试3: 检测不同视频 - print("\n3. 检测不同视频...") - video_id2 = "BV1yy411c7mE" - region3 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id2, use_cache=True) - print(f"✓ 新视频检测完成，区域: {region3}") - print(f"✓ 缓存大小: {detector.get_cache_size()}") - - # 测试4: 查看所有缓存 - print("\n4. 查看所有缓存...") - cached = detector.get_cached_regions() - print(f"✓ 缓存内容:") - for vid, reg in cached.items(): - print(f" {vid}: {reg}") - - # 测试5: 清空指定视频缓存 - print("\n5. 清空指定视频缓存...") - detector.clear_cache(video_id=video_id1) - print(f"✓ 已删除 {video_id1}") - print(f"✓ 缓存大小: {detector.get_cache_size()}") - - # 测试6: 清空所有缓存 - print("\n6. 清空所有缓存...") - detector.clear_cache() - print(f"✓ 缓存已清空") - print(f"✓ 缓存大小: {detector.get_cache_size()}") - - # 测试7: 缓存容量限制（LRU） - print("\n7. 测试缓存容量限制（LRU）...") - for i in range(5): - vid = f"BV{i:012d}" - detector.auto_detect_subtitle_region(test_frame, video_id=vid, use_cache=True) - print(f" 添加视频 {vid}，缓存大小: {detector.get_cache_size()}") - - cached = detector.get_cached_regions() - print(f"✓ 缓存中有 {len(cached)} 个视频（应为最大容量3）") - print(f"✓ 保留的视频: {list(cached.keys())}") - - print("\n测试3完成!") - - -def test_integration(): - """测试4: 集成测试""" - print("\n" + "="*60) - print("测试4: 集成测试") - print("="*60) - - detector = SubtitleDetector() - test_frame = create_test_frame_with_text() - height, width = test_frame.shape[:2] - - print("\n模拟B站视频字幕检测流程...") - - # 步骤1: 尝试自动检测 - print("\n步骤1: 自动检测字幕区域") - video_id = "BV1xx411c7mD" - region = detector.auto_detect_subtitle_region(test_frame, video_id=video_id) - print(f"✓ 自动检测结果: {region}") - - # 步骤2: 后续帧使用缓存 - print("\n步骤2: 后续帧使用缓存") - for i in range(3): - cached_region = detector.auto_detect_subtitle_region(test_frame, video_id=video_id) - if cached_region == region: - print(f"✓ 帧 {i+1}: 使用缓存，结果一致") - else: - print(f"✗ 帧 {i+1}: 缓存结果异常") - - # 步骤3: 对比预设区域 - print("\n步骤3: 对比预设区域") - bottom_region = detector.get_preset_region(detector.REGION_BOTTOM, height, width) - print(f" 检测区域: y={region[1]}, height={region[3]}") - print(f" 底部预设: y={bottom_region[1]}, height={bottom_region[3]}") - - # 步骤4: 提取字幕区域 - print("\n步骤4: 提取字幕区域用于OCR") - x, y, w, h = region - subtitle_roi = test_frame[y:y+h, x:x+w] - subtitle_path = "/tmp/subtitle_roi.png" - cv2.imwrite(subtitle_path, subtitle_roi) - print(f"✓ 字幕ROI已保存到: {subtitle_path}") - print(f" ROI尺寸: {w}x{h}") - - print("\n测试4完成!") - - -def main(): - """主测试函数""" - print("\n" + "="*60) - print("字幕检测器完整测试") - print("="*60) - - try: - # 运行所有测试 - test_preset_regions() - test_auto_detection() - test_cache_mechanism() - test_integration() - - print("\n" + "="*60) - print("所有测试完成！") - print("="*60) - - print("\n生成的测试文件:") - print(" - /tmp/test_subtitle_frame.png (原始测试帧)") - print(" - /tmp/test_subtitle_detected.png (带检测框的结果)") - print(" - /tmp/subtitle_roi.png (提取的字幕区域)") - - except Exception as e: - print(f"\n✗ 测试失败: {e}") - import traceback - traceback.print_exc() - return 1 - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/obsidian-kb/tests/test_template_renderer.py b/skills/obsidian-kb/tests/test_template_renderer.py deleted file mode 100644 index 15fbea8..0000000 --- a/skills/obsidian-kb/tests/test_template_renderer.py +++ /dev/null @@ -1,257 +0,0 @@ -""" -Unit tests for template renderer module. -""" -import unittest -import tempfile -from pathlib import Path - -import sys -sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) - -from template_renderer import TemplateRenderer, render_content -from models import Content, ContentType, ContentStatus, ContentAnalysis - - -class TestTemplateRenderer(unittest.TestCase): - """Test TemplateRenderer class.""" - - def setUp(self): - """Set up test fixtures.""" - # Use the actual templates directory - self.templates_dir = Path(__file__).parent.parent / 'src' / 'templates' - self.renderer = TemplateRenderer(str(self.templates_dir)) - - def test_initialization(self): - """Test renderer initialization.""" - self.assertTrue(self.renderer.templates_dir.exists()) - - def test_get_template_name_video(self): - """Test template selection for video content.""" - content = Content( - id="test_001", - source="bilibili", - title="Test Video", - url="https://example.com", - duration=600, # 10 minutes - content_type=ContentType.VIDEO - ) - - template_name = self.renderer._get_template_name(content) - self.assertEqual(template_name, 'light_video.md') - - def test_get_template_name_long_video(self): - """Test template selection for long video.""" - content = Content( - id="test_002", - source="bilibili", - title="Long Tutorial", - url="https://example.com", - duration=3600, # 60 minutes - content_type=ContentType.VIDEO - ) - - template_name = self.renderer._get_template_name(content) - self.assertEqual(template_name, 'tutorial_video.md') - - def test_get_template_name_article(self): - """Test template selection for article.""" - content = Content( - id="test_003", - source="zhihu", - title="Test Article", - url="https://example.com", - content_type=ContentType.ARTICLE - ) - - template_name = self.renderer._get_template_name(content) - self.assertEqual(template_name, 'article.md') - - def test_prepare_context(self): - """Test context preparation.""" - content = Content( - id="test_004", - source="bilibili", - title="Test Title", - url="https://example.com", - author="Test Author", - duration=300, - tags=["python", "tutorial"] - ) - - context = self.renderer._prepare_context(content) - - self.assertEqual(context['title'], "Test Title") - self.assertEqual(context['source'], "bilibili") - self.assertEqual(context['author'], "Test Author") - self.assertEqual(context['tags'], ["python", "tutorial"]) - - def test_prepare_context_with_analysis(self): - """Test context preparation with analysis data.""" - analysis = ContentAnalysis( - one_liner="Quick summary", - prerequisites=["Basic Python"], - outline=[{"title": "Part 1", "summary": "Introduction"}], - pitfalls=["Common mistake"], - practice_advice=["Try this"] - ) - - content = Content( - id="test_005", - source="bilibili", - title="Test", - url="https://example.com", - analysis=analysis - ) - - context = self.renderer._prepare_context(content) - - self.assertEqual(context['prerequisites'], ["Basic Python"]) - self.assertEqual(context['outline'], [{"title": "Part 1", "summary": "Introduction"}]) - - def test_render_light_video(self): - """Test rendering light video template.""" - content = Content( - id="test_006", - source="bilibili", - title="Quick Tips", - url="https://example.com", - author="Creator", - duration=300, - tags=["tips", "productivity"] - ) - content.analysis = ContentAnalysis( - one_liner="Great productivity tips", - summary="This video covers essential tips.", - key_points=["Point 1", "Point 2"], - quality_score=85 - ) - - result = self.renderer.render(content) - - self.assertIn("Quick Tips", result) - self.assertIn("Great productivity tips", result) - self.assertIn("Point 1", result) - self.assertIn("#tips", result) - - def test_render_tutorial_video(self): - """Test rendering tutorial video template.""" - content = Content( - id="test_007", - source="bilibili", - title="Python Tutorial", - url="https://example.com", - author="PythonMaster", - duration=3600, - tags=["python", "programming"] - ) - content.analysis = ContentAnalysis( - one_liner="Complete Python guide", - prerequisites=["Basic programming"], - outline=[{ - "title": "Introduction", - "time_range": "00:00-05:00", - "summary": "Getting started" - }], - key_points=["Install Python", "Write first script"], - pitfalls=["Don't use Python 2"], - practice_advice=["Practice daily"], - quality_score=95 - ) - - result = self.renderer.render(content) - - self.assertIn("Python Tutorial", result) - self.assertIn("Complete Python guide", result) - self.assertIn("Basic programming", result) - - def test_render_article(self): - """Test rendering article template.""" - content = Content( - id="test_008", - source="zhihu", - title="Deep Dive Article", - url="https://zhihu.com/article/test", - author="Writer", - content_type=ContentType.ARTICLE, - tags=["technology", "ai"] - ) - content.analysis = ContentAnalysis( - one_liner="Comprehensive analysis", - summary="Detailed article content.", - key_points=["Insight 1", "Insight 2"], - quality_score=90 - ) - - result = self.renderer.render(content) - - self.assertIn("Deep Dive Article", result) - self.assertIn("Comprehensive analysis", result) - - def test_render_to_file(self): - """Test rendering to file.""" - with tempfile.TemporaryDirectory() as tmpdir: - content = Content( - id="test_009", - source="bilibili", - title="File Test", - url="https://example.com", - duration=600 - ) - content.analysis = ContentAnalysis( - one_liner="Test one-liner", - key_points=["Point A"] - ) - - output_path = Path(tmpdir) / "output.md" - result_path = self.renderer.render_to_file(content, str(output_path)) - - self.assertTrue(Path(result_path).exists()) - content_text = Path(result_path).read_text(encoding='utf-8') - self.assertIn("File Test", content_text) - - def test_simple_render_fallback(self): - """Test simple rendering fallback (no Jinja2).""" - template = "Title: {{ title }}, Author: {{ author }}" - context = {'title': 'Test', 'author': 'User'} - - result = self.renderer._simple_render(template, context) - - self.assertIn("Title: Test", result) - self.assertIn("Author: User", result) - - def test_simple_render_with_list(self): - """Test simple rendering with list values.""" - template = "Tags: {{ tags }}" - context = {'tags': ['python', 'coding']} - - result = self.renderer._simple_render(template, context) - - self.assertIn("Tags: python, coding", result) - - -class TestRenderContentFunction(unittest.TestCase): - """Test render_content convenience function.""" - - def test_render_content(self): - """Test convenience function.""" - templates_dir = Path(__file__).parent.parent / 'src' / 'templates' - content = Content( - id="test_010", - source="bilibili", - title="Convenience Test", - url="https://example.com", - duration=300 - ) - content.analysis = ContentAnalysis( - one_liner="Simple test", - key_points=["Test point"] - ) - - result = render_content(content, str(templates_dir)) - - self.assertIn("Convenience Test", result) - self.assertIn("Simple test", result) - - -if __name__ == '__main__': - unittest.main() diff --git a/skills/obsidian-kb/tests/test_zhihu_playwright.py b/skills/obsidian-kb/tests/test_zhihu_playwright.py deleted file mode 100644 index 73fc452..0000000 --- a/skills/obsidian-kb/tests/test_zhihu_playwright.py +++ /dev/null @@ -1,736 +0,0 @@ -"""知乎 Playwright Mock 测试""" - -import pytest -import sys -from pathlib import Path -from unittest.mock import Mock, patch, MagicMock, Mock as MockObj - -# 添加路径 -sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) - -# Mock playwright 模块 -mock_playwright = MockObj() -mock_playwright.sync_api = MockObj() -mock_playwright.sync_api.sync_playwright = MockObj(return_value=MockObj( - start=MockObj(return_value=MockObj( - chromium=MockObj( - launch=MockObj(return_value=MockObj( - new_context=MockObj(return_value=MockObj( - new_page=MockObj(return_value=MockObj( - goto=MockObj(), - wait_for_selector=MockObj(), - query_selector=MockObj(return_value=MockObj( - inner_text=MockObj(return_value='测试内容'), - inner_html=MockObj(return_value='

测试

'), - get_attribute=MockObj(return_value='2024-01-01') - )), - query_selector_all=MockObj() - )) - )) - )) - ) - )) -)) - -sys.modules['playwright'] = mock_playwright -sys.modules['playwright.sync_api'] = mock_playwright.sync_api -sys.modules['playwright.sync_api.sync_playwright'] = mock_playwright.sync_api.sync_playwright - - -class TestURLDetection: - """测试 URL 识别""" - - def test_zhihu_answer_url(self): - """测试知乎回答 URL 识别""" - try: - from sources.zhihu import ZhihuSource - - source = ZhihuSource() - url = "https://www.zhihu.com/question/123456/answer/789012" - - # 测试 can_handle - assert source.can_handle(url) is True - - # 测试 extract_id - content_id = source.extract_id(url) - assert content_id == "789012" - - print("✅ 知乎回答 URL 识别测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 导入失败（BaseSource 不存在）: {e}") - - def test_zhihu_article_url(self): - """测试知乎文章 URL 识别""" - try: - from sources.zhihu import ZhihuSource - - source = ZhihuSource() - url = "https://zhuanlan.zhihu.com/p/123456789" - - # 测试 can_handle - assert source.can_handle(url) is True - - # 测试 extract_id - content_id = source.extract_id(url) - assert content_id == "123456789" - - print("✅ 知乎文章 URL 识别测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 导入失败（BaseSource 不存在）: {e}") - - def test_zhihu_question_url(self): - """测试知乎问题 URL 识别""" - try: - from sources.zhihu import ZhihuSource - - source = ZhihuSource() - url = "https://www.zhihu.com/question/123456" - - # 测试 can_handle - assert source.can_handle(url) is True - - # 测试 extract_id - content_id = source.extract_id(url) - assert content_id == "123456" - - print("✅ 知乎问题 URL 识别测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 导入失败（BaseSource 不存在）: {e}") - - def test_non_zhihu_url(self): - """测试非知乎 URL 识别""" - try: - from sources.zhihu import ZhihuSource - - source = ZhihuSource() - url = "https://www.example.com/article/123" - - assert source.can_handle(url) is False - - print("✅ 非知乎 URL 识别测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 导入失败（BaseSource 不存在）: {e}") - - def test_url_pattern_direct(self): - """直接测试 URL 模式匹配""" - import re - - # 定义知乎 URL 模式 - patterns = [ - r'zhihu\.com/question/\d+/answer/(\d+)', - r'zhuanlan\.zhihu\.com/p/(\d+)', - r'zhihu\.com/question/(\d+)', - ] - - test_cases = [ - ("https://www.zhihu.com/question/123456/answer/789012", "789012"), - ("https://zhuanlan.zhihu.com/p/123456789", "123456789"), - ("https://www.zhihu.com/question/999888", "999888"), - ] - - for url, expected_id in test_cases: - extracted = None - for pattern in patterns: - match = re.search(pattern, url) - if match: - extracted = match.group(1) - break - - assert extracted == expected_id, f"URL {url} 应提取到 {expected_id}，实际得到 {extracted}" - - print("✅ URL 模式匹配测试通过") - - -class TestPlaywrightBrowserMock: - """测试 Mock Playwright 浏览器""" - - def test_mock_playwright_browser_start(self): - """测试 Mock Playwright 浏览器启动""" - try: - from sources.zhihu_playwright_extractor import PlaywrightBrowser - - browser = PlaywrightBrowser() - browser.start() - - # 验证 mock 被调用 - assert mock_playwright.sync_api.sync_playwright.called - - print("✅ Mock Playwright 浏览器启动测试通过") - except ImportError as e: - pytest.skip(f"PlaywrightBrowser 导入失败: {e}") - - def test_mock_playwright_browser_close(self): - """测试 Mock Playwright 浏览器关闭""" - try: - from sources.zhihu_playwright_extractor import PlaywrightBrowser - - # 设置 mock 返回值 - mock_playwright.sync_api.sync_playwright.return_value.start.return_value.chromium.launch.return_value.new_context.return_value.new_page.return_value.close = Mock() - - browser = PlaywrightBrowser() - browser.browser = mock_playwright.sync_api.sync_playwright.return_value.start.return_value.chromium.launch.return_value.new_context.return_value.new_page.return_value - browser.context = browser.browser.new_context.return_value - browser.page = browser.browser.new_page.return_value - browser.playwright = mock_playwright.sync_api.sync_playwright.return_value - - browser.close() - - # 验证关闭被调用 - assert browser.page.close.called - - print("✅ Mock Playwright 浏览器关闭测试通过") - except ImportError as e: - pytest.skip(f"PlaywrightBrowser 导入失败: {e}") - - def test_mock_playwright_goto_and_query(self): - """测试 Mock Playwright 导航和查询""" - try: - from sources.zhihu_playwright_extractor import PlaywrightBrowser - - # 创建浏览器实例 - browser = PlaywrightBrowser() - browser.browser = mock_playwright.sync_api.sync_playwright.return_value.start.return_value.chromium.launch.return_value.new_context.return_value.new_page.return_value - browser.context = browser.browser.new_context.return_value - browser.page = browser.browser.new_page.return_value - - # 测试导航 - url = "https://www.zhihu.com/question/123456" - browser.goto(url) - assert browser.page.goto.called - - # 测试查询 - selector = "h1.QuestionHeader-title" - browser.query_selector(selector) - assert browser.page.query_selector.called - - # 测试查询所有 - browser.query_selector_all(selector) - assert browser.page.query_selector_all.called - - print("✅ Mock Playwright 导航和查询测试通过") - except ImportError as e: - pytest.skip(f"PlaywrightBrowser 导入失败: {e}") - - -class TestPlaywrightTimeoutHandling: - """测试 Playwright 超时处理""" - - def test_playwright_start_timeout(self): - """测试 Playwright 启动超时""" - try: - from sources.zhihu_playwright_extractor import PlaywrightBrowser - - # 设置 mock 抛出超时异常 - mock_playwright.sync_api.sync_playwright.side_effect = TimeoutError("Playwright 启动超时") - - browser = PlaywrightBrowser() - - # 测试启动超时 - with pytest.raises(TimeoutError): - browser.start() - - print("✅ Playwright 启动超时测试通过") - except ImportError as e: - pytest.skip(f"PlaywrightBrowser 导入失败: {e}") - - def test_playwright_goto_timeout(self): - """测试 Playwright 页面导航超时""" - try: - from sources.zhihu_playwright_extractor import PlaywrightBrowser - - # 设置 mock page goto 抛出超时异常 - mock_page = mock_playwright.sync_api.sync_playwright.return_value.start.return_value.chromium.launch.return_value.new_context.return_value.new_page.return_value - mock_page.goto.side_effect = TimeoutError("页面加载超时") - - browser = PlaywrightBrowser() - browser.browser = mock_page - browser.context = mock_page.new_context.return_value - browser.page = mock_page - - # 测试导航超时 - with pytest.raises(TimeoutError): - browser.goto("https://www.zhihu.com/question/123456") - - print("✅ Playwright 页面导航超时测试通过") - except ImportError as e: - pytest.skip(f"PlaywrightBrowser 导入失败: {e}") - - def test_playwright_wait_for_selector_timeout(self): - """测试 Playwright 等待选择器超时""" - try: - from sources.zhihu_playwright_extractor import PlaywrightBrowser - - # 设置 mock page wait_for_selector 抛出超时异常 - mock_page = mock_playwright.sync_api.sync_playwright.return_value.start.return_value.chromium.launch.return_value.new_context.return_value.new_page.return_value - mock_page.wait_for_selector.side_effect = TimeoutError("等待选择器超时") - - browser = PlaywrightBrowser() - browser.browser = mock_page - browser.context = mock_page.new_context.return_value - browser.page = mock_page - - # 测试等待选择器超时 - with pytest.raises(TimeoutError): - browser.wait_for_selector(".RichContent-inner") - - print("✅ Playwright 等待选择器超时测试通过") - except ImportError as e: - pytest.skip(f"PlaywrightBrowser 导入失败: {e}") - - -class TestPlaywrightForbiddenError: - """测试 Playwright 403 错误处理""" - - def test_playwright_goto_403_error(self): - """测试 Playwright 遇到 403 错误""" - try: - from sources.zhihu_playwright_extractor import PlaywrightBrowser - - # 创建 mock response - mock_response = Mock() - mock_response.status = 403 - mock_response.status_text = "Forbidden" - - mock_page = mock_playwright.sync_api.sync_playwright.return_value.start.return_value.chromium.launch.return_value.new_context.return_value.new_page.return_value - mock_page.goto.return_value = mock_response - - # 设置 close 方法避免后续错误 - mock_page.close = Mock() - mock_page.new_context.return_value.close = Mock() - mock_page.new_context.return_value.new_page.return_value.close = Mock() - - browser = PlaywrightBrowser() - browser.browser = mock_page - browser.context = mock_page.new_context.return_value - browser.page = mock_page - - # 测试导航到返回 403 的页面 - result = browser.goto("https://www.zhihu.com/question/123456") - - # 验证返回的响应状态 - assert result.status == 403 - assert result.status_text == "Forbidden" - - print("✅ Playwright 403 错误测试通过") - except ImportError as e: - pytest.skip(f"PlaywrightBrowser 导入失败: {e}") - - -class TestPlaywrightFallbackToHTTP: - """测试 Playwright 降级到 HTTP 的机制""" - - def test_playwright_timeout_fallback_to_http(self): - """测试 Playwright 超时降级到 HTTP""" - try: - from sources.zhihu import ZhihuSource - - # 设置 Playwright 导入失败 - with patch('sources.zhihu_playwright_extractor.ZhihuPlaywrightExtractor') as MockExtractor: - mock_extractor_instance = MagicMock() - mock_extractor_instance.extract.side_effect = TimeoutError("Playwright 启动超时") - MockExtractor.return_value = mock_extractor_instance - - source = ZhihuSource() - url = "https://www.zhihu.com/question/123456/answer/789012" - - # 测试 get_transcript 的降级逻辑 - with patch.object(source, '_fetch_with_http', return_value="HTTP 内容"): - transcript = source.get_transcript(source.fetch(url)) - - # 验证是否降级到 HTTP - assert transcript == "HTTP 内容" - - print("✅ Playwright 超时降级测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 导入失败: {e}") - - def test_playwright_import_error_fallback(self): - """测试 Playwright 导入错误降级""" - try: - from sources.zhihu import ZhihuSource - - # 设置 Playwright 导入失败 - with patch('sources.zhihu_playwright_extractor.ZhihuPlaywrightExtractor', side_effect=ImportError("Playwright 未安装")): - source = ZhihuSource() - url = "https://www.zhihu.com/question/123456/answer/789012" - - # 测试 get_transcript 的降级逻辑 - with patch.object(source, '_fetch_with_http', return_value="HTTP 内容"): - transcript = source.get_transcript(source.fetch(url)) - - # 验证是否降级到 HTTP - assert transcript == "HTTP 内容" - - print("✅ Playwright 导入错误降级测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 导入失败: {e}") - - def test_playwright_runtime_error_fallback(self): - """测试 Playwright 运行时错误降级""" - try: - from sources.zhihu import ZhihuSource - - # 设置 Playwright 提取器抛出运行时错误 - with patch('sources.zhihu_playwright_extractor.ZhihuPlaywrightExtractor') as MockExtractor: - mock_extractor_instance = MagicMock() - mock_extractor_instance.extract.side_effect = RuntimeError("浏览器崩溃") - MockExtractor.return_value = mock_extractor_instance - - source = ZhihuSource() - url = "https://www.zhihu.com/question/123456/answer/789012" - - # 测试 get_transcript 的降级逻辑 - with patch.object(source, '_fetch_with_http', return_value="HTTP 内容"): - transcript = source.get_transcript(source.fetch(url)) - - # 验证是否降级到 HTTP - assert transcript == "HTTP 内容" - - print("✅ Playwright 运行时错误降级测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 导入失败: {e}") - - -class TestZhihuPlaywrightExtractorMock: - """测试 Mock ZhihuPlaywrightExtractor""" - - def test_extract_answer_success(self): - """测试成功提取回答""" - try: - from sources.zhihu_playwright_extractor import ZhihuPlaywrightExtractor - - # 设置 mock 元素 - 创建多个 mock 元素以匹配不同的选择器 - mock_title = Mock() - mock_title.inner_text.return_value = "测试问题" - - mock_author = Mock() - mock_author.inner_text.return_value = "测试作者" - - mock_content = Mock() - mock_content.inner_html.return_value = "

测试内容

" - - mock_time = Mock() - mock_time.get_attribute.return_value = "2024-01-01" - - mock_vote = Mock() - mock_vote.inner_text.return_value = "100" - - def mock_query_selector(selector): - if 'QuestionHeader-title' in selector: - return mock_title - elif 'AuthorInfo-name' in selector: - return mock_author - elif 'RichContent-inner' in selector: - return mock_content - elif 'ContentItem-time' in selector: - return mock_time - elif 'VoteButton--up' in selector: - return mock_vote - return None - - mock_page = mock_playwright.sync_api.sync_playwright.return_value.start.return_value.chromium.launch.return_value.new_context.return_value.new_page.return_value - mock_page.query_selector = mock_query_selector - - # 设置 close 方法避免后续错误 - mock_page.close = Mock() - mock_page.new_context.return_value.close = Mock() - mock_page.new_context.return_value.new_page.return_value.close = Mock() - - # 创建提取器并提取内容 - extractor = ZhihuPlaywrightExtractor() - url = "https://www.zhihu.com/question/123456/answer/789012" - result = extractor.extract(url) - - # 验证结果 - assert result is not None - assert result['type'] == 'answer' - assert result['title'] == '测试问题' - assert result['author'] == '测试作者' - assert result['content_id'] == '789012' - - print("✅ 成功提取回答测试通过") - except ImportError as e: - pytest.skip(f"ZhihuPlaywrightExtractor 导入失败: {e}") - - def test_extract_article_success(self): - """测试成功提取文章""" - try: - from sources.zhihu_playwright_extractor import ZhihuPlaywrightExtractor - - # 设置 mock 元素 - 创建多个 mock 元素以匹配不同的选择器 - mock_title = Mock() - mock_title.inner_text.return_value = "测试文章标题" - - mock_author = Mock() - mock_author.inner_text.return_value = "测试作者" - - mock_content = Mock() - mock_content.inner_html.return_value = "

测试文章内容

" - - mock_time = Mock() - mock_time.get_attribute.return_value = "2024-01-01" - - mock_vote = Mock() - mock_vote.inner_text.return_value = "200" - - def mock_query_selector(selector): - if 'Post-Title' in selector: - return mock_title - elif 'AuthorInfo-name' in selector: - return mock_author - elif 'Post-RichText' in selector: - return mock_content - elif 'ContentItem-time' in selector: - return mock_time - elif 'VoteButton--up' in selector: - return mock_vote - return None - - mock_page = mock_playwright.sync_api.sync_playwright.return_value.start.return_value.chromium.launch.return_value.new_context.return_value.new_page.return_value - mock_page.query_selector = mock_query_selector - - # 设置 close 方法避免后续错误 - mock_page.close = Mock() - mock_page.new_context.return_value.close = Mock() - mock_page.new_context.return_value.new_page.return_value.close = Mock() - - # 创建提取器并提取内容 - extractor = ZhihuPlaywrightExtractor() - url = "https://zhuanlan.zhihu.com/p/123456789" - result = extractor.extract(url) - - # 验证结果 - assert result is not None - assert result['type'] == 'article' - assert result['title'] == '测试文章标题' - assert result['content_id'] == '123456789' - - print("✅ 成功提取文章测试通过") - except ImportError as e: - pytest.skip(f"ZhihuPlaywrightExtractor 导入失败: {e}") - - -class TestMarkdownConverter: - """测试 Markdown 转换器""" - - def test_simple_html_conversion(self): - """测试简单 HTML 转 Markdown""" - try: - from sources.markdown_converter import MarkdownConverter - - converter = MarkdownConverter() - html = "

测试

内容

" - md = converter.convert(html) - - assert "# 测试" in md - assert "内容" in md - - print("✅ 简单 HTML 转换测试通过") - except ImportError as e: - pytest.skip(f"MarkdownConverter 未安装: {e}") - - def test_complex_html_conversion(self): - """测试复杂 HTML 转 Markdown""" - try: - from sources.markdown_converter import MarkdownConverter - - converter = MarkdownConverter() - html = """ -
-

标题

-

段落加粗和斜体

-
    -
  • 列表项 1
    • -
  • 列表项 2
    • -
-
引用内容
-
- """ - md = converter.convert(html) - - assert "## 标题" in md - assert "**加粗**" in md - assert "*斜体*" in md - assert "* 列表项 1" in md or "- 列表项 1" in md - assert ">" in md # 引用格式 - - print("✅ 复杂 HTML 转换测试通过") - except ImportError as e: - pytest.skip(f"MarkdownConverter 未安装: {e}") - - def test_code_blocks_conversion(self): - """测试代码块转换""" - try: - from sources.markdown_converter import MarkdownConverter - - converter = MarkdownConverter() - html = """ - 
def hello():
-    print("Hello, World!")
-
- """ - md = converter.convert(html) - - assert "```" in md or " def hello():" in md - - print("✅ 代码块转换测试通过") - except ImportError as e: - pytest.skip(f"MarkdownConverter 未安装: {e}") - - def test_script_and_style_stripping(self): - """测试移除 script 和 style 标签""" - try: - from sources.markdown_converter import MarkdownConverter - - converter = MarkdownConverter() - html = """ - - -

实际内容

- """ - md = converter.convert(html) - - # markdownify 可能保留 script/style 内容，但至少应该有实际内容 - assert "实际内容" in md - - print("✅ Script 和 Style 移除测试通过") - except ImportError as e: - pytest.skip(f"MarkdownConverter 未安装: {e}") - - -class TestErrorHandling: - """测试错误处理""" - - def test_timeout_handling_mock(self): - """测试超时处理（Mock 版本）""" - # 模拟超时异常 - def mock_fetch_with_timeout(url, timeout=1): - """模拟带超时的请求""" - raise TimeoutError(f"Request to {url} timed out after {timeout}s") - - # 测试超时 - with pytest.raises(TimeoutError) as exc_info: - mock_fetch_with_timeout("https://www.zhihu.com", timeout=30) - - assert "timed out" in str(exc_info.value) - - print("✅ 超时处理测试通过") - - def test_timeout_handling_sync(self): - """测试超时处理（同步版本）""" - # 模拟一个可能超时的函数 - def mock_fetch_with_possible_timeout(url, should_timeout=False): - """模拟可能超时的请求""" - if should_timeout: - raise TimeoutError(f"Request to {url} timed out") - return "success" - - # 测试正常情况 - result = mock_fetch_with_possible_timeout("https://www.zhihu.com", should_timeout=False) - assert result == "success" - - # 测试超时情况 - with pytest.raises(TimeoutError): - mock_fetch_with_possible_timeout("https://www.zhihu.com", should_timeout=True) - - print("✅ 超时处理（同步）测试通过") - - def test_403_error_handling(self): - """测试 403 错误处理""" - # 模拟 403 响应 - class MockResponse: - def __init__(self, status_code): - self.status_code = status_code - self.url = "https://www.zhihu.com/question/123456" - - def raise_for_status(self): - if self.status_code == 403: - from requests.exceptions import HTTPError - raise HTTPError(f"403 Forbidden for {self.url}") - - # 测试 403 错误 - response = MockResponse(403) - with pytest.raises(Exception): # HTTPError - response.raise_for_status() - - print("✅ 403 错误处理测试通过") - - def test_file_permission_error(self): - """测试文件权限错误""" - try: - with pytest.raises(PermissionError): - open("/tmp/test.txt", "w") - except AssertionError: - # 在某些环境下可能没有权限错误，跳过 - pytest.skip("无法在当前环境下测试文件权限错误") - - print("✅ 文件权限错误测试通过") - - def test_network_error_handling(self): - """测试网络错误处理""" - # 模拟网络连接错误 - class MockNetworkError(Exception): - pass - - def mock_fetch_url(url): - if url.startswith("https://www.zhihu.com"): - raise MockNetworkError("Network unreachable") - return "success" - - # 测试网络错误 - with pytest.raises(MockNetworkError): - mock_fetch_url("https://www.zhihu.com/question/123456") - - print("✅ 网络错误处理测试通过") - - -class TestIntegration: - """集成测试""" - - def test_zhihu_source_content_type(self): - """测试内容类型识别""" - try: - from sources.zhihu import ZhihuSource - try: - from src.models import ContentType - except ImportError: - from models import ContentType - - source = ZhihuSource() - - # 测试文章类型 - article_url = "https://zhuanlan.zhihu.com/p/123456789" - article_content = source.fetch(article_url) - assert article_content is not None - assert article_content.content_type == ContentType.ARTICLE - - print("✅ 内容类型识别测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 或 ContentType 导入失败: {e}") - - def test_url_pattern_matching(self): - """测试 URL 模式匹配""" - try: - from sources.zhihu import ZhihuSource - - source = ZhihuSource() - - # 测试所有 URL 模式 - test_urls = [ - ("https://www.zhihu.com/question/123456/answer/789012", "789012"), - ("https://zhuanlan.zhihu.com/p/123456789", "123456789"), - ("https://www.zhihu.com/question/999888", "999888"), - ] - - for url, expected_id in test_urls: - extracted_id = source.extract_id(url) - assert extracted_id == expected_id, f"Failed for {url}: got {extracted_id}, expected {expected_id}" - - print("✅ URL 模式匹配测试通过") - except ImportError as e: - pytest.skip(f"ZhihuSource 导入失败: {e}") - - -if __name__ == "__main__": - # 运行所有测试 - pytest.main([__file__, "-v"]) diff --git a/skills/obsidian-kb/verify_cookie_config.py b/skills/obsidian-kb/verify_cookie_config.py deleted file mode 100644 index bb954af..0000000 --- a/skills/obsidian-kb/verify_cookie_config.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -""" -验证知乎 Cookie 配置 -""" -import json -import os -import sys - -# 添加 src 目录到 Python 路径 -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) - -from sources.zhihu_drission_extractor import ZhihuDrissionExtractor - - -def verify_cookie_config(): - """验证 Cookie 配置""" - print("=" * 60) - print("知乎 Cookie 配置验证") - print("=" * 60) - print() - - # 1. 检查 Cookie 文件路径 - cookie_file = os.path.expanduser('~/.obsidian-kb/zhihu_cookies.json') - print(f"1. Cookie 文件路径") - print(f" {cookie_file}") - if os.path.exists(cookie_file): - print(f" ✅ 文件存在") - else: - print(f" ❌ 文件不存在") - print(f" 请上传 Cookie 文件到此路径") - print() - return False - - print() - - # 2. 检查 Cookie 文件格式 - print(f"2. Cookie 文件格式") - try: - with open(cookie_file, 'r', encoding='utf-8') as f: - cookies = json.load(f) - print(f" ✅ JSON 格式正确") - print(f" Cookie 数量: {len(cookies)}") - except json.JSONDecodeError as e: - print(f" ❌ JSON 格式错误: {e}") - return False - except Exception as e: - print(f" ❌ 读取失败: {e}") - return False - - print() - - # 3. 检查关键 Cookie - print(f"3. 关键 Cookie 检查") - cookie_names = [c['name'] for c in cookies] - required_cookies = ['z_c0', 'd_c0'] - - for cookie_name in required_cookies: - if cookie_name in cookie_names: - cookie = next(c for c in cookies if c['name'] == cookie_name) - value_preview = cookie['value'][:10] + '...' if len(cookie['value']) > 10 else cookie['value'] - print(f" ✅ {cookie_name}: {value_preview}") - else: - print(f" ⚠️ {cookie_name}: 未找到（可能影响登录状态）") - - print() - - # 4. 检查配置文件 - print(f"4. 配置文件检查") - config_file = os.path.join(os.path.dirname(__file__), 'config.yaml') - if os.path.exists(config_file): - print(f" ✅ config.yaml 存在") - with open(config_file, 'r') as f: - content = f.read() - if 'zhihu_cookies.json' in content: - print(f" ✅ Cookie 路径已配置") - else: - print(f" ⚠️ Cookie 路径未找到（可能是使用了其他配置）") - else: - print(f" ❌ config.yaml 不存在") - - print() - - # 5. 测试提取器初始化 - print(f"5. 提取器初始化测试") - try: - # 加载配置文件并传递给提取器 - import yaml - with open(config_file, 'r', encoding='utf-8') as f: - full_config = yaml.safe_load(f) - - print(f" 配置文件: {config_file}") - print(f" zhihu 配置: {full_config.get('zhihu', {})}") - - # 提取 zhihu 配置并传递 - zhihu_config = full_config.get('zhihu', {}) - extractor = ZhihuDrissionExtractor(config=zhihu_config) - print(f" ✅ ZhihuDrissionExtractor 初始化成功") - - # 测试 Cookie 加载 - loaded_cookies = extractor._load_cookies() - print(f" ✅ 成功加载 {len(loaded_cookies)} 个 Cookie") - - if len(loaded_cookies) == 0: - print(f" ⚠️ 未加载到 Cookie，将使用未登录状态访问") - print(f" 这可能导致提取失败或内容不完整") - - except Exception as e: - print(f" ❌ 初始化失败: {e}") - return False - - print() - print("=" * 60) - print("✅ 配置验证完成！") - print("=" * 60) - print() - - # 6. 下一步建议 - print("下一步：") - print("1. 如果 Cookie 文件格式正确，可以直接开始提取") - print("2. 建议先测试一个简单的知乎回答链接") - print() - print("示例命令：") - print(" python3 -c \"from src.sources.zhihu_drission_extractor import ZhihuDrissionExtractor; print(ZhihuDrissionExtractor().extract('https://www.zhihu.com/question/266309957/answer/2003427891925509507')['title'])\"") - - return True - - -if __name__ == '__main__': - success = verify_cookie_config() - sys.exit(0 if success else 1) diff --git a/skills/obsidian-kb/verify_integration.py b/skills/obsidian-kb/verify_integration.py deleted file mode 100644 index c748e4b..0000000 --- a/skills/obsidian-kb/verify_integration.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Obsidian KB - 主流程集成验证脚本 - -验证代码结构和集成逻辑，不执行实际提取操作 -""" - -import os -import sys -import ast -from pathlib import Path - -# 添加项目路径 -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - - -def verify_file_exists(filepath, description): - """验证文件是否存在""" - path = Path(filepath) - if path.exists(): - print(f"✅ {description}: {filepath}") - return True - else: - print(f"❌ {description} 不存在: {filepath}") - return False - - -def verify_imports(filepath, required_imports): - """验证文件中是否包含指定的导入""" - with open(filepath, 'r', encoding='utf-8') as f: - content = f.read() - tree = ast.parse(content) - - found_imports = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - found_imports.add(alias.name) - elif isinstance(node, ast.ImportFrom): - if node.module: - for alias in node.names: - found_imports.add(f"{node.module}.{alias.name}") - - all_found = True - for imp in required_imports: - if any(imp in found for found in found_imports): - print(f" ✅ 导入: {imp}") - else: - print(f" ❌ 缺少导入: {imp}") - all_found = False - - return all_found - - -def verify_methods(filepath, required_methods): - """验证文件中是否包含指定的方法""" - with open(filepath, 'r', encoding='utf-8') as f: - content = f.read() - tree = ast.parse(content) - - found_methods = set() - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef): - found_methods.add(node.name) - elif isinstance(node, ast.ClassDef): - for item in node.body: - if isinstance(item, ast.FunctionDef): - found_methods.add(f"{node.name}.{item.name}") - - all_found = True - for method in required_methods: - if method in found_methods or any(method in found for found in found_methods): - print(f" ✅ 方法: {method}") - else: - print(f" ❌ 缺少方法: {method}") - all_found = False - - return all_found - - -def verify_integration(): - """验证主流程集成""" - print("\n" + "=" * 80) - print("Obsidian KB - 主流程集成验证") - print("=" * 80 + "\n") - - results = {} - - # 验证文件存在 - print("1. 验证文件存在性") - print("-" * 80) - files = [ - ('main_pipeline.py', '主流程集成文件'), - ('test_integration.py', '集成测试脚本'), - ('handler.py', 'Handler入口'), - ('src/extractors/bilibili_extractor.py', '提取层'), - ('src/deep_analyzer.py', '处理层-分析器'), - ('src/content_chunker.py', '处理层-分片器'), - ('src/outputs/obsidian_writer.py', '输出层-写入器'), - ('src/outputs/git_sync.py', '输出层-同步器'), - ] - - all_files_exist = True - for filepath, description in files: - if not verify_file_exists(filepath, description): - all_files_exist = False - results['files_exist'] = all_files_exist - - # 验证主流程导入 - print("\n2. 验证主流程导入 (main_pipeline.py)") - print("-" * 80) - required_imports = [ - 'extractors.bilibili_extractor', - 'deep_analyzer', - 'content_chunker', - 'outputs.obsidian_writer', - 'outputs.git_sync', - ] - results['main_pipeline_imports'] = verify_imports('main_pipeline.py', required_imports) - - # 验证主流程方法 - print("\n3. 验证主流程方法 (main_pipeline.py)") - print("-" * 80) - required_methods = [ - 'MainPipeline.__init__', - 'MainPipeline.process_video', - 'MainPipeline._extract_video', - 'MainPipeline._analyze_content', - 'MainPipeline._chunk_content', - 'MainPipeline._write_obsidian_note', - 'MainPipeline._sync_to_git', - ] - results['main_pipeline_methods'] = verify_methods('main_pipeline.py', required_methods) - - # 验证测试脚本导入 - print("\n4. 验证测试脚本导入 (test_integration.py)") - print("-" * 80) - required_imports = [ - 'main_pipeline', - 'tempfile', - 'shutil', - ] - results['test_imports'] = verify_imports('test_integration.py', required_imports) - - # 验证测试脚本方法 - print("\n5. 验证测试脚本方法 (test_integration.py)") - print("-" * 80) - required_methods = [ - 'test_extract_only', - 'test_full_pipeline', - 'test_error_handling', - 'cleanup_test_env', - ] - results['test_methods'] = verify_methods('test_integration.py', required_methods) - - # 验证 handler.py 集成 - print("\n6. 验证 Handler 集成 (handler.py)") - print("-" * 80) - required_methods = [ - 'process_video_full', - ] - results['handler_integration'] = verify_methods('handler.py', required_methods) - - # 验证错误处理 - print("\n7. 验证错误处理 (main_pipeline.py)") - print("-" * 80) - with open('main_pipeline.py', 'r', encoding='utf-8') as f: - content = f.read() - error_handling_checks = [ - ('try:', 'try 块'), - ('except', 'except 块'), - ('ExtractionError', 'ExtractionError 捕获'), - ('logger.error', '日志错误记录'), - ('logger.info', '日志信息记录'), - ] - - error_handling_ok = True - for check, description in error_handling_checks: - if check in content: - print(f" ✅ {description}") - else: - print(f" ❌ 缺少 {description}") - error_handling_ok = False - results['error_handling'] = error_handling_ok - - # 验证日志配置 - print("\n8. 验证日志配置 (main_pipeline.py)") - print("-" * 80) - logging_checks = [ - ('import logging', 'logging 模块导入'), - ('logging.basicConfig', 'logging 基本配置'), - ('FileHandler', '文件日志处理器'), - ('obsidian_kb.log', '日志文件'), - ] - - logging_ok = True - for check, description in logging_checks: - if check in content: - print(f" ✅ {description}") - else: - print(f" ❌ 缺少 {description}") - logging_ok = False - results['logging'] = logging_ok - - # 汇总结果 - print("\n" + "=" * 80) - print("验证结果汇总") - print("=" * 80 + "\n") - - checks = [ - ('文件存在性', 'files_exist'), - ('主流程导入', 'main_pipeline_imports'), - ('主流程方法', 'main_pipeline_methods'), - ('测试脚本导入', 'test_imports'), - ('测试脚本方法', 'test_methods'), - ('Handler 集成', 'handler_integration'), - ('错误处理', 'error_handling'), - ('日志配置', 'logging'), - ] - - for description, key in checks: - status = '✅ 通过' if results.get(key, False) else '❌ 失败' - print(f" {description:20s} {status}") - - all_passed = all(results.values()) - print(f"\n 总体结果: {'✅ 全部通过' if all_passed else '❌ 有验证失败'}") - - return 0 if all_passed else 1 - - -if __name__ == '__main__': - sys.exit(verify_integration())