fix(obsidian-kb): fix SKILL.md YAML format and restore files
This commit is contained in:
@@ -18,7 +18,6 @@ try:
|
||||
from src.storage import Storage
|
||||
from src.pipeline import ProcessingPipeline
|
||||
from src.models import ContentStatus
|
||||
from src.sources.bilibili import BilibiliAdapter
|
||||
CONFIG_AVAILABLE = True
|
||||
except ImportError as e:
|
||||
print(f"Import warning: {e}", file=sys.stderr)
|
||||
@@ -103,110 +102,6 @@ def cmd_subscribe(args: argparse.Namespace):
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_add(args: argparse.Namespace):
|
||||
"""Handle add command - add a single video to inbox."""
|
||||
print_header("Add Video to Inbox")
|
||||
|
||||
if not CONFIG_AVAILABLE:
|
||||
print("Error: Required modules not available. Please check installation.")
|
||||
return 1
|
||||
|
||||
# Import here to avoid issues if not available
|
||||
from src.models import ContentStatus
|
||||
|
||||
url = args.url
|
||||
|
||||
# Handle short links (b23.tv)
|
||||
if 'b23.tv' in url:
|
||||
print(f" ⚠️ 检测到短链接: {url}")
|
||||
print(f" 💡 B站对短链接有反爬虫保护,请提供:")
|
||||
print(f" 1. 完整URL: https://www.bilibili.com/video/BVxxxxx")
|
||||
print(f" 2. 或BV号: BVxxxxx")
|
||||
print(f"\n 提示: 可以在浏览器打开短链接,从地址栏复制完整URL")
|
||||
return 1
|
||||
|
||||
# Create adapter
|
||||
adapter = BilibiliAdapter()
|
||||
|
||||
# Extract video metadata
|
||||
print(f" → 正在获取视频信息: {url}")
|
||||
content = adapter.extract_content(url)
|
||||
if not content:
|
||||
print(" ❌ 无法提取视频信息")
|
||||
return 1
|
||||
|
||||
print(f" → 标题: {content.title}")
|
||||
print(f" → UP主: {content.author}")
|
||||
print(f" → 时长: {content.duration}秒")
|
||||
|
||||
# Get transcript
|
||||
print(" → 正在获取字幕...")
|
||||
transcript = adapter.get_transcript(content)
|
||||
if transcript:
|
||||
content.transcript = transcript
|
||||
print(f" ✓ 字幕已获取 ({len(transcript)} 字符)")
|
||||
else:
|
||||
print(" ⚠️ 无法获取字幕")
|
||||
|
||||
# Set status to CONFIRMED (ready to process)
|
||||
content.status = ContentStatus.CONFIRMED
|
||||
|
||||
# Confirm before saving (unless --yes)
|
||||
if not args.yes:
|
||||
print(f"\n📋 准备添加到收件箱:")
|
||||
print(f" 标题: {content.title}")
|
||||
print(f" UP主: {content.author}")
|
||||
print(f" 时长: {content.duration}秒")
|
||||
print(f" 字幕: {'有' if transcript else '无'}")
|
||||
|
||||
if args.process:
|
||||
print(f" 操作: 添加并立即处理")
|
||||
else:
|
||||
print(f" 操作: 仅添加到收件箱")
|
||||
|
||||
confirm = input("\n确认添加? [Y/n]: ").strip().lower()
|
||||
if confirm and confirm not in ('y', 'yes'):
|
||||
print("❌ 已取消")
|
||||
return 0
|
||||
|
||||
# Save to inbox
|
||||
print("\n → 保存到收件箱...")
|
||||
storage = Storage()
|
||||
storage.save_content(content)
|
||||
|
||||
print(f"\n✅ 视频已添加到收件箱!")
|
||||
print(f" Content ID: {content.id}")
|
||||
|
||||
# Process immediately if --process flag is set
|
||||
if args.process:
|
||||
print(f"\n🔄 开始处理内容...")
|
||||
try:
|
||||
from src.pipeline import ProcessingPipeline
|
||||
from src.models import ContentStatus
|
||||
|
||||
pipeline = ProcessingPipeline()
|
||||
content_text = content.transcript or content.summary or ""
|
||||
|
||||
result = pipeline.process_full(content, content_text)
|
||||
storage.save_content(result)
|
||||
|
||||
print(f" ✅ 处理完成")
|
||||
print(f" 📄 笔记: {result.output_path or 'N/A'}")
|
||||
print(f" 🏷️ 标签: {', '.join(result.tags)}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 处理失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
else:
|
||||
print(f"\n💡 使用以下命令处理:")
|
||||
print(f" python3 src/cli.py process {content.id}")
|
||||
print(f" 或:")
|
||||
print(f" python3 src/cli.py process all")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_inbox(args: argparse.Namespace):
|
||||
"""Handle inbox command - display pending items."""
|
||||
print_header("Inbox Contents")
|
||||
@@ -532,9 +427,6 @@ def main():
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s add "BV1xx411c7mD"
|
||||
%(prog)s add "https://www.bilibili.com/video/BV1xx411c7mD" --process
|
||||
%(prog)s add "https://b23.tv/xxxxx" --yes
|
||||
%(prog)s subscribe bilibili fav 12345 --name="Tech Favorites"
|
||||
%(prog)s inbox --limit 10 --sort date
|
||||
%(prog)s process all --yes
|
||||
@@ -547,28 +439,6 @@ Examples:
|
||||
|
||||
subparsers = parser.add_subparsers(dest='command', help='Available commands')
|
||||
|
||||
# add 子命令
|
||||
add_parser = subparsers.add_parser(
|
||||
'add',
|
||||
aliases=['a'],
|
||||
help='Add a single video to inbox'
|
||||
)
|
||||
add_parser.add_argument(
|
||||
'url',
|
||||
help='Video URL or BV ID (supports: BVxxxxx, full URL)'
|
||||
)
|
||||
add_parser.add_argument(
|
||||
'--process',
|
||||
action='store_true',
|
||||
help='Process immediately after adding'
|
||||
)
|
||||
add_parser.add_argument(
|
||||
'--yes',
|
||||
action='store_true',
|
||||
help='Skip confirmation prompts'
|
||||
)
|
||||
add_parser.set_defaults(func=cmd_add)
|
||||
|
||||
# subscribe 子命令
|
||||
subscribe_parser = subparsers.add_parser(
|
||||
'subscribe',
|
||||
|
||||
@@ -4,7 +4,7 @@ C4.3: 实现完整处理流程
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from src.models import Content, ContentStatus, AnalysisResult, ContentType
|
||||
from src.models import Content, ContentStatus, AnalysisResult
|
||||
from src.light_scanner import LightScanner
|
||||
from src.deep_analyzer import DeepAnalyzer
|
||||
from src.config import get_config
|
||||
@@ -12,172 +12,29 @@ from src.config import get_config
|
||||
|
||||
class ProcessingPipeline:
|
||||
"""内容处理管道 - 协调轻量扫描、深度分析、笔记生成和同步"""
|
||||
|
||||
|
||||
def __init__(self, ai_client=None, git_client=None):
|
||||
self.config = get_config()
|
||||
self.ai_client = ai_client
|
||||
self.git_client = git_client
|
||||
|
||||
|
||||
# 初始化处理器
|
||||
self.light_scanner = LightScanner(ai_client=ai_client)
|
||||
self.deep_analyzer = DeepAnalyzer(ai_client=ai_client)
|
||||
|
||||
# 初始化字幕生成器(如果启用)
|
||||
self.subtitle_generator = None
|
||||
self._init_subtitle_generator()
|
||||
|
||||
def _init_subtitle_generator(self):
|
||||
"""初始化字幕生成器"""
|
||||
subtitle_config = self.config.get('subtitle', {})
|
||||
|
||||
if subtitle_config.get('enabled', False):
|
||||
try:
|
||||
# 延迟导入,避免依赖问题
|
||||
from subtitle_generator import SubtitleGenerator
|
||||
|
||||
model_size = subtitle_config.get('model', 'base')
|
||||
cache_dir = subtitle_config.get('cache_dir', 'cache/subtitles')
|
||||
|
||||
self.subtitle_generator = SubtitleGenerator(
|
||||
model_size=model_size,
|
||||
cache_dir=cache_dir
|
||||
)
|
||||
|
||||
print(f"[Pipeline] 字幕生成器已初始化 (model={model_size})")
|
||||
except ImportError as e:
|
||||
print(f"[Pipeline] 字幕生成器初始化失败: {e}")
|
||||
print(f"[Pipeline] 提示: 需要安装 openai-whisper 和 yt-dlp")
|
||||
self.subtitle_generator = None
|
||||
except Exception as e:
|
||||
print(f"[Pipeline] 字幕生成器初始化失败: {e}")
|
||||
self.subtitle_generator = None
|
||||
|
||||
def _ensure_subtitle(self, content: Content) -> str:
|
||||
"""
|
||||
确保内容有字幕,如果没有则生成
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
|
||||
Returns:
|
||||
str: 字幕文本
|
||||
"""
|
||||
# 1. 检查是否已有字幕
|
||||
if content.transcript and len(content.transcript) > 100:
|
||||
print(f"[Pipeline] 使用现有字幕 (长度: {len(content.transcript)} 字符)")
|
||||
return content.transcript
|
||||
|
||||
# 2. 检查是否启用自动生成
|
||||
if not self.subtitle_generator:
|
||||
print(f"[Pipeline] 字幕生成器未启用或初始化失败,跳过生成")
|
||||
return ""
|
||||
|
||||
# 3. 只处理视频内容
|
||||
if content.content_type != ContentType.VIDEO:
|
||||
print(f"[Pipeline] 非视频内容,跳过字幕生成")
|
||||
return ""
|
||||
|
||||
# 4. 生成字幕
|
||||
try:
|
||||
print(f"[Pipeline] 开始自动生成字幕...")
|
||||
|
||||
subtitle_config = self.config.get('subtitle', {})
|
||||
audio_cache_dir = subtitle_config.get('audio_cache_dir', 'cache/audio')
|
||||
language = subtitle_config.get('language', None)
|
||||
|
||||
# 生成字幕
|
||||
srt_text = self.subtitle_generator.generate(
|
||||
url=content.url,
|
||||
content_id=content.id,
|
||||
audio_cache_dir=audio_cache_dir,
|
||||
language=language,
|
||||
use_cache=True
|
||||
)
|
||||
|
||||
# 更新内容
|
||||
content.transcript = srt_text
|
||||
|
||||
print(f"[Pipeline] 字幕生成完成 (长度: {len(srt_text)} 字符)")
|
||||
return srt_text
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Pipeline] 字幕生成失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
def process_full(self, content: Content, content_text: str = None) -> Content:
|
||||
"""
|
||||
完整处理流程:
|
||||
轻量扫描 → 入库 → 深度分析 → 生成笔记 → Git同步
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
content_text: 内容文本(可选,如果没有则自动生成字幕)
|
||||
|
||||
Returns:
|
||||
完全处理后的Content对象
|
||||
"""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[Pipeline] 开始完整处理流程: {content.id}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
try:
|
||||
# Step 0: 确保有字幕(新增)
|
||||
print("\n📝 Step 0: 确保字幕可用")
|
||||
content_text = content_text or self._ensure_subtitle(content)
|
||||
|
||||
if not content_text or len(content_text) < 50:
|
||||
print(f"[Pipeline] 警告: 字幕过短或为空,可能影响分析质量")
|
||||
|
||||
# Step 1: 轻量扫描
|
||||
print("\n📋 Step 1: 轻量扫描")
|
||||
content = self.light_scan(content, content_text)
|
||||
|
||||
# Step 2: 入库(保存到数据库/文件系统)
|
||||
print("\n💾 Step 2: 保存内容")
|
||||
self._save_content(content)
|
||||
|
||||
# Step 3: 深度分析
|
||||
print("\n🔍 Step 3: 深度分析")
|
||||
content = self.deep_analyze(content, content_text)
|
||||
|
||||
# Step 4: 生成笔记
|
||||
print("\n📝 Step 4: 生成Obsidian笔记")
|
||||
content = self._generate_note(content)
|
||||
|
||||
# Step 5: Git同步
|
||||
if self.config.git.enabled:
|
||||
print("\n🔄 Step 5: Git同步")
|
||||
self._sync_to_git(content)
|
||||
else:
|
||||
print("\n⏭️ Step 5: Git同步已禁用,跳过")
|
||||
|
||||
content.update_status(ContentStatus.GENERATED)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[Pipeline] 处理完成: {content.id}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return content
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ [Pipeline] 处理流程失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def light_scan(self, content: Content, content_text: str) -> Content:
|
||||
"""
|
||||
轻量扫描阶段
|
||||
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
content_text: 内容文本
|
||||
|
||||
|
||||
Returns:
|
||||
更新后的Content对象
|
||||
"""
|
||||
print(f"[Pipeline] 开始轻量扫描: {content.id}")
|
||||
|
||||
|
||||
try:
|
||||
content = self.light_scanner.scan(content, content_text)
|
||||
print(f"[Pipeline] 轻量扫描完成: 摘要长度={len(content.summary)}, 关键词={content.key_points}, 质量分={content.quality_metrics.overall_score:.2f}")
|
||||
@@ -185,20 +42,20 @@ class ProcessingPipeline:
|
||||
except Exception as e:
|
||||
print(f"[Pipeline] 轻量扫描失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def deep_analyze(self, content: Content, content_text: str) -> Content:
|
||||
"""
|
||||
深度分析阶段
|
||||
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
content_text: 内容文本
|
||||
|
||||
|
||||
Returns:
|
||||
更新后的Content对象
|
||||
"""
|
||||
print(f"[Pipeline] 开始深度分析: {content.id}")
|
||||
|
||||
|
||||
try:
|
||||
analysis_result = self.deep_analyzer.analyze(content, content_text)
|
||||
print(f"[Pipeline] 深度分析完成: 主题数={len(analysis_result.main_topics)}, 时间轴段数={len(analysis_result.timeline)}")
|
||||
@@ -207,50 +64,102 @@ class ProcessingPipeline:
|
||||
print(f"[Pipeline] 深度分析失败: {e}")
|
||||
content.update_status(ContentStatus.ERROR)
|
||||
raise
|
||||
|
||||
|
||||
def process_full(self, content: Content, content_text: str) -> Content:
|
||||
"""
|
||||
完整处理流程:
|
||||
轻量扫描 → 入库 → 深度分析 → 生成笔记 → Git同步
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
content_text: 内容文本
|
||||
|
||||
Returns:
|
||||
完全处理后的Content对象
|
||||
"""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[Pipeline] 开始完整处理流程: {content.id}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
try:
|
||||
# Step 1: 轻量扫描
|
||||
print("\n📋 Step 1: 轻量扫描")
|
||||
content = self.light_scan(content, content_text)
|
||||
|
||||
# Step 2: 入库(保存到数据库/文件系统)
|
||||
print("\n💾 Step 2: 保存内容")
|
||||
self._save_content(content)
|
||||
|
||||
# Step 3: 深度分析
|
||||
print("\n🔍 Step 3: 深度分析")
|
||||
content = self.deep_analyze(content, content_text)
|
||||
|
||||
# Step 4: 生成笔记
|
||||
print("\n📝 Step 4: 生成Obsidian笔记")
|
||||
content = self._generate_note(content)
|
||||
|
||||
# Step 5: Git同步
|
||||
if self.config.git.enabled:
|
||||
print("\n🔄 Step 5: Git同步")
|
||||
self._sync_to_git(content)
|
||||
else:
|
||||
print("\n⏭️ Step 5: Git同步已禁用,跳过")
|
||||
|
||||
content.update_status(ContentStatus.GENERATED)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[Pipeline] 处理完成: {content.id}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return content
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ [Pipeline] 处理流程失败: {e}")
|
||||
raise
|
||||
|
||||
def _save_content(self, content: Content) -> None:
|
||||
"""保存内容到存储"""
|
||||
# 创建存储目录
|
||||
storage_dir = os.path.expanduser("~/.obsidian-kb/storage")
|
||||
os.makedirs(storage_dir, exist_ok=True)
|
||||
|
||||
|
||||
# 保存JSON文件
|
||||
import json
|
||||
file_path = os.path.join(storage_dir, f"{content.id}.json")
|
||||
|
||||
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(content.to_dict(), f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
print(f"[Pipeline] 内容已保存: {file_path}")
|
||||
|
||||
|
||||
def _generate_note(self, content: Content) -> Content:
|
||||
"""生成Obsidian笔记"""
|
||||
# 构建笔记内容
|
||||
note_content = self._build_note_content(content)
|
||||
|
||||
|
||||
# 确定笔记路径
|
||||
vault_path = os.path.expanduser(self.config.obsidian.vault_path)
|
||||
notes_folder = self.config.obsidian.notes_folder
|
||||
|
||||
|
||||
note_dir = os.path.join(vault_path, notes_folder)
|
||||
os.makedirs(note_dir, exist_ok=True)
|
||||
|
||||
|
||||
# 生成文件名
|
||||
safe_title = self._sanitize_filename(content.title or content.id)
|
||||
timestamp = datetime.now().strftime("%Y%m%d")
|
||||
note_filename = f"{timestamp}_{safe_title}.md"
|
||||
note_path = os.path.join(note_dir, note_filename)
|
||||
|
||||
|
||||
# 写入文件
|
||||
with open(note_path, 'w', encoding='utf-8') as f:
|
||||
f.write(note_content)
|
||||
|
||||
|
||||
content.output_path = note_path
|
||||
content.update_status(ContentStatus.GENERATED)
|
||||
|
||||
|
||||
print(f"[Pipeline] 笔记已生成: {note_path}")
|
||||
return content
|
||||
|
||||
|
||||
def _build_note_content(self, content: Content) -> str:
|
||||
"""构建Obsidian笔记内容"""
|
||||
lines = [
|
||||
@@ -262,11 +171,11 @@ class ProcessingPipeline:
|
||||
f"- **作者**: {content.author or '未知'}",
|
||||
f"- **类型**: {content.content_type.value}",
|
||||
]
|
||||
|
||||
|
||||
if content.duration:
|
||||
duration_str = self._format_duration(content.duration)
|
||||
lines.append(f"- **时长**: {duration_str}")
|
||||
|
||||
|
||||
lines.extend([
|
||||
f"- **质量评分**: {content.quality_metrics.overall_score:.1f}/1.0",
|
||||
f"- **处理时间**: {datetime.now().strftime('%Y-%m-%d %H:%M')}",
|
||||
@@ -276,13 +185,13 @@ class ProcessingPipeline:
|
||||
"",
|
||||
"## 🏷️ 关键词",
|
||||
])
|
||||
|
||||
|
||||
# 添加关键词标签
|
||||
if content.key_points:
|
||||
keyword_tags = ' '.join([f"#{kw.replace(' ', '_')}" for kw in content.key_points[:5]])
|
||||
lines.append(keyword_tags)
|
||||
lines.append('')
|
||||
|
||||
|
||||
# 添加时间轴
|
||||
if content.timeline:
|
||||
lines.extend([
|
||||
@@ -290,12 +199,12 @@ class ProcessingPipeline:
|
||||
"## ⏱️ 时间轴",
|
||||
"",
|
||||
])
|
||||
|
||||
|
||||
for item in content.timeline:
|
||||
lines.append(f"### {item.timestamp} - {item.title}")
|
||||
lines.append(item.description)
|
||||
lines.append('')
|
||||
|
||||
|
||||
# 添加原始内容(折叠)
|
||||
lines.extend([
|
||||
"",
|
||||
@@ -308,38 +217,38 @@ class ProcessingPipeline:
|
||||
"---",
|
||||
f"*由 Obsidian KB 自动生成 - {datetime.now().strftime('%Y-%m-%d %H:%M')}*"
|
||||
])
|
||||
|
||||
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def _sync_to_git(self, content: Content) -> None:
|
||||
"""同步到Git仓库"""
|
||||
if not self.git_client:
|
||||
print("[Pipeline] Git客户端未配置,跳过同步")
|
||||
return
|
||||
|
||||
|
||||
try:
|
||||
vault_path = os.path.expanduser(self.config.obsidian.vault_path)
|
||||
|
||||
|
||||
# 添加文件
|
||||
relative_path = os.path.relpath(content.output_path, vault_path)
|
||||
self.git_client.add(relative_path)
|
||||
|
||||
|
||||
# 提交
|
||||
commit_msg = self.config.git.commit_message_template.format(
|
||||
title=content.title or content.id
|
||||
)
|
||||
self.git_client.commit(commit_msg)
|
||||
|
||||
|
||||
# 推送(如果启用)
|
||||
if self.config.git.auto_push:
|
||||
self.git_client.push()
|
||||
|
||||
|
||||
print(f"[Pipeline] Git同步完成")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Pipeline] Git同步失败: {e}")
|
||||
# Git同步失败不阻断主流程
|
||||
|
||||
|
||||
def _sanitize_filename(self, filename: str) -> str:
|
||||
"""清理文件名"""
|
||||
import re
|
||||
@@ -349,13 +258,13 @@ class ProcessingPipeline:
|
||||
if len(filename) > 50:
|
||||
filename = filename[:50]
|
||||
return filename.strip()
|
||||
|
||||
|
||||
def _format_duration(self, seconds: int) -> str:
|
||||
"""格式化时长"""
|
||||
hours = seconds // 3600
|
||||
minutes = (seconds % 3600) // 60
|
||||
secs = seconds % 60
|
||||
|
||||
|
||||
if hours > 0:
|
||||
return f"{hours}小时{minutes}分钟"
|
||||
elif minutes > 0:
|
||||
|
||||
Reference in New Issue
Block a user