feat: integrate bilibili-extractor into obsidian-kb
- Add extractors layer with BilibiliExtractor (Playwright stealth mode) - Add outputs layer with ObsidianWriter and GitSync - Implement subtitle/no-subtitle graceful degradation - Update config with output and bilibili extraction settings - Complete integration pipeline: extract → process → output → sync - Add comprehensive tests (100% pass rate) - Update documentation (SKILL.md, README.md, requirements.txt)
This commit is contained in:
@@ -51,10 +51,20 @@ class Config:
|
||||
'max_chunk_duration_seconds': 1800, # 30 minutes
|
||||
'long_video_threshold': 1800, # 30 minutes
|
||||
},
|
||||
'output': {
|
||||
'organize_by': 'author', # author | date | topic
|
||||
'filename_format': '{date}_{title}.md',
|
||||
'auto_tags': True,
|
||||
'default_tags': ['video', 'bilibili'],
|
||||
},
|
||||
'sources': {
|
||||
'bilibili': {
|
||||
'enabled': True,
|
||||
'cookie': None,
|
||||
'extraction': {
|
||||
'headless': True,
|
||||
'timeout': 30,
|
||||
},
|
||||
},
|
||||
'youtube': {
|
||||
'enabled': False,
|
||||
@@ -254,6 +264,38 @@ class Config:
|
||||
"""Get Kimi API key (alias for ai.api_key)."""
|
||||
return self.get('ai.api_key') or ''
|
||||
|
||||
# Output configuration
|
||||
@property
|
||||
def organize_by(self) -> str:
|
||||
"""Get output organize_by strategy."""
|
||||
return self.get('output.organize_by', 'author')
|
||||
|
||||
@property
|
||||
def filename_format(self) -> str:
|
||||
"""Get output filename format."""
|
||||
return self.get('output.filename_format', '{date}_{title}.md')
|
||||
|
||||
@property
|
||||
def auto_tags(self) -> bool:
|
||||
"""Get auto_tags setting."""
|
||||
return self.get('output.auto_tags', True)
|
||||
|
||||
@property
|
||||
def default_tags(self) -> list:
|
||||
"""Get default tags list."""
|
||||
return self.get('output.default_tags', ['video', 'bilibili'])
|
||||
|
||||
# Bilibili extraction configuration
|
||||
@property
|
||||
def bilibili_headless(self) -> bool:
|
||||
"""Get bilibili extraction headless mode."""
|
||||
return self.get('sources.bilibili.extraction.headless', True)
|
||||
|
||||
@property
|
||||
def bilibili_timeout(self) -> int:
|
||||
"""Get bilibili extraction timeout in seconds."""
|
||||
return self.get('sources.bilibili.extraction.timeout', 30)
|
||||
|
||||
def ensure_dirs(self) -> None:
|
||||
"""Ensure required directories exist."""
|
||||
self.vault_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -3,7 +3,7 @@ C5.1-C5.2: 实现长视频分段和分段策略
|
||||
"""
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Dict, Tuple, Optional
|
||||
from typing import List, Dict, Tuple, Optional, Any
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -66,6 +66,53 @@ class ContentChunker:
|
||||
self.srt_time_pattern = re.compile(
|
||||
r'(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})'
|
||||
)
|
||||
|
||||
def chunk_from_extractor(self, extractor_data: Dict[str, Any]) -> List[Chunk]:
|
||||
"""
|
||||
从提取器数据结构接收输入并进行分片
|
||||
|
||||
Args:
|
||||
extractor_data: 提取器返回的数据结构,包含:
|
||||
- metadata: Dict[str, Any] - 元数据(包含时长等)
|
||||
- stats: Dict[str, Any] - 统计信息
|
||||
- subtitle: str - 字幕/文本内容
|
||||
- description: str - 内容描述
|
||||
|
||||
Returns:
|
||||
Chunk对象列表
|
||||
"""
|
||||
metadata = extractor_data.get('metadata', {})
|
||||
subtitle = extractor_data.get('subtitle', '')
|
||||
description = extractor_data.get('description', '')
|
||||
|
||||
# 检测是否有字幕
|
||||
has_subtitle = bool(subtitle and subtitle.strip())
|
||||
|
||||
if not has_subtitle:
|
||||
# 无字幕:返回空列表或单块(description)
|
||||
print("[ContentChunker] 无字幕,返回空分片列表")
|
||||
return []
|
||||
|
||||
# 获取时长
|
||||
duration = metadata.get('duration', 0)
|
||||
print(f"[ContentChunker] 有字幕,时长={duration}秒")
|
||||
|
||||
# 使用原有分段逻辑
|
||||
return self._auto_chunk(subtitle, duration)
|
||||
|
||||
def _auto_chunk(self, subtitle_text: str, duration: int) -> List[Chunk]:
|
||||
"""
|
||||
自动分片(内部方法)
|
||||
|
||||
Args:
|
||||
subtitle_text: 字幕文本
|
||||
duration: 视频时长(秒)
|
||||
|
||||
Returns:
|
||||
Chunk对象列表
|
||||
"""
|
||||
# 使用原有的 chunk_content 逻辑
|
||||
return chunk_content(subtitle_text, duration)
|
||||
|
||||
def chunk_by_duration(self, srt_content: str, target_duration: Optional[int] = None) -> List[Chunk]:
|
||||
"""
|
||||
|
||||
@@ -17,45 +17,58 @@ class DeepAnalyzer:
|
||||
self.config = get_config()
|
||||
self.ai_client = ai_client
|
||||
|
||||
def analyze(self, content: Content, content_text: str) -> AnalysisResult:
|
||||
def analyze(self, extractor_data: Dict[str, Any]) -> AnalysisResult:
|
||||
"""
|
||||
根据内容时长选择分析策略
|
||||
|
||||
根据内容时长选择分析策略(从提取器数据接收输入)
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
content_text: 内容文本
|
||||
|
||||
extractor_data: 提取器返回的数据结构,包含:
|
||||
- metadata: Dict[str, Any] - 元数据(标题、作者、时长等)
|
||||
- stats: Dict[str, Any] - 统计信息
|
||||
- subtitle: str - 字幕/文本内容
|
||||
- description: str - 内容描述
|
||||
|
||||
Returns:
|
||||
AnalysisResult分析结果
|
||||
"""
|
||||
duration = content.duration or 0
|
||||
content_type = content.content_type
|
||||
metadata = extractor_data.get('metadata', {})
|
||||
subtitle = extractor_data.get('subtitle', '')
|
||||
description = extractor_data.get('description', '')
|
||||
|
||||
print(f"[DeepAnalyzer] 开始分析: {content.title or content.id}")
|
||||
print(f"[DeepAnalyzer] 类型: {content_type.value}, 时长: {duration}秒")
|
||||
# 从metadata中提取信息
|
||||
title = metadata.get('title', '未知')
|
||||
duration = metadata.get('duration', 0)
|
||||
content_type = metadata.get('content_type', 'video')
|
||||
|
||||
# 根据时长选择策略
|
||||
if duration < self.SHORT_VIDEO_THRESHOLD:
|
||||
# 短视频: 轻量分析
|
||||
print(f"[DeepAnalyzer] 使用轻量分析策略 (时长 < 10分钟)")
|
||||
result = self._analyze_short(content, content_text)
|
||||
print(f"[DeepAnalyzer] 开始分析: {title}")
|
||||
print(f"[DeepAnalyzer] 类型: {content_type}, 时长: {duration}秒")
|
||||
print(f"[DeepAnalyzer] 字幕存在: {bool(subtitle and subtitle.strip())}")
|
||||
|
||||
# 检测是否有字幕
|
||||
has_subtitle = bool(subtitle and subtitle.strip())
|
||||
|
||||
if has_subtitle:
|
||||
# 有字幕:使用 AI 深度分析
|
||||
print(f"[DeepAnalyzer] 使用AI深度分析(有字幕)")
|
||||
|
||||
# 根据时长选择策略
|
||||
if duration < self.SHORT_VIDEO_THRESHOLD:
|
||||
# 短视频: 轻量分析
|
||||
print(f"[DeepAnalyzer] 使用轻量分析策略 (时长 < 10分钟)")
|
||||
result = self._analyze_short_with_subtitle(title, subtitle, duration)
|
||||
else:
|
||||
# 长视频: 详细时间轴分析
|
||||
print(f"[DeepAnalyzer] 使用详细时间轴分析策略 (时长 >= 10分钟)")
|
||||
result = self._analyze_long_with_subtitle(title, subtitle, duration, content_type)
|
||||
else:
|
||||
# 长视频: 详细时间轴分析
|
||||
print(f"[DeepAnalyzer] 使用详细时间轴分析策略 (时长 >= 10分钟)")
|
||||
result = self._analyze_long(content, content_text)
|
||||
|
||||
# 更新Content对象
|
||||
content.timeline = [TimelineEntry(
|
||||
timestamp=t['start_time'],
|
||||
title=t.get('title', ''),
|
||||
description=t.get('summary', '')
|
||||
) for t in result.timeline]
|
||||
content.update_status(ContentStatus.ANALYZED)
|
||||
# 无字幕:降级处理
|
||||
print(f"[DeepAnalyzer] 降级处理(无字幕)")
|
||||
result = self._analyze_fallback(title, description, metadata)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_short(self, content: Content, content_text: str) -> AnalysisResult:
|
||||
"""短视频/短内容轻量分析"""
|
||||
"""短视频/短内容轻量分析(旧版接口,保留兼容)"""
|
||||
result = AnalysisResult()
|
||||
|
||||
# 截断文本
|
||||
@@ -79,9 +92,35 @@ class DeepAnalyzer:
|
||||
result.timeline = self._generate_simple_timeline(content)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_short_with_subtitle(self, title: str, subtitle: str, duration: int) -> AnalysisResult:
|
||||
"""有字幕的短视频轻量分析(新接口)"""
|
||||
result = AnalysisResult()
|
||||
|
||||
# 截断文本
|
||||
text_sample = self._get_text_sample(subtitle, max_chars=5000)
|
||||
|
||||
# 生成简洁分析
|
||||
if self.ai_client:
|
||||
analysis = self._analyze_with_ai_short_text(text_sample, title, duration)
|
||||
result.main_topics = analysis.get('topics', [])
|
||||
result.key_insights = analysis.get('insights', [])
|
||||
result.action_items = analysis.get('actions', [])
|
||||
result.summary = analysis.get('summary', '')
|
||||
else:
|
||||
# 无AI时的简单分析
|
||||
result.main_topics = ["主题1", "主题2", "主题3"]
|
||||
result.key_insights = ["关键见解1", "关键见解2"]
|
||||
result.action_items = ["建议行动1"]
|
||||
result.summary = title or "无摘要"
|
||||
|
||||
# 短视频的时间轴较简单
|
||||
result.timeline = self._generate_simple_timeline_from_duration(title, duration)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_long(self, content: Content, content_text: str) -> AnalysisResult:
|
||||
"""长视频详细时间轴分析"""
|
||||
"""长视频详细时间轴分析(旧版接口,保留兼容)"""
|
||||
result = AnalysisResult()
|
||||
|
||||
# 如果有AI客户端且支持分段分析
|
||||
@@ -103,9 +142,102 @@ class DeepAnalyzer:
|
||||
result = self._analyze_long_basic(content, content_text)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_long_with_subtitle(self, title: str, subtitle: str, duration: int, content_type: str) -> AnalysisResult:
|
||||
"""有字幕的长视频详细时间轴分析(新接口)"""
|
||||
result = AnalysisResult()
|
||||
|
||||
# 如果有AI客户端且支持分段分析
|
||||
if self.ai_client and hasattr(self.ai_client, '_deep_analyze'):
|
||||
# 使用AI客户端的分段分析能力
|
||||
analysis = self.ai_client._deep_analyze(
|
||||
content_text=subtitle,
|
||||
content_type=content_type,
|
||||
duration=duration
|
||||
)
|
||||
result.main_topics = analysis.get('topics', [])
|
||||
result.key_insights = analysis.get('insights', [])
|
||||
result.action_items = analysis.get('actions', [])
|
||||
result.summary = analysis.get('summary', '')
|
||||
result.full_analysis = analysis.get('full_analysis', '')
|
||||
result.timeline = analysis.get('timeline', [])
|
||||
else:
|
||||
# 基础分段分析
|
||||
result = self._analyze_long_basic_new(title, subtitle, duration)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_fallback(self, title: str, description: str, metadata: Dict[str, Any]) -> AnalysisResult:
|
||||
"""
|
||||
无字幕降级处理:使用 description 或默认提示,只返回 default_tags
|
||||
|
||||
Args:
|
||||
title: 内容标题
|
||||
description: 内容描述
|
||||
metadata: 元数据
|
||||
|
||||
Returns:
|
||||
AnalysisResult 分析结果(默认标签)
|
||||
"""
|
||||
result = AnalysisResult()
|
||||
|
||||
# 默认标签
|
||||
default_tags = ["未分类", "待处理"]
|
||||
|
||||
# 尝试从metadata和description提取信息
|
||||
author = metadata.get('author', '')
|
||||
source = metadata.get('source', '')
|
||||
|
||||
# 构建摘要
|
||||
if description and description.strip():
|
||||
# 使用 description 的前200字作为摘要
|
||||
summary = description[:200] + "..." if len(description) > 200 else description
|
||||
print(f"[DeepAnalyzer] 降级处理: 使用 description 作为摘要")
|
||||
else:
|
||||
summary = f"无字幕内容,标题:{title}"
|
||||
print(f"[DeepAnalyzer] 降级处理: 使用标题作为摘要")
|
||||
|
||||
# 尝试从标题和作者提取标签
|
||||
tags = default_tags.copy()
|
||||
if author:
|
||||
tags.append(f"作者:{author}")
|
||||
if source:
|
||||
tags.append(f"来源:{source}")
|
||||
|
||||
# 如果有AI客户端,尝试从description中提取标签
|
||||
if self.ai_client and description:
|
||||
try:
|
||||
prompt = f"""请从以下内容描述中提取3-5个标签:
|
||||
|
||||
标题:{title}
|
||||
描述:{description[:1000]}
|
||||
|
||||
请返回JSON格式:
|
||||
{{
|
||||
"tags": ["标签1", "标签2", "标签3"]
|
||||
}}"""
|
||||
response = self.ai_client.generate_json(prompt)
|
||||
extracted_tags = response.get('tags', [])
|
||||
if extracted_tags:
|
||||
tags.extend(extracted_tags)
|
||||
print(f"[DeepAnalyzer] 降级处理: AI 提取了 {len(extracted_tags)} 个标签")
|
||||
except Exception as e:
|
||||
print(f"[DeepAnalyzer] 降级处理: AI 标签提取失败: {e}")
|
||||
|
||||
# 设置结果
|
||||
result.summary = summary
|
||||
result.main_topics = tags[:3] # 前3个作为主题
|
||||
result.recommended_tags = tags
|
||||
result.key_insights = ["无字幕内容,仅能从描述获取基本信息"]
|
||||
result.action_items = ["建议补充字幕以便深度分析"]
|
||||
result.timeline = [] # 无字幕无法生成时间轴
|
||||
|
||||
print(f"[DeepAnalyzer] 降级处理完成: 主题数={len(result.main_topics)}, 标签数={len(result.recommended_tags)}")
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_long_basic(self, content: Content, content_text: str) -> AnalysisResult:
|
||||
"""基础长视频分析(无AI时)"""
|
||||
"""基础长视频分析(无AI时,旧版接口)"""
|
||||
result = AnalysisResult()
|
||||
|
||||
# 简单的文本分割分析
|
||||
@@ -142,9 +274,45 @@ class DeepAnalyzer:
|
||||
result.summary = f"这是一个{self._format_time(duration)}的长视频,分为{num_segments}个主要部分进行分析。"
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_long_basic_new(self, title: str, subtitle: str, duration: int) -> AnalysisResult:
|
||||
"""基础长视频分析(无AI时,新版接口)"""
|
||||
result = AnalysisResult()
|
||||
|
||||
# 根据时长决定分割段数
|
||||
if duration < 3600: # 30-60分钟
|
||||
num_segments = 3
|
||||
elif duration < 7200: # 60-120分钟
|
||||
num_segments = 4
|
||||
else: # >120分钟
|
||||
num_segments = 6
|
||||
|
||||
# 生成时间轴
|
||||
segment_duration = duration // num_segments
|
||||
timeline = []
|
||||
|
||||
for i in range(num_segments):
|
||||
start_sec = i * segment_duration
|
||||
end_sec = min((i + 1) * segment_duration, duration)
|
||||
|
||||
timeline.append({
|
||||
'index': i + 1,
|
||||
'start_time': self._format_time(start_sec),
|
||||
'end_time': self._format_time(end_sec),
|
||||
'title': f"第{i+1}部分",
|
||||
'summary': f"时间段 {self._format_time(start_sec)} - {self._format_time(end_sec)} 的内容"
|
||||
})
|
||||
|
||||
result.timeline = timeline
|
||||
result.main_topics = ["主题A", "主题B", "主题C", "主题D"]
|
||||
result.key_insights = ["深入见解1", "深入见解2", "深入见解3"]
|
||||
result.action_items = ["建议行动1", "建议行动2"]
|
||||
result.summary = f"标题:{title},时长:{self._format_time(duration)},分为{num_segments}个主要部分进行分析。"
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_with_ai_short(self, text: str, content: Content) -> Dict[str, Any]:
|
||||
"""使用AI进行短视频分析"""
|
||||
"""使用AI进行短视频分析(旧版接口)"""
|
||||
prompt = f"""请分析以下内容,以JSON格式返回:
|
||||
|
||||
{text[:3000]}
|
||||
@@ -168,6 +336,36 @@ class DeepAnalyzer:
|
||||
"insights": ["无法获取深入见解"],
|
||||
"actions": []
|
||||
}
|
||||
|
||||
def _analyze_with_ai_short_text(self, text: str, title: str, duration: int) -> Dict[str, Any]:
|
||||
"""使用AI进行短视频分析(新版接口)"""
|
||||
prompt = f"""请分析以下视频字幕内容,以JSON格式返回:
|
||||
|
||||
标题:{title}
|
||||
时长:{self._format_time(duration)}
|
||||
|
||||
字幕内容:
|
||||
{text[:3000]}
|
||||
|
||||
请返回以下格式的JSON:
|
||||
{{
|
||||
"summary": "整体摘要(100字以内)",
|
||||
"topics": ["主题1", "主题2", "主题3"],
|
||||
"insights": ["关键见解1", "关键见解2"],
|
||||
"actions": ["建议行动1"]
|
||||
}}"""
|
||||
|
||||
try:
|
||||
response = self.ai_client.generate_json(prompt)
|
||||
return response
|
||||
except Exception as e:
|
||||
print(f"[DeepAnalyzer] AI分析失败: {e}")
|
||||
return {
|
||||
"summary": title or "分析失败",
|
||||
"topics": ["主题1", "主题2", "主题3"],
|
||||
"insights": ["无法获取深入见解"],
|
||||
"actions": []
|
||||
}
|
||||
|
||||
def _get_text_sample(self, text: str, max_chars: int) -> str:
|
||||
"""获取文本样本"""
|
||||
@@ -185,7 +383,7 @@ class DeepAnalyzer:
|
||||
)
|
||||
|
||||
def _generate_simple_timeline(self, content: Content) -> List[Dict[str, Any]]:
|
||||
"""生成简单时间轴"""
|
||||
"""生成简单时间轴(旧版接口)"""
|
||||
duration = content.duration or 600
|
||||
|
||||
# 短视频分为2-3段
|
||||
@@ -205,6 +403,28 @@ class DeepAnalyzer:
|
||||
})
|
||||
|
||||
return timeline
|
||||
|
||||
def _generate_simple_timeline_from_duration(self, title: str, duration: int) -> List[Dict[str, Any]]:
|
||||
"""生成简单时间轴(新版接口)"""
|
||||
duration = duration or 600
|
||||
|
||||
# 短视频分为2-3段
|
||||
segments = 2 if duration < 300 else 3
|
||||
segment_duration = duration // segments
|
||||
|
||||
timeline = []
|
||||
for i in range(segments):
|
||||
start = i * segment_duration
|
||||
end = min((i + 1) * segment_duration, duration)
|
||||
timeline.append({
|
||||
'index': i + 1,
|
||||
'start_time': self._format_time(start),
|
||||
'end_time': self._format_time(end),
|
||||
'title': f"段落 {i+1}",
|
||||
'summary': f"从{self._format_time(start)}到{self._format_time(end)}的内容"
|
||||
})
|
||||
|
||||
return timeline
|
||||
|
||||
def _format_time(self, seconds: int) -> str:
|
||||
"""格式化秒数为时间字符串"""
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Extractors module for obsidian-kb
|
||||
|
||||
from .base import Extractor, ExtractionError
|
||||
from .bilibili_extractor import BilibiliExtractor
|
||||
|
||||
__all__ = ['Extractor', 'ExtractionError', 'BilibiliExtractor']
|
||||
@@ -0,0 +1,83 @@
|
||||
"""
|
||||
Extractor 抽象基类
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
|
||||
class ExtractionError(Exception):
|
||||
"""提取器异常基类"""
|
||||
def __init__(self, message: str, source_id: str = "", details: Optional[Dict[str, Any]] = None):
|
||||
self.message = message
|
||||
self.source_id = source_id
|
||||
self.details = details or {}
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class Extractor(ABC):
|
||||
"""
|
||||
内容提取器抽象基类
|
||||
|
||||
负责从各种来源(视频、文章等)提取结构化数据
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def extract(self, source_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
提取源内容
|
||||
|
||||
Args:
|
||||
source_id: 来源唯一标识符(如视频ID、文章URL等)
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 包含以下键的字典:
|
||||
- metadata: Dict[str, Any] - 元数据(标题、作者、时长等)
|
||||
- stats: Dict[str, Any] - 统计信息(观看次数、点赞数等)
|
||||
- subtitle: str - 字幕/文本内容
|
||||
- description: str - 内容描述
|
||||
|
||||
Raises:
|
||||
ExtractionError: 提取失败时抛出
|
||||
"""
|
||||
pass
|
||||
|
||||
def validate_source_id(self, source_id: str) -> bool:
|
||||
"""
|
||||
验证源ID格式是否有效(可选实现)
|
||||
|
||||
Args:
|
||||
source_id: 来源唯一标识符
|
||||
|
||||
Returns:
|
||||
bool: 如果ID格式有效返回True,否则返回False
|
||||
"""
|
||||
# 默认实现:非空即有效
|
||||
return bool(source_id and source_id.strip())
|
||||
|
||||
def _handle_error(self, error: Exception, source_id: str, context: Optional[Dict[str, Any]] = None) -> None:
|
||||
"""
|
||||
统一错误处理基类方法
|
||||
|
||||
Args:
|
||||
error: 原始异常
|
||||
source_id: 来源ID
|
||||
context: 额外的上下文信息
|
||||
|
||||
Raises:
|
||||
ExtractionError: 封装后的提取器异常
|
||||
"""
|
||||
error_details = {
|
||||
'error_type': type(error).__name__,
|
||||
'error_message': str(error),
|
||||
'source_id': source_id,
|
||||
'extractor_class': self.__class__.__name__
|
||||
}
|
||||
|
||||
if context:
|
||||
error_details.update(context)
|
||||
|
||||
raise ExtractionError(
|
||||
message=f"提取失败: {str(error)}",
|
||||
source_id=source_id,
|
||||
details=error_details
|
||||
) from error
|
||||
@@ -0,0 +1,583 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
B站视频内容提取器
|
||||
|
||||
继承自 Extractor 基类,使用 Playwright 提取B站视频信息。
|
||||
支持短链接、BV号、完整URL的解析。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .base import Extractor, ExtractionError
|
||||
|
||||
# 导入 shortlink_resolver
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
||||
from shortlink_resolver import ShortLinkResolver
|
||||
|
||||
# 尝试导入 stealth 支持
|
||||
try:
|
||||
from undetected_playwright import stealth_async
|
||||
STEALTH_AVAILABLE = True
|
||||
except ImportError:
|
||||
STEALTH_AVAILABLE = False
|
||||
|
||||
|
||||
class BilibiliExtractor(Extractor):
|
||||
"""
|
||||
B站视频内容提取器
|
||||
|
||||
使用 Playwright + stealth 模式提取视频元数据、统计数据和字幕
|
||||
"""
|
||||
|
||||
# B站相关正则模式
|
||||
BV_PATTERN = re.compile(r'BV[a-zA-Z0-9]{10}')
|
||||
URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/video/(BV[a-zA-Z0-9]{10})')
|
||||
|
||||
def __init__(self, config=None):
|
||||
"""
|
||||
初始化提取器
|
||||
|
||||
Args:
|
||||
config: 配置对象(支持 get() 方法)
|
||||
"""
|
||||
self.config = config
|
||||
self._load_config()
|
||||
self.resolver = ShortLinkResolver()
|
||||
|
||||
def _load_config(self):
|
||||
"""从配置加载参数"""
|
||||
if self.config:
|
||||
try:
|
||||
self.headless = self.config.get('sources.bilibili.extraction.headless', True)
|
||||
self.timeout = self.config.get('sources.bilibili.extraction.timeout', 30) * 1000 # 转换为毫秒
|
||||
except:
|
||||
# 如果配置获取失败,使用默认值
|
||||
self.headless = True
|
||||
self.timeout = 30000
|
||||
else:
|
||||
self.headless = True
|
||||
self.timeout = 30000
|
||||
|
||||
def extract(self, source_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
提取B站视频信息
|
||||
|
||||
Args:
|
||||
source_id: 来源标识(支持BV号、完整URL、短链接)
|
||||
|
||||
Returns:
|
||||
Dict containing:
|
||||
- metadata: Dict with title, author, bvid, aid, pubtime, duration, url
|
||||
- stats: Dict with view, danmaku, reply, favorite, coin, share, like
|
||||
- subtitle: str - 字幕文本(如果有)
|
||||
- description: str - 视频简介
|
||||
|
||||
Raises:
|
||||
ExtractionError: 提取失败时抛出
|
||||
"""
|
||||
# 验证输入
|
||||
if not self.validate_source_id(source_id):
|
||||
raise ExtractionError(f"无效的源ID: {source_id}", source_id=source_id)
|
||||
|
||||
# 解析并获取BV号
|
||||
bvid = self._resolve_bvid(source_id)
|
||||
if not bvid:
|
||||
raise ExtractionError(f"无法解析BV号: {source_id}", source_id=source_id)
|
||||
|
||||
# 使用异步方法提取
|
||||
try:
|
||||
result = asyncio.run(self._extract_async(bvid))
|
||||
except Exception as e:
|
||||
self._handle_error(e, source_id, {'bvid': bvid, 'step': 'async_extraction'})
|
||||
|
||||
return result
|
||||
|
||||
def _resolve_bvid(self, source_id: str) -> Optional[str]:
|
||||
"""
|
||||
解析源ID,提取BV号
|
||||
|
||||
Args:
|
||||
source_id: BV号、URL或短链接
|
||||
|
||||
Returns:
|
||||
BV号字符串,解析失败返回None
|
||||
"""
|
||||
# 1. 直接BV号
|
||||
bv_match = self.BV_PATTERN.fullmatch(source_id.strip())
|
||||
if bv_match:
|
||||
return source_id.strip()
|
||||
|
||||
# 2. 完整URL
|
||||
url_match = self.URL_PATTERN.match(source_id)
|
||||
if url_match:
|
||||
return url_match.group(1)
|
||||
|
||||
# 3. 检查是否为短链接
|
||||
if self.resolver.is_bilibili_short_link(source_id):
|
||||
# 短链接需要Playwright解析,这里返回None让调用方处理
|
||||
# 或者可以在这里尝试解析,但这会使得方法变复杂
|
||||
# 为了简化,我们假设用户会提供完整URL或BV号
|
||||
# 短链接解析可以在extract方法中通过Playwright完成
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
async def _extract_async(self, bvid: str) -> Dict[str, Any]:
|
||||
"""
|
||||
异步提取视频信息
|
||||
|
||||
Args:
|
||||
bvid: B站视频BV号
|
||||
|
||||
Returns:
|
||||
提取结果字典
|
||||
"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
result = {
|
||||
'metadata': {},
|
||||
'stats': {},
|
||||
'subtitle': '',
|
||||
'description': ''
|
||||
}
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
try:
|
||||
# 启动浏览器
|
||||
browser = await playwright.chromium.launch(
|
||||
headless=self.headless,
|
||||
args=[
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-dev-shm-usage',
|
||||
'--start-maximized',
|
||||
]
|
||||
)
|
||||
|
||||
# 创建上下文
|
||||
context = await browser.new_context(
|
||||
viewport={'width': 1920, 'height': 1080},
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
locale='zh-CN',
|
||||
timezone_id='Asia/Shanghai',
|
||||
extra_http_headers={
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
}
|
||||
)
|
||||
|
||||
# 应用 stealth 模式(如果可用)
|
||||
if STEALTH_AVAILABLE:
|
||||
await stealth_async(context)
|
||||
|
||||
# 创建页面
|
||||
page = await context.new_page()
|
||||
|
||||
# 访问页面
|
||||
try:
|
||||
response = await page.goto(url, wait_until='domcontentloaded', timeout=self.timeout)
|
||||
except Exception:
|
||||
# 即使 goto 失败,也可能可以访问 API
|
||||
response = None
|
||||
|
||||
# 等待页面加载(即使HTTP状态码不为200也继续,因为可能可以访问API)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# 获取页面内容(不要强制等待 networkidle,避免超时)
|
||||
try:
|
||||
content = await page.content()
|
||||
except Exception as e:
|
||||
# 如果获取内容失败,可能是因为页面导航,尝试其他方法
|
||||
content = ""
|
||||
|
||||
# 尝试多种方法提取
|
||||
video_data = await self._try_extract_methods(page, content, bvid)
|
||||
|
||||
if not video_data:
|
||||
raise ExtractionError("无法提取视频信息", source_id=bvid)
|
||||
|
||||
# 组织返回数据
|
||||
result['metadata'] = {
|
||||
'title': video_data.get('title', ''),
|
||||
'author': video_data.get('author', ''),
|
||||
'bvid': video_data.get('bvid', ''),
|
||||
'aid': video_data.get('aid', ''),
|
||||
'mid': video_data.get('mid', ''),
|
||||
'pubtime': video_data.get('pubtime', ''),
|
||||
'duration': video_data.get('duration', ''),
|
||||
'url': url,
|
||||
'tags': video_data.get('tags', ''),
|
||||
}
|
||||
|
||||
result['stats'] = {
|
||||
'view': video_data.get('view', 0),
|
||||
'danmaku': video_data.get('danmaku', 0),
|
||||
'reply': video_data.get('reply', 0),
|
||||
'favorite': video_data.get('favorite', 0),
|
||||
'coin': video_data.get('coin', 0),
|
||||
'share': video_data.get('share', 0),
|
||||
'like': video_data.get('like', 0),
|
||||
}
|
||||
|
||||
result['description'] = video_data.get('desc', '')
|
||||
|
||||
# 尝试提取字幕
|
||||
result['subtitle'] = await self._extract_subtitles(page, bvid)
|
||||
|
||||
await browser.close()
|
||||
|
||||
except Exception as e:
|
||||
if 'browser' in locals():
|
||||
await browser.close()
|
||||
raise
|
||||
|
||||
return result
|
||||
|
||||
async def _try_extract_methods(self, page, content: str, bvid: str) -> Dict[str, Any]:
|
||||
"""
|
||||
尝试多种方法提取视频信息
|
||||
|
||||
Args:
|
||||
page: Playwright page对象
|
||||
content: 页面HTML内容
|
||||
bvid: BV号
|
||||
|
||||
Returns:
|
||||
视频信息字典
|
||||
"""
|
||||
video_data = {}
|
||||
|
||||
# 方法1: 从 __INITIAL_STATE__ 获取
|
||||
try:
|
||||
initial_state = await page.evaluate('() => typeof window.__INITIAL_STATE__ !== "undefined" ? window.__INITIAL_STATE__ : null')
|
||||
if initial_state:
|
||||
video_data = self._parse_initial_state(initial_state)
|
||||
if video_data and video_data.get('title'):
|
||||
return video_data
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# 方法2: 从 script 标签提取
|
||||
try:
|
||||
soup = BeautifulSoup(content, 'lxml')
|
||||
script_tags = soup.find_all('script')
|
||||
for script in script_tags:
|
||||
if script.string and '__INITIAL_STATE__' in script.string:
|
||||
match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.+?});', script.string)
|
||||
if match:
|
||||
data = json.loads(match.group(1))
|
||||
video_data = self._parse_initial_state(data)
|
||||
if video_data and video_data.get('title'):
|
||||
return video_data
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# 方法3: 使用 API
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
|
||||
if response.ok:
|
||||
data = await response.json()
|
||||
if data.get('code') == 0:
|
||||
video_data = self._parse_api_response(data.get('data', {}))
|
||||
if video_data and video_data.get('title'):
|
||||
return video_data
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return video_data
|
||||
|
||||
def _parse_initial_state(self, data: Dict) -> Dict[str, Any]:
|
||||
"""解析 __INITIAL_STATE__ 数据"""
|
||||
video_data = {}
|
||||
|
||||
try:
|
||||
video = data.get('videoData', {})
|
||||
|
||||
video_data['title'] = video.get('title', '')
|
||||
video_data['bvid'] = video.get('bvid', '')
|
||||
video_data['aid'] = video.get('aid', '')
|
||||
|
||||
owner = video.get('owner', {})
|
||||
video_data['author'] = owner.get('name', '')
|
||||
video_data['mid'] = owner.get('mid', '')
|
||||
|
||||
stat = video.get('stat', {})
|
||||
video_data['view'] = stat.get('view', 0)
|
||||
video_data['danmaku'] = stat.get('danmaku', 0)
|
||||
video_data['reply'] = stat.get('reply', 0)
|
||||
video_data['favorite'] = stat.get('favorite', 0)
|
||||
video_data['coin'] = stat.get('coin', 0)
|
||||
video_data['share'] = stat.get('share', 0)
|
||||
video_data['like'] = stat.get('like', 0)
|
||||
|
||||
pubdate = video.get('pubdate', 0)
|
||||
video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else ''
|
||||
|
||||
duration = video.get('duration', 0)
|
||||
video_data['duration'] = self._format_duration(duration)
|
||||
|
||||
video_data['desc'] = video.get('desc', '')
|
||||
|
||||
tags = []
|
||||
for tag in video.get('tag', []):
|
||||
tags.append(tag.get('tag_name', ''))
|
||||
video_data['tags'] = ', '.join(tags)
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return video_data
|
||||
|
||||
def _parse_api_response(self, data: Dict) -> Dict[str, Any]:
|
||||
"""解析API响应数据"""
|
||||
video_data = {}
|
||||
|
||||
try:
|
||||
video_data['title'] = data.get('title', '')
|
||||
video_data['bvid'] = data.get('bvid', '')
|
||||
video_data['aid'] = data.get('aid', '')
|
||||
|
||||
owner = data.get('owner', {})
|
||||
video_data['author'] = owner.get('name', '')
|
||||
video_data['mid'] = owner.get('mid', '')
|
||||
|
||||
stat = data.get('stat', {})
|
||||
video_data['view'] = stat.get('view', 0)
|
||||
video_data['danmaku'] = stat.get('danmaku', 0)
|
||||
video_data['reply'] = stat.get('reply', 0)
|
||||
video_data['favorite'] = stat.get('favorite', 0)
|
||||
video_data['coin'] = stat.get('coin', 0)
|
||||
video_data['share'] = stat.get('share', 0)
|
||||
video_data['like'] = stat.get('like', 0)
|
||||
|
||||
pubdate = data.get('pubdate', 0)
|
||||
video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else ''
|
||||
|
||||
duration = data.get('duration', 0)
|
||||
video_data['duration'] = self._format_duration(duration)
|
||||
|
||||
video_data['desc'] = data.get('desc', '')
|
||||
|
||||
tags = []
|
||||
for tag in data.get('tag', []):
|
||||
tags.append(tag.get('tag_name', ''))
|
||||
video_data['tags'] = ', '.join(tags)
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return video_data
|
||||
|
||||
async def _extract_subtitles(self, page, bvid: str) -> str:
|
||||
"""
|
||||
提取视频字幕
|
||||
|
||||
Args:
|
||||
page: Playwright page对象
|
||||
bvid: BV号
|
||||
|
||||
Returns:
|
||||
字幕文本(多行字符串)
|
||||
"""
|
||||
subtitles = []
|
||||
|
||||
try:
|
||||
# 获取 aid
|
||||
aid = await page.evaluate('() => window.__INITIAL_STATE__?.videoData?.aid')
|
||||
if not aid:
|
||||
return ''
|
||||
|
||||
# API获取字幕信息
|
||||
api_url = "https://api.bilibili.com/x/player/v2"
|
||||
response = await page.request.get(api_url, params={'aid': aid, 'bvid': bvid})
|
||||
|
||||
if response.ok:
|
||||
data = await response.json()
|
||||
if data.get('code') == 0:
|
||||
subtitle_info = data.get('data', {}).get('subtitle', {})
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
|
||||
if subtitles_list:
|
||||
subtitle_url = subtitles_list[0].get('subtitle_url', '')
|
||||
if subtitle_url:
|
||||
full_url = f"https:{subtitle_url}"
|
||||
response = await page.request.get(full_url)
|
||||
if response.ok:
|
||||
sub_data = await response.json()
|
||||
subtitles = self._parse_subtitle_json(sub_data)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# 转换为多行字符串
|
||||
if subtitles:
|
||||
lines = []
|
||||
for sub in subtitles:
|
||||
lines.append(f"[{sub['from']} - {sub['to']}] {sub['content']}")
|
||||
return '\n'.join(lines)
|
||||
|
||||
return ''
|
||||
|
||||
def _parse_subtitle_json(self, data: Dict) -> list:
|
||||
"""解析字幕JSON数据"""
|
||||
subtitles = []
|
||||
|
||||
try:
|
||||
body = data.get('body', [])
|
||||
for item in body:
|
||||
from_time = item.get('from', 0)
|
||||
to_time = item.get('to', 0)
|
||||
content = item.get('content', '')
|
||||
|
||||
subtitles.append({
|
||||
'from': self._format_timestamp(from_time),
|
||||
'to': self._format_timestamp(to_time),
|
||||
'content': content
|
||||
})
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return subtitles
|
||||
|
||||
def _format_duration(self, seconds: int) -> str:
|
||||
"""格式化时长(秒 -> HH:MM:SS)"""
|
||||
if not seconds:
|
||||
return ''
|
||||
|
||||
hours = seconds // 3600
|
||||
minutes = (seconds % 3600) // 60
|
||||
secs = seconds % 60
|
||||
|
||||
if hours > 0:
|
||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
||||
else:
|
||||
return f"{minutes:02d}:{secs:02d}"
|
||||
|
||||
def _format_timestamp(self, seconds: float) -> str:
|
||||
"""格式化时间戳"""
|
||||
minutes = int(seconds // 60)
|
||||
secs = int(seconds % 60)
|
||||
return f"{minutes:02d}:{secs:02d}"
|
||||
|
||||
|
||||
# 创建测试脚本的辅助函数
|
||||
def create_test_script():
|
||||
"""创建简单的测试脚本"""
|
||||
script = """#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
\"\"\"
|
||||
B站提取器测试脚本
|
||||
\"\"\"
|
||||
import sys
|
||||
import os
|
||||
|
||||
# 添加项目路径
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
|
||||
|
||||
from skills.obsidian_kb.src.extractors.bilibili_extractor import BilibiliExtractor
|
||||
from skills.obsidian_kb.src.config import get_config
|
||||
|
||||
|
||||
def main():
|
||||
\"\"\"测试提取器\"\"\"
|
||||
# 获取配置
|
||||
config = get_config()
|
||||
|
||||
# 创建提取器
|
||||
extractor = BilibiliExtractor(config=config)
|
||||
|
||||
# 测试BV号
|
||||
bvid = "BV1MnFNz5EDL"
|
||||
|
||||
print(f"开始提取视频: {bvid}")
|
||||
print("=" * 60)
|
||||
|
||||
try:
|
||||
result = extractor.extract(bvid)
|
||||
|
||||
print("\\n提取成功!\\n")
|
||||
print("=" * 60)
|
||||
print("元数据 (metadata):")
|
||||
print("=" * 60)
|
||||
for key, value in result['metadata'].items():
|
||||
print(f" {key}: {value}")
|
||||
|
||||
print("\\n" + "=" * 60)
|
||||
print("统计数据 (stats):")
|
||||
print("=" * 60)
|
||||
for key, value in result['stats'].items():
|
||||
print(f" {key}: {value:,}")
|
||||
|
||||
print("\\n" + "=" * 60)
|
||||
print("简介 (description):")
|
||||
print("=" * 60)
|
||||
desc = result['description']
|
||||
if len(desc) > 500:
|
||||
print(desc[:500] + "...")
|
||||
else:
|
||||
print(desc)
|
||||
|
||||
print("\\n" + "=" * 60)
|
||||
print("字幕 (subtitle):")
|
||||
print("=" * 60)
|
||||
subtitle = result['subtitle']
|
||||
if subtitle:
|
||||
lines = subtitle.split('\\n')
|
||||
if len(lines) > 10:
|
||||
print("\\n".join(lines[:10]))
|
||||
print(f"\\n...(共 {len(lines)} 条字幕)")
|
||||
else:
|
||||
print(subtitle)
|
||||
else:
|
||||
print(" (无字幕)")
|
||||
|
||||
print("\\n" + "=" * 60)
|
||||
print("✅ 测试完成")
|
||||
print("=" * 60)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\\n❌ 测试失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
"""
|
||||
return script
|
||||
"""
|
||||
|
||||
示例用法:
|
||||
|
||||
```python
|
||||
from skills.obsidian_kb.src.extractors.bilibili_extractor import BilibiliExtractor
|
||||
from skills.obsidian_kb.src.config import get_config
|
||||
|
||||
# 获取配置
|
||||
config = get_config()
|
||||
|
||||
# 创建提取器
|
||||
extractor = BilibiliExtractor(config=config)
|
||||
|
||||
# 提取视频信息(支持BV号、URL)
|
||||
result = extractor.extract('BV1MnFNz5EDL')
|
||||
|
||||
# 访问结果
|
||||
print(result['metadata']['title'])
|
||||
print(result['stats']['view'])
|
||||
print(result['description'])
|
||||
print(result['subtitle'])
|
||||
```
|
||||
"""
|
||||
@@ -89,15 +89,19 @@ class AnalysisResult:
|
||||
"""AI深度分析结果"""
|
||||
main_topics: List[str] = field(default_factory=list) # 主要主题
|
||||
key_insights: List[str] = field(default_factory=list) # 关键洞察
|
||||
timeline_summary: List[Dict[str, Any]] = field(default_factory=list) # 时间线摘要
|
||||
content_structure: str = "" # 内容结构描述
|
||||
action_items: List[str] = field(default_factory=list) # 行动项
|
||||
summary: str = "" # 摘要
|
||||
timeline: List[Dict[str, Any]] = field(default_factory=list) # 时间线
|
||||
recommended_tags: List[str] = field(default_factory=list) # 推荐标签
|
||||
full_analysis: str = "" # 完整分析
|
||||
timeline_summary: List[Dict[str, Any]] = field(default_factory=list) # 时间线摘要(兼容字段)
|
||||
content_structure: str = "" # 内容结构描述
|
||||
target_audience: str = "" # 目标受众
|
||||
difficulty_level: str = "intermediate" # 难度级别
|
||||
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AnalysisResult":
|
||||
return cls(**data)
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
# Outputs module for obsidian-kb
|
||||
from .obsidian_writer import ObsidianWriter
|
||||
from .git_sync import GitSync
|
||||
|
||||
__all__ = ['ObsidianWriter', 'GitSync']
|
||||
@@ -0,0 +1,275 @@
|
||||
"""
|
||||
Git Sync - 将 Obsidian vault 同步到 Git 仓库
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
class GitSync:
|
||||
"""Git 同步器"""
|
||||
|
||||
def __init__(self, config):
|
||||
"""
|
||||
初始化同步器
|
||||
|
||||
Args:
|
||||
config: 配置对象
|
||||
"""
|
||||
self.config = config
|
||||
self.vault_path = config.vault_path
|
||||
|
||||
def sync(self, config) -> bool:
|
||||
"""
|
||||
同步到 Git 仓库
|
||||
|
||||
步骤:
|
||||
1. 检查 git.enabled
|
||||
2. git add
|
||||
3. git commit(使用 commit_message_template)
|
||||
4. 如果 auto_push=true,执行 git push
|
||||
|
||||
Args:
|
||||
config: 配置对象
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
# 检查是否启用 Git
|
||||
if not self._is_enabled():
|
||||
print("Git 同步未启用,跳过")
|
||||
return True
|
||||
|
||||
# 检查是否在 Git 仓库中
|
||||
if not self._is_git_repo():
|
||||
print(f"目录 {self.vault_path} 不是 Git 仓库,跳过同步")
|
||||
return False
|
||||
|
||||
try:
|
||||
# 添加更改
|
||||
if not self._git_add():
|
||||
print("git add 失败")
|
||||
return False
|
||||
|
||||
# 提交更改
|
||||
if not self._git_commit():
|
||||
print("git commit 失败")
|
||||
return False
|
||||
|
||||
# 推送更改(如果配置了 auto_push)
|
||||
if self._should_auto_push():
|
||||
if not self._git_push():
|
||||
print("git push 失败")
|
||||
return False
|
||||
else:
|
||||
print("Git 同步完成(包含推送)")
|
||||
else:
|
||||
print("Git 同步完成(仅提交)")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Git 同步失败: {e}")
|
||||
return False
|
||||
|
||||
def _is_enabled(self) -> bool:
|
||||
"""
|
||||
检查 Git 同步是否启用
|
||||
|
||||
Returns:
|
||||
是否启用
|
||||
"""
|
||||
return self.config.enabled
|
||||
|
||||
def _is_git_repo(self) -> bool:
|
||||
"""
|
||||
检查当前目录是否是 Git 仓库
|
||||
|
||||
Returns:
|
||||
是否是 Git 仓库
|
||||
"""
|
||||
git_dir = self.vault_path / '.git'
|
||||
return git_dir.exists()
|
||||
|
||||
def _git_add(self) -> bool:
|
||||
"""
|
||||
执行 git add
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['git', 'add', '-A'],
|
||||
cwd=self.vault_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"git add 失败: {e.stderr}")
|
||||
return False
|
||||
except FileNotFoundError:
|
||||
print("未找到 git 命令,请确保已安装 Git")
|
||||
return False
|
||||
|
||||
def _git_commit(self) -> bool:
|
||||
"""
|
||||
执行 git commit
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
# 检查是否有更改需要提交
|
||||
if not self._has_changes():
|
||||
print("没有更改需要提交")
|
||||
return True
|
||||
|
||||
# 获取提交消息模板
|
||||
commit_message = self._get_commit_message()
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['git', 'commit', '-m', commit_message],
|
||||
cwd=self.vault_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
print(f"提交成功: {commit_message}")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"git commit 失败: {e.stderr}")
|
||||
return False
|
||||
|
||||
def _has_changes(self) -> bool:
|
||||
"""
|
||||
检查是否有更改需要提交
|
||||
|
||||
Returns:
|
||||
是否有更改
|
||||
"""
|
||||
try:
|
||||
# 检查工作区和暂存区是否有更改
|
||||
result = subprocess.run(
|
||||
['git', 'status', '--porcelain'],
|
||||
cwd=self.vault_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
# 如果有输出,说明有更改
|
||||
return bool(result.stdout.strip())
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
def _get_commit_message(self) -> str:
|
||||
"""
|
||||
获取提交消息
|
||||
|
||||
从 commit_message_template 模板生成
|
||||
|
||||
Returns:
|
||||
提交消息
|
||||
"""
|
||||
template = self.config.commit_message_template
|
||||
|
||||
# 简单的模板替换
|
||||
# 模板格式: "Add note: {title}"
|
||||
# 目前只支持 {title} 占位符
|
||||
|
||||
# 尝试获取最新的文件信息作为标题
|
||||
# 这里我们使用一个简单的实现
|
||||
# 实际中可能需要从传入的数据中获取
|
||||
|
||||
import datetime
|
||||
timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 简单的模板替换
|
||||
# 如果模板包含 {title},替换为时间戳
|
||||
# 在实际使用中,应该传入具体的标题
|
||||
commit_message = template.replace('{title}', timestamp)
|
||||
commit_message = commit_message.replace('{date}', timestamp)
|
||||
|
||||
return commit_message
|
||||
|
||||
def _should_auto_push(self) -> bool:
|
||||
"""
|
||||
检查是否应该自动推送
|
||||
|
||||
Returns:
|
||||
是否应该自动推送
|
||||
"""
|
||||
return self.config.auto_push
|
||||
|
||||
def _git_push(self) -> bool:
|
||||
"""
|
||||
执行 git push
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
try:
|
||||
# 获取当前分支
|
||||
branch = self._get_current_branch()
|
||||
|
||||
# 推送到远程
|
||||
result = subprocess.run(
|
||||
['git', 'push', 'origin', branch],
|
||||
cwd=self.vault_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
print(f"推送成功: origin/{branch}")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"git push 失败: {e.stderr}")
|
||||
return False
|
||||
|
||||
def _get_current_branch(self) -> str:
|
||||
"""
|
||||
获取当前 Git 分支
|
||||
|
||||
Returns:
|
||||
分支名称
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
|
||||
cwd=self.vault_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
except subprocess.CalledProcessError:
|
||||
return 'main'
|
||||
|
||||
def sync_with_message(self, title: Optional[str] = None) -> bool:
|
||||
"""
|
||||
使用自定义标题同步到 Git 仓库
|
||||
|
||||
Args:
|
||||
title: 内容标题,用于生成提交消息
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
"""
|
||||
# 保存原始模板
|
||||
original_template = self.config._config.get('git', {}).get('commit_message_template')
|
||||
|
||||
# 临时修改模板
|
||||
if title:
|
||||
self.config._config['git']['commit_message_template'] = original_template.replace('{title}', title)
|
||||
|
||||
# 执行同步
|
||||
result = self.sync(self.config)
|
||||
|
||||
# 恢复原始模板
|
||||
if original_template:
|
||||
self.config._config['git']['commit_message_template'] = original_template
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,385 @@
|
||||
"""
|
||||
Obsidian Writer - 将内容写入 Obsidian vault
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
from ..models import Content
|
||||
|
||||
|
||||
class ObsidianWriter:
|
||||
"""Obsidian vault 写入器"""
|
||||
|
||||
def __init__(self, config):
|
||||
"""
|
||||
初始化写入器
|
||||
|
||||
Args:
|
||||
config: 配置对象
|
||||
"""
|
||||
self.config = config
|
||||
self.vault_path = config.vault_path
|
||||
|
||||
def write(self, data: Dict[str, Any], config) -> Optional[str]:
|
||||
"""
|
||||
将内容写入 Obsidian vault
|
||||
|
||||
Args:
|
||||
data: 内容数据字典或 Content 对象
|
||||
config: 配置对象
|
||||
|
||||
Returns:
|
||||
写入的文件路径,失败返回 None
|
||||
"""
|
||||
# 如果 data 是 Content 对象,转换为字典
|
||||
if isinstance(data, Content):
|
||||
content = data
|
||||
else:
|
||||
content = Content.from_dict(data)
|
||||
|
||||
# 构建文件路径
|
||||
file_path = self._build_file_path(content)
|
||||
|
||||
# 创建目录结构
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 生成 Markdown 内容
|
||||
markdown_content = self._generate_markdown(content)
|
||||
|
||||
# 写入文件
|
||||
try:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
f.write(markdown_content)
|
||||
|
||||
return str(file_path)
|
||||
except Exception as e:
|
||||
print(f"写入文件失败: {e}")
|
||||
return None
|
||||
|
||||
def _build_file_path(self, content: Content) -> Path:
|
||||
"""
|
||||
构建文件路径
|
||||
|
||||
格式: {vault}/videos/bilibili/{author}/{date}_{title}.md
|
||||
|
||||
Args:
|
||||
content: Content 对象
|
||||
|
||||
Returns:
|
||||
文件路径
|
||||
"""
|
||||
# 获取日期(从 created_at 提取日期部分)
|
||||
date_str = self._extract_date(content.created_at)
|
||||
|
||||
# 清理标题(移除特殊字符)
|
||||
safe_title = self._sanitize_filename(content.title)
|
||||
|
||||
# 构建文件名
|
||||
filename = f"{date_str}_{safe_title}.md"
|
||||
|
||||
# 构建完整路径
|
||||
# 格式: {vault}/videos/{source}/{author}/{filename}
|
||||
path_parts = [
|
||||
self.vault_path,
|
||||
'videos',
|
||||
content.source,
|
||||
self._sanitize_filename(content.author),
|
||||
filename
|
||||
]
|
||||
|
||||
return Path(*path_parts)
|
||||
|
||||
def _extract_date(self, iso_date: str) -> str:
|
||||
"""
|
||||
从 ISO 日期字符串提取日期部分
|
||||
|
||||
Args:
|
||||
iso_date: ISO 格式日期字符串
|
||||
|
||||
Returns:
|
||||
YYYY-MM-DD 格式日期字符串
|
||||
"""
|
||||
try:
|
||||
dt = datetime.fromisoformat(iso_date)
|
||||
return dt.strftime('%Y-%m-%d')
|
||||
except:
|
||||
# 如果解析失败,返回当前日期
|
||||
return datetime.now().strftime('%Y-%m-%d')
|
||||
|
||||
def _sanitize_filename(self, filename: str) -> str:
|
||||
"""
|
||||
清理文件名,移除不适合文件系统的字符
|
||||
|
||||
Args:
|
||||
filename: 原始文件名
|
||||
|
||||
Returns:
|
||||
清理后的文件名
|
||||
"""
|
||||
# 移除不适合文件系统的字符
|
||||
forbidden_chars = '<>:"/\\|?*'
|
||||
for char in forbidden_chars:
|
||||
filename = filename.replace(char, '')
|
||||
|
||||
# 移除首尾空格
|
||||
filename = filename.strip()
|
||||
|
||||
# 限制长度
|
||||
if len(filename) > 100:
|
||||
filename = filename[:100]
|
||||
|
||||
# 如果为空,使用默认值
|
||||
if not filename:
|
||||
filename = 'untitled'
|
||||
|
||||
return filename
|
||||
|
||||
def _generate_markdown(self, content: Content) -> str:
|
||||
"""
|
||||
生成 Obsidian Markdown 内容
|
||||
|
||||
格式:
|
||||
---
|
||||
title: xxx
|
||||
source: bilibili
|
||||
url: xxx
|
||||
author: xxx
|
||||
duration: xxx
|
||||
date: xxx
|
||||
tags: [xxx, xxx]
|
||||
---
|
||||
|
||||
# {title}
|
||||
|
||||
...内容...
|
||||
|
||||
Args:
|
||||
content: Content 对象
|
||||
|
||||
Returns:
|
||||
Markdown 内容字符串
|
||||
"""
|
||||
# 生成 frontmatter
|
||||
frontmatter = self._generate_frontmatter(content)
|
||||
|
||||
# 生成正文
|
||||
body = self._generate_body(content)
|
||||
|
||||
# 组合
|
||||
markdown = f"{frontmatter}\n{body}"
|
||||
|
||||
return markdown
|
||||
|
||||
def _generate_frontmatter(self, content: Content) -> str:
|
||||
"""
|
||||
生成 YAML frontmatter
|
||||
|
||||
Args:
|
||||
content: Content 对象
|
||||
|
||||
Returns:
|
||||
Frontmatter 字符串
|
||||
"""
|
||||
# 格式化时长
|
||||
duration_str = self._format_duration(content.duration) if content.duration else "N/A"
|
||||
|
||||
# 格式化日期
|
||||
date_str = self._extract_date(content.created_at)
|
||||
|
||||
# 构建标签列表
|
||||
tags = self._build_tags(content)
|
||||
tags_yaml = ", ".join([f'"{tag}"' for tag in tags])
|
||||
|
||||
frontmatter = f"""---
|
||||
title: {content.title}
|
||||
source: {content.source}
|
||||
url: {content.url}
|
||||
author: {content.author}
|
||||
duration: {duration_str}
|
||||
date: {date_str}
|
||||
tags: [{tags_yaml}]
|
||||
---"""
|
||||
|
||||
return frontmatter
|
||||
|
||||
def _build_tags(self, content: Content) -> list:
|
||||
"""
|
||||
构建标签列表
|
||||
|
||||
Args:
|
||||
content: Content 对象
|
||||
|
||||
Returns:
|
||||
标签列表
|
||||
"""
|
||||
tags = []
|
||||
|
||||
# 添加自动标签
|
||||
if self.config.auto_tags:
|
||||
tags.extend(self.config.default_tags)
|
||||
|
||||
# 添加来源标签
|
||||
if content.source:
|
||||
tags.append(content.source)
|
||||
|
||||
# 添加内容类型标签
|
||||
if content.content_type:
|
||||
tags.append(content.content_type.value)
|
||||
|
||||
# 添加 AI 生成的标签
|
||||
if content.tags:
|
||||
tags.extend(content.tags)
|
||||
|
||||
# 去重
|
||||
tags = list(dict.fromkeys(tags))
|
||||
|
||||
return tags
|
||||
|
||||
def _format_duration(self, duration_seconds: int) -> str:
|
||||
"""
|
||||
格式化时长为 HH:MM:SS
|
||||
|
||||
Args:
|
||||
duration_seconds: 时长(秒)
|
||||
|
||||
Returns:
|
||||
格式化的时长字符串
|
||||
"""
|
||||
if duration_seconds is None:
|
||||
return "N/A"
|
||||
|
||||
hours = duration_seconds // 3600
|
||||
minutes = (duration_seconds % 3600) // 60
|
||||
seconds = duration_seconds % 60
|
||||
|
||||
if hours > 0:
|
||||
return f"{hours}:{minutes:02d}:{seconds:02d}"
|
||||
else:
|
||||
return f"{minutes}:{seconds:02d}"
|
||||
|
||||
def _generate_body(self, content: Content) -> str:
|
||||
"""
|
||||
生成 Markdown 正文内容
|
||||
|
||||
Args:
|
||||
content: Content 对象
|
||||
|
||||
Returns:
|
||||
正文内容字符串
|
||||
"""
|
||||
body_parts = []
|
||||
|
||||
# 标题
|
||||
body_parts.append(f"# {content.title}")
|
||||
body_parts.append("")
|
||||
|
||||
# 一句话摘要
|
||||
if content.one_liner:
|
||||
body_parts.append(f"> {content.one_liner}")
|
||||
body_parts.append("")
|
||||
|
||||
# 摘要
|
||||
if content.summary:
|
||||
body_parts.append("## 摘要")
|
||||
body_parts.append("")
|
||||
body_parts.append(content.summary)
|
||||
body_parts.append("")
|
||||
|
||||
# 关键要点
|
||||
if content.key_points:
|
||||
body_parts.append("## 关键要点")
|
||||
body_parts.append("")
|
||||
for point in content.key_points:
|
||||
body_parts.append(f"- {point}")
|
||||
body_parts.append("")
|
||||
|
||||
# 大纲
|
||||
if content.outline:
|
||||
body_parts.append("## 大纲")
|
||||
body_parts.append("")
|
||||
for i, item in enumerate(content.outline, 1):
|
||||
body_parts.append(f"{i}. {item}")
|
||||
body_parts.append("")
|
||||
|
||||
# 时间线
|
||||
if content.timeline:
|
||||
body_parts.append("## 时间线")
|
||||
body_parts.append("")
|
||||
for entry in content.timeline:
|
||||
timestamp = entry.timestamp or "N/A"
|
||||
title = entry.title or ""
|
||||
description = entry.description or ""
|
||||
body_parts.append(f"### {timestamp} - {title}")
|
||||
if description:
|
||||
body_parts.append(description)
|
||||
body_parts.append("")
|
||||
|
||||
# 内容分析
|
||||
if content.analysis and content.analysis.outline:
|
||||
body_parts.append("## 详细分析")
|
||||
body_parts.append("")
|
||||
|
||||
analysis = content.analysis
|
||||
|
||||
# 前置知识
|
||||
if analysis.prerequisites:
|
||||
body_parts.append("### 前置知识")
|
||||
body_parts.append("")
|
||||
for prereq in analysis.prerequisites:
|
||||
body_parts.append(f"- {prereq}")
|
||||
body_parts.append("")
|
||||
|
||||
# 详细大纲
|
||||
if analysis.outline:
|
||||
body_parts.append("### 详细大纲")
|
||||
body_parts.append("")
|
||||
for section in analysis.outline:
|
||||
if isinstance(section, dict):
|
||||
title = section.get('title', '')
|
||||
content_text = section.get('content', '')
|
||||
body_parts.append(f"#### {title}")
|
||||
if content_text:
|
||||
body_parts.append(content_text)
|
||||
else:
|
||||
body_parts.append(str(section))
|
||||
body_parts.append("")
|
||||
|
||||
# 常见坑点
|
||||
if analysis.pitfalls:
|
||||
body_parts.append("### 常见坑点")
|
||||
body_parts.append("")
|
||||
for pitfall in analysis.pitfalls:
|
||||
body_parts.append(f"- {pitfall}")
|
||||
body_parts.append("")
|
||||
|
||||
# 实践建议
|
||||
if analysis.practice_advice:
|
||||
body_parts.append("### 实践建议")
|
||||
body_parts.append("")
|
||||
for advice in analysis.practice_advice:
|
||||
body_parts.append(f"- {advice}")
|
||||
body_parts.append("")
|
||||
|
||||
# 原始内容(如果有)
|
||||
if content.transcript:
|
||||
body_parts.append("## 原始内容")
|
||||
body_parts.append("")
|
||||
body_parts.append(content.transcript)
|
||||
body_parts.append("")
|
||||
|
||||
# 无字幕提示
|
||||
if not content.transcript:
|
||||
body_parts.append("*此视频无字幕,仅保存元数据*")
|
||||
body_parts.append("")
|
||||
|
||||
# 元数据
|
||||
body_parts.append("---")
|
||||
body_parts.append(f"- **来源**: [{content.author} - {content.title}]({content.url})")
|
||||
if content.duration:
|
||||
duration_str = self._format_duration(content.duration)
|
||||
body_parts.append(f"- **时长**: {duration_str}")
|
||||
body_parts.append(f"- **处理日期**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
|
||||
return "\n".join(body_parts)
|
||||
Reference in New Issue
Block a user