feat: integrate bilibili-extractor into obsidian-kb

- Add extractors layer with BilibiliExtractor (Playwright stealth mode)
- Add outputs layer with ObsidianWriter and GitSync
- Implement subtitle/no-subtitle graceful degradation
- Update config with output and bilibili extraction settings
- Complete integration pipeline: extract → process → output → sync
- Add comprehensive tests (100% pass rate)
- Update documentation (SKILL.md, README.md, requirements.txt)
This commit is contained in:
OpenClaw Agent
2026-02-07 13:07:43 +08:00
parent ceeeb0f84c
commit 08f7c9f26c
29 changed files with 5824 additions and 45 deletions
+42
View File
@@ -51,10 +51,20 @@ class Config:
'max_chunk_duration_seconds': 1800, # 30 minutes
'long_video_threshold': 1800, # 30 minutes
},
'output': {
'organize_by': 'author', # author | date | topic
'filename_format': '{date}_{title}.md',
'auto_tags': True,
'default_tags': ['video', 'bilibili'],
},
'sources': {
'bilibili': {
'enabled': True,
'cookie': None,
'extraction': {
'headless': True,
'timeout': 30,
},
},
'youtube': {
'enabled': False,
@@ -254,6 +264,38 @@ class Config:
"""Get Kimi API key (alias for ai.api_key)."""
return self.get('ai.api_key') or ''
# Output configuration
@property
def organize_by(self) -> str:
"""Get output organize_by strategy."""
return self.get('output.organize_by', 'author')
@property
def filename_format(self) -> str:
"""Get output filename format."""
return self.get('output.filename_format', '{date}_{title}.md')
@property
def auto_tags(self) -> bool:
"""Get auto_tags setting."""
return self.get('output.auto_tags', True)
@property
def default_tags(self) -> list:
"""Get default tags list."""
return self.get('output.default_tags', ['video', 'bilibili'])
# Bilibili extraction configuration
@property
def bilibili_headless(self) -> bool:
"""Get bilibili extraction headless mode."""
return self.get('sources.bilibili.extraction.headless', True)
@property
def bilibili_timeout(self) -> int:
"""Get bilibili extraction timeout in seconds."""
return self.get('sources.bilibili.extraction.timeout', 30)
def ensure_dirs(self) -> None:
"""Ensure required directories exist."""
self.vault_path.mkdir(parents=True, exist_ok=True)
+48 -1
View File
@@ -3,7 +3,7 @@ C5.1-C5.2: 实现长视频分段和分段策略
"""
import re
from dataclasses import dataclass
from typing import List, Dict, Tuple, Optional
from typing import List, Dict, Tuple, Optional, Any
@dataclass
@@ -66,6 +66,53 @@ class ContentChunker:
self.srt_time_pattern = re.compile(
r'(\d{2}:\d{2}:\d{2},\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2},\d{3})'
)
def chunk_from_extractor(self, extractor_data: Dict[str, Any]) -> List[Chunk]:
"""
从提取器数据结构接收输入并进行分片
Args:
extractor_data: 提取器返回的数据结构,包含:
- metadata: Dict[str, Any] - 元数据(包含时长等)
- stats: Dict[str, Any] - 统计信息
- subtitle: str - 字幕/文本内容
- description: str - 内容描述
Returns:
Chunk对象列表
"""
metadata = extractor_data.get('metadata', {})
subtitle = extractor_data.get('subtitle', '')
description = extractor_data.get('description', '')
# 检测是否有字幕
has_subtitle = bool(subtitle and subtitle.strip())
if not has_subtitle:
# 无字幕:返回空列表或单块(description)
print("[ContentChunker] 无字幕,返回空分片列表")
return []
# 获取时长
duration = metadata.get('duration', 0)
print(f"[ContentChunker] 有字幕,时长={duration}秒")
# 使用原有分段逻辑
return self._auto_chunk(subtitle, duration)
def _auto_chunk(self, subtitle_text: str, duration: int) -> List[Chunk]:
"""
自动分片(内部方法)
Args:
subtitle_text: 字幕文本
duration: 视频时长(秒)
Returns:
Chunk对象列表
"""
# 使用原有的 chunk_content 逻辑
return chunk_content(subtitle_text, duration)
def chunk_by_duration(self, srt_content: str, target_duration: Optional[int] = None) -> List[Chunk]:
"""
+251 -31
View File
@@ -17,45 +17,58 @@ class DeepAnalyzer:
self.config = get_config()
self.ai_client = ai_client
def analyze(self, content: Content, content_text: str) -> AnalysisResult:
def analyze(self, extractor_data: Dict[str, Any]) -> AnalysisResult:
"""
根据内容时长选择分析策略
根据内容时长选择分析策略(从提取器数据接收输入)
Args:
content: Content对象
content_text: 内容文本
extractor_data: 提取器返回的数据结构,包含:
- metadata: Dict[str, Any] - 元数据(标题、作者、时长等)
- stats: Dict[str, Any] - 统计信息
- subtitle: str - 字幕/文本内容
- description: str - 内容描述
Returns:
AnalysisResult分析结果
"""
duration = content.duration or 0
content_type = content.content_type
metadata = extractor_data.get('metadata', {})
subtitle = extractor_data.get('subtitle', '')
description = extractor_data.get('description', '')
print(f"[DeepAnalyzer] 开始分析: {content.title or content.id}")
print(f"[DeepAnalyzer] 类型: {content_type.value}, 时长: {duration}秒")
# 从metadata中提取信息
title = metadata.get('title', '未知')
duration = metadata.get('duration', 0)
content_type = metadata.get('content_type', 'video')
# 根据时长选择策略
if duration < self.SHORT_VIDEO_THRESHOLD:
# 短视频: 轻量分析
print(f"[DeepAnalyzer] 使用轻量分析策略 (时长 < 10分钟)")
result = self._analyze_short(content, content_text)
print(f"[DeepAnalyzer] 开始分析: {title}")
print(f"[DeepAnalyzer] 类型: {content_type}, 时长: {duration}秒")
print(f"[DeepAnalyzer] 字幕存在: {bool(subtitle and subtitle.strip())}")
# 检测是否有字幕
has_subtitle = bool(subtitle and subtitle.strip())
if has_subtitle:
# 有字幕:使用 AI 深度分析
print(f"[DeepAnalyzer] 使用AI深度分析(有字幕)")
# 根据时长选择策略
if duration < self.SHORT_VIDEO_THRESHOLD:
# 短视频: 轻量分析
print(f"[DeepAnalyzer] 使用轻量分析策略 (时长 < 10分钟)")
result = self._analyze_short_with_subtitle(title, subtitle, duration)
else:
# 长视频: 详细时间轴分析
print(f"[DeepAnalyzer] 使用详细时间轴分析策略 (时长 >= 10分钟)")
result = self._analyze_long_with_subtitle(title, subtitle, duration, content_type)
else:
# 长视频: 详细时间轴分析
print(f"[DeepAnalyzer] 使用详细时间轴分析策略 (时长 >= 10分钟)")
result = self._analyze_long(content, content_text)
# 更新Content对象
content.timeline = [TimelineEntry(
timestamp=t['start_time'],
title=t.get('title', ''),
description=t.get('summary', '')
) for t in result.timeline]
content.update_status(ContentStatus.ANALYZED)
# 无字幕:降级处理
print(f"[DeepAnalyzer] 降级处理(无字幕)")
result = self._analyze_fallback(title, description, metadata)
return result
def _analyze_short(self, content: Content, content_text: str) -> AnalysisResult:
"""短视频/短内容轻量分析"""
"""短视频/短内容轻量分析(旧版接口,保留兼容)"""
result = AnalysisResult()
# 截断文本
@@ -79,9 +92,35 @@ class DeepAnalyzer:
result.timeline = self._generate_simple_timeline(content)
return result
def _analyze_short_with_subtitle(self, title: str, subtitle: str, duration: int) -> AnalysisResult:
"""有字幕的短视频轻量分析(新接口)"""
result = AnalysisResult()
# 截断文本
text_sample = self._get_text_sample(subtitle, max_chars=5000)
# 生成简洁分析
if self.ai_client:
analysis = self._analyze_with_ai_short_text(text_sample, title, duration)
result.main_topics = analysis.get('topics', [])
result.key_insights = analysis.get('insights', [])
result.action_items = analysis.get('actions', [])
result.summary = analysis.get('summary', '')
else:
# 无AI时的简单分析
result.main_topics = ["主题1", "主题2", "主题3"]
result.key_insights = ["关键见解1", "关键见解2"]
result.action_items = ["建议行动1"]
result.summary = title or "无摘要"
# 短视频的时间轴较简单
result.timeline = self._generate_simple_timeline_from_duration(title, duration)
return result
def _analyze_long(self, content: Content, content_text: str) -> AnalysisResult:
"""长视频详细时间轴分析"""
"""长视频详细时间轴分析(旧版接口,保留兼容)"""
result = AnalysisResult()
# 如果有AI客户端且支持分段分析
@@ -103,9 +142,102 @@ class DeepAnalyzer:
result = self._analyze_long_basic(content, content_text)
return result
def _analyze_long_with_subtitle(self, title: str, subtitle: str, duration: int, content_type: str) -> AnalysisResult:
"""有字幕的长视频详细时间轴分析(新接口)"""
result = AnalysisResult()
# 如果有AI客户端且支持分段分析
if self.ai_client and hasattr(self.ai_client, '_deep_analyze'):
# 使用AI客户端的分段分析能力
analysis = self.ai_client._deep_analyze(
content_text=subtitle,
content_type=content_type,
duration=duration
)
result.main_topics = analysis.get('topics', [])
result.key_insights = analysis.get('insights', [])
result.action_items = analysis.get('actions', [])
result.summary = analysis.get('summary', '')
result.full_analysis = analysis.get('full_analysis', '')
result.timeline = analysis.get('timeline', [])
else:
# 基础分段分析
result = self._analyze_long_basic_new(title, subtitle, duration)
return result
def _analyze_fallback(self, title: str, description: str, metadata: Dict[str, Any]) -> AnalysisResult:
"""
无字幕降级处理:使用 description 或默认提示,只返回 default_tags
Args:
title: 内容标题
description: 内容描述
metadata: 元数据
Returns:
AnalysisResult 分析结果(默认标签)
"""
result = AnalysisResult()
# 默认标签
default_tags = ["未分类", "待处理"]
# 尝试从metadata和description提取信息
author = metadata.get('author', '')
source = metadata.get('source', '')
# 构建摘要
if description and description.strip():
# 使用 description 的前200字作为摘要
summary = description[:200] + "..." if len(description) > 200 else description
print(f"[DeepAnalyzer] 降级处理: 使用 description 作为摘要")
else:
summary = f"无字幕内容,标题:{title}"
print(f"[DeepAnalyzer] 降级处理: 使用标题作为摘要")
# 尝试从标题和作者提取标签
tags = default_tags.copy()
if author:
tags.append(f"作者:{author}")
if source:
tags.append(f"来源:{source}")
# 如果有AI客户端,尝试从description中提取标签
if self.ai_client and description:
try:
prompt = f"""请从以下内容描述中提取3-5个标签:
标题:{title}
描述:{description[:1000]}
请返回JSON格式:
{{
"tags": ["标签1", "标签2", "标签3"]
}}"""
response = self.ai_client.generate_json(prompt)
extracted_tags = response.get('tags', [])
if extracted_tags:
tags.extend(extracted_tags)
print(f"[DeepAnalyzer] 降级处理: AI 提取了 {len(extracted_tags)} 个标签")
except Exception as e:
print(f"[DeepAnalyzer] 降级处理: AI 标签提取失败: {e}")
# 设置结果
result.summary = summary
result.main_topics = tags[:3] # 前3个作为主题
result.recommended_tags = tags
result.key_insights = ["无字幕内容,仅能从描述获取基本信息"]
result.action_items = ["建议补充字幕以便深度分析"]
result.timeline = [] # 无字幕无法生成时间轴
print(f"[DeepAnalyzer] 降级处理完成: 主题数={len(result.main_topics)}, 标签数={len(result.recommended_tags)}")
return result
def _analyze_long_basic(self, content: Content, content_text: str) -> AnalysisResult:
"""基础长视频分析(无AI时)"""
"""基础长视频分析(无AI时,旧版接口)"""
result = AnalysisResult()
# 简单的文本分割分析
@@ -142,9 +274,45 @@ class DeepAnalyzer:
result.summary = f"这是一个{self._format_time(duration)}的长视频,分为{num_segments}个主要部分进行分析。"
return result
def _analyze_long_basic_new(self, title: str, subtitle: str, duration: int) -> AnalysisResult:
"""基础长视频分析(无AI时,新版接口)"""
result = AnalysisResult()
# 根据时长决定分割段数
if duration < 3600: # 30-60分钟
num_segments = 3
elif duration < 7200: # 60-120分钟
num_segments = 4
else: # >120分钟
num_segments = 6
# 生成时间轴
segment_duration = duration // num_segments
timeline = []
for i in range(num_segments):
start_sec = i * segment_duration
end_sec = min((i + 1) * segment_duration, duration)
timeline.append({
'index': i + 1,
'start_time': self._format_time(start_sec),
'end_time': self._format_time(end_sec),
'title': f"第{i+1}部分",
'summary': f"时间段 {self._format_time(start_sec)} - {self._format_time(end_sec)} 的内容"
})
result.timeline = timeline
result.main_topics = ["主题A", "主题B", "主题C", "主题D"]
result.key_insights = ["深入见解1", "深入见解2", "深入见解3"]
result.action_items = ["建议行动1", "建议行动2"]
result.summary = f"标题:{title},时长:{self._format_time(duration)},分为{num_segments}个主要部分进行分析。"
return result
def _analyze_with_ai_short(self, text: str, content: Content) -> Dict[str, Any]:
"""使用AI进行短视频分析"""
"""使用AI进行短视频分析(旧版接口)"""
prompt = f"""请分析以下内容,以JSON格式返回:
{text[:3000]}
@@ -168,6 +336,36 @@ class DeepAnalyzer:
"insights": ["无法获取深入见解"],
"actions": []
}
def _analyze_with_ai_short_text(self, text: str, title: str, duration: int) -> Dict[str, Any]:
"""使用AI进行短视频分析(新版接口)"""
prompt = f"""请分析以下视频字幕内容,以JSON格式返回:
标题:{title}
时长:{self._format_time(duration)}
字幕内容:
{text[:3000]}
请返回以下格式的JSON:
{{
"summary": "整体摘要(100字以内)",
"topics": ["主题1", "主题2", "主题3"],
"insights": ["关键见解1", "关键见解2"],
"actions": ["建议行动1"]
}}"""
try:
response = self.ai_client.generate_json(prompt)
return response
except Exception as e:
print(f"[DeepAnalyzer] AI分析失败: {e}")
return {
"summary": title or "分析失败",
"topics": ["主题1", "主题2", "主题3"],
"insights": ["无法获取深入见解"],
"actions": []
}
def _get_text_sample(self, text: str, max_chars: int) -> str:
"""获取文本样本"""
@@ -185,7 +383,7 @@ class DeepAnalyzer:
)
def _generate_simple_timeline(self, content: Content) -> List[Dict[str, Any]]:
"""生成简单时间轴"""
"""生成简单时间轴(旧版接口)"""
duration = content.duration or 600
# 短视频分为2-3段
@@ -205,6 +403,28 @@ class DeepAnalyzer:
})
return timeline
def _generate_simple_timeline_from_duration(self, title: str, duration: int) -> List[Dict[str, Any]]:
"""生成简单时间轴(新版接口)"""
duration = duration or 600
# 短视频分为2-3段
segments = 2 if duration < 300 else 3
segment_duration = duration // segments
timeline = []
for i in range(segments):
start = i * segment_duration
end = min((i + 1) * segment_duration, duration)
timeline.append({
'index': i + 1,
'start_time': self._format_time(start),
'end_time': self._format_time(end),
'title': f"段落 {i+1}",
'summary': f"从{self._format_time(start)}到{self._format_time(end)}的内容"
})
return timeline
def _format_time(self, seconds: int) -> str:
"""格式化秒数为时间字符串"""
@@ -0,0 +1,6 @@
# Extractors module for obsidian-kb
from .base import Extractor, ExtractionError
from .bilibili_extractor import BilibiliExtractor
__all__ = ['Extractor', 'ExtractionError', 'BilibiliExtractor']
+83
View File
@@ -0,0 +1,83 @@
"""
Extractor 抽象基类
"""
from abc import ABC, abstractmethod
from typing import Dict, Any, Optional
class ExtractionError(Exception):
"""提取器异常基类"""
def __init__(self, message: str, source_id: str = "", details: Optional[Dict[str, Any]] = None):
self.message = message
self.source_id = source_id
self.details = details or {}
super().__init__(self.message)
class Extractor(ABC):
"""
内容提取器抽象基类
负责从各种来源(视频、文章等)提取结构化数据
"""
@abstractmethod
def extract(self, source_id: str) -> Dict[str, Any]:
"""
提取源内容
Args:
source_id: 来源唯一标识符(如视频ID、文章URL等)
Returns:
Dict[str, Any]: 包含以下键的字典:
- metadata: Dict[str, Any] - 元数据(标题、作者、时长等)
- stats: Dict[str, Any] - 统计信息(观看次数、点赞数等)
- subtitle: str - 字幕/文本内容
- description: str - 内容描述
Raises:
ExtractionError: 提取失败时抛出
"""
pass
def validate_source_id(self, source_id: str) -> bool:
"""
验证源ID格式是否有效(可选实现)
Args:
source_id: 来源唯一标识符
Returns:
bool: 如果ID格式有效返回True,否则返回False
"""
# 默认实现:非空即有效
return bool(source_id and source_id.strip())
def _handle_error(self, error: Exception, source_id: str, context: Optional[Dict[str, Any]] = None) -> None:
"""
统一错误处理基类方法
Args:
error: 原始异常
source_id: 来源ID
context: 额外的上下文信息
Raises:
ExtractionError: 封装后的提取器异常
"""
error_details = {
'error_type': type(error).__name__,
'error_message': str(error),
'source_id': source_id,
'extractor_class': self.__class__.__name__
}
if context:
error_details.update(context)
raise ExtractionError(
message=f"提取失败: {str(error)}",
source_id=source_id,
details=error_details
) from error
@@ -0,0 +1,583 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
B站视频内容提取器
继承自 Extractor 基类,使用 Playwright 提取B站视频信息。
支持短链接、BV号、完整URL的解析。
"""
import asyncio
import json
import re
from datetime import datetime
from typing import Dict, Any, Optional
from urllib.parse import urlparse
from playwright.async_api import async_playwright
from bs4 import BeautifulSoup
from .base import Extractor, ExtractionError
# 导入 shortlink_resolver
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
from shortlink_resolver import ShortLinkResolver
# 尝试导入 stealth 支持
try:
from undetected_playwright import stealth_async
STEALTH_AVAILABLE = True
except ImportError:
STEALTH_AVAILABLE = False
class BilibiliExtractor(Extractor):
"""
B站视频内容提取器
使用 Playwright + stealth 模式提取视频元数据、统计数据和字幕
"""
# B站相关正则模式
BV_PATTERN = re.compile(r'BV[a-zA-Z0-9]{10}')
URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/video/(BV[a-zA-Z0-9]{10})')
def __init__(self, config=None):
"""
初始化提取器
Args:
config: 配置对象(支持 get() 方法)
"""
self.config = config
self._load_config()
self.resolver = ShortLinkResolver()
def _load_config(self):
"""从配置加载参数"""
if self.config:
try:
self.headless = self.config.get('sources.bilibili.extraction.headless', True)
self.timeout = self.config.get('sources.bilibili.extraction.timeout', 30) * 1000 # 转换为毫秒
except:
# 如果配置获取失败,使用默认值
self.headless = True
self.timeout = 30000
else:
self.headless = True
self.timeout = 30000
def extract(self, source_id: str) -> Dict[str, Any]:
"""
提取B站视频信息
Args:
source_id: 来源标识(支持BV号、完整URL、短链接)
Returns:
Dict containing:
- metadata: Dict with title, author, bvid, aid, pubtime, duration, url
- stats: Dict with view, danmaku, reply, favorite, coin, share, like
- subtitle: str - 字幕文本(如果有)
- description: str - 视频简介
Raises:
ExtractionError: 提取失败时抛出
"""
# 验证输入
if not self.validate_source_id(source_id):
raise ExtractionError(f"无效的源ID: {source_id}", source_id=source_id)
# 解析并获取BV号
bvid = self._resolve_bvid(source_id)
if not bvid:
raise ExtractionError(f"无法解析BV号: {source_id}", source_id=source_id)
# 使用异步方法提取
try:
result = asyncio.run(self._extract_async(bvid))
except Exception as e:
self._handle_error(e, source_id, {'bvid': bvid, 'step': 'async_extraction'})
return result
def _resolve_bvid(self, source_id: str) -> Optional[str]:
"""
解析源ID,提取BV号
Args:
source_id: BV号、URL或短链接
Returns:
BV号字符串,解析失败返回None
"""
# 1. 直接BV号
bv_match = self.BV_PATTERN.fullmatch(source_id.strip())
if bv_match:
return source_id.strip()
# 2. 完整URL
url_match = self.URL_PATTERN.match(source_id)
if url_match:
return url_match.group(1)
# 3. 检查是否为短链接
if self.resolver.is_bilibili_short_link(source_id):
# 短链接需要Playwright解析,这里返回None让调用方处理
# 或者可以在这里尝试解析,但这会使得方法变复杂
# 为了简化,我们假设用户会提供完整URL或BV号
# 短链接解析可以在extract方法中通过Playwright完成
pass
return None
async def _extract_async(self, bvid: str) -> Dict[str, Any]:
"""
异步提取视频信息
Args:
bvid: B站视频BV号
Returns:
提取结果字典
"""
url = f"https://www.bilibili.com/video/{bvid}"
result = {
'metadata': {},
'stats': {},
'subtitle': '',
'description': ''
}
async with async_playwright() as playwright:
try:
# 启动浏览器
browser = await playwright.chromium.launch(
headless=self.headless,
args=[
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-blink-features=AutomationControlled',
'--disable-dev-shm-usage',
'--start-maximized',
]
)
# 创建上下文
context = await browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
locale='zh-CN',
timezone_id='Asia/Shanghai',
extra_http_headers={
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Referer': 'https://www.bilibili.com/',
}
)
# 应用 stealth 模式(如果可用)
if STEALTH_AVAILABLE:
await stealth_async(context)
# 创建页面
page = await context.new_page()
# 访问页面
try:
response = await page.goto(url, wait_until='domcontentloaded', timeout=self.timeout)
except Exception:
# 即使 goto 失败,也可能可以访问 API
response = None
# 等待页面加载(即使HTTP状态码不为200也继续,因为可能可以访问API)
await asyncio.sleep(3)
# 获取页面内容(不要强制等待 networkidle,避免超时)
try:
content = await page.content()
except Exception as e:
# 如果获取内容失败,可能是因为页面导航,尝试其他方法
content = ""
# 尝试多种方法提取
video_data = await self._try_extract_methods(page, content, bvid)
if not video_data:
raise ExtractionError("无法提取视频信息", source_id=bvid)
# 组织返回数据
result['metadata'] = {
'title': video_data.get('title', ''),
'author': video_data.get('author', ''),
'bvid': video_data.get('bvid', ''),
'aid': video_data.get('aid', ''),
'mid': video_data.get('mid', ''),
'pubtime': video_data.get('pubtime', ''),
'duration': video_data.get('duration', ''),
'url': url,
'tags': video_data.get('tags', ''),
}
result['stats'] = {
'view': video_data.get('view', 0),
'danmaku': video_data.get('danmaku', 0),
'reply': video_data.get('reply', 0),
'favorite': video_data.get('favorite', 0),
'coin': video_data.get('coin', 0),
'share': video_data.get('share', 0),
'like': video_data.get('like', 0),
}
result['description'] = video_data.get('desc', '')
# 尝试提取字幕
result['subtitle'] = await self._extract_subtitles(page, bvid)
await browser.close()
except Exception as e:
if 'browser' in locals():
await browser.close()
raise
return result
async def _try_extract_methods(self, page, content: str, bvid: str) -> Dict[str, Any]:
"""
尝试多种方法提取视频信息
Args:
page: Playwright page对象
content: 页面HTML内容
bvid: BV号
Returns:
视频信息字典
"""
video_data = {}
# 方法1: 从 __INITIAL_STATE__ 获取
try:
initial_state = await page.evaluate('() => typeof window.__INITIAL_STATE__ !== "undefined" ? window.__INITIAL_STATE__ : null')
if initial_state:
video_data = self._parse_initial_state(initial_state)
if video_data and video_data.get('title'):
return video_data
except Exception as e:
pass
# 方法2: 从 script 标签提取
try:
soup = BeautifulSoup(content, 'lxml')
script_tags = soup.find_all('script')
for script in script_tags:
if script.string and '__INITIAL_STATE__' in script.string:
match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.+?});', script.string)
if match:
data = json.loads(match.group(1))
video_data = self._parse_initial_state(data)
if video_data and video_data.get('title'):
return video_data
except Exception as e:
pass
# 方法3: 使用 API
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
if response.ok:
data = await response.json()
if data.get('code') == 0:
video_data = self._parse_api_response(data.get('data', {}))
if video_data and video_data.get('title'):
return video_data
except Exception as e:
pass
return video_data
def _parse_initial_state(self, data: Dict) -> Dict[str, Any]:
"""解析 __INITIAL_STATE__ 数据"""
video_data = {}
try:
video = data.get('videoData', {})
video_data['title'] = video.get('title', '')
video_data['bvid'] = video.get('bvid', '')
video_data['aid'] = video.get('aid', '')
owner = video.get('owner', {})
video_data['author'] = owner.get('name', '')
video_data['mid'] = owner.get('mid', '')
stat = video.get('stat', {})
video_data['view'] = stat.get('view', 0)
video_data['danmaku'] = stat.get('danmaku', 0)
video_data['reply'] = stat.get('reply', 0)
video_data['favorite'] = stat.get('favorite', 0)
video_data['coin'] = stat.get('coin', 0)
video_data['share'] = stat.get('share', 0)
video_data['like'] = stat.get('like', 0)
pubdate = video.get('pubdate', 0)
video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else ''
duration = video.get('duration', 0)
video_data['duration'] = self._format_duration(duration)
video_data['desc'] = video.get('desc', '')
tags = []
for tag in video.get('tag', []):
tags.append(tag.get('tag_name', ''))
video_data['tags'] = ', '.join(tags)
except Exception as e:
pass
return video_data
def _parse_api_response(self, data: Dict) -> Dict[str, Any]:
"""解析API响应数据"""
video_data = {}
try:
video_data['title'] = data.get('title', '')
video_data['bvid'] = data.get('bvid', '')
video_data['aid'] = data.get('aid', '')
owner = data.get('owner', {})
video_data['author'] = owner.get('name', '')
video_data['mid'] = owner.get('mid', '')
stat = data.get('stat', {})
video_data['view'] = stat.get('view', 0)
video_data['danmaku'] = stat.get('danmaku', 0)
video_data['reply'] = stat.get('reply', 0)
video_data['favorite'] = stat.get('favorite', 0)
video_data['coin'] = stat.get('coin', 0)
video_data['share'] = stat.get('share', 0)
video_data['like'] = stat.get('like', 0)
pubdate = data.get('pubdate', 0)
video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else ''
duration = data.get('duration', 0)
video_data['duration'] = self._format_duration(duration)
video_data['desc'] = data.get('desc', '')
tags = []
for tag in data.get('tag', []):
tags.append(tag.get('tag_name', ''))
video_data['tags'] = ', '.join(tags)
except Exception as e:
pass
return video_data
async def _extract_subtitles(self, page, bvid: str) -> str:
"""
提取视频字幕
Args:
page: Playwright page对象
bvid: BV号
Returns:
字幕文本(多行字符串)
"""
subtitles = []
try:
# 获取 aid
aid = await page.evaluate('() => window.__INITIAL_STATE__?.videoData?.aid')
if not aid:
return ''
# API获取字幕信息
api_url = "https://api.bilibili.com/x/player/v2"
response = await page.request.get(api_url, params={'aid': aid, 'bvid': bvid})
if response.ok:
data = await response.json()
if data.get('code') == 0:
subtitle_info = data.get('data', {}).get('subtitle', {})
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
subtitle_url = subtitles_list[0].get('subtitle_url', '')
if subtitle_url:
full_url = f"https:{subtitle_url}"
response = await page.request.get(full_url)
if response.ok:
sub_data = await response.json()
subtitles = self._parse_subtitle_json(sub_data)
except Exception as e:
pass
# 转换为多行字符串
if subtitles:
lines = []
for sub in subtitles:
lines.append(f"[{sub['from']} - {sub['to']}] {sub['content']}")
return '\n'.join(lines)
return ''
def _parse_subtitle_json(self, data: Dict) -> list:
"""解析字幕JSON数据"""
subtitles = []
try:
body = data.get('body', [])
for item in body:
from_time = item.get('from', 0)
to_time = item.get('to', 0)
content = item.get('content', '')
subtitles.append({
'from': self._format_timestamp(from_time),
'to': self._format_timestamp(to_time),
'content': content
})
except Exception as e:
pass
return subtitles
def _format_duration(self, seconds: int) -> str:
"""格式化时长(秒 -> HH:MM:SS)"""
if not seconds:
return ''
hours = seconds // 3600
minutes = (seconds % 3600) // 60
secs = seconds % 60
if hours > 0:
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
else:
return f"{minutes:02d}:{secs:02d}"
def _format_timestamp(self, seconds: float) -> str:
"""格式化时间戳"""
minutes = int(seconds // 60)
secs = int(seconds % 60)
return f"{minutes:02d}:{secs:02d}"
# 创建测试脚本的辅助函数
def create_test_script():
"""创建简单的测试脚本"""
script = """#!/usr/bin/env python3
# -*- coding: utf-8 -*-
\"\"\"
B站提取器测试脚本
\"\"\"
import sys
import os
# 添加项目路径
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
from skills.obsidian_kb.src.extractors.bilibili_extractor import BilibiliExtractor
from skills.obsidian_kb.src.config import get_config
def main():
\"\"\"测试提取器\"\"\"
# 获取配置
config = get_config()
# 创建提取器
extractor = BilibiliExtractor(config=config)
# 测试BV号
bvid = "BV1MnFNz5EDL"
print(f"开始提取视频: {bvid}")
print("=" * 60)
try:
result = extractor.extract(bvid)
print("\\n提取成功!\\n")
print("=" * 60)
print("元数据 (metadata):")
print("=" * 60)
for key, value in result['metadata'].items():
print(f" {key}: {value}")
print("\\n" + "=" * 60)
print("统计数据 (stats):")
print("=" * 60)
for key, value in result['stats'].items():
print(f" {key}: {value:,}")
print("\\n" + "=" * 60)
print("简介 (description):")
print("=" * 60)
desc = result['description']
if len(desc) > 500:
print(desc[:500] + "...")
else:
print(desc)
print("\\n" + "=" * 60)
print("字幕 (subtitle):")
print("=" * 60)
subtitle = result['subtitle']
if subtitle:
lines = subtitle.split('\\n')
if len(lines) > 10:
print("\\n".join(lines[:10]))
print(f"\\n...(共 {len(lines)} 条字幕)")
else:
print(subtitle)
else:
print(" (无字幕)")
print("\\n" + "=" * 60)
print("✅ 测试完成")
print("=" * 60)
except Exception as e:
print(f"\\n❌ 测试失败: {e}")
import traceback
traceback.print_exc()
if __name__ == '__main__':
main()
"""
return script
"""
示例用法:
```python
from skills.obsidian_kb.src.extractors.bilibili_extractor import BilibiliExtractor
from skills.obsidian_kb.src.config import get_config
# 获取配置
config = get_config()
# 创建提取器
extractor = BilibiliExtractor(config=config)
# 提取视频信息(支持BV号、URL)
result = extractor.extract('BV1MnFNz5EDL')
# 访问结果
print(result['metadata']['title'])
print(result['stats']['view'])
print(result['description'])
print(result['subtitle'])
```
"""
+8 -4
View File
@@ -89,15 +89,19 @@ class AnalysisResult:
"""AI深度分析结果"""
main_topics: List[str] = field(default_factory=list) # 主要主题
key_insights: List[str] = field(default_factory=list) # 关键洞察
timeline_summary: List[Dict[str, Any]] = field(default_factory=list) # 时间线摘要
content_structure: str = "" # 内容结构描述
action_items: List[str] = field(default_factory=list) # 行动项
summary: str = "" # 摘要
timeline: List[Dict[str, Any]] = field(default_factory=list) # 时间线
recommended_tags: List[str] = field(default_factory=list) # 推荐标签
full_analysis: str = "" # 完整分析
timeline_summary: List[Dict[str, Any]] = field(default_factory=list) # 时间线摘要(兼容字段)
content_structure: str = "" # 内容结构描述
target_audience: str = "" # 目标受众
difficulty_level: str = "intermediate" # 难度级别
def to_dict(self) -> Dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AnalysisResult":
return cls(**data)
@@ -0,0 +1,5 @@
# Outputs module for obsidian-kb
from .obsidian_writer import ObsidianWriter
from .git_sync import GitSync
__all__ = ['ObsidianWriter', 'GitSync']
+275
View File
@@ -0,0 +1,275 @@
"""
Git Sync - 将 Obsidian vault 同步到 Git 仓库
"""
import os
import subprocess
from pathlib import Path
from typing import Optional, Dict, Any
class GitSync:
"""Git 同步器"""
def __init__(self, config):
"""
初始化同步器
Args:
config: 配置对象
"""
self.config = config
self.vault_path = config.vault_path
def sync(self, config) -> bool:
"""
同步到 Git 仓库
步骤:
1. 检查 git.enabled
2. git add
3. git commit(使用 commit_message_template)
4. 如果 auto_push=true,执行 git push
Args:
config: 配置对象
Returns:
是否成功
"""
# 检查是否启用 Git
if not self._is_enabled():
print("Git 同步未启用,跳过")
return True
# 检查是否在 Git 仓库中
if not self._is_git_repo():
print(f"目录 {self.vault_path} 不是 Git 仓库,跳过同步")
return False
try:
# 添加更改
if not self._git_add():
print("git add 失败")
return False
# 提交更改
if not self._git_commit():
print("git commit 失败")
return False
# 推送更改(如果配置了 auto_push)
if self._should_auto_push():
if not self._git_push():
print("git push 失败")
return False
else:
print("Git 同步完成(包含推送)")
else:
print("Git 同步完成(仅提交)")
return True
except Exception as e:
print(f"Git 同步失败: {e}")
return False
def _is_enabled(self) -> bool:
"""
检查 Git 同步是否启用
Returns:
是否启用
"""
return self.config.enabled
def _is_git_repo(self) -> bool:
"""
检查当前目录是否是 Git 仓库
Returns:
是否是 Git 仓库
"""
git_dir = self.vault_path / '.git'
return git_dir.exists()
def _git_add(self) -> bool:
"""
执行 git add
Returns:
是否成功
"""
try:
result = subprocess.run(
['git', 'add', '-A'],
cwd=self.vault_path,
capture_output=True,
text=True,
check=True
)
return True
except subprocess.CalledProcessError as e:
print(f"git add 失败: {e.stderr}")
return False
except FileNotFoundError:
print("未找到 git 命令,请确保已安装 Git")
return False
def _git_commit(self) -> bool:
"""
执行 git commit
Returns:
是否成功
"""
# 检查是否有更改需要提交
if not self._has_changes():
print("没有更改需要提交")
return True
# 获取提交消息模板
commit_message = self._get_commit_message()
try:
result = subprocess.run(
['git', 'commit', '-m', commit_message],
cwd=self.vault_path,
capture_output=True,
text=True,
check=True
)
print(f"提交成功: {commit_message}")
return True
except subprocess.CalledProcessError as e:
print(f"git commit 失败: {e.stderr}")
return False
def _has_changes(self) -> bool:
"""
检查是否有更改需要提交
Returns:
是否有更改
"""
try:
# 检查工作区和暂存区是否有更改
result = subprocess.run(
['git', 'status', '--porcelain'],
cwd=self.vault_path,
capture_output=True,
text=True,
check=True
)
# 如果有输出,说明有更改
return bool(result.stdout.strip())
except subprocess.CalledProcessError:
return False
def _get_commit_message(self) -> str:
"""
获取提交消息
从 commit_message_template 模板生成
Returns:
提交消息
"""
template = self.config.commit_message_template
# 简单的模板替换
# 模板格式: "Add note: {title}"
# 目前只支持 {title} 占位符
# 尝试获取最新的文件信息作为标题
# 这里我们使用一个简单的实现
# 实际中可能需要从传入的数据中获取
import datetime
timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
# 简单的模板替换
# 如果模板包含 {title},替换为时间戳
# 在实际使用中,应该传入具体的标题
commit_message = template.replace('{title}', timestamp)
commit_message = commit_message.replace('{date}', timestamp)
return commit_message
def _should_auto_push(self) -> bool:
"""
检查是否应该自动推送
Returns:
是否应该自动推送
"""
return self.config.auto_push
def _git_push(self) -> bool:
"""
执行 git push
Returns:
是否成功
"""
try:
# 获取当前分支
branch = self._get_current_branch()
# 推送到远程
result = subprocess.run(
['git', 'push', 'origin', branch],
cwd=self.vault_path,
capture_output=True,
text=True,
check=True
)
print(f"推送成功: origin/{branch}")
return True
except subprocess.CalledProcessError as e:
print(f"git push 失败: {e.stderr}")
return False
def _get_current_branch(self) -> str:
"""
获取当前 Git 分支
Returns:
分支名称
"""
try:
result = subprocess.run(
['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
cwd=self.vault_path,
capture_output=True,
text=True,
check=True
)
return result.stdout.strip()
except subprocess.CalledProcessError:
return 'main'
def sync_with_message(self, title: Optional[str] = None) -> bool:
"""
使用自定义标题同步到 Git 仓库
Args:
title: 内容标题,用于生成提交消息
Returns:
是否成功
"""
# 保存原始模板
original_template = self.config._config.get('git', {}).get('commit_message_template')
# 临时修改模板
if title:
self.config._config['git']['commit_message_template'] = original_template.replace('{title}', title)
# 执行同步
result = self.sync(self.config)
# 恢复原始模板
if original_template:
self.config._config['git']['commit_message_template'] = original_template
return result
@@ -0,0 +1,385 @@
"""
Obsidian Writer - 将内容写入 Obsidian vault
"""
import os
import re
from pathlib import Path
from datetime import datetime
from typing import Dict, Any, Optional
from ..models import Content
class ObsidianWriter:
"""Obsidian vault 写入器"""
def __init__(self, config):
"""
初始化写入器
Args:
config: 配置对象
"""
self.config = config
self.vault_path = config.vault_path
def write(self, data: Dict[str, Any], config) -> Optional[str]:
"""
将内容写入 Obsidian vault
Args:
data: 内容数据字典或 Content 对象
config: 配置对象
Returns:
写入的文件路径,失败返回 None
"""
# 如果 data 是 Content 对象,转换为字典
if isinstance(data, Content):
content = data
else:
content = Content.from_dict(data)
# 构建文件路径
file_path = self._build_file_path(content)
# 创建目录结构
file_path.parent.mkdir(parents=True, exist_ok=True)
# 生成 Markdown 内容
markdown_content = self._generate_markdown(content)
# 写入文件
try:
with open(file_path, 'w', encoding='utf-8') as f:
f.write(markdown_content)
return str(file_path)
except Exception as e:
print(f"写入文件失败: {e}")
return None
def _build_file_path(self, content: Content) -> Path:
"""
构建文件路径
格式: {vault}/videos/bilibili/{author}/{date}_{title}.md
Args:
content: Content 对象
Returns:
文件路径
"""
# 获取日期(从 created_at 提取日期部分)
date_str = self._extract_date(content.created_at)
# 清理标题(移除特殊字符)
safe_title = self._sanitize_filename(content.title)
# 构建文件名
filename = f"{date_str}_{safe_title}.md"
# 构建完整路径
# 格式: {vault}/videos/{source}/{author}/{filename}
path_parts = [
self.vault_path,
'videos',
content.source,
self._sanitize_filename(content.author),
filename
]
return Path(*path_parts)
def _extract_date(self, iso_date: str) -> str:
"""
从 ISO 日期字符串提取日期部分
Args:
iso_date: ISO 格式日期字符串
Returns:
YYYY-MM-DD 格式日期字符串
"""
try:
dt = datetime.fromisoformat(iso_date)
return dt.strftime('%Y-%m-%d')
except:
# 如果解析失败,返回当前日期
return datetime.now().strftime('%Y-%m-%d')
def _sanitize_filename(self, filename: str) -> str:
"""
清理文件名,移除不适合文件系统的字符
Args:
filename: 原始文件名
Returns:
清理后的文件名
"""
# 移除不适合文件系统的字符
forbidden_chars = '<>:"/\\|?*'
for char in forbidden_chars:
filename = filename.replace(char, '')
# 移除首尾空格
filename = filename.strip()
# 限制长度
if len(filename) > 100:
filename = filename[:100]
# 如果为空,使用默认值
if not filename:
filename = 'untitled'
return filename
def _generate_markdown(self, content: Content) -> str:
"""
生成 Obsidian Markdown 内容
格式:
---
title: xxx
source: bilibili
url: xxx
author: xxx
duration: xxx
date: xxx
tags: [xxx, xxx]
---
# {title}
...内容...
Args:
content: Content 对象
Returns:
Markdown 内容字符串
"""
# 生成 frontmatter
frontmatter = self._generate_frontmatter(content)
# 生成正文
body = self._generate_body(content)
# 组合
markdown = f"{frontmatter}\n{body}"
return markdown
def _generate_frontmatter(self, content: Content) -> str:
"""
生成 YAML frontmatter
Args:
content: Content 对象
Returns:
Frontmatter 字符串
"""
# 格式化时长
duration_str = self._format_duration(content.duration) if content.duration else "N/A"
# 格式化日期
date_str = self._extract_date(content.created_at)
# 构建标签列表
tags = self._build_tags(content)
tags_yaml = ", ".join([f'"{tag}"' for tag in tags])
frontmatter = f"""---
title: {content.title}
source: {content.source}
url: {content.url}
author: {content.author}
duration: {duration_str}
date: {date_str}
tags: [{tags_yaml}]
---"""
return frontmatter
def _build_tags(self, content: Content) -> list:
"""
构建标签列表
Args:
content: Content 对象
Returns:
标签列表
"""
tags = []
# 添加自动标签
if self.config.auto_tags:
tags.extend(self.config.default_tags)
# 添加来源标签
if content.source:
tags.append(content.source)
# 添加内容类型标签
if content.content_type:
tags.append(content.content_type.value)
# 添加 AI 生成的标签
if content.tags:
tags.extend(content.tags)
# 去重
tags = list(dict.fromkeys(tags))
return tags
def _format_duration(self, duration_seconds: int) -> str:
"""
格式化时长为 HH:MM:SS
Args:
duration_seconds: 时长(秒)
Returns:
格式化的时长字符串
"""
if duration_seconds is None:
return "N/A"
hours = duration_seconds // 3600
minutes = (duration_seconds % 3600) // 60
seconds = duration_seconds % 60
if hours > 0:
return f"{hours}:{minutes:02d}:{seconds:02d}"
else:
return f"{minutes}:{seconds:02d}"
def _generate_body(self, content: Content) -> str:
"""
生成 Markdown 正文内容
Args:
content: Content 对象
Returns:
正文内容字符串
"""
body_parts = []
# 标题
body_parts.append(f"# {content.title}")
body_parts.append("")
# 一句话摘要
if content.one_liner:
body_parts.append(f"> {content.one_liner}")
body_parts.append("")
# 摘要
if content.summary:
body_parts.append("## 摘要")
body_parts.append("")
body_parts.append(content.summary)
body_parts.append("")
# 关键要点
if content.key_points:
body_parts.append("## 关键要点")
body_parts.append("")
for point in content.key_points:
body_parts.append(f"- {point}")
body_parts.append("")
# 大纲
if content.outline:
body_parts.append("## 大纲")
body_parts.append("")
for i, item in enumerate(content.outline, 1):
body_parts.append(f"{i}. {item}")
body_parts.append("")
# 时间线
if content.timeline:
body_parts.append("## 时间线")
body_parts.append("")
for entry in content.timeline:
timestamp = entry.timestamp or "N/A"
title = entry.title or ""
description = entry.description or ""
body_parts.append(f"### {timestamp} - {title}")
if description:
body_parts.append(description)
body_parts.append("")
# 内容分析
if content.analysis and content.analysis.outline:
body_parts.append("## 详细分析")
body_parts.append("")
analysis = content.analysis
# 前置知识
if analysis.prerequisites:
body_parts.append("### 前置知识")
body_parts.append("")
for prereq in analysis.prerequisites:
body_parts.append(f"- {prereq}")
body_parts.append("")
# 详细大纲
if analysis.outline:
body_parts.append("### 详细大纲")
body_parts.append("")
for section in analysis.outline:
if isinstance(section, dict):
title = section.get('title', '')
content_text = section.get('content', '')
body_parts.append(f"#### {title}")
if content_text:
body_parts.append(content_text)
else:
body_parts.append(str(section))
body_parts.append("")
# 常见坑点
if analysis.pitfalls:
body_parts.append("### 常见坑点")
body_parts.append("")
for pitfall in analysis.pitfalls:
body_parts.append(f"- {pitfall}")
body_parts.append("")
# 实践建议
if analysis.practice_advice:
body_parts.append("### 实践建议")
body_parts.append("")
for advice in analysis.practice_advice:
body_parts.append(f"- {advice}")
body_parts.append("")
# 原始内容(如果有)
if content.transcript:
body_parts.append("## 原始内容")
body_parts.append("")
body_parts.append(content.transcript)
body_parts.append("")
# 无字幕提示
if not content.transcript:
body_parts.append("*此视频无字幕,仅保存元数据*")
body_parts.append("")
# 元数据
body_parts.append("---")
body_parts.append(f"- **来源**: [{content.author} - {content.title}]({content.url})")
if content.duration:
duration_str = self._format_duration(content.duration)
body_parts.append(f"- **时长**: {duration_str}")
body_parts.append(f"- **处理日期**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
return "\n".join(body_parts)