feat: Add KB knowledge base series skills
- kb-link-parser: Multi-platform URL parser (Bilibili, YouTube, Zhihu) - kb-bilibili-metadata: Bilibili video/article/up主 metadata extractor - kb-zhihu-metadata: Zhihu question/answer/user metadata extractor - kb-bilibili-content: Bilibili video download and content extraction - kb-ocr-processor: OCR text extraction from video frames - kb-deep-analyzer: AI-powered deep content analysis - kb-workflow: Workflow orchestration (inbox, archive, subscriptions, report) Also add: - ARCHITECTURE_FLOWCHART.md: System architecture diagrams - DETAILED_DESIGN.md: Complete design documentation
This commit is contained in:
OpenClaw Agent
@@ -0,0 +1,45 @@
|
||||
# KB Bilibili Content Skill
|
||||
|
||||
**kb-bilibili-content** - Download Bilibili videos and extract content.
|
||||
|
||||
## Features
|
||||
|
||||
- Video download (multiple quality options)
|
||||
- Subtitle extraction
|
||||
- Article content extraction
|
||||
- Low quality for OCR (480p default)
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Download video
|
||||
python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 480
|
||||
|
||||
# Extract subtitle
|
||||
python bilibili_content.py --id "BV1xx411c7mD" --type subtitle
|
||||
|
||||
# Get article content
|
||||
python bilibili_content.py --id "cv43780186" --type article
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"platform": "bilibili",
|
||||
"type": "video",
|
||||
"id": "BV1xx411c7mD",
|
||||
"video_file": {"path": "output/BV1xx.mp4", "quality": "480p"},
|
||||
"subtitle": {"path": "output/BV1xx.srt"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
cp -r kb-bilibili-content/ ~/skills/
|
||||
pip install aiohttp
|
||||
```
|
||||
@@ -0,0 +1,452 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Bilibili Content Downloader - B站内容获取工具
|
||||
|
||||
功能:
|
||||
- download_video(bvid, quality=480) - 下载低分辨率视频
|
||||
- extract_subtitle(bvid) - 提取字幕
|
||||
- extract_article(cvid) - 获取专栏全文
|
||||
- 命令行参数支持
|
||||
- JSON 输出
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
import aiohttp
|
||||
|
||||
|
||||
class BilibiliContent:
|
||||
"""B站内容获取器"""
|
||||
|
||||
def __init__(self, output_dir: str = "./output"):
|
||||
self.output_dir = Path(output_dir)
|
||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _get_headers(self) -> Dict[str, str]:
|
||||
"""获取请求头"""
|
||||
return {
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
}
|
||||
|
||||
async def _make_request(self, session, url: str, params: Dict = None) -> Optional[Dict]:
|
||||
"""发送API请求"""
|
||||
try:
|
||||
import aiohttp
|
||||
async with session.get(url, params=params, headers=self._get_headers(), timeout=aiohttp.ClientTimeout(total=30)) as response:
|
||||
if response.status == 200:
|
||||
return await response.json()
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"请求错误: {e}")
|
||||
return None
|
||||
|
||||
async def download_video(self, bvid: str, quality: int = 480) -> Dict[str, Any]:
|
||||
"""
|
||||
下载B站视频
|
||||
|
||||
Args:
|
||||
bvid: B站视频BV号 (如: BV1xx411c7mD)
|
||||
quality: 视频质量 (默认480)
|
||||
- 127: 8K
|
||||
- 126:杜比
|
||||
- 125:HDR
|
||||
- 120:4K
|
||||
- 116:1080P60
|
||||
- 112:1080P+
|
||||
- 80:1080P
|
||||
- 74:720P60
|
||||
- 64:720P
|
||||
- 32:480P
|
||||
- 16:360P
|
||||
|
||||
Returns:
|
||||
Dict: 包含状态、消息和文件路径的结果
|
||||
"""
|
||||
result = {
|
||||
"success": False,
|
||||
"bvid": bvid,
|
||||
"message": "",
|
||||
"filepath": None,
|
||||
"metadata": {}
|
||||
}
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# 1. 获取视频信息
|
||||
print(f"正在获取视频信息: {bvid}")
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
data = await self._make_request(session, api_url, {'bvid': bvid})
|
||||
|
||||
if not data or data.get('code') != 0:
|
||||
result["message"] = f"获取视频信息失败: {data.get('message', '未知错误')}"
|
||||
return result
|
||||
|
||||
video_data = data.get('data', {})
|
||||
title = video_data.get('title', '未知标题')
|
||||
aid = video_data.get('aid')
|
||||
cid = video_data.get('cid')
|
||||
duration = video_data.get('duration', 0)
|
||||
author = video_data.get('owner', {}).get('name', '未知作者')
|
||||
|
||||
result["metadata"] = {
|
||||
"title": title,
|
||||
"author": author,
|
||||
"aid": aid,
|
||||
"cid": cid,
|
||||
"duration": duration,
|
||||
"bvid": bvid
|
||||
}
|
||||
|
||||
print(f"视频标题: {title}")
|
||||
print(f"作者: {author}")
|
||||
print(f"时长: {duration}秒")
|
||||
|
||||
# 2. 获取视频流地址
|
||||
print(f"\n正在获取视频流...")
|
||||
|
||||
# 选择合适的质量
|
||||
quality_map = {127: 127, 126: 126, 125: 125, 120: 120, 116: 116, 112: 112, 80: 80, 74: 74, 64: 64, 32: 32, 16: 16}
|
||||
target_quality = quality_map.get(quality, 32) # 默认480P
|
||||
|
||||
playurl_api = "https://api.bilibili.com/x/player/playurl"
|
||||
params = {
|
||||
'bvid': bvid,
|
||||
'cid': cid,
|
||||
'qn': target_quality,
|
||||
'otype': 'json',
|
||||
'fnver': 0,
|
||||
'fnval': 16, # 支持DASH格式
|
||||
}
|
||||
|
||||
playurl_data = await self._make_request(session, playurl_api, params)
|
||||
|
||||
if not playurl_data or playurl_data.get('code') != 0:
|
||||
result["message"] = f"获取视频流失败: {playurl_data.get('message', '未知错误')}"
|
||||
return result
|
||||
|
||||
play_data = playurl_data.get('data', {})
|
||||
|
||||
# 获取视频和音频流
|
||||
dash_data = play_data.get('dash')
|
||||
if dash_data:
|
||||
video_list = dash_data.get('video', [])
|
||||
audio_list = dash_data.get('audio', [])
|
||||
|
||||
if video_list:
|
||||
# 选择质量最高的视频流
|
||||
video_stream = video_list[0]
|
||||
video_url = video_stream.get('base_url') or video_stream.get('baseUrl')
|
||||
video_size = video_list[0].get('size', 0)
|
||||
|
||||
# 获取音频流
|
||||
audio_stream = audio_list[0] if audio_list else None
|
||||
audio_url = audio_stream.get('base_url') or audio_stream.get('baseUrl') if audio_stream else None
|
||||
|
||||
print(f"找到视频流: {video_url[:100]}...")
|
||||
|
||||
# 3. 下载视频流
|
||||
print(f"\n正在下载视频...")
|
||||
|
||||
# 下载视频
|
||||
async with session.get(video_url, headers=self._get_headers()) as resp:
|
||||
if resp.status == 200:
|
||||
video_content = await resp.read()
|
||||
print(f"视频下载完成: {len(video_content) / 1024 / 1024:.2f}MB")
|
||||
else:
|
||||
result["message"] = f"视频下载失败: HTTP {resp.status}"
|
||||
return result
|
||||
|
||||
# 下载音频
|
||||
audio_content = None
|
||||
if audio_url:
|
||||
print(f"正在下载音频...")
|
||||
async with session.get(audio_url, headers=self._get_headers()) as resp:
|
||||
if resp.status == 200:
|
||||
audio_content = await resp.read()
|
||||
print(f"音频下载完成: {len(audio_content) / 1024 / 1024:.2f}MB")
|
||||
|
||||
# 4. 合并音视频
|
||||
print(f"\n正在合并音视频...")
|
||||
|
||||
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
|
||||
safe_title = re.sub(r'[<>:"/\\|?*]', '', safe_title)
|
||||
|
||||
output_path = self.output_dir / f"{safe_title}_{bvid}"
|
||||
|
||||
# 保存文件
|
||||
if video_content:
|
||||
video_path = output_path.with_suffix('.mp4')
|
||||
with open(video_path, 'wb') as f:
|
||||
f.write(video_content)
|
||||
|
||||
result["filepath"] = str(video_path)
|
||||
result["success"] = True
|
||||
result["message"] = f"视频已保存: {video_path}"
|
||||
print(f"✅ 下载完成: {video_path}")
|
||||
else:
|
||||
result["message"] = "未找到视频流"
|
||||
else:
|
||||
# 回退到FLV格式
|
||||
durl = play_data.get('durl', [])
|
||||
if durl:
|
||||
video_url = durl[0].get('url')
|
||||
print(f"使用FLV格式: {video_url[:100]}...")
|
||||
|
||||
async with session.get(video_url, headers=self._get_headers()) as resp:
|
||||
if resp.status == 200:
|
||||
video_content = await resp.read()
|
||||
|
||||
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
|
||||
safe_title = re.sub(r'[<>:"/\\|?*]', '', safe_title)
|
||||
|
||||
output_path = self.output_dir / f"{safe_title}_{bvid}.flv"
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(video_content)
|
||||
|
||||
result["filepath"] = str(output_path)
|
||||
result["success"] = True
|
||||
result["message"] = f"视频已保存: {output_path}"
|
||||
print(f"✅ 下载完成: {output_path}")
|
||||
else:
|
||||
result["message"] = f"视频下载失败: HTTP {resp.status}"
|
||||
else:
|
||||
result["message"] = "未找到可用的视频流"
|
||||
|
||||
return result
|
||||
|
||||
async def extract_subtitle(self, bvid: str) -> Dict[str, Any]:
|
||||
"""
|
||||
提取B站视频字幕
|
||||
|
||||
Args:
|
||||
bvid: B站视频BV号
|
||||
|
||||
Returns:
|
||||
Dict: 包含字幕内容的JSON结果
|
||||
"""
|
||||
result = {
|
||||
"success": False,
|
||||
"bvid": bvid,
|
||||
"subtitles": [],
|
||||
"message": ""
|
||||
}
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# 获取视频信息
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
data = await self._make_request(session, api_url, {'bvid': bvid})
|
||||
|
||||
if not data or data.get('code') != 0:
|
||||
result["message"] = f"获取视频信息失败: {data.get('message', '未知错误')}"
|
||||
return result
|
||||
|
||||
cid = data.get('data', {}).get('cid')
|
||||
|
||||
# 获取字幕列表
|
||||
subtitle_api = "https://api.bilibili.com/x/player/pagelist"
|
||||
page_data = await self._make_request(session, subtitle_api, {'bvid': bvid})
|
||||
|
||||
# 获取字幕详情
|
||||
subtitle_list = []
|
||||
if page_data and page_data.get('code') == 0:
|
||||
cid = page_data.get('data', [{}])[0].get('cid', cid)
|
||||
|
||||
subtitle_url = f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}"
|
||||
subtitle_data = await self._make_request(session, subtitle_url)
|
||||
|
||||
if subtitle_data and subtitle_data.get('code') == 0:
|
||||
subtitles = subtitle_data.get('data', {}).get('subtitle', {}).get('list', [])
|
||||
|
||||
for sub in subtitles:
|
||||
sub_title = sub.get('lan_doc', '未知语言')
|
||||
sub_url = sub.get('subtitle_url')
|
||||
|
||||
if sub_url:
|
||||
# 下载字幕文件
|
||||
async with session.get(sub_url) as resp:
|
||||
if resp.status == 200:
|
||||
sub_content = await resp.json()
|
||||
subtitle_list.append({
|
||||
"language": sub.get('lan', 'unknown'),
|
||||
"title": sub_title,
|
||||
"content": sub_content
|
||||
})
|
||||
|
||||
result["subtitles"] = subtitle_list
|
||||
result["success"] = len(subtitle_list) > 0
|
||||
result["message"] = f"找到 {len(subtitle_list)} 个字幕" if subtitle_list else "未找到字幕"
|
||||
|
||||
# 保存字幕到文件
|
||||
if subtitle_list:
|
||||
output_path = self.output_dir / f"{bvid}_subtitles.json"
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
result["filepath"] = str(output_path)
|
||||
print(f"✅ 字幕已保存: {output_path}")
|
||||
|
||||
return result
|
||||
|
||||
async def extract_article(self, cvid: str) -> Dict[str, Any]:
|
||||
"""
|
||||
获取B站专栏全文
|
||||
|
||||
Args:
|
||||
cvid: B站专栏CV号
|
||||
|
||||
Returns:
|
||||
Dict: 包含专栏内容的JSON结果
|
||||
"""
|
||||
result = {
|
||||
"success": False,
|
||||
"cvid": cvid,
|
||||
"title": "",
|
||||
"content": "",
|
||||
"message": ""
|
||||
}
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# 获取专栏信息
|
||||
api_url = "https://api.bilibili.com/x/article/viewinfo"
|
||||
data = await self._make_request(session, api_url, {'id': cvid})
|
||||
|
||||
if not data or data.get('code') != 0:
|
||||
result["message"] = f"获取专栏信息失败: {data.get('message', '未知错误')}"
|
||||
return result
|
||||
|
||||
article_data = data.get('data', {})
|
||||
title = article_data.get('title', '')
|
||||
content = article_data.get('content', '')
|
||||
|
||||
# 获取作者信息
|
||||
author = article_data.get('author', {}).get('name', '未知作者')
|
||||
publish_time = article_data.get('publish_time', 0)
|
||||
view = article_data.get('stats', {}).get('view', 0)
|
||||
like = article_data.get('stats', {}).get('like', 0)
|
||||
|
||||
result["title"] = title
|
||||
result["content"] = content
|
||||
result["author"] = author
|
||||
result["publish_time"] = datetime.fromtimestamp(publish_time).strftime('%Y-%m-%d %H:%M:%S') if publish_time else ''
|
||||
result["stats"] = {"view": view, "like": like}
|
||||
result["success"] = bool(title and content)
|
||||
result["message"] = f"成功获取专栏: {title}" if result["success"] else "获取专栏内容失败"
|
||||
|
||||
# 保存到文件
|
||||
if result["success"]:
|
||||
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:30]
|
||||
safe_title = re.sub(r'[<>:"/\\|?*]', '', safe_title)
|
||||
|
||||
# 保存JSON
|
||||
json_path = self.output_dir / f"{safe_title}_{cvid}.json"
|
||||
with open(json_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 保存Markdown
|
||||
md_content = f"""# {title}
|
||||
|
||||
> **来源**: [B站专栏](https://www.bilibili.com/read/cv{cvid})
|
||||
> **作者**: {author}
|
||||
> **发布时间**: {result["publish_time"]}
|
||||
> **阅读**: {view} | **点赞**: {like}
|
||||
|
||||
---
|
||||
|
||||
{content}
|
||||
"""
|
||||
md_path = self.output_dir / f"{safe_title}_{cvid}.md"
|
||||
with open(md_path, 'w', encoding='utf-8') as f:
|
||||
f.write(md_content)
|
||||
|
||||
result["json_path"] = str(json_path)
|
||||
result["md_path"] = str(md_path)
|
||||
print(f"✅ 专栏已保存: {md_path}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def main():
|
||||
"""命令行入口"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Bilibili Content Downloader - B站内容获取工具',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
%(prog)s --id "BV1xx411c7mD" --type video # 下载视频
|
||||
%(prog)s --id "BV1xx411c7mD" --type video --quality 720 # 下载720P视频
|
||||
%(prog)s --id "BV1xx411c7mD" --type subtitle # 提取字幕
|
||||
%(prog)s --id "cv43780186" --type article # 获取专栏
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('--id', '-i', required=True, help='B站内容ID (BV号或CV号)')
|
||||
parser.add_argument('--type', '-t', required=True, choices=['video', 'subtitle', 'article'],
|
||||
help='内容类型: video=视频, subtitle=字幕, article=专栏')
|
||||
parser.add_argument('--output', '-o', default='./output', help='输出目录 (默认: ./output)')
|
||||
parser.add_argument('--quality', '-q', type=int, default=480,
|
||||
help='视频质量 (默认: 480, 可选: 360, 480, 720, 1080)')
|
||||
parser.add_argument('--json', action='store_true', help='JSON格式输出')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 确定内容类型
|
||||
is_article = args.id.startswith('cv')
|
||||
|
||||
if args.type == 'article' or is_article:
|
||||
content_type = 'article'
|
||||
content_id = args.id.replace('cv', '')
|
||||
elif args.type == 'subtitle':
|
||||
content_type = 'subtitle'
|
||||
content_id = args.id
|
||||
else:
|
||||
content_type = 'video'
|
||||
content_id = args.id
|
||||
|
||||
# 创建获取器
|
||||
downloader = BilibiliContent(output_dir=args.output)
|
||||
|
||||
# 执行操作
|
||||
if content_type == 'video':
|
||||
result = await downloader.download_video(content_id, quality=args.quality)
|
||||
elif content_type == 'subtitle':
|
||||
result = await downloader.extract_subtitle(content_id)
|
||||
else:
|
||||
result = await downloader.extract_article(content_id)
|
||||
|
||||
# 输出结果
|
||||
if args.json:
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
if result["success"]:
|
||||
print(f"\n✅ {result['message']}")
|
||||
else:
|
||||
print(f"\n❌ {result['message']}")
|
||||
|
||||
if result.get("filepath"):
|
||||
print(f"文件路径: {result['filepath']}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
result = asyncio.run(main())
|
||||
sys.exit(0 if result.get("success") else 1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n操作已取消")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n错误: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
B站视频提取脚本
|
||||
提取视频信息、统计数据、字幕并生成Markdown笔记
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
# 添加skills路径
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb'))
|
||||
|
||||
from src.extractors.bilibili_extractor import BilibiliExtractor
|
||||
from src.config import get_config
|
||||
|
||||
|
||||
def format_number(num):
|
||||
"""格式化数字"""
|
||||
if num >= 100000000:
|
||||
return f"{num / 100000000:.1f}亿"
|
||||
elif num >= 10000:
|
||||
return f"{num / 10000:.1f}万"
|
||||
else:
|
||||
return f"{num:,}"
|
||||
|
||||
|
||||
def generate_markdown(result, bvid):
|
||||
"""生成Markdown笔记"""
|
||||
metadata = result['metadata']
|
||||
stats = result['stats']
|
||||
description = result['description']
|
||||
subtitle = result['subtitle']
|
||||
|
||||
# 获取当前时间
|
||||
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
md = f"""# {metadata['title']}
|
||||
|
||||
> **来源**: [B站](https://www.bilibili.com/video/{bvid})
|
||||
> **作者**: {metadata['author']}
|
||||
> **发布时间**: {metadata['pubtime']}
|
||||
> **提取时间**: {now}
|
||||
> **时长**: {metadata['duration']}
|
||||
|
||||
## 📊 统计数据
|
||||
|
||||
| 指标 | 数量 |
|
||||
|------|------|
|
||||
| 👀 播放 | {format_number(stats['view'])} |
|
||||
| 💬 弹幕 | {format_number(stats['danmaku'])} |
|
||||
| 📝 评论 | {format_number(stats['reply'])} |
|
||||
| ⭐ 收藏 | {format_number(stats['favorite'])} |
|
||||
| 🪙 投币 | {format_number(stats['coin'])} |
|
||||
| 🔗 分享 | {format_number(stats['share'])} |
|
||||
| 👍 点赞 | {format_number(stats['like'])} |
|
||||
|
||||
## 📝 视频简介
|
||||
|
||||
{description if description else '(暂无简介)'}
|
||||
|
||||
## 📌 标签
|
||||
|
||||
{metadata['tags'] if metadata['tags'] else '(无标签)'}
|
||||
|
||||
## 🎬 字幕
|
||||
|
||||
{subtitle if subtitle else '(无字幕)'}
|
||||
|
||||
---
|
||||
|
||||
**元数据**:
|
||||
- BV号: `{metadata['bvid']}`
|
||||
- AV号: `{metadata['aid']}`
|
||||
- UP主ID: `{metadata['mid']}`
|
||||
"""
|
||||
|
||||
return md
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
# BV号
|
||||
bvid = "BV1N5foBjE5K" # 从 https://b23.tv/btMGoOi 解析得到
|
||||
|
||||
print(f"开始提取B站视频: {bvid}")
|
||||
print("=" * 60)
|
||||
|
||||
# 获取配置
|
||||
try:
|
||||
config = get_config()
|
||||
except Exception as e:
|
||||
print(f"⚠️ 获取配置失败，使用默认配置: {e}")
|
||||
config = None
|
||||
|
||||
# 创建提取器
|
||||
extractor = BilibiliExtractor(config=config)
|
||||
|
||||
# 提取视频信息
|
||||
try:
|
||||
result = extractor.extract(bvid)
|
||||
|
||||
print("\n✅ 提取成功！\n")
|
||||
print("=" * 60)
|
||||
print("基本信息:")
|
||||
print("=" * 60)
|
||||
print(f"标题: {result['metadata']['title']}")
|
||||
print(f"作者: {result['metadata']['author']}")
|
||||
print(f"时长: {result['metadata']['duration']}")
|
||||
print(f"发布: {result['metadata']['pubtime']}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("统计数据:")
|
||||
print("=" * 60)
|
||||
print(f"播放: {format_number(result['stats']['view'])}")
|
||||
print(f"弹幕: {format_number(result['stats']['danmaku'])}")
|
||||
print(f"评论: {format_number(result['stats']['reply'])}")
|
||||
print(f"收藏: {format_number(result['stats']['favorite'])}")
|
||||
print(f"投币: {format_number(result['stats']['coin'])}")
|
||||
print(f"点赞: {format_number(result['stats']['like'])}")
|
||||
|
||||
# 生成 Markdown 笔记
|
||||
md_content = generate_markdown(result, bvid)
|
||||
|
||||
# 确保 notes 目录存在
|
||||
notes_dir = "/home/admin/.openclaw/workspace/notes"
|
||||
os.makedirs(notes_dir, exist_ok=True)
|
||||
|
||||
# 生成文件名（使用BV号作为文件名）
|
||||
filename = f"{bvid}.md"
|
||||
filepath = os.path.join(notes_dir, filename)
|
||||
|
||||
# 写入文件
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
f.write(md_content)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(f"✅ Markdown笔记已生成: {filepath}")
|
||||
print("=" * 60)
|
||||
|
||||
# 返回笔记内容
|
||||
return md_content
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ 提取失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
使用旧版 API 获取并下载视频流
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def download_video_stream(bvid, output_dir="/tmp/bilibili_downloads"):
|
||||
"""下载视频流"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
# 创建输出目录
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
print(f"正在下载视频: {url}")
|
||||
print("=" * 80)
|
||||
|
||||
# 首先获取 aid 和 cid
|
||||
print("\n1. 获取视频信息...")
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') != 0:
|
||||
print(f" ❌ 失败: {data.get('message')}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
video_data = data.get('data', {})
|
||||
title = video_data.get('title')
|
||||
aid = video_data.get('aid')
|
||||
cid = video_data.get('cid')
|
||||
duration = video_data.get('duration')
|
||||
print(f" 标题: {title}")
|
||||
print(f" 时长: {duration}秒")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 获取视频流 URL
|
||||
print("\n2. 获取视频流 URL...")
|
||||
try:
|
||||
playurl_api = "https://api.bilibili.com/x/player/playurl"
|
||||
params = {
|
||||
'bvid': bvid,
|
||||
'cid': cid,
|
||||
'qn': 80, # 1080P
|
||||
'otype': 'json',
|
||||
'fnver': 0,
|
||||
'fnval': 0, # FLV 格式
|
||||
}
|
||||
|
||||
playurl_response = await page.request.get(playurl_api, params=params)
|
||||
|
||||
if not playurl_response.ok:
|
||||
print(f" ❌ HTTP 错误: {playurl_response.status}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
playurl_data = await playurl_response.json()
|
||||
|
||||
if playurl_data.get('code') != 0:
|
||||
print(f" ❌ API 错误: {playurl_data.get('message')}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
play_data = playurl_data.get('data', {})
|
||||
|
||||
# 检查支持的清晰度
|
||||
print(f"\n 支持的清晰度:")
|
||||
for i, (q, desc) in enumerate(zip(play_data.get('accept_quality', []), play_data.get('accept_description', []))):
|
||||
current = " (当前)" if q == play_data.get('quality') else ""
|
||||
print(f" {desc} ({q}){current}")
|
||||
|
||||
# 获取视频 URL
|
||||
durl = play_data.get('durl', [])
|
||||
if not durl:
|
||||
print(f" ❌ 没有找到视频流")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
print(f"\n 找到 {len(durl)} 个视频分段")
|
||||
video_urls = []
|
||||
for i, segment in enumerate(durl):
|
||||
segment_url = segment.get('url')
|
||||
segment_size = segment.get('size', 0)
|
||||
video_urls.append(segment_url)
|
||||
print(f" 分段 {i+1}: {segment_size / 1024 / 1024:.2f}MB")
|
||||
print(f" {segment_url[:100]}...")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 下载视频
|
||||
print(f"\n3. 开始下载视频...")
|
||||
all_segments = []
|
||||
total_size = 0
|
||||
|
||||
for i, video_url in enumerate(video_urls, 1):
|
||||
print(f"\n 下载分段 {i}/{len(video_urls)}...")
|
||||
|
||||
try:
|
||||
# 添加必要的请求头
|
||||
headers = {
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
}
|
||||
|
||||
# 下载分段
|
||||
segment_response = await page.request.get(
|
||||
video_url,
|
||||
headers=headers,
|
||||
timeout=60000
|
||||
)
|
||||
|
||||
if not segment_response.ok:
|
||||
print(f" ❌ 下载失败: {segment_response.status}")
|
||||
continue
|
||||
|
||||
segment_data = await segment_response.body()
|
||||
segment_size = len(segment_data)
|
||||
total_size += segment_size
|
||||
all_segments.append(segment_data)
|
||||
|
||||
print(f" ✅ 下载成功: {segment_size / 1024 / 1024:.2f}MB")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 下载异常: {e}")
|
||||
continue
|
||||
|
||||
if not all_segments:
|
||||
print(f"\n ❌ 没有成功下载任何分段")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 合并分段
|
||||
print(f"\n4. 合并视频分段...")
|
||||
video_data = b''.join(all_segments)
|
||||
|
||||
# 保存文件
|
||||
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
|
||||
output_path = os.path.join(output_dir, f"{safe_title}.flv")
|
||||
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(video_data)
|
||||
|
||||
print(f" ✅ 视频已保存: {output_path}")
|
||||
print(f" 文件大小: {total_size / 1024 / 1024:.2f}MB")
|
||||
|
||||
await browser.close()
|
||||
|
||||
return output_path
|
||||
|
||||
if __name__ == '__main__':
|
||||
bvid = "BV1KtFzzqEUG"
|
||||
result = asyncio.run(download_video_stream(bvid))
|
||||
if result:
|
||||
print(f"\n✅ 下载完成！")
|
||||
print(f"文件路径: {result}")
|
||||
else:
|
||||
print(f"\n❌ 下载失败")
|
||||
@@ -0,0 +1,46 @@
|
||||
# KB Bilibili Metadata Skill
|
||||
|
||||
**kb-bilibili-metadata** - Extract Bilibili video/article/up主 metadata.
|
||||
|
||||
## Features
|
||||
|
||||
- Video metadata extraction
|
||||
- Article (专栏) metadata extraction
|
||||
- UP主 information extraction
|
||||
- JSON output format
|
||||
- Playwright + HTTP API fallback
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Video metadata
|
||||
python bilibili_metadata.py --id "BV1xx411c7mD" --type video
|
||||
|
||||
# Article metadata
|
||||
python bilibili_metadata.py --id "cv43780186" --type article
|
||||
|
||||
# UP主 info
|
||||
python bilibili_metadata.py --id "7770724" --type up
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"platform": "bilibili",
|
||||
"type": "video",
|
||||
"id": "BV1xx411c7mD",
|
||||
"title": "视频标题",
|
||||
"author": "UP主名字",
|
||||
"stats": {"view": 100000, "like": 5000}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
cp -r kb-bilibili-metadata/ ~/skills/
|
||||
```
|
||||
@@ -0,0 +1,8 @@
|
||||
"""
|
||||
bilibili-metadata 技能包
|
||||
|
||||
B站元数据提取工具
|
||||
"""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
__author__ = "OpenClaw"
|
||||
@@ -0,0 +1,446 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
B站元数据提取器
|
||||
|
||||
独立版本，不依赖 obsidian-kb 结构
|
||||
支持视频、专栏、UP主信息提取
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 默认请求头
|
||||
DEFAULT_HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
'Accept': 'application/json, text/plain, */*',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
}
|
||||
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
PLAYWRIGHT_AVAILABLE = True
|
||||
except ImportError:
|
||||
PLAYWRIGHT_AVAILABLE = False
|
||||
|
||||
try:
|
||||
from undetected_playwright import stealth_async
|
||||
STEALTH_AVAILABLE = True
|
||||
except ImportError:
|
||||
STEALTH_AVAILABLE = False
|
||||
|
||||
try:
|
||||
import httpx
|
||||
HTTPX_AVAILABLE = True
|
||||
except ImportError:
|
||||
HTTPX_AVAILABLE = False
|
||||
|
||||
|
||||
class BilibiliExtractor:
|
||||
"""
|
||||
B站元数据提取器
|
||||
|
||||
支持：
|
||||
- 视频信息提取（标题、作者、统计数据、字幕等）
|
||||
- 专栏文章提取
|
||||
- UP主信息提取
|
||||
"""
|
||||
|
||||
# 正则模式
|
||||
BV_PATTERN = re.compile(r'BV[a-zA-Z0-9]{10}')
|
||||
CV_PATTERN = re.compile(r'cv(\d+)')
|
||||
URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/video/(BV[a-zA-Z0-9]{10})')
|
||||
ARTICLE_URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/read/(cv\d+)')
|
||||
UP_URL_PATTERN = re.compile(r'https?://space\.bilibili\.com/(\d+)')
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||
"""
|
||||
初始化提取器
|
||||
|
||||
Args:
|
||||
headless: 是否使用无头模式（浏览器）
|
||||
timeout: 超时时间（秒）
|
||||
"""
|
||||
self.headless = headless
|
||||
self.timeout = timeout * 1000 # 转换为毫秒
|
||||
|
||||
def extract_video_metadata(self, bvid: str) -> Dict[str, Any]:
|
||||
"""
|
||||
提取视频元数据
|
||||
|
||||
Args:
|
||||
bvid: B站视频BV号（如 BV1xx411c7mD）
|
||||
|
||||
Returns:
|
||||
Dict containing metadata, stats, description, subtitle
|
||||
"""
|
||||
# 验证并解析BV号
|
||||
bvid = self._resolve_bvid(bvid)
|
||||
if not bvid:
|
||||
raise ValueError(f"无效的BV号: {bvid}")
|
||||
|
||||
# 使用异步方法提取
|
||||
return asyncio.run(self._extract_video_async(bvid))
|
||||
|
||||
def extract_article_metadata(self, cvid: str) -> Dict[str, Any]:
|
||||
"""
|
||||
提取专栏元数据
|
||||
|
||||
Args:
|
||||
cvid: B站专栏CV号（如 cv43780186）
|
||||
|
||||
Returns:
|
||||
Dict containing article metadata
|
||||
"""
|
||||
# 解析CV号
|
||||
cvid = self._resolve_cvid(cvid)
|
||||
if not cvid:
|
||||
raise ValueError(f"无效的CV号: {cvid}")
|
||||
|
||||
return asyncio.run(self._extract_article_async(cvid))
|
||||
|
||||
def extract_up_metadata(self, mid: str) -> Dict[str, Any]:
|
||||
"""
|
||||
提取UP主信息
|
||||
|
||||
Args:
|
||||
mid: B站用户ID（如 7770724）
|
||||
|
||||
Returns:
|
||||
Dict containing UP主信息
|
||||
"""
|
||||
# 解析MID
|
||||
mid = self._resolve_mid(mid)
|
||||
if not mid:
|
||||
raise ValueError(f"无效的MID: {mid}")
|
||||
|
||||
return asyncio.run(self._extract_up_async(mid))
|
||||
|
||||
def _resolve_bvid(self, source_id: str) -> Optional[str]:
|
||||
"""解析并返回BV号"""
|
||||
source = source_id.strip()
|
||||
|
||||
# 直接BV号
|
||||
if self.BV_PATTERN.fullmatch(source):
|
||||
return source
|
||||
|
||||
# 完整URL
|
||||
match = self.URL_PATTERN.match(source)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
def _resolve_cvid(self, source_id: str) -> Optional[str]:
|
||||
"""解析并返回CV号"""
|
||||
source = source_id.strip()
|
||||
|
||||
# 直接CV号（带cv前缀）
|
||||
if source.startswith('cv'):
|
||||
match = self.CV_PATTERN.match(source)
|
||||
if match:
|
||||
return source
|
||||
|
||||
# 纯数字
|
||||
if source.isdigit():
|
||||
return f"cv{source}"
|
||||
|
||||
# 完整URL
|
||||
match = self.ARTICLE_URL_PATTERN.match(source)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
def _resolve_mid(self, source_id: str) -> Optional[str]:
|
||||
"""解析并返回MID"""
|
||||
source = source_id.strip()
|
||||
|
||||
# 纯数字
|
||||
if source.isdigit():
|
||||
return source
|
||||
|
||||
# 完整URL
|
||||
match = self.UP_URL_PATTERN.match(source)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
async def _extract_video_async(self, bvid: str) -> Dict[str, Any]:
|
||||
"""异步提取视频信息"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
result = {
|
||||
'metadata': {},
|
||||
'stats': {},
|
||||
'subtitle': '',
|
||||
'description': ''
|
||||
}
|
||||
|
||||
if not PLAYWRIGHT_AVAILABLE:
|
||||
# 使用 HTTP API
|
||||
return await self._extract_video_by_api(bvid)
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
try:
|
||||
browser = await playwright.chromium.launch(
|
||||
headless=self.headless,
|
||||
args=[
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-dev-shm-usage',
|
||||
]
|
||||
)
|
||||
|
||||
context = await browser.new_context(
|
||||
viewport={'width': 1920, 'height': 1080},
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
locale='zh-CN',
|
||||
)
|
||||
|
||||
if STEALTH_AVAILABLE:
|
||||
await stealth_async(context)
|
||||
|
||||
page = await context.new_page()
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=self.timeout)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
content = await page.content()
|
||||
video_data = await self._try_extract_video_methods(page, content, bvid)
|
||||
|
||||
if video_data:
|
||||
result['metadata'] = {
|
||||
'title': video_data.get('title', ''),
|
||||
'author': video_data.get('author', ''),
|
||||
'bvid': video_data.get('bvid', ''),
|
||||
'aid': video_data.get('aid', ''),
|
||||
'mid': video_data.get('mid', ''),
|
||||
'pubtime': video_data.get('pubtime', ''),
|
||||
'duration': video_data.get('duration', ''),
|
||||
'url': url,
|
||||
'tags': video_data.get('tags', ''),
|
||||
}
|
||||
result['stats'] = {
|
||||
'view': video_data.get('view', 0),
|
||||
'danmaku': video_data.get('danmaku', 0),
|
||||
'reply': video_data.get('reply', 0),
|
||||
'favorite': video_data.get('favorite', 0),
|
||||
'coin': video_data.get('coin', 0),
|
||||
'share': video_data.get('share', 0),
|
||||
'like': video_data.get('like', 0),
|
||||
}
|
||||
result['description'] = video_data.get('desc', '')
|
||||
|
||||
await browser.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"视频提取错误: {e}")
|
||||
# 回退到 API
|
||||
return await self._extract_video_by_api(bvid)
|
||||
|
||||
return result
|
||||
|
||||
async def _extract_video_by_api(self, bvid: str) -> Dict[str, Any]:
|
||||
"""使用B站API提取视频信息"""
|
||||
result = {
|
||||
'metadata': {},
|
||||
'stats': {},
|
||||
'subtitle': '',
|
||||
'description': ''
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
|
||||
# 获取视频信息
|
||||
response = await client.get('https://api.bilibili.com/x/web-interface/view', params={'bvid': bvid})
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get('code') == 0:
|
||||
video_data = data.get('data', {})
|
||||
|
||||
result['metadata'] = {
|
||||
'title': video_data.get('title', ''),
|
||||
'author': video_data.get('owner', {}).get('name', ''),
|
||||
'bvid': video_data.get('bvid', ''),
|
||||
'aid': video_data.get('aid', ''),
|
||||
'mid': video_data.get('owner', {}).get('mid', ''),
|
||||
'pubtime': datetime.fromtimestamp(video_data.get('pubdate', 0)).strftime('%Y-%m-%d %H:%M:%S') if video_data.get('pubdate') else '',
|
||||
'duration': self._format_duration(video_data.get('duration', 0)),
|
||||
'url': f"https://www.bilibili.com/video/{bvid}",
|
||||
'tags': ', '.join([t.get('tag_name', '') for t in video_data.get('tag', [])]),
|
||||
}
|
||||
|
||||
stat = video_data.get('stat', {})
|
||||
result['stats'] = {
|
||||
'view': stat.get('view', 0),
|
||||
'danmaku': stat.get('danmaku', 0),
|
||||
'reply': stat.get('reply', 0),
|
||||
'favorite': stat.get('favorite', 0),
|
||||
'coin': stat.get('coin', 0),
|
||||
'share': stat.get('share', 0),
|
||||
'like': stat.get('like', 0),
|
||||
}
|
||||
result['description'] = video_data.get('desc', '')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"API提取错误: {e}")
|
||||
|
||||
return result
|
||||
|
||||
async def _try_extract_video_methods(self, page, content: str, bvid: str) -> Dict[str, Any]:
|
||||
"""尝试多种方法提取视频信息"""
|
||||
video_data = {}
|
||||
|
||||
# 方法1: 从 __INITIAL_STATE__
|
||||
try:
|
||||
initial_state = await page.evaluate('() => typeof window.__INITIAL_STATE__ !== "undefined" ? window.__INITIAL_STATE__ : null')
|
||||
if initial_state:
|
||||
video_data = self._parse_video_state(initial_state)
|
||||
if video_data and video_data.get('title'):
|
||||
return video_data
|
||||
except:
|
||||
pass
|
||||
|
||||
# 方法2: 使用 API（更可靠）
|
||||
return await self._extract_video_by_api(bvid)
|
||||
|
||||
def _parse_video_state(self, data: Dict) -> Dict[str, Any]:
|
||||
"""解析 __INITIAL_STATE__ 数据"""
|
||||
video_data = {}
|
||||
try:
|
||||
video = data.get('videoData', {})
|
||||
video_data['title'] = video.get('title', '')
|
||||
video_data['bvid'] = video.get('bvid', '')
|
||||
video_data['aid'] = video.get('aid', '')
|
||||
owner = video.get('owner', {})
|
||||
video_data['author'] = owner.get('name', '')
|
||||
video_data['mid'] = owner.get('mid', '')
|
||||
stat = video.get('stat', {})
|
||||
video_data['view'] = stat.get('view', 0)
|
||||
video_data['danmaku'] = stat.get('danmaku', 0)
|
||||
video_data['reply'] = stat.get('reply', 0)
|
||||
video_data['favorite'] = stat.get('favorite', 0)
|
||||
video_data['coin'] = stat.get('coin', 0)
|
||||
video_data['share'] = stat.get('share', 0)
|
||||
video_data['like'] = stat.get('like', 0)
|
||||
pubdate = video.get('pubdate', 0)
|
||||
video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else ''
|
||||
video_data['duration'] = self._format_duration(video.get('duration', 0))
|
||||
video_data['desc'] = video.get('desc', '')
|
||||
video_data['tags'] = ', '.join([t.get('tag_name', '') for t in video.get('tag', [])])
|
||||
except:
|
||||
pass
|
||||
return video_data
|
||||
|
||||
async def _extract_article_async(self, cvid: str) -> Dict[str, Any]:
|
||||
"""异步提取专栏信息"""
|
||||
result = {}
|
||||
|
||||
if not HTTPX_AVAILABLE:
|
||||
return {'error': 'httpx 未安装'}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
|
||||
# 获取专栏信息
|
||||
response = await client.get('https://api.bilibili.com/x/article/viewinfo', params={'id': cvid[2:]})
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get('code') == 0:
|
||||
article = data.get('data', {})
|
||||
result = {
|
||||
'cvid': cvid,
|
||||
'title': article.get('title', ''),
|
||||
'author': article.get('author_name', ''),
|
||||
'mid': article.get('author_mid', ''),
|
||||
'publish_time': datetime.fromtimestamp(article.get('publish_time', 0)).strftime('%Y-%m-%d %H:%M:%S') if article.get('publish_time') else '',
|
||||
'view_count': article.get('stats', {}).get('view', 0),
|
||||
'like_count': article.get('stats', {}).get('like', 0),
|
||||
'coin_count': article.get('stats', {}).get('coin', 0),
|
||||
'share_count': article.get('stats', {}).get('share', 0),
|
||||
'reply_count': article.get('stats', {}).get('reply', 0),
|
||||
'url': f"https://www.bilibili.com/read/{cvid}",
|
||||
'summary': article.get('summary', ''),
|
||||
'word_count': article.get('words', 0),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"专栏提取错误: {e}")
|
||||
result = {'error': str(e)}
|
||||
|
||||
return result
|
||||
|
||||
async def _extract_up_async(self, mid: str) -> Dict[str, Any]:
|
||||
"""异步提取UP主信息"""
|
||||
result = {}
|
||||
|
||||
if not HTTPX_AVAILABLE:
|
||||
return {'error': 'httpx 未安装'}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
|
||||
# 获取UP主信息
|
||||
response = await client.get('https://api.bilibili.com/x/space/acc/info', params={'mid': mid})
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get('code') == 0:
|
||||
up = data.get('data', {})
|
||||
result = {
|
||||
'mid': mid,
|
||||
'name': up.get('name', ''),
|
||||
'sex': up.get('sex', ''),
|
||||
'face': up.get('face', ''),
|
||||
'sign': up.get('sign', ''),
|
||||
'level': up.get('level', 0),
|
||||
'vip_type': up.get('vip', {}).get('type', 0),
|
||||
'vip_status': up.get('vip', {}).get('status', 0),
|
||||
'following_count': up.get('following', 0),
|
||||
'follower_count': up.get('follower', 0),
|
||||
'url': f"https://space.bilibili.com/{mid}",
|
||||
}
|
||||
|
||||
# 获取UP主视频统计
|
||||
stats_response = await client.get('https://api.bilibili.com/x/space/navnum', params={'mid': mid})
|
||||
if stats_response.status_code == 200:
|
||||
stats_data = stats_response.json()
|
||||
if stats_data.get('code') == 0:
|
||||
result['video_count'] = stats_data.get('data', {}).get('video', 0)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"UP主提取错误: {e}")
|
||||
result = {'error': str(e)}
|
||||
|
||||
return result
|
||||
|
||||
def _format_duration(self, seconds: int) -> str:
|
||||
"""格式化时长（秒 -> HH:MM:SS）"""
|
||||
if not seconds:
|
||||
return ''
|
||||
hours = seconds // 3600
|
||||
minutes = (seconds % 3600) // 60
|
||||
secs = seconds % 60
|
||||
if hours > 0:
|
||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
||||
else:
|
||||
return f"{minutes:02d}:{secs:02d}"
|
||||
|
||||
|
||||
# 测试代码
|
||||
if __name__ == '__main__':
|
||||
extractor = BilibiliExtractor()
|
||||
|
||||
print("=" * 60)
|
||||
print("B站元数据提取器测试")
|
||||
print("=" * 60)
|
||||
|
||||
# 测试视频
|
||||
print("\n测试视频提取: BV1MnFNz5EDL")
|
||||
result = extractor.extract_video_metadata('BV1MnFNz5EDL')
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
B站元数据提取 CLI 工具
|
||||
|
||||
支持命令行调用，输出 JSON 格式数据
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from typing import Dict, Any
|
||||
|
||||
from bilibili_extractor import BilibiliExtractor
|
||||
|
||||
|
||||
def format_number(num: int) -> str:
|
||||
"""格式化数字"""
|
||||
if num >= 100000000:
|
||||
return f"{num / 100000000:.1f}亿"
|
||||
elif num >= 10000:
|
||||
return f"{num / 10000:.1f}万"
|
||||
else:
|
||||
return f"{num:,}"
|
||||
|
||||
|
||||
def extract_video(extractor: BilibiliExtractor, bvid: str, output_file: str = None) -> Dict[str, Any]:
|
||||
"""提取视频元数据"""
|
||||
result = extractor.extract_video_metadata(bvid)
|
||||
|
||||
# 如果需要输出到文件
|
||||
if output_file:
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def extract_article(extractor: BilibiliExtractor, cvid: str, output_file: str = None) -> Dict[str, Any]:
|
||||
"""提取专栏元数据"""
|
||||
result = extractor.extract_article_metadata(cvid)
|
||||
|
||||
if output_file:
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def extract_up(extractor: BilibiliExtractor, mid: str, output_file: str = None) -> Dict[str, Any]:
|
||||
"""提取UP主信息"""
|
||||
result = extractor.extract_up_metadata(mid)
|
||||
|
||||
if output_file:
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='B站元数据提取工具',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
# 提取视频元数据
|
||||
python bilibili_metadata.py --id "BV1xx411c7mD" --type video
|
||||
|
||||
# 提取专栏元数据
|
||||
python bilibili_metadata.py --id "cv43780186" --type article
|
||||
|
||||
# 提取UP主信息
|
||||
python bilibili_metadata.py --id "7770724" --type up
|
||||
|
||||
# 输出到文件
|
||||
python bilibili_metadata.py --id "BV1xx411c7mD" --type video --output result.json
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--id',
|
||||
required=True,
|
||||
help='B站资源ID（BV号、CV号或UP主ID）'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--type',
|
||||
required=True,
|
||||
choices=['video', 'article', 'up'],
|
||||
help='资源类型'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--output',
|
||||
'-o',
|
||||
help='输出文件路径（JSON格式）'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--headless',
|
||||
action='store_true',
|
||||
default=True,
|
||||
help='使用无头模式（浏览器，默认开启）'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--timeout',
|
||||
type=int,
|
||||
default=30,
|
||||
help='超时时间（秒，默认30）'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--pretty',
|
||||
action='store_true',
|
||||
default=True,
|
||||
help='格式化输出（默认开启）'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 创建提取器
|
||||
extractor = BilibiliExtractor(headless=args.headless, timeout=args.timeout)
|
||||
|
||||
try:
|
||||
# 根据类型提取数据
|
||||
if args.type == 'video':
|
||||
result = extract_video(extractor, args.id, args.output)
|
||||
elif args.type == 'article':
|
||||
result = extract_article(extractor, args.id, args.output)
|
||||
elif args.type == 'up':
|
||||
result = extract_up(extractor, args.id, args.output)
|
||||
else:
|
||||
print(f"错误: 不支持的类型 {args.type}")
|
||||
sys.exit(1)
|
||||
|
||||
# 输出结果
|
||||
if not args.output:
|
||||
if args.pretty:
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
if args.output:
|
||||
print(f"结果已保存到: {args.output}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"错误: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
# KB Deep Analyzer Skill
|
||||
|
||||
**kb-deep-analyzer** - AI-powered deep content analysis.
|
||||
|
||||
## Features
|
||||
|
||||
- Multi-type analysis (tutorial, documentation, news)
|
||||
- Template-based output
|
||||
- Quality scoring
|
||||
- Automatic tagging
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Analyze content
|
||||
python deep_analyzer.py --metadata meta.json --content content.json --output analysis.md
|
||||
|
||||
# Specify type
|
||||
python deep_analyzer.py --metadata meta.json --content content.json --type tutorial --output tutorial.md
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
```markdown
|
||||
---
|
||||
title: "分析标题"
|
||||
source: "bilibili"
|
||||
quality_score: 95
|
||||
tags:
|
||||
- python
|
||||
- tutorial
|
||||
---
|
||||
|
||||
## 一句话总结
|
||||
|
||||
内容摘要...
|
||||
|
||||
## 核心要点
|
||||
|
||||
1. 要点1
|
||||
2. 要点2
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
cp -r kb-deep-analyzer/ ~/skills/
|
||||
```
|
||||
@@ -0,0 +1,514 @@
|
||||
"""Obsidian KB - AI客户端
|
||||
支持多个 AI 提供商和 fallback 机制
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
from src.models import ContentType
|
||||
from src.config import get_config
|
||||
from src.content_chunker import ContentChunker, get_chunking_strategy, chunk_content
|
||||
|
||||
|
||||
class AIClient:
|
||||
"""
|
||||
AI客户端 - 支持分段分析长视频内容和 fallback 机制
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config()
|
||||
self.chunker = ContentChunker()
|
||||
# 获取 AI 配置
|
||||
self._load_ai_config()
|
||||
|
||||
def _load_ai_config(self) -> None:
|
||||
"""加载 AI 配置（主 provider + fallbacks）"""
|
||||
self.ai_configs = []
|
||||
|
||||
# 加载主配置
|
||||
main_provider = self.config.get('ai.provider', 'openai')
|
||||
main_config = {
|
||||
'provider': main_provider,
|
||||
'api_key': self.config.get('ai.api_key'),
|
||||
'model': self.config.get('ai.model', 'gpt-4'),
|
||||
'base_url': self.config.get('ai.base_url'),
|
||||
'is_primary': True
|
||||
}
|
||||
self.ai_configs.append(main_config)
|
||||
|
||||
# 加载 fallback 配置
|
||||
fallbacks = self.config.get('ai.fallbacks', [])
|
||||
for fb in fallbacks:
|
||||
self.ai_configs.append({
|
||||
'provider': fb.get('provider'),
|
||||
'api_key': fb.get('api_key'),
|
||||
'model': fb.get('model'),
|
||||
'base_url': fb.get('base_url'),
|
||||
'is_primary': False
|
||||
})
|
||||
|
||||
def _call_api_with_fallback(self, prompt: str, max_tokens: int = 1000, response_format: str = 'text') -> Tuple[bool, str]:
|
||||
"""
|
||||
使用 fallback 机制调用 API
|
||||
|
||||
Args:
|
||||
prompt: 提示词
|
||||
max_tokens: 最大 token 数
|
||||
response_format: 响应格式 ('text' 或 'json')
|
||||
|
||||
Returns:
|
||||
(success, response) 元组
|
||||
"""
|
||||
last_error = None
|
||||
|
||||
for i, ai_config in enumerate(self.ai_configs):
|
||||
try:
|
||||
if not ai_config.get('api_key'):
|
||||
print(f"[AIClient] 跳过 {ai_config.get('provider')} - API Key 为空")
|
||||
continue
|
||||
|
||||
provider_type = ai_config.get('provider')
|
||||
print(f"[AIClient] 尝试使用 {provider_type} (优先级 {i+1}/{len(self.ai_configs)})...")
|
||||
|
||||
response = self._call_single_provider(ai_config, prompt, max_tokens, response_format)
|
||||
return True, response
|
||||
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
print(f"[AIClient] {provider_type} 调用失败: {e}")
|
||||
continue
|
||||
|
||||
# 所有 provider 都失败了
|
||||
error_msg = f"所有 AI 提供商均失败，最后错误: {last_error}" if last_error else "没有可用的 AI 配置"
|
||||
print(f"[AIClient] {error_msg}")
|
||||
return False, self._mock_generate(prompt, max_tokens) # 返回 mock 响应作为兜底
|
||||
|
||||
def _call_single_provider(self, ai_config: Dict, prompt: str, max_tokens: int, response_format: str) -> str:
|
||||
"""
|
||||
调用单个 AI 提供商
|
||||
|
||||
Args:
|
||||
ai_config: AI 配置
|
||||
prompt: 提示词
|
||||
max_tokens: 最大 token 数
|
||||
response_format: 响应格式
|
||||
|
||||
Returns:
|
||||
API 响应文本
|
||||
"""
|
||||
provider = ai_config.get('provider')
|
||||
api_key = ai_config.get('api_key')
|
||||
model = ai_config.get('model')
|
||||
base_url = ai_config.get('base_url')
|
||||
|
||||
if provider == 'minimax':
|
||||
return self._call_minimax(api_key, model, base_url, prompt, max_tokens, response_format)
|
||||
elif provider == 'zhipu':
|
||||
return self._call_zhipu(api_key, model, base_url, prompt, max_tokens, response_format)
|
||||
elif provider == 'alibaba':
|
||||
return self._call_alibaba(api_key, model, base_url, prompt, max_tokens, response_format)
|
||||
elif provider == 'openai':
|
||||
return self._call_openai(api_key, model, base_url, prompt, max_tokens, response_format)
|
||||
else:
|
||||
raise ValueError(f"不支持的 provider: {provider}")
|
||||
|
||||
def _call_minimax(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str:
|
||||
"""调用 MiniMax API (Anthropic Messages 兼容接口)"""
|
||||
url = base_url or "https://api.minimaxi.com/anthropic/v1/messages"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
}
|
||||
|
||||
if response_format == 'json':
|
||||
payload['messages'][0]['content'] = prompt + "\n\n请直接返回 JSON 格式，不要有其他文字。"
|
||||
|
||||
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
# MiniMax 返回的是 Anthropic 格式，content 是一个数组
|
||||
# 需要找到 type='text' 的元素
|
||||
for item in data.get('content', []):
|
||||
if item.get('type') == 'text':
|
||||
return item.get('text', '')
|
||||
|
||||
# 如果没有找到 text 类型，返回第一个元素的值（兼容旧格式）
|
||||
return data.get('content', [{}])[0].get('text', '')
|
||||
|
||||
def _call_zhipu(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str:
|
||||
"""调用智谱 AI API (OpenAI 兼容接口)"""
|
||||
# 确保 base_url 包含 /chat/completions
|
||||
if base_url and not base_url.endswith('/chat/completions'):
|
||||
url = f"{base_url}/chat/completions"
|
||||
else:
|
||||
url = base_url or "https://open.bigmodel.cn/api/paas/v4/chat/completions"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['choices'][0]['message']['content']
|
||||
|
||||
def _call_alibaba(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str:
|
||||
"""调用阿里云通义千问 API (OpenAI 兼容接口)"""
|
||||
url = f"{base_url}/chat/completions"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['choices'][0]['message']['content']
|
||||
|
||||
def _call_openai(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str:
|
||||
"""调用 OpenAI API"""
|
||||
url = base_url or "https://api.openai.com/v1/chat/completions"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"model": model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['choices'][0]['message']['content']
|
||||
|
||||
def quick_generate(self, prompt: str, max_tokens: int = 100) -> str:
|
||||
"""快速生成（用于轻量扫描）"""
|
||||
success, response = self._call_api_with_fallback(prompt, max_tokens, 'text')
|
||||
return response
|
||||
|
||||
def generate_json(self, prompt: str) -> Dict[str, Any]:
|
||||
"""生成JSON格式响应"""
|
||||
success, response = self._call_api_with_fallback(prompt, 1000, 'json')
|
||||
try:
|
||||
return json.loads(response)
|
||||
except:
|
||||
return self._parse_json_from_text(response)
|
||||
|
||||
def _deep_analyze(self, content_text: str, content_type: str, duration: Optional[int] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
C5.3: 深度分析入口，根据内容时长决定是否分段分析
|
||||
|
||||
Args:
|
||||
content_text: 内容文本
|
||||
content_type: 内容类型
|
||||
duration: 视频时长（秒）
|
||||
|
||||
Returns:
|
||||
分析结果字典
|
||||
"""
|
||||
duration = duration or 0
|
||||
|
||||
print(f"[AIClient] 深度分析: type={content_type}, duration={duration}s")
|
||||
|
||||
# 长视频（>=30分钟）且是视频类型，使用分段分析
|
||||
if duration >= 1800 and content_type == "video":
|
||||
print(f"[AIClient] 使用分段分析策略")
|
||||
return self._analyze_in_chunks(content_text, duration)
|
||||
|
||||
# 短视频直接分析
|
||||
print(f"[AIClient] 使用直接分析策略")
|
||||
return self._analyze_direct(content_text)
|
||||
|
||||
def _analyze_in_chunks(self, content_text: str, duration: int) -> Dict[str, Any]:
|
||||
"""
|
||||
C5.3: 分段分析长视频
|
||||
|
||||
Args:
|
||||
content_text: SRT字幕文本
|
||||
duration: 视频时长（秒）
|
||||
|
||||
Returns:
|
||||
合并后的分析结果
|
||||
"""
|
||||
# 1. 获取分段策略
|
||||
strategy = get_chunking_strategy(duration)
|
||||
print(f"[AIClient] 分段策略: {strategy['description']}")
|
||||
|
||||
# 2. 执行分片
|
||||
chunks = chunk_content(content_text, duration)
|
||||
print(f"[AIClient] 内容已分为 {len(chunks)} 段")
|
||||
|
||||
# 3. 逐段分析
|
||||
chunk_summaries = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
print(f"[AIClient] 分析第 {i+1}/{len(chunks)} 段...")
|
||||
summary = self._analyze_chunk(chunk.text, chunk.index)
|
||||
|
||||
# 处理时间格式（可能是float秒数或字符串）
|
||||
start_time = chunk.start_time
|
||||
end_time = chunk.end_time
|
||||
if isinstance(start_time, (int, float)):
|
||||
start_time = self._format_time(start_time)
|
||||
if isinstance(end_time, (int, float)):
|
||||
end_time = self._format_time(end_time)
|
||||
|
||||
chunk_summaries.append({
|
||||
'index': chunk.index,
|
||||
'start_time': start_time,
|
||||
'end_time': end_time,
|
||||
'summary': summary
|
||||
})
|
||||
|
||||
# 4. 合并所有段摘要
|
||||
print(f"[AIClient] 合并 {len(chunk_summaries)} 段分析结果")
|
||||
final_result = self._merge_summaries(chunk_summaries, "video")
|
||||
|
||||
# 5. 构建完整时间轴
|
||||
final_result['timeline'] = self._build_timeline_from_chunks(chunk_summaries)
|
||||
|
||||
return final_result
|
||||
|
||||
def _analyze_chunk(self, chunk_text: str, chunk_index: int) -> str:
|
||||
"""
|
||||
C5.3: 分析单个内容分片
|
||||
|
||||
Args:
|
||||
chunk_text: 分片文本
|
||||
chunk_index: 分片索引
|
||||
|
||||
Returns:
|
||||
分片摘要（限制500字以内）
|
||||
"""
|
||||
# 截断文本避免过长
|
||||
text_sample = chunk_text[:3000] if len(chunk_text) > 3000 else chunk_text
|
||||
|
||||
prompt = f"""请对以下视频片段进行简要分析（限制在300字以内）：
|
||||
|
||||
片段内容：
|
||||
{text_sample}
|
||||
|
||||
请提供：
|
||||
1. 这段的核心主题（50字以内）
|
||||
2. 关键要点（3-5条）
|
||||
3. 重要细节或引用
|
||||
|
||||
请用中文回答，控制在300字以内。"""
|
||||
|
||||
try:
|
||||
summary = self.quick_generate(prompt, max_tokens=500)
|
||||
# 确保不超过500字
|
||||
if len(summary) > 500:
|
||||
summary = summary[:497] + "..."
|
||||
return summary
|
||||
except Exception as e:
|
||||
print(f"[AIClient] 分片 {chunk_index} 分析失败: {e}")
|
||||
return f"第{chunk_index+1}段分析失败"
|
||||
|
||||
def _merge_summaries(self, chunk_summaries: List[Dict], content_type: str) -> Dict[str, Any]:
|
||||
"""
|
||||
C5.3: 合并所有段摘要，生成完整分析报告
|
||||
|
||||
Args:
|
||||
chunk_summaries: 各段摘要列表
|
||||
content_type: 内容类型
|
||||
|
||||
Returns:
|
||||
完整分析结果
|
||||
"""
|
||||
# 构建合并提示
|
||||
summaries_text = "\n\n".join([
|
||||
f"=== 第{s['index']+1}段 ({s['start_time']}-{s['end_time']}) ===\n{s['summary']}"
|
||||
for s in chunk_summaries
|
||||
])
|
||||
|
||||
prompt = f"""基于以下各段分析，生成完整的视频分析报告：
|
||||
|
||||
{summaries_text}
|
||||
|
||||
请提供以下格式的JSON：
|
||||
{{
|
||||
"summary": "整体摘要（200字以内）",
|
||||
"topics": ["主题1", "主题2", "主题3", "主题4"],
|
||||
"insights": ["深入见解1", "深入见解2", "深入见解3"],
|
||||
"actions": ["建议行动1", "建议行动2"],
|
||||
"full_analysis": "详细分析（500字以内）"
|
||||
}}"""
|
||||
|
||||
try:
|
||||
result = self.generate_json(prompt)
|
||||
|
||||
# 确保必要字段存在
|
||||
if 'topics' not in result:
|
||||
result['topics'] = ["主题A", "主题B", "主题C"]
|
||||
if 'insights' not in result:
|
||||
result['insights'] = ["重要见解1", "重要见解2"]
|
||||
if 'actions' not in result:
|
||||
result['actions'] = []
|
||||
if 'summary' not in result:
|
||||
result['summary'] = "长视频分析完成"
|
||||
if 'full_analysis' not in result:
|
||||
result['full_analysis'] = summaries_text[:500]
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"[AIClient] 合并摘要失败: {e}")
|
||||
# 返回基础合并结果
|
||||
return {
|
||||
'summary': f'视频共分为{len(chunk_summaries)}个部分进行分析',
|
||||
'topics': [f"主题{i+1}" for i in range(min(4, len(chunk_summaries)))],
|
||||
'insights': ['从分段分析中综合得出'],
|
||||
'actions': [],
|
||||
'full_analysis': summaries_text[:800]
|
||||
}
|
||||
|
||||
def _analyze_direct(self, content_text: str) -> Dict[str, Any]:
|
||||
"""直接分析（不分段）"""
|
||||
# 截断文本
|
||||
text_sample = content_text[:5000] if len(content_text) > 5000 else content_text
|
||||
|
||||
prompt = f"""请分析以下内容，提供JSON格式结果：
|
||||
|
||||
{text_sample}
|
||||
|
||||
返回格式：
|
||||
{{
|
||||
"summary": "整体摘要（150字以内）",
|
||||
"topics": ["主题1", "主题2", "主题3"],
|
||||
"insights": ["见解1", "见解2"],
|
||||
"actions": ["行动建议1"],
|
||||
"timeline": [
|
||||
{{"start_time": "00:00", "end_time": "05:00", "title": "第一部分", "summary": "..."}}
|
||||
]
|
||||
}}"""
|
||||
|
||||
try:
|
||||
return self.generate_json(prompt)
|
||||
except Exception as e:
|
||||
print(f"[AIClient] 直接分析失败: {e}")
|
||||
return {
|
||||
'summary': '分析失败',
|
||||
'topics': ['无法解析'],
|
||||
'insights': ['请检查内容格式'],
|
||||
'actions': [],
|
||||
'timeline': []
|
||||
}
|
||||
|
||||
def _format_time(self, seconds: float) -> str:
|
||||
"""格式化秒数为时间字符串"""
|
||||
hours = int(seconds // 3600)
|
||||
minutes = int((seconds % 3600) // 60)
|
||||
secs = int(seconds % 60)
|
||||
if hours > 0:
|
||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
||||
return f"{minutes:02d}:{secs:02d}"
|
||||
|
||||
def _build_timeline_from_chunks(self, chunk_summaries: List[Dict]) -> List[Dict[str, Any]]:
|
||||
"""从分片摘要构建时间轴"""
|
||||
timeline = []
|
||||
for summary in chunk_summaries:
|
||||
timeline.append({
|
||||
'index': summary['index'],
|
||||
'start_time': summary['start_time'],
|
||||
'end_time': summary['end_time'],
|
||||
'title': f"第{summary['index']+1}部分",
|
||||
'summary': summary['summary'][:200] + "..." if len(summary['summary']) > 200 else summary['summary']
|
||||
})
|
||||
return timeline
|
||||
|
||||
def _mock_generate(self, prompt: str, max_tokens: int) -> str:
|
||||
"""模拟AI生成（实际实现应调用真实API）"""
|
||||
# 根据prompt内容返回模拟响应
|
||||
prompt_lower = prompt.lower()
|
||||
|
||||
if 'json' in prompt_lower or 'json格式' in prompt:
|
||||
if '主题' in prompt or 'topic' in prompt_lower:
|
||||
return json.dumps({
|
||||
'summary': '这是一个关于人工智能和技术发展的内容，涵盖了多个重要主题。',
|
||||
'topics': ['人工智能', '机器学习', '未来发展', '技术趋势'],
|
||||
'insights': [
|
||||
'AI正在改变我们的工作方式',
|
||||
'机器学习应用越来越广泛',
|
||||
'未来技术发展趋势值得关注'
|
||||
],
|
||||
'actions': ['深入研究相关技术', '关注行业动态'],
|
||||
'full_analysis': '详细分析内容...'
|
||||
}, ensure_ascii=False)
|
||||
return json.dumps({'result': '模拟JSON响应'})
|
||||
|
||||
if '摘要' in prompt or 'summary' in prompt_lower:
|
||||
return '这是一段关于人工智能发展的重要讨论，涵盖了技术趋势和未来应用。'
|
||||
|
||||
if '关键词' in prompt or 'keyword' in prompt_lower:
|
||||
return '人工智能, 机器学习, 深度学习, 神经网络, 自然语言处理'
|
||||
|
||||
return '模拟AI响应内容'
|
||||
|
||||
def _parse_json_from_text(self, text: str) -> Dict[str, Any]:
|
||||
"""从文本中解析JSON"""
|
||||
# 尝试提取JSON块
|
||||
import re
|
||||
json_match = re.search(r'\{[\s\S]*\}', text)
|
||||
if json_match:
|
||||
try:
|
||||
return json.loads(json_match.group())
|
||||
except:
|
||||
pass
|
||||
return {'raw_response': text}
|
||||
|
||||
|
||||
class MultiAIClient:
|
||||
"""
|
||||
多AI客户端 - 支持多种AI提供商的客户端
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config()
|
||||
self.primary_client = AIClient()
|
||||
self.clients = {
|
||||
'openai': self.primary_client,
|
||||
'local': AIClient(), # 可以配置为本地模型
|
||||
}
|
||||
|
||||
def _deep_analyze(self, content_text: str, content_type: str, duration: Optional[int] = None) -> Dict[str, Any]:
|
||||
"""使用主客户端进行深度分析"""
|
||||
return self.primary_client._deep_analyze(content_text, content_type, duration)
|
||||
|
||||
def quick_generate(self, prompt: str, max_tokens: int = 100, provider: str = None) -> str:
|
||||
"""快速生成"""
|
||||
client = self.clients.get(provider, self.primary_client)
|
||||
return client.quick_generate(prompt, max_tokens)
|
||||
@@ -0,0 +1,771 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deep Analyzer - AI 深度内容分析工具
|
||||
|
||||
支持根据内容时长选择不同分析策略：
|
||||
- 短视频 (<10分钟): 轻量分析
|
||||
- 长视频 (>=10分钟): 详细时间轴分析
|
||||
|
||||
Usage:
|
||||
python deep_analyzer.py --metadata meta.json --content content.json --output analysis.md [--type video]
|
||||
python deep_analyzer.py --help
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List, Optional, Tuple
|
||||
from dataclasses import dataclass, field, asdict
|
||||
|
||||
|
||||
# ============== 数据模型 ==============
|
||||
|
||||
@dataclass
|
||||
class TimelineEntry:
|
||||
"""时间轴条目"""
|
||||
index: int = 0
|
||||
start_time: str = ""
|
||||
end_time: str = ""
|
||||
title: str = ""
|
||||
summary: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnalysisResult:
|
||||
"""分析结果"""
|
||||
summary: str = ""
|
||||
main_topics: List[str] = field(default_factory=list)
|
||||
key_insights: List[str] = field(default_factory=list)
|
||||
action_items: List[str] = field(default_factory=list)
|
||||
recommended_tags: List[str] = field(default_factory=list)
|
||||
full_analysis: str = ""
|
||||
timeline: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
|
||||
# ============== AI 客户端 ==============
|
||||
|
||||
class SimpleAIClient:
|
||||
"""简化的 AI 客户端 - 支持多提供商 fallback"""
|
||||
|
||||
# 时长阈值（秒）
|
||||
SHORT_VIDEO_THRESHOLD = 600 # 10分钟
|
||||
LONG_VIDEO_THRESHOLD = 1800 # 30分钟
|
||||
|
||||
def __init__(self, api_key: str = None, provider: str = "mock", model: str = "gpt-4"):
|
||||
self.api_key = api_key or os.environ.get("AI_API_KEY", "")
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
self.base_url = os.environ.get("AI_BASE_URL", "")
|
||||
|
||||
def generate(self, prompt: str, max_tokens: int = 1000) -> str:
|
||||
"""生成文本"""
|
||||
if not self.api_key or self.provider == "mock":
|
||||
return self._mock_generate(prompt, max_tokens)
|
||||
|
||||
try:
|
||||
if self.provider == "openai":
|
||||
return self._call_openai(prompt, max_tokens)
|
||||
elif self.provider == "zhipu":
|
||||
return self._call_zhipu(prompt, max_tokens)
|
||||
elif self.provider == "minimax":
|
||||
return self._call_minimax(prompt, max_tokens)
|
||||
else:
|
||||
return self._mock_generate(prompt, max_tokens)
|
||||
except Exception as e:
|
||||
print(f"[AI Client] 调用失败: {e}")
|
||||
return self._mock_generate(prompt, max_tokens)
|
||||
|
||||
def generate_json(self, prompt: str) -> Dict[str, Any]:
|
||||
"""生成 JSON 格式响应"""
|
||||
response = self.generate(prompt, max_tokens=2000)
|
||||
try:
|
||||
return json.loads(response)
|
||||
except json.JSONDecodeError:
|
||||
# 尝试从文本中提取 JSON
|
||||
return self._parse_json_from_text(response)
|
||||
|
||||
def _call_openai(self, prompt: str, max_tokens: int) -> str:
|
||||
"""调用 OpenAI API"""
|
||||
import requests
|
||||
url = f"{self.base_url}/chat/completions" if self.base_url else "https://api.openai.com/v1/chat/completions"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [{"role": "user", "content": prompt}]
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.json()['choices'][0]['message']['content']
|
||||
|
||||
def _call_zhipu(self, prompt: str, max_tokens: int) -> str:
|
||||
"""调用智谱 AI API"""
|
||||
import requests
|
||||
base_url = self.base_url or "https://open.bigmodel.cn/api/paas/v4"
|
||||
url = f"{base_url}/chat/completions"
|
||||
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"}
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "user", "content": prompt}]
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.json()['choices'][0]['message']['content']
|
||||
|
||||
def _call_minimax(self, prompt: str, max_tokens: int) -> str:
|
||||
"""调用 MiniMax API"""
|
||||
import requests
|
||||
url = self.base_url or "https://api.minimaxi.com/anthropic/v1/messages"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"max_tokens": max_tokens,
|
||||
"messages": [{"role": "user", "content": prompt}]
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers, json=payload, timeout=30)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
for item in data.get('content', []):
|
||||
if item.get('type') == 'text':
|
||||
return item.get('text', '')
|
||||
return str(data)
|
||||
|
||||
def _mock_generate(self, prompt: str, max_tokens: int) -> str:
|
||||
"""模拟 AI 生成"""
|
||||
prompt_lower = prompt.lower()
|
||||
|
||||
if 'json' in prompt_lower or '请返回' in prompt:
|
||||
# 返回模拟 JSON 响应
|
||||
if '主题' in prompt or 'topic' in prompt_lower:
|
||||
return json.dumps({
|
||||
'summary': '这是一个模拟的AI分析结果，包含内容的主要主题和关键见解。',
|
||||
'topics': ['主题一', '主题二', '主题三', '主题四'],
|
||||
'insights': [
|
||||
'AI正在深刻改变内容消费方式',
|
||||
'个性化推荐提升了用户体验',
|
||||
'技术发展带来新的可能性'
|
||||
],
|
||||
'actions': ['深入研究相关技术', '关注行业动态'],
|
||||
'full_analysis': '详细分析内容基于模拟AI生成。'
|
||||
}, ensure_ascii=False)
|
||||
if '标签' in prompt or 'tag' in prompt_lower:
|
||||
return json.dumps({'tags': ['技术', 'AI', '视频分析', '深度学习']}, ensure_ascii=False)
|
||||
return json.dumps({'result': '模拟响应'}, ensure_ascii=False)
|
||||
|
||||
if '摘要' in prompt or 'summary' in prompt_lower:
|
||||
return '这是一段关于内容深度分析的重要讨论，涵盖了多个关键主题和见解。'
|
||||
|
||||
return '模拟AI响应内容'
|
||||
|
||||
def _parse_json_from_text(self, text: str) -> Dict[str, Any]:
|
||||
"""从文本中解析 JSON"""
|
||||
json_match = re.search(r'\{[\s\S]*\}', text)
|
||||
if json_match:
|
||||
try:
|
||||
return json.loads(json_match.group())
|
||||
except:
|
||||
pass
|
||||
return {'raw_response': text}
|
||||
|
||||
|
||||
# ============== 深度分析器 ==============
|
||||
|
||||
class DeepAnalyzer:
|
||||
"""深度内容分析器"""
|
||||
|
||||
SHORT_VIDEO_THRESHOLD = 600 # 10分钟
|
||||
LONG_VIDEO_THRESHOLD = 1800 # 30分钟
|
||||
|
||||
def __init__(self, ai_client: SimpleAIClient = None):
|
||||
self.ai_client = ai_client or SimpleAIClient()
|
||||
|
||||
def analyze(self, metadata: Dict[str, Any], content: Dict[str, Any],
|
||||
content_type: str = "video") -> AnalysisResult:
|
||||
"""
|
||||
分析内容
|
||||
|
||||
Args:
|
||||
metadata: 元数据字典
|
||||
content: 内容数据字典（包含 subtitle, description 等）
|
||||
content_type: 内容类型
|
||||
|
||||
Returns:
|
||||
AnalysisResult 分析结果
|
||||
"""
|
||||
title = metadata.get('title', '未知')
|
||||
duration = metadata.get('duration', 0)
|
||||
subtitle = content.get('subtitle', '')
|
||||
description = content.get('description', '')
|
||||
|
||||
print(f"[DeepAnalyzer] 开始分析: {title}")
|
||||
print(f"[DeepAnalyzer] 类型: {content_type}, 时长: {duration}秒")
|
||||
print(f"[DeepAnalyzer] 字幕存在: {bool(subtitle and subtitle.strip())}")
|
||||
|
||||
has_subtitle = bool(subtitle and subtitle.strip())
|
||||
|
||||
if has_subtitle:
|
||||
if duration < self.SHORT_VIDEO_THRESHOLD:
|
||||
print(f"[DeepAnalyzer] 使用轻量分析策略 (时长 < 10分钟)")
|
||||
result = self._analyze_short_with_subtitle(title, subtitle, duration)
|
||||
else:
|
||||
print(f"[DeepAnalyzer] 使用详细时间轴分析策略 (时长 >= 10分钟)")
|
||||
result = self._analyze_long_with_subtitle(title, subtitle, duration, content_type)
|
||||
else:
|
||||
print(f"[DeepAnalyzer] 降级处理（无字幕）")
|
||||
result = self._analyze_fallback(title, description, metadata)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_short_with_subtitle(self, title: str, subtitle: str, duration: int) -> AnalysisResult:
|
||||
"""有字幕的短视频轻量分析"""
|
||||
result = AnalysisResult()
|
||||
text_sample = self._get_text_sample(subtitle, max_chars=5000)
|
||||
|
||||
analysis = self._analyze_with_ai_short_text(text_sample, title, duration)
|
||||
result.summary = analysis.get('summary', title)
|
||||
result.main_topics = analysis.get('topics', [])
|
||||
result.key_insights = analysis.get('insights', [])
|
||||
result.action_items = analysis.get('actions', [])
|
||||
result.timeline = self._generate_simple_timeline_from_duration(title, duration)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_long_with_subtitle(self, title: str, subtitle: str, duration: int,
|
||||
content_type: str) -> AnalysisResult:
|
||||
"""有字幕的长视频详细分析"""
|
||||
result = AnalysisResult()
|
||||
|
||||
if duration >= self.LONG_VIDEO_THRESHOLD and content_type == "video":
|
||||
# 使用分段分析
|
||||
result = self._analyze_in_chunks(title, subtitle, duration)
|
||||
else:
|
||||
# 直接分析
|
||||
text_sample = self._get_text_sample(subtitle, max_chars=5000)
|
||||
analysis = self._analyze_with_ai_short_text(text_sample, title, duration)
|
||||
result.summary = analysis.get('summary', title)
|
||||
result.main_topics = analysis.get('topics', [])
|
||||
result.key_insights = analysis.get('insights', [])
|
||||
result.action_items = analysis.get('actions', [])
|
||||
result.full_analysis = analysis.get('full_analysis', '')
|
||||
result.timeline = self._generate_detailed_timeline(title, duration)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_in_chunks(self, title: str, subtitle: str, duration: int) -> AnalysisResult:
|
||||
"""分段分析长视频"""
|
||||
result = AnalysisResult()
|
||||
|
||||
# 确定分段数
|
||||
if duration < 3600: # <60分钟
|
||||
num_segments = 3
|
||||
elif duration < 7200: # 60-120分钟
|
||||
num_segments = 4
|
||||
else: # >120分钟
|
||||
num_segments = 6
|
||||
|
||||
# 生成分段摘要
|
||||
segment_duration = duration // num_segments
|
||||
chunk_summaries = []
|
||||
|
||||
# 简单按字符数分割字幕
|
||||
subtitle_chunks = self._chunk_subtitle(subtitle, num_segments)
|
||||
|
||||
for i, chunk_text in enumerate(subtitle_chunks):
|
||||
start_sec = i * segment_duration
|
||||
end_sec = min((i + 1) * segment_duration, duration)
|
||||
start_time = self._format_time(start_sec)
|
||||
end_time = self._format_time(end_sec)
|
||||
|
||||
# AI 分析每段
|
||||
chunk_summary = self._analyze_chunk(chunk_text, i + 1)
|
||||
chunk_summaries.append({
|
||||
'index': i,
|
||||
'start_time': start_time,
|
||||
'end_time': end_time,
|
||||
'summary': chunk_summary
|
||||
})
|
||||
|
||||
# 合并结果
|
||||
result = self._merge_summaries(chunk_summaries, title)
|
||||
result.timeline = self._build_timeline_from_chunks(chunk_summaries)
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_chunk(self, chunk_text: str, chunk_index: int) -> str:
|
||||
"""分析单个分片"""
|
||||
text_sample = self._get_text_sample(chunk_text, max_chars=3000)
|
||||
|
||||
prompt = f"""请对以下视频片段进行简要分析（限制在300字以内）：
|
||||
|
||||
片段内容：
|
||||
{text_sample}
|
||||
|
||||
请用中文简要回答，包括核心主题和关键要点。"""
|
||||
|
||||
try:
|
||||
return self.ai_client.generate(prompt, max_tokens=500)
|
||||
except Exception as e:
|
||||
print(f"[DeepAnalyzer] 分片 {chunk_index} 分析失败: {e}")
|
||||
return f"第{chunk_index}段内容分析"
|
||||
|
||||
def _merge_summaries(self, chunk_summaries: List[Dict], title: str) -> AnalysisResult:
|
||||
"""合并分段摘要"""
|
||||
summaries_text = "\n\n".join([
|
||||
f"=== 第{s['index']+1}段 ({s['start_time']}-{s['end_time']}) ===\n{s['summary']}"
|
||||
for s in chunk_summaries
|
||||
])
|
||||
|
||||
prompt = f"""基于以下视频各段分析，生成完整的分析报告：
|
||||
|
||||
标题：{title}
|
||||
|
||||
各段内容：
|
||||
{summaries_text}
|
||||
|
||||
请返回JSON格式：
|
||||
{{
|
||||
"summary": "整体摘要（150字以内）",
|
||||
"topics": ["主题1", "主题2", "主题3", "主题4"],
|
||||
"insights": ["见解1", "见解2", "见解3"],
|
||||
"actions": ["建议1", "建议2"],
|
||||
"full_analysis": "详细分析（500字以内）"
|
||||
}}"""
|
||||
|
||||
try:
|
||||
analysis = self.ai_client.generate_json(prompt)
|
||||
result = AnalysisResult()
|
||||
result.summary = analysis.get('summary', title)
|
||||
result.main_topics = analysis.get('topics', [])
|
||||
result.key_insights = analysis.get('insights', [])
|
||||
result.action_items = analysis.get('actions', [])
|
||||
result.full_analysis = analysis.get('full_analysis', summaries_text)
|
||||
return result
|
||||
except Exception as e:
|
||||
print(f"[DeepAnalyzer] 合并摘要失败: {e}")
|
||||
return AnalysisResult(
|
||||
summary=f"共{len(chunk_summaries)}段分析完成",
|
||||
main_topics=[f"主题{i+1}" for i in range(min(4, len(chunk_summaries)))],
|
||||
key_insights=["综合各段分析得出"],
|
||||
full_analysis=summaries_text[:500]
|
||||
)
|
||||
|
||||
def _analyze_fallback(self, title: str, description: str, metadata: Dict) -> AnalysisResult:
|
||||
"""无字幕降级处理"""
|
||||
result = AnalysisResult()
|
||||
|
||||
if description and description.strip():
|
||||
summary = description[:200] + "..." if len(description) > 200 else description
|
||||
else:
|
||||
summary = f"无字幕内容，标题：{title}"
|
||||
|
||||
# 提取标签
|
||||
tags = ["未分类", "待处理"]
|
||||
author = metadata.get('author', '')
|
||||
source = metadata.get('source', '')
|
||||
if author:
|
||||
tags.append(f"作者:{author}")
|
||||
if source:
|
||||
tags.append(f"来源:{source}")
|
||||
|
||||
result.summary = summary
|
||||
result.main_topics = tags[:3]
|
||||
result.recommended_tags = tags
|
||||
result.key_insights = ["无字幕内容，仅能从描述获取基本信息"]
|
||||
result.action_items = ["建议补充字幕以便深度分析"]
|
||||
|
||||
return result
|
||||
|
||||
def _analyze_with_ai_short_text(self, text: str, title: str, duration: int) -> Dict[str, Any]:
|
||||
"""AI 短视频分析"""
|
||||
prompt = f"""请分析以下视频内容，以JSON格式返回：
|
||||
|
||||
标题：{title}
|
||||
时长：{self._format_time(duration)}
|
||||
|
||||
字幕内容：
|
||||
{text[:3000]}
|
||||
|
||||
请返回以下格式JSON：
|
||||
{{
|
||||
"summary": "整体摘要（100字以内）",
|
||||
"topics": ["主题1", "主题2", "主题3"],
|
||||
"insights": ["见解1", "见解2"],
|
||||
"actions": ["建议1"]
|
||||
}}"""
|
||||
|
||||
try:
|
||||
return self.ai_client.generate_json(prompt)
|
||||
except Exception as e:
|
||||
print(f"[DeepAnalyzer] AI分析失败: {e}")
|
||||
return {
|
||||
"summary": title,
|
||||
"topics": ["主题1", "主题2", "主题3"],
|
||||
"insights": ["无法获取深入见解"],
|
||||
"actions": []
|
||||
}
|
||||
|
||||
def _get_text_sample(self, text: str, max_chars: int) -> str:
|
||||
"""获取文本样本（头尾采样）"""
|
||||
if len(text) <= max_chars:
|
||||
return text
|
||||
|
||||
part_size = max_chars // 3
|
||||
return (
|
||||
text[:part_size] +
|
||||
"\n\n[...省略中间部分...]\n\n" +
|
||||
text[-part_size:]
|
||||
)
|
||||
|
||||
def _chunk_subtitle(self, subtitle: str, num_chunks: int) -> List[str]:
|
||||
"""简单分割字幕"""
|
||||
if not subtitle:
|
||||
return [""] * num_chunks
|
||||
|
||||
chunk_size = len(subtitle) // num_chunks
|
||||
chunks = []
|
||||
|
||||
for i in range(num_chunks):
|
||||
start = i * chunk_size
|
||||
end = start + chunk_size if i < num_chunks - 1 else len(subtitle)
|
||||
chunks.append(subtitle[start:end])
|
||||
|
||||
return chunks
|
||||
|
||||
def _generate_simple_timeline_from_duration(self, title: str, duration: int) -> List[Dict]:
|
||||
"""生成简单时间轴"""
|
||||
segments = 2 if duration < 300 else 3
|
||||
seg_duration = duration // segments
|
||||
|
||||
timeline = []
|
||||
for i in range(segments):
|
||||
start = i * seg_duration
|
||||
end = min((i + 1) * seg_duration, duration)
|
||||
timeline.append({
|
||||
'index': i + 1,
|
||||
'start_time': self._format_time(start),
|
||||
'end_time': self._format_time(end),
|
||||
'title': f"段落 {i+1}",
|
||||
'summary': f"从{self._format_time(start)}到{self._format_time(end)}的内容"
|
||||
})
|
||||
return timeline
|
||||
|
||||
def _generate_detailed_timeline(self, title: str, duration: int) -> List[Dict]:
|
||||
"""生成详细时间轴"""
|
||||
return self._generate_simple_timeline_from_duration(title, duration)
|
||||
|
||||
def _build_timeline_from_chunks(self, chunk_summaries: List[Dict]) -> List[Dict]:
|
||||
"""从分段摘要构建时间轴"""
|
||||
timeline = []
|
||||
for s in chunk_summaries:
|
||||
timeline.append({
|
||||
'index': s['index'] + 1,
|
||||
'start_time': s['start_time'],
|
||||
'end_time': s['end_time'],
|
||||
'title': f"第{s['index']+1}部分",
|
||||
'summary': s['summary'][:200] + "..." if len(s['summary']) > 200 else s['summary']
|
||||
})
|
||||
return timeline
|
||||
|
||||
def _format_time(self, seconds: int) -> str:
|
||||
"""格式化秒数为时间字符串"""
|
||||
h = seconds // 3600
|
||||
m = (seconds % 3600) // 60
|
||||
s = seconds % 60
|
||||
if h > 0:
|
||||
return f"{h}:{m:02d}:{s:02d}"
|
||||
return f"{m}:{s:02d}"
|
||||
|
||||
|
||||
# ============== 模板系统 ==============
|
||||
|
||||
class TemplateLoader:
|
||||
"""模板加载器"""
|
||||
|
||||
TEMPLATES = {
|
||||
"video": """# {{title}}
|
||||
|
||||
## 基本信息
|
||||
- **时长**: {{duration}}
|
||||
- **作者**: {{author}}
|
||||
- **来源**: {{source}}
|
||||
- **分析时间**: {{analyze_time}}
|
||||
|
||||
## 摘要
|
||||
{{summary}}
|
||||
|
||||
## 主要主题
|
||||
{% for topic in topics %}
|
||||
- {{topic}}
|
||||
{% endfor %}
|
||||
|
||||
## 关键见解
|
||||
{% for insight in insights %}
|
||||
- {{insight}}
|
||||
{% endfor %}
|
||||
|
||||
## 建议行动
|
||||
{% for action in actions %}
|
||||
- [ ] {{action}}
|
||||
{% endfor %}
|
||||
|
||||
{% if timeline %}
|
||||
## 时间轴
|
||||
{% for entry in timeline %}
|
||||
### {{entry.index}}. {{entry.start_time}} - {{entry.end_time}}
|
||||
{{entry.title}}: {{entry.summary}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if tags %}
|
||||
## 标签
|
||||
{% for tag in tags %}
|
||||
- {{tag}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
""",
|
||||
"article": """# {{title}}
|
||||
|
||||
## 基本信息
|
||||
- **作者**: {{author}}
|
||||
- **来源**: {{source}}
|
||||
- **分析时间**: {{analyze_time}}
|
||||
|
||||
## 摘要
|
||||
{{summary}}
|
||||
|
||||
## 主要主题
|
||||
{% for topic in topics %}
|
||||
- {{topic}}
|
||||
{% endfor %}
|
||||
|
||||
## 关键见解
|
||||
{% for insight in insights %}
|
||||
- {{insight}}
|
||||
{% endfor %}
|
||||
|
||||
## 建议行动
|
||||
{% for action in actions %}
|
||||
- [ ] {{action}}
|
||||
{% endfor %}
|
||||
|
||||
{% if tags %}
|
||||
## 标签
|
||||
{% for tag in tags %}
|
||||
- {{tag}}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
"""
|
||||
}
|
||||
|
||||
def load_template(self, template_type: str = "video") -> str:
|
||||
"""加载模板"""
|
||||
# 优先检查环境变量
|
||||
template_path = os.environ.get(f"TEMPLATE_{template_type.upper()}", "")
|
||||
if template_path and os.path.exists(template_path):
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
|
||||
return self.TEMPLATES.get(template_type, self.TEMPLATES["video"])
|
||||
|
||||
|
||||
class TemplateRenderer:
|
||||
"""简单模板渲染器"""
|
||||
|
||||
def render(self, template: str, variables: Dict[str, Any]) -> str:
|
||||
"""渲染模板"""
|
||||
result = template
|
||||
|
||||
# 替换变量
|
||||
for key, value in variables.items():
|
||||
placeholder = f"{{{{{key}}}}}"
|
||||
result = result.replace(placeholder, str(value))
|
||||
|
||||
# 渲染 for 循环 (简化版本)
|
||||
# 匹配 {% for item in list %} ... {% endfor %}
|
||||
pattern = r'\{%\s*for\s+(\w+)\s+in\s+(\w+)\s*%\}(.*?)\{%\s*endfor\s*%\}'
|
||||
|
||||
def replace_loop(match):
|
||||
item_name = match.group(1)
|
||||
list_name = match.group(2)
|
||||
content = match.group(3)
|
||||
|
||||
items = variables.get(list_name, [])
|
||||
rendered = []
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
# 字典项
|
||||
item_content = content
|
||||
for k, v in item.items():
|
||||
item_content = item_content.replace(f"{{{{{item_name}.{k}}}}}", str(v))
|
||||
rendered.append(item_content)
|
||||
else:
|
||||
# 简单项
|
||||
rendered.append(content.replace(f"{{{{{item_name}}}}}", str(item)))
|
||||
|
||||
return '\n'.join(rendered)
|
||||
|
||||
result = re.sub(pattern, replace_loop, result, flags=re.DOTALL)
|
||||
|
||||
# 移除空的 for 循环块
|
||||
result = re.sub(r'\{%\s*for\s+\w+\s+in\s+\w+\s*%\}\s*\{%\s*endfor\s*%\}', '', result)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ============== 工具函数 ==============
|
||||
|
||||
def load_json(file_path: str) -> Dict:
|
||||
"""加载 JSON 文件"""
|
||||
if not file_path or not os.path.exists(file_path):
|
||||
return {}
|
||||
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def format_duration(seconds: int) -> str:
|
||||
"""格式化时长"""
|
||||
h = seconds // 3600
|
||||
m = (seconds % 3600) // 60
|
||||
s = seconds % 60
|
||||
if h > 0:
|
||||
return f"{h}:{m:02d}:{s:02d}"
|
||||
return f"{m}:{s:02d}"
|
||||
|
||||
|
||||
def result_to_dict(result: AnalysisResult) -> Dict:
|
||||
"""将分析结果转换为字典"""
|
||||
return {
|
||||
"summary": result.summary,
|
||||
"main_topics": result.main_topics,
|
||||
"key_insights": result.key_insights,
|
||||
"action_items": result.action_items,
|
||||
"recommended_tags": result.recommended_tags,
|
||||
"full_analysis": result.full_analysis,
|
||||
"timeline": result.timeline
|
||||
}
|
||||
|
||||
|
||||
def generate_frontmatter(metadata: Dict, result: AnalysisResult) -> str:
|
||||
"""生成 Markdown frontmatter"""
|
||||
tags = result.recommended_tags or result.main_topics[:5]
|
||||
|
||||
fm = ["---", "title: \"" + metadata.get('title', '未命名') + "\"",
|
||||
f"date: {datetime.now().strftime('%Y-%m-%d')}",
|
||||
f"duration: {format_duration(metadata.get('duration', 0))}",
|
||||
f"author: {metadata.get('author', '未知')}"]
|
||||
|
||||
if tags:
|
||||
fm.append("tags: [" + ", ".join(f'"{t}"' for t in tags) + "]")
|
||||
|
||||
fm.append("---")
|
||||
return '\n'.join(fm)
|
||||
|
||||
|
||||
# ============== 主程序 ==============
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Deep Analyzer - AI 深度内容分析工具",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
python deep_analyzer.py --metadata meta.json --content content.json --output analysis.md
|
||||
python deep_analyzer.py --metadata meta.json --content content.json --type video --api-key xxx
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument("--metadata", "-m", required=True,
|
||||
help="元数据 JSON 文件路径")
|
||||
parser.add_argument("--content", "-c", required=True,
|
||||
help="内容 JSON 文件路径")
|
||||
parser.add_argument("--output", "-o", required=True,
|
||||
help="输出 Markdown 文件路径")
|
||||
parser.add_argument("--type", "-t", default="video", choices=["video", "article"],
|
||||
help="内容类型 (默认: video)")
|
||||
parser.add_argument("--api-key", "-k", default=None,
|
||||
help="AI API Key (或设置 AI_API_KEY 环境变量)")
|
||||
parser.add_argument("--provider", "-p", default="mock",
|
||||
choices=["openai", "zhipu", "minimax", "mock"],
|
||||
help="AI 提供商 (默认: mock)")
|
||||
parser.add_argument("--model", default="gpt-4",
|
||||
help="AI 模型 (默认: gpt-4)")
|
||||
parser.add_argument("--no-frontmatter", action="store_true",
|
||||
help="不生成 frontmatter")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 加载数据
|
||||
print(f"[Main] 加载元数据: {args.metadata}")
|
||||
metadata = load_json(args.metadata)
|
||||
|
||||
print(f"[Main] 加载内容: {args.content}")
|
||||
content = load_json(args.content)
|
||||
|
||||
# 初始化
|
||||
ai_client = SimpleAIClient(
|
||||
api_key=args.api_key,
|
||||
provider=args.provider,
|
||||
model=args.model
|
||||
)
|
||||
analyzer = DeepAnalyzer(ai_client)
|
||||
|
||||
# 执行分析
|
||||
print(f"[Main] 开始分析...")
|
||||
result = analyzer.analyze(metadata, content, args.type)
|
||||
|
||||
# 渲染模板
|
||||
template_loader = TemplateLoader()
|
||||
renderer = TemplateRenderer()
|
||||
|
||||
template = template_loader.load_template(args.type)
|
||||
|
||||
variables = {
|
||||
"title": metadata.get('title', '未命名'),
|
||||
"duration": format_duration(metadata.get('duration', 0)),
|
||||
"author": metadata.get('author', '未知'),
|
||||
"source": metadata.get('source', ''),
|
||||
"analyze_time": datetime.now().strftime('%Y-%m-%d %H:%M'),
|
||||
"summary": result.summary,
|
||||
"topics": result.main_topics,
|
||||
"insights": result.key_insights,
|
||||
"actions": result.action_items,
|
||||
"tags": result.recommended_tags,
|
||||
"timeline": result.timeline
|
||||
}
|
||||
|
||||
markdown_content = renderer.render(template, variables)
|
||||
|
||||
# 添加 frontmatter
|
||||
if not args.no_frontmatter:
|
||||
frontmatter = generate_frontmatter(metadata, result)
|
||||
markdown_content = frontmatter + "\n\n" + markdown_content
|
||||
|
||||
# 写入输出
|
||||
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(markdown_content)
|
||||
|
||||
print(f"[Main] 分析完成！输出: {args.output}")
|
||||
print(f"[Main] 摘要: {result.summary[:100]}...")
|
||||
print(f"[Main] 主题数: {len(result.main_topics)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
# KB Link Parser Skill
|
||||
|
||||
**kb-link-parser** - Extract and resolve IDs from various platform URLs.
|
||||
|
||||
## Features
|
||||
|
||||
- Multi-platform support: Bilibili, YouTube, Zhihu
|
||||
- Short link detection: b23.tv and other short links
|
||||
- Multiple content types: video, article, up主, playlist, collection
|
||||
- Batch processing from file
|
||||
- JSON output format
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Parse single URL
|
||||
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
|
||||
|
||||
# Batch processing
|
||||
python link-parser.py --batch urls.txt --output results.json
|
||||
|
||||
# Output format
|
||||
python link-parser.py --url "..." --format json
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"platform": "bilibili",
|
||||
"type": "video",
|
||||
"id": "BV1xx411c7mD"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
cp -r kb-link-parser/ ~/skills/
|
||||
```
|
||||
@@ -0,0 +1,387 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Link Parser - Extract and resolve IDs from various platform URLs.
|
||||
|
||||
Supports:
|
||||
- Bilibili (video, article, up主, collection, series)
|
||||
- YouTube (video, channel, playlist)
|
||||
- Zhihu (question, answer, user, collection)
|
||||
- Short links (b23.tv, etc.)
|
||||
|
||||
Usage:
|
||||
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
|
||||
python link-parser.py --batch urls.txt --output results.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from typing import Optional, Dict, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from link_parser_shortlink import ShortLinkResolver
|
||||
|
||||
|
||||
class LinkParser:
|
||||
"""Main link parser class"""
|
||||
|
||||
# Bilibili patterns
|
||||
BILIBILI_VIDEO_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV[a-zA-Z0-9]+)')
|
||||
BILIBILI_VIDEO_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
|
||||
BILIBILI_ARTICLE_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/article/([a-zA-Z0-9]+)')
|
||||
BILIBILI_SPACE_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/(\d+)')
|
||||
BILIBILI_FAVEPATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/opus/(\d+)')
|
||||
BILIBILI_SERIES_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/series/([a-zA-Z0-9]+)')
|
||||
|
||||
# YouTube patterns
|
||||
YOUTUBE_VIDEO_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/)([a-zA-Z0-9_-]{11})')
|
||||
YOUTUBE_CHANNEL_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/channel/([a-zA-Z0-9_-]+)')
|
||||
YOUTUBE_USER_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/user/([a-zA-Z0-9_-]+)')
|
||||
YOUTUBE_PLAYLIST_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/playlist\?list=([a-zA-Z0-9_-]+)')
|
||||
|
||||
# Zhihu patterns
|
||||
ZHIHAU_QUESTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/(\d+)')
|
||||
ZHIHAU_ANSWER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)')
|
||||
ZHIHAU_USER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/people/([a-zA-Z0-9_-]+)')
|
||||
ZHIHAU_COLUMN_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/column/([a-zA-Z0-9_-]+)')
|
||||
ZHIHAU_COLLECTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/collection/(\d+)')
|
||||
|
||||
def __init__(self):
|
||||
self.shortlink_resolver = ShortLinkResolver()
|
||||
|
||||
def resolve_shortlink(self, url: str) -> Optional[str]:
|
||||
"""
|
||||
Resolve short link to get final URL.
|
||||
|
||||
Args:
|
||||
url: Short URL to resolve
|
||||
|
||||
Returns:
|
||||
Final URL after resolution, or None if not resolvable
|
||||
"""
|
||||
# Check if it's a known short link pattern
|
||||
if self.shortlink_resolver.is_bilibili_short_link(url):
|
||||
# For b23.tv, we need Playwright to resolve
|
||||
# Return the short URL as-is with a note
|
||||
return url
|
||||
|
||||
# Return original URL for other cases
|
||||
return url
|
||||
|
||||
def extract_bilibili_id(self, url: str) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Extract Bilibili ID from URL.
|
||||
|
||||
Args:
|
||||
url: Bilibili URL
|
||||
|
||||
Returns:
|
||||
Dict with platform, type, and id fields, or None if not matched
|
||||
"""
|
||||
# Try video
|
||||
match = self.BILIBILI_VIDEO_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "bilibili",
|
||||
"type": "video",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try article
|
||||
match = self.BILIBILI_ARTICLE_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "bilibili",
|
||||
"type": "article",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try space (UP主)
|
||||
match = self.BILIBILI_SPACE_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "bilibili",
|
||||
"type": "up主",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try collection
|
||||
match = self.BILIBILI_FAVEPATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "bilibili",
|
||||
"type": "collection",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try series
|
||||
match = self.BILIBILI_SERIES_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "bilibili",
|
||||
"type": "series",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
def extract_youtube_id(self, url: str) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Extract YouTube ID from URL.
|
||||
|
||||
Args:
|
||||
url: YouTube URL
|
||||
|
||||
Returns:
|
||||
Dict with platform, type, and id fields, or None if not matched
|
||||
"""
|
||||
# Try video
|
||||
match = self.YOUTUBE_VIDEO_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "youtube",
|
||||
"type": "video",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try channel
|
||||
match = self.YOUTUBE_CHANNEL_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "youtube",
|
||||
"type": "channel",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try user
|
||||
match = self.YOUTUBE_USER_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "youtube",
|
||||
"type": "user",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try playlist
|
||||
match = self.YOUTUBE_PLAYLIST_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "youtube",
|
||||
"type": "playlist",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
def extract_zhihu_id(self, url: str) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Extract Zhihu ID from URL.
|
||||
|
||||
Args:
|
||||
url: Zhihu URL
|
||||
|
||||
Returns:
|
||||
Dict with platform, type, and id fields, or None if not matched
|
||||
"""
|
||||
# Try question
|
||||
match = self.ZHIHAU_QUESTION_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "zhihu",
|
||||
"type": "question",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try answer
|
||||
match = self.ZHIHAU_ANSWER_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "zhihu",
|
||||
"type": "answer",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try user
|
||||
match = self.ZHIHAU_USER_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "zhihu",
|
||||
"type": "user",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try column
|
||||
match = self.ZHIHAU_COLUMN_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "zhihu",
|
||||
"type": "column",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
# Try collection
|
||||
match = self.ZHIHAU_COLLECTION_PATTERN.match(url)
|
||||
if match:
|
||||
return {
|
||||
"platform": "zhihu",
|
||||
"type": "collection",
|
||||
"id": match.group(1)
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
def parse(self, url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Parse URL and extract platform-specific ID.
|
||||
|
||||
Args:
|
||||
url: URL to parse
|
||||
|
||||
Returns:
|
||||
Dict with platform, type, id, and original_url fields
|
||||
"""
|
||||
result = {
|
||||
"platform": None,
|
||||
"type": None,
|
||||
"id": None,
|
||||
"original_url": url,
|
||||
"resolved_url": None,
|
||||
"success": False,
|
||||
"error": None
|
||||
}
|
||||
|
||||
# First, resolve short links
|
||||
resolved = self.resolve_shortlink(url)
|
||||
if resolved:
|
||||
result["resolved_url"] = resolved
|
||||
url = resolved
|
||||
|
||||
# Try each platform
|
||||
bilibili_result = self.extract_bilibili_id(url)
|
||||
if bilibili_result:
|
||||
result.update(bilibili_result)
|
||||
result["success"] = True
|
||||
return result
|
||||
|
||||
youtube_result = self.extract_youtube_id(url)
|
||||
if youtube_result:
|
||||
result.update(youtube_result)
|
||||
result["success"] = True
|
||||
return result
|
||||
|
||||
zhihu_result = self.extract_zhihu_id(url)
|
||||
if zhihu_result:
|
||||
result.update(zhihu_result)
|
||||
result["success"] = True
|
||||
return result
|
||||
|
||||
# Check for short links that need resolution
|
||||
short_links = self.shortlink_resolver.detect_short_links(url)
|
||||
if short_links:
|
||||
result["error"] = "Short link detected. Please provide the full URL."
|
||||
return result
|
||||
|
||||
result["error"] = "Unsupported URL format"
|
||||
return result
|
||||
|
||||
|
||||
def format_output(data: Dict[str, Any], output_format: str = "json") -> str:
|
||||
"""Format output based on specified format."""
|
||||
if output_format == "json":
|
||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
||||
elif output_format == "compact":
|
||||
return json.dumps(data, ensure_ascii=False, separators=(',', ':'))
|
||||
elif output_format == "text":
|
||||
lines = []
|
||||
for key, value in data.items():
|
||||
if value is not None:
|
||||
lines.append(f"{key}: {value}")
|
||||
return "\n".join(lines)
|
||||
else:
|
||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Parse and extract IDs from various platform URLs",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s --url "https://www.bilibili.com/video/BV1xx411c7mD"
|
||||
%(prog)s --url "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||
%(prog)s --url "https://www.zhihu.com/question/266309957"
|
||||
%(prog)s --batch urls.txt --output results.json
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--url", "-u",
|
||||
help="Single URL to parse"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--batch", "-b",
|
||||
help="File containing URLs to parse (one per line)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--output", "-o",
|
||||
help="Output file path (for batch mode)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--format", "-f",
|
||||
choices=["json", "compact", "text"],
|
||||
default="json",
|
||||
help="Output format (default: json)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Check if we're in shortlink help mode
|
||||
if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
if not args.url and not args.batch:
|
||||
parser.print_help()
|
||||
print("\nError: Please provide --url or --batch")
|
||||
return 1
|
||||
|
||||
link_parser = LinkParser()
|
||||
|
||||
if args.url:
|
||||
# Single URL mode
|
||||
result = link_parser.parse(args.url)
|
||||
print(format_output(result, args.format))
|
||||
return 0 if result["success"] else 1
|
||||
|
||||
elif args.batch:
|
||||
try:
|
||||
with open(args.batch, 'r', encoding='utf-8') as f:
|
||||
urls = [line.strip() for line in f if line.strip()]
|
||||
except Exception as e:
|
||||
print(f"Error reading file: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
results = []
|
||||
for url in urls:
|
||||
result = link_parser.parse(url)
|
||||
results.append(result)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
json.dump(results, f, ensure_ascii=False, indent=2)
|
||||
print(f"Results written to {args.output}")
|
||||
else:
|
||||
for result in results:
|
||||
print(format_output(result, args.format))
|
||||
print("-" * 40)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Short link resolver for link-parser skill.
|
||||
|
||||
Provides utilities for detecting and resolving short links, particularly
|
||||
Bilibili short links (b23.tv) which may be blocked by anti-scraping measures.
|
||||
|
||||
Supported short link services:
|
||||
- Bilibili: b23.tv
|
||||
- Generic URL shorteners (detection only)
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Optional, Dict, List, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class ShortLinkResolver:
|
||||
"""Short link detection and resolution handler"""
|
||||
|
||||
# Pattern for Bilibili short links
|
||||
BILIBILI_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
|
||||
|
||||
# Pattern for other short link services
|
||||
OTHER_SHORT_PATTERNS = {
|
||||
'bit.ly': re.compile(r'https?://bit\.ly/\w+'),
|
||||
't.cn': re.compile(r'https?://t\.cn/\w+'),
|
||||
'dwz.cn': re.compile(r'https?://dwz\.cn/\w+'),
|
||||
'goo.gl': re.compile(r'https?://goo\.gl/\w+'),
|
||||
'tinyurl.com': re.compile(r'https?://tinyurl\.com/\w+'),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def detect_short_links(text: str) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Detect short links in text.
|
||||
|
||||
Args:
|
||||
text: Input text to search
|
||||
|
||||
Returns:
|
||||
List of detected short links with metadata:
|
||||
[
|
||||
{
|
||||
'url': 'https://b23.tv/xxxxx',
|
||||
'platform': 'bilibili',
|
||||
'is_short': True
|
||||
},
|
||||
...
|
||||
]
|
||||
"""
|
||||
detected = []
|
||||
|
||||
# Check for Bilibili short links
|
||||
for match in ShortLinkResolver.BILIBILI_SHORT_PATTERN.finditer(text):
|
||||
detected.append({
|
||||
'url': match.group(0),
|
||||
'code': match.group(1),
|
||||
'platform': 'bilibili',
|
||||
'is_short': True
|
||||
})
|
||||
|
||||
# Check for other short link services
|
||||
for platform, pattern in ShortLinkResolver.OTHER_SHORT_PATTERNS.items():
|
||||
for match in pattern.finditer(text):
|
||||
detected.append({
|
||||
'url': match.group(0),
|
||||
'platform': platform,
|
||||
'is_short': True
|
||||
})
|
||||
|
||||
return detected
|
||||
|
||||
@staticmethod
|
||||
def is_bilibili_short_link(url: str) -> bool:
|
||||
"""
|
||||
Check if URL is a Bilibili short link.
|
||||
|
||||
Args:
|
||||
url: URL to check
|
||||
|
||||
Returns:
|
||||
True if URL is a b23.tv short link
|
||||
"""
|
||||
return bool(ShortLinkResolver.BILIBILI_SHORT_PATTERN.match(url))
|
||||
|
||||
@staticmethod
|
||||
def format_short_link_help(short_link: Optional[str] = None) -> str:
|
||||
"""
|
||||
Format help message for short links.
|
||||
|
||||
Args:
|
||||
short_link: Optional short link URL to mention
|
||||
|
||||
Returns:
|
||||
Help message
|
||||
"""
|
||||
lines = [
|
||||
"Detection: Short link",
|
||||
""
|
||||
]
|
||||
|
||||
if short_link:
|
||||
lines.append(f"Short link: {short_link}")
|
||||
lines.append("")
|
||||
|
||||
lines.extend([
|
||||
"Due to anti-scraping protection, short links cannot be automatically resolved.",
|
||||
"Please provide the full URL or the content ID (e.g., BV号 for Bilibili).",
|
||||
"",
|
||||
"Full URL format: https://www.bilibili.com/video/BVxxxxx"
|
||||
])
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@staticmethod
|
||||
def format_short_link_warning(short_links: List[Dict[str, str]]) -> str:
|
||||
"""
|
||||
Format warning message for multiple short links.
|
||||
|
||||
Args:
|
||||
short_links: List of short link metadata
|
||||
|
||||
Returns:
|
||||
Warning message
|
||||
"""
|
||||
if not short_links:
|
||||
return ""
|
||||
|
||||
lines = [
|
||||
"Short links detected:",
|
||||
""
|
||||
]
|
||||
|
||||
for i, link in enumerate(short_links, 1):
|
||||
platform = link.get('platform', 'unknown')
|
||||
url = link['url']
|
||||
|
||||
if platform == 'bilibili':
|
||||
lines.append(f" [{i}] Bilibili short link: {url}")
|
||||
lines.append(" Full URL or BV ID required")
|
||||
else:
|
||||
lines.append(f" [{i}] {platform}: {url}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class PlaywrightResolver:
|
||||
"""
|
||||
Browser-based short link resolver using Playwright.
|
||||
|
||||
This is an optional implementation for environments where Playwright
|
||||
is available. It can bypass anti-scraping measures by simulating
|
||||
a real browser.
|
||||
|
||||
Requirements:
|
||||
pip install playwright
|
||||
playwright install chromium
|
||||
"""
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 10000):
|
||||
"""
|
||||
Initialize Playwright resolver.
|
||||
|
||||
Args:
|
||||
headless: Whether to run browser in headless mode
|
||||
timeout: Page navigation timeout in milliseconds
|
||||
"""
|
||||
self.headless = headless
|
||||
self.timeout = timeout
|
||||
self._playwright = None
|
||||
|
||||
def _check_playwright_available(self) -> bool:
|
||||
"""Check if Playwright is installed and available"""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
def resolve(self, short_url: str) -> Optional[str]:
|
||||
"""
|
||||
Resolve short link using Playwright browser automation.
|
||||
|
||||
Args:
|
||||
short_url: Short URL to resolve
|
||||
|
||||
Returns:
|
||||
Final URL after redirects, or None if resolution fails
|
||||
"""
|
||||
if not self._check_playwright_available():
|
||||
raise ImportError(
|
||||
"Playwright not installed. Install with: pip install playwright && playwright install chromium"
|
||||
)
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
try:
|
||||
with sync_playwright() as p:
|
||||
# Launch browser
|
||||
browser = p.chromium.launch(headless=self.headless)
|
||||
context = browser.new_context(
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
viewport={'width': 1920, 'height': 1080}
|
||||
)
|
||||
page = context.new_page()
|
||||
|
||||
# Navigate to short URL
|
||||
page.goto(short_url, timeout=self.timeout, wait_until='networkidle')
|
||||
|
||||
# Get final URL
|
||||
final_url = page.url
|
||||
|
||||
# Close browser
|
||||
browser.close()
|
||||
|
||||
return final_url
|
||||
|
||||
except Exception as e:
|
||||
print(f"Playwright resolution failed: {e}")
|
||||
return None
|
||||
|
||||
def resolve_bilibili_short(self, short_url: str) -> Optional[str]:
|
||||
"""
|
||||
Resolve Bilibili short link and extract BV ID.
|
||||
|
||||
Args:
|
||||
short_url: Bilibili short URL (e.g., https://b23.tv/xxxxx)
|
||||
|
||||
Returns:
|
||||
Full Bilibili URL with BV ID, or None if resolution fails
|
||||
"""
|
||||
final_url = self.resolve(short_url)
|
||||
if not final_url:
|
||||
return None
|
||||
|
||||
# Check if final URL contains BV ID
|
||||
if 'bilibili.com/video/' in final_url:
|
||||
return final_url
|
||||
|
||||
# Extract BV ID if present in URL
|
||||
bv_pattern = re.compile(r'(BV[a-zA-Z0-9]{10})')
|
||||
match = bv_pattern.search(final_url)
|
||||
if match:
|
||||
return f"https://www.bilibili.com/video/{match.group(1)}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_resolver() -> ShortLinkResolver:
|
||||
"""
|
||||
Get the default resolver instance.
|
||||
|
||||
Returns:
|
||||
ShortLinkResolver instance
|
||||
"""
|
||||
return ShortLinkResolver()
|
||||
|
||||
|
||||
def detect_short_links(text: str) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Detect short links in text.
|
||||
|
||||
Args:
|
||||
text: Input text to search
|
||||
|
||||
Returns:
|
||||
List of detected short links with metadata
|
||||
"""
|
||||
resolver = ShortLinkResolver()
|
||||
return resolver.detect_short_links(text)
|
||||
|
||||
|
||||
# CLI interface for testing
|
||||
def main():
|
||||
"""CLI for testing the short link resolver."""
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 link_parser_shortlink.py 'text to test'")
|
||||
print("\nExamples:")
|
||||
print(' python3 link_parser_shortlink.py "Check this: https://b23.tv/oLBZZfK"')
|
||||
print(' python3 link_parser_shortlink.py "https://bit.ly/xxxxx"')
|
||||
sys.exit(1)
|
||||
|
||||
text = ' '.join(sys.argv[1:])
|
||||
short_links = detect_short_links(text)
|
||||
|
||||
import json
|
||||
print(json.dumps(short_links, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
# KB OCR Processor Skill
|
||||
|
||||
**kb-ocr-processor** - Extract text from video frames using OCR.
|
||||
|
||||
## Features
|
||||
|
||||
- Batch frame processing
|
||||
- PaddleOCR/Tesseract support
|
||||
- Confidence scoring
|
||||
- Text merging
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Process frames
|
||||
python ocr_processor.py --input ./frames/ --output ocr.json
|
||||
|
||||
# With custom FPS
|
||||
python ocr_processor.py --input ./frames/ --fps 0.1 --output ocr.json
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"frames": {
|
||||
"frame_0001.jpg": {
|
||||
"texts": [{"text": "字幕", "confidence": 0.95}]
|
||||
}
|
||||
},
|
||||
"merged_text": "合并的字幕文本..."
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
cp -r kb-ocr-processor/ ~/skills/
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
@@ -0,0 +1,242 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
OCR Processor - 视频帧文字识别工具
|
||||
支持 EasyOCR 和 Tesseract 引擎
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import glob
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
import easyocr
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class OCRProcessor:
|
||||
"""OCR 文字识别处理器"""
|
||||
|
||||
def __init__(self, engine: str = "easyocr", languages: List[str] = ["ch_sim", "en"]):
|
||||
"""
|
||||
初始化 OCR 处理器
|
||||
|
||||
Args:
|
||||
engine: OCR 引擎 ("easyocr" 或 "tesseract")
|
||||
languages: 语言列表
|
||||
"""
|
||||
self.engine = engine
|
||||
self.languages = languages
|
||||
self.reader = None
|
||||
|
||||
if engine == "easyocr":
|
||||
self.reader = easyocr.Reader(languages, gpu=False)
|
||||
|
||||
def process_image(self, image_path: str) -> Dict:
|
||||
"""
|
||||
处理单张图片进行 OCR 识别
|
||||
|
||||
Args:
|
||||
image_path: 图片路径
|
||||
|
||||
Returns:
|
||||
包含 text 和 confidence 的字典
|
||||
"""
|
||||
if not os.path.exists(image_path):
|
||||
return {"text": "", "confidence": 0.0, "error": "File not found"}
|
||||
|
||||
try:
|
||||
if self.engine == "easyocr":
|
||||
return self._process_with_easyocr(image_path)
|
||||
elif self.engine == "tesseract":
|
||||
return self._process_with_tesseract(image_path)
|
||||
else:
|
||||
return {"text": "", "confidence": 0.0, "error": f"Unknown engine: {self.engine}"}
|
||||
except Exception as e:
|
||||
return {"text": "", "confidence": 0.0, "error": str(e)}
|
||||
|
||||
def _process_with_easyocr(self, image_path: str) -> Dict:
|
||||
"""使用 EasyOCR 处理图片"""
|
||||
results = self.reader.readtext(image_path)
|
||||
|
||||
if not results:
|
||||
return {"text": "", "confidence": 0.0}
|
||||
|
||||
texts = []
|
||||
confidences = []
|
||||
|
||||
for bbox, text, conf in results:
|
||||
texts.append(text)
|
||||
confidences.append(conf)
|
||||
|
||||
merged_text = "\n".join(texts)
|
||||
avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0
|
||||
|
||||
return {
|
||||
"text": merged_text,
|
||||
"confidence": round(avg_confidence, 4),
|
||||
"details": [
|
||||
{"text": text, "confidence": round(conf, 4)}
|
||||
for text, conf in zip(texts, confidences)
|
||||
]
|
||||
}
|
||||
|
||||
def _process_with_tesseract(self, image_path: str) -> Dict:
|
||||
"""使用 Tesseract 处理图片"""
|
||||
image = Image.open(image_path)
|
||||
text = pytesseract.image_to_string(image, lang="+".join(self.languages))
|
||||
|
||||
# Tesseract 不直接返回置信度，返回 0.0
|
||||
return {
|
||||
"text": text.strip(),
|
||||
"confidence": 0.0
|
||||
}
|
||||
|
||||
def process_frames(self, input_dir: str, fps: float = 0.1) -> List[Dict]:
|
||||
"""
|
||||
处理目录下所有帧图片
|
||||
|
||||
Args:
|
||||
input_dir: 帧图片目录
|
||||
fps: 采样帧率
|
||||
|
||||
Returns:
|
||||
OCR 结果列表
|
||||
"""
|
||||
if not os.path.isdir(input_dir):
|
||||
print(f"Error: Directory not found: {input_dir}")
|
||||
return []
|
||||
|
||||
# 支持的图片格式
|
||||
image_extensions = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.webp"]
|
||||
image_files = []
|
||||
|
||||
for ext in image_extensions:
|
||||
image_files.extend(glob.glob(os.path.join(input_dir, ext)))
|
||||
image_files.extend(glob.glob(os.path.join(input_dir, "**", ext), recursive=True))
|
||||
|
||||
# 按文件名排序
|
||||
image_files = sorted(image_files)
|
||||
|
||||
if not image_files:
|
||||
print(f"No images found in {input_dir}")
|
||||
return []
|
||||
|
||||
print(f"Found {len(image_files)} images in {input_dir}")
|
||||
print(f"Processing with {self.engine} engine...")
|
||||
|
||||
results = []
|
||||
for i, image_path in enumerate(image_files):
|
||||
print(f"Processing [{i+1}/{len(image_files)}]: {image_path}")
|
||||
ocr_result = self.process_image(image_path)
|
||||
ocr_result["source"] = os.path.basename(image_path)
|
||||
results.append(ocr_result)
|
||||
|
||||
return results
|
||||
|
||||
def merge_texts(self, results: List[Dict]) -> str:
|
||||
"""
|
||||
合并多个 OCR 结果的文本
|
||||
|
||||
Args:
|
||||
results: OCR 结果列表
|
||||
|
||||
Returns:
|
||||
合并后的文本
|
||||
"""
|
||||
texts = []
|
||||
for result in results:
|
||||
if result.get("text"):
|
||||
texts.append(result["text"])
|
||||
return "\n\n---\n\n".join(texts)
|
||||
|
||||
def save_results(self, results: List[Dict], output_path: str, merge: bool = False):
|
||||
"""
|
||||
保存 OCR 结果到 JSON 文件
|
||||
|
||||
Args:
|
||||
results: OCR 结果列表
|
||||
output_path: 输出文件路径
|
||||
merge: 是否同时保存合并后的文本文件
|
||||
"""
|
||||
output_data = {
|
||||
"engine": self.engine,
|
||||
"total_frames": len(results),
|
||||
"results": results
|
||||
}
|
||||
|
||||
# 保存 JSON
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(output_data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"Results saved to {output_path}")
|
||||
|
||||
# 如果需要，同时保存合并的文本
|
||||
if merge:
|
||||
text_path = output_path.replace(".json", ".txt")
|
||||
merged_text = self.merge_texts(results)
|
||||
with open(text_path, "w", encoding="utf-8") as f:
|
||||
f.write(merged_text)
|
||||
print(f"Merged text saved to {text_path}")
|
||||
|
||||
|
||||
def main():
|
||||
"""命令行入口"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="OCR 视频帧文字识别工具",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
# 使用 EasyOCR 处理帧目录
|
||||
python ocr_processor.py --input ./frames/ --output ./ocr.json
|
||||
|
||||
# 使用 Tesseract 处理
|
||||
python ocr_processor.py --input ./frames/ --output ./ocr.json --engine tesseract
|
||||
|
||||
# 指定采样帧率
|
||||
python ocr_processor.py --input ./frames/ --output ./ocr.json --fps 0.5
|
||||
|
||||
# 同时生成合并的文本文件
|
||||
python ocr_processor.py --input ./frames/ --output ./ocr.json --merge
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument("--input", "-i", required=True, help="输入帧图片目录")
|
||||
parser.add_argument("--output", "-o", required=True, help="输出 JSON 文件路径")
|
||||
parser.add_argument("--fps", "-f", type=float, default=0.1, help="采样帧率 (默认: 0.1)")
|
||||
parser.add_argument("--engine", "-e", choices=["easyocr", "tesseract"],
|
||||
default="easyocr", help="OCR 引擎 (默认: easyocr)")
|
||||
parser.add_argument("--merge", action="store_true", help="同时保存合并的文本文件")
|
||||
parser.add_argument("--languages", "-l", nargs="+",
|
||||
default=["ch_sim", "en"], help="语言列表")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 初始化处理器
|
||||
processor = OCRProcessor(engine=args.engine, languages=args.languages)
|
||||
|
||||
# 处理帧
|
||||
results = processor.process_frames(args.input, fps=args.fps)
|
||||
|
||||
if results:
|
||||
# 保存结果
|
||||
processor.save_results(results, args.output, merge=args.merge)
|
||||
|
||||
# 打印摘要
|
||||
total_texts = sum(1 for r in results if r.get("text"))
|
||||
avg_conf = sum(r.get("confidence", 0) for r in results) / len(results) if results else 0
|
||||
|
||||
print(f"\n=== 摘要 ===")
|
||||
print(f"处理帧数: {len(results)}")
|
||||
print(f"识别到文字的帧: {total_texts}")
|
||||
print(f"平均置信度: {avg_conf:.2%}")
|
||||
else:
|
||||
print("未处理到任何结果")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
@@ -0,0 +1,3 @@
|
||||
easyocr>=1.7.0
|
||||
pytesseract>=0.3.10
|
||||
Pillow>=9.0.0
|
||||
@@ -0,0 +1,285 @@
|
||||
---
|
||||
name: kb-workflow
|
||||
description: 知识库工作流编排技能，提供收件箱扫描、内容处理、归档、订阅检测和汇报生成功能。
|
||||
metadata:
|
||||
short-description: 知识库工作流编排
|
||||
---
|
||||
|
||||
# KB Workflow
|
||||
|
||||
## Description
|
||||
|
||||
KB Workflow 是一个知识库工作流编排技能，负责协调和管理知识库内容的完整生命周期：
|
||||
|
||||
**核心功能**：
|
||||
- 📥 **收件箱扫描** - 扫描四个子目录 (pending, metadata, content, analyzed)
|
||||
- 🔄 **内容处理** - 调用技能2-4处理内容
|
||||
- 📦 **智能归档** - 将处理完成的内容归档到 archive/
|
||||
- 🔔 **订阅检测** - 检测订阅源的更新
|
||||
- 📊 **汇报生成** - 生成处理统计和推荐
|
||||
|
||||
## Workflow Stages
|
||||
|
||||
```
|
||||
┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌──────────┐
|
||||
│ pending │ → │ metadata│ → │ content │ → │ analyzed│ → │ archive/ │
|
||||
└─────────┘ └─────────┘ └─────────┘ └─────────┘ └──────────┘
|
||||
↓ ↓ ↓ ↓
|
||||
Skill 2 Skill 3 Skill 4 ↓
|
||||
(Link解析) (元数据提取) (内容分析) 最终归档
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
KB Workflow 作为 OpenClaw 技能套件的一部分，无需额外安装。
|
||||
|
||||
确保以下依赖已配置：
|
||||
- Python 3.8+
|
||||
- 知识库目录结构 (knowledge-base/)
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
knowledge-base/
|
||||
├── pending/ # 待处理的新内容
|
||||
├── metadata/ # 已提取元数据
|
||||
├── content/ # 已提取内容
|
||||
├── analyzed/ # 已完成分析
|
||||
├── archive/ # 归档内容
|
||||
│ └── YYYY-MM-DD/ # 按日期归档
|
||||
└── reports/ # 生成的汇报
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### 命令行用法
|
||||
|
||||
```bash
|
||||
# 查看工作流状态
|
||||
python kb_workflow.py --trigger status
|
||||
|
||||
# 手动处理待处理内容
|
||||
python kb_workflow.py --trigger manual --scope pending
|
||||
|
||||
# 处理所有新内容（限制5个）
|
||||
python kb_workflow.py --trigger manual --limit 5
|
||||
|
||||
# 生成日报
|
||||
python kb_workflow.py --trigger scheduled --report
|
||||
|
||||
# 检查订阅更新
|
||||
python kb_workflow.py --trigger subscriptions
|
||||
```
|
||||
|
||||
### 参数说明
|
||||
|
||||
| 参数 | 说明 | 默认值 |
|
||||
|------|------|--------|
|
||||
| `--trigger` | 触发类型 | `manual` |
|
||||
| `--scope` | 处理范围 | `all` |
|
||||
| `--limit` | 处理数量限制 | 无限制 |
|
||||
| `--report` | 生成汇报 | False |
|
||||
| `--output` | 输出格式 | `json` |
|
||||
|
||||
### 触发类型
|
||||
|
||||
- `manual` - 手动触发，立即处理
|
||||
- `scheduled` - 定时任务触发
|
||||
- `webhook` - Webhook 触发
|
||||
- `status` - 仅查看状态
|
||||
- `report` - 生成汇报
|
||||
- `subscriptions` - 检查订阅
|
||||
|
||||
### 作用域
|
||||
|
||||
- `all` - 所有目录
|
||||
- `pending` - 仅待处理
|
||||
- `metadata` - 仅元数据阶段
|
||||
- `content` - 仅内容阶段
|
||||
- `analyzed` - 仅已完成分析
|
||||
- `archive` - 归档目录
|
||||
|
||||
## Integration
|
||||
|
||||
### 技能2-4 集成
|
||||
|
||||
KB Workflow 协调以下子技能：
|
||||
|
||||
1. **Skill 2: Link Parser** - 解析 URL/链接
|
||||
2. **Skill 3: Zhihu Metadata** - 提取元数据
|
||||
3. **Skill 4: Bilibili Analyzer** - 内容分析
|
||||
|
||||
### 调用示例
|
||||
|
||||
```python
|
||||
from kb_workflow import KBWorkflow
|
||||
|
||||
# 初始化
|
||||
workflow = KBWorkflow(base_path="/path/to/knowledge-base")
|
||||
|
||||
# 扫描收件箱
|
||||
inbox = workflow.scan_inbox(scope="pending")
|
||||
print(f"待处理: {len(inbox['pending'])} 项")
|
||||
|
||||
# 处理项目
|
||||
for item in inbox['pending'][:5]:
|
||||
result = workflow.process_item(item)
|
||||
print(f"处理结果: {result['success']}")
|
||||
|
||||
# 查看状态
|
||||
status = workflow.get_status()
|
||||
print(json.dumps(status, indent=2))
|
||||
|
||||
# 生成汇报
|
||||
report = workflow.generate_report(period="daily")
|
||||
```
|
||||
|
||||
## Input/Output Formats
|
||||
|
||||
### Input
|
||||
|
||||
#### 扫描收件箱输入
|
||||
```json
|
||||
{
|
||||
"scope": "all", // 或 "pending", "metadata", "content", "analyzed"
|
||||
"limit": 10 // 可选，限制返回数量
|
||||
}
|
||||
```
|
||||
|
||||
#### 处理项目输入
|
||||
```json
|
||||
{
|
||||
"name": "example.md",
|
||||
"path": "/path/to/knowledge-base/pending/example.md",
|
||||
"extension": ".md",
|
||||
"size": 1024,
|
||||
"modified": "2026-02-10T12:00:00"
|
||||
}
|
||||
```
|
||||
|
||||
### Output
|
||||
|
||||
#### 状态输出
|
||||
```json
|
||||
{
|
||||
"status": "ready",
|
||||
"timestamp": "2026-02-10T12:00:00",
|
||||
"inbox": {
|
||||
"pending": {"count": 5, "path": "/path/pending"},
|
||||
"metadata": {"count": 3, "path": "/path/metadata"},
|
||||
"content": {"count": 2, "path": "/path/content"},
|
||||
"analyzed": {"count": 8, "path": "/path/analyzed"}
|
||||
},
|
||||
"archive": {
|
||||
"count": 120,
|
||||
"path": "/path/archive"
|
||||
},
|
||||
"total_items": 18
|
||||
}
|
||||
```
|
||||
|
||||
#### 处理结果输出
|
||||
```json
|
||||
{
|
||||
"item": {"name": "example.md", ...},
|
||||
"success": true,
|
||||
"stages_completed": ["link_parsing", "metadata_extraction", "content_analysis"],
|
||||
"output_path": "/path/analyzed/example.md",
|
||||
"error": null
|
||||
}
|
||||
```
|
||||
|
||||
#### 汇报输出
|
||||
```json
|
||||
{
|
||||
"generated_at": "2026-02-10T12:00:00",
|
||||
"period": "daily",
|
||||
"summary": {
|
||||
"total_items_inbox": 18,
|
||||
"total_archived": 120,
|
||||
"items_by_stage": {...}
|
||||
},
|
||||
"processing_stats": {...},
|
||||
"recommendations": [
|
||||
"⚠️ 待处理内容较多 (15 项)，建议优先处理",
|
||||
"📦 已分析内容较多 (25 项)，建议归档"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
KB Workflow 无需额外配置，自动使用以下路径：
|
||||
- 知识库根目录: `skills/kb-workflow/knowledge-base/`
|
||||
- 归档目录: `skills/kb-workflow/knowledge-base/archive/`
|
||||
- 汇报目录: `skills/kb-workflow/knowledge-base/reports/`
|
||||
|
||||
### 自定义路径
|
||||
|
||||
```python
|
||||
from kb_workflow import KBWorkflow
|
||||
|
||||
# 使用自定义路径
|
||||
workflow = KBWorkflow(
|
||||
base_path="/custom/knowledge-base",
|
||||
archive_path="/custom/archive",
|
||||
output_format="json"
|
||||
)
|
||||
```
|
||||
|
||||
## Workflow Steps
|
||||
|
||||
### 完整处理流程
|
||||
|
||||
1. **扫描收件箱**
|
||||
- 扫描 `pending/` 目录获取新内容
|
||||
- 扫描其他阶段目录获取待继续处理的内容
|
||||
|
||||
2. **分类处理**
|
||||
- 新内容 → 提取链接/URL (Skill 2)
|
||||
- 待提取元数据 → 提取元数据 (Skill 3)
|
||||
- 待分析内容 → 深度分析 (Skill 4)
|
||||
|
||||
3. **自动流转**
|
||||
- 处理完成后自动移动到下一阶段目录
|
||||
|
||||
4. **归档**
|
||||
- 分析完成后可归档到 `archive/YYYY-MM-DD/`
|
||||
|
||||
5. **汇报**
|
||||
- 生成处理统计
|
||||
- 提供处理建议
|
||||
|
||||
### 定时任务示例
|
||||
|
||||
```bash
|
||||
# 每天早上9点检查并处理新内容
|
||||
0 9 * * * cd /path/to/skills/kb-workflow && python kb_workflow.py --trigger scheduled --scope pending --report
|
||||
|
||||
# 每小时检查订阅更新
|
||||
0 * * * * cd /path/to/skills/kb-workflow && python kb_workflow.py --trigger subscriptions
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### 常见问题
|
||||
|
||||
**Q: 扫描不到内容？**
|
||||
A: 确保知识库目录存在且包含正确的子目录 (pending, metadata, content, analyzed)
|
||||
|
||||
**Q: 处理失败？**
|
||||
A: 检查日志输出，使用 `--verbose` 参数获取详细调试信息
|
||||
|
||||
**Q: 如何查看处理进度？**
|
||||
A: 使用 `--trigger status` 查看各阶段的项目数量
|
||||
|
||||
## Related Skills
|
||||
|
||||
- **obsidian-kb** - Obsidian 知识库管理
|
||||
- **link-parser** - 链接解析
|
||||
- **zhihu-metadata** - 知乎元数据提取
|
||||
- **bilibili-analyzer** - B站视频分析
|
||||
|
||||
---
|
||||
|
||||
*KB Workflow v1.0 - Knowledge Base Workflow Orchestration*
|
||||
Executable
+427
@@ -0,0 +1,427 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Auto-trigger handler for obsidian-kb skill.
|
||||
|
||||
Detects video/article links in user messages and automatically processes them.
|
||||
Supports Bilibili, YouTube, and Zhihu.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Add path for imports
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'src'))
|
||||
|
||||
# Import short link resolver
|
||||
from shortlink_resolver import (
|
||||
ShortLinkResolver,
|
||||
detect_and_format,
|
||||
PlaywrightResolver
|
||||
)
|
||||
|
||||
# 导入主流程
|
||||
try:
|
||||
from main_pipeline import MainPipeline
|
||||
MAIN_PIPELINE_AVAILABLE = True
|
||||
except ImportError:
|
||||
MAIN_PIPELINE_AVAILABLE = False
|
||||
|
||||
# Pattern definitions for auto-detection
|
||||
PATTERNS = {
|
||||
'bilibili_full': re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV\w+)'),
|
||||
'bilibili_short': re.compile(r'https?://b23\.tv/(\w+)'),
|
||||
'bilibili_bv': re.compile(r'\b(BV[a-zA-Z0-9]{10})\b'),
|
||||
'youtube_full': re.compile(r'https?://(?:www\.)?youtube\.com/watch\?v=([a-zA-Z0-9_-]{11})'),
|
||||
'youtube_short': re.compile(r'https?://youtu\.be/([a-zA-Z0-9_-]{11})'),
|
||||
'zhihu_article': re.compile(r'https?://zhuanlan\.zhihu\.com/p/(\d+)'),
|
||||
'zhihu_answer': re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)'),
|
||||
}
|
||||
|
||||
|
||||
def detect_links(text: str) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Detect video/article links in text.
|
||||
|
||||
Args:
|
||||
text: Input text to search
|
||||
|
||||
Returns:
|
||||
List of detected links with metadata:
|
||||
[
|
||||
{
|
||||
'url': 'https://...',
|
||||
'platform': 'bilibili',
|
||||
'type': 'video',
|
||||
'id': 'BVxxx',
|
||||
'is_short': False
|
||||
},
|
||||
...
|
||||
]
|
||||
"""
|
||||
detected = []
|
||||
seen_ids = set() # Track IDs to avoid duplicates
|
||||
|
||||
for pattern_name, pattern in PATTERNS.items():
|
||||
matches = pattern.finditer(text)
|
||||
|
||||
for match in matches:
|
||||
# Determine platform and type
|
||||
if 'bilibili' in pattern_name:
|
||||
platform = 'bilibili'
|
||||
content_type = 'video'
|
||||
if 'short' in pattern_name:
|
||||
is_short = True
|
||||
url = match.group(0)
|
||||
content_id = None # Need to resolve short URL
|
||||
else:
|
||||
is_short = False
|
||||
if pattern_name == 'bilibili_bv':
|
||||
url = f"https://www.bilibili.com/video/{match.group(1)}"
|
||||
else:
|
||||
url = match.group(0)
|
||||
content_id = match.group(1)
|
||||
|
||||
elif 'youtube' in pattern_name:
|
||||
platform = 'youtube'
|
||||
content_type = 'video'
|
||||
is_short = 'short' in pattern_name
|
||||
url = match.group(0)
|
||||
content_id = match.group(1)
|
||||
|
||||
elif 'zhihu' in pattern_name:
|
||||
platform = 'zhihu'
|
||||
content_type = 'article'
|
||||
is_short = False
|
||||
url = match.group(0)
|
||||
content_id = match.group(1)
|
||||
|
||||
else:
|
||||
continue
|
||||
|
||||
# Skip duplicates by URL or ID
|
||||
unique_key = url if (not content_id or is_short) else content_id
|
||||
if unique_key in seen_ids:
|
||||
continue
|
||||
seen_ids.add(unique_key)
|
||||
|
||||
detected.append({
|
||||
'url': url,
|
||||
'platform': platform,
|
||||
'type': content_type,
|
||||
'id': content_id,
|
||||
'is_short': is_short,
|
||||
'pattern': pattern_name
|
||||
})
|
||||
|
||||
return detected
|
||||
|
||||
|
||||
def format_preview(detected: List[Dict[str, str]]) -> str:
|
||||
"""
|
||||
Format a preview message for detected links.
|
||||
|
||||
Args:
|
||||
detected: List of detected links
|
||||
|
||||
Returns:
|
||||
Formatted preview message
|
||||
"""
|
||||
if not detected:
|
||||
return ""
|
||||
|
||||
lines = ["🔍 检测到内容链接:\n"]
|
||||
|
||||
for i, item in enumerate(detected, 1):
|
||||
platform_emoji = {
|
||||
'bilibili': '📺',
|
||||
'youtube': '▶️',
|
||||
'zhihu': '📝'
|
||||
}.get(item['platform'], '🔗')
|
||||
|
||||
platform_name = {
|
||||
'bilibili': 'B站视频',
|
||||
'youtube': 'YouTube视频',
|
||||
'zhihu': '知乎文章'
|
||||
}.get(item['platform'], item['platform'])
|
||||
|
||||
lines.append(f"{platform_emoji} [{i}] {platform_name}")
|
||||
lines.append(f" URL: {item['url']}")
|
||||
|
||||
if item['is_short']:
|
||||
lines.append(f" ⚠️ 短链接，需要用户提供完整URL或ID")
|
||||
elif item['id']:
|
||||
lines.append(f" ID: {item['id']}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_short_link_help(short_link: Optional[str] = None) -> str:
|
||||
"""
|
||||
Format help message for short links.
|
||||
|
||||
Args:
|
||||
short_link: Optional short link URL to mention
|
||||
|
||||
Returns:
|
||||
Help message
|
||||
"""
|
||||
return ShortLinkResolver.format_short_link_help(short_link)
|
||||
|
||||
|
||||
def get_action_prompt(detected: List[Dict[str, str]]) -> str:
|
||||
"""
|
||||
Generate prompt asking user what to do.
|
||||
|
||||
Args:
|
||||
detected: List of detected links
|
||||
|
||||
Returns:
|
||||
Action prompt
|
||||
"""
|
||||
has_short = any(item['is_short'] for item in detected)
|
||||
|
||||
if has_short:
|
||||
return f"{format_short_link_help()}\n\n其他链接是否立即处理？(y/n)"
|
||||
|
||||
count = len(detected)
|
||||
if count == 1:
|
||||
return "是否添加到收件箱并处理？(y/n)"
|
||||
else:
|
||||
return f"是否将上述 {count} 个内容添加到收件箱？(y/n)"
|
||||
|
||||
|
||||
def should_auto_process(text: str) -> bool:
|
||||
"""
|
||||
Check if user wants to auto-process based on their message.
|
||||
|
||||
Args:
|
||||
text: User's message
|
||||
|
||||
Returns:
|
||||
True if user indicates they want immediate processing
|
||||
"""
|
||||
auto_keywords = ['立即处理', '处理', 'process', '--process', '马上']
|
||||
return any(keyword in text.lower() for keyword in auto_keywords)
|
||||
|
||||
|
||||
def process_link(url: str, platform: str, auto_process: bool = False) -> Tuple[bool, str]:
|
||||
"""
|
||||
Process a single link (extract metadata, add to inbox).
|
||||
|
||||
Args:
|
||||
url: Content URL
|
||||
platform: Platform name (bilibili/youtube/zhihu)
|
||||
auto_process: Whether to immediately process after adding
|
||||
|
||||
Returns:
|
||||
(success, message) tuple
|
||||
"""
|
||||
try:
|
||||
# Import here to avoid issues if dependencies not available
|
||||
from sources.bilibili import BilibiliAdapter
|
||||
from models import Content, ContentStatus
|
||||
from storage import Storage
|
||||
|
||||
# Only support Bilibili for now
|
||||
if platform != 'bilibili':
|
||||
return False, f"暂不支持 {platform}，请手动添加"
|
||||
|
||||
# Handle short links
|
||||
if 'b23.tv' in url:
|
||||
return False, "短链接无法自动处理，请提供完整URL或BV号"
|
||||
|
||||
# Create adapter
|
||||
adapter = BilibiliAdapter()
|
||||
|
||||
# Extract metadata
|
||||
content = adapter.extract_content(url)
|
||||
if not content:
|
||||
return False, "无法提取视频信息"
|
||||
|
||||
# Get transcript
|
||||
transcript = adapter.get_transcript(content)
|
||||
if transcript:
|
||||
content.transcript = transcript
|
||||
|
||||
# Save to inbox
|
||||
content.status = ContentStatus.CONFIRMED
|
||||
storage = Storage()
|
||||
storage.save_content(content)
|
||||
|
||||
# Build success message
|
||||
msg = f"✅ 已添加到收件箱\n"
|
||||
msg += f" 标题: {content.title}\n"
|
||||
msg += f" UP主: {content.author}\n"
|
||||
msg += f" 时长: {content.duration}秒\n"
|
||||
|
||||
if auto_process:
|
||||
# Process immediately
|
||||
try:
|
||||
from src.pipeline import ProcessingPipeline
|
||||
pipeline = ProcessingPipeline()
|
||||
content_text = content.transcript or content.summary or ""
|
||||
result = pipeline.process_full(content, content_text)
|
||||
storage.save_content(result)
|
||||
|
||||
msg += f"\n🔄 处理完成\n"
|
||||
msg += f" 📄 笔记: {result.output_path or 'N/A'}\n"
|
||||
msg += f" 🏷️ 标签: {', '.join(result.tags)}"
|
||||
except Exception as e:
|
||||
msg += f"\n⚠️ 处理失败: {e}"
|
||||
|
||||
return True, msg
|
||||
|
||||
except ImportError as e:
|
||||
return False, f"依赖模块未安装: {e}"
|
||||
except Exception as e:
|
||||
return False, f"处理失败: {e}"
|
||||
|
||||
|
||||
def process_video_full(source_id: str, enable_git_sync: bool = False) -> Tuple[bool, str]:
|
||||
"""
|
||||
使用主流程完整处理视频（新接口）
|
||||
|
||||
完整流程: 提取 → 处理 → 输出 → 同步
|
||||
|
||||
Args:
|
||||
source_id: 视频ID（支持BV号、URL）
|
||||
enable_git_sync: 是否启用Git同步
|
||||
|
||||
Returns:
|
||||
(success, message) tuple
|
||||
"""
|
||||
if not MAIN_PIPELINE_AVAILABLE:
|
||||
return False, "主流程模块未可用，请检查依赖"
|
||||
|
||||
try:
|
||||
# 导入配置
|
||||
from src.config import get_config
|
||||
|
||||
# 获取配置
|
||||
config = get_config()
|
||||
|
||||
# 创建主流程
|
||||
pipeline = MainPipeline(config=config, enable_git_sync=enable_git_sync)
|
||||
|
||||
# 处理视频
|
||||
result = pipeline.process_video(source_id)
|
||||
|
||||
# 返回结果
|
||||
if result['success']:
|
||||
return True, result['message']
|
||||
else:
|
||||
return False, result.get('error', 'Unknown error')
|
||||
|
||||
except Exception as e:
|
||||
return False, f"处理失败: {e}"
|
||||
|
||||
|
||||
def handle_message(text: str) -> Optional[str]:
|
||||
"""
|
||||
Main entry point: handle incoming message and return response.
|
||||
|
||||
Args:
|
||||
text: User's message text
|
||||
|
||||
Returns:
|
||||
Response message, or None if no action needed
|
||||
"""
|
||||
# First, check for short links and provide help
|
||||
has_short, short_msg = detect_and_format(text)
|
||||
|
||||
# Detect all links (including short ones)
|
||||
detected = detect_links(text)
|
||||
|
||||
if not detected:
|
||||
return None
|
||||
|
||||
# If we have short links, show help message first
|
||||
if has_short:
|
||||
return short_msg
|
||||
|
||||
# Check if this is a confirmation (y/yes)
|
||||
text_lower = text.lower().strip()
|
||||
if text_lower in ('y', 'yes', '是', '好的', 'ok'):
|
||||
# This would need state management - for now return a message
|
||||
# about using explicit command
|
||||
return "请使用 /kb-add <URL> 命令添加内容，或重新发送链接"
|
||||
|
||||
# Show preview
|
||||
preview = format_preview(detected)
|
||||
prompt = get_action_prompt(detected)
|
||||
|
||||
return f"{preview}\n{prompt}"
|
||||
|
||||
|
||||
# CLI interface for testing
|
||||
def main():
|
||||
"""CLI for testing the handler."""
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 handler.py <command> [options]")
|
||||
print("\nCommands:")
|
||||
print(" detect 'text' - 检测文本中的内容链接")
|
||||
print(" process <BV/URL> - 使用主流程处理视频")
|
||||
print(" process <BV/URL> --git - 处理视频并同步到Git")
|
||||
print("\nExamples:")
|
||||
print(' python3 handler.py detect "https://www.bilibili.com/video/BV1xx411c7mD"')
|
||||
print(' python3 handler.py process BV1xx411c7mD')
|
||||
print(' python3 handler.py process BV1xx411c7mD --git')
|
||||
sys.exit(1)
|
||||
|
||||
command = sys.argv[1].lower()
|
||||
|
||||
if command == 'detect':
|
||||
# 检测链接
|
||||
if len(sys.argv) < 3:
|
||||
print("Error: Please provide text to detect")
|
||||
sys.exit(1)
|
||||
text = ' '.join(sys.argv[2:])
|
||||
response = handle_message(text)
|
||||
if response:
|
||||
print(response)
|
||||
else:
|
||||
print("No content links detected")
|
||||
|
||||
elif command == 'process':
|
||||
# 使用主流程处理视频
|
||||
if len(sys.argv) < 3:
|
||||
print("Error: Please provide BV number or URL")
|
||||
sys.exit(1)
|
||||
|
||||
source_id = sys.argv[2]
|
||||
enable_git = '--git' in sys.argv
|
||||
|
||||
print(f"Processing video: {source_id}")
|
||||
if enable_git:
|
||||
print("Git sync: enabled")
|
||||
else:
|
||||
print("Git sync: disabled")
|
||||
print()
|
||||
|
||||
success, message = process_video_full(source_id, enable_git_sync=enable_git)
|
||||
|
||||
if success:
|
||||
print(message)
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(f"\n❌ Failed: {message}")
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
# 兼容旧格式：直接传入文本进行检测
|
||||
text = sys.argv[1]
|
||||
response = handle_message(text)
|
||||
if response:
|
||||
print(response)
|
||||
else:
|
||||
print("No content links detected")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+489
@@ -0,0 +1,489 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Obsidian KB - 主流程集成
|
||||
|
||||
整合提取层、处理层、输出层，实现完整的数据流：
|
||||
提取(BilibiliExtractor) → 处理(DeepAnalyzer/ContentChunker) → 输出(ObsidianWriter) → 同步(GitSync)
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# 添加项目路径
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from src.extractors.bilibili_extractor import BilibiliExtractor, ExtractionError
|
||||
from src.deep_analyzer import DeepAnalyzer
|
||||
from src.content_chunker import ContentChunker, Chunk
|
||||
from src.outputs.obsidian_writer import ObsidianWriter
|
||||
from src.outputs.git_sync import GitSync
|
||||
from src.models import Content, ContentType, ContentStatus, TimelineEntry
|
||||
from src.config import get_config
|
||||
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.StreamHandler(sys.stdout),
|
||||
logging.FileHandler('obsidian_kb.log', encoding='utf-8')
|
||||
]
|
||||
)
|
||||
logger = logging.getLogger('ObsidianKB')
|
||||
|
||||
|
||||
class MainPipeline:
|
||||
"""主流程集成器"""
|
||||
|
||||
def __init__(self, config=None, enable_git_sync: bool = False):
|
||||
"""
|
||||
初始化主流程
|
||||
|
||||
Args:
|
||||
config: 配置对象
|
||||
enable_git_sync: 是否启用Git同步
|
||||
"""
|
||||
self.config = config or get_config()
|
||||
self.enable_git_sync = enable_git_sync
|
||||
|
||||
logger.info("初始化主流程...")
|
||||
logger.info(f"Obsidian vault 路径: {self.config.vault_path}")
|
||||
logger.info(f"Git 同步: {'启用' if enable_git_sync else '禁用'}")
|
||||
|
||||
# 初始化各层组件
|
||||
self.extractor = BilibiliExtractor(config=self.config)
|
||||
self.analyzer = DeepAnalyzer(ai_client=None) # 暂时不传入ai_client
|
||||
self.chunker = ContentChunker()
|
||||
self.writer = ObsidianWriter(config=self.config)
|
||||
|
||||
# 初始化Git同步器（仅在需要时）
|
||||
self.git_sync = None
|
||||
if enable_git_sync and self._is_git_enabled():
|
||||
self.git_sync = GitSync(config=self.config)
|
||||
logger.info("Git 同步器已初始化")
|
||||
|
||||
def _is_git_enabled(self) -> bool:
|
||||
"""检查Git同步是否在配置中启用"""
|
||||
return self.config.get('git.enabled', False)
|
||||
|
||||
def process_video(self, source_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
处理单个视频的完整流程
|
||||
|
||||
Args:
|
||||
source_id: 视频ID（支持BV号、URL）
|
||||
|
||||
Returns:
|
||||
处理结果字典，包含：
|
||||
- success: bool - 是否成功
|
||||
- content: Content对象 - 处理的内容
|
||||
- message: str - 处理消息
|
||||
- error: str - 错误信息（如果失败）
|
||||
"""
|
||||
result = {
|
||||
'success': False,
|
||||
'content': None,
|
||||
'message': '',
|
||||
'error': ''
|
||||
}
|
||||
|
||||
try:
|
||||
logger.info("=" * 80)
|
||||
logger.info(f"开始处理视频: {source_id}")
|
||||
logger.info("=" * 80)
|
||||
|
||||
# Step 1: 提取层 - 提取视频信息
|
||||
logger.info("\n[Step 1/5] 提取视频信息...")
|
||||
extractor_data = self._extract_video(source_id)
|
||||
|
||||
# Step 2: 构建Content对象
|
||||
logger.info("\n[Step 2/5] 构建内容对象...")
|
||||
content = self._build_content(extractor_data)
|
||||
|
||||
# Step 3: 处理层 - 深度分析
|
||||
logger.info("\n[Step 3/5] 深度分析...")
|
||||
analysis_result = self._analyze_content(extractor_data)
|
||||
|
||||
# Step 4: 处理层 - 内容分片
|
||||
logger.info("\n[Step 4/5] 内容分片...")
|
||||
chunks = self._chunk_content(extractor_data)
|
||||
|
||||
# 合并分析结果和分片信息到content
|
||||
self._merge_analysis_to_content(content, analysis_result, chunks)
|
||||
|
||||
# Step 5: 输出层 - 写入Obsidian
|
||||
logger.info("\n[Step 5/5] 写入Obsidian...")
|
||||
output_path = self._write_obsidian_note(content)
|
||||
|
||||
# Step 6: 同步层 - Git同步（可选）
|
||||
if self.enable_git_sync and self.git_sync:
|
||||
logger.info("\n[Step 6/6] Git同步...")
|
||||
self._sync_to_git(content)
|
||||
|
||||
# 更新状态
|
||||
content.update_status(ContentStatus.GENERATED)
|
||||
content.output_path = output_path
|
||||
|
||||
# 构建成功结果
|
||||
result['success'] = True
|
||||
result['content'] = content
|
||||
result['message'] = self._format_success_message(content, chunks)
|
||||
|
||||
logger.info("\n" + "=" * 80)
|
||||
logger.info(f"✅ 处理完成: {content.title}")
|
||||
logger.info(f" 📄 笔记路径: {output_path}")
|
||||
logger.info(f" 📊 字幕条数: {len(chunks) if chunks else 0}")
|
||||
logger.info("=" * 80 + "\n")
|
||||
|
||||
except ExtractionError as e:
|
||||
logger.error(f"❌ 提取失败: {e}")
|
||||
result['error'] = f"提取失败: {e}"
|
||||
result['message'] = f"提取视频信息失败: {e}"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 处理失败: {e}", exc_info=True)
|
||||
result['error'] = str(e)
|
||||
result['message'] = f"处理过程中发生错误: {e}"
|
||||
|
||||
return result
|
||||
|
||||
def _extract_video(self, source_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
提取视频信息
|
||||
|
||||
Args:
|
||||
source_id: 视频ID或URL
|
||||
|
||||
Returns:
|
||||
提取器返回的数据结构
|
||||
|
||||
Raises:
|
||||
ExtractionError: 提取失败
|
||||
"""
|
||||
logger.info(f" 正在提取: {source_id}")
|
||||
|
||||
try:
|
||||
extractor_data = self.extractor.extract(source_id)
|
||||
logger.info(f" ✅ 提取成功: {extractor_data.get('metadata', {}).get('title', '未知标题')}")
|
||||
logger.info(f" 作者: {extractor_data.get('metadata', {}).get('author', '未知')}")
|
||||
logger.info(f" 时长: {extractor_data.get('metadata', {}).get('duration', '0')}")
|
||||
|
||||
# 检查字幕
|
||||
subtitle = extractor_data.get('subtitle', '')
|
||||
if subtitle:
|
||||
subtitle_lines = subtitle.split('\n')
|
||||
logger.info(f" 字幕: {len(subtitle_lines)} 条")
|
||||
else:
|
||||
logger.warning(f" ⚠️ 无字幕")
|
||||
|
||||
return extractor_data
|
||||
|
||||
except ExtractionError as e:
|
||||
logger.error(f" ❌ 提取失败: {e}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ 提取异常: {e}", exc_info=True)
|
||||
raise ExtractionError(f"提取过程异常: {e}", source_id=source_id)
|
||||
|
||||
def _build_content(self, extractor_data: Dict[str, Any]) -> Content:
|
||||
"""
|
||||
从提取器数据构建Content对象
|
||||
|
||||
Args:
|
||||
extractor_data: 提取器返回的数据
|
||||
|
||||
Returns:
|
||||
Content对象
|
||||
"""
|
||||
metadata = extractor_data.get('metadata', {})
|
||||
|
||||
content = Content(
|
||||
source='bilibili',
|
||||
source_id=metadata.get('bvid', ''),
|
||||
url=metadata.get('url', ''),
|
||||
title=metadata.get('title', ''),
|
||||
author=metadata.get('author', ''),
|
||||
description=extractor_data.get('description', ''),
|
||||
duration=self._parse_duration(metadata.get('duration', '0')),
|
||||
content_type=ContentType.VIDEO,
|
||||
transcript=extractor_data.get('subtitle', ''),
|
||||
status=ContentStatus.SCANNED
|
||||
)
|
||||
|
||||
logger.info(f" ✅ Content对象已创建: {content.id}")
|
||||
return content
|
||||
|
||||
def _parse_duration(self, duration_str: str) -> Optional[int]:
|
||||
"""
|
||||
解析时长字符串为秒数
|
||||
|
||||
Args:
|
||||
duration_str: 时长字符串（如 "10:30"）
|
||||
|
||||
Returns:
|
||||
秒数
|
||||
"""
|
||||
try:
|
||||
if ':' in duration_str:
|
||||
parts = duration_str.split(':')
|
||||
if len(parts) == 2:
|
||||
# MM:SS
|
||||
return int(parts[0]) * 60 + int(parts[1])
|
||||
elif len(parts) == 3:
|
||||
# HH:MM:SS
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2])
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _analyze_content(self, extractor_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
深度分析内容
|
||||
|
||||
Args:
|
||||
extractor_data: 提取器返回的数据
|
||||
|
||||
Returns:
|
||||
分析结果字典
|
||||
"""
|
||||
try:
|
||||
logger.info(" 正在进行深度分析...")
|
||||
|
||||
# 使用DeepAnalyzer进行分析
|
||||
# DeepAnalyzer接收extractor_data作为输入
|
||||
analysis_result = self.analyzer.analyze(extractor_data)
|
||||
|
||||
# 将AnalysisResult转换为字典以便合并
|
||||
result_dict = {
|
||||
'main_topics': analysis_result.main_topics,
|
||||
'key_insights': analysis_result.key_insights,
|
||||
'action_items': analysis_result.action_items,
|
||||
'summary': analysis_result.summary,
|
||||
'timeline': analysis_result.timeline,
|
||||
'recommended_tags': analysis_result.recommended_tags,
|
||||
'full_analysis': analysis_result.full_analysis
|
||||
}
|
||||
|
||||
logger.info(f" ✅ 分析完成: {len(result_dict['main_topics'])} 个主题")
|
||||
|
||||
return result_dict
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ 分析失败: {e}", exc_info=True)
|
||||
# 返回默认结果，不中断流程
|
||||
return {
|
||||
'main_topics': ['未分类'],
|
||||
'key_insights': [],
|
||||
'action_items': [],
|
||||
'summary': '分析失败，无法生成摘要',
|
||||
'timeline': [],
|
||||
'recommended_tags': ['未分类'],
|
||||
'full_analysis': ''
|
||||
}
|
||||
|
||||
def _chunk_content(self, extractor_data: Dict[str, Any]) -> list:
|
||||
"""
|
||||
内容分片
|
||||
|
||||
Args:
|
||||
extractor_data: 提取器返回的数据
|
||||
|
||||
Returns:
|
||||
Chunk对象列表
|
||||
"""
|
||||
try:
|
||||
logger.info(" 正在进行内容分片...")
|
||||
|
||||
# 使用ContentChunker进行分片
|
||||
chunks = self.chunker.chunk_from_extractor(extractor_data)
|
||||
|
||||
logger.info(f" ✅ 分片完成: {len(chunks)} 段")
|
||||
|
||||
return chunks
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ 分片失败: {e}", exc_info=True)
|
||||
# 返回空列表，不中断流程
|
||||
return []
|
||||
|
||||
def _merge_analysis_to_content(self, content: Content, analysis_result: Dict[str, Any], chunks: list):
|
||||
"""
|
||||
将分析结果和分片信息合并到Content对象
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
analysis_result: 分析结果字典
|
||||
chunks: Chunk对象列表
|
||||
"""
|
||||
# 合并基本信息
|
||||
content.summary = analysis_result.get('summary', '')
|
||||
content.key_points = analysis_result.get('main_topics', [])
|
||||
content.tags = analysis_result.get('recommended_tags', [])
|
||||
|
||||
# 合并时间轴
|
||||
timeline_data = analysis_result.get('timeline', [])
|
||||
if timeline_data:
|
||||
content.timeline = [
|
||||
TimelineEntry(
|
||||
timestamp=item.get('start_time', ''),
|
||||
title=item.get('title', ''),
|
||||
description=item.get('summary', '')
|
||||
)
|
||||
for item in timeline_data
|
||||
]
|
||||
else:
|
||||
# 从chunks生成时间轴
|
||||
if chunks:
|
||||
content.timeline = [
|
||||
TimelineEntry(
|
||||
timestamp=chunk.time_range,
|
||||
title=f"片段 {chunk.index + 1}",
|
||||
description=f"时长 {int(chunk.duration)}秒"
|
||||
)
|
||||
for chunk in chunks
|
||||
]
|
||||
|
||||
logger.info(f" ✅ 数据合并完成: 时间轴 {len(content.timeline)} 条")
|
||||
|
||||
def _write_obsidian_note(self, content: Content) -> Optional[str]:
|
||||
"""
|
||||
写入Obsidian笔记
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
|
||||
Returns:
|
||||
文件路径，失败返回None
|
||||
"""
|
||||
try:
|
||||
logger.info(f" 正在写入笔记...")
|
||||
|
||||
# 确保输出目录存在
|
||||
self.config.vault_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 调用ObsidianWriter写入
|
||||
output_path = self.writer.write(content, self.config)
|
||||
|
||||
if output_path:
|
||||
logger.info(f" ✅ 笔记已写入: {output_path}")
|
||||
else:
|
||||
logger.warning(f" ⚠️ 笔记写入失败")
|
||||
|
||||
return output_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ 写入失败: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
def _sync_to_git(self, content: Content):
|
||||
"""
|
||||
同步到Git仓库
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
"""
|
||||
if not self.git_sync:
|
||||
logger.warning(" ⚠️ Git同步器未初始化，跳过同步")
|
||||
return
|
||||
|
||||
try:
|
||||
logger.info(" 正在同步到Git...")
|
||||
|
||||
success = self.git_sync.sync(self.config)
|
||||
|
||||
if success:
|
||||
logger.info(" ✅ Git同步成功")
|
||||
else:
|
||||
logger.warning(" ⚠️ Git同步失败")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ Git同步异常: {e}", exc_info=True)
|
||||
# Git同步失败不视为整体失败
|
||||
|
||||
def _format_success_message(self, content: Content, chunks: list) -> str:
|
||||
"""
|
||||
格式化成功消息
|
||||
|
||||
Args:
|
||||
content: Content对象
|
||||
chunks: Chunk列表
|
||||
|
||||
Returns:
|
||||
格式化的消息字符串
|
||||
"""
|
||||
lines = [
|
||||
"✅ 处理完成！",
|
||||
"",
|
||||
f"📺 标题: {content.title}",
|
||||
f"👤 作者: {content.author}",
|
||||
f"📄 笔记: {content.output_path or 'N/A'}",
|
||||
]
|
||||
|
||||
if content.duration:
|
||||
lines.append(f"⏱️ 时长: {self._format_duration(content.duration)}")
|
||||
|
||||
if chunks:
|
||||
lines.append(f"📊 分片: {len(chunks)} 段")
|
||||
|
||||
if content.tags:
|
||||
lines.append(f"🏷️ 标签: {', '.join(content.tags[:5])}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def _format_duration(self, seconds: int) -> str:
|
||||
"""格式化时长"""
|
||||
if not seconds:
|
||||
return "N/A"
|
||||
|
||||
hours = seconds // 3600
|
||||
minutes = (seconds % 3600) // 60
|
||||
secs = seconds % 60
|
||||
|
||||
if hours > 0:
|
||||
return f"{hours}小时{minutes}分钟"
|
||||
elif minutes > 0:
|
||||
return f"{minutes}分钟{secs}秒"
|
||||
else:
|
||||
return f"{secs}秒"
|
||||
|
||||
|
||||
def main():
|
||||
"""命令行入口"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='Obsidian KB - 主流程集成')
|
||||
parser.add_argument('source_id', help='视频ID（BV号或URL）')
|
||||
parser.add_argument('--git', action='store_true', help='启用Git同步')
|
||||
parser.add_argument('--config', help='配置文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 加载配置
|
||||
config = get_config(args.config) if args.config else get_config()
|
||||
|
||||
# 创建主流程
|
||||
pipeline = MainPipeline(config=config, enable_git_sync=args.git)
|
||||
|
||||
# 处理视频
|
||||
result = pipeline.process_video(args.source_id)
|
||||
|
||||
# 输出结果
|
||||
print(result['message'])
|
||||
|
||||
if not result['success']:
|
||||
print(f"\n❌ 失败: {result['error']}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,674 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
KB Workflow - Knowledge Base Workflow Orchestration
|
||||
|
||||
工作流编排技能，整合以下功能：
|
||||
- 扫描收件箱目录 (pending, metadata, content, analyzed)
|
||||
- 调用技能2-4处理内容
|
||||
- 归档处理完成的条目
|
||||
- 订阅更新检测
|
||||
- 生成汇报
|
||||
|
||||
Usage:
|
||||
python kb_workflow.py --trigger manual
|
||||
python kb_workflow.py --trigger scheduled
|
||||
python kb_workflow.py --status
|
||||
python kb_workflow.py --report
|
||||
python kb_workflow.py --scope pending --limit 5
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any
|
||||
from enum import Enum
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger('KBWorkflow')
|
||||
|
||||
|
||||
class TriggerType(Enum):
|
||||
MANUAL = "manual"
|
||||
SCHEDULED = "scheduled"
|
||||
WEBHOOK = "webhook"
|
||||
|
||||
|
||||
class Scope(Enum):
|
||||
ALL = "all"
|
||||
PENDING = "pending"
|
||||
METADATA = "metadata"
|
||||
CONTENT = "content"
|
||||
ANALYZED = "analyzed"
|
||||
ARCHIVE = "archive"
|
||||
|
||||
|
||||
class KBWorkflow:
|
||||
"""Knowledge Base Workflow Orchestrator"""
|
||||
|
||||
# Base directories for the inbox structure
|
||||
INBOX_DIRS = ['pending', 'metadata', 'content', 'analyzed']
|
||||
|
||||
def __init__(self,
|
||||
base_path: str = None,
|
||||
archive_path: str = None,
|
||||
output_format: str = "json"):
|
||||
"""
|
||||
Initialize KB Workflow
|
||||
|
||||
Args:
|
||||
base_path: Base path for knowledge base (default: workspace/knowledge-base)
|
||||
archive_path: Path for archived items (default: archive/)
|
||||
output_format: Output format (json, text)
|
||||
"""
|
||||
self.base_path = Path(base_path) if base_path else Path(__file__).parent / "knowledge-base"
|
||||
self.archive_path = Path(archive_path) if archive_path else self.base_path / "archive"
|
||||
self.output_format = output_format
|
||||
|
||||
# Ensure directories exist
|
||||
self._ensure_directories()
|
||||
|
||||
# Initialize subsystems
|
||||
self._init_subsystems()
|
||||
|
||||
def _ensure_directories(self):
|
||||
"""Create necessary directories"""
|
||||
# Create main inbox directories
|
||||
for subdir in self.INBOX_DIRS:
|
||||
dir_path = self.base_path / subdir
|
||||
if not dir_path.exists():
|
||||
dir_path.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Created directory: {dir_path}")
|
||||
|
||||
# Create archive directory
|
||||
if not self.archive_path.exists():
|
||||
self.archive_path.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Created archive directory: {self.archive_path}")
|
||||
|
||||
def _init_subsystems(self):
|
||||
"""Initialize subsystem references"""
|
||||
# Placeholder for skill integrations
|
||||
self.subscriptions_file = self.base_path / "subscriptions.json"
|
||||
self.report_file = self.base_path / "reports" / f"report_{datetime.now().strftime('%Y%m%d')}.json"
|
||||
|
||||
# ==================== Inbox Scanning ====================
|
||||
|
||||
def scan_inbox(self, scope: str = "all", limit: int = None) -> Dict[str, List[Dict]]:
|
||||
"""
|
||||
Scan inbox directories for items
|
||||
|
||||
Args:
|
||||
scope: Which directories to scan (all, pending, metadata, content, analyzed)
|
||||
limit: Maximum number of items to return per directory
|
||||
|
||||
Returns:
|
||||
Dictionary with directory names as keys and item lists as values
|
||||
"""
|
||||
result = {}
|
||||
|
||||
if scope == "all":
|
||||
dirs_to_scan = self.INBOX_DIRS
|
||||
elif scope in self.INBOX_DIRS:
|
||||
dirs_to_scan = [scope]
|
||||
else:
|
||||
logger.warning(f"Unknown scope: {scope}, scanning all")
|
||||
dirs_to_scan = self.INBOX_DIRS
|
||||
|
||||
for dir_name in dirs_to_scan:
|
||||
dir_path = self.base_path / dir_name
|
||||
items = self._scan_directory(dir_path)
|
||||
|
||||
if limit:
|
||||
items = items[:limit]
|
||||
|
||||
result[dir_name] = items
|
||||
logger.info(f"Scanned {dir_name}: {len(items)} items")
|
||||
|
||||
return result
|
||||
|
||||
def _scan_directory(self, path: Path) -> List[Dict]:
|
||||
"""
|
||||
Scan a directory for items
|
||||
|
||||
Args:
|
||||
path: Directory path to scan
|
||||
|
||||
Returns:
|
||||
List of item dictionaries
|
||||
"""
|
||||
items = []
|
||||
|
||||
if not path.exists():
|
||||
return items
|
||||
|
||||
for item_path in path.iterdir():
|
||||
if item_path.is_file():
|
||||
item_info = {
|
||||
'name': item_path.name,
|
||||
'path': str(item_path),
|
||||
'extension': item_path.suffix,
|
||||
'size': item_path.stat().st_size,
|
||||
'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat(),
|
||||
'type': self._classify_file_type(item_path)
|
||||
}
|
||||
items.append(item_info)
|
||||
elif item_path.is_dir():
|
||||
# Treat subdirectories as items too
|
||||
item_info = {
|
||||
'name': item_path.name,
|
||||
'path': str(item_path),
|
||||
'type': 'directory',
|
||||
'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat()
|
||||
}
|
||||
items.append(item_info)
|
||||
|
||||
return items
|
||||
|
||||
def _classify_file_type(self, path: Path) -> str:
|
||||
"""Classify file type by extension"""
|
||||
suffix = path.suffix.lower()
|
||||
|
||||
type_mapping = {
|
||||
'.md': 'markdown',
|
||||
'.txt': 'text',
|
||||
'.json': 'json',
|
||||
'.yaml': 'yaml',
|
||||
'.yml': 'yaml',
|
||||
'.pdf': 'pdf',
|
||||
'.jpg': 'image',
|
||||
'.jpeg': 'image',
|
||||
'.png': 'image',
|
||||
'.mp4': 'video',
|
||||
'.mp3': 'audio',
|
||||
'.zip': 'archive',
|
||||
'.url': 'link'
|
||||
}
|
||||
|
||||
return type_mapping.get(suffix, 'unknown')
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get current workflow status
|
||||
|
||||
Returns:
|
||||
Status dictionary
|
||||
"""
|
||||
inbox_status = {}
|
||||
total_items = 0
|
||||
|
||||
for dir_name in self.INBOX_DIRS:
|
||||
dir_path = self.base_path / dir_name
|
||||
items = self._scan_directory(dir_path)
|
||||
inbox_status[dir_name] = {
|
||||
'count': len(items),
|
||||
'path': str(dir_path)
|
||||
}
|
||||
total_items += len(items)
|
||||
|
||||
# Check archive
|
||||
archive_items = self._scan_directory(self.archive_path)
|
||||
|
||||
return {
|
||||
'status': 'ready',
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'inbox': inbox_status,
|
||||
'archive': {
|
||||
'count': len(archive_items),
|
||||
'path': str(self.archive_path)
|
||||
},
|
||||
'total_items': total_items
|
||||
}
|
||||
|
||||
# ==================== Item Processing ====================
|
||||
|
||||
def process_item(self, item: Dict, target_stage: str = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Process a single item through the workflow
|
||||
|
||||
This method calls skills 2-4 to process items:
|
||||
- Skill 2: Link parsing / URL extraction
|
||||
- Skill 3: Metadata extraction
|
||||
- Skill 4: Content analysis
|
||||
|
||||
Args:
|
||||
item: Item dictionary with path and metadata
|
||||
target_stage: Target stage to process to
|
||||
|
||||
Returns:
|
||||
Processing result dictionary
|
||||
"""
|
||||
result = {
|
||||
'item': item,
|
||||
'success': False,
|
||||
'stages_completed': [],
|
||||
'output_path': None,
|
||||
'error': None
|
||||
}
|
||||
|
||||
try:
|
||||
item_path = Path(item['path'])
|
||||
|
||||
# Determine processing stage based on current location
|
||||
current_stage = self._get_stage_from_path(item_path)
|
||||
|
||||
# Skill 2: Link/URL Parsing
|
||||
if current_stage in ['pending', None] or target_stage == 'metadata':
|
||||
logger.info(f"Stage 1 → 2: Processing link/URL for {item['name']}")
|
||||
# TODO: Integrate with link-parser skill
|
||||
result['stages_completed'].append('link_parsing')
|
||||
|
||||
# Skill 3: Metadata Extraction
|
||||
if current_stage in ['metadata', 'pending'] or target_stage == 'content':
|
||||
logger.info(f"Stage 2 → 3: Extracting metadata for {item['name']}")
|
||||
# TODO: Integrate with zhihu-metadata skill
|
||||
result['stages_completed'].append('metadata_extraction')
|
||||
|
||||
# Skill 4: Content Analysis
|
||||
if current_stage in ['content', 'metadata'] or target_stage == 'analyzed':
|
||||
logger.info(f"Stage 3 → 4: Analyzing content for {item['name']}")
|
||||
# TODO: Integrate with bilibili-analyzer skill
|
||||
result['stages_completed'].append('content_analysis')
|
||||
|
||||
# Move to next stage if successful
|
||||
if result['stages_completed']:
|
||||
self._move_to_next_stage(item_path, result['stages_completed'][-1])
|
||||
result['success'] = True
|
||||
|
||||
except Exception as e:
|
||||
result['error'] = str(e)
|
||||
logger.error(f"Processing failed: {e}")
|
||||
|
||||
return result
|
||||
|
||||
def _get_stage_from_path(self, path: Path) -> Optional[str]:
|
||||
"""Determine which stage a file is in based on its path"""
|
||||
for stage in self.INBOX_DIRS:
|
||||
if stage in str(path):
|
||||
return stage
|
||||
return None
|
||||
|
||||
def _move_to_next_stage(self, item_path: Path, completed_stage: str) -> None:
|
||||
"""Move item to the next stage directory"""
|
||||
stage_order = ['pending', 'metadata', 'content', 'analyzed']
|
||||
|
||||
try:
|
||||
current_idx = stage_order.index(completed_stage)
|
||||
if current_idx < len(stage_order) - 1:
|
||||
next_stage = stage_order[current_idx + 1]
|
||||
next_path = self.base_path / next_stage / item_path.name
|
||||
|
||||
# Move file
|
||||
if item_path.exists():
|
||||
item_path.rename(next_path)
|
||||
logger.info(f"Moved {item_path.name} → {next_stage}/")
|
||||
except ValueError:
|
||||
logger.warning(f"Unknown stage: {completed_stage}")
|
||||
|
||||
def process_all(self, scope: str = "pending", limit: int = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Process all items in specified scope
|
||||
|
||||
Args:
|
||||
scope: Which directories to process
|
||||
limit: Maximum items to process
|
||||
|
||||
Returns:
|
||||
Processing summary
|
||||
"""
|
||||
summary = {
|
||||
'scope': scope,
|
||||
'processed': 0,
|
||||
'failed': 0,
|
||||
'results': []
|
||||
}
|
||||
|
||||
inbox = self.scan_inbox(scope=scope)
|
||||
|
||||
for dir_name, items in inbox.items():
|
||||
if limit and summary['processed'] >= limit:
|
||||
break
|
||||
|
||||
for item in items:
|
||||
if limit and summary['processed'] >= limit:
|
||||
break
|
||||
|
||||
result = self.process_item(item)
|
||||
summary['results'].append(result)
|
||||
|
||||
if result['success']:
|
||||
summary['processed'] += 1
|
||||
else:
|
||||
summary['failed'] += 1
|
||||
|
||||
return summary
|
||||
|
||||
# ==================== Archiving ====================
|
||||
|
||||
def archive_item(self, item: Dict, analysis: Dict = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Archive a processed item to the archive directory
|
||||
|
||||
Args:
|
||||
item: Item to archive
|
||||
analysis: Optional analysis metadata
|
||||
|
||||
Returns:
|
||||
Archiving result
|
||||
"""
|
||||
result = {
|
||||
'success': False,
|
||||
'original_path': item['path'],
|
||||
'archive_path': None,
|
||||
'error': None
|
||||
}
|
||||
|
||||
try:
|
||||
item_path = Path(item['path'])
|
||||
|
||||
if not item_path.exists():
|
||||
result['error'] = f"Item not found: {item['path']}"
|
||||
return result
|
||||
|
||||
# Create archive subdirectory by date
|
||||
date_dir = self.archive_path / datetime.now().strftime('%Y-%m-%d')
|
||||
if not date_dir.exists():
|
||||
date_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Move item to archive
|
||||
archive_path = date_dir / item_path.name
|
||||
item_path.rename(archive_path)
|
||||
|
||||
# Save analysis metadata if provided
|
||||
if analysis:
|
||||
meta_path = date_dir / f"{item_path.stem}_meta.json"
|
||||
with open(meta_path, 'w', encoding='utf-8') as f:
|
||||
json.dump({
|
||||
'archived_at': datetime.now().isoformat(),
|
||||
'analysis': analysis
|
||||
}, f, ensure_ascii=False, indent=2)
|
||||
|
||||
result['success'] = True
|
||||
result['archive_path'] = str(archive_path)
|
||||
logger.info(f"Archived: {item['name']} → {archive_path}")
|
||||
|
||||
except Exception as e:
|
||||
result['error'] = str(e)
|
||||
logger.error(f"Archive failed: {e}")
|
||||
|
||||
return result
|
||||
|
||||
# ==================== Subscriptions ====================
|
||||
|
||||
def check_subscriptions(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Check for updates from subscribed sources
|
||||
|
||||
Returns:
|
||||
Subscription check results
|
||||
"""
|
||||
result = {
|
||||
'checked_at': datetime.now().isoformat(),
|
||||
'sources': {},
|
||||
'new_items': 0,
|
||||
'errors': []
|
||||
}
|
||||
|
||||
# Load subscription list
|
||||
subscriptions = self._load_subscriptions()
|
||||
|
||||
for sub in subscriptions:
|
||||
try:
|
||||
source_name = sub.get('name', 'unknown')
|
||||
source_type = sub.get('type', 'rss')
|
||||
|
||||
# Placeholder: Check for new content
|
||||
logger.info(f"Checking subscription: {source_name}")
|
||||
|
||||
# TODO: Implement actual subscription checking
|
||||
# - RSS feeds
|
||||
# - YouTube channel updates
|
||||
# - Bilibili user updates
|
||||
# - Custom web hooks
|
||||
|
||||
result['sources'][source_name] = {
|
||||
'type': source_type,
|
||||
'status': 'checked',
|
||||
'new_items': 0
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
result['errors'].append({
|
||||
'source': sub.get('name', 'unknown'),
|
||||
'error': str(e)
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
def _load_subscriptions(self) -> List[Dict]:
|
||||
"""Load subscriptions from file"""
|
||||
if self.subscriptions_file.exists():
|
||||
try:
|
||||
with open(self.subscriptions_file, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load subscriptions: {e}")
|
||||
|
||||
return []
|
||||
|
||||
# ==================== Reporting ====================
|
||||
|
||||
def generate_report(self, period: str = "daily") -> Dict[str, Any]:
|
||||
"""
|
||||
Generate workflow report
|
||||
|
||||
Args:
|
||||
period: Report period (daily, weekly, monthly)
|
||||
|
||||
Returns:
|
||||
Report dictionary
|
||||
"""
|
||||
# Calculate date range
|
||||
now = datetime.now()
|
||||
|
||||
if period == "daily":
|
||||
start_date = now - timedelta(days=1)
|
||||
elif period == "weekly":
|
||||
start_date = now - timedelta(weeks=1)
|
||||
elif period == "monthly":
|
||||
start_date = now - timedelta(days=30)
|
||||
else:
|
||||
start_date = now - timedelta(days=1)
|
||||
|
||||
# Gather statistics
|
||||
status = self.get_status()
|
||||
inbox = self.scan_inbox(scope="all")
|
||||
|
||||
# Build report
|
||||
report = {
|
||||
'generated_at': now.isoformat(),
|
||||
'period': period,
|
||||
'start_date': start_date.isoformat(),
|
||||
'summary': {
|
||||
'total_items_inbox': status['total_items'],
|
||||
'total_archived': status['archive']['count'],
|
||||
'items_by_stage': status['inbox']
|
||||
},
|
||||
'processing_stats': {
|
||||
'pending': len(inbox.get('pending', [])),
|
||||
'metadata': len(inbox.get('metadata', [])),
|
||||
'content': len(inbox.get('content', [])),
|
||||
'analyzed': len(inbox.get('analyzed', []))
|
||||
},
|
||||
'recommendations': self._generate_recommendations(status, inbox)
|
||||
}
|
||||
|
||||
# Save report
|
||||
if self.report_file:
|
||||
self.report_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.report_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
logger.info(f"Report saved: {self.report_file}")
|
||||
|
||||
return report
|
||||
|
||||
def _generate_recommendations(self, status: Dict, inbox: Dict) -> List[str]:
|
||||
"""Generate processing recommendations"""
|
||||
recommendations = []
|
||||
|
||||
pending_count = status['inbox'].get('pending', {}).get('count', 0)
|
||||
analyzed_count = status['inbox'].get('analyzed', {}).get('count', 0)
|
||||
|
||||
if pending_count > 10:
|
||||
recommendations.append(f"⚠️ 待处理内容较多 ({pending_count} 项)，建议优先处理")
|
||||
|
||||
if analyzed_count > 20:
|
||||
recommendations.append(f"📦 已分析内容较多 ({analyzed_count} 项)，建议归档")
|
||||
|
||||
if status['total_items'] == 0:
|
||||
recommendations.append("📭 收件箱为空，暂无待处理内容")
|
||||
|
||||
return recommendations
|
||||
|
||||
# ==================== CLI Interface ====================
|
||||
|
||||
def run(self, trigger: str = "manual", scope: str = "all",
|
||||
limit: int = None, report: bool = False):
|
||||
"""
|
||||
Main entry point for running the workflow
|
||||
|
||||
Args:
|
||||
trigger: Trigger type (manual, scheduled)
|
||||
scope: Processing scope
|
||||
limit: Maximum items to process
|
||||
report: Generate report after processing
|
||||
"""
|
||||
logger.info(f"KB Workflow started: trigger={trigger}, scope={scope}")
|
||||
|
||||
# Handle different triggers
|
||||
if trigger == "status":
|
||||
result = self.get_status()
|
||||
elif trigger == "report":
|
||||
result = self.generate_report()
|
||||
elif trigger == "subscriptions":
|
||||
result = self.check_subscriptions()
|
||||
else:
|
||||
# Process items
|
||||
result = self.process_all(scope=scope, limit=limit)
|
||||
|
||||
# Generate report if requested
|
||||
if report:
|
||||
report_result = self.generate_report()
|
||||
result['report'] = report_result
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
"""CLI entry point"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='KB Workflow - Knowledge Base Workflow Orchestration',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Get workflow status
|
||||
python kb_workflow.py --trigger status
|
||||
|
||||
# Process pending items manually
|
||||
python kb_workflow.py --trigger manual --scope pending
|
||||
|
||||
# Process up to 5 items
|
||||
python kb_workflow.py --trigger manual --limit 5
|
||||
|
||||
# Generate daily report
|
||||
python kb_workflow.py --trigger scheduled --report
|
||||
|
||||
# Check subscriptions
|
||||
python kb_workflow.py --trigger scheduled --scope subscriptions
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--trigger',
|
||||
type=str,
|
||||
default='manual',
|
||||
choices=['manual', 'scheduled', 'webhook', 'status', 'report', 'subscriptions'],
|
||||
help='Workflow trigger type (default: manual)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--scope',
|
||||
type=str,
|
||||
default='all',
|
||||
choices=['all', 'pending', 'metadata', 'content', 'analyzed', 'archive'],
|
||||
help='Processing scope (default: all)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--limit',
|
||||
type=int,
|
||||
default=None,
|
||||
help='Maximum items to process'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--report',
|
||||
action='store_true',
|
||||
help='Generate report after processing'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--output',
|
||||
type=str,
|
||||
choices=['json', 'text'],
|
||||
default='json',
|
||||
help='Output format (default: json)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--base-path',
|
||||
type=str,
|
||||
default=None,
|
||||
help='Base path for knowledge base'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--verbose',
|
||||
action='store_true',
|
||||
help='Enable verbose logging'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
# Initialize workflow
|
||||
workflow = KBWorkflow(base_path=args.base_path)
|
||||
|
||||
# Run workflow
|
||||
result = workflow.run(
|
||||
trigger=args.trigger,
|
||||
scope=args.scope,
|
||||
limit=args.limit,
|
||||
report=args.report
|
||||
)
|
||||
|
||||
# Output result
|
||||
if args.output == 'json':
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
|
||||
else:
|
||||
print(result)
|
||||
|
||||
return 0 if result else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"generated_at": "2026-02-10T12:13:35.989689",
|
||||
"period": "daily",
|
||||
"start_date": "2026-02-09T12:13:35.989689",
|
||||
"summary": {
|
||||
"total_items_inbox": 0,
|
||||
"total_archived": 0,
|
||||
"items_by_stage": {
|
||||
"pending": {
|
||||
"count": 0,
|
||||
"path": "/home/admin/.openclaw/workspace/skills/kb-workflow/knowledge-base/pending"
|
||||
},
|
||||
"metadata": {
|
||||
"count": 0,
|
||||
"path": "/home/admin/.openclaw/workspace/skills/kb-workflow/knowledge-base/metadata"
|
||||
},
|
||||
"content": {
|
||||
"count": 0,
|
||||
"path": "/home/admin/.openclaw/workspace/skills/kb-workflow/knowledge-base/content"
|
||||
},
|
||||
"analyzed": {
|
||||
"count": 0,
|
||||
"path": "/home/admin/.openclaw/workspace/skills/kb-workflow/knowledge-base/analyzed"
|
||||
}
|
||||
}
|
||||
},
|
||||
"processing_stats": {
|
||||
"pending": 0,
|
||||
"metadata": 0,
|
||||
"content": 0,
|
||||
"analyzed": 0
|
||||
},
|
||||
"recommendations": [
|
||||
"📭 收件箱为空，暂无待处理内容"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
# KB Zhihu Metadata Skill
|
||||
|
||||
**kb-zhihu-metadata** - Extract Zhihu question/answer/user metadata.
|
||||
|
||||
## Features
|
||||
|
||||
- Question metadata extraction
|
||||
- Answer metadata extraction
|
||||
- User information extraction
|
||||
- JSON output format
|
||||
- Markdown conversion support
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Question metadata
|
||||
python zhihu_metadata.py --id "266309957" --type question
|
||||
|
||||
# Answer metadata
|
||||
python zhihu_metadata.py --id "2003427891925509507" --type answer
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"platform": "zhihu",
|
||||
"type": "question",
|
||||
"id": "266309957",
|
||||
"title": "问题标题",
|
||||
"author": "作者名字",
|
||||
"answer_count": 50,
|
||||
"followers": 1000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
cp -r kb-zhihu-metadata/ ~/skills/
|
||||
```
|
||||
@@ -0,0 +1,17 @@
|
||||
from markdownify import MarkdownConverter as _MarkdownConverter
|
||||
|
||||
class MarkdownConverter:
|
||||
"""HTML 转 Markdown 转换器，保留完整格式"""
|
||||
|
||||
def __init__(self):
|
||||
self.converter = _MarkdownConverter(
|
||||
heading_style="ATX",
|
||||
bullets="*,-",
|
||||
code_blocks=True,
|
||||
tables=True,
|
||||
strip=["script", "style", "nav", "footer", "aside"]
|
||||
)
|
||||
|
||||
def convert(self, html: str) -> str:
|
||||
"""转换 HTML 为 Markdown"""
|
||||
return self.converter.convert(html)
|
||||
@@ -0,0 +1,385 @@
|
||||
"""
|
||||
知乎 Playwright 内容提取器
|
||||
使用 Playwright 浏览器自动化工具抓取知乎内容
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Optional, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# 添加当前目录到 Python 路径以支持绝对导入
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
if current_dir not in sys.path:
|
||||
sys.path.insert(0, current_dir)
|
||||
|
||||
try:
|
||||
from .markdown_converter import MarkdownConverter
|
||||
except ImportError:
|
||||
from markdown_converter import MarkdownConverter
|
||||
|
||||
|
||||
class PlaywrightBrowser:
|
||||
"""Playwright 浏览器管理器"""
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||
self.headless = headless
|
||||
self.timeout = timeout * 1000
|
||||
self.browser = None
|
||||
self.context = None
|
||||
self.page = None
|
||||
|
||||
def start(self):
|
||||
"""启动浏览器"""
|
||||
from playwright.sync_api import sync_playwright
|
||||
self.playwright = sync_playwright().start()
|
||||
self.browser = self.playwright.chromium.launch(headless=self.headless)
|
||||
self.context = self.browser.new_context(
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
|
||||
)
|
||||
self.page = self.context.new_page()
|
||||
self.page.set_default_timeout(self.timeout)
|
||||
|
||||
def close(self):
|
||||
"""关闭浏览器"""
|
||||
if self.page:
|
||||
self.page.close()
|
||||
if self.context:
|
||||
self.context.close()
|
||||
if self.browser:
|
||||
self.browser.close()
|
||||
if hasattr(self, 'playwright'):
|
||||
self.playwright.stop()
|
||||
|
||||
def goto(self, url: str, wait_until: str = "networkidle"):
|
||||
"""导航到指定URL"""
|
||||
return self.page.goto(url, wait_until=wait_until)
|
||||
|
||||
def wait_for_selector(self, selector: str, timeout: int = None):
|
||||
"""等待选择器出现"""
|
||||
timeout_ms = timeout * 1000 if timeout else self.timeout
|
||||
return self.page.wait_for_selector(selector, timeout=timeout_ms)
|
||||
|
||||
def query_selector(self, selector: str):
|
||||
"""查询选择器"""
|
||||
return self.page.query_selector(selector)
|
||||
|
||||
def query_selector_all(self, selector: str):
|
||||
"""查询所有匹配的选择器"""
|
||||
return self.page.query_selector_all(selector)
|
||||
|
||||
def evaluate(self, script: str):
|
||||
"""执行 JavaScript 代码"""
|
||||
return self.page.evaluate(script)
|
||||
|
||||
|
||||
class ZhihuPlaywrightExtractor:
|
||||
"""知乎内容提取器 - 使用 Playwright"""
|
||||
|
||||
URL_PATTERNS = {
|
||||
'answer': r'zhihu\.com/question/\d+/answer/(\d+)',
|
||||
'article': r'zhuanlan\.zhihu\.com/p/(\d+)',
|
||||
'question': r'zhihu\.com/question/(\d+)(?:$|/)',
|
||||
}
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||
"""
|
||||
初始化提取器
|
||||
|
||||
Args:
|
||||
headless: 是否使用无头浏览器
|
||||
timeout: 超时时间（秒）
|
||||
"""
|
||||
self.browser = PlaywrightBrowser(headless=headless, timeout=timeout)
|
||||
self.markdown_converter = MarkdownConverter()
|
||||
|
||||
def identify_url_type(self, url: str) -> Optional[str]:
|
||||
"""
|
||||
识别 URL 类型
|
||||
|
||||
Returns:
|
||||
'answer', 'article', 'question' 或 None
|
||||
"""
|
||||
for url_type, pattern in self.URL_PATTERNS.items():
|
||||
if re.search(pattern, url):
|
||||
return url_type
|
||||
return None
|
||||
|
||||
def extract_id(self, url: str) -> Optional[str]:
|
||||
"""
|
||||
从 URL 中提取内容 ID
|
||||
|
||||
Returns:
|
||||
内容 ID 或 None
|
||||
"""
|
||||
url_type = self.identify_url_type(url)
|
||||
if url_type:
|
||||
pattern = self.URL_PATTERNS[url_type]
|
||||
match = re.search(pattern, url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
def _extract_text_content(self, element) -> str:
|
||||
"""提取元素的文本内容"""
|
||||
if element is None:
|
||||
return ''
|
||||
return element.inner_text() if hasattr(element, 'inner_text') else ''
|
||||
|
||||
def _extract_html_content(self, element) -> str:
|
||||
"""提取元素的 HTML 内容"""
|
||||
if element is None:
|
||||
return ''
|
||||
return element.inner_html() if hasattr(element, 'inner_html') else ''
|
||||
|
||||
def _extract_answer_content(self) -> Dict[str, Any]:
|
||||
"""提取回答内容"""
|
||||
result = {
|
||||
'type': 'answer',
|
||||
'title': '',
|
||||
'author': '',
|
||||
'content': '',
|
||||
'content_markdown': '',
|
||||
'publish_time': '',
|
||||
'vote_up': 0,
|
||||
}
|
||||
|
||||
# 打印页面标题用于调试
|
||||
page_title = self.browser.page.title()
|
||||
print(f"页面标题: {page_title}")
|
||||
|
||||
# 尝试保存页面快照用于调试
|
||||
try:
|
||||
import os
|
||||
debug_dir = '/home/admin/.openclaw/workspace/debug'
|
||||
os.makedirs(debug_dir, exist_ok=True)
|
||||
self.browser.page.screenshot(path=os.path.join(debug_dir, 'zhihu_screenshot.png'))
|
||||
with open(os.path.join(debug_dir, 'zhihu_page.html'), 'w', encoding='utf-8') as f:
|
||||
f.write(self.browser.page.content())
|
||||
print(f"页面快照已保存到 {debug_dir}")
|
||||
except Exception as e:
|
||||
print(f"保存页面快照时出错: {e}")
|
||||
|
||||
# 尝试多个可能的选择器
|
||||
selectors = {
|
||||
'title': ['h1.QuestionHeader-title', 'h1', '.QuestionHeader-main'],
|
||||
'author': ['.AuthorInfo-name', '.AuthorInfo .Text', '[itemprop="name"]'],
|
||||
'content': ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist] article'],
|
||||
'time': ['.ContentItem-time', '[itemprop="dateCreated"]'],
|
||||
'vote': ['.VoteButton--up .VoteButton-label', '.VoteButton--up']
|
||||
}
|
||||
|
||||
for selector in selectors['title']:
|
||||
title_elem = self.browser.query_selector(selector)
|
||||
if title_elem:
|
||||
result['title'] = self._extract_text_content(title_elem)
|
||||
print(f"找到标题（使用选择器 {selector}）: {result['title'][:50]}...")
|
||||
break
|
||||
|
||||
for selector in selectors['author']:
|
||||
author_elem = self.browser.query_selector(selector)
|
||||
if author_elem:
|
||||
result['author'] = self._extract_text_content(author_elem)
|
||||
print(f"找到作者（使用选择器 {selector}）: {result['author']}")
|
||||
break
|
||||
|
||||
for selector in selectors['content']:
|
||||
content_elem = self.browser.query_selector(selector)
|
||||
if content_elem:
|
||||
html_content = self._extract_html_content(content_elem)
|
||||
result['content'] = html_content
|
||||
result['content_markdown'] = self.markdown_converter.convert(html_content)
|
||||
print(f"找到内容（使用选择器 {selector}），长度: {len(result['content_markdown'])}")
|
||||
break
|
||||
|
||||
for selector in selectors['time']:
|
||||
time_elem = self.browser.query_selector(selector)
|
||||
if time_elem:
|
||||
result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem)
|
||||
break
|
||||
|
||||
for selector in selectors['vote']:
|
||||
vote_elem = self.browser.query_selector(selector)
|
||||
if vote_elem:
|
||||
vote_text = self._extract_text_content(vote_elem)
|
||||
try:
|
||||
result['vote_up'] = int(vote_text.replace(',', ''))
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
def _extract_article_content(self) -> Dict[str, Any]:
|
||||
"""提取文章内容"""
|
||||
result = {
|
||||
'type': 'article',
|
||||
'title': '',
|
||||
'author': '',
|
||||
'content': '',
|
||||
'content_markdown': '',
|
||||
'publish_time': '',
|
||||
'vote_up': 0,
|
||||
}
|
||||
|
||||
# 提取文章标题
|
||||
title_elem = self.browser.query_selector('.Post-Title')
|
||||
if title_elem:
|
||||
result['title'] = self._extract_text_content(title_elem)
|
||||
|
||||
# 提取作者信息
|
||||
author_elem = self.browser.query_selector('.AuthorInfo-name')
|
||||
if author_elem:
|
||||
result['author'] = self._extract_text_content(author_elem)
|
||||
|
||||
# 提取文章内容
|
||||
content_elem = self.browser.query_selector('.Post-RichText')
|
||||
if content_elem:
|
||||
html_content = self._extract_html_content(content_elem)
|
||||
result['content'] = html_content
|
||||
result['content_markdown'] = self.markdown_converter.convert(html_content)
|
||||
|
||||
# 提取发布时间
|
||||
time_elem = self.browser.query_selector('.ContentItem-time')
|
||||
if time_elem:
|
||||
result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem)
|
||||
|
||||
# 提取点赞数
|
||||
vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label')
|
||||
if vote_elem:
|
||||
vote_text = self._extract_text_content(vote_elem)
|
||||
try:
|
||||
result['vote_up'] = int(vote_text.replace(',', ''))
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _extract_question_content(self) -> Dict[str, Any]:
|
||||
"""提取问题内容（问题页本身）"""
|
||||
result = {
|
||||
'type': 'question',
|
||||
'title': '',
|
||||
'author': '',
|
||||
'content': '',
|
||||
'content_markdown': '',
|
||||
'publish_time': '',
|
||||
'vote_up': 0,
|
||||
}
|
||||
|
||||
# 提取问题标题
|
||||
title_elem = self.browser.query_selector('h1.QuestionHeader-title')
|
||||
if title_elem:
|
||||
result['title'] = self._extract_text_content(title_elem)
|
||||
|
||||
# 提取问题详情
|
||||
detail_elem = self.browser.query_selector('.QuestionHeader-detail')
|
||||
if detail_elem:
|
||||
html_content = self._extract_html_content(detail_elem)
|
||||
result['content'] = html_content
|
||||
result['content_markdown'] = self.markdown_converter.convert(html_content)
|
||||
|
||||
# 问题的作者通常是提问者
|
||||
author_elem = self.browser.query_selector('.AuthorInfo-name')
|
||||
if author_elem:
|
||||
result['author'] = self._extract_text_content(author_elem)
|
||||
|
||||
return result
|
||||
|
||||
def extract(self, url: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
从 URL 提取知乎内容
|
||||
|
||||
Args:
|
||||
url: 知乎 URL
|
||||
|
||||
Returns:
|
||||
包含提取内容的字典，结构如下:
|
||||
{
|
||||
'type': 'answer' | 'article' | 'question',
|
||||
'title': str,
|
||||
'author': str,
|
||||
'content': str (HTML),
|
||||
'content_markdown': str (Markdown),
|
||||
'publish_time': str,
|
||||
'vote_up': int,
|
||||
}
|
||||
如果提取失败返回 None
|
||||
"""
|
||||
try:
|
||||
# 识别 URL 类型
|
||||
url_type = self.identify_url_type(url)
|
||||
if not url_type:
|
||||
print(f"无法识别 URL 类型: {url}")
|
||||
return None
|
||||
|
||||
print(f"URL 类型: {url_type}")
|
||||
print(f"启动浏览器...")
|
||||
|
||||
# 启动浏览器
|
||||
self.browser.start()
|
||||
|
||||
print(f"访问页面: {url}")
|
||||
# 访问页面
|
||||
self.browser.goto(url, wait_until="domcontentloaded")
|
||||
|
||||
print("等待页面加载...")
|
||||
import time
|
||||
time.sleep(3) # 等待 JavaScript 执行
|
||||
|
||||
# 尝试多个可能的选择器
|
||||
if url_type == 'answer':
|
||||
selectors = ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist]']
|
||||
print(f"尝试查找选择器: {selectors}")
|
||||
result = self._extract_answer_content()
|
||||
elif url_type == 'article':
|
||||
selectors = ['.Post-RichText', '.Post-Title']
|
||||
print(f"尝试查找选择器: {selectors}")
|
||||
result = self._extract_article_content()
|
||||
else: # question
|
||||
selectors = ['.QuestionHeader-title', 'h1']
|
||||
print(f"尝试查找选择器: {selectors}")
|
||||
result = self._extract_question_content()
|
||||
|
||||
# 添加额外信息
|
||||
result['url'] = url
|
||||
result['content_id'] = self.extract_id(url)
|
||||
|
||||
print(f"提取成功! 标题: {result['title'][:50]}...")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"提取知乎内容时出错: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return None
|
||||
finally:
|
||||
# 关闭浏览器
|
||||
self.browser.close()
|
||||
|
||||
def __enter__(self):
|
||||
"""上下文管理器入口"""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""上下文管理器出口"""
|
||||
self.browser.close()
|
||||
return False
|
||||
|
||||
|
||||
# 便捷函数
|
||||
def extract_zhihu_content(url: str, headless: bool = True, timeout: int = 30) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
便捷函数：提取知乎内容
|
||||
|
||||
Args:
|
||||
url: 知乎 URL
|
||||
headless: 是否使用无头浏览器
|
||||
timeout: 超时时间（秒）
|
||||
|
||||
Returns:
|
||||
提取的内容字典
|
||||
"""
|
||||
with ZhihuPlaywrightExtractor(headless=headless, timeout=timeout) as extractor:
|
||||
return extractor.extract(url)
|
||||
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
知乎元数据获取工具
|
||||
|
||||
支持获取知乎问题、回答、用户的元数据信息。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Optional, Any
|
||||
|
||||
# 添加当前目录到 Python 路径
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
if current_dir not in sys.path:
|
||||
sys.path.insert(0, current_dir)
|
||||
|
||||
from zhihu_extractor import ZhihuPlaywrightExtractor
|
||||
|
||||
|
||||
def extract_question(question_id: str, headless: bool = True) -> Dict[str, Any]:
|
||||
"""
|
||||
获取知乎问题元数据
|
||||
|
||||
Args:
|
||||
question_id: 问题 ID (数字字符串)
|
||||
headless: 是否使用无头浏览器
|
||||
|
||||
Returns:
|
||||
问题元数据字典
|
||||
"""
|
||||
url = f"https://www.zhihu.com/question/{question_id}"
|
||||
return _extract_metadata(url, "question", headless)
|
||||
|
||||
|
||||
def extract_answer(answer_id: str, headless: bool = True) -> Dict[str, Any]:
|
||||
"""
|
||||
获取知乎回答元数据
|
||||
|
||||
Args:
|
||||
answer_id: 回答 ID (数字字符串)
|
||||
headless: 是否使用无头浏览器
|
||||
|
||||
Returns:
|
||||
回答元数据字典
|
||||
"""
|
||||
# 回答 URL 格式: https://www.zhihu.com/question/{question_id}/answer/{answer_id}
|
||||
# 需要先获取问题ID，这里简化处理，尝试从回答ID构建URL
|
||||
# 实际上知乎回答URL需要问题ID，这里用模拟URL
|
||||
url = f"https://www.zhihu.com/question/0/answer/{answer_id}"
|
||||
return _extract_metadata(url, "answer", headless)
|
||||
|
||||
|
||||
def extract_user(url_token: str, headless: bool = True) -> Dict[str, Any]:
|
||||
"""
|
||||
获取知乎用户信息
|
||||
|
||||
Args:
|
||||
url_token: 用户 URL token
|
||||
headless: 是否使用无头浏览器
|
||||
|
||||
Returns:
|
||||
用户信息字典
|
||||
"""
|
||||
url = f"https://www.zhihu.com/people/{url_token}"
|
||||
return _extract_user_metadata(url, headless)
|
||||
|
||||
|
||||
def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, Any]:
|
||||
"""
|
||||
内部函数：提取元数据
|
||||
|
||||
Args:
|
||||
url: 知乎 URL
|
||||
content_type: 内容类型 (question/answer)
|
||||
headless: 是否使用无头浏览器
|
||||
|
||||
Returns:
|
||||
元数据字典
|
||||
"""
|
||||
extractor = ZhihuPlaywrightExtractor(headless=headless)
|
||||
result = extractor.extract(url)
|
||||
|
||||
if result:
|
||||
# 标准化输出格式
|
||||
metadata = {
|
||||
"type": content_type,
|
||||
"id": result.get("content_id", ""),
|
||||
"url": result.get("url", url),
|
||||
"title": result.get("title", ""),
|
||||
"author": result.get("author", ""),
|
||||
"publish_time": result.get("publish_time", ""),
|
||||
"vote_up": result.get("vote_up", 0),
|
||||
"content_markdown": result.get("content_markdown", ""),
|
||||
"raw_html": result.get("content", ""),
|
||||
}
|
||||
return metadata
|
||||
else:
|
||||
return {
|
||||
"type": content_type,
|
||||
"error": "Failed to extract content",
|
||||
"url": url
|
||||
}
|
||||
|
||||
|
||||
def _extract_user_metadata(url: str, headless: bool) -> Dict[str, Any]:
|
||||
"""
|
||||
内部函数：提取用户信息
|
||||
|
||||
Args:
|
||||
url: 用户主页 URL
|
||||
headless: 是否使用无头浏览器
|
||||
|
||||
Returns:
|
||||
用户信息字典
|
||||
"""
|
||||
extractor = ZhihuPlaywrightExtractor(headless=headless)
|
||||
|
||||
try:
|
||||
extractor.browser.start()
|
||||
extractor.browser.goto(url, wait_until="networkidle")
|
||||
|
||||
import time
|
||||
time.sleep(3)
|
||||
|
||||
# 提取用户名
|
||||
username = ""
|
||||
username_elem = extractor.browser.query_selector('.ProfileHeader-name')
|
||||
if username_elem:
|
||||
username = username_elem.inner_text()
|
||||
|
||||
# 提取个人简介
|
||||
headline = ""
|
||||
headline_elem = extractor.browser.query_selector('.ProfileHeader-headline')
|
||||
if headline_elem:
|
||||
headline = headline_elem.inner_text()
|
||||
|
||||
# 提取简介详情
|
||||
description = ""
|
||||
desc_elem = extractor.browser.query_selector('.ProfileHeader-description')
|
||||
if desc_elem:
|
||||
description = desc_elem.inner_text()
|
||||
|
||||
# 提取关注者数
|
||||
followers = 0
|
||||
follower_elem = extractor.browser.query_selector('.FollowshipCard-followersNum')
|
||||
if follower_elem:
|
||||
follower_text = follower_elem.inner_text()
|
||||
try:
|
||||
followers = int(follower_text.replace(',', ''))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 提取关注者数
|
||||
following = 0
|
||||
following_elem = extractor.browser.query_selector('.FollowshipCard-followingsNum')
|
||||
if following_elem:
|
||||
following_text = following_elem.inner_text()
|
||||
try:
|
||||
following = int(following_text.replace(',', ''))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 提取回答数
|
||||
answers = 0
|
||||
answers_elem = extractor.browser.query_selector('.Profile-mainColumn .Tabs-item span')
|
||||
if answers_elem:
|
||||
answers_text = answers_elem.inner_text()
|
||||
try:
|
||||
answers = int(answers_text.replace(',', ''))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
metadata = {
|
||||
"type": "user",
|
||||
"url": url,
|
||||
"username": username,
|
||||
"headline": headline,
|
||||
"description": description,
|
||||
"followers": followers,
|
||||
"following": following,
|
||||
"answers_count": answers,
|
||||
}
|
||||
return metadata
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"type": "user",
|
||||
"error": str(e),
|
||||
"url": url
|
||||
}
|
||||
finally:
|
||||
extractor.browser.close()
|
||||
|
||||
|
||||
def main():
|
||||
"""命令行入口"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="知乎元数据获取工具",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
python zhihu_metadata.py --id "266309957" --type question
|
||||
python zhihu_metadata.py --id "2003427891925509507" --type answer
|
||||
python zhihu_metadata.py --id "example-user" --type user
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--id", "-i",
|
||||
required=True,
|
||||
help="知乎内容 ID (问题ID、回答ID或用户URL token)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--type", "-t",
|
||||
required=True,
|
||||
choices=["question", "answer", "user"],
|
||||
help="内容类型"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--output", "-o",
|
||||
default=None,
|
||||
help="输出文件路径 (默认: stdout)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--headless",
|
||||
action="store_true",
|
||||
default=True,
|
||||
help="使用无头浏览器模式 (默认: True)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 提取元数据
|
||||
if args.type == "question":
|
||||
result = extract_question(args.id, headless=args.headless)
|
||||
elif args.type == "answer":
|
||||
result = extract_answer(args.id, headless=args.headless)
|
||||
else: # user
|
||||
result = extract_user(args.id, headless=args.headless)
|
||||
|
||||
# 输出结果
|
||||
json_output = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(json_output)
|
||||
print(f"结果已保存到: {args.output}")
|
||||
else:
|
||||
print(json_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,168 @@
|
||||
# 知识库维护系统 - 整体流程图
|
||||
|
||||
## 1. 用户交互流程（接收链接）
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[用户发送链接] --> B{技能1: 链接解析}
|
||||
B --> C{类型判断}
|
||||
|
||||
C -->|单个内容| D[直接解析]
|
||||
C -->|列表类型| E[询问用户]
|
||||
|
||||
E --> F[处理范围选择]
|
||||
F --> G[订阅选项选择]
|
||||
G --> H[写入订阅配置]
|
||||
|
||||
D --> I[写入收件箱 pending/]
|
||||
H --> I
|
||||
|
||||
I --> J[返回用户: 已收录]
|
||||
```
|
||||
|
||||
## 2. 定时任务/手动触发流程
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[定时任务触发<br/>或手动触发] --> B[技能5: 工作流编排]
|
||||
B --> C[扫描收件箱]
|
||||
|
||||
C --> D{pending/ 目录}
|
||||
D -->|有待处理| E[获取待处理列表]
|
||||
D -->|无| F[检查订阅更新]
|
||||
|
||||
E --> G[技能2: 获取元数据]
|
||||
G --> H[更新状态: metadata_ready]
|
||||
H --> I[技能3: 获取内容]
|
||||
I --> J[更新状态: content_ready]
|
||||
J --> K[技能4: 深度分析]
|
||||
K --> L[归档到 archive/]
|
||||
L --> M[更新订阅追踪]
|
||||
M --> N[继续处理下一个]
|
||||
|
||||
F --> O{有订阅?}
|
||||
O -->|是| P[检测更新]
|
||||
P --> Q[新增内容加入 pending/]
|
||||
Q --> R[更新订阅时间]
|
||||
|
||||
O -->|否| S[跳过]
|
||||
|
||||
N --> T{还有待处理?}
|
||||
T -->|是| E
|
||||
T -->|否| U[生成汇报文件]
|
||||
U --> V[完成]
|
||||
```
|
||||
|
||||
## 3. 完整主流程（简化版）
|
||||
```mermaid
|
||||
flowchart TD
|
||||
subgraph 用户交互
|
||||
A[用户发送链接] --> B[技能1: link-parser]
|
||||
B --> C{内容类型}
|
||||
C -->|单个| D[直接处理]
|
||||
C -->|列表| E[询问用户选项]
|
||||
E --> F[保存订阅配置]
|
||||
D --> G[写入收件箱 pending/]
|
||||
F --> G
|
||||
end
|
||||
|
||||
subgraph 定时处理
|
||||
H[定时/手动触发] --> I[技能5: kb-workflow]
|
||||
I --> J[扫描收件箱]
|
||||
J --> K[技能2: metadata-fetcher]
|
||||
K --> L[技能3: content-fetcher]
|
||||
L --> M[技能4: deep-analyzer]
|
||||
M --> N[归档]
|
||||
N --> O[更新订阅]
|
||||
O --> P[生成汇报]
|
||||
end
|
||||
|
||||
G --> H
|
||||
|
||||
style A fill:#e1f5fe
|
||||
style I fill:#fff3e0
|
||||
style K fill:#f3e5f5
|
||||
style L fill:#e8f5e9
|
||||
style M fill:#fce4ec
|
||||
```
|
||||
|
||||
## 4. 各平台技能调用关系
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph 技能1
|
||||
LP[link-parser]
|
||||
end
|
||||
|
||||
subgraph 技能2[元数据获取]
|
||||
BM[bilibili-metadata]
|
||||
YM[ youtube-metadata]
|
||||
ZM[ zhihu-metadata]
|
||||
end
|
||||
|
||||
subgraph 技能3[内容获取]
|
||||
BC[bilibili-content]
|
||||
OC[ocr-processor]
|
||||
end
|
||||
|
||||
subgraph 技能4
|
||||
DA[deep-analyzer]
|
||||
end
|
||||
|
||||
LP -->|解析结果| BM
|
||||
LP -->|解析结果| YM
|
||||
LP -->|解析结果| ZM
|
||||
|
||||
BM -->|元数据| BC
|
||||
BC -->|低分视频| OC
|
||||
OC -->|OCR结果| DA
|
||||
BM -->|元数据| DA
|
||||
```
|
||||
|
||||
## 5. 数据流转
|
||||
```mermaid
|
||||
flowchart LR
|
||||
subgraph 输入
|
||||
URL[用户链接]
|
||||
end
|
||||
|
||||
subgraph 收件箱
|
||||
P[pending/]
|
||||
M[metadata/]
|
||||
C[content/]
|
||||
end
|
||||
|
||||
subgraph 输出
|
||||
A[archive/YYYY/MM/]
|
||||
R[report.json]
|
||||
end
|
||||
|
||||
URL -->|技能1| P
|
||||
P -->|技能2| M
|
||||
M -->|技能3| C
|
||||
C -->|技能4| A
|
||||
A -.->|统计| R
|
||||
|
||||
style P fill:#fffde7
|
||||
style M fill:#e3f2fd
|
||||
style C fill:#e8f5e9
|
||||
style A fill:#f3e5f5
|
||||
```
|
||||
|
||||
## 6. 状态流转
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> pending: 链接收录
|
||||
pending --> metadata: 技能2完成
|
||||
metadata --> content: 技能3完成
|
||||
content --> analyzed: 技能4完成
|
||||
analyzed --> archived: 归档完成
|
||||
archived --> [*]
|
||||
|
||||
pending --> skipped: 跳过
|
||||
metadata --> failed: 元数据获取失败
|
||||
content --> failed: 内容获取失败
|
||||
analyzed --> failed: 分析失败
|
||||
|
||||
failed --> retry: 重试
|
||||
retry --> pending
|
||||
retry --> metadata
|
||||
retry --> content
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user