starmen1265/kb-skills
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加技能模块

Browse Source
This commit is contained in:
starmen1265
2026-02-15 01:17:39 +08:00
parent c417a8f7a9
commit 86daba5771
48 changed files with 10121 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kb-bilibili-content/SKILL.md
+91
View File
@@ -0,0 +1,91 @@
# KB Bilibili Content Skill
**kb-bilibili-content** - Download Bilibili videos and extract content.
## Features
- Video download using **yt-dlp** (recommended)
- Subtitle extraction
- Article content extraction
- Multiple quality options (360P/480P/720P/1080P)
- Supports `BILIBILI_COOKIE` environment variable
## Environment Variables
| Variable | Required | Description |
|----------|----------|-------------|
| `BILIBILI_COOKIE` | No | B站登录 Cookie (for member-only videos) |
## Commands
```bash
# Download video (720P default)
python bilibili_content.py --id "BV1xx411c7mD" --type video --output "/path/to/downloads/"
# Download video (1080P)
python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 80 --output "/path/to/downloads/"
# Download video (480P)
python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 32 --output "/path/to/downloads/"
# Extract subtitle
python bilibili_content.py --id "BV1xx411c7mD" --type subtitle --output "/path/to/subtitles/"
# Get article content
python bilibili_content.py --id "cv43780186" --type article --output "/path/to/article.md"
```
## Output
```json
{
"success": true,
"type": "video",
"filename": "video_title.mp4",
"filepath": "/path/to/downloads/video_title.mp4",
"message": "下载完成: video_title.mp4"
}
```
## Cookie Setup
### Get Cookie from Browser
1. Login to bilibili.com
2. Press F12 → Network → Refresh page
3. Copy Cookie from request headers
### Set Environment Variable
```bash
# Add to ~/.bashrc for persistence
export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx; DedeUserID=xxx; DedeUserID__ckMd5=xxx"
# Or set temporarily
export BILIBILI_COOKIE="your_cookie_here"
```
## Installation
```bash
cp -r kb-bilibili-content/ ~/skills/
pip install aiohttp playwright
playwright install chromium
```
## How It Works
Video download uses **yt-dlp** with Netscape-format cookie file:
1. Parses `BILIBILI_COOKIE` environment variable
2. Creates temporary Netscape-format cookie file
3. Runs `yt-dlp --cookies <file> -f <format> -o <output>`
4. Cleans up temporary file
## Quality Options
| Quality | ID | Resolution | Notes |
|---------|-----|------------|-------|
| 16 | 360P | 640x360 | Fastest |
| 32 | 480P | 852x480 | Default for low bandwidth |
| 64 | 720P | 1280x720 | **Recommended** |
| 80 | 1080P | 1920x1080 | Best quality |
Default quality: **64 (720P)**
kb-bilibili-content/__pycache__/bilibili_content.cpython-312.pyc
BIN
View File
Binary file not shown.
kb-bilibili-content/bilibili_content.py
+419
View File
@@ -0,0 +1,419 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Bilibili Content Downloader - B站内容获取工具
功能:
- download_video(bvid, quality=480) - 下载视频（使用 yt-dlp）
- extract_subtitle(bvid) - 提取字幕
- extract_article(cvid) - 获取专栏全文（使用Playwright）
- 命令行参数支持
- JSON 输出
"""
import argparse
import asyncio
import json
import os
import re
import subprocess
import sys
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Optional, Dict, Any, List
class BilibiliContent:
"""B站内容获取器"""
def __init__(self, output_dir: str = "./output", cookie: str = None):
self.output_dir = Path(output_dir)
self.output_dir.mkdir(parents=True, exist_ok=True)
self.cookie = cookie
def _create_cookie_file(self) -> str:
"""创建 Netscape 格式的 Cookie 文件"""
if not self.cookie:
return None
# 解析 Cookie 字符串
cookies = {}
for item in self.cookie.split(';'):
item = item.strip()
if '=' in item:
key, value = item.split('=', 1)
cookies[key.strip()] = value.strip()
# 创建 Netscape 格式的 Cookie 文件
cookie_content = """# Netscape HTTP Cookie File
# Generated by bilibili_content.py
"""
for name, value in cookies.items():
cookie_content += f".bilibili.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n"
# 写入临时文件
fd, path = tempfile.mkstemp(suffix='.txt')
with os.fdopen(fd, 'w') as f:
f.write(cookie_content)
return path
async def download_video(self, bvid: str, quality: int = 64) -> Dict[str, Any]:
"""下载B站视频（使用 yt-dlp）"""
result = {"success": False, "bvid": bvid, "message": "", "filepath": None, "metadata": {}}
try:
# 先获取视频信息
print(f"获取视频信息: {bvid}")
cookie_file = self._create_cookie_file()
url = f"https://www.bilibili.com/video/{bvid}"
try:
# 获取视频标题（用于文件名）
if cookie_file:
cmd = ["yt-dlp", "--cookies", cookie_file, "--get-title", url]
else:
cmd = ["yt-dlp", "--get-title", url]
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
title = stdout.decode('utf-8').strip()
if not title or 'error' in title.lower():
# 尝试备用方法
title = f"video_{bvid}"
except Exception as e:
title = f"video_{bvid}"
# 构建 yt-dlp 命令
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_', '丨')).strip()[:50]
# 选择视频格式
if quality == 80:
# 1080P
format_str = "30112+30280"
elif quality == 64:
# 720P
format_str = "30064+30280"
elif quality == 32:
# 480P
format_str = "30032+30280"
else:
# 360P
format_str = "30016+30280"
print(f"下载视频: {title}")
print(f"格式: {format_str}")
output_template = str(self.output_dir / f"{safe_title}.%(ext)s")
cmd = [
"yt-dlp",
"-f", format_str,
"-o", output_template,
"--no-download-archive",
"-q",
url
]
if cookie_file:
cmd.extend(["--cookies", cookie_file])
print(f"执行命令: {' '.join(cmd[:4])}...")
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
# 查找下载的文件
downloaded_files = list(self.output_dir.glob(f"{safe_title}.*"))
if downloaded_files:
video_file = downloaded_files[0]
result["success"] = True
result["filepath"] = str(video_file)
result["message"] = f"下载完成: {video_file.name}"
result["metadata"] = {"title": title, "bvid": bvid}
print(f"✅ {result['message']}")
else:
result["message"] = "未找到下载的文件"
print(f"❌ {result['message']}")
# 清理临时 Cookie 文件
if cookie_file and os.path.exists(cookie_file):
os.remove(cookie_file)
except Exception as e:
result["message"] = f"错误: {str(e)}"
print(f"❌ {result['message']}")
return result
async def extract_subtitle(self, bvid: str) -> Dict[str, Any]:
"""
提取字幕
注意：没有找到官方字幕是正常的！很多视频没有自动生成的字幕。
返回 success=True 即使没有字幕，让工作流继续执行后续步骤（如 Whisper）。
"""
result = {"success": False, "bvid": bvid, "subtitles": [], "has_subtitle": False, "message": ""}
try:
import aiohttp
api_url = f"https://api.bilibili.com/x/player/pagelist?bvid={bvid}"
async with aiohttp.ClientSession() as session:
async with session.get(api_url) as resp:
data = await resp.json()
pages = data.get('data', [])
if not pages:
result["message"] = "未找到分P（这是正常的）"
# 仍然返回 success=True，让工作流继续
result["success"] = True
return result
cid = pages[0].get('cid', '')
async with aiohttp.ClientSession() as session:
async with session.get(f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}") as resp:
sub_data = await resp.json()
subtitles = sub_data.get('data', {}).get('subtitle', {}).get('list', [])
if not subtitles:
result["message"] = "没有官方字幕（这是正常的），将使用 Whisper 进行语音识别"
# 仍然返回 success=True，让工作流继续
result["success"] = True
return result
result["success"] = True
result["has_subtitle"] = True
result["subtitles"] = [{"lan": s.get('lan'), "title": s.get('lan_doc')} for s in subtitles]
result["message"] = f"✅ 找到 {len(subtitles)} 个字幕，将使用官方字幕进行深度分析"
print(result["message"])
except Exception as e:
result["message"] = f"获取字幕出错（这是正常的），将使用 Whisper: {str(e)}"
# 出错时也返回 success=True，让工作流继续
result["success"] = True
return result
async def download_subtitle(self, bvid: str) -> Dict[str, Any]:
"""
下载字幕并转换为 SRT 格式
如果没有官方字幕，返回 success=True 但 has_subtitle=False，
让工作流继续使用 Whisper 进行语音识别。
"""
result = {"success": False, "bvid": bvid, "filepath": None, "has_subtitle": False, "message": ""}
try:
import aiohttp
# 获取 CID
api_url = f"https://api.bilibili.com/x/player/pagelist?bvid={bvid}"
async with aiohttp.ClientSession() as session:
async with session.get(api_url) as resp:
data = await resp.json()
pages = data.get('data', [])
if not pages:
result["message"] = "未找到分P，将使用 Whisper 进行语音识别"
result["success"] = True
return result
cid = pages[0].get('cid', '')
# 获取字幕列表
async with aiohttp.ClientSession() as session:
async with session.get(f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}") as resp:
sub_data = await resp.json()
subtitles = sub_data.get('data', {}).get('subtitle', {}).get('list', [])
if not subtitles:
result["message"] = "没有官方字幕，将使用 Whisper 进行语音识别"
result["success"] = True
return result
# 选择中文字幕，如果没有则选第一个
zh_sub = None
for s in subtitles:
if s.get('lan') in ('ai-zh', 'zh-CN', 'zh'):
zh_sub = s
break
if not zh_sub:
zh_sub = subtitles[0]
# 下载字幕文件
sub_url = zh_sub.get('subtitle_url', '')
if not sub_url:
result["message"] = "字幕URL为空，将使用 Whisper"
result["success"] = True
return result
if sub_url.startswith('//'):
sub_url = 'https:' + sub_url
async with aiohttp.ClientSession() as session:
async with session.get(sub_url) as resp:
sub_content = await resp.json()
# 转换为 SRT 格式
body = sub_content.get('body', [])
if not body:
result["message"] = "字幕内容为空，将使用 Whisper"
result["success"] = True
return result
def format_srt_time(t: float) -> str:
"""将秒数转换为 SRT 时间格式"""
h = int(t // 3600)
m = int((t % 3600) // 60)
s = int(t % 60)
ms = int((t - int(t)) * 1000)
return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
srt_lines = []
for i, line in enumerate(body, 1):
start = line.get('from', 0)
end = line.get('to', 0)
content = line.get('content', '')
srt_lines.extend([str(i), f"{format_srt_time(start)} --> {format_srt_time(end)}", content, ""])
srt_content = "\n".join(srt_lines)
# 保存 SRT 文件
output_path = self.output_dir / f"{bvid}.srt"
with open(output_path, 'w', encoding='utf-8') as f:
f.write(srt_content)
result["success"] = True
result["has_subtitle"] = True
result["filepath"] = str(output_path)
result["message"] = f"✅ 官方字幕已下载: {output_path.name} ({len(body)}条)"
print(result["message"])
except Exception as e:
result["message"] = f"下载字幕出错: {str(e)}，将使用 Whisper"
result["success"] = True # 仍然返回成功，让工作流继续
return result
async def extract_article(self, cvid: str) -> Dict[str, Any]:
"""获取专栏全文（使用Playwright异步API）"""
result = {"success": False, "cvid": cvid, "title": "", "content": "", "message": ""}
try:
from playwright.async_api import async_playwright
cvid_clean = cvid.replace('cv', '')
url = f"https://www.bilibili.com/read/cv{cvid_clean}"
print(f"获取专栏: {url}")
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
page = await browser.new_page()
await page.goto(url, wait_until="networkidle")
await page.wait_for_timeout(3000)
# 提取标题
title_elem = await page.query_selector('h1')
if title_elem:
result["title"] = (await title_elem.inner_text()).strip()
print(f"标题: {result['title'][:50]}...")
# 提取正文内容
content_elem = await page.query_selector('.read-article-holder')
if content_elem:
html_content = await content_elem.inner_html()
text_content = await content_elem.inner_text()
result["content"] = text_content.strip()
result["html_content"] = html_content.strip()
result["success"] = bool(result["title"] and result["content"])
print(f"内容长度: {len(result['content'])} 字符")
else:
result["message"] = "未找到专栏内容"
await browser.close()
if result["success"]:
result["message"] = f"✅ 获取成功: {result['title'][:30]}..."
else:
result["message"] = f"获取失败: {result.get('message', '未知错误')}"
except Exception as e:
result["message"] = f"错误: {str(e)}"
print(f"错误: {e}")
return result
async def main():
parser = argparse.ArgumentParser(description='Bilibili Content Downloader')
parser.add_argument('--id', '-i', required=True, help='B站内容ID')
parser.add_argument('--type', '-t', required=True, choices=['video', 'subtitle', 'subtitle-download', 'article'], help='内容类型')
parser.add_argument('--output', '-o', default='./output', help='输出目录')
parser.add_argument('--quality', '-q', type=int, default=64, help='视频质量: 16=360P, 32=480P, 64=720P, 80=1080P')
args = parser.parse_args()
cookie = os.environ.get('BILIBILI_COOKIE')
downloader = BilibiliContent(output_dir=args.output, cookie=cookie)
if args.type == 'video':
result = await downloader.download_video(args.id, quality=args.quality)
elif args.type == 'subtitle':
result = await downloader.extract_subtitle(args.id)
elif args.type == 'subtitle-download':
# 下载并转换字幕为 SRT 格式
result = await downloader.download_subtitle(args.id)
else:
result = await downloader.extract_article(args.id.replace('cv', ''))
# 统一输出格式
if result.get("success"):
output = {
"success": True,
"type": args.type,
"has_subtitle": result.get("has_subtitle", False),
"filename": Path(result.get('filepath', '')).name if result.get('filepath') else None,
"filepath": str(Path(result.get('filepath', '')).resolve()) if result.get('filepath') else None,
"message": result.get('message', '处理成功')
}
else:
output = {
"success": False,
"error": result.get('message', '处理失败'),
"detail": f"ID: {args.id}, 类型: {args.type}"
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return result
if __name__ == '__main__':
try:
result = asyncio.run(main())
sys.exit(0 if result.get("success") else 1)
except KeyboardInterrupt:
print(json.dumps({"success": False, "error": "用户取消"}, ensure_ascii=False, indent=2))
sys.exit(1)
except Exception as e:
print(json.dumps({"success": False, "error": str(e)}, ensure_ascii=False, indent=2))
sys.exit(1)
kb-bilibili-content/bilibili_extractor.py
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
B站视频提取脚本
提取视频信息、统计数据、字幕并生成Markdown笔记
"""
import sys
import os
import json
import re
from datetime import datetime
# 添加skills路径
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb'))
from src.extractors.bilibili_extractor import BilibiliExtractor
from src.config import get_config
def format_number(num):
"""格式化数字"""
if num >= 100000000:
return f"{num / 100000000:.1f}亿"
elif num >= 10000:
return f"{num / 10000:.1f}万"
else:
return f"{num:,}"
def generate_markdown(result, bvid):
"""生成Markdown笔记"""
metadata = result['metadata']
stats = result['stats']
description = result['description']
subtitle = result['subtitle']
# 获取当前时间
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
md = f"""# {metadata['title']}
> **来源**: [B站](https://www.bilibili.com/video/{bvid})
> **作者**: {metadata['author']}
> **发布时间**: {metadata['pubtime']}
> **提取时间**: {now}
> **时长**: {metadata['duration']}
## 📊 统计数据
| 指标 | 数量 |
|------|------|
| 👀 播放 | {format_number(stats['view'])} |
| 💬 弹幕 | {format_number(stats['danmaku'])} |
| 📝 评论 | {format_number(stats['reply'])} |
| ⭐ 收藏 | {format_number(stats['favorite'])} |
| 🪙 投币 | {format_number(stats['coin'])} |
| 🔗 分享 | {format_number(stats['share'])} |
| 👍 点赞 | {format_number(stats['like'])} |
## 📝 视频简介
{description if description else '(暂无简介)'}
## 📌 标签
{metadata['tags'] if metadata['tags'] else '(无标签)'}
## 🎬 字幕
{subtitle if subtitle else '(无字幕)'}
---
**元数据**:
- BV号: `{metadata['bvid']}`
- AV号: `{metadata['aid']}`
- UP主ID: `{metadata['mid']}`
"""
return md
def main():
"""主函数"""
# BV号
bvid = "BV1N5foBjE5K" # 从 https://b23.tv/btMGoOi 解析得到
print(f"开始提取B站视频: {bvid}")
print("=" * 60)
# 获取配置
try:
config = get_config()
except Exception as e:
print(f"⚠️ 获取配置失败，使用默认配置: {e}")
config = None
# 创建提取器
extractor = BilibiliExtractor(config=config)
# 提取视频信息
try:
result = extractor.extract(bvid)
print("\n✅ 提取成功！\n")
print("=" * 60)
print("基本信息:")
print("=" * 60)
print(f"标题: {result['metadata']['title']}")
print(f"作者: {result['metadata']['author']}")
print(f"时长: {result['metadata']['duration']}")
print(f"发布: {result['metadata']['pubtime']}")
print("\n" + "=" * 60)
print("统计数据:")
print("=" * 60)
print(f"播放: {format_number(result['stats']['view'])}")
print(f"弹幕: {format_number(result['stats']['danmaku'])}")
print(f"评论: {format_number(result['stats']['reply'])}")
print(f"收藏: {format_number(result['stats']['favorite'])}")
print(f"投币: {format_number(result['stats']['coin'])}")
print(f"点赞: {format_number(result['stats']['like'])}")
# 生成 Markdown 笔记
md_content = generate_markdown(result, bvid)
# 确保 notes 目录存在
notes_dir = "/home/admin/.openclaw/workspace/notes"
os.makedirs(notes_dir, exist_ok=True)
# 生成文件名（使用BV号作为文件名）
filename = f"{bvid}.md"
filepath = os.path.join(notes_dir, filename)
# 写入文件
with open(filepath, 'w', encoding='utf-8') as f:
f.write(md_content)
print("\n" + "=" * 60)
print(f"✅ Markdown笔记已生成: {filepath}")
print("=" * 60)
# 返回笔记内容
return md_content
except Exception as e:
print(f"\n❌ 提取失败: {e}")
import traceback
traceback.print_exc()
return None
if __name__ == '__main__':
main()
kb-bilibili-content/video_downloader.py
+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
使用旧版 API 获取并下载视频流
"""
import asyncio
import json
import os
from playwright.async_api import async_playwright
async def download_video_stream(bvid, output_dir="/tmp/bilibili_downloads"):
"""下载视频流"""
url = f"https://www.bilibili.com/video/{bvid}"
# 创建输出目录
os.makedirs(output_dir, exist_ok=True)
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
print(f"正在下载视频: {url}")
print("=" * 80)
# 首先获取 aid 和 cid
print("\n1. 获取视频信息...")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') != 0:
print(f" ❌ 失败: {data.get('message')}")
await browser.close()
return None
video_data = data.get('data', {})
title = video_data.get('title')
aid = video_data.get('aid')
cid = video_data.get('cid')
duration = video_data.get('duration')
print(f" 标题: {title}")
print(f" 时长: {duration}秒")
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
return None
# 获取视频流 URL
print("\n2. 获取视频流 URL...")
try:
playurl_api = "https://api.bilibili.com/x/player/playurl"
params = {
'bvid': bvid,
'cid': cid,
'qn': 80, # 1080P
'otype': 'json',
'fnver': 0,
'fnval': 0, # FLV 格式
}
playurl_response = await page.request.get(playurl_api, params=params)
if not playurl_response.ok:
print(f" ❌ HTTP 错误: {playurl_response.status}")
await browser.close()
return None
playurl_data = await playurl_response.json()
if playurl_data.get('code') != 0:
print(f" ❌ API 错误: {playurl_data.get('message')}")
await browser.close()
return None
play_data = playurl_data.get('data', {})
# 检查支持的清晰度
print(f"\n 支持的清晰度:")
for i, (q, desc) in enumerate(zip(play_data.get('accept_quality', []), play_data.get('accept_description', []))):
current = " (当前)" if q == play_data.get('quality') else ""
print(f" {desc} ({q}){current}")
# 获取视频 URL
durl = play_data.get('durl', [])
if not durl:
print(f" ❌ 没有找到视频流")
await browser.close()
return None
print(f"\n 找到 {len(durl)} 个视频分段")
video_urls = []
for i, segment in enumerate(durl):
segment_url = segment.get('url')
segment_size = segment.get('size', 0)
video_urls.append(segment_url)
print(f" 分段 {i+1}: {segment_size / 1024 / 1024:.2f}MB")
print(f" {segment_url[:100]}...")
except Exception as e:
print(f" ❌ 异常: {e}")
import traceback
traceback.print_exc()
await browser.close()
return None
# 下载视频
print(f"\n3. 开始下载视频...")
all_segments = []
total_size = 0
for i, video_url in enumerate(video_urls, 1):
print(f"\n 下载分段 {i}/{len(video_urls)}...")
try:
# 添加必要的请求头
headers = {
'Referer': 'https://www.bilibili.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
}
# 下载分段
segment_response = await page.request.get(
video_url,
headers=headers,
timeout=60000
)
if not segment_response.ok:
print(f" ❌ 下载失败: {segment_response.status}")
continue
segment_data = await segment_response.body()
segment_size = len(segment_data)
total_size += segment_size
all_segments.append(segment_data)
print(f" ✅ 下载成功: {segment_size / 1024 / 1024:.2f}MB")
except Exception as e:
print(f" ❌ 下载异常: {e}")
continue
if not all_segments:
print(f"\n ❌ 没有成功下载任何分段")
await browser.close()
return None
# 合并分段
print(f"\n4. 合并视频分段...")
video_data = b''.join(all_segments)
# 保存文件
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
output_path = os.path.join(output_dir, f"{safe_title}.flv")
with open(output_path, 'wb') as f:
f.write(video_data)
print(f" ✅ 视频已保存: {output_path}")
print(f" 文件大小: {total_size / 1024 / 1024:.2f}MB")
await browser.close()
return output_path
if __name__ == '__main__':
bvid = "BV1KtFzzqEUG"
result = asyncio.run(download_video_stream(bvid))
if result:
print(f"\n✅ 下载完成！")
print(f"文件路径: {result}")
else:
print(f"\n❌ 下载失败")
kb-bilibili-metadata/SKILL.md
+46
View File
@@ -0,0 +1,46 @@
# KB Bilibili Metadata Skill
**kb-bilibili-metadata** - Extract Bilibili video/article/up主 metadata.
## Features
- Video metadata extraction
- Article (专栏) metadata extraction
- UP主 information extraction
- JSON output format
- Playwright + HTTP API fallback
## Commands
```bash
# Video metadata
python bilibili_metadata.py --id "BV1xx411c7mD" --type video
# Article metadata
python bilibili_metadata.py --id "cv43780186" --type article
# UP主 info
python bilibili_metadata.py --id "7770724" --type up
```
## Output
```json
{
"success": true,
"data": {
"platform": "bilibili",
"type": "video",
"id": "BV1xx411c7mD",
"title": "视频标题",
"author": "UP主名字",
"stats": {"view": 100000, "like": 5000}
}
}
```
## Installation
```bash
cp -r kb-bilibili-metadata/ ~/skills/
```
kb-bilibili-metadata/__init__.py
+8
View File
@@ -0,0 +1,8 @@
"""
bilibili-metadata 技能包
B站元数据提取工具
"""
__version__ = "1.0.0"
__author__ = "OpenClaw"
kb-bilibili-metadata/__pycache__/bilibili_extractor.cpython-312.pyc
BIN
View File
Binary file not shown.
kb-bilibili-metadata/bilibili_extractor.py
+666
View File
@@ -0,0 +1,666 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
B站元数据提取器
独立版本，不依赖 obsidian-kb 结构
支持视频、专栏、UP主信息提取
"""
import asyncio
import json
import logging
import re
from datetime import datetime
from typing import Dict, Any, Optional
from urllib.parse import urlparse
logger = logging.getLogger(__name__)
# 默认请求头
DEFAULT_HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Referer': 'https://www.bilibili.com/',
'Accept': 'application/json, text/plain, */*',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}
try:
from playwright.async_api import async_playwright
PLAYWRIGHT_AVAILABLE = True
except ImportError:
PLAYWRIGHT_AVAILABLE = False
try:
from undetected_playwright import stealth_async
STEALTH_AVAILABLE = True
except ImportError:
STEALTH_AVAILABLE = False
try:
import httpx
HTTPX_AVAILABLE = True
except ImportError:
HTTPX_AVAILABLE = False
class BilibiliExtractor:
"""
B站元数据提取器
支持：
- 视频信息提取（标题、作者、统计数据、字幕等）
- 专栏文章提取
- UP主信息提取
"""
# 正则模式
BV_PATTERN = re.compile(r'BV[a-zA-Z0-9]{10}')
CV_PATTERN = re.compile(r'cv(\d+)')
URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/video/(BV[a-zA-Z0-9]{10})')
ARTICLE_URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/read/(cv\d+)')
UP_URL_PATTERN = re.compile(r'https?://space\.bilibili\.com/(\d+)')
def __init__(self, headless: bool = True, timeout: int = 30):
"""
初始化提取器
Args:
headless: 是否使用无头模式（浏览器）
timeout: 超时时间（秒）
"""
self.headless = headless
self.timeout = timeout * 1000 # 转换为毫秒
def extract_video_metadata(self, bvid: str) -> Dict[str, Any]:
"""
提取视频元数据
Args:
bvid: B站视频BV号（如 BV1xx411c7mD）
Returns:
Dict containing metadata, stats, description, subtitle
"""
# 验证并解析BV号
bvid = self._resolve_bvid(bvid)
if not bvid:
raise ValueError(f"无效的BV号: {bvid}")
# 使用异步方法提取
return asyncio.run(self._extract_video_async(bvid))
def extract_article_metadata(self, cvid: str) -> Dict[str, Any]:
"""
提取专栏元数据
Args:
cvid: B站专栏CV号（如 cv43780186）
Returns:
Dict containing article metadata
"""
# 解析CV号
cvid = self._resolve_cvid(cvid)
if not cvid:
raise ValueError(f"无效的CV号: {cvid}")
return asyncio.run(self._extract_article_async(cvid))
def extract_up_metadata(self, mid: str) -> Dict[str, Any]:
"""
提取UP主信息
Args:
mid: B站用户ID（如 7770724）
Returns:
Dict containing UP主信息
"""
# 解析MID
mid = self._resolve_mid(mid)
if not mid:
raise ValueError(f"无效的MID: {mid}")
return asyncio.run(self._extract_up_async(mid))
def _resolve_bvid(self, source_id: str) -> Optional[str]:
"""解析并返回BV号"""
source = source_id.strip()
# 直接BV号
if self.BV_PATTERN.fullmatch(source):
return source
# 完整URL
match = self.URL_PATTERN.match(source)
if match:
return match.group(1)
return None
def _resolve_cvid(self, source_id: str) -> Optional[str]:
"""解析并返回CV号"""
source = source_id.strip()
# 直接CV号（带cv前缀）
if source.startswith('cv'):
match = self.CV_PATTERN.match(source)
if match:
return source
# 纯数字
if source.isdigit():
return f"cv{source}"
# 完整URL
match = self.ARTICLE_URL_PATTERN.match(source)
if match:
return match.group(1)
return None
def _resolve_mid(self, source_id: str) -> Optional[str]:
"""解析并返回MID"""
source = source_id.strip()
# 纯数字
if source.isdigit():
return source
# 完整URL
match = self.UP_URL_PATTERN.match(source)
if match:
return match.group(1)
return None
async def _extract_video_async(self, bvid: str) -> Dict[str, Any]:
"""异步提取视频信息"""
url = f"https://www.bilibili.com/video/{bvid}"
result = {
'metadata': {},
'stats': {},
'subtitle': '',
'description': ''
}
if not PLAYWRIGHT_AVAILABLE:
# 使用 HTTP API
return await self._extract_video_by_api(bvid)
async with async_playwright() as playwright:
try:
browser = await playwright.chromium.launch(
headless=self.headless,
args=[
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-blink-features=AutomationControlled',
'--disable-dev-shm-usage',
]
)
context = await browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
locale='zh-CN',
)
if STEALTH_AVAILABLE:
await stealth_async(context)
page = await context.new_page()
await page.goto(url, wait_until='domcontentloaded', timeout=self.timeout)
await asyncio.sleep(2)
content = await page.content()
video_data = await self._try_extract_video_methods(page, content, bvid)
if video_data:
result['metadata'] = {
'title': video_data.get('title', ''),
'author': video_data.get('author', ''),
'bvid': video_data.get('bvid', ''),
'aid': video_data.get('aid', ''),
'mid': video_data.get('mid', ''),
'pubtime': video_data.get('pubtime', ''),
'duration': video_data.get('duration', ''),
'url': url,
'tags': video_data.get('tags', ''),
}
result['stats'] = {
'view': video_data.get('view', 0),
'danmaku': video_data.get('danmaku', 0),
'reply': video_data.get('reply', 0),
'favorite': video_data.get('favorite', 0),
'coin': video_data.get('coin', 0),
'share': video_data.get('share', 0),
'like': video_data.get('like', 0),
}
result['description'] = video_data.get('desc', '')
await browser.close()
except Exception as e:
logger.error(f"视频提取错误: {e}")
# 回退到 API
return await self._extract_video_by_api(bvid)
return result
async def _extract_video_by_api(self, bvid: str) -> Dict[str, Any]:
"""使用B站API提取视频信息"""
result = {
'metadata': {},
'stats': {},
'subtitle': '',
'description': ''
}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取视频信息
response = await client.get('https://api.bilibili.com/x/web-interface/view', params={'bvid': bvid})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
video_data = data.get('data', {})
result['metadata'] = {
'title': video_data.get('title', ''),
'author': video_data.get('owner', {}).get('name', ''),
'bvid': video_data.get('bvid', ''),
'aid': video_data.get('aid', ''),
'mid': video_data.get('owner', {}).get('mid', ''),
'pubtime': datetime.fromtimestamp(video_data.get('pubdate', 0)).strftime('%Y-%m-%d %H:%M:%S') if video_data.get('pubdate') else '',
'duration': self._format_duration(video_data.get('duration', 0)),
'url': f"https://www.bilibili.com/video/{bvid}",
'tags': ', '.join([t.get('tag_name', '') for t in video_data.get('tag', [])]),
}
stat = video_data.get('stat', {})
result['stats'] = {
'view': stat.get('view', 0),
'danmaku': stat.get('danmaku', 0),
'reply': stat.get('reply', 0),
'favorite': stat.get('favorite', 0),
'coin': stat.get('coin', 0),
'share': stat.get('share', 0),
'like': stat.get('like', 0),
}
result['description'] = video_data.get('desc', '')
except Exception as e:
logger.error(f"API提取错误: {e}")
return result
async def _try_extract_video_methods(self, page, content: str, bvid: str) -> Dict[str, Any]:
"""尝试多种方法提取视频信息"""
video_data = {}
# 方法1: 从 __INITIAL_STATE__
try:
initial_state = await page.evaluate('() => typeof window.__INITIAL_STATE__ !== "undefined" ? window.__INITIAL_STATE__ : null')
if initial_state:
video_data = self._parse_video_state(initial_state)
if video_data and video_data.get('title'):
return video_data
except:
pass
# 方法2: 使用 API（更可靠）
return await self._extract_video_by_api(bvid)
def _parse_video_state(self, data: Dict) -> Dict[str, Any]:
"""解析 __INITIAL_STATE__ 数据"""
video_data = {}
try:
video = data.get('videoData', {})
video_data['title'] = video.get('title', '')
video_data['bvid'] = video.get('bvid', '')
video_data['aid'] = video.get('aid', '')
owner = video.get('owner', {})
video_data['author'] = owner.get('name', '')
video_data['mid'] = owner.get('mid', '')
stat = video.get('stat', {})
video_data['view'] = stat.get('view', 0)
video_data['danmaku'] = stat.get('danmaku', 0)
video_data['reply'] = stat.get('reply', 0)
video_data['favorite'] = stat.get('favorite', 0)
video_data['coin'] = stat.get('coin', 0)
video_data['share'] = stat.get('share', 0)
video_data['like'] = stat.get('like', 0)
pubdate = video.get('pubdate', 0)
video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else ''
video_data['duration'] = self._format_duration(video.get('duration', 0))
video_data['desc'] = video.get('desc', '')
video_data['tags'] = ', '.join([t.get('tag_name', '') for t in video.get('tag', [])])
except:
pass
return video_data
async def _extract_article_async(self, cvid: str) -> Dict[str, Any]:
"""异步提取专栏信息"""
result = {}
if not HTTPX_AVAILABLE:
return {'error': 'httpx 未安装'}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取专栏信息
response = await client.get('https://api.bilibili.com/x/article/viewinfo', params={'id': cvid[2:]})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
article = data.get('data', {})
result = {
'cvid': cvid,
'title': article.get('title', ''),
'author': article.get('author_name', ''),
'mid': article.get('author_mid', ''),
'publish_time': datetime.fromtimestamp(article.get('publish_time', 0)).strftime('%Y-%m-%d %H:%M:%S') if article.get('publish_time') else '',
'view_count': article.get('stats', {}).get('view', 0),
'like_count': article.get('stats', {}).get('like', 0),
'coin_count': article.get('stats', {}).get('coin', 0),
'share_count': article.get('stats', {}).get('share', 0),
'reply_count': article.get('stats', {}).get('reply', 0),
'url': f"https://www.bilibili.com/read/{cvid}",
'summary': article.get('summary', ''),
'word_count': article.get('words', 0),
}
except Exception as e:
logger.error(f"专栏提取错误: {e}")
result = {'error': str(e)}
return result
async def _extract_up_async(self, mid: str) -> Dict[str, Any]:
"""
异步提取UP主信息（支持订阅跟踪）
Args:
mid: UP主ID
Returns:
Dict containing UP主完整信息
"""
result = {
'type': 'up',
'mid': mid,
'subscribed': False,
'basic_info': {},
'subscription_info': {}
}
if not HTTPX_AVAILABLE:
return {'error': 'httpx 未安装'}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取UP主基本信息
response = await client.get('https://api.bilibili.com/x/space/acc/info', params={'mid': mid})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
up = data.get('data', {})
result['basic_info'] = {
'name': up.get('name', ''),
'sex': up.get('sex', ''),
'face': up.get('face', ''),
'sign': up.get('sign', ''),
'level': up.get('level', 0),
'vip_type': up.get('vip', {}).get('type', 0),
'vip_status': up.get('vip', {}).get('status', 0),
'following_count': up.get('following', 0),
'follower_count': up.get('follower', 0),
'url': f"https://space.bilibili.com/{mid}",
}
# 获取UP主视频统计
stats_response = await client.get('https://api.bilibili.com/x/space/navnum', params={'mid': mid})
if stats_response.status_code == 200:
stats_data = stats_response.json()
if stats_data.get('code') == 0:
result['basic_info']['video_count'] = stats_data.get('data', {}).get('video', 0)
# 获取UP主专栏数量
article_response = await client.get('https://api.bilibili.com/x/space/article', params={'mid': mid, 'pn': 1, 'ps': 1})
if article_response.status_code == 200:
article_data = article_response.json()
if article_data.get('code') == 0:
result['basic_info']['article_count'] = article_data.get('data', {}).get('count', 0)
# 获取UP主最近视频（用于跟踪更新）
video_response = await client.get('https://api.bilibili.com/x/space/arc/search',
params={'mid': mid, 'pn': 1, 'ps': 1, 'order': 'pubtime'})
if video_response.status_code == 200:
video_data = video_response.json()
if video_data.get('code') == 0:
videos = video_data.get('data', {}).get('list', {}).get('vlist', [])
if videos:
latest_video = videos[0]
result['subscription_info'] = {
'last_video_id': latest_video.get('bvid', ''),
'last_video_title': latest_video.get('title', ''),
'last_video_time': datetime.fromtimestamp(latest_video.get('created', 0)).strftime('%Y-%m-%d %H:%M:%S') if latest_video.get('created') else '',
}
# 获取UP主最近专栏（用于跟踪更新）
article_response = await client.get('https://api.bilibili.com/x/space/article',
params={'mid': mid, 'pn': 1, 'ps': 1, 'sort': 'publish_time'})
if article_response.status_code == 200:
article_data = article_response.json()
if article_data.get('code') == 0:
articles = article_data.get('data', {}).get('articles', [])
if articles:
latest_article = articles[0]
if 'last_article' not in result['subscription_info']:
result['subscription_info']['last_article'] = {}
result['subscription_info']['last_article'] = {
'id': latest_article.get('id', 0),
'title': latest_article.get('title', ''),
'time': datetime.fromtimestamp(latest_article.get('publish_time', 0)).strftime('%Y-%m-%d %H:%M:%S') if latest_article.get('publish_time') else '',
}
except Exception as e:
logger.error(f"UP主提取错误: {e}")
result = {'error': str(e)}
return result
def _format_duration(self, seconds: int) -> str:
"""格式化时长（秒 -> HH:MM:SS）"""
if not seconds:
return ''
hours = seconds // 3600
minutes = (seconds % 3600) // 60
secs = seconds % 60
if hours > 0:
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
else:
return f"{minutes:02d}:{secs:02d}"
async def _extract_series_async(self, mid: str, series_id: str) -> Dict[str, Any]:
"""
异步提取系列合集信息（不包含列表内容）
Args:
mid: UP主ID
series_id: 系列ID
Returns:
Dict containing series basic info
"""
result = {
'type': 'series',
'series_id': series_id,
'mid': mid,
'subscribed': False,
'basic_info': {}
}
if not HTTPX_AVAILABLE:
return {'error': 'httpx 未安装'}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取系列信息
response = await client.get('https://api.bilibili.com/x/space/series',
params={'mid': mid, 'series_id': series_id, 'pn': 1, 'ps': 1})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
series_info = data.get('data', {}).get('list', {})
result['basic_info'] = {
'title': series_info.get('title', ''),
'description': series_info.get('desc', ''),
'cover': series_info.get('cover', ''),
'mid': mid,
'creator_name': '',
'total_count': series_info.get('cnt', 0),
'last_updated': '',
'update_count': 0
}
# 获取创作者信息
up_info = await client.get('https://api.bilibili.com/x/space/acc/info',
params={'mid': mid})
if up_info.status_code == 200:
up_data = up_info.json()
if up_data.get('code') == 0:
result['basic_info']['creator_name'] = up_data.get('data', {}).get('name', '')
except Exception as e:
logger.error(f"系列合集提取错误: {e}")
result = {'error': str(e)}
return result
async def _extract_readlist_async(self, rl_id: str) -> Dict[str, Any]:
"""
异步提取专栏合集信息（不包含列表内容）
Args:
rl_id: 专栏合集ID (rl开头)
Returns:
Dict containing readlist basic info
"""
result = {
'type': 'readlist',
'readlist_id': rl_id,
'subscribed': False,
'basic_info': {}
}
if not HTTPX_AVAILABLE:
return {'error': 'httpx 未安装'}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取专栏合集信息
response = await client.get('https://api.bilibili.com/x/article/readlist',
params={'id': rl_id.replace('rl', '')})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
list_info = data.get('data', {})
result['basic_info'] = {
'title': list_info.get('name', ''),
'description': list_info.get('description', ''),
'cover': list_info.get('cover', ''),
'mid': list_info.get('mid', ''),
'creator_name': '',
'total_count': list_info.get('articles_count', 0),
'last_updated': '',
'is_public': list_info.get('public', True)
}
# 获取创作者信息
mid = list_info.get('mid', '')
if mid:
up_info = await client.get('https://api.bilibili.com/x/space/acc/info',
params={'mid': mid})
if up_info.status_code == 200:
up_data = up_info.json()
if up_data.get('code') == 0:
result['basic_info']['creator_name'] = up_data.get('data', {}).get('name', '')
except Exception as e:
logger.error(f"专栏合集提取错误: {e}")
result = {'error': str(e)}
return result
async def _extract_favlist_async(self, fid: str) -> Dict[str, Any]:
"""
异步提取收藏夹信息（不包含列表内容）
Args:
fid: 收藏夹ID
Returns:
Dict containing favlist basic info
"""
result = {
'type': 'favlist',
'favlist_id': fid,
'subscribed': False,
'basic_info': {}
}
if not HTTPX_AVAILABLE:
return {'error': 'httpx 未安装'}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取收藏夹信息
response = await client.get('https://api.bilibili.com/x/space/favlist',
params={'fid': fid, 'pn': 1, 'ps': 1})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
fav_info = data.get('data', {})
result['basic_info'] = {
'title': fav_info.get('title', ''),
'description': fav_info.get('intro', ''),
'cover': fav_info.get('cover', ''),
'mid': fav_info.get('mid', ''),
'creator_name': '',
'total_count': fav_info.get('media_count', 0),
'is_public': fav_info.get('public', True),
'like_count': fav_info.get('like', 0)
}
# 获取创作者信息
mid = fav_info.get('mid', '')
if mid:
up_info = await client.get('https://api.bilibili.com/x/space/acc/info',
params={'mid': mid})
if up_info.status_code == 200:
up_data = up_info.json()
if up_data.get('code') == 0:
result['basic_info']['creator_name'] = up_data.get('data', {}).get('name', '')
except Exception as e:
logger.error(f"收藏夹提取错误: {e}")
result = {'error': str(e)}
return result
# 测试代码
if __name__ == '__main__':
extractor = BilibiliExtractor()
print("=" * 60)
print("B站元数据提取器测试")
print("=" * 60)
# 测试视频
print("\n测试视频提取: BV1MnFNz5EDL")
result = extractor.extract_video_metadata('BV1MnFNz5EDL')
print(json.dumps(result, ensure_ascii=False, indent=2))
kb-bilibili-metadata/bilibili_metadata.py
+228
View File
@@ -0,0 +1,228 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
B站元数据提取 CLI 工具
支持命令行调用，输出 JSON 格式数据
"""
import argparse
import asyncio
import json
import os
import sys
from pathlib import Path
from typing import Dict, Any
from bilibili_extractor import BilibiliExtractor
def format_number(num: int) -> str:
"""格式化数字"""
if num >= 100000000:
return f"{num / 100000000:.1f}亿"
elif num >= 10000:
return f"{num / 10000:.1f}万"
else:
return f"{num:,}"
def extract_video(extractor: BilibiliExtractor, bvid: str, output_file: str = None) -> Dict[str, Any]:
"""提取视频元数据"""
result = extractor.extract_video_metadata(bvid)
# 如果需要输出到文件
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def extract_article(extractor: BilibiliExtractor, cvid: str, output_file: str = None) -> Dict[str, Any]:
"""提取专栏元数据"""
result = extractor.extract_article_metadata(cvid)
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def extract_up(extractor: BilibiliExtractor, mid: str, output_file: str = None) -> Dict[str, Any]:
"""提取UP主信息（包含订阅跟踪所需信息）"""
result = asyncio.run(extractor._extract_up_async(mid))
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def extract_series(extractor: BilibiliExtractor, mid: str, series_id: str, output_file: str = None) -> Dict[str, Any]:
"""提取系列合集元数据"""
result = asyncio.run(extractor._extract_series_async(mid, series_id))
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def extract_readlist(extractor: BilibiliExtractor, rl_id: str, output_file: str = None) -> Dict[str, Any]:
"""提取专栏合集元数据"""
result = asyncio.run(extractor._extract_readlist_async(rl_id))
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def extract_favlist(extractor: BilibiliExtractor, fid: str, output_file: str = None) -> Dict[str, Any]:
"""提取收藏夹元数据"""
result = asyncio.run(extractor._extract_favlist_async(fid))
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def main():
"""主函数"""
parser = argparse.ArgumentParser(
description='B站元数据提取工具',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
# 提取视频元数据
python bilibili_metadata.py --id "BV1xx411c7mD" --type video
# 提取专栏元数据
python bilibili_metadata.py --id "cv43780186" --type article
# 提取UP主信息
python bilibili_metadata.py --id "7770724" --type up
# 提取系列合集信息
python bilibili_metadata.py --id "123456" --type series --mid "7770724"
# 提取专栏合集信息
python bilibili_metadata.py --id "rl123456" --type readlist
# 提取收藏夹信息
python bilibili_metadata.py --id "123456" --type favlist
# 输出到文件
python bilibili_metadata.py --id "BV1xx411c7mD" --type video --output result.json
"""
)
parser.add_argument(
'--id',
required=True,
help='B站资源ID（BV号、CV号或UP主ID）'
)
parser.add_argument(
'--type',
required=True,
choices=['video', 'article', 'up', 'series', 'readlist', 'favlist'],
help='资源类型'
)
parser.add_argument(
'--mid',
help='UP主ID (用于series类型)'
)
parser.add_argument(
'--output',
'-o',
help='输出文件路径（JSON格式）'
)
parser.add_argument(
'--headless',
action='store_true',
default=True,
help='使用无头模式（浏览器，默认开启，使用API）'
)
parser.add_argument(
'--timeout',
type=int,
default=30,
help='超时时间（秒，默认30）'
)
parser.add_argument(
'--pretty',
action='store_true',
default=True,
help='格式化输出（默认开启）'
)
args = parser.parse_args()
# 创建提取器
extractor = BilibiliExtractor(headless=args.headless, timeout=args.timeout)
try:
# 根据类型提取数据
if args.type == 'video':
result = extract_video(extractor, args.id, args.output)
elif args.type == 'article':
result = extract_article(extractor, args.id, args.output)
elif args.type == 'up':
result = extract_up(extractor, args.id, args.output)
elif args.type == 'series':
if not args.mid:
print(json.dumps({
"success": False,
"error": "series类型需要提供 --mid 参数"
}, ensure_ascii=False, indent=2))
sys.exit(1)
result = extract_series(extractor, args.mid, args.id, args.output)
elif args.type == 'readlist':
result = extract_readlist(extractor, args.id, args.output)
elif args.type == 'favlist':
result = extract_favlist(extractor, args.id, args.output)
else:
print(json.dumps({
"success": False,
"error": f"不支持的类型 {args.type}"
}, ensure_ascii=False, indent=2))
sys.exit(1)
# 生成文件名
platform = "bilibili"
filename = f"{platform}_{args.id}.meta.json"
filepath = args.output or filename
# 统一输出格式
output = {
"success": True,
"filename": filename,
"filepath": str(Path(filepath).resolve()) if filepath else None,
"message": f"{args.type}元数据提取成功"
}
print(json.dumps(output, ensure_ascii=False, indent=2))
# 如果指定了输出路径，则写入文件
if args.output:
with open(args.output, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
except Exception as e:
print(json.dumps({
"success": False,
"error": str(e),
"detail": type(e).__name__
}, ensure_ascii=False, indent=2))
sys.exit(1)
if __name__ == '__main__':
main()
kb-deep-analyzer/SKILL.md
+94
View File
@@ -0,0 +1,94 @@
# KB Deep Analyzer Skill
**kb-deep-analyzer** - AI-powered deep content analysis.
## Features
- Multi-type analysis (tutorial, documentation, news)
- Template-based output
- Quality scoring
- Automatic tagging
- **多格式元数据支持**: 支持扁平结构 `{"title":"xxx"}` 和嵌套结构 `{"data":{"metadata":{...}}}`
## Commands
```bash
# 分析内容（推荐：使用 -u 参数强制无缓冲输出）
AI_API_KEY="your-api-key" AI_PROVIDER="zhipu" AI_MODEL="glm-4.7" \
python3 -u deep_analyzer.py --metadata meta.json --content content.json --output analysis.md
# 指定类型
python deep_analyzer.py --metadata meta.json --content content.json --type tutorial --output tutorial.md
# 指定 AI 提供商
python deep_analyzer.py -m meta.json -c content.json -o out.md --provider zhipu --model glm-4
```
## 环境变量配置
**必须设置以下环境变量**：
```bash
export AI_API_KEY="your-api-key"
export AI_PROVIDER="zhipu" # 或 openai, minimax
export AI_MODEL="glm-4.7" # 模型名称
export AI_BASE_URL="https://..." # 可选，自定义 API 地址
```
### 重要提示
**必须使用 `-u` 参数运行 Python**，确保输出实时显示在终端：
```bash
python3 -u deep_analyzer.py ...
```
如果不使用 `-u` 参数，Python 的 stdout 缓冲会导致输出延迟或看不到进度。
## 执行方式选择
### 方式1: 并发执行（推荐，默认）
- 使用 `batch_size=3`（默认）
- 6个分片约 100 秒完成
- 适合快速分析
```bash
python3 -u deep_analyzer.py --metadata meta.json --content content.json -o out.md
```
### 方式2: 顺序执行
- 使用 `batch_size=1`
- 6个分片约 735 秒完成
- 适合调试和问题排查
```bash
# 在代码中设置 batch_size=1
# analyzer.analyze(metadata, content, "video", batch_size=1)
```
## Metadata Format
支持两种元数据格式：
```json
// 格式1: 扁平结构
{
"title": "视频标题",
"author": "作者名",
"duration": 619,
"bvid": "BVxxx"
}
// 格式2: 嵌套结构（kb-bilibili-metadata 输出）
{
"data": {
"metadata": {
"title": "视频标题",
"author": "作者名",
"duration": "10:19"
}
}
}
```
会自动转换为扁平格式进行分析。
kb-deep-analyzer/__pycache__/deep_analyzer.cpython-312.pyc
BIN
View File
Binary file not shown.
kb-deep-analyzer/ai_client.py
+514
View File
@@ -0,0 +1,514 @@
"""Obsidian KB - AI客户端
支持多个 AI 提供商和 fallback 机制
"""
import os
import json
import requests
from typing import Dict, Any, List, Optional, Tuple
from src.models import ContentType
from src.config import get_config
from src.content_chunker import ContentChunker, get_chunking_strategy, chunk_content
class AIClient:
"""
AI客户端 - 支持分段分析长视频内容和 fallback 机制
"""
def __init__(self):
self.config = get_config()
self.chunker = ContentChunker()
# 获取 AI 配置
self._load_ai_config()
def _load_ai_config(self) -> None:
"""加载 AI 配置（主 provider + fallbacks）"""
self.ai_configs = []
# 加载主配置
main_provider = self.config.get('ai.provider', 'openai')
main_config = {
'provider': main_provider,
'api_key': self.config.get('ai.api_key'),
'model': self.config.get('ai.model', 'gpt-4'),
'base_url': self.config.get('ai.base_url'),
'is_primary': True
}
self.ai_configs.append(main_config)
# 加载 fallback 配置
fallbacks = self.config.get('ai.fallbacks', [])
for fb in fallbacks:
self.ai_configs.append({
'provider': fb.get('provider'),
'api_key': fb.get('api_key'),
'model': fb.get('model'),
'base_url': fb.get('base_url'),
'is_primary': False
})
def _call_api_with_fallback(self, prompt: str, max_tokens: int = 1000, response_format: str = 'text') -> Tuple[bool, str]:
"""
使用 fallback 机制调用 API
Args:
prompt: 提示词
max_tokens: 最大 token 数
response_format: 响应格式 ('text' 或 'json')
Returns:
(success, response) 元组
"""
last_error = None
for i, ai_config in enumerate(self.ai_configs):
try:
if not ai_config.get('api_key'):
print(f"[AIClient] 跳过 {ai_config.get('provider')} - API Key 为空")
continue
provider_type = ai_config.get('provider')
print(f"[AIClient] 尝试使用 {provider_type} (优先级 {i+1}/{len(self.ai_configs)})...")
response = self._call_single_provider(ai_config, prompt, max_tokens, response_format)
return True, response
except Exception as e:
last_error = e
print(f"[AIClient] {provider_type} 调用失败: {e}")
continue
# 所有 provider 都失败了
error_msg = f"所有 AI 提供商均失败，最后错误: {last_error}" if last_error else "没有可用的 AI 配置"
print(f"[AIClient] {error_msg}")
return False, self._mock_generate(prompt, max_tokens) # 返回 mock 响应作为兜底
def _call_single_provider(self, ai_config: Dict, prompt: str, max_tokens: int, response_format: str) -> str:
"""
调用单个 AI 提供商
Args:
ai_config: AI 配置
prompt: 提示词
max_tokens: 最大 token 数
response_format: 响应格式
Returns:
API 响应文本
"""
provider = ai_config.get('provider')
api_key = ai_config.get('api_key')
model = ai_config.get('model')
base_url = ai_config.get('base_url')
if provider == 'minimax':
return self._call_minimax(api_key, model, base_url, prompt, max_tokens, response_format)
elif provider == 'zhipu':
return self._call_zhipu(api_key, model, base_url, prompt, max_tokens, response_format)
elif provider == 'alibaba':
return self._call_alibaba(api_key, model, base_url, prompt, max_tokens, response_format)
elif provider == 'openai':
return self._call_openai(api_key, model, base_url, prompt, max_tokens, response_format)
else:
raise ValueError(f"不支持的 provider: {provider}")
def _call_minimax(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str:
"""调用 MiniMax API (Anthropic Messages 兼容接口)"""
url = base_url or "https://api.minimaxi.com/anthropic/v1/messages"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
payload = {
"model": model,
"max_tokens": max_tokens,
"messages": [
{"role": "user", "content": prompt}
]
}
if response_format == 'json':
payload['messages'][0]['content'] = prompt + "\n\n请直接返回 JSON 格式，不要有其他文字。"
response = requests.post(url, headers=headers, json=payload, timeout=30)
response.raise_for_status()
data = response.json()
# MiniMax 返回的是 Anthropic 格式，content 是一个数组
# 需要找到 type='text' 的元素
for item in data.get('content', []):
if item.get('type') == 'text':
return item.get('text', '')
# 如果没有找到 text 类型，返回第一个元素的值（兼容旧格式）
return data.get('content', [{}])[0].get('text', '')
def _call_zhipu(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str:
"""调用智谱 AI API (OpenAI 兼容接口)"""
# 确保 base_url 包含 /chat/completions
if base_url and not base_url.endswith('/chat/completions'):
url = f"{base_url}/chat/completions"
else:
url = base_url or "https://open.bigmodel.cn/api/paas/v4/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
payload = {
"model": model,
"max_tokens": max_tokens,
"messages": [
{"role": "user", "content": prompt}
]
}
response = requests.post(url, headers=headers, json=payload, timeout=30)
response.raise_for_status()
data = response.json()
return data['choices'][0]['message']['content']
def _call_alibaba(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str:
"""调用阿里云通义千问 API (OpenAI 兼容接口)"""
url = f"{base_url}/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
payload = {
"model": model,
"max_tokens": max_tokens,
"messages": [
{"role": "user", "content": prompt}
]
}
response = requests.post(url, headers=headers, json=payload, timeout=30)
response.raise_for_status()
data = response.json()
return data['choices'][0]['message']['content']
def _call_openai(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str:
"""调用 OpenAI API"""
url = base_url or "https://api.openai.com/v1/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
payload = {
"model": model,
"max_tokens": max_tokens,
"messages": [
{"role": "user", "content": prompt}
]
}
response = requests.post(url, headers=headers, json=payload, timeout=30)
response.raise_for_status()
data = response.json()
return data['choices'][0]['message']['content']
def quick_generate(self, prompt: str, max_tokens: int = 100) -> str:
"""快速生成（用于轻量扫描）"""
success, response = self._call_api_with_fallback(prompt, max_tokens, 'text')
return response
def generate_json(self, prompt: str) -> Dict[str, Any]:
"""生成JSON格式响应"""
success, response = self._call_api_with_fallback(prompt, 1000, 'json')
try:
return json.loads(response)
except:
return self._parse_json_from_text(response)
def _deep_analyze(self, content_text: str, content_type: str, duration: Optional[int] = None) -> Dict[str, Any]:
"""
C5.3: 深度分析入口，根据内容时长决定是否分段分析
Args:
content_text: 内容文本
content_type: 内容类型
duration: 视频时长（秒）
Returns:
分析结果字典
"""
duration = duration or 0
print(f"[AIClient] 深度分析: type={content_type}, duration={duration}s")
# 长视频（>=30分钟）且是视频类型，使用分段分析
if duration >= 1800 and content_type == "video":
print(f"[AIClient] 使用分段分析策略")
return self._analyze_in_chunks(content_text, duration)
# 短视频直接分析
print(f"[AIClient] 使用直接分析策略")
return self._analyze_direct(content_text)
def _analyze_in_chunks(self, content_text: str, duration: int) -> Dict[str, Any]:
"""
C5.3: 分段分析长视频
Args:
content_text: SRT字幕文本
duration: 视频时长（秒）
Returns:
合并后的分析结果
"""
# 1. 获取分段策略
strategy = get_chunking_strategy(duration)
print(f"[AIClient] 分段策略: {strategy['description']}")
# 2. 执行分片
chunks = chunk_content(content_text, duration)
print(f"[AIClient] 内容已分为 {len(chunks)} 段")
# 3. 逐段分析
chunk_summaries = []
for i, chunk in enumerate(chunks):
print(f"[AIClient] 分析第 {i+1}/{len(chunks)} 段...")
summary = self._analyze_chunk(chunk.text, chunk.index)
# 处理时间格式（可能是float秒数或字符串）
start_time = chunk.start_time
end_time = chunk.end_time
if isinstance(start_time, (int, float)):
start_time = self._format_time(start_time)
if isinstance(end_time, (int, float)):
end_time = self._format_time(end_time)
chunk_summaries.append({
'index': chunk.index,
'start_time': start_time,
'end_time': end_time,
'summary': summary
})
# 4. 合并所有段摘要
print(f"[AIClient] 合并 {len(chunk_summaries)} 段分析结果")
final_result = self._merge_summaries(chunk_summaries, "video")
# 5. 构建完整时间轴
final_result['timeline'] = self._build_timeline_from_chunks(chunk_summaries)
return final_result
def _analyze_chunk(self, chunk_text: str, chunk_index: int) -> str:
"""
C5.3: 分析单个内容分片
Args:
chunk_text: 分片文本
chunk_index: 分片索引
Returns:
分片摘要（限制500字以内）
"""
# 截断文本避免过长
text_sample = chunk_text[:3000] if len(chunk_text) > 3000 else chunk_text
prompt = f"""请对以下视频片段进行简要分析（限制在300字以内）：
片段内容：
{text_sample}
请提供：
1. 这段的核心主题（50字以内）
2. 关键要点（3-5条）
3. 重要细节或引用
请用中文回答，控制在300字以内。"""
try:
summary = self.quick_generate(prompt, max_tokens=500)
# 确保不超过500字
if len(summary) > 500:
summary = summary[:497] + "..."
return summary
except Exception as e:
print(f"[AIClient] 分片 {chunk_index} 分析失败: {e}")
return f"第{chunk_index+1}段分析失败"
def _merge_summaries(self, chunk_summaries: List[Dict], content_type: str) -> Dict[str, Any]:
"""
C5.3: 合并所有段摘要，生成完整分析报告
Args:
chunk_summaries: 各段摘要列表
content_type: 内容类型
Returns:
完整分析结果
"""
# 构建合并提示
summaries_text = "\n\n".join([
f"=== 第{s['index']+1}段 ({s['start_time']}-{s['end_time']}) ===\n{s['summary']}"
for s in chunk_summaries
])
prompt = f"""基于以下各段分析，生成完整的视频分析报告：
{summaries_text}
请提供以下格式的JSON：
{{
"summary": "整体摘要（200字以内）",
"topics": ["主题1", "主题2", "主题3", "主题4"],
"insights": ["深入见解1", "深入见解2", "深入见解3"],
"actions": ["建议行动1", "建议行动2"],
"full_analysis": "详细分析（500字以内）"
}}"""
try:
result = self.generate_json(prompt)
# 确保必要字段存在
if 'topics' not in result:
result['topics'] = ["主题A", "主题B", "主题C"]
if 'insights' not in result:
result['insights'] = ["重要见解1", "重要见解2"]
if 'actions' not in result:
result['actions'] = []
if 'summary' not in result:
result['summary'] = "长视频分析完成"
if 'full_analysis' not in result:
result['full_analysis'] = summaries_text[:500]
return result
except Exception as e:
print(f"[AIClient] 合并摘要失败: {e}")
# 返回基础合并结果
return {
'summary': f'视频共分为{len(chunk_summaries)}个部分进行分析',
'topics': [f"主题{i+1}" for i in range(min(4, len(chunk_summaries)))],
'insights': ['从分段分析中综合得出'],
'actions': [],
'full_analysis': summaries_text[:800]
}
def _analyze_direct(self, content_text: str) -> Dict[str, Any]:
"""直接分析（不分段）"""
# 截断文本
text_sample = content_text[:5000] if len(content_text) > 5000 else content_text
prompt = f"""请分析以下内容，提供JSON格式结果：
{text_sample}
返回格式：
{{
"summary": "整体摘要（150字以内）",
"topics": ["主题1", "主题2", "主题3"],
"insights": ["见解1", "见解2"],
"actions": ["行动建议1"],
"timeline": [
{{"start_time": "00:00", "end_time": "05:00", "title": "第一部分", "summary": "..."}}
]
}}"""
try:
return self.generate_json(prompt)
except Exception as e:
print(f"[AIClient] 直接分析失败: {e}")
return {
'summary': '分析失败',
'topics': ['无法解析'],
'insights': ['请检查内容格式'],
'actions': [],
'timeline': []
}
def _format_time(self, seconds: float) -> str:
"""格式化秒数为时间字符串"""
hours = int(seconds // 3600)
minutes = int((seconds % 3600) // 60)
secs = int(seconds % 60)
if hours > 0:
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
return f"{minutes:02d}:{secs:02d}"
def _build_timeline_from_chunks(self, chunk_summaries: List[Dict]) -> List[Dict[str, Any]]:
"""从分片摘要构建时间轴"""
timeline = []
for summary in chunk_summaries:
timeline.append({
'index': summary['index'],
'start_time': summary['start_time'],
'end_time': summary['end_time'],
'title': f"第{summary['index']+1}部分",
'summary': summary['summary'][:200] + "..." if len(summary['summary']) > 200 else summary['summary']
})
return timeline
def _mock_generate(self, prompt: str, max_tokens: int) -> str:
"""模拟AI生成（实际实现应调用真实API）"""
# 根据prompt内容返回模拟响应
prompt_lower = prompt.lower()
if 'json' in prompt_lower or 'json格式' in prompt:
if '主题' in prompt or 'topic' in prompt_lower:
return json.dumps({
'summary': '这是一个关于人工智能和技术发展的内容，涵盖了多个重要主题。',
'topics': ['人工智能', '机器学习', '未来发展', '技术趋势'],
'insights': [
'AI正在改变我们的工作方式',
'机器学习应用越来越广泛',
'未来技术发展趋势值得关注'
],
'actions': ['深入研究相关技术', '关注行业动态'],
'full_analysis': '详细分析内容...'
}, ensure_ascii=False)
return json.dumps({'result': '模拟JSON响应'})
if '摘要' in prompt or 'summary' in prompt_lower:
return '这是一段关于人工智能发展的重要讨论，涵盖了技术趋势和未来应用。'
if '关键词' in prompt or 'keyword' in prompt_lower:
return '人工智能, 机器学习, 深度学习, 神经网络, 自然语言处理'
return '模拟AI响应内容'
def _parse_json_from_text(self, text: str) -> Dict[str, Any]:
"""从文本中解析JSON"""
# 尝试提取JSON块
import re
json_match = re.search(r'\{[\s\S]*\}', text)
if json_match:
try:
return json.loads(json_match.group())
except:
pass
return {'raw_response': text}
class MultiAIClient:
"""
多AI客户端 - 支持多种AI提供商的客户端
"""
def __init__(self):
self.config = get_config()
self.primary_client = AIClient()
self.clients = {
'openai': self.primary_client,
'local': AIClient(), # 可以配置为本地模型
}
def _deep_analyze(self, content_text: str, content_type: str, duration: Optional[int] = None) -> Dict[str, Any]:
"""使用主客户端进行深度分析"""
return self.primary_client._deep_analyze(content_text, content_type, duration)
def quick_generate(self, prompt: str, max_tokens: int = 100, provider: str = None) -> str:
"""快速生成"""
client = self.clients.get(provider, self.primary_client)
return client.quick_generate(prompt, max_tokens)
kb-deep-analyzer/deep_analyzer.py
+511
View File
@@ -0,0 +1,511 @@
#!/usr/bin/env python3
"""Deep Analyzer - AI 深度内容分析工具
一次性调用 AI 生成完整深度分析报告。
Usage:
python deep_analyzer.py --metadata meta.json --content content.json --output analysis.md [--type video]
python deep_analyzer.py --help
"""
import argparse
import json
import os
import sys
import re
from datetime import datetime
from pathlib import Path
from typing import Dict, Any, List
from dataclasses import dataclass, field
# ============== 数据模型 ==============
@dataclass
class AnalysisResult:
"""分析结果"""
summary: str = ""
main_topics: List[str] = field(default_factory=list)
key_insights: List[str] = field(default_factory=list)
recommended_tags: List[str] = field(default_factory=list)
full_analysis: str = ""
# ============== AI 客户端 ==============
class SimpleAIClient:
"""AI 客户端"""
def __init__(self, api_key: str = None, provider: str = None, model: str = None, base_url: str = None):
self.api_key = api_key or os.environ.get("AI_API_KEY", "")
self.provider = provider or os.environ.get("AI_PROVIDER", "")
self.model = model or os.environ.get("AI_MODEL", "gpt-4")
self.base_url = base_url or os.environ.get("AI_BASE_URL", "")
self.max_retries = 3
def generate(self, prompt: str, max_tokens: int = 16000) -> str:
"""生成文本"""
api_key = os.environ.get("AI_API_KEY", self.api_key)
provider = os.environ.get("AI_PROVIDER", self.provider)
model = os.environ.get("AI_MODEL", self.model)
base_url = os.environ.get("AI_BASE_URL", self.base_url)
if not api_key:
raise ValueError("AI_API_KEY 未设置")
for attempt in range(self.max_retries):
try:
import time
start_time = time.time()
print(f"[AI Client] 开始调用 AI...")
if provider == "openai":
result = self._call_openai(prompt, max_tokens, api_key, model, base_url)
elif provider == "zhipu":
result = self._call_zhipu(prompt, max_tokens, api_key, model, base_url)
elif provider == "minimax":
result = self._call_minimax(prompt, max_tokens, api_key, model, base_url)
else:
raise ValueError(f"不支持的 AI 提供商: {provider}")
elapsed = time.time() - start_time
print(f"[AI Client] AI 调用完成，耗时: {elapsed:.2f}秒")
return result
except Exception as e:
if attempt < self.max_retries - 1:
print(f"[AI Client] 第 {attempt + 1} 次调用失败，{self.max_retries - attempt - 1} 次重试剩余: {e}")
continue
else:
raise
def _call_openai(self, prompt: str, max_tokens: int, api_key: str, model: str, base_url: str) -> str:
import requests
url = f"{base_url}/chat/completions" if base_url else "https://api.openai.com/v1/chat/completions"
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
payload = {"model": model, "max_tokens": max_tokens, "messages": [{"role": "user", "content": prompt}]}
response = requests.post(url, headers=headers, json=payload, timeout=600)
response.raise_for_status()
return response.json()['choices'][0]['message']['content']
def _call_zhipu(self, prompt: str, max_tokens: int, api_key: str, model: str, base_url: str) -> str:
import requests
base_url = base_url or "https://open.bigmodel.cn/api/coding/paas/v4"
url = f"{base_url}/chat/completions"
headers = {"Authorization": f"Bearer {api_key}"}
payload = {"model": model, "messages": [{"role": "user", "content": prompt}], "max_tokens": max_tokens}
response = requests.post(url, headers=headers, json=payload, timeout=600)
response.raise_for_status()
data = response.json()
message = data.get('choices', [{}])[0].get('message', {})
content = message.get('content', '')
reasoning = message.get('reasoning_content', '')
if not content and reasoning:
sentences = reasoning.strip().split('。')
useful = [s.strip() for s in sentences if s.strip() and len(s.strip()) > 10]
if useful:
return '。'.join(useful[-3:]) + '。'
return reasoning.strip()[:500]
return content
def _call_minimax(self, prompt: str, max_tokens: int, api_key: str, model: str, base_url: str) -> str:
import requests
url = base_url or "https://api.minimaxi.com/anthropic/v1/messages"
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
payload = {"model": model, "max_tokens": max_tokens, "messages": [{"role": "user", "content": prompt}]}
response = requests.post(url, headers=headers, json=payload, timeout=600)
response.raise_for_status()
data = response.json()
for item in data.get('content', []):
if item.get('type') == 'text':
return item.get('text', '')
return str(data)
# ============== 分析器 ==============
class DeepAnalyzer:
"""深度内容分析器"""
def __init__(self, ai_client: SimpleAIClient = None):
self.ai_client = ai_client or SimpleAIClient()
def analyze(self, metadata: Dict[str, Any], content: Dict[str, Any]) -> AnalysisResult:
"""一次性调用 AI 生成完整分析报告"""
# 标准化元数据
metadata = self._normalize_metadata(metadata)
title = metadata.get('title', '未知')
duration = metadata.get('duration', 0)
author = metadata.get('author', '')
pubtime = metadata.get('pubtime', '')
url = metadata.get('url', '')
bvid = metadata.get('bvid', '')
# 获取内容 - 支持多种字段名
text = content.get('text', '') or content.get('subtitle', '') or content.get('subtitles', '') or content.get('description', '')
print(f"[DeepAnalyzer] 开始分析: {title}")
print(f"[DeepAnalyzer] 时长: {duration}秒, 内容长度: {len(text)} 字符")
# 计算分片数（按10分钟分片）
num_chunks = max(1, (duration + 599) // 600) # 向上取整到10分钟
print(f"[DeepAnalyzer] 将分 {num_chunks} 段分析")
# 构建 prompt
prompt = self._build_prompt(
title=title,
duration=duration,
duration_str=self._format_duration(duration),
author=author,
pubtime=pubtime,
url=url,
bvid=bvid,
text=text,
num_chunks=num_chunks
)
# 调用 AI
try:
full_analysis = self.ai_client.generate(prompt, max_tokens=16000)
except Exception as e:
print(f"[DeepAnalyzer] AI 调用失败: {e}")
raise
# 解析结果
result = self._parse_analysis(full_analysis, text, duration)
return result
def _build_prompt(self, title: str, duration: int, duration_str: str, author: str,
pubtime: str, url: str, bvid: str, text: str, num_chunks: int) -> str:
"""构建分析 prompt"""
# 截取适当长度的文本（20-30K 字符）
max_chars = 30000
if len(text) > max_chars:
text_sample = text[:max_chars] + "\n\n[...]"
else:
text_sample = text
prompt = f"""你是一个专业的视频内容分析师。请根据以下信息生成深度分析报告：
【视频信息】
标题：{title}
作者：{author}
时长：{duration_str}（{duration}秒）
发布日期：{pubtime}
平台：bilibili
URL：{url}
【内容】（完整字幕）
{text_sample}
【输出要求】必须使用以下格式：
## 标题
[自己总结的简洁标题，禁止使用原标题]
## 摘要
[200-400字]
## 主要主题
[从内容中提取不超过10个主题，每个不超过10字]
- 主题1
- 主题2
...
## 关键见解
[若干条深刻见解]
- 见解1
- 见解2
...
## 标签
[从内容中提取3-5个精炼标签，每个不超过10字]
- 标签1
- 标签2
...
## 时间轴
### 第 1 部分 (00:00 - 10:00)
[详细分析，至少500字]
### 第 2 部分 (10:00 - 20:00)
[详细分析，至少500字]
...（以此类推）
【重要】分析应足够详尽，读者阅读后可了解视频核心内容，无需再看原视频。"""
return prompt
def _parse_analysis(self, analysis_text: str, original_text: str, duration: int) -> AnalysisResult:
"""解析 AI 返回的分析结果"""
result = AnalysisResult()
result.full_analysis = analysis_text
# 提取各部分（使用正则）
lines = analysis_text.split('\n')
# 简单解析：按 ## 标题分割
current_section = ""
in_section = None
for line in lines:
if line.startswith('##'):
# 保存上一个 section
if in_section:
self._store_section(result, in_section, current_section)
# 开始新 section
section_name = line.lstrip('#').strip()
in_section = section_name
current_section = ""
elif in_section:
current_section += line + "\n"
# 存储最后一个 section
if in_section:
self._store_section(result, in_section, current_section)
# 如果解析失败，使用原始文本
if not result.summary:
result.summary = analysis_text[:500]
return result
def _store_section(self, result: AnalysisResult, section_name: str, content: str):
"""存储解析到的 section"""
content = content.strip()
if not content:
return
section_lower = section_name.lower()
if '摘要' in section_lower:
result.summary = content[:500]
elif '主题' in section_lower:
# 提取主题
topics = []
for line in content.split('\n'):
line = line.strip().lstrip('-').strip()
if line and len(line) <= 15:
topics.append(line)
result.main_topics = topics[:10]
elif '见解' in section_lower or '观点' in section_lower:
# 提取见解
insights = []
for line in content.split('\n'):
line = line.strip().lstrip('-').strip()
if line and len(line) > 5:
insights.append(line)
result.key_insights = insights
elif '标签' in section_lower:
# 提取标签
tags = []
for line in content.split('\n'):
line = line.strip().lstrip('-').strip()
if line and len(line) <= 10:
tags.append(line)
result.recommended_tags = tags[:5]
def _normalize_metadata(self, metadata: Dict) -> Dict:
"""标准化元数据"""
if not metadata:
return {}
# 如果已经扁平
if 'title' in metadata:
duration = metadata.get('duration', 0)
if isinstance(duration, str) and ':' in duration:
parts = duration.split(':')
try:
metadata['duration'] = int(parts[0]) * 60 + int(parts[1])
except:
pass
return metadata
# 嵌套结构
if 'metadata' in metadata:
meta = metadata['metadata']
normalized = {}
for k, v in meta.items():
normalized[k] = v
# 处理 duration
duration = normalized.get('duration', 0)
if isinstance(duration, str) and ':' in duration:
parts = duration.split(':')
try:
normalized['duration'] = int(parts[0]) * 60 + int(parts[1])
except:
pass
return normalized
return metadata
def _format_duration(self, seconds: int) -> str:
"""格式化时长"""
h = seconds // 3600
m = (seconds % 3600) // 60
s = seconds % 60
if h > 0:
return f"{h}:{m:02d}:{s:02d}"
return f"{m}:{s:02d}"
# ============== 工具函数 ==============
def load_json(file_path: str) -> Dict:
"""加载 JSON 文件"""
if not file_path or not os.path.exists(file_path):
return {}
with open(file_path, 'r', encoding='utf-8') as f:
return json.load(f)
def generate_frontmatter(metadata: Dict, result: AnalysisResult) -> str:
"""生成 Markdown frontmatter"""
# 提取标签
tags = result.recommended_tags or []
# 格式化日期
pubtime = metadata.get('pubtime', '')
if pubtime:
date_str = str(pubtime)[:10]
else:
date_str = datetime.now().strftime('%Y-%m-%d')
fm = ["---",
f"title: \"{metadata.get('title', '未命名')}\"",
f"date: {date_str}",
f"duration: {metadata.get('duration_str', metadata.get('duration', 0))}",
f"author: {metadata.get('author', '未知')}",
f"bvid: {metadata.get('bvid', '')}",
f"platform: {metadata.get('platform', 'bilibili')}"]
# 添加 URL
url = metadata.get('url', '')
if url:
fm.append(f"url: {url}")
if tags:
fm.append("tags: [" + ", ".join(f'"{t}"' for t in tags) + "]")
fm.append("---")
return '\n'.join(fm)
# ============== 主程序 ==============
def main():
parser = argparse.ArgumentParser(description="Deep Analyzer - AI 深度内容分析工具")
parser.add_argument("--metadata", "-m", required=True, help="元数据 JSON 文件路径")
parser.add_argument("--content", "-c", required=True, help="内容 JSON 文件路径")
parser.add_argument("--output", "-o", required=True, help="输出 Markdown 文件路径")
parser.add_argument("--type", "-t", default="video", choices=["video", "article"], help="内容类型")
parser.add_argument("--api-key", "-k", default=None, help="AI API Key")
parser.add_argument("--provider", "-p", default=None, choices=["openai", "zhipu", "minimax"], help="AI 提供商")
parser.add_argument("--model", default=None, help="AI 模型")
parser.add_argument("--base-url", "-u", default=None, help="API Base URL")
parser.add_argument("--no-frontmatter", action="store_true", help="不生成 frontmatter")
args = parser.parse_args()
# 加载数据
print(f"[Main] 加载元数据: {args.metadata}")
raw_metadata = load_json(args.metadata)
print(f"[Main] 加载内容: {args.content}")
content = load_json(args.content)
# 标准化元数据
metadata = {}
if 'title' in raw_metadata:
metadata = raw_metadata.copy()
# 处理 duration
duration = metadata.get('duration', 0)
if isinstance(duration, str) and ':' in duration:
parts = duration.split(':')
try:
metadata['duration'] = int(parts[0]) * 60 + int(parts[1])
except:
pass
metadata['duration_str'] = format_duration(metadata.get('duration', 0))
elif 'metadata' in raw_metadata:
meta = raw_metadata['metadata']
metadata = meta.copy()
duration = metadata.get('duration', 0)
if isinstance(duration, str) and ':' in duration:
parts = duration.split(':')
try:
metadata['duration'] = int(parts[0]) * 60 + int(parts[1])
except:
pass
metadata['duration_str'] = format_duration(metadata.get('duration', 0))
# 初始化
ai_client = SimpleAIClient(
api_key=args.api_key or os.environ.get("AI_API_KEY", ""),
provider=args.provider or os.environ.get("AI_PROVIDER", ""),
model=args.model or os.environ.get("AI_MODEL", "gpt-4"),
base_url=args.base_url or os.environ.get("AI_BASE_URL", "")
)
analyzer = DeepAnalyzer(ai_client)
# 执行分析
print(f"[Main] 开始分析...")
result = analyzer.analyze(metadata, content)
# 使用 AI 生成的原始内容（保留 Markdown 格式）
full_analysis = result.full_analysis
# 删除内容中的"标签" section（只保留 frontmatter 中的 tags）
# 匹配 ## 标签 或 标签（无##）到下一个 ## 或文件结尾之间的内容
full_analysis = re.sub(r'(## ?)?标签\n[\s\S]*?(?=\n## ?|\n时间轴|---|\Z)', '', full_analysis)
# 清理多余的空行
full_analysis = re.sub(r'\n{3,}', '\n\n', full_analysis)
# 写入输出
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
with open(args.output, 'w', encoding='utf-8') as f:
# 添加 frontmatter
if not args.no_frontmatter:
fm = generate_frontmatter(metadata, result)
f.write(fm + "\n\n")
# 写入分析内容
f.write(full_analysis)
# 输出结果
output = {
"success": True,
"filename": Path(args.output).name,
"filepath": str(Path(args.output).resolve()),
"message": "深度分析完成",
"stats": {
"summary_length": len(result.summary) if result.summary else 0,
"topics_count": len(result.main_topics),
"insights_count": len(result.key_insights),
"tags_count": len(result.recommended_tags)
}
}
print(json.dumps(output, ensure_ascii=False, indent=2))
def format_duration(seconds: int) -> str:
"""格式化时长"""
h = seconds // 3600
m = (seconds % 3600) // 60
s = seconds % 60
if h > 0:
return f"{h}:{m:02d}:{s:02d}"
return f"{m}:{s:02d}"
if __name__ == "__main__":
try:
main()
except Exception as e:
print(json.dumps({"success": False, "error": str(e)}, ensure_ascii=False, indent=2))
sys.exit(1)
kb-deep-analyzer/deep_analyzer.py.bak
+1727
View File
File diff suppressed because it is too large Load Diff
kb-funasr/.env.example
+2
View File
@@ -0,0 +1,2 @@
# FunASR 模型缓存目录
FUNASR_CACHE_DIR=~/.cache/modelscope/hub
kb-funasr/README.md
+212
View File
@@ -0,0 +1,212 @@
# KB FunASR - 本地语音识别
基于阿里 FunASR 的本地语音识别工具，支持将音视频文件转换为文本和 JSON 格式。
## 功能特性
- 🎯 **本地执行** - 完全离线运行，保护隐私
- 🚀 **多模型支持** - Paraformer、SenseVoice 等模型可选
- 📝 **多格式输出** - JSON、TXT 两种格式
- ⚡ **异步执行** - 支持后台运行，不阻塞主代理
- 🎬 **广泛兼容** - 支持常见音视频格式（MP3、MP4、WAV 等）
- 🇨🇳 **中文优化** - Paraformer 模型针对中文语音优化
## 安装前提
### 1. 安装依赖
```bash
pip install funasr modelscope
```
### 2. 配置环境变量（可选）
在 `~/.clawdbot/clawdbot.json` 中添加：
```json5
{
skills: {
entries: {
"kb-funasr": {
env: {
FUNASR_CACHE_DIR: "~/.cache/modelscope/hub"
}
}
}
}
}
```
或通过命令行配置：
```bash
clawdbot skills config kb-funasr FUNASR_CACHE_DIR "~/.cache/modelscope/hub"
```
## 使用方法
### 基本用法
```bash
# 使用默认 Paraformer 中文模型处理
python funasr.py --input video.mp4
# 后台异步执行
python funasr.py --input video.mp4 --background
```
### 指定模型
```bash
# Paraformer 中文模型（推荐）
python funasr.py --input video.mp4 --model paraformer-zh
# Paraformer 多语言模型
python funasr.py --input video.mp4 --model paraformer
# SenseVoice 多语言模型
python funasr.py --input video.mp4 --model sensevoice
```
### 指定输出目录
```bash
python funasr.py --input video.mp4 --output ./transcripts
```
## 输出文件
对于输入 `video.mp4`，将生成以下文件：
```
output/
├── video.json # 详细结果（含时间戳）
└── video.txt # 纯文本
```
### JSON 格式
```json
{
"success": true,
"input_file": "video.mp4",
"model": "paraformer-zh",
"output_files": {
"json": "video.json",
"txt": "video.txt"
},
"duration_seconds": 120.5,
"segments": [
{
"id": 0,
"start": 0.0,
"end": 3.2,
"text": "这是第一段识别结果"
},
{
"id": 1,
"start": 3.2,
"end": 6.8,
"text": "这是第二段识别结果"
}
]
}
```
## 模型对比
| 模型 | 速度 | 精度 | 适用场景 |
|------|------|------|----------|
| paraformer-zh | ⚡⚡⚡ | ⭐⭐⭐⭐ | 中文语音（推荐） |
| paraformer | ⚡⚡ | ⭐⭐⭐⭐ | 多语言识别 |
| sensevoice | ⚡ | ⭐⭐⭐⭐⭐ | 高精度多语言 |
## 完整示例
### 示例 1: 转录播客
```bash
python funasr.py --input podcast.mp3 --model paraformer-zh --background
```
### 示例 2: 生成视频识别结果
```bash
python funasr.py --input tutorial.mp4 --model paraformer
```
### 示例 3: 批量处理（配合脚本）
```bash
#!/bin/bash
for video in videos/*.mp4; do
python funasr.py --input "$video" --model paraformer-zh
done
```
## 在 Workflow 中使用
在知识库处理流程中，如果视频没有识别结果：
```bash
# 步骤4: 下载视频后
# 步骤5: 使用 FunASR 生成识别结果
python funasr.py --input ${video_path} --model paraformer-zh --background
# 等待 FunASR 完成，然后继续后续步骤
```
## 技术细节
- **引擎**: 阿里 FunASR
- **模型框架**: ModelScope
- **音频采样率**: 自动转换为 16kHz
- **语言**: 根据模型支持不同语言
## 故障排查
### 问题: 模型下载失败
```bash
# 首次运行会自动下载，确保网络稳定
python funasr.py --input test.mp4
# 手动检查缓存目录
ls ~/.cache/modelscope/hub/
```
### 问题: 音频格式不支持
```bash
# 转换音频格式
ffmpeg -i input.avi -acodec libmp3lame output.mp3
python funasr.py --input output.mp3
```
### 问题: 依赖安装失败
```bash
# 使用清华镜像源
pip install funasr modelscope -i https://pypi.tuna.tsinghua.edu.cn/simple
```
## 性能参考
在 CPU 环境下（Intel i7）：
- **paraformer-zh 模型**: ~2x 实时速度（1分钟音频 ~30秒处理）
- **paraformer 模型**: ~1.5x 实时速度（1分钟音频 ~40秒处理）
- **sensevoice 模型**: ~1x 实时速度（1分钟音频 ~1分钟处理）
## 与 Whisper 对比
| 特性 | FunASR | Whisper |
|------|--------|---------|
| 中文识别 | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ |
| 速度 | ⚡⚡⚡ | ⚡⚡ |
| 模型大小 | 较小 | 较大 |
| 多语言支持 | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
## 许可证
本技能使用 MIT 许可证，FunASR 使用 Apache 2.0 许可证。
kb-funasr/SKILL.md
+113
View File
@@ -0,0 +1,113 @@
---
name: kb-funasr
description: 本地语音识别，支持音视频转文字（基于阿里 FunASR）
---
# KB FunASR Skill
**kb-funasr** - 本地语音识别，使用阿里 FunASR 将音视频文件转换为文本和 JSON 格式输出。
## 什么时候使用
当用户需要：
- 将视频/音频文件转换为文字
- 获取带时间戳的详细识别结果
- 需要中文优化的语音识别（Paraformer 模型）
## 环境变量
在使用之前，必须检查环境变量是否已经设置。
| 变量名 | 用途 | 示例 |
|--------|------|------|
| `FUNASR_CACHE_DIR` | FunASR 模型缓存目录 | `~/.cache/modelscope/hub` |
## 使用方法
```bash
# 基本用法（使用 Paraformer 模型）
python funasr.py --input video.mp4
# 指定模型
python funasr.py --input video.mp4 --model paraformer-zh
# 指定输出目录
python funasr.py --input video.mp4 --output ./output
# 异步执行（推荐）
python funasr.py --input video.mp4 --background
```
## 命令参数
| 参数 | 说明 | 默认值 |
|------|------|--------|
| `--input` | 输入音视频文件路径 | 必需 |
| `--model` | FunASR 模型类型 | `paraformer-zh` |
| `--output` | 输出目录 | 文件所在目录 |
| `--background` | 后台异步执行 | False |
## 模型选项
- `paraformer-zh` - Paraformer 中文模型（轻量，中文优化，推荐）
- `paraformer` - Paraformer 多语言模型
- `sensevoice` - SenseVoice 多语言模型
## 输出文件
对于输入 `video.mp4`，将生成：
- `video.json` - 详细结果（包含时间戳、分段信息）
- `video.txt` - 纯文本内容
## 输出格式示例
### JSON 格式
```json
{
"success": true,
"input_file": "video.mp4",
"model": "paraformer-zh",
"output_files": {
"json": "video.json",
"txt": "video.txt"
},
"duration_seconds": 120.5,
"segments": [
{
"id": 0,
"start": 0.0,
"end": 3.2,
"text": "这是第一段识别结果"
}
]
}
```
## Workflow 中的使用
```bash
# 步骤4之后：如果已有音频但无字幕，可以使用 FunASR
python funasr.py --input ${video_path} --model paraformer-zh --background
```
## 安装
```bash
# 1. 安装依赖
pip install funasr modelscope
# 2. 首次运行时会自动下载模型到缓存目录
python funasr.py --input test.mp4
# 3. 复制技能到 skills 目录
cp -r kb-funasr/ ~/skills/
```
## 注意事项
- **异步执行**: 使用 `--background` 参数时，命令会在后台运行，不阻塞主代理
- **模型选择**: paraformer-zh 模型适合中文语音识别，速度较快，精度较高
- **文件格式**: 支持 MP3、MP4、WAV、FLAC、OGG、M4A 等常见格式
- **首次运行**: 首次运行时会自动下载模型文件，需要稳定的网络连接
kb-funasr/__pycache__/funasr.cpython-312.pyc
BIN
View File
Binary file not shown.
kb-funasr/funasr_tool.py
+313
View File
@@ -0,0 +1,313 @@
#!/usr/bin/env python3
"""
KB FunASR - 本地语音识别工具
基于阿里 FunASR 实现音视频转文字
"""
import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from datetime import datetime
from typing import Dict, List, Optional
class FunASRTranscriber:
"""FunASR 语音识别器"""
# 模型映射
MODEL_MAP = {
"paraformer-zh": "damo/speech_paraformer-large-vad_punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
"paraformer": "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
"sensevoice": "FunASR/SenseVoiceSmall"
}
def __init__(self, model: str = "paraformer-zh", cache_dir: Optional[str] = None):
"""初始化 FunASR 识别器
Args:
model: 模型名称
cache_dir: 模型缓存目录
"""
self.model_name = model
self.model_id = self.MODEL_MAP.get(model, self.MODEL_MAP["paraformer-zh"])
self.cache_dir = cache_dir or os.path.expanduser("~/.cache/modelscope/hub")
self._model = None
def _load_model(self):
"""延迟加载模型"""
if self._model is None:
try:
from funasr import AutoModel
print(f"正在加载模型: {self.model_name} ({self.model_id})")
print(f"缓存目录: {self.cache_dir}")
self._model = AutoModel(
model=self.model_id,
disable_pbar=False,
disable_log=False,
)
print("模型加载完成")
except ImportError:
print("错误: 未安装 funasr 库")
print("请运行: pip install funasr modelscope")
sys.exit(1)
def _convert_audio(self, input_path: str) -> str:
"""转换音频格式为 16kHz WAV
Args:
input_path: 输入文件路径
Returns:
转换后的文件路径（如果不需要转换则返回原路径）
"""
# 检查是否已经是 WAV 格式
if input_path.lower().endswith('.wav'):
# 检查采样率
try:
result = subprocess.run(
['ffprobe', '-v', 'error', '-show_entries',
'stream=sample_rate', '-of', 'default=noprint_wrappers=1:nokey=1',
input_path],
capture_output=True, text=True
)
if result.returncode == 0:
sample_rate = int(result.stdout.strip())
if sample_rate == 16000:
return input_path
except Exception:
pass
# 需要转换
output_path = input_path.rsplit('.', 1)[0] + '_converted.wav'
print(f"正在转换音频格式: {input_path} -> {output_path}")
try:
subprocess.run([
'ffmpeg', '-y', '-i', input_path,
'-acodec', 'pcm_s16le',
'-ar', '16000',
'-ac', '1',
output_path
], check=True, capture_output=True)
print("音频转换完成")
return output_path
except subprocess.CalledProcessError as e:
print(f"音频转换失败: {e}")
# 如果转换失败，尝试使用原文件
print("尝试使用原文件继续处理")
return input_path
except FileNotFoundError:
print("警告: 未找到 ffmpeg，跳过音频转换")
return input_path
def transcribe(self, input_path: str) -> Dict:
"""转录音频文件
Args:
input_path: 输入文件路径
Returns:
识别结果字典
"""
input_path = os.path.abspath(input_path)
if not os.path.exists(input_path):
return {
"success": False,
"error": f"文件不存在: {input_path}"
}
print(f"\n开始处理: {input_path}")
print(f"使用模型: {self.model_name}")
# 转换音频格式
audio_path = self._convert_audio(input_path)
# 加载模型
self._load_model()
# 执行识别
print("正在进行语音识别...")
try:
result = self._model.generate(
input=audio_path,
batch_size_s=300,
)
except Exception as e:
return {
"success": False,
"error": f"识别失败: {str(e)}"
}
# 解析结果
if not result or not isinstance(result, list):
return {
"success": False,
"error": "识别结果格式错误"
}
# FunASR 返回格式处理
segments = []
full_text = ""
for seg in result:
if isinstance(seg, dict):
text = seg.get("text", "").strip()
if "timestamp" in seg and seg["timestamp"]:
# 包含时间戳
timestamp = seg["timestamp"]
if len(timestamp) >= 2:
start_time = timestamp[0][0] if timestamp[0] else 0
end_time = timestamp[-1][1] if timestamp[-1] else start_time
else:
start_time = 0
end_time = 0
else:
start_time = 0
end_time = 0
if text:
segments.append({
"id": len(segments),
"start": round(start_time, 2),
"end": round(end_time, 2),
"text": text
})
full_text += text + "\n"
# 清理临时文件
if audio_path != input_path and "_converted.wav" in audio_path:
try:
os.remove(audio_path)
except Exception:
pass
# 计算时长
duration = segments[-1]["end"] if segments else 0
return {
"success": True,
"input_file": os.path.basename(input_path),
"model": self.model_name,
"output_files": {},
"duration_seconds": round(duration, 2),
"segments": segments,
"text": full_text.strip()
}
def save_outputs(self, result: Dict, output_dir: str):
"""保存输出文件
Args:
result: 识别结果
output_dir: 输出目录
"""
if not result.get("success"):
return
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
basename = Path(result["input_file"]).stem
# 保存 JSON
json_path = output_dir / f"{basename}.json"
with open(json_path, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, indent=2)
print(f"已保存: {json_path}")
result["output_files"]["json"] = str(json_path)
# 保存 TXT
txt_path = output_dir / f"{basename}.txt"
with open(txt_path, "w", encoding="utf-8") as f:
f.write(result.get("text", ""))
print(f"已保存: {txt_path}")
result["output_files"]["txt"] = str(txt_path)
def run_as_background(input_path: str, model: str, output_dir: str):
"""后台运行识别
Args:
input_path: 输入文件路径
model: 模型名称
output_dir: 输出目录
"""
# 创建日志文件
log_file = os.path.join(output_dir, f"funasr_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log")
# 重新执行命令（后台模式）
cmd = [
sys.executable, __file__,
"--input", input_path,
"--model", model,
"--output", output_dir
]
with open(log_file, "w") as f:
process = subprocess.Popen(
cmd,
stdout=f,
stderr=subprocess.STDOUT,
start_new_session=True
)
print(f"\n✓ 后台任务已启动 (PID: {process.pid})")
print(f"日志文件: {log_file}")
print(f"输出目录: {output_dir}")
print("\n使用以下命令查看进度:")
print(f" tail -f {log_file}")
def main():
parser = argparse.ArgumentParser(
description="KB FunASR - 本地语音识别工具",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
python funasr.py --input video.mp4
python funasr.py --input podcast.mp3 --model paraformer-zh
python funasr.py --input video.mp4 --output ./transcripts --background
"""
)
parser.add_argument("--input", "-i", required=True, help="输入音视频文件路径")
parser.add_argument("--model", "-m", default="paraformer-zh",
choices=["paraformer-zh", "paraformer", "sensevoice"],
help="模型名称 (默认: paraformer-zh)")
parser.add_argument("--output", "-o", default=None, help="输出目录 (默认: 文件所在目录)")
parser.add_argument("--background", "-b", action="store_true", help="后台异步执行")
args = parser.parse_args()
# 确定输出目录
if args.output:
output_dir = args.output
else:
output_dir = os.path.dirname(args.input) or "."
# 后台模式
if args.background:
run_as_background(args.input, args.model, output_dir)
return
# 前台模式
transcriber = FunASRTranscriber(model=args.model)
result = transcriber.transcribe(args.input)
if result.get("success"):
transcriber.save_outputs(result, output_dir)
print(f"\n✓ 处理完成!")
print(f" 时长: {result['duration_seconds']} 秒")
print(f" 分段数: {len(result['segments'])}")
else:
print(f"\n✗ 处理失败: {result.get('error')}")
sys.exit(1)
if __name__ == "__main__":
main()
kb-link-parser/SKILL.md
+43
View File
@@ -0,0 +1,43 @@
# KB Link Parser Skill
**kb-link-parser** - Extract and resolve IDs from various platform URLs.
## Features
- Multi-platform support: Bilibili, YouTube, Zhihu
- Short link detection: b23.tv and other short links
- Multiple content types: video, article, up主, playlist, collection
- Batch processing from file
- JSON output format
## Commands
```bash
# Parse single URL
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
# Batch processing
python link-parser.py --batch urls.txt --output results.json
# Output format
python link-parser.py --url "..." --format json
```
## Output
```json
{
"success": true,
"data": {
"platform": "bilibili",
"type": "video",
"id": "BV1xx411c7mD"
}
}
```
## Installation
```bash
cp -r kb-link-parser/ ~/skills/
```
kb-link-parser/__pycache__/link_parser_shortlink.cpython-312.pyc
BIN
View File
Binary file not shown.
kb-link-parser/link-parser.py
+637
View File
@@ -0,0 +1,637 @@
#!/usr/bin/env python3
"""
Link Parser - Extract and resolve IDs from various platform URLs.
Supports:
- Bilibili (video, article, up主, collection, series)
- YouTube (video, channel, playlist)
- Zhihu (question, answer, user, collection)
- Xiaohongshu (note, user)
- Short links (b23.tv, xhslink.com, etc.)
Usage:
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
python link-parser.py --batch urls.txt --output results.json
"""
import argparse
import json
import re
import sys
from typing import Optional, Dict, Any
from urllib.parse import urlparse
from link_parser_shortlink import ShortLinkResolver
class LinkParser:
"""Main link parser class"""
# Bilibili patterns
BILIBILI_VIDEO_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV[a-zA-Z0-9]+)')
BILIBILI_VIDEO_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
BILIBILI_ARTICLE_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/(?:article|read)/([a-zA-Z0-9]+)')
BILIBILI_SPACE_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/(\d+)')
BILIBILI_MOBILE_SPACE_PATTERN = re.compile(r'https?://m\.bilibili\.com/space/(\d+)')
BILIBILI_MOBILE_OPUS_PATTERN = re.compile(r'https?://m\.bilibili\.com/opus/(\d+)')
BILIBILI_FAVLIST_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/favlist\?fid=(\d+)')
BILIBILI_OPUS_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/opus/(\d+)')
BILIBILI_SERIES_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/lists/(\d+)\?type=series')
BILIBILI_READLIST_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/read/readlist/(rl[a-zA-Z0-9]+)')
# YouTube patterns
YOUTUBE_VIDEO_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/)([a-zA-Z0-9_-]{11})')
YOUTUBE_CHANNEL_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/channel/([a-zA-Z0-9_-]+)')
YOUTUBE_USER_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/user/([a-zA-Z0-9_-]+)')
YOUTUBE_PLAYLIST_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/playlist\?list=([a-zA-Z0-9_-]+)')
# Xiaohongshu (小红书) patterns
XIAOHONGSHU_SHORT_PATTERN = re.compile(r'https?://xhslink\.com/o/([a-zA-Z0-9]+)')
XIAOHONGSHU_NOTE_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/explore/([a-zA-Z0-9]+)')
XIAOHONGSHU_NOTE_PATTERN2 = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/notes/([a-zA-Z0-9]+)')
XIAOHONGSHU_DISCOVERY_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/discovery/item/([a-zA-Z0-9]+)')
XIAOHONGSHU_USER_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/user/profile/(\d+)')
# Zhihu patterns
ZHIHAU_QUESTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/(\d+)')
ZHIHAU_ANSWER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)')
ZHIHAU_USER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/people/([a-zA-Z0-9_-]+)')
ZHIHAU_COLUMN_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/column/([a-zA-Z0-9_-]+)')
ZHIHAU_COLLECTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/collection/(\d+)')
# Additional Zhihu patterns
ZHIHAU_ZHUANLAN_PATTERN = re.compile(r'https?://zhuanlan\.zhihu\.com/p/(\d+)')
ZHIHAU_ORG_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/([a-zA-Z0-9_-]+)')
ZHIHAU_ORG_ANSWERS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/answers')
ZHIHAU_ORG_VIDEOS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/zvideos')
ZHIHAU_ORG_ASKS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/asks')
ZHIHAU_ORG_POSTS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/posts')
ZHIHAU_ORG_PINS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/pins')
ZHIHAU_ZVIDEO_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/zvideo/(\d+)')
def __init__(self):
self.shortlink_resolver = ShortLinkResolver()
def resolve_shortlink(self, url: str) -> Optional[str]:
"""
Resolve short link to get final URL.
Args:
url: Short URL to resolve
Returns:
Final URL after resolution, or None if not resolvable
"""
# Check if it's a known short link pattern
if self.shortlink_resolver.is_bilibili_short_link(url):
# Try to resolve using requests (follows redirects)
try:
import requests
response = requests.head(url, allow_redirects=True, timeout=10)
return response.url
except Exception:
# Fallback: return original URL
return url
# Check if it's Xiaohongshu short link
if self.XIAOHONGSHU_SHORT_PATTERN.match(url):
try:
import requests
response = requests.get(url, allow_redirects=True, timeout=10)
return response.url
except Exception:
return url
return url
def extract_bilibili_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Bilibili ID from URL.
Args:
url: Bilibili URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try series/playlist (must check before space pattern)
match = self.BILIBILI_SERIES_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "playlist",
"id": match.group(1)
}
# Try readlist (专栏合集)
match = self.BILIBILI_READLIST_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "readlist",
"id": match.group(1)
}
# Try favlist (收藏夹)
match = self.BILIBILI_FAVLIST_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "favlist",
"id": match.group(1)
}
# Try opus (动态)
match = self.BILIBILI_OPUS_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "opus",
"id": match.group(1)
}
# Try mobile opus (移动端动态)
match = self.BILIBILI_MOBILE_OPUS_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "opus",
"id": match.group(1)
}
# Try video
match = self.BILIBILI_VIDEO_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "video",
"id": match.group(1)
}
# Try article/read
match = self.BILIBILI_ARTICLE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "article",
"id": match.group(1)
}
# Try space (UP主) - must check after specific patterns
match = self.BILIBILI_SPACE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "up主",
"id": match.group(1)
}
# Try mobile space (移动端UP主空间)
match = self.BILIBILI_MOBILE_SPACE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "up主",
"id": match.group(1)
}
return None
def extract_youtube_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract YouTube ID from URL.
Args:
url: YouTube URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try video
match = self.YOUTUBE_VIDEO_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "video",
"id": match.group(1)
}
# Try channel
match = self.YOUTUBE_CHANNEL_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "channel",
"id": match.group(1)
}
# Try user
match = self.YOUTUBE_USER_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "user",
"id": match.group(1)
}
# Try playlist
match = self.YOUTUBE_PLAYLIST_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "playlist",
"id": match.group(1)
}
return None
def extract_zhihu_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Zhihu ID from URL.
Args:
url: Zhihu URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try question
match = self.ZHIHAU_QUESTION_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "question",
"id": match.group(1)
}
# Try answer
match = self.ZHIHAU_ANSWER_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "answer",
"id": match.group(1)
}
# Try user
match = self.ZHIHAU_USER_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "user",
"id": match.group(1)
}
# Try column
match = self.ZHIHAU_COLUMN_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "column",
"id": match.group(1)
}
# Try collection
match = self.ZHIHAU_COLLECTION_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "collection",
"id": match.group(1)
}
# Try zhuanlan article (专栏文章)
match = self.ZHIHAU_ZHUANLAN_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "zhuanlan",
"id": match.group(1)
}
# Try org page (机构页面)
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org",
"id": match.group(1)
}
# Try org answers (机构回答)
if self.ZHIHAU_ORG_ANSWERS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_answers",
"id": match.group(1)
}
# Try org zvideos (机构视频)
if self.ZHIHAU_ORG_VIDEOS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_zvideos",
"id": match.group(1)
}
# Try org asks (机构提问)
if self.ZHIHAU_ORG_ASKS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_asks",
"id": match.group(1)
}
# Try org posts (机构文章)
if self.ZHIHAU_ORG_POSTS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_posts",
"id": match.group(1)
}
# Try org pins (机构想法)
if self.ZHIHAU_ORG_PINS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_pins",
"id": match.group(1)
}
# Try zvideo (知乎视频)
match = self.ZHIHAU_ZVIDEO_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "zvideo",
"id": match.group(1)
}
return None
def extract_xiaohongshu_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Xiaohongshu ID from URL.
Args:
url: Xiaohongshu URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try short link
match = self.XIAOHONGSHU_SHORT_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try explore link
match = self.XIAOHONGSHU_NOTE_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try notes link
match = self.XIAOHONGSHU_NOTE_PATTERN2.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try discovery item link (resolved from short link)
match = self.XIAOHONGSHU_DISCOVERY_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try user profile
match = self.XIAOHONGSHU_USER_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "user",
"id": match.group(1)
}
return None
def parse(self, url: str) -> Dict[str, Any]:
"""
Parse URL and extract platform-specific ID.
Args:
url: URL to parse
Returns:
Dict with platform, type, id, and original_url fields
"""
result = {
"platform": None,
"type": None,
"id": None,
"original_url": url,
"resolved_url": None,
"success": False,
"error": None
}
# First, resolve short links
resolved = self.resolve_shortlink(url)
if resolved:
result["resolved_url"] = resolved
url = resolved
# Try each platform
bilibili_result = self.extract_bilibili_id(url)
if bilibili_result:
result.update(bilibili_result)
result["success"] = True
return result
youtube_result = self.extract_youtube_id(url)
if youtube_result:
result.update(youtube_result)
result["success"] = True
return result
zhihu_result = self.extract_zhihu_id(url)
if zhihu_result:
result.update(zhihu_result)
result["success"] = True
return result
# Try Xiaohongshu
xiaohongshu_result = self.extract_xiaohongshu_id(url)
if xiaohongshu_result:
result.update(xiaohongshu_result)
result["success"] = True
return result
# Check for short links that need resolution
short_links = self.shortlink_resolver.detect_short_links(url)
if short_links:
result["error"] = "Short link detected. Please provide the full URL."
return result
result["error"] = "Unsupported URL format"
return result
def format_output(data: Dict[str, Any], output_format: str = "json") -> str:
"""Format output based on specified format."""
if output_format == "json":
return json.dumps(data, ensure_ascii=False, indent=2)
elif output_format == "compact":
return json.dumps(data, ensure_ascii=False, separators=(',', ':'))
elif output_format == "text":
lines = []
for key, value in data.items():
if value is not None:
lines.append(f"{key}: {value}")
return "\n".join(lines)
else:
return json.dumps(data, ensure_ascii=False, indent=2)
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(
description="Parse and extract IDs from various platform URLs",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s --url "https://www.bilibili.com/video/BV1xx411c7mD"
%(prog)s --url "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
%(prog)s --url "https://www.zhihu.com/question/266309957"
%(prog)s --batch urls.txt --output results.json
"""
)
parser.add_argument(
"--url", "-u",
help="Single URL to parse"
)
parser.add_argument(
"--batch", "-b",
help="File containing URLs to parse (one per line)"
)
parser.add_argument(
"--output", "-o",
help="Output file path (for batch mode)"
)
parser.add_argument(
"--format", "-f",
choices=["json", "compact", "text"],
default="json",
help="Output format (default: json)"
)
args = parser.parse_args()
# Check if we're in shortlink help mode
if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']:
parser.print_help()
return 0
if not args.url and not args.batch:
parser.print_help()
print("\nError: Please provide --url or --batch")
return 1
link_parser = LinkParser()
if args.url:
# Single URL mode
parsed = link_parser.parse(args.url)
# 统一输出格式
if parsed["success"]:
output = {
"success": True,
"platform": parsed.get("platform"),
"type": parsed.get("type"),
"id": parsed.get("id"),
"filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None,
"message": "URL解析成功"
}
else:
output = {
"success": False,
"error": parsed.get("error", "解析失败"),
"detail": f"URL: {args.url}"
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return 0 if output["success"] else 1
elif args.batch:
try:
with open(args.batch, 'r', encoding='utf-8') as f:
urls = [line.strip() for line in f if line.strip()]
except Exception as e:
print(json.dumps({
"success": False,
"error": "读取文件失败",
"detail": str(e)
}, ensure_ascii=False, indent=2))
return 1
results = []
for url in urls:
parsed = link_parser.parse(url)
if parsed["success"]:
results.append({
"success": True,
"platform": parsed.get("platform"),
"type": parsed.get("type"),
"id": parsed.get("id"),
"filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None
})
else:
results.append({
"success": False,
"error": parsed.get("error", "解析失败"),
"detail": f"URL: {url}"
})
if args.output:
with open(args.output, 'w', encoding='utf-8') as f:
json.dump(results, f, ensure_ascii=False, indent=2)
print(json.dumps({
"success": True,
"message": f"结果已保存到 {args.output}",
"total": len(results)
}, ensure_ascii=False, indent=2))
else:
for result in results:
print(json.dumps(result, ensure_ascii=False, indent=2))
print("-" * 40)
return 0
if __name__ == '__main__':
sys.exit(main())
kb-link-parser/link_parser_shortlink.py
+296
View File
@@ -0,0 +1,296 @@
#!/usr/bin/env python3
"""
Short link resolver for link-parser skill.
Provides utilities for detecting and resolving short links, particularly
Bilibili short links (b23.tv) which may be blocked by anti-scraping measures.
Supported short link services:
- Bilibili: b23.tv
- Generic URL shorteners (detection only)
"""
import re
from typing import Optional, Dict, List, Tuple
from urllib.parse import urlparse
class ShortLinkResolver:
"""Short link detection and resolution handler"""
# Pattern for Bilibili short links
BILIBILI_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
# Pattern for other short link services
OTHER_SHORT_PATTERNS = {
'bit.ly': re.compile(r'https?://bit\.ly/\w+'),
't.cn': re.compile(r'https?://t\.cn/\w+'),
'dwz.cn': re.compile(r'https?://dwz\.cn/\w+'),
'goo.gl': re.compile(r'https?://goo\.gl/\w+'),
'tinyurl.com': re.compile(r'https?://tinyurl\.com/\w+'),
}
@staticmethod
def detect_short_links(text: str) -> List[Dict[str, str]]:
"""
Detect short links in text.
Args:
text: Input text to search
Returns:
List of detected short links with metadata:
[
{
'url': 'https://b23.tv/xxxxx',
'platform': 'bilibili',
'is_short': True
},
...
]
"""
detected = []
# Check for Bilibili short links
for match in ShortLinkResolver.BILIBILI_SHORT_PATTERN.finditer(text):
detected.append({
'url': match.group(0),
'code': match.group(1),
'platform': 'bilibili',
'is_short': True
})
# Check for other short link services
for platform, pattern in ShortLinkResolver.OTHER_SHORT_PATTERNS.items():
for match in pattern.finditer(text):
detected.append({
'url': match.group(0),
'platform': platform,
'is_short': True
})
return detected
@staticmethod
def is_bilibili_short_link(url: str) -> bool:
"""
Check if URL is a Bilibili short link.
Args:
url: URL to check
Returns:
True if URL is a b23.tv short link
"""
return bool(ShortLinkResolver.BILIBILI_SHORT_PATTERN.match(url))
@staticmethod
def format_short_link_help(short_link: Optional[str] = None) -> str:
"""
Format help message for short links.
Args:
short_link: Optional short link URL to mention
Returns:
Help message
"""
lines = [
"Detection: Short link",
""
]
if short_link:
lines.append(f"Short link: {short_link}")
lines.append("")
lines.extend([
"Due to anti-scraping protection, short links cannot be automatically resolved.",
"Please provide the full URL or the content ID (e.g., BV号 for Bilibili).",
"",
"Full URL format: https://www.bilibili.com/video/BVxxxxx"
])
return "\n".join(lines)
@staticmethod
def format_short_link_warning(short_links: List[Dict[str, str]]) -> str:
"""
Format warning message for multiple short links.
Args:
short_links: List of short link metadata
Returns:
Warning message
"""
if not short_links:
return ""
lines = [
"Short links detected:",
""
]
for i, link in enumerate(short_links, 1):
platform = link.get('platform', 'unknown')
url = link['url']
if platform == 'bilibili':
lines.append(f" [{i}] Bilibili short link: {url}")
lines.append(" Full URL or BV ID required")
else:
lines.append(f" [{i}] {platform}: {url}")
lines.append("")
return "\n".join(lines)
class PlaywrightResolver:
"""
Browser-based short link resolver using Playwright.
This is an optional implementation for environments where Playwright
is available. It can bypass anti-scraping measures by simulating
a real browser.
Requirements:
pip install playwright
playwright install chromium
"""
def __init__(self, headless: bool = True, timeout: int = 10000):
"""
Initialize Playwright resolver.
Args:
headless: Whether to run browser in headless mode
timeout: Page navigation timeout in milliseconds
"""
self.headless = headless
self.timeout = timeout
self._playwright = None
def _check_playwright_available(self) -> bool:
"""Check if Playwright is installed and available"""
try:
from playwright.sync_api import sync_playwright
return True
except ImportError:
return False
def resolve(self, short_url: str) -> Optional[str]:
"""
Resolve short link using Playwright browser automation.
Args:
short_url: Short URL to resolve
Returns:
Final URL after redirects, or None if resolution fails
"""
if not self._check_playwright_available():
raise ImportError(
"Playwright not installed. Install with: pip install playwright && playwright install chromium"
)
from playwright.sync_api import sync_playwright
try:
with sync_playwright() as p:
# Launch browser
browser = p.chromium.launch(headless=self.headless)
context = browser.new_context(
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
viewport={'width': 1920, 'height': 1080}
)
page = context.new_page()
# Navigate to short URL
page.goto(short_url, timeout=self.timeout, wait_until='networkidle')
# Get final URL
final_url = page.url
# Close browser
browser.close()
return final_url
except Exception as e:
print(f"Playwright resolution failed: {e}")
return None
def resolve_bilibili_short(self, short_url: str) -> Optional[str]:
"""
Resolve Bilibili short link and extract BV ID.
Args:
short_url: Bilibili short URL (e.g., https://b23.tv/xxxxx)
Returns:
Full Bilibili URL with BV ID, or None if resolution fails
"""
final_url = self.resolve(short_url)
if not final_url:
return None
# Check if final URL contains BV ID
if 'bilibili.com/video/' in final_url:
return final_url
# Extract BV ID if present in URL
bv_pattern = re.compile(r'(BV[a-zA-Z0-9]{10})')
match = bv_pattern.search(final_url)
if match:
return f"https://www.bilibili.com/video/{match.group(1)}"
return None
def get_resolver() -> ShortLinkResolver:
"""
Get the default resolver instance.
Returns:
ShortLinkResolver instance
"""
return ShortLinkResolver()
def detect_short_links(text: str) -> List[Dict[str, str]]:
"""
Detect short links in text.
Args:
text: Input text to search
Returns:
List of detected short links with metadata
"""
resolver = ShortLinkResolver()
return resolver.detect_short_links(text)
# CLI interface for testing
def main():
"""CLI for testing the short link resolver."""
import sys
if len(sys.argv) < 2:
print("Usage: python3 link_parser_shortlink.py 'text to test'")
print("\nExamples:")
print(' python3 link_parser_shortlink.py "Check this: https://b23.tv/oLBZZfK"')
print(' python3 link_parser_shortlink.py "https://bit.ly/xxxxx"')
sys.exit(1)
text = ' '.join(sys.argv[1:])
short_links = detect_short_links(text)
import json
print(json.dumps(short_links, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()
kb-ocr-processor/SKILL.md
+66
View File
@@ -0,0 +1,66 @@
# KB OCR Processor Skill
**kb-ocr-processor** - Extract text from video frames using OCR.
## Features
- **视频截帧**: 直接从视频文件截取帧图片
- **批量处理**: 支持处理帧目录
- **双引擎**: EasyOCR / Tesseract
- **置信度评分**: 每帧识别结果带置信度
- **文本合并**: 自动合并所有帧的识别结果
## Commands
```bash
# 方式1: 直接从视频截帧并OCR（推荐）
python ocr_processor.py --video ./video.flv --output ocr.json
# 方式2: 处理已截取的帧目录
python ocr_processor.py --input ./frames/ --output ocr.json
# 指定截帧帧率（默认1帧/秒）
python ocr_processor.py --video ./video.flv --fps 2 --output ocr.json
# 同时生成合并的文本文件
python ocr_processor.py --video ./video.flv --output ocr.json --merge
# 使用 Tesseract 引擎
python ocr_processor.py --input ./frames/ --engine tesseract --output ocr.json
```
## Workflow 中的使用
```bash
# 步骤5: OCR处理（视频无字幕时）
python ocr_processor.py --video ${CONTENT_VIDEO_PATH} --output ${CONTENT_DIR}/ocr.json --merge
```
## Output
```json
{
"success": true,
"engine": "easyocr",
"total_frames": 619,
"results": [
{
"source": "0001.jpg",
"text": "字幕文字",
"confidence": 0.95
}
]
}
```
## Installation
```bash
cp -r kb-ocr-processor/ ~/skills/
pip install -r requirements.txt
# 依赖说明
# - easyocr: 主要OCR引擎（自动下载模型）
# - pytesseract: Tesseract引擎
# - ffmpeg: 视频截帧（apt install ffmpeg）
```
kb-ocr-processor/ocr_processor.py
+360
View File
@@ -0,0 +1,360 @@
#!/usr/bin/env python3
"""
OCR Processor - 视频帧文字识别工具
支持 EasyOCR 和 Tesseract 引擎
"""
import argparse
import json
import os
import glob
from pathlib import Path
from typing import List, Dict, Optional
import easyocr
import pytesseract
from PIL import Image
class OCRProcessor:
"""OCR 文字识别处理器"""
def __init__(self, engine: str = "easyocr", languages: List[str] = ["ch_sim", "en"]):
"""
初始化 OCR 处理器
Args:
engine: OCR 引擎 ("easyocr" 或 "tesseract")
languages: 语言列表
"""
self.engine = engine
self.languages = languages
self.reader = None
if engine == "easyocr":
self.reader = easyocr.Reader(languages, gpu=False)
def process_image(self, image_path: str) -> Dict:
"""
处理单张图片进行 OCR 识别
Args:
image_path: 图片路径
Returns:
包含 text 和 confidence 的字典
"""
if not os.path.exists(image_path):
return {"text": "", "confidence": 0.0, "error": "File not found"}
try:
if self.engine == "easyocr":
return self._process_with_easyocr(image_path)
elif self.engine == "tesseract":
return self._process_with_tesseract(image_path)
else:
return {"text": "", "confidence": 0.0, "error": f"Unknown engine: {self.engine}"}
except Exception as e:
return {"text": "", "confidence": 0.0, "error": str(e)}
def _process_with_easyocr(self, image_path: str) -> Dict:
"""使用 EasyOCR 处理图片"""
results = self.reader.readtext(image_path)
if not results:
return {"text": "", "confidence": 0.0}
texts = []
confidences = []
for bbox, text, conf in results:
texts.append(text)
confidences.append(conf)
merged_text = "\n".join(texts)
avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0
return {
"text": merged_text,
"confidence": round(avg_confidence, 4),
"details": [
{"text": text, "confidence": round(conf, 4)}
for text, conf in zip(texts, confidences)
]
}
def _process_with_tesseract(self, image_path: str) -> Dict:
"""使用 Tesseract 处理图片"""
image = Image.open(image_path)
text = pytesseract.image_to_string(image, lang="+".join(self.languages))
# Tesseract 不直接返回置信度，返回 0.0
return {
"text": text.strip(),
"confidence": 0.0
}
def process_frames(self, input_dir: str, fps: float = 0.1, progress_file: Optional[str] = None) -> List[Dict]:
"""
处理目录下所有帧图片
Args:
input_dir: 帧图片目录
fps: 采样帧率
progress_file: 可选，进度文件路径（会实时更新）
Returns:
OCR 结果列表
"""
if not os.path.isdir(input_dir):
print(f"Error: Directory not found: {input_dir}")
return []
# 支持的图片格式
image_extensions = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.webp"]
image_files = []
for ext in image_extensions:
image_files.extend(glob.glob(os.path.join(input_dir, ext)))
image_files.extend(glob.glob(os.path.join(input_dir, "**", ext), recursive=True))
# 按文件名排序
image_files = sorted(image_files)
if not image_files:
print(f"No images found in {input_dir}")
return []
total = len(image_files)
print(f"Found {total} images in {input_dir}")
print(f"Processing with {self.engine} engine...")
def update_progress(current, status="running", last_result=None):
"""更新进度文件"""
if progress_file:
import json
import time
prog = {
"current": current,
"total": total,
"percent": round(current / total * 100, 1) if total > 0 else 0,
"status": status,
"last_result": last_result,
"updated_at": time.strftime("%Y-%m-%d %H:%M:%S")
}
with open(progress_file, 'w', encoding='utf-8') as f:
json.dump(prog, f, ensure_ascii=False, indent=2)
# 初始化进度
update_progress(0, "started")
results = []
for i, image_path in enumerate(image_files, 1):
basename = os.path.basename(image_path)
print(f"Processing [{i}/{total}]: {basename}")
ocr_result = self.process_image(image_path)
ocr_result["source"] = basename
# 记录结果摘要
text_count = len(ocr_result.get("text", "").split()) if ocr_result.get("text") else 0
last_result = f"✓ {text_count} 处文字" if ocr_result.get("text") else "✗ 无文字"
results.append(ocr_result)
# 实时更新进度
update_progress(i, "running", last_result)
# 完成
update_progress(total, "completed", f"完成 {total} 张")
print(f"\n✅ Processed {total} frames")
return results
def merge_texts(self, results: List[Dict]) -> str:
"""
合并多个 OCR 结果的文本
Args:
results: OCR 结果列表
Returns:
合并后的文本
"""
texts = []
for result in results:
if result.get("text"):
texts.append(result["text"])
return "\n\n---\n\n".join(texts)
def save_results(self, results: List[Dict], output_path: str, merge: bool = False):
"""
保存 OCR 结果到 JSON 文件
Args:
results: OCR 结果列表
output_path: 输出文件路径
merge: 是否同时保存合并后的文本文件
"""
output_data = {
"engine": self.engine,
"total_frames": len(results),
"results": results
}
# 保存 JSON
with open(output_path, "w", encoding="utf-8") as f:
json.dump(output_data, f, ensure_ascii=False, indent=2)
print(f"Results saved to {output_path}")
# 如果需要，同时保存合并的文本
if merge:
text_path = output_path.replace(".json", ".txt")
merged_text = self.merge_texts(results)
with open(text_path, "w", encoding="utf-8") as f:
f.write(merged_text)
print(f"Merged text saved to {text_path}")
def extract_frames(video_path: str, output_dir: str, fps: float = 1.0) -> str:
"""
使用 ffmpeg 从视频中截取帧图片
Args:
video_path: 视频文件路径
output_dir: 输出目录
fps: 每秒截取帧数 (默认: 1)
Returns:
帧图片目录路径
"""
import subprocess
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# ffmpeg 命令: -i 输入 -vf fps=fps 输出格式
cmd = [
"ffmpeg", "-y",
"-i", video_path,
"-vf", f"fps={fps}",
"-q:v", "2", # 质量等级 (2=高质量)
str(output_dir / "%04d.jpg")
]
print(f"🔄 截取视频帧: {video_path}")
print(f" 输出目录: {output_dir}")
try:
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
# 统计截取的帧数
frame_count = len(list(output_dir.glob("*.jpg")))
print(f"✅ 截取完成: {frame_count} 帧")
return str(output_dir)
else:
print(f"❌ ffmpeg 错误: {result.stderr}")
return ""
except FileNotFoundError:
print("❌ 错误: 未找到 ffmpeg，请先安装: apt install ffmpeg")
return ""
def main():
"""命令行入口"""
parser = argparse.ArgumentParser(
description="OCR 视频帧文字识别工具",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
# 使用 EasyOCR 处理帧目录
python ocr_processor.py --input ./frames/ --output ./ocr.json
# 直接从视频截帧并 OCR
python ocr_processor.py --video ./video.flv --output ./ocr.json
# 使用 Tesseract 处理
python ocr_processor.py --input ./frames/ --output ./ocr.json --engine tesseract
# 指定截帧帧率
python ocr_processor.py --video ./video.flv --output ./ocr.json --fps 2
# 同时生成合并的文本文件
python ocr_processor.py --input ./frames/ --output ./ocr.json --merge
"""
)
# 输入源：帧目录 或 视频文件（二选一）
input_group = parser.add_mutually_exclusive_group(required=True)
input_group.add_argument("--input", "-i", help="输入帧图片目录")
input_group.add_argument("--video", "-v", help="输入视频文件（会自动截帧）")
parser.add_argument("--output", "-o", required=True, help="输出 JSON 文件路径")
parser.add_argument("--fps", "-f", type=float, default=1.0, help="截帧帧率 (默认: 1帧/秒)")
parser.add_argument("--engine", "-e", choices=["easyocr", "tesseract"],
default="easyocr", help="OCR 引擎 (默认: easyocr)")
parser.add_argument("--merge", action="store_true", help="同时保存合并的文本文件")
parser.add_argument("--languages", "-l", nargs="+",
default=["ch_sim", "en"], help="语言列表")
parser.add_argument("--progress", "-p", help="可选：进度文件路径（会实时更新）")
args = parser.parse_args()
# 如果指定了视频文件，先截帧
input_dir = args.input
if args.video:
if not os.path.exists(args.video):
print(json.dumps({
"success": False,
"error": f"视频文件不存在: {args.video}"
}, ensure_ascii=False, indent=2))
return 1
# 创建临时帧目录
frames_dir = Path(args.video).stem + "_frames"
input_dir = extract_frames(args.video, frames_dir, fps=args.fps)
if not input_dir:
print(json.dumps({
"success": False,
"error": "截帧失败"
}, ensure_ascii=False, indent=2))
return 1
# 初始化处理器
processor = OCRProcessor(engine=args.engine, languages=args.languages)
# 处理帧
results = processor.process_frames(input_dir, fps=args.fps, progress_file=args.progress)
if results:
# 保存结果
processor.save_results(results, args.output, merge=args.merge)
# 统一输出格式
total_texts = sum(1 for r in results if r.get("text"))
avg_conf = sum(r.get("confidence", 0) for r in results) / len(results) if results else 0
output = {
"success": True,
"filename": Path(args.output).name,
"filepath": str(Path(args.output).resolve()),
"message": f"OCR处理完成",
"stats": {
"total_frames": len(results),
"frames_with_text": total_texts,
"avg_confidence": round(avg_conf, 4)
}
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return 0
else:
print(json.dumps({
"success": False,
"error": "未处理到任何结果"
}, ensure_ascii=False, indent=2))
return 1
if __name__ == "__main__":
exit(main())
kb-ocr-processor/requirements.txt
+3
View File
@@ -0,0 +1,3 @@
easyocr>=1.7.0
pytesseract>=0.3.10
Pillow>=9.0.0
kb-whisper-diarization/.env.example
+9
View File
@@ -0,0 +1,9 @@
# KB Whisper Diarization 环境变量
# Whisper.cpp 安装路径
WHISPER_CPP_PATH=~/whisper.cpp
# HuggingFace Token（必需，用于下载 pyannote 模型）
# 获取方式: https://huggingface.co/settings/tokens
# 注意: 需要先申请 pyannote/speaker-diarization 模型访问权限
HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
kb-whisper-diarization/SKILL.md
+127
View File
@@ -0,0 +1,127 @@
# KB Whisper Diarization Skill
**kb-whisper-diarization** - 使用 Whisper + pyannote.audio 实现语音识别和说话者分离。
## 什么时候使用
当用户需要：
- 区分多个说话者（如访谈、对话）
- 获取带说话者标签的字幕
- 分析对话中谁说了什么
## 环境变量
在使用之前，必须检查环境变量是否已经设置。
| 变量名 | 用途 | 示例 |
|--------|------|------|
| `WHISPER_CPP_PATH` | whisper.cpp 安装路径 | `~/whisper.cpp` |
| `HF_TOKEN` | HuggingFace Token | `hf_xxx...` |
### 获取 HuggingFace Token
pyannote.audio 需要从 HuggingFace 下载模型，需要先：
1. 注册 HuggingFace 账号：https://huggingface.co
2. 申请模型访问权限：https://huggingface.co/pyannote/speaker-diarization
3. 创建 Token：https://huggingface.co/settings/tokens
## 使用方法
```bash
# 基本用法（带说话者分离）
python whisper_diarization.py --input video.mp4
# 说话者分离
python whisper_diarization.py --input interview.mp4 --diarize
# 指定说话者数量
python whisper_diarization.py --input meeting.mp4 --diarize --num-speakers 2
# 指定语言
python whisper_diarization.py --input video.mp4 --language zh
```
## 命令参数
| 参数 | 说明 | 默认值 |
|------|------|--------|
| `--input` | 输入音视频文件路径 | 必需 |
| `--output` | 输出目录 | 文件所在目录 |
| `--diarize` | 启用说话者分离 | False |
| `--num-speakers` | 估计说话者数量 | None（自动检测） |
| `--language` | 语言 | 自动检测 |
| `--model` | Whisper 模型 | base |
| `--background` | 后台异步执行 | False |
## 模型选项
- **Whisper 模型**: tiny, base, small, medium, large
- **说话者分离**: pyannote/speaker-diarization (需申请权限)
## 输出文件
对于输入 `interview.mp4`，将生成：
- `interview.mp4.json` - 完整结果（包含说话者标签）
- `interview.mp4.txt` - 纯文本
- `interview.mp4.srt` - 字幕格式（带说话者标签）
## 输出格式示例
### JSON 格式
```json
{
"success": true,
"input_file": "interview.mp4",
"diarization": true,
"speakers": ["SPEAKER_00", "SPEAKER_01"],
"segments": [
{
"id": 0,
"speaker": "SPEAKER_00",
"start": 0.0,
"end": 5.2,
"text": "大家好，欢迎来到访谈节目"
},
{
"id": 1,
"speaker": "SPEAKER_01",
"start": 5.2,
"end": 10.8,
"text": "谢谢邀请，我很荣幸"
}
]
}
```
### SRT 字幕格式
```
1
00:00:00,000 --> 00:00:05,200
[SPEAKER_00] 大家好，欢迎来到访谈节目
2
00:00:05,200 --> 00:00:10,800
[SPEAKER_01] 谢谢邀请，我很荣幸
```
## 安装
```bash
# 1. 安装依赖
pip install pyannote.audio torch
# 2. 设置 HuggingFace Token
export HF_TOKEN="hf_xxx..."
# 3. 首次运行会自动下载模型
python whisper_diarization.py --input test.mp4
```
## 注意事项
- **说话者分离需要申请权限**: https://huggingface.co/pyannote/speaker-diarization
- **处理时间较长**: 说话者分离会增加处理时间
- **对立体声效果更好**: 多声道音频的分离效果更佳
kb-whisper-diarization/requirements.txt
+8
View File
@@ -0,0 +1,8 @@
# KB Whisper Diarization 依赖
# 核心依赖
torch>=2.0.0
pyannote.audio>=1.0.0
# 可选: GPU 加速
# torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
kb-whisper-diarization/whisper_diarization.py
+339
View File
@@ -0,0 +1,339 @@
#!/usr/bin/env python3
"""
KB Whisper Diarization - 语音识别 + 说话者分离
使用 Whisper 进行语音识别，pyannote.audio 进行说话者分离
"""
import argparse
import json
import os
import subprocess
import sys
import tempfile
from datetime import datetime
from pathlib import Path
import re
def parse_arguments():
"""解析命令行参数"""
parser = argparse.ArgumentParser(
description='KB Whisper Diarization - 语音识别 + 说话者分离',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
python whisper_diarization.py --input video.mp4
python whisper_diarization.py --input interview.mp4 --diarize
python whisper_diarization.py --input meeting.mp4 --diarize --num-speakers 2
"""
)
parser.add_argument('--input', required=True, help='输入音视频文件路径')
parser.add_argument('--output', help='输出目录 (默认: 文件所在目录)')
parser.add_argument('--diarize', action='store_true', help='启用说话者分离')
parser.add_argument('--num-speakers', type=int, help='估计说话者数量')
parser.add_argument('--language', help='指定语言 (默认: 自动检测)')
parser.add_argument('--model', default='base', choices=['tiny', 'base', 'small', 'medium', 'large'],
help='Whisper 模型 (默认: base)')
parser.add_argument('--background', action='store_true', help='后台异步执行')
return parser.parse_args()
def get_whisper_path():
"""获取 whisper.cpp 路径"""
path = os.environ.get('WHISPER_CPP_PATH')
if path:
path = os.path.expanduser(path)
if os.path.exists(path):
return path
return os.path.expanduser('~/whisper.cpp')
def check_whisper_model(whisper_path, model):
"""检查 Whisper 模型"""
model_path = os.path.join(whisper_path, 'models', f'ggml-{model}.bin')
if not os.path.exists(model_path):
raise FileNotFoundError(f"Whisper 模型不存在: {model_path}")
return model_path
def convert_to_wav(input_path, output_dir):
"""转换为 16kHz WAV"""
if input_path.lower().endswith('.wav'):
try:
result = subprocess.run(
['ffprobe', '-v', 'error', '-show_entries', 'stream=sample_rate',
'-of', 'default=noprint_wrappers=1:nokey=1', input_path],
capture_output=True, text=True
)
if result.returncode == 0 and int(result.stdout.strip()) == 16000:
return input_path
except Exception:
pass
output_path = os.path.join(output_dir, f"{Path(input_path).stem}_audio.wav")
print(f"转换音频: {input_path} -> {output_path}")
subprocess.run([
'ffmpeg', '-y', '-i', input_path,
'-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',
'-loglevel', 'error', output_path
], check=True, capture_output=True)
print("✓ 音频转换完成")
return output_path
def run_whisper(whisper_path, audio_path, model, output_dir, language=None):
"""运行 Whisper 识别"""
basename = Path(audio_path).stem.replace('_audio', '')
output_file = os.path.join(output_dir, basename)
cmd = [
os.path.join(whisper_path, 'whisper-cli'),
'-m', os.path.join(whisper_path, 'models', f'ggml-{model}.bin'),
'-f', audio_path,
'-of', output_file,
'-l', language or 'auto',
'--output-txt', '--output-srt', '--output-json',
]
print(f"执行 Whisper: {' '.join(cmd[:6])} ...")
result = subprocess.run(
cmd, capture_output=True, text=True, cwd=whisper_path, timeout=3600
)
if result.returncode != 0 and not os.path.exists(f"{output_file}.txt"):
raise RuntimeError(f"Whisper 失败: {result.stderr}")
return basename
def run_diarization(audio_path, num_speakers=None):
"""运行 pyannote.audio 说话者分离"""
print("\n🎙️ 步骤2: 说话者分离...")
try:
import torch
from pyannote.audio import Pipeline
except ImportError as e:
print(f"✗ 缺少依赖: {e}")
print("请安装: pip install pyannote.audio torch")
raise
# 检查 HuggingFace Token
hf_token = os.environ.get('HF_TOKEN')
if not hf_token:
raise RuntimeError("需要设置 HF_TOKEN 环境变量\n"
"获取方式: https://huggingface.co/settings/tokens")
# 新版 pyannote 需要先登录
from huggingface_hub import login
print("登录 HuggingFace...")
login(token=hf_token)
print("加载 pyannote 流水线...")
pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization")
# 设置设备
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"使用设备: {device}")
pipeline.to(device)
# 运行分离
print("执行说话者分离...")
diarization = pipeline(
audio_path,
num_speakers=num_speakers
)
# 解析结果
speakers = {}
for turn, _, speaker in diarization.itertracks(yield_label=True):
if speaker not in speakers:
speakers[speaker] = []
speakers[speaker].append({
"start": round(turn.start, 2),
"end": round(turn.end, 2)
})
print(f"✓ 检测到 {len(speakers)} 个说话者: {list(speakers.keys())}")
return speakers
def merge_results(whisper_txt, whisper_srt, diarization, output_dir, basename):
"""合并 Whisper 和说话者分离结果"""
print("\n📝 步骤3: 合并结果...")
# 读取 Whisper 文本
text_segments = []
if os.path.exists(whisper_txt):
with open(whisper_txt, 'r', encoding='utf-8') as f:
content = f.read()
# 简单按行分段
for i, line in enumerate(content.strip().split('\n')):
if line.strip():
text_segments.append({
"id": i,
"text": line.strip(),
"start": i * 3.0, # 估算
"end": (i + 1) * 3.0
})
# 读取 SRT 获取精确时间
srt_segments = []
if os.path.exists(whisper_srt):
with open(whisper_srt, 'r', encoding='utf-8') as f:
content = f.read()
# 解析 SRT
pattern = r'(\d+)\n(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\n(.+?)(?=\n\n|\Z)'
matches = re.findall(pattern, content, re.DOTALL)
def parse_time(t):
h, m, s_ms = t.split(':')
s, ms = s_ms.split(',')
return float(h) * 3600 + float(m) * 60 + float(s) + float(ms) / 1000
for idx, num, start, end, text in matches:
srt_segments.append({
"id": int(idx),
"start": parse_time(start),
"end": parse_time(end),
"text": text.strip()
})
# 为每个 Whisper 段落分配说话者
merged_segments = []
for seg in srt_segments:
speaker = assign_speaker(seg['start'], seg['end'], diarization)
merged_segments.append({
"id": len(merged_segments),
"speaker": speaker,
"start": seg['start'],
"end": seg['end'],
"text": seg['text']
})
# 生成带说话者的 SRT
output_srt = os.path.join(output_dir, f"{basename}_diarized.srt")
with open(output_srt, 'w', encoding='utf-8') as f:
for seg in merged_segments:
start = format_time(seg['start'])
end = format_time(seg['end'])
f.write(f"{seg['id'] + 1}\n")
f.write(f"{start} --> {end}\n")
f.write(f"[{seg['speaker']}] {seg['text']}\n\n")
# 生成合并的 TXT
output_txt = os.path.join(output_dir, f"{basename}_diarized.txt")
with open(output_txt, 'w', encoding='utf-8') as f:
for seg in merged_segments:
f.write(f"[{seg['speaker']}] {seg['text']}\n")
# 生成 JSON
output_json = os.path.join(output_dir, f"{basename}_diarized.json")
json_data = {
"success": True,
"tool": "whisper-diarization",
"diarization": True,
"speakers": list(diarization.keys()),
"input_file": basename,
"output_files": {
"txt": f"{basename}_diarized.txt",
"srt": f"{basename}_diarized.srt",
"json": f"{basename}_diarized.json"
},
"segments": merged_segments
}
with open(output_json, 'w', encoding='utf-8') as f:
json.dump(json_data, f, ensure_ascii=False, indent=2)
print(f"✓ 生成文件:")
print(f" - {output_srt}")
print(f" - {output_txt}")
print(f" - {output_json}")
return json_data
def assign_speaker(start, end, diarization):
"""将时间段分配给说话者"""
best_speaker = "UNKNOWN"
max_overlap = 0
for speaker, segments in diarization.items():
for seg in segments:
overlap = min(end, seg['end']) - max(start, seg['start'])
if overlap > max_overlap:
max_overlap = overlap
best_speaker = speaker
return best_speaker
def format_time(seconds):
"""秒数转换为 SRT 时间格式"""
hours = int(seconds // 3600)
minutes = int((seconds % 3600) // 60)
secs = int(seconds % 60)
ms = int((seconds % 1) * 1000)
return f"{hours:02d}:{minutes:02d}:{secs:02d},{ms:03d}"
def main():
args = parse_arguments()
input_file = os.path.abspath(args.input)
if not os.path.exists(input_file):
print(f"✗ 文件不存在: {input_file}")
sys.exit(1)
output_dir = os.path.abspath(args.output) if args.output else os.path.dirname(input_file)
os.makedirs(output_dir, exist_ok=True)
basename = Path(input_file).stem
print(f"\n📁 输入: {input_file}")
print(f"📁 输出: {output_dir}")
print(f"🎙️ 说话者分离: {'是' if args.diarize else '否'}")
try:
# 步骤1: Whisper 识别
print(f"\n🎙️ 步骤1: 语音识别...")
whisper_path = get_whisper_path()
check_whisper_model(whisper_path, args.model)
audio_path = convert_to_wav(input_file, output_dir)
run_whisper(whisper_path, audio_path, args.model, output_dir, args.language)
# 步骤2: 说话者分离
if args.diarize:
diarization = run_diarization(audio_path, args.num_speakers)
else:
diarization = {}
# 步骤3: 合并结果
whisper_txt = os.path.join(output_dir, f"{basename}.txt")
whisper_srt = os.path.join(output_dir, f"{basename}.srt")
result = merge_results(whisper_txt, whisper_srt, diarization, output_dir, basename)
# 清理临时音频
if audio_path != input_path and audio_path.endswith('_audio.wav'):
try:
os.remove(audio_path)
except Exception:
pass
print(f"\n✅ 处理完成!")
print(f" 说话者数量: {len(diarization) if args.diarize else 1}")
print(f" 段落数量: {len(result['segments'])}")
except Exception as e:
print(f"\n✗ 错误: {e}")
sys.exit(1)
if __name__ == '__main__':
main()
kb-whisper/.env.example
+5
View File
@@ -0,0 +1,5 @@
# KB Whisper 环境变量配置示例
# 复制此文件为 .env 并填写实际值
# Whisper.cpp 安装路径
WHISPER_CPP_PATH=~/whisper.cpp
kb-whisper/README.md
+211
View File
@@ -0,0 +1,211 @@
# KB Whisper - 本地语音识别
基于 Whisper.cpp 的本地语音识别工具，支持将音视频文件转换为文本、字幕和 JSON 格式。
## 功能特性
- 🎯 **本地执行** - 完全离线运行，保护隐私
- 🚀 **多模型支持** - 从 tiny 到 large，根据需求选择速度/精度
- 📝 **多格式输出** - JSON、SRT、TXT 三种格式
- ⚡ **异步执行** - 支持后台运行，不阻塞主代理
- 🎬 **广泛兼容** - 支持常见音视频格式（MP3、MP4、WAV 等）
## 安装前提
### 1. 安装 whisper.cpp
```bash
# 克隆仓库
git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp
# 编译
cd ~/whisper.cpp
make
# 下载模型（至少需要一个模型）
./models/download-ggml-model.sh tiny
./models/download-ggml-model.sh base # 可选
```
### 2. 配置环境变量
在 `~/.clawdbot/clawdbot.json` 中添加：
```json5
{
skills: {
entries: {
"kb-whisper": {
env: {
WHISPER_CPP_PATH: "~/whisper.cpp"
}
}
}
}
}
```
或通过命令行配置：
```bash
clawdbot skills config kb-whisper WHISPER_CPP_PATH "~/whisper.cpp"
```
## 使用方法
### 基本用法
```bash
# 使用默认 tiny 模型处理
python whisper.py --input video.mp4
# 使用 base 模型（推荐，平衡速度和精度）
python whisper.py --input video.mp4 --model base
# 后台异步执行
python whisper.py --input video.mp4 --background
```
### 指定输出目录
```bash
python whisper.py --input video.mp4 --output ./transcripts
```
### 处理音频文件
```bash
python whisper.py --input podcast.mp3 --model base
```
## 输出文件
对于输入 `video.mp4`，将生成以下文件：
```
output/
├── video.json # 详细结果（含时间戳）
├── video.srt # 字幕格式
└── video.txt # 纯文本
```
### JSON 格式
```json
{
"success": true,
"input_file": "video.mp4",
"model": "base",
"output_files": {
"json": "video.json",
"srt": "video.srt",
"txt": "video.txt"
},
"duration_seconds": 120.5,
"segments": [
{
"id": 0,
"start": 0.0,
"end": 3.2,
"text": "这是第一段字幕"
},
{
"id": 1,
"start": 3.2,
"end": 6.8,
"text": "这是第二段字幕"
}
]
}
```
## 模型对比
| 模型 | 速度 | 精度 | 适用场景 |
|------|------|------|----------|
| tiny | ⚡⚡⚡ | ⭐ | 快速预览、实时转录 |
| base | ⚡⚡ | ⭐⭐⭐ | 日常使用（推荐） |
| small | ⚡ | ⭐⭐⭐⭐ | 高质量需求 |
| medium | ⭐ | ⭐⭐⭐⭐⭐ | 专业用途 |
| large | 🐢 | ⭐⭐⭐⭐⭐ | 最高精度要求 |
## 完整示例
### 示例 1: 转录播客
```bash
python whisper.py --input podcast.mp3 --model base --background
```
### 示例 2: 生成视频字幕
```bash
python whisper.py --input tutorial.mp4 --model small
```
### 示例 3: 批量处理（配合脚本）
```bash
#!/bin/bash
for video in videos/*.mp4; do
python whisper.py --input "$video" --model base
done
```
## 在 Workflow 中使用
在知识库处理流程中，如果视频没有字幕：
```bash
# 步骤4: 下载视频后，如果没有字幕
# 步骤5: 使用 Whisper 生成字幕
python whisper.py --input ${video_path} --model base --background
# 等待 Whisper 完成，然后继续后续步骤
```
## 技术细节
- **引擎**: Whisper.cpp（C++ 实现，性能优异）
- **音频采样率**: 自动转换为 16kHz
- **语言**: 自动检测（支持 99 种语言）
- **GPU 支持**: whisper.cpp 支持 CUDA 加速（需编译时启用）
## 故障排查
### 问题: 找不到 whisper.cpp
```bash
# 检查环境变量
echo $WHISPER_CPP_PATH
# 手动测试
~/whisper.cpp/main --help
```
### 问题: 模型未下载
```bash
cd ~/whisper.cpp
./models/download-ggml-model.sh tiny
```
### 问题: 音频格式不支持
```bash
# 转换音频格式
ffmpeg -i input.avi -acodec libmp3lame output.mp3
python whisper.py --input output.mp3
```
## 性能参考
在 CPU 环境下（Intel i7）：
- **tiny 模型**: ~1x 实时速度（1分钟音频 ~1分钟处理）
- **base 模型**: ~0.5x 实时速度（1分钟音频 ~2分钟处理）
- **small 模型**: ~0.25x 实时速度（1分钟音频 ~4分钟处理）
## 许可证
本技能使用 MIT 许可证，Whisper.cpp 使用 MIT 许可证。
kb-whisper/SKILL.md
+124
View File
@@ -0,0 +1,124 @@
---
name: kb-whisper
description: 本地语音识别，支持音视频转文字（基于 Whisper.cpp）
---
# KB Whisper Skill
**kb-whisper** - 本地语音识别，使用 Whisper.cpp 将音视频文件转换为文本、字幕和 JSON 格式输出。
## 什么时候使用
当用户需要：
- 将视频/音频文件转换为文字
- 生成视频字幕（SRT 格式）
- 获取带时间戳的详细识别结果
## 环境变量
在使用之前，必须检查环境变量是否已经设置。
| 变量名 | 用途 | 示例 |
|--------|------|------|
| `WHISPER_CPP_PATH` | whisper.cpp 安装路径 | `~/whisper.cpp` |
## 使用方法
```bash
# 基本用法
python whisper.py --input video.mp4
# 指定模型（默认 tiny）
python whisper.py --input video.mp4 --model base
# 指定输出目录
python whisper.py --input video.mp4 --output ./output
# 异步执行（推荐）
python whisper.py --input video.mp4 --background
```
## 命令参数
| 参数 | 说明 | 默认值 |
|------|------|--------|
| `--input` | 输入音视频文件路径 | 必需 |
| `--model` | Whisper 模型类型 | `tiny` |
| `--output` | 输出目录 | 文件所在目录 |
| `--background` | 后台异步执行 | False |
## 模型选项
- `tiny` - 最小模型，速度快，精度较低（推荐快速识别）
- `base` - 基础模型，平衡速度和精度（推荐）
- `small` - 小型模型，精度更高，速度较慢
- `medium` - 中型模型，高精度
- `large` - 大型模型，最高精度，速度慢
## 输出文件
对于输入 `video.mp4`，将生成：
- `video.json` - 详细结果（包含时间戳、分段信息）
- `video.srt` - 字幕格式文件
- `video.txt` - 纯文本内容
## 输出格式示例
### JSON 格式
```json
{
"success": true,
"input_file": "video.mp4",
"model": "tiny",
"output_files": {
"json": "video.json",
"srt": "video.srt",
"txt": "video.txt"
},
"duration_seconds": 120.5,
"segments": [
{
"id": 0,
"start": 0.0,
"end": 3.2,
"text": "这是第一段字幕"
}
]
}
```
## Workflow 中的使用
```bash
# 步骤4之后：如果已有音频但无字幕，可以使用 whisper
python whisper.py --input ${video_path} --model base --background
```
## 安装
```bash
# 1. 克隆 whisper.cpp
git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp
# 2. 编译
cd ~/whisper.cpp
make
# 3. 下载模型（tiny）
./models/download-ggml-model.sh tiny
# 4. 下载更多模型（可选）
# ./models/download-ggml-model.sh base
# ./models/download-ggml-model.sh small
# 5. 复制技能到 skills 目录
cp -r kb-whisper/ ~/skills/
```
## 注意事项
- **异步执行**: 使用 `--background` 参数时，命令会在后台运行，不阻塞主代理
- **模型选择**: tiny 模型适合快速识别，base 模型适合一般用途，large 模型适合高精度需求
- **文件格式**: 支持 MP3、MP4、WAV、FLAC、OGG、M4A 等常见格式
kb-whisper/__pycache__/whisper.cpython-312.pyc
BIN
View File
Binary file not shown.
kb-whisper/examples.md
+273
View File
@@ -0,0 +1,273 @@
# KB Whisper 使用示例
## 快速开始
### 1. 基础使用
```bash
# 使用默认 tiny 模型转录视频
python whisper.py --input video.mp4
# 使用 base 模型（推荐）
python whisper.py --input video.mp4 --model base
```
### 2. 异步执行
```bash
# 后台处理，不阻塞主代理
python whisper.py --input video.mp4 --model base --background
```
### 3. 指定输出目录
```bash
python whisper.py --input video.mp4 --output ./transcripts
```
### 4. 指定语言
```bash
# 指定中文
python whisper.py --input video.mp4 --language zh
# 指定英文
python whisper.py --input video.mp4 --language en
# 指定日语
python whisper.py --input video.mp4 --language ja
```
## 实际应用场景
### 场景 1: 转录播客
```bash
python whisper.py --input podcast.mp3 --model base --background
```
**输出**:
- `podcast.json` - 带时间戳的详细结果
- `podcast.srt` - 字幕格式
- `podcast.txt` - 纯文本
### 场景 2: 生成视频字幕
```bash
python whisper.py --input tutorial.mp4 --model small
```
**用途**:
- 为教程视频生成字幕
- 导出 SRT 文件到视频编辑软件
- 生成多语言字幕版本
### 场景 3: 会议录音转文字
```bash
python whisper.py --input meeting.mp3 --model base --output ./meetings
```
**输出示例**:
```
meetings/
├── meeting.json # 完整 JSON（含时间戳）
├── meeting.srt # 字幕格式
└── meeting.txt # 纯文本
```
### 场景 4: 批量处理
创建脚本 `batch_transcribe.sh`:
```bash
#!/bin/bash
OUTPUT_DIR="./transcripts"
mkdir -p "$OUTPUT_DIR"
for video in videos/*.mp4; do
echo "处理: $video"
python whisper.py --input "$video" --output "$OUTPUT_DIR" --model base
done
echo "批量处理完成!"
```
```bash
chmod +x batch_transcribe.sh
./batch_transcribe.sh
```
## 在 Workflow 中使用
### 知识库处理流程
在 `kb-workflow` 技能中，如果视频没有字幕：
```bash
# 步骤 4: 下载视频后
# 检查是否有字幕文件
if [ ! -f "${subtitle_file}" ]; then
echo "无字幕文件，使用 Whisper 生成..."
# 使用 Whisper 生成字幕（后台执行）
python /path/to/whisper.py \
--input "${video_path}" \
--model base \
--output "${content_dir}" \
--background
# 等待 Whisper 完成
sleep 5 # 等待后台任务启动
while ! [ -f "${content_dir}/$(basename ${video_path} .mp4).srt" ]; do
sleep 2
done
echo "Whisper 处理完成"
fi
```
## 输出文件使用
### JSON 格式
```python
import json
# 读取 JSON 结果
with open('video.json', 'r', encoding='utf-8') as f:
data = json.load(f)
# 访问分段
for segment in data['segments']:
print(f"[{segment['start']:.2f}-{segment['end']:.2f}] {segment['text']}")
```
### SRT 格式
```bash
# 直接在视频播放器中使用
mpv video.mp4 --sub-file=video.srt
# 或转换字幕格式
ffmpeg -i video.srt video.ass
```
### TXT 格式
```bash
# 搜索关键词
grep "重要信息" video.txt
# 统计字数
wc -m video.txt
```
## 性能优化建议
### 1. 根据内容选择模型
```bash
# 快速预览（短视频、测试）
python whisper.py --input video.mp4 --model tiny
# 日常使用（播客、会议）
python whisper.py --input video.mp4 --model base
# 高质量需求（正式内容、多语言）
python whisper.py --input video.mp4 --model small
```
### 2. 使用后台执行
```bash
# 处理长文件时使用后台模式
python whisper.py --input long_video.mp4 --model base --background
```
### 3. 指定语言提升准确度
```bash
# 如果知道内容语言，指定后准确度更高
python whisper.py --input video.mp4 --language zh
```
## 故障排查
### 问题 1: whisper.cpp 未找到
```bash
# 检查环境变量
echo $WHISPER_CPP_PATH
# 或配置
clawdbot skills config kb-whisper WHISPER_CPP_PATH "~/whisper.cpp"
```
### 问题 2: 模型未下载
```bash
cd ~/whisper.cpp
./models/download-ggml-model.sh base
```
### 问题 3: 音频格式不支持
```bash
# 转换格式
ffmpeg -i input.avi -acodec libmp3lame output.mp3
python whisper.py --input output.mp3
```
### 问题 4: 后台任务未完成
```bash
# 查看日志
cat video_whisper.log
# 检查进程
ps aux | grep whisper
```
## 集成到其他工具
### 集成到 Python 脚本
```python
import subprocess
import json
def transcribe_audio(audio_path, model='base'):
"""调用 whisper.py 转录音频"""
cmd = [
'python3', '/path/to/whisper.py',
'--input', audio_path,
'--model', model
]
result = subprocess.run(cmd, capture_output=True, text=True)
return json.loads(result.stdout)
# 使用
result = transcribe_audio('podcast.mp3', 'base')
print(f"识别到 {result['total_segments']} 个片段")
```
### 集成到 Shell 脚本
```bash
#!/bin/bash
transcribe() {
local file=$1
local model=${2:-base}
echo "转录: $file"
python3 whisper.py --input "$file" --model "$model" --background
}
# 批量处理
transcribe "video1.mp4"
transcribe "video2.mp4" "tiny"
```
kb-whisper/whisper.py
+403
View File
@@ -0,0 +1,403 @@
#!/usr/bin/env python3
"""
KB Whisper - 本地语音识别工具
基于 Whisper.cpp 的音视频转文字
功能：
- 自动将视频/音频转换为 16kHz WAV 格式
- 支持指定语言（默认中文）
- 输出标准 JSON 格式供 KB Workflow 使用
"""
import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
import re
def parse_arguments():
"""解析命令行参数"""
parser = argparse.ArgumentParser(
description='KB Whisper - 本地语音识别工具',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
python whisper.py --input video.mp4
python whisper.py --input video.mp4 --model base
python whisper.py --input video.mp4 --output ./output --background
"""
)
parser.add_argument(
'--input',
required=True,
help='输入音视频文件路径'
)
parser.add_argument(
'--model',
default='base',
choices=['tiny', 'base', 'small', 'medium', 'large'],
help='Whisper 模型类型 (默认: base)'
)
parser.add_argument(
'--output',
help='输出目录 (默认: 输入文件所在目录)'
)
parser.add_argument(
'--background',
action='store_true',
help='后台异步执行'
)
parser.add_argument(
'--language',
default='zh',
help='指定语言 (默认: zh)，例如: en, zh, ja'
)
return parser.parse_args()
def get_whisper_cpp_path():
"""获取 whisper.cpp 路径"""
# 优先从环境变量获取
path = os.environ.get('WHISPER_CPP_PATH')
if path:
path = os.path.expanduser(path)
if os.path.exists(path):
return path
# 尝试常见位置
common_paths = [
os.path.expanduser('~/whisper.cpp'),
'/usr/local/bin/whisper',
'/opt/whisper.cpp',
]
for cp in common_paths:
if os.path.exists(cp):
return cp
raise FileNotFoundError(
"未找到 whisper.cpp。请设置环境变量 WHISPER_CPP_PATH "
"或安装到 ~/whisper.cpp\n"
"安装方法: git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp && cd ~/whisper.cpp && make"
)
def check_model(whisper_path, model):
"""检查模型是否存在"""
model_path = os.path.join(whisper_path, 'models', f'ggml-{model}.bin')
if not os.path.exists(model_path):
raise FileNotFoundError(
f"模型文件不存在: {model_path}\n"
f"请运行以下命令下载模型:\n"
f"cd {whisper_path} && ./models/download-ggml-model.sh {model}"
)
return model_path
def convert_to_wav(input_path, output_dir):
"""将音视频转换为 16kHz WAV 格式
Whisper 需要 WAV 格式的输入文件。
"""
input_path = os.path.abspath(input_path)
# 如果已经是 WAV 且采样率正确，直接返回
if input_path.lower().endswith('.wav'):
try:
result = subprocess.run(
['ffprobe', '-v', 'error', '-show_entries',
'stream=sample_rate', '-of', 'default=noprint_wrappers=1:nokey=1',
input_path],
capture_output=True, text=True
)
if result.returncode == 0 and int(result.stdout.strip()) == 16000:
print(f"输入文件已是 16kHz WAV 格式，直接使用")
return input_path
except Exception:
pass
# 转换
output_path = os.path.join(output_dir, f"{Path(input_path).stem}_audio.wav")
print(f"转换音频格式: {input_path} -> {output_path}")
try:
subprocess.run([
'ffmpeg', '-y', '-i', input_path,
'-acodec', 'pcm_s16le',
'-ar', '16000',
'-ac', '1',
'-loglevel', 'error',
output_path
], check=True, capture_output=True)
print("音频转换完成")
return output_path
except subprocess.CalledProcessError as e:
print(f"音频转换失败: {e}")
raise RuntimeError(f"无法转换音频格式: {e}")
except FileNotFoundError:
raise RuntimeError("未找到 ffmpeg，请安装: sudo apt-get install ffmpeg")
def run_whisper(whisper_path, audio_path, model, output_dir, language='zh'):
"""运行 Whisper 语音识别
Args:
whisper_path: whisper.cpp 目录
audio_path: WAV 音频文件路径
model: 模型名称
output_dir: 输出目录
language: 语言代码
"""
# 输出文件名（使用原始输入文件名，不是 audio.wav）
# 从 output_dir 和 audio_path 推断原始文件名
original_stem = Path(audio_path).stem.replace('_audio', '')
output_basename = original_stem
# 构建命令
cmd = [
os.path.join(whisper_path, 'whisper-cli'),
'-m', os.path.join(whisper_path, 'models', f'ggml-{model}.bin'),
'-f', audio_path,
'-of', os.path.join(output_dir, output_basename),
'-l', language,
'--output-txt',
'--output-srt',
'--output-json',
]
print(f"执行命令: {' '.join(cmd)}")
# 运行
result = subprocess.run(
cmd,
capture_output=True,
text=True,
cwd=whisper_path,
timeout=3600 # 1小时超时
)
if result.returncode != 0:
# 检查是否有警告但仍成功
if "error" in result.stderr.lower() and not os.path.exists(
os.path.join(output_dir, f"{output_basename}.txt")
):
raise RuntimeError(
f"Whisper 执行失败:\n"
f"标准输出: {result.stdout}\n"
f"错误输出: {result.stderr}"
)
return result
def parse_srt_for_duration(srt_file):
"""从 SRT 文件解析时长"""
if not os.path.exists(srt_file):
return 0
with open(srt_file, 'r', encoding='utf-8') as f:
content = f.read()
# 找到最后一个时间戳
pattern = r'(\d{2}:\d{2}:\d{2}),\d{3} --> \d{2}:\d{2}:\d{2},\d{3}'
matches = re.findall(pattern, content)
if not matches:
return 0
# 解析最后一个时间戳
last_time = matches[-1]
h, m, s = last_time.split(':')
return int(h) * 3600 + int(m) * 60 + int(s)
def create_workflow_json(input_file, output_dir, model, language, duration):
"""创建标准格式的 JSON 输出，供 KB Workflow 使用"""
input_path = Path(input_file)
basename = input_path.stem
txt_file = os.path.join(output_dir, f"{basename}.txt")
srt_file = os.path.join(output_dir, f"{basename}.srt")
json_file = os.path.join(output_dir, f"{basename}.json")
# 读取文本内容
text_content = ""
if os.path.exists(txt_file):
with open(txt_file, 'r', encoding='utf-8') as f:
text_content = f.read().strip()
# 解析 SRT 获取分段
segments = []
if os.path.exists(srt_file):
with open(srt_file, 'r', encoding='utf-8') as f:
content = f.read()
# SRT 格式: 序号 -> 时间 -> 文本
pattern = r'(\d+)\n(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\n(.+?)(?=\n\n|\Z)'
matches = re.findall(pattern, content, re.DOTALL)
def parse_time(t):
h, m, s_ms = t.split(':')
s, ms = s_ms.split(',')
return float(h) * 3600 + float(m) * 60 + float(s) + float(ms) / 1000
for idx, (num, start, end, text) in enumerate(matches):
segments.append({
"id": idx,
"start": round(parse_time(start), 2),
"end": round(parse_time(end), 2),
"text": text.strip()
})
# 构建 JSON
workflow_json = {
"success": True,
"tool": "whisper",
"model": model,
"language": language,
"input_file": str(input_path),
"output_files": {
"txt": f"{basename}.txt",
"srt": f"{basename}.srt",
"json": f"{basename}.json"
},
"duration_seconds": duration,
"segments": segments,
"text": text_content
}
# 写入 JSON
with open(json_file, 'w', encoding='utf-8') as f:
json.dump(workflow_json, f, ensure_ascii=False, indent=2)
return workflow_json
def main():
"""主函数"""
args = parse_arguments()
# 检查输入文件
input_file = os.path.abspath(args.input)
if not os.path.exists(input_file):
print(f"错误: 输入文件不存在: {input_file}", file=sys.stderr)
sys.exit(1)
# 获取输出目录
if args.output:
output_dir = os.path.abspath(args.output)
else:
output_dir = os.path.dirname(input_file)
os.makedirs(output_dir, exist_ok=True)
# 获取 whisper.cpp 路径
try:
whisper_path = get_whisper_cpp_path()
print(f"✓ Whisper.cpp 路径: {whisper_path}")
except FileNotFoundError as e:
print(f"✗ 错误: {e}", file=sys.stderr)
sys.exit(1)
# 检查模型
try:
check_model(whisper_path, args.model)
print(f"✓ 使用模型: {args.model}")
except FileNotFoundError as e:
print(f"✗ 错误: {e}", file=sys.stderr)
sys.exit(1)
# 后台执行
if args.background:
print("启动后台处理...")
log_file = os.path.join(output_dir, f"{Path(input_file).stem}_whisper.log")
with open(log_file, 'w') as log:
subprocess.Popen(
[sys.executable, __file__,
'--input', input_file,
'--model', args.model,
'--output', output_dir,
'--language', args.language],
stdout=log,
stderr=log,
start_new_session=True
)
print(f"✓ 后台任务已启动")
print(f" 日志: {log_file}")
print(f" 输出: {output_dir}")
result = {
"success": True,
"background": True,
"status": "running",
"log_file": log_file,
"output_dir": output_dir
}
print(json.dumps(result, ensure_ascii=False, indent=2))
return
# 前台执行
print(f"\n📁 处理文件: {input_file}")
print(f"📁 输出目录: {output_dir}")
try:
# 1. 转换为 WAV
print(f"\n🔄 步骤1: 转换音频格式...")
audio_path = convert_to_wav(input_file, output_dir)
# 2. 运行 Whisper
print(f"\n🎙️ 步骤2: 语音识别 (语言: {args.language})...")
run_whisper(whisper_path, audio_path, args.model, output_dir, args.language)
# 3. 清理临时 WAV 文件
if audio_path != input_file and audio_path.endswith('_audio.wav'):
try:
os.remove(audio_path)
print("✓ 清理临时文件")
except Exception:
pass
# 4. 获取时长并创建 JSON
print(f"\n📝 步骤3: 生成输出文件...")
basename = Path(input_file).stem
srt_file = os.path.join(output_dir, f"{basename}.srt")
duration = parse_srt_for_duration(srt_file)
workflow_json = create_workflow_json(
input_file, output_dir, args.model, args.language, duration
)
# 输出结果
print(f"\n✅ 处理完成!")
print(f" 时长: {duration} 秒")
print(f" 分段数: {len(workflow_json['segments'])}")
print(f"\n📄 输出文件:")
for fmt in ['txt', 'srt', 'json']:
fpath = os.path.join(output_dir, f"{basename}.{fmt}")
if os.path.exists(fpath):
size = os.path.getsize(fpath)
print(f" • {fmt.upper()}: {fpath} ({size:,} bytes)")
print(f"\n📋 Workflow JSON:")
print(json.dumps({
"success": workflow_json["success"],
"tool": "whisper",
"text_length": len(workflow_json["text"]),
"segments_count": len(workflow_json["segments"])
}, ensure_ascii=False, indent=2))
except Exception as e:
print(f"\n✗ 错误: {e}", file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()
kb-workflow/.env
+14
View File
@@ -0,0 +1,14 @@
# KB Workflow 环境变量配置
export KB_BASE_PATH="~/work/knowledge-base"
# 可选: B站 Cookie（用于下载会员视频）
# export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx"
# 可选: AI 分析配置
# export AI_API_KEY="your-api-key"
# export AI_PROVIDER="openai" # 或 zhipu, minimax
# export AI_MODEL="gpt-4"
# 可选: Git 配置
# export KB_GIT_USER="Your Name"
# export KB_GIT_EMAIL="your@email.com"
kb-workflow/.env.example
+14
View File
@@ -0,0 +1,14 @@
# KB Workflow 环境变量配置
export KB_BASE_PATH="~/work/knowledge-base"
# 可选: B站 Cookie（用于下载会员视频）
# export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx"
# 可选: AI 分析配置
# export AI_API_KEY="your-api-key"
# export AI_PROVIDER="openai" # 或 zhipu, minimax
# export AI_MODEL="gpt-4"
# 可选: Git 配置
# export KB_GIT_USER="Your Name"
# export KB_GIT_EMAIL="your@email.com"
kb-workflow/README.md
+247
View File
@@ -0,0 +1,247 @@
# KB Workflow 环境安装脚本
一键检查并安装 kb-workflow 知识库工作流所需的全部环境依赖。
## 功能特性
- ✅ 自动检测系统类型 (Ubuntu/Debian/Fedora/Arch)
- ✅ 检查系统依赖 (Python3, pip3, ffmpeg, tesseract, git)
- ✅ 检查 Python 依赖包 (httpx, aiohttp, playwright, easyocr 等)
- ✅ 自动创建知识库目录结构
- ✅ 生成环境变量配置示例
- ✅ 汇总显示所有需要执行的安装命令
## 使用方法
### 运行检查脚本
```bash
cd /home/starmen1265/.openclaw/workspace/skills
./install-kb-workflow.sh
```
脚本会检查所有依赖，并显示需要执行的安装命令。
### 示例输出
```
========================================
第一步: 检查系统依赖
========================================
[SUCCESS] Python3 已安装: Python 3.12.3
[WARN] pip3 未安装
[WARN] ffmpeg 未安装
...
========================================
需要安装的系统依赖
========================================
检测到以下依赖未安装:
- python3-pip
- ffmpeg
- tesseract-ocr
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
请运行以下命令安装:
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
sudo apt-get update
sudo apt-get install -y python3-pip ffmpeg tesseract-ocr tesseract-ocr-chi-sim tesseract-ocr-eng
```
## 依赖清单
### 系统依赖
| 依赖 | 用途 | 安装命令 |
|------|------|----------|
| **python3** | Python 运行时 | `sudo apt-get install python3` |
| **python3-pip** | Python 包管理器 | `sudo apt-get install python3-pip` |
| **ffmpeg** | 视频处理（OCR 截帧） | `sudo apt-get install ffmpeg` |
| **tesseract-ocr** | OCR 引擎（备用方案） | `sudo apt-get install tesseract-ocr` |
| **git** | 版本控制（归档提交） | `sudo apt-get install git` |
### Tesseract 语言包
```bash
# Ubuntu/Debian
sudo apt-get install tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-eng
# Fedora/RHEL
sudo dnf install tesseract-langpack-chi_sim tesseract-langpack-chi_tra tesseract-langpack-eng
```
### Python 依赖
```bash
# 基础 HTTP 库
pip3 install --user httpx aiohttp requests
# 图像和 OCR
pip3 install --user Pillow easyocr pytesseract
# 浏览器自动化
pip3 install --user playwright
python3 -m playwright install chromium
```
## 目录结构
安装完成后会创建以下目录结构：
```
~/work/knowledge-base/ # 知识库根目录（可配置）
├── archive/ # 归档目录
│ ├── pending/ # 待处理
│ └── subscriptions/ # 订阅信息
├── metadata/ # 元数据文件
├── content/ # 内容文件
│ ├── downloads/ # 视频下载
│ └── subtitles/ # 字幕文件
└── analyzed/ # 深度分析报告
```
## 环境变量
### 必需
- `KB_BASE_PATH` - 知识库根目录（默认: `~/work/knowledge-base`）
### 可选
- `BILIBILI_COOKIE` - B站 Cookie，用于下载会员视频
- `AI_API_KEY` - AI API 密钥，用于深度内容分析
- `AI_PROVIDER` - AI 提供商（openai/zhipu/minimax）
- `AI_MODEL` - AI 模型名称（如 gpt-4）
- `AI_BASE_URL` - 自定义 API 端点
- `KB_GIT_USER` - Git 用户名（用于归档提交）
- `KB_GIT_EMAIL` - Git 邮箱（用于归档提交）
- `KB_GIT_REMOTE` - Git 远程仓库地址
### 配置方法
将以下内容添加到 `~/.bashrc` 或 `~/.zshrc`：
```bash
# KB Workflow 配置
export KB_BASE_PATH="~/work/knowledge-base"
# 可选：B站 Cookie
export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx"
# 可选：AI 配置
export AI_API_KEY="your-api-key"
export AI_PROVIDER="openai"
export AI_MODEL="gpt-4"
```
然后重新加载配置：
```bash
source ~/.bashrc # 或 source ~/.zshrc
```
## 故障排除
### 1. pip3 命令不存在
```bash
# 方法 1: 安装 pip3
sudo apt-get install python3-pip
# 方法 2: 使用 python3 -m pip
python3 -m pip install --user <package>
```
### 2. Playwright 浏览器安装失败
```bash
# 手动安装 Chromium
python3 -m playwright install chromium
# 如果仍然失败，尝试安装依赖
sudo apt-get install libnss3 libatk-bridge2.0-0 libxss1 libgtk-3-0
```
### 3. EasyOCR 安装失败
EasyOCR 依赖 PyTorch，可能需要较长时间安装：
```bash
# 先安装 PyTorch
pip3 install --user torch torchvision --index-url https://download.pytorch.org/whl/cpu
# 再安装 easyocr
pip3 install --user easyocr
```
### 4. 权限问题
如果使用 `--user` 安装的包无法找到，检查 PATH：
```bash
# 查看用户安装路径
python3 -m site --user-base
# 添加到 PATH（添加到 ~/.bashrc）
export PATH="$PATH:$(python3 -m site --user-base)/bin"
```
### 5. OCR 中文识别问题
确保已安装中文语言包：
```bash
# 检查已安装的语言包
tesseract --list-langs
# 应该能看到 chi_sim 和 chi_tra
```
## 更新
如果技能更新了依赖，可以重新运行检查脚本：
```bash
./install-kb-workflow.sh
```
## 子技能清单
| 技能 | 路径 | 功能 |
|------|------|------|
| kb-workflow | `kb-workflow/kb_workflow.py` | 主流程协调 |
| kb-link-parser | `kb-link-parser/link-parser.py` | 链接解析 |
| kb-bilibili-metadata | `kb-bilibili-metadata/bilibili_metadata.py` | B站元数据提取 |
| kb-bilibili-content | `kb-bilibili-content/bilibili_content.py` | B站内容下载 |
| kb-ocr-processor | `kb-ocr-processor/ocr_processor.py` | 视频 OCR 识别 |
| kb-deep-analyzer | `kb-deep-analyzer/deep_analyzer.py` | AI 深度分析 |
## 使用 kb-workflow
安装完成后，可以通过以下方式使用：
### 方式 1：通过主代理
直接向主代理发送消息：
```
添加到知识库 https://www.bilibili.com/video/BVxxxxx
```
### 方式 2：手动运行子技能
```bash
# 1. 链接解析
python3 kb-link-parser/link-parser.py --url "https://www.bilibili.com/video/BVxxxxx"
# 2. 元数据提取
python3 kb-bilibili-metadata/bilibili_metadata.py --id "BVxxxxx" --type video
# 3. 内容下载
python3 kb-bilibili-content/bilibili_content.py --id "BVxxxxx" --type subtitle
# 4. 深度分析
python3 kb-deep-analyzer/deep_analyzer.py --metadata meta.json --content content.json --output analysis.md
```
## 许可证
MIT License
kb-workflow/SKILL.md
+360
View File
@@ -0,0 +1,360 @@
---
name: kb-workflow
description: 当收到"添加到知识库 + 链接" 或"保存到知识库 + 链接"时使用
---
# KB Workflow
## 什么时候使用
当用户发送包含以下模式的消息时，此技能自动触发：
| 触发词 | 行为 | 示例 |
|--------|------|------|
| `添加到知识库` + 链接 | 添加到知识库并处理 | "添加到知识库 https://www.bilibili.com/video/BVxxx" |
| `保存到知识库` + 链接 | 同上 | "保存到知识库 https://www.bilibili.com/video/BVxxx" |
## 环境变量
### 执行前必须检查
在执行 workflow 之前，必须检查环境变量是否已经设置。
**必须的环境变量**（缺失则报错退出）：
配置方式：将变量写入 `~/.kb_env` 文件：
```bash
export AI_API_KEY="your-api-key"
export AI_PROVIDER="zhipu"
export AI_MODEL="glm-5"
export AI_BASE_URL="https://open.bigmodel.cn/api/coding/paas/v4"
```
**可选的环境变量**：
| 变量名 | 用途 | 示例 |
|--------|------|------|
| `BILIBILI_COOKIE` | 下载会员视频 | `SESSDATA=xxx; bili_jct=xxx` |
| `KB_GIT_USER` | Git提交用户名 | `Your Name` |
| `KB_GIT_EMAIL` | Git提交邮箱 | `your@email.com` |
### 检查命令
```bash
# 检查必须的环境变量
echo "AI_API_KEY: ${AI_API_KEY:0:10}..."
echo "AI_PROVIDER: $AI_PROVIDER"
echo "AI_MODEL: $AI_MODEL"
echo "AI_BASE_URL: $AI_BASE_URL"
# 验证方式1: 检查变量是否存在
if [ -z "$AI_API_KEY" ]; then
echo "❌ AI_API_KEY 未设置"
exit 1
fi
# 验证方式2: 执行测试调用
python3 -c "
import os
if not os.environ.get('AI_API_KEY'):
raise RuntimeError('AI_API_KEY 未设置')
print('✅ 环境变量检查通过')
"
```
## Workflow 执行方式
主代理直接执行以下步骤，无需启动 kb-workflow 子代理：
1. 主代理启动子代理调用 kb-link-parser
2. 主代理启动子代理调用 kb-平台-metadata
3. 主代理启动子代理调用 kb-平台-content
4. 主代理启动子代理调用 kb-ocr-processor（如需要）
5. 主代理启动子代理调用 kb-deep-analyzer
6. 主代理执行 `kb_workflow.py --archive-file` 归档
## Workflow 执行步骤
当触发后，按照以下步骤执行：
### 步骤 1: 环境变量检查
**❌ 禁止跳过此步骤！必须先检查环境变量。**
```bash
# 0. 从 ~/.kb_env 加载环境变量（专门的知识库配置文件，不受 bashrc 交互式检查影响）
if [ -f ~/.kb_env ]; then
source ~/.kb_env
fi
# 1. 检查必须的环境变量是否存在
if [ -z "$AI_API_KEY" ] || [ -z "$AI_PROVIDER" ] || [ -z "$AI_MODEL" ] || [ -z "$AI_BASE_URL" ]; then
echo "❌ 错误：必须的环境变量未设置"
echo "请配置以下环境变量："
echo " - AI_API_KEY"
echo " - AI_PROVIDER (zhipu/openai/minimax)"
echo " - AI_MODEL"
echo " - AI_BASE_URL"
exit 1
fi
echo "✅ 环境变量检查通过"
echo " AI_PROVIDER: $AI_PROVIDER"
echo " AI_MODEL: $AI_MODEL"
```
**如果检查不通过，必须**：
1. 报错提示用户
2. 退出，不执行后续步骤
3. 等待用户配置好环境变量后重新执行
## ⚠️ 严格失败处理原则
> **深度分析步骤必须成功，否则整个流程失败**
在执行工作流时，必须遵守以下原则：
1. **深度分析不可跳过**：kb-deep-analyzer 是必须成功执行的步骤
2. **失败即终止**：如果深度分析返回 `success: false`，必须立即停止并汇报失败
3. **禁止降级处理**：不能使用预设模板或简单摘要来"假装成功"
4. **每次执行都要使用 AI**：必须有真实的 AI API 调用记录
### 步骤 2: 链接解析
1. 获取目录路径：
```bash
python3 kb_workflow.py --pending
```
2. 启动子代理调用 kb-link-parser：
```bash
python3 /path/to/link-parser.py --url "[URL]"
```
3. 解析输出 JSON，提取 filename 和 platform
### 步骤 3: 元数据提取
1. 获取元数据目录路径：
```bash
python3 kb_workflow.py --metadata
# 返回: {"path": "/home/admin/work/knowledge-base/metadata"}
```
2. 构建输出文件路径：`${metadata_dir}/${platform}_${id}.meta.json`
3. 启动子代理调用对应平台的 metadata 技能（传入 `--output`）：
```bash
# Bilibili
python3 /path/to/bilibili_metadata.py --id "[ID]" --type video --output "${metadata_dir}/bilibili_[ID].meta.json"
# Zhihu
python3 /path/to/zhihu_metadata.py --id "[ID]" --type question --output "${metadata_dir}/zhihu_[ID].meta.json"
```
4. 解析输出 JSON，提取 filename
### 步骤 4: 内容下载
1. 获取目录路径：
```bash
python3 kb_workflow.py --content_downloads
python3 kb_workflow.py --content
```
2. 启动子代理调用对应平台的 content 技能：
```bash
# Bilibili 字幕下载（自动检测是否有官方字幕）
python3 /path/to/bilibili_content.py --id "[ID]" --type subtitle-download --output [content_dir]
# 返回 JSON 包含 has_subtitle 字段
```
3. 解析输出 JSON：
- **has_subtitle = true**: 有官方字幕，直接进入步骤 6
- **has_subtitle = false**: 没有官方字幕，继续步骤 5
> ⚠️ **重要提示**：没有找到官方字幕是**正常的**！很多视频没有自动生成的字幕。不要因为没有字幕就停止工作流，继续执行 Whisper 步骤即可。
### 步骤 5: 语音识别转文字（仅当没有官方字幕时执行）
> ⚠️ **重要**：此步骤仅在**没有官方字幕**时执行。如果有官方字幕，**跳过此步骤**。
此步骤使用 Whisper 进行语音识别：
- **严禁跳过**：必须等待识别完成
- **定时汇报**：每2-3分钟向用户汇报进度
- **超时时间**：30分钟
- **耐心等待**：不要中途启动其他步骤，必须串行执行
1. 获取目录路径：
```bash
python3 kb_workflow.py --content
```
2. 启动子代理调用 kb-whisper（传入视频路径）：
```bash
# 必须传入 --output 完整路径
python3 /path/to/whisper.py --input [video_path] --model base --output [content_dir]
```
3. 等待 Whisper 完成（定时汇报进度）
4. 解析输出 JSON，生成 content 文件供步骤6使用
### 步骤 6: 深度分析
1. 获取目录路径：
```bash
python3 kb_workflow.py --analyzed
```
2. 准备 content JSON 文件（包含 subtitle/description）
3. 如果有官方字幕，使用官方字幕的 SRT 文件
4. 如果没有官方字幕，使用 Whisper 识别的结果
5. 启动子代理调用 kb-deep-analyzer（使用 g5 模型）：
```bash
python3 /path/to/deep_analyzer.py --metadata [meta_path] --content [content_path] --output [analyzed_dir]/[filename].md --provider zhipu --model glm-5
```
**环境变量设置**（在子代理中设置）：
- `AI_PROVIDER=zhipu`
- `AI_MODEL=glm-5`
6. 解析输出 JSON，检查 `success` 字段：
> ⚠️ **深度分析失败 = 整个流程失败**
> - 如果 `success = true`：继续步骤 7（归档）
> - 如果 `success = false`：**立即终止流程**，向用户汇报失败原因
> - **严禁**继续执行后续步骤或使用降级内容"假装成功"
### 步骤 7: 归档
运行归档命令：
```bash
python3 kb_workflow.py --archive-file "[分析报告文件路径]"
```
### 步骤 8: 汇报完成
向用户汇报：
- 视频标题
- 处理阶段完成情况
- 最终文件路径
- Git commit 信息
## 子技能清单
| 步骤 | 子技能 | 命令示例 | 输出 | 备注 |
|------|--------|----------|------|------|
| 2 | kb-link-parser | `--url "BVxxx"` | `{p}_{id}.link.json` | |
| 3 | kb-bilibili-metadata | `--id "BVxxx" --type video` | `{p}_{id}.meta.json` | |
| 3 | kb-zhihu-metadata | `--id "xxx" --type question` | `{p}_{id}.meta.json` | |
| 4 | kb-bilibili-content | `--id "BVxxx" --type subtitle-download` | `{p}_{id}.srt` | 如果没有字幕是正常的 |
| 4 | kb-bilibili-content | `--id "BVxxx" --type video` | `{p}_{id}.mp4` | 如果需要下载视频 |
| 5 | kb-whisper | `--input xxx.mp4 --model base` | `{p}_{id}.json`, `{p}_{id}.txt` | 仅当没有官方字幕时执行 |
| 6 | kb-deep-analyzer | `--metadata x.meta.json --content x.json --output x.md --provider zhipu --model glm-5` | `{p}_{id}.md` | **使用 g5 模型，必须成功** |
## 条件分支逻辑
```
步骤3: 获取元数据
↓
步骤4: 下载字幕 (--type subtitle-download)
↓
┌─────────────────────────────────────┐
│ has_subtitle = true? │
├─────────────────────────────────────┤
│ 是 → 直接进入步骤6（深度分析） │
│ 否 → 进入步骤5（Whisper语音识别） │
└─────────────────────────────────────┘
↓
步骤6: 深度分析
↓
┌─────────────────────────────────────┐
│ kb-deep-analyzer 执行结果 │
├─────────────────────────────────────┤
│ success = true → 步骤7（归档） │
│ success = false → ❌ 整个流程失败 │
└─────────────────────────────────────┘
```
> ⚠️ **重要：深度分析失败则整个流程失败**
> - 深度分析必须使用正确的 AI 模型（glm-4.7）完整分析字幕内容
> - 如果 `kb-deep-analyzer` 返回 `success: false`，**必须汇报失败**，不能继续执行后续步骤
> - 不能使用预设模板或降级内容来"假装成功"
> - 失败时必须向用户报告具体的错误原因
> ⚠️ **重要**：没有找到官方字幕是**正常的**！很多视频没有自动生成的字幕。
> - 如果 `has_subtitle = false`，继续执行 Whisper 步骤
> - 如果 `has_subtitle = true`，跳过 Whisper 步骤，直接使用官方字幕
## 目录结构
```
~/work/knowledge-base/
├── archive/
│ └── YYYY-MM-DD/
├── pending/
│ └── {platform}_{id}.link.json
├── metadata/
│ └── {platform}_{id}.meta.json
├── content/
│ ├── downloads/
│ │ └── {platform}_{id}.mp4
│ ├── {platform}_{id}.txt # Whisper 识别的文本
│ └── {platform}_{id}.json # Whisper 完整结果
└── analyzed/
└── {platform}_{id}.md
```
## 辅助命令
```bash
# ------------------- 查重与归档 -------------------
# 查重
python3 kb_workflow.py --check --url "[URL]"
# 归档分析报告
python3 kb_workflow.py --archive-file "[分析报告路径]"
# ------------------- 目录路径查询 -------------------
# 获取知识库根目录（默认 ~/work/knowledge-base）
python3 kb_workflow.py --base-path "~/custom/path"
# 获取待处理目录（pending）
python3 kb_workflow.py --pending
# 获取元数据目录
python3 kb_workflow.py --metadata
# 获取内容目录
python3 kb_workflow.py --content
# 获取内容下载目录
python3 kb_workflow.py --content_downloads
# 获取字幕目录
python3 kb_workflow.py --content_subtitles
# 获取深度分析目录
python3 kb_workflow.py --analyzed
# 获取归档目录
python3 kb_workflow.py --archive
# 获取订阅信息目录
python3 kb_workflow.py --archive_subscriptions
# ------------------- 工具函数 -------------------
# 从文件名提取平台名称
python3 kb_workflow.py --extract_platform "bilibili_BVxxx.mp4"
```
kb-workflow/SkillBug.md
+101
View File
@@ -0,0 +1,101 @@
# SkillBug.md
## 问题记录
### 2026-02-10: 子代理权限问题
**问题：** `sessions_spawn` 调用子代理时返回 `forbidden` 错误。
**原因：** OpenClaw 配置中 allowlist 为空，Python 脚本没有权限调用子代理。
**解决方案：** 改用 Prompt-based 模式（方案B）。
---
## 架构重构
### 之前（Script-based）
```
kb_workflow.py (Python脚本)
↓
sessions_spawn() # ❌ 失败（脚本没有权限）
```
### 之后（Prompt-based）
```
主代理读取 SKILL.md
↓
按照指导调用 sessions_spawn # ✅ 成功（主代理有权限）
```
### 文件变更
| 文件 | 变更 | 说明 |
|------|------|------|
| `SKILL.md` | 重写 | 详细定义 workflow 步骤，主代理直接读取执行 |
| `kb_workflow.py` | 简化 | 保留为辅助工具函数库（查重、归档、Git） |
| `skill_main.py` | 简化 | 只做触发检测，返回指导信息 |
| `SkillBug.md` | 更新 | 记录架构变更 |
### SKILL.md 结构
```
1. 触发条件
2. 步骤 1: 查重
3. 步骤 2: kb-link-parser
4. 步骤 3: 元数据技能
5. 步骤 4: 内容下载
6. 步骤 5: OCR（30分钟超时）
7. 步骤 6: 深度分析
8. 步骤 7: 归档和 Git
9. 步骤 8: 汇报完成
10. 目录结构
11. 支持平台
12. 注意事项
```
### kb_workflow.py 保留的函数
```python
extract_platform(filename) # 从文件名提取平台
extract_base_name(filename) # 从文件名提取基础名称
get_knowledge_base_path() # 获取知识库根目录
ensure_directories(base_path) # 确保目录存在
check_duplicate(url) # 查重
archive_and_git(analyzed_file) # 归档和 Git
generate_summary(...) # 生成汇报
```
---
### 2026-02-12: B站视频下载失败 - 403 Forbidden
**问题：** 使用 kb-bilibili-content 下载视频时返回 403 Forbidden 错误。
**原因：**
1. 视频可能需要B站会员权限
2. 或者需要登录cookie才能下载
3. 当前未配置 `BILIBILI_COOKIE` 环境变量
**复现场景：**
- 链接：https://www.bilibili.com/video/BV1Z9nXzUEhv
- 下载命令：`python3 bilibili_content.py --id "BV1Z9nXzUEhv" --type video`
- 结果：返回403错误页面，文件只有548字节
**影响：** 无法进行后续的OCR处理和深度分析
**建议解决方案：**
1. 配置 `BILIBILI_COOKIE` 环境变量
2. 或提供cookies文件路径
3. 或跳过OCR步骤，使用元数据+简介进行深度分析
**当前进度：**
- ✅ 步骤1：环境变量检查完成
- ✅ 步骤2：链接解析成功（bilibili_BV1Z9nXzUEhv）
- ✅ 步骤3：元数据提取成功（标题：新加坡分区漫游指南...）
- ❌ 步骤4：视频下载失败（403）
- ⏸️ 步骤5：OCR处理（阻塞）
- ⏸️ 步骤6：深度分析（阻塞）
- ⏸️ 步骤7：归档（阻塞）
kb-workflow/install-env.sh
+321
View File
@@ -0,0 +1,321 @@
#!/bin/bash
# =============================================================================
# KB Workflow 环境一键安装脚本
# =============================================================================
# 功能: 检查并安装 kb-workflow 技能所需的全部环境依赖
# 支持: Ubuntu/Debian 系统
#
# 使用方法:
# ./install-env.sh # 检查环境并提示安装命令（检查模式）
# ./install-env.sh --auto # 自动安装所有缺失依赖（自动模式）
# =============================================================================
# 颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
# 日志函数
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_section() {
echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}$1${NC}"
echo -e "${BLUE}========================================${NC}"
}
# 获取脚本目录
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DEFAULT_KB_PATH="${HOME}/work/knowledge-base"
# 解析参数
AUTO_MODE=false
if [ "$1" = "--auto" ]; then
AUTO_MODE=true
log_info "自动安装模式"
fi
# 检测 Linux 发行版
detect_distro() {
if [ -f /etc/os-release ]; then
. /etc/os-release
echo "$ID"
elif [ -f /etc/debian_version ]; then
echo "debian"
elif [ -f /etc/redhat-release ]; then
echo "rhel"
fi
}
DISTRO=$(detect_distro)
# 检查命令是否存在
command_exists() {
command -v "$1" > /dev/null 2>&1
}
# 检查 Python 包
check_python_package() {
python3 -c "import $1" 2>/dev/null
}
# =============================================================================
# 1. 检查系统依赖
# =============================================================================
log_section "第一步: 检查系统依赖"
MISSING_DEPS=()
# 检查 Python3
if command_exists python3; then
PYTHON_VERSION=$(python3 --version | cut -d' ' -f2)
log_success "Python3 已安装: $PYTHON_VERSION"
else
log_warn "Python3 未安装"
MISSING_DEPS+=("python3")
fi
# 检查 pip3
if python3 -m pip --version > /dev/null 2>&1 || command_exists pip3; then
log_success "pip3 已安装"
else
log_warn "pip3 未安装"
MISSING_DEPS+=("python3-pip")
fi
# 检查 ffmpeg
if command_exists ffmpeg; then
log_success "ffmpeg 已安装"
else
log_warn "ffmpeg 未安装"
MISSING_DEPS+=("ffmpeg")
fi
# 检查 tesseract
if command_exists tesseract; then
log_success "Tesseract OCR 已安装"
else
log_warn "Tesseract OCR 未安装"
MISSING_DEPS+=("tesseract-ocr")
fi
# 检查 git
if command_exists git; then
log_success "Git 已安装"
else
log_warn "Git 未安装"
MISSING_DEPS+=("git")
fi
# =============================================================================
# 2. 自动安装系统依赖（自动模式）
# =============================================================================
if [ ${#MISSING_DEPS[@]} -gt 0 ]; then
if [ "$AUTO_MODE" = true ]; then
log_section "自动安装系统依赖"
log_info "检测到系统: $DISTRO"
case "$DISTRO" in
ubuntu|debian)
log_info "更新包列表..."
sudo apt-get update -qq
log_info "安装依赖: ${MISSING_DEPS[*]}"
sudo apt-get install -y -qq "${MISSING_DEPS[@]}" tesseract-ocr-chi-sim tesseract-ocr-eng
;;
fedora|rhel|centos)
log_info "更新包列表..."
sudo dnf check-update -y || true
log_info "安装依赖: ${MISSING_DEPS[*]}"
sudo dnf install -y "${MISSING_DEPS[@]}" tesseract-langpack-chi_sim tesseract-langpack-eng
;;
arch)
log_info "安装依赖: ${MISSING_DEPS[*]}"
sudo pacman -S --noconfirm "${MISSING_DEPS[@]}" tesseract-data-chi_sim tesseract-data-eng
;;
*)
log_error "不支持的系统: $DISTRO"
exit 1
;;
esac
log_success "系统依赖安装完成"
MISSING_DEPS=()
else
log_section "需要安装的系统依赖"
echo ""
echo "检测到以下依赖未安装:"
for dep in "${MISSING_DEPS[@]}"; do
echo " - $dep"
done
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "请运行以下命令安装:"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo -e "${YELLOW}sudo apt-get update${NC}"
echo -e "${YELLOW}sudo apt-get install -y ${MISSING_DEPS[*]} tesseract-ocr-chi-sim tesseract-ocr-eng${NC}"
echo ""
fi
fi
# =============================================================================
# 3. 检查 Python 依赖
# =============================================================================
log_section "第二步: 检查 Python 依赖"
PYTHON_PACKAGES=(
"httpx:httpx"
"aiohttp:aiohttp"
"requests:requests"
"Pillow:PIL"
)
MISSING_PY_PKGS=()
for pkg_info in "${PYTHON_PACKAGES[@]}"; do
pkg_name=$(echo "$pkg_info" | cut -d':' -f1)
import_name=$(echo "$pkg_info" | cut -d':' -f2)
if check_python_package "$import_name" 2>/dev/null; then
log_success "$pkg_name 已安装"
else
log_warn "$pkg_name 未安装"
MISSING_PY_PKGS+=("$pkg_name")
fi
done
# 检查特殊包
check_python_package "playwright" 2>/dev/null && log_success "playwright 已安装" || MISSING_PY_PKGS+=("playwright")
check_python_package "easyocr" 2>/dev/null && log_success "easyocr 已安装" || MISSING_PY_PKGS+=("easyocr")
check_python_package "pytesseract" 2>/dev/null && log_success "pytesseract 已安装" || MISSING_PY_PKGS+=("pytesseract")
# =============================================================================
# 4. 自动安装 Python 依赖（自动模式）
# =============================================================================
if [ ${#MISSING_PY_PKGS[@]} -gt 0 ]; then
if [ "$AUTO_MODE" = true ]; then
log_section "自动安装 Python 包"
log_info "安装: ${MISSING_PY_PKGS[*]}"
pip3 install --user "${MISSING_PY_PKGS[@]}" -q
log_success "Python 包安装完成"
# 安装 Playwright 浏览器
if [[ " ${MISSING_PY_PKGS[*]} " =~ " playwright " ]]; then
log_info "安装 Playwright Chromium 浏览器..."
python3 -m playwright install chromium
log_success "Chromium 浏览器安装完成"
fi
MISSING_PY_PKGS=()
else
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "需要安装以下 Python 包:"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
for pkg in "${MISSING_PY_PKGS[@]}"; do
echo " - $pkg"
done
echo ""
echo "请运行以下命令安装:"
echo ""
echo -e "${YELLOW}pip3 install --user ${MISSING_PY_PKGS[*]}${NC}"
if [[ " ${MISSING_PY_PKGS[*]} " =~ " playwright " ]]; then
echo ""
echo -e "${YELLOW}安装完 playwright 后，还需要运行:${NC}"
echo -e "${YELLOW} python3 -m playwright install chromium${NC}"
fi
echo ""
fi
fi
# =============================================================================
# 5. 创建知识库目录结构
# =============================================================================
log_section "第三步: 创建知识库目录结构"
KB_BASE_PATH="${KB_BASE_PATH:-$DEFAULT_KB_PATH}"
log_info "知识库路径: $KB_BASE_PATH"
mkdir -p "$KB_BASE_PATH"/{archive/{subscriptions,pending},metadata,content/{downloads,subtitles},analyzed}
log_success "目录结构创建完成"
# =============================================================================
# 6. 环境变量配置提示
# =============================================================================
log_section "第四步: 环境变量配置"
if [ -z "$KB_BASE_PATH_SET" ]; then
log_warn "KB_BASE_PATH 环境变量未设置"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "建议添加到 ~/.bashrc 或 ~/.zshrc:"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo -e "${YELLOW}export KB_BASE_PATH=\"$KB_BASE_PATH\"${NC}"
echo ""
else
log_success "KB_BASE_PATH 已设置: $KB_BASE_PATH"
fi
# 创建环境变量示例文件
ENV_FILE="${SCRIPT_DIR}/.env.example"
cat > "$ENV_FILE" << 'EOF'
# KB Workflow 环境变量配置
export KB_BASE_PATH="~/work/knowledge-base"
# 可选: B站 Cookie（用于下载会员视频）
# export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx"
# 可选: AI 分析配置
# export AI_API_KEY="your-api-key"
# export AI_PROVIDER="openai" # 或 zhipu, minimax
# export AI_MODEL="gpt-4"
# 可选: Git 配置
# export KB_GIT_USER="Your Name"
# export KB_GIT_EMAIL="your@email.com"
EOF
log_info "环境变量示例已保存到: $ENV_FILE"
# =============================================================================
# 7. 完成总结
# =============================================================================
log_section "安装完成"
echo ""
if [ ${#MISSING_DEPS[@]} -eq 0 ] && [ ${#MISSING_PY_PKGS[@]} -eq 0 ]; then
echo -e "${GREEN}✅ 所有依赖已安装完成！${NC}"
else
echo -e "${YELLOW}⚠️ 部分依赖需要手动安装${NC}"
echo ""
if [ ${#MISSING_DEPS[@]} -gt 0 ]; then
echo "系统依赖:"
echo -e "${YELLOW}sudo apt-get install -y ${MISSING_DEPS[*]} tesseract-ocr-chi-sim tesseract-ocr-eng${NC}"
fi
if [ ${#MISSING_PY_PKGS[@]} -gt 0 ]; then
echo ""
echo "Python 包:"
echo -e "${YELLOW}pip3 install --user ${MISSING_PY_PKGS[*]}${NC}"
fi
fi
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "使用方法:"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " export KB_BASE_PATH=\"~/work/knowledge-base\"
echo " python3 kb_workflow.py --check --url \"[URL]\""
echo ""
kb-workflow/kb_workflow.py
+337
View File
@@ -0,0 +1,337 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
KB Workflow - Knowledge Base Workflow Orchestrator
知识库工作流辅助工具函数。
此模块提供查重、文件操作、Git 管理等工具函数，
供主代理在执行 SKILL.md 中定义的 workflow 时调用。
"""
import os
import sys
import json
import argparse
import subprocess
import shutil
import re
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Any
# ==================== 子技能映射 ====================
METADATA_AGENTS = {
"bilibili": "kb-bilibili-metadata",
"zhihu": "kb-zhihu-metadata",
}
CONTENT_AGENTS = {
"bilibili": "kb-bilibili-content",
}
# ==================== 工具函数 ====================
def extract_platform(filename: str) -> str:
"""从文件名提取平台（第一个下划线之前）"""
match = re.match(r'^([a-zA-Z0-9]+)_(.+)$', filename)
if match:
return match.group(1)
return "unknown"
def extract_base_name(filename: str) -> str:
"""从文件名提取基础名称（去掉扩展名）"""
return Path(filename).stem
def get_knowledge_base_path() -> Path:
"""获取知识库根目录"""
base_path_str = os.environ.get('KB_BASE_PATH', '')
if base_path_str:
return Path(base_path_str).expanduser()
return Path.home() / "work" / "knowledge-base"
def ensure_directories(base_path: Path = None) -> Dict[str, Path]:
"""确保目录存在"""
if base_path is None:
base_path = get_knowledge_base_path()
dirs = {
"base": base_path,
"pending": base_path / "pending",
"metadata": base_path / "metadata",
"content": base_path / "content",
"content_downloads": base_path / "content" / "downloads",
"analyzed": base_path / "analyzed",
"archive": base_path / "archive",
"archive_subscriptions": base_path / "archive" / "subscriptions",
}
for name, dir_path in dirs.items():
if isinstance(dir_path, Path):
dir_path.mkdir(parents=True, exist_ok=True)
return dirs
def check_duplicate(base_path: Path = None, url: str = None, platform: str = None, content_id: str = None) -> Dict[str, Any]:
"""查重检查"""
if base_path is None:
base_path = get_knowledge_base_path()
metadata_dir = base_path / "metadata"
archive_dir = base_path / "archive"
# 从URL提取ID
if url and not content_id:
bv_match = re.search(r'BV[a-zA-Z0-9]+', url)
content_id = bv_match.group() if bv_match else None
result = {"duplicate": False, "status": None, "files": []}
if metadata_dir.exists():
for f in metadata_dir.iterdir():
if f.is_file() and content_id and content_id in f.name:
result["duplicate"] = True
result["status"] = "metadata"
result["files"].append(str(f))
if archive_dir.exists():
for date_dir in archive_dir.iterdir():
if date_dir.is_dir():
for f in date_dir.iterdir():
if f.is_file() and content_id and content_id in f.name:
result["duplicate"] = True
result["status"] = "archived"
result["files"].append(str(f))
return result
def archive_and_git(
analyzed_file: str,
base_path: Path = None,
git_root: Path = None,
git_user: str = None,
git_email: str = None,
git_remote: str = None
) -> Dict[str, Any]:
"""归档并提交Git"""
if base_path is None:
base_path = get_knowledge_base_path()
if git_root is None:
git_root = base_path / "archive"
if git_user is None:
git_user = os.environ.get('KB_GIT_USER', '')
if git_email is None:
git_email = os.environ.get('KB_GIT_EMAIL', '')
if git_remote is None:
git_remote = os.environ.get('KB_GIT_REMOTE', '')
result = {
"success": False,
"archive_path": "",
"git_commit": False,
"git_push": None,
"error": None
}
try:
# 1. 检查并初始化 Git 仓库
git_dir = git_root / ".git"
if not git_dir.exists():
# 初始化仓库
subprocess.run(['git', 'init'], cwd=str(git_root), capture_output=True)
subprocess.run(['git', 'config', 'user.name', git_user], cwd=str(git_root))
subprocess.run(['git', 'config', 'user.email', git_email], cwd=str(git_root))
# 关联远端仓库
if git_remote:
subprocess.run(['git', 'remote', 'add', 'origin', git_remote], cwd=str(git_root), capture_output=True)
# 拉取远端分支
subprocess.run(['git', 'fetch', 'origin'], cwd=str(git_root), capture_output=True)
# 设置 main 分支并拉取
subprocess.run(['git', 'branch', '--set-upstream-to=origin/main', 'main'], cwd=str(git_root), capture_output=True)
subprocess.run(['git', 'pull', 'origin', 'main', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
else:
# 已有仓库，检查远端是否配置
if git_remote:
remote_check = subprocess.run(['git', 'remote', '-v'], cwd=str(git_root), capture_output=True, text=True)
if 'origin' not in remote_check.stdout:
subprocess.run(['git', 'remote', 'add', 'origin', git_remote], cwd=str(git_root), capture_output=True)
# 拉取远端最新代码
subprocess.run(['git', 'fetch', 'origin'], cwd=str(git_root), capture_output=True)
# 尝试拉取 main 分支，如果失败则尝试 master
pull_result = subprocess.run(['git', 'pull', 'origin', 'main', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
if pull_result.returncode != 0:
subprocess.run(['git', 'pull', 'origin', 'master', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
# 2. 创建归档目录
date_str = datetime.now().strftime('%Y-%m-%d')
date_dir = base_path / "archive" / date_str
date_dir.mkdir(parents=True, exist_ok=True)
# 3. 复制分析报告
src_file = Path(analyzed_file)
archive_file = date_dir / src_file.name
shutil.copy(src_file, archive_file)
result["archive_path"] = str(archive_file)
# 4. 复制元数据
meta_file = base_path / "metadata" / f"{src_file.stem}.meta.json"
if meta_file.exists():
shutil.copy(meta_file, date_dir / meta_file.name)
# 5. Git 提交并推送
if (git_root / ".git").exists():
# 添加所有变更
subprocess.run(['git', 'add', '.'], cwd=str(git_root), capture_output=True)
# 检查是否有变更需要提交
status_result = subprocess.run(['git', 'status', '--porcelain'], cwd=str(git_root), capture_output=True, text=True)
if status_result.stdout.strip():
commit_msg = f"Archive: {src_file.name} - {datetime.now().isoformat()}"
commit_result = subprocess.run(
['git', 'commit', '-m', commit_msg],
cwd=str(git_root), capture_output=True, text=True
)
result["git_commit"] = commit_result.returncode == 0
# 推送到远端
if git_remote and commit_result.returncode == 0:
push_result = subprocess.run(
['git', 'push', 'origin', 'main'],
cwd=str(git_root), capture_output=True, text=True, timeout=60
)
if push_result.returncode != 0:
# 如果 main 分支推送失败，尝试 master
push_result = subprocess.run(
['git', 'push', 'origin', 'master'],
cwd=str(git_root), capture_output=True, text=True, timeout=60
)
result["git_push"] = {"success": push_result.returncode == 0, "output": push_result.stdout}
result["success"] = True
except Exception as e:
result["error"] = str(e)
return result
def generate_summary(url: str, stages: List[Dict], final_file: str, archive_result: Dict = None) -> str:
"""生成处理完成汇报"""
# 提取视频标题（如果有）
title_match = re.search(r'com/video/([^?]+)', url)
title = title_match.group(1) if title_match else "未知"
summary = f"""
✅ **处理完成！**
**视频:** {title}
**处理阶段:**
"""
for stage in stages:
stage_name = stage.get('stage', 'unknown')
emoji = "✓" if stage.get('success') else "○"
summary += f"{emoji} {stage_name}\n"
if final_file:
summary += f"\n**最终文件:** `{final_file}`"
if archive_result:
if archive_result.get('git_commit'):
summary += f"\n**Git:** 已提交"
if archive_result.get('git_push', {}).get('success'):
summary += " ✓"
elif archive_result.get('git_push'):
summary += " ✗"
return summary
# ==================== CLI ====================
def main():
"""CLI入口 - 用于命令行测试"""
parser = argparse.ArgumentParser(
description='KB Workflow - Knowledge Base Workflow Orchestrator',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
使用示例:
# 查重
python kb_workflow.py --check --url "https://www.bilibili.com/video/BVxxx"
# 归档
python kb_workflow.py --archive "/path/to/analyzed.md"
"""
)
parser.add_argument('--check', action='store_true', help='查重')
parser.add_argument('--url', type=str, help='要检查的URL')
parser.add_argument('--archive-file', type=str, help='归档的文件路径')
parser.add_argument('--base-path', type=str, help='获取知识库根目录')
parser.add_argument('--pending', action='store_true', help='获取url解析目录')
parser.add_argument('--metadata', action='store_true', help='获取元数据目录')
parser.add_argument('--content', action='store_true', help='获取内容目录')
parser.add_argument('--content_downloads', action='store_true', help='获取主内容下载目录')
parser.add_argument('--analyzed', action='store_true', help='获取深度分析目录')
parser.add_argument('--archive_dir', action='store_true', help='获取归档目录')
parser.add_argument('--archive_subscriptions', action='store_true', help='获取订阅信息目录')
parser.add_argument('--extract_platform', type=str, help='要提取平台名称的文件名')
args = parser.parse_args()
base_path = Path(args.base_path) if args.base_path else None
dirs = ensure_directories(base_path)
if args.check and args.url:
result = check_duplicate(base_path, args.url)
print(json.dumps(result, ensure_ascii=False, indent=2))
elif args.archive_file:
result = archive_and_git(args.archive_file, base_path)
print(json.dumps(result, ensure_ascii=False, indent=2))
elif args.pending:
print(json.dumps({"path": str(dirs["pending"])}, ensure_ascii=False, indent=2))
elif args.metadata:
print(json.dumps({"path": str(dirs["metadata"])}, ensure_ascii=False, indent=2))
elif args.content:
print(json.dumps({"path": str(dirs["content"])}, ensure_ascii=False, indent=2))
elif args.content_downloads:
print(json.dumps({"path": str(dirs["content_downloads"])}, ensure_ascii=False, indent=2))
elif args.analyzed:
print(json.dumps({"path": str(dirs["analyzed"])}, ensure_ascii=False, indent=2))
elif args.archive_dir:
print(json.dumps({"path": str(dirs["archive"])}, ensure_ascii=False, indent=2))
elif args.archive_subscriptions:
print(json.dumps({"path": str(dirs["archive_subscriptions"])}, ensure_ascii=False, indent=2))
elif args.extract_platform:
platform = extract_platform(args.extract_platform)
print(json.dumps({"platform": platform}, ensure_ascii=False, indent=2))
elif args.base_path:
print(json.dumps({"path": str(dirs["base"])}, ensure_ascii=False, indent=2))
else:
parser.print_help()
if __name__ == '__main__':
main()
kb-zhihu-metadata/SKILL.md
+44
View File
@@ -0,0 +1,44 @@
# KB Zhihu Metadata Skill
**kb-zhihu-metadata** - Extract Zhihu question/answer/user metadata.
## Features
- Question metadata extraction
- Answer metadata extraction
- User information extraction
- JSON output format
- Markdown conversion support
## Commands
```bash
# Question metadata
python zhihu_metadata.py --id "266309957" --type question
# Answer metadata
python zhihu_metadata.py --id "2003427891925509507" --type answer
```
## Output
```json
{
"success": true,
"data": {
"platform": "zhihu",
"type": "question",
"id": "266309957",
"title": "问题标题",
"author": "作者名字",
"answer_count": 50,
"followers": 1000
}
}
```
## Installation
```bash
cp -r kb-zhihu-metadata/ ~/skills/
```
kb-zhihu-metadata/markdown_converter.py
+17
View File
@@ -0,0 +1,17 @@
from markdownify import MarkdownConverter as _MarkdownConverter
class MarkdownConverter:
"""HTML 转 Markdown 转换器，保留完整格式"""
def __init__(self):
self.converter = _MarkdownConverter(
heading_style="ATX",
bullets="*,-",
code_blocks=True,
tables=True,
strip=["script", "style", "nav", "footer", "aside"]
)
def convert(self, html: str) -> str:
"""转换 HTML 为 Markdown"""
return self.converter.convert(html)
kb-zhihu-metadata/zhihu_extractor.py
+335
View File
@@ -0,0 +1,335 @@
"""
知乎 Playwright 内容提取器
使用 Playwright 浏览器自动化工具抓取知乎内容
"""
import re
import sys
import os
from typing import Dict, Optional, Any
from urllib.parse import urlparse
# 添加当前目录到 Python 路径以支持绝对导入
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
try:
from .markdown_converter import MarkdownConverter
except ImportError:
from markdown_converter import MarkdownConverter
class PlaywrightBrowser:
"""Playwright 浏览器管理器"""
def __init__(self, headless: bool = True, timeout: int = 30):
self.headless = headless
self.timeout = timeout * 1000
self.browser = None
self.context = None
self.page = None
def start(self):
"""启动浏览器"""
from playwright.sync_api import sync_playwright
self.playwright = sync_playwright().start()
self.browser = self.playwright.chromium.launch(headless=self.headless)
self.context = self.browser.new_context(
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
)
self.page = self.context.new_page()
self.page.set_default_timeout(self.timeout)
def close(self):
"""关闭浏览器"""
if self.page:
self.page.close()
if self.context:
self.context.close()
if self.browser:
self.browser.close()
if hasattr(self, 'playwright'):
self.playwright.stop()
def goto(self, url: str, wait_until: str = "networkidle"):
"""导航到指定URL"""
return self.page.goto(url, wait_until=wait_until)
def wait_for_selector(self, selector: str, timeout: int = None):
"""等待选择器出现"""
timeout_ms = timeout * 1000 if timeout else self.timeout
return self.page.wait_for_selector(selector, timeout=timeout_ms)
def query_selector(self, selector: str):
"""查询选择器"""
return self.page.query_selector(selector)
def query_selector_all(self, selector: str):
"""查询所有匹配的选择器"""
return self.page.query_selector_all(selector)
def evaluate(self, script: str):
"""执行 JavaScript 代码"""
return self.page.evaluate(script)
class ZhihuPlaywrightExtractor:
"""知乎内容提取器 - 使用 Playwright"""
URL_PATTERNS = {
'zvideo': r'zhihu\.com/zvideo/(\d+)',
'zhuanlan': r'zhuanlan\.zhihu\.com/p/(\d+)',
}
def __init__(self, headless: bool = True, timeout: int = 30):
"""
初始化提取器
Args:
headless: 是否使用无头浏览器
timeout: 超时时间（秒）
"""
self.browser = PlaywrightBrowser(headless=headless, timeout=timeout)
self.markdown_converter = MarkdownConverter()
def identify_url_type(self, url: str) -> Optional[str]:
"""
识别 URL 类型
Returns:
'answer', 'article', 'question' 或 None
"""
for url_type, pattern in self.URL_PATTERNS.items():
if re.search(pattern, url):
return url_type
return None
def extract_id(self, url: str) -> Optional[str]:
"""
从 URL 中提取内容 ID
Returns:
内容 ID 或 None
"""
url_type = self.identify_url_type(url)
if url_type:
pattern = self.URL_PATTERNS[url_type]
match = re.search(pattern, url)
if match:
return match.group(1)
return None
def _extract_text_content(self, element) -> str:
"""提取元素的文本内容"""
if element is None:
return ''
return element.inner_text() if hasattr(element, 'inner_text') else ''
def _extract_html_content(self, element) -> str:
"""提取元素的 HTML 内容"""
if element is None:
return ''
return element.inner_html() if hasattr(element, 'inner_html') else ''
def _extract_zvideo_content(self) -> Dict[str, Any]:
"""提取知乎视频内容"""
result = {
'type': 'zvideo',
'title': '',
'author': '',
'content': '',
'content_markdown': '',
'publish_time': '',
'vote_up': 0,
}
# 打印页面标题用于调试
page_title = self.browser.page.title()
print(f"页面标题: {page_title}")
# 提取视频标题
title_elem = self.browser.query_selector('.VideoPlaylistInfo-title')
if title_elem:
result['title'] = self._extract_text_content(title_elem)
# 备用：尝试其他选择器
if not result['title']:
title_elem = self.browser.query_selector('h1')
if title_elem:
result['title'] = self._extract_text_content(title_elem)
# 提取作者信息
author_elem = self.browser.query_selector('.AuthorInfo-name')
if author_elem:
result['author'] = self._extract_text_content(author_elem)
# 提取视频描述
desc_elem = self.browser.query_selector('.Video-desc')
if desc_elem:
html_content = self._extract_html_content(desc_elem)
result['content'] = html_content
result['content_markdown'] = self.markdown_converter.convert(html_content)
# 提取发布时间
time_elem = self.browser.query_selector('.Video-meta time')
if time_elem:
result['publish_time'] = time_elem.get_attribute('datetime') or self._extract_text_content(time_elem)
# 提取点赞数
vote_elem = self.browser.query_selector('.VideoLikeButton-count')
if vote_elem:
vote_text = self._extract_text_content(vote_elem)
try:
result['vote_up'] = int(vote_text.replace(',', ''))
except (ValueError, AttributeError):
pass
return result
def _extract_zhuanlan_content(self) -> Dict[str, Any]:
"""提取知乎专栏文章内容"""
result = {
'type': 'zhuanlan',
'title': '',
'author': '',
'content': '',
'content_markdown': '',
'publish_time': '',
'vote_up': 0,
}
# 打印页面标题用于调试
page_title = self.browser.page.title()
print(f"页面标题: {page_title}")
# 提取文章标题
title_elem = self.browser.query_selector('.Post-Title')
if title_elem:
result['title'] = self._extract_text_content(title_elem)
# 备用
if not result['title']:
title_elem = self.browser.query_selector('h1')
if title_elem:
result['title'] = self._extract_text_content(title_elem)
# 提取作者信息
author_elem = self.browser.query_selector('.AuthorInfo-name')
if author_elem:
result['author'] = self._extract_text_content(author_elem)
# 提取文章内容
content_elem = self.browser.query_selector('.Post-RichText')
if content_elem:
html_content = self._extract_html_content(content_elem)
result['content'] = html_content
result['content_markdown'] = self.markdown_converter.convert(html_content)
# 提取发布时间
time_elem = self.browser.query_selector('.ContentItem-time')
if time_elem:
result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem)
# 提取点赞数
vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label')
if vote_elem:
vote_text = self._extract_text_content(vote_elem)
try:
result['vote_up'] = int(vote_text.replace(',', ''))
except (ValueError, AttributeError):
pass
return result
def extract(self, url: str) -> Optional[Dict[str, Any]]:
"""
从 URL 提取知乎内容
Args:
url: 知乎 URL
Returns:
包含提取内容的字典，结构如下:
{
'type': 'zvideo' | 'zhuanlan',
'title': str,
'author': str,
'content': str (HTML),
'content_markdown': str (Markdown),
'publish_time': str,
'vote_up': int,
}
如果提取失败返回 None
"""
try:
# 识别 URL 类型
url_type = self.identify_url_type(url)
if not url_type:
print(f"无法识别 URL 类型: {url}")
return None
print(f"URL 类型: {url_type}")
print(f"启动浏览器...")
# 启动浏览器
self.browser.start()
print(f"访问页面: {url}")
# 访问页面
self.browser.goto(url, wait_until="domcontentloaded")
print("等待页面加载...")
import time
time.sleep(3) # 等待 JavaScript 执行
# 根据类型提取内容
if url_type == 'zvideo':
result = self._extract_zvideo_content()
elif url_type == 'zhuanlan':
result = self._extract_zhuanlan_content()
else:
print(f"不支持的类型: {url_type}")
return None
# 添加额外信息
result['url'] = url
result['content_id'] = self.extract_id(url)
print(f"提取成功! 标题: {result.get('title', '')[:50]}...")
return result
except Exception as e:
print(f"提取知乎内容时出错: {e}")
import traceback
traceback.print_exc()
return None
finally:
# 关闭浏览器
self.browser.close()
def __enter__(self):
"""上下文管理器入口"""
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""上下文管理器出口"""
self.browser.close()
return False
# 便捷函数
def extract_zhihu_content(url: str, headless: bool = True, timeout: int = 30) -> Optional[Dict[str, Any]]:
"""
便捷函数：提取知乎内容
Args:
url: 知乎 URL
headless: 是否使用无头浏览器
timeout: 超时时间（秒）
Returns:
提取的内容字典
"""
with ZhihuPlaywrightExtractor(headless=headless, timeout=timeout) as extractor:
return extractor.extract(url)
kb-zhihu-metadata/zhihu_metadata.py
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env python3
"""
知乎元数据获取工具
仅支持两种内容类型：
- zvideo: 知乎视频 (https://www.zhihu.com/zvideo/xxx)
- zhuanlan: 知乎专栏文章 (https://zhuanlan.zhihu.com/p/xxx)
"""
import argparse
import json
import sys
import os
from typing import Dict, Optional, Any
# 添加当前目录到 Python 路径
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
from zhihu_extractor import ZhihuPlaywrightExtractor
def extract_zvideo(video_id: str, headless: bool = True) -> Dict[str, Any]:
"""
获取知乎视频元数据
Args:
video_id: 视频 ID (数字字符串)
headless: 是否使用无头浏览器
Returns:
视频元数据字典
"""
url = f"https://www.zhihu.com/zvideo/{video_id}"
return _extract_metadata(url, "zvideo", headless)
def extract_zhuanlan(article_id: str, headless: bool = True) -> Dict[str, Any]:
"""
获取知乎专栏文章元数据
Args:
article_id: 文章 ID (数字字符串)
headless: 是否使用无头浏览器
Returns:
文章元数据字典
"""
url = f"https://zhuanlan.zhihu.com/p/{article_id}"
return _extract_metadata(url, "zhuanlan", headless)
def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, Any]:
"""
内部函数：提取元数据
Args:
url: 知乎 URL
content_type: 内容类型 (zvideo/zhuanlan)
headless: 是否使用无头浏览器
Returns:
元数据字典
"""
extractor = ZhihuPlaywrightExtractor(headless=headless)
result = extractor.extract(url)
if result:
# 标准化输出格式
metadata = {
"type": content_type,
"id": result.get("content_id", ""),
"url": result.get("url", url),
"title": result.get("title", ""),
"author": result.get("author", ""),
"publish_time": result.get("publish_time", ""),
"vote_up": result.get("vote_up", 0),
"content_markdown": result.get("content_markdown", ""),
"raw_html": result.get("content", ""),
}
return metadata
else:
return {
"type": content_type,
"error": "Failed to extract content",
"url": url
}
def main():
"""命令行入口"""
parser = argparse.ArgumentParser(
description="知乎元数据获取工具",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
支持的类型:
zvideo - 知乎视频 (https://www.zhihu.com/zvideo/123456)
zhuanlan - 知乎专栏文章 (https://zhuanlan.zhihu.com/p/123456)
示例:
python zhihu_metadata.py --id "123456" --type zvideo
python zhihu_metadata.py --id "123456" --type zhuanlan
"""
)
parser.add_argument(
"--id", "-i",
required=True,
help="知乎内容 ID (视频ID或文章ID)"
)
parser.add_argument(
"--type", "-t",
required=True,
choices=["zvideo", "zhuanlan"],
help="内容类型: zvideo=视频, zhuanlan=专栏文章"
)
parser.add_argument(
"--output", "-o",
default=None,
help="输出文件路径 (默认: stdout)"
)
parser.add_argument(
"--headless",
action="store_true",
default=True,
help="使用无头浏览器模式 (默认: True)"
)
args = parser.parse_args()
# 提取元数据
if args.type == "zvideo":
result = extract_zvideo(args.id, headless=args.headless)
else: # zhuanlan
result = extract_zhuanlan(args.id, headless=args.headless)
# 输出结果
json_output = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
with open(args.output, 'w', encoding='utf-8') as f:
f.write(json_output)
print(f"结果已保存到: {args.output}")
else:
print(json_output)
if __name__ == "__main__":
main()