feat: Add KB knowledge base series skills

- kb-link-parser: Multi-platform URL parser (Bilibili, YouTube, Zhihu)
- kb-bilibili-metadata: Bilibili video/article/up主 metadata extractor
- kb-zhihu-metadata: Zhihu question/answer/user metadata extractor
- kb-bilibili-content: Bilibili video download and content extraction
- kb-ocr-processor: OCR text extraction from video frames
- kb-deep-analyzer: AI-powered deep content analysis
- kb-workflow: Workflow orchestration (inbox, archive, subscriptions, report)

Also add:
- ARCHITECTURE_FLOWCHART.md: System architecture diagrams
- DETAILED_DESIGN.md: Complete design documentation
This commit is contained in:
OpenClaw Agent
2026-02-10 12:18:52 +08:00
parent 2f5a269866
commit 5507b1d986
28 changed files with 7715 additions and 0 deletions
+46
View File
@@ -0,0 +1,46 @@
# KB Bilibili Metadata Skill
**kb-bilibili-metadata** - Extract Bilibili video/article/up主 metadata.
## Features
- Video metadata extraction
- Article (专栏) metadata extraction
- UP主 information extraction
- JSON output format
- Playwright + HTTP API fallback
## Commands
```bash
# Video metadata
python bilibili_metadata.py --id "BV1xx411c7mD" --type video
# Article metadata
python bilibili_metadata.py --id "cv43780186" --type article
# UP主 info
python bilibili_metadata.py --id "7770724" --type up
```
## Output
```json
{
"success": true,
"data": {
"platform": "bilibili",
"type": "video",
"id": "BV1xx411c7mD",
"title": "视频标题",
"author": "UP主名字",
"stats": {"view": 100000, "like": 5000}
}
}
```
## Installation
```bash
cp -r kb-bilibili-metadata/ ~/skills/
```
+8
View File
@@ -0,0 +1,8 @@
"""
bilibili-metadata 技能包
B站元数据提取工具
"""
__version__ = "1.0.0"
__author__ = "OpenClaw"
@@ -0,0 +1,446 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
B站元数据提取器
独立版本,不依赖 obsidian-kb 结构
支持视频、专栏、UP主信息提取
"""
import asyncio
import json
import logging
import re
from datetime import datetime
from typing import Dict, Any, Optional
from urllib.parse import urlparse
logger = logging.getLogger(__name__)
# 默认请求头
DEFAULT_HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Referer': 'https://www.bilibili.com/',
'Accept': 'application/json, text/plain, */*',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}
try:
from playwright.async_api import async_playwright
PLAYWRIGHT_AVAILABLE = True
except ImportError:
PLAYWRIGHT_AVAILABLE = False
try:
from undetected_playwright import stealth_async
STEALTH_AVAILABLE = True
except ImportError:
STEALTH_AVAILABLE = False
try:
import httpx
HTTPX_AVAILABLE = True
except ImportError:
HTTPX_AVAILABLE = False
class BilibiliExtractor:
"""
B站元数据提取器
支持:
- 视频信息提取(标题、作者、统计数据、字幕等)
- 专栏文章提取
- UP主信息提取
"""
# 正则模式
BV_PATTERN = re.compile(r'BV[a-zA-Z0-9]{10}')
CV_PATTERN = re.compile(r'cv(\d+)')
URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/video/(BV[a-zA-Z0-9]{10})')
ARTICLE_URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/read/(cv\d+)')
UP_URL_PATTERN = re.compile(r'https?://space\.bilibili\.com/(\d+)')
def __init__(self, headless: bool = True, timeout: int = 30):
"""
初始化提取器
Args:
headless: 是否使用无头模式(浏览器)
timeout: 超时时间(秒)
"""
self.headless = headless
self.timeout = timeout * 1000 # 转换为毫秒
def extract_video_metadata(self, bvid: str) -> Dict[str, Any]:
"""
提取视频元数据
Args:
bvid: B站视频BV号(如 BV1xx411c7mD)
Returns:
Dict containing metadata, stats, description, subtitle
"""
# 验证并解析BV号
bvid = self._resolve_bvid(bvid)
if not bvid:
raise ValueError(f"无效的BV号: {bvid}")
# 使用异步方法提取
return asyncio.run(self._extract_video_async(bvid))
def extract_article_metadata(self, cvid: str) -> Dict[str, Any]:
"""
提取专栏元数据
Args:
cvid: B站专栏CV号(如 cv43780186)
Returns:
Dict containing article metadata
"""
# 解析CV号
cvid = self._resolve_cvid(cvid)
if not cvid:
raise ValueError(f"无效的CV号: {cvid}")
return asyncio.run(self._extract_article_async(cvid))
def extract_up_metadata(self, mid: str) -> Dict[str, Any]:
"""
提取UP主信息
Args:
mid: B站用户ID(如 7770724)
Returns:
Dict containing UP主信息
"""
# 解析MID
mid = self._resolve_mid(mid)
if not mid:
raise ValueError(f"无效的MID: {mid}")
return asyncio.run(self._extract_up_async(mid))
def _resolve_bvid(self, source_id: str) -> Optional[str]:
"""解析并返回BV号"""
source = source_id.strip()
# 直接BV号
if self.BV_PATTERN.fullmatch(source):
return source
# 完整URL
match = self.URL_PATTERN.match(source)
if match:
return match.group(1)
return None
def _resolve_cvid(self, source_id: str) -> Optional[str]:
"""解析并返回CV号"""
source = source_id.strip()
# 直接CV号(带cv前缀)
if source.startswith('cv'):
match = self.CV_PATTERN.match(source)
if match:
return source
# 纯数字
if source.isdigit():
return f"cv{source}"
# 完整URL
match = self.ARTICLE_URL_PATTERN.match(source)
if match:
return match.group(1)
return None
def _resolve_mid(self, source_id: str) -> Optional[str]:
"""解析并返回MID"""
source = source_id.strip()
# 纯数字
if source.isdigit():
return source
# 完整URL
match = self.UP_URL_PATTERN.match(source)
if match:
return match.group(1)
return None
async def _extract_video_async(self, bvid: str) -> Dict[str, Any]:
"""异步提取视频信息"""
url = f"https://www.bilibili.com/video/{bvid}"
result = {
'metadata': {},
'stats': {},
'subtitle': '',
'description': ''
}
if not PLAYWRIGHT_AVAILABLE:
# 使用 HTTP API
return await self._extract_video_by_api(bvid)
async with async_playwright() as playwright:
try:
browser = await playwright.chromium.launch(
headless=self.headless,
args=[
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-blink-features=AutomationControlled',
'--disable-dev-shm-usage',
]
)
context = await browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
locale='zh-CN',
)
if STEALTH_AVAILABLE:
await stealth_async(context)
page = await context.new_page()
await page.goto(url, wait_until='domcontentloaded', timeout=self.timeout)
await asyncio.sleep(2)
content = await page.content()
video_data = await self._try_extract_video_methods(page, content, bvid)
if video_data:
result['metadata'] = {
'title': video_data.get('title', ''),
'author': video_data.get('author', ''),
'bvid': video_data.get('bvid', ''),
'aid': video_data.get('aid', ''),
'mid': video_data.get('mid', ''),
'pubtime': video_data.get('pubtime', ''),
'duration': video_data.get('duration', ''),
'url': url,
'tags': video_data.get('tags', ''),
}
result['stats'] = {
'view': video_data.get('view', 0),
'danmaku': video_data.get('danmaku', 0),
'reply': video_data.get('reply', 0),
'favorite': video_data.get('favorite', 0),
'coin': video_data.get('coin', 0),
'share': video_data.get('share', 0),
'like': video_data.get('like', 0),
}
result['description'] = video_data.get('desc', '')
await browser.close()
except Exception as e:
logger.error(f"视频提取错误: {e}")
# 回退到 API
return await self._extract_video_by_api(bvid)
return result
async def _extract_video_by_api(self, bvid: str) -> Dict[str, Any]:
"""使用B站API提取视频信息"""
result = {
'metadata': {},
'stats': {},
'subtitle': '',
'description': ''
}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取视频信息
response = await client.get('https://api.bilibili.com/x/web-interface/view', params={'bvid': bvid})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
video_data = data.get('data', {})
result['metadata'] = {
'title': video_data.get('title', ''),
'author': video_data.get('owner', {}).get('name', ''),
'bvid': video_data.get('bvid', ''),
'aid': video_data.get('aid', ''),
'mid': video_data.get('owner', {}).get('mid', ''),
'pubtime': datetime.fromtimestamp(video_data.get('pubdate', 0)).strftime('%Y-%m-%d %H:%M:%S') if video_data.get('pubdate') else '',
'duration': self._format_duration(video_data.get('duration', 0)),
'url': f"https://www.bilibili.com/video/{bvid}",
'tags': ', '.join([t.get('tag_name', '') for t in video_data.get('tag', [])]),
}
stat = video_data.get('stat', {})
result['stats'] = {
'view': stat.get('view', 0),
'danmaku': stat.get('danmaku', 0),
'reply': stat.get('reply', 0),
'favorite': stat.get('favorite', 0),
'coin': stat.get('coin', 0),
'share': stat.get('share', 0),
'like': stat.get('like', 0),
}
result['description'] = video_data.get('desc', '')
except Exception as e:
logger.error(f"API提取错误: {e}")
return result
async def _try_extract_video_methods(self, page, content: str, bvid: str) -> Dict[str, Any]:
"""尝试多种方法提取视频信息"""
video_data = {}
# 方法1: 从 __INITIAL_STATE__
try:
initial_state = await page.evaluate('() => typeof window.__INITIAL_STATE__ !== "undefined" ? window.__INITIAL_STATE__ : null')
if initial_state:
video_data = self._parse_video_state(initial_state)
if video_data and video_data.get('title'):
return video_data
except:
pass
# 方法2: 使用 API(更可靠)
return await self._extract_video_by_api(bvid)
def _parse_video_state(self, data: Dict) -> Dict[str, Any]:
"""解析 __INITIAL_STATE__ 数据"""
video_data = {}
try:
video = data.get('videoData', {})
video_data['title'] = video.get('title', '')
video_data['bvid'] = video.get('bvid', '')
video_data['aid'] = video.get('aid', '')
owner = video.get('owner', {})
video_data['author'] = owner.get('name', '')
video_data['mid'] = owner.get('mid', '')
stat = video.get('stat', {})
video_data['view'] = stat.get('view', 0)
video_data['danmaku'] = stat.get('danmaku', 0)
video_data['reply'] = stat.get('reply', 0)
video_data['favorite'] = stat.get('favorite', 0)
video_data['coin'] = stat.get('coin', 0)
video_data['share'] = stat.get('share', 0)
video_data['like'] = stat.get('like', 0)
pubdate = video.get('pubdate', 0)
video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else ''
video_data['duration'] = self._format_duration(video.get('duration', 0))
video_data['desc'] = video.get('desc', '')
video_data['tags'] = ', '.join([t.get('tag_name', '') for t in video.get('tag', [])])
except:
pass
return video_data
async def _extract_article_async(self, cvid: str) -> Dict[str, Any]:
"""异步提取专栏信息"""
result = {}
if not HTTPX_AVAILABLE:
return {'error': 'httpx 未安装'}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取专栏信息
response = await client.get('https://api.bilibili.com/x/article/viewinfo', params={'id': cvid[2:]})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
article = data.get('data', {})
result = {
'cvid': cvid,
'title': article.get('title', ''),
'author': article.get('author_name', ''),
'mid': article.get('author_mid', ''),
'publish_time': datetime.fromtimestamp(article.get('publish_time', 0)).strftime('%Y-%m-%d %H:%M:%S') if article.get('publish_time') else '',
'view_count': article.get('stats', {}).get('view', 0),
'like_count': article.get('stats', {}).get('like', 0),
'coin_count': article.get('stats', {}).get('coin', 0),
'share_count': article.get('stats', {}).get('share', 0),
'reply_count': article.get('stats', {}).get('reply', 0),
'url': f"https://www.bilibili.com/read/{cvid}",
'summary': article.get('summary', ''),
'word_count': article.get('words', 0),
}
except Exception as e:
logger.error(f"专栏提取错误: {e}")
result = {'error': str(e)}
return result
async def _extract_up_async(self, mid: str) -> Dict[str, Any]:
"""异步提取UP主信息"""
result = {}
if not HTTPX_AVAILABLE:
return {'error': 'httpx 未安装'}
try:
async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client:
# 获取UP主信息
response = await client.get('https://api.bilibili.com/x/space/acc/info', params={'mid': mid})
if response.status_code == 200:
data = response.json()
if data.get('code') == 0:
up = data.get('data', {})
result = {
'mid': mid,
'name': up.get('name', ''),
'sex': up.get('sex', ''),
'face': up.get('face', ''),
'sign': up.get('sign', ''),
'level': up.get('level', 0),
'vip_type': up.get('vip', {}).get('type', 0),
'vip_status': up.get('vip', {}).get('status', 0),
'following_count': up.get('following', 0),
'follower_count': up.get('follower', 0),
'url': f"https://space.bilibili.com/{mid}",
}
# 获取UP主视频统计
stats_response = await client.get('https://api.bilibili.com/x/space/navnum', params={'mid': mid})
if stats_response.status_code == 200:
stats_data = stats_response.json()
if stats_data.get('code') == 0:
result['video_count'] = stats_data.get('data', {}).get('video', 0)
except Exception as e:
logger.error(f"UP主提取错误: {e}")
result = {'error': str(e)}
return result
def _format_duration(self, seconds: int) -> str:
"""格式化时长(秒 -> HH:MM:SS)"""
if not seconds:
return ''
hours = seconds // 3600
minutes = (seconds % 3600) // 60
secs = seconds % 60
if hours > 0:
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
else:
return f"{minutes:02d}:{secs:02d}"
# 测试代码
if __name__ == '__main__':
extractor = BilibiliExtractor()
print("=" * 60)
print("B站元数据提取器测试")
print("=" * 60)
# 测试视频
print("\n测试视频提取: BV1MnFNz5EDL")
result = extractor.extract_video_metadata('BV1MnFNz5EDL')
print(json.dumps(result, ensure_ascii=False, indent=2))
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
B站元数据提取 CLI 工具
支持命令行调用,输出 JSON 格式数据
"""
import argparse
import json
import sys
from typing import Dict, Any
from bilibili_extractor import BilibiliExtractor
def format_number(num: int) -> str:
"""格式化数字"""
if num >= 100000000:
return f"{num / 100000000:.1f}亿"
elif num >= 10000:
return f"{num / 10000:.1f}万"
else:
return f"{num:,}"
def extract_video(extractor: BilibiliExtractor, bvid: str, output_file: str = None) -> Dict[str, Any]:
"""提取视频元数据"""
result = extractor.extract_video_metadata(bvid)
# 如果需要输出到文件
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def extract_article(extractor: BilibiliExtractor, cvid: str, output_file: str = None) -> Dict[str, Any]:
"""提取专栏元数据"""
result = extractor.extract_article_metadata(cvid)
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def extract_up(extractor: BilibiliExtractor, mid: str, output_file: str = None) -> Dict[str, Any]:
"""提取UP主信息"""
result = extractor.extract_up_metadata(mid)
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(result, f, ensure_ascii=False, indent=2)
return result
def main():
"""主函数"""
parser = argparse.ArgumentParser(
description='B站元数据提取工具',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
# 提取视频元数据
python bilibili_metadata.py --id "BV1xx411c7mD" --type video
# 提取专栏元数据
python bilibili_metadata.py --id "cv43780186" --type article
# 提取UP主信息
python bilibili_metadata.py --id "7770724" --type up
# 输出到文件
python bilibili_metadata.py --id "BV1xx411c7mD" --type video --output result.json
"""
)
parser.add_argument(
'--id',
required=True,
help='B站资源ID(BV号、CV号或UP主ID)'
)
parser.add_argument(
'--type',
required=True,
choices=['video', 'article', 'up'],
help='资源类型'
)
parser.add_argument(
'--output',
'-o',
help='输出文件路径(JSON格式)'
)
parser.add_argument(
'--headless',
action='store_true',
default=True,
help='使用无头模式(浏览器,默认开启)'
)
parser.add_argument(
'--timeout',
type=int,
default=30,
help='超时时间(秒,默认30)'
)
parser.add_argument(
'--pretty',
action='store_true',
default=True,
help='格式化输出(默认开启)'
)
args = parser.parse_args()
# 创建提取器
extractor = BilibiliExtractor(headless=args.headless, timeout=args.timeout)
try:
# 根据类型提取数据
if args.type == 'video':
result = extract_video(extractor, args.id, args.output)
elif args.type == 'article':
result = extract_article(extractor, args.id, args.output)
elif args.type == 'up':
result = extract_up(extractor, args.id, args.output)
else:
print(f"错误: 不支持的类型 {args.type}")
sys.exit(1)
# 输出结果
if not args.output:
if args.pretty:
print(json.dumps(result, ensure_ascii=False, indent=2))
else:
print(json.dumps(result, ensure_ascii=False))
if args.output:
print(f"结果已保存到: {args.output}")
except Exception as e:
print(f"错误: {e}", file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()