From 5507b1d98671e8ee960dd8e8e6daebc1b6267696 Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Tue, 10 Feb 2026 12:18:52 +0800 Subject: [PATCH] feat: Add KB knowledge base series skills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - kb-link-parser: Multi-platform URL parser (Bilibili, YouTube, Zhihu) - kb-bilibili-metadata: Bilibili video/article/up主 metadata extractor - kb-zhihu-metadata: Zhihu question/answer/user metadata extractor - kb-bilibili-content: Bilibili video download and content extraction - kb-ocr-processor: OCR text extraction from video frames - kb-deep-analyzer: AI-powered deep content analysis - kb-workflow: Workflow orchestration (inbox, archive, subscriptions, report) Also add: - ARCHITECTURE_FLOWCHART.md: System architecture diagrams - DETAILED_DESIGN.md: Complete design documentation --- skills/kb-bilibili-content/SKILL.md | 45 + .../kb-bilibili-content/bilibili_content.py | 452 +++++++ .../kb-bilibili-content/bilibili_extractor.py | 155 +++ .../kb-bilibili-content/video_downloader.py | 175 +++ skills/kb-bilibili-metadata/SKILL.md | 46 + skills/kb-bilibili-metadata/__init__.py | 8 + .../bilibili_extractor.py | 446 +++++++ .../kb-bilibili-metadata/bilibili_metadata.py | 150 +++ skills/kb-deep-analyzer/SKILL.md | 48 + skills/kb-deep-analyzer/ai_client.py | 514 ++++++++ skills/kb-deep-analyzer/deep_analyzer.py | 771 ++++++++++++ skills/kb-link-parser/SKILL.md | 43 + skills/kb-link-parser/link-parser.py | 387 ++++++ .../kb-link-parser/link_parser_shortlink.py | 296 +++++ skills/kb-ocr-processor/SKILL.md | 43 + skills/kb-ocr-processor/ocr_processor.py | 242 ++++ skills/kb-ocr-processor/requirements.txt | 3 + skills/kb-workflow/SKILL.md | 285 +++++ skills/kb-workflow/kb_handler.py | 427 +++++++ skills/kb-workflow/kb_pipeline.py | 489 ++++++++ skills/kb-workflow/kb_workflow.py | 674 ++++++++++ .../reports/report_20260210.json | 36 + skills/kb-zhihu-metadata/SKILL.md | 44 + .../kb-zhihu-metadata/markdown_converter.py | 17 + skills/kb-zhihu-metadata/zhihu_extractor.py | 385 ++++++ skills/kb-zhihu-metadata/zhihu_metadata.py | 258 ++++ skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md | 168 +++ skills/obsidian-kb/DETAILED_DESIGN.md | 1108 +++++++++++++++++ 28 files changed, 7715 insertions(+) create mode 100644 skills/kb-bilibili-content/SKILL.md create mode 100644 skills/kb-bilibili-content/bilibili_content.py create mode 100644 skills/kb-bilibili-content/bilibili_extractor.py create mode 100644 skills/kb-bilibili-content/video_downloader.py create mode 100644 skills/kb-bilibili-metadata/SKILL.md create mode 100644 skills/kb-bilibili-metadata/__init__.py create mode 100644 skills/kb-bilibili-metadata/bilibili_extractor.py create mode 100755 skills/kb-bilibili-metadata/bilibili_metadata.py create mode 100644 skills/kb-deep-analyzer/SKILL.md create mode 100644 skills/kb-deep-analyzer/ai_client.py create mode 100644 skills/kb-deep-analyzer/deep_analyzer.py create mode 100644 skills/kb-link-parser/SKILL.md create mode 100644 skills/kb-link-parser/link-parser.py create mode 100644 skills/kb-link-parser/link_parser_shortlink.py create mode 100644 skills/kb-ocr-processor/SKILL.md create mode 100644 skills/kb-ocr-processor/ocr_processor.py create mode 100644 skills/kb-ocr-processor/requirements.txt create mode 100644 skills/kb-workflow/SKILL.md create mode 100755 skills/kb-workflow/kb_handler.py create mode 100755 skills/kb-workflow/kb_pipeline.py create mode 100644 skills/kb-workflow/kb_workflow.py create mode 100644 skills/kb-workflow/knowledge-base/reports/report_20260210.json create mode 100644 skills/kb-zhihu-metadata/SKILL.md create mode 100644 skills/kb-zhihu-metadata/markdown_converter.py create mode 100644 skills/kb-zhihu-metadata/zhihu_extractor.py create mode 100644 skills/kb-zhihu-metadata/zhihu_metadata.py create mode 100644 skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md create mode 100644 skills/obsidian-kb/DETAILED_DESIGN.md diff --git a/skills/kb-bilibili-content/SKILL.md b/skills/kb-bilibili-content/SKILL.md new file mode 100644 index 0000000..8fb3c77 --- /dev/null +++ b/skills/kb-bilibili-content/SKILL.md @@ -0,0 +1,45 @@ +# KB Bilibili Content Skill + +**kb-bilibili-content** - Download Bilibili videos and extract content. + +## Features + +- Video download (multiple quality options) +- Subtitle extraction +- Article content extraction +- Low quality for OCR (480p default) + +## Commands + +```bash +# Download video +python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 480 + +# Extract subtitle +python bilibili_content.py --id "BV1xx411c7mD" --type subtitle + +# Get article content +python bilibili_content.py --id "cv43780186" --type article +``` + +## Output + +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "video", + "id": "BV1xx411c7mD", + "video_file": {"path": "output/BV1xx.mp4", "quality": "480p"}, + "subtitle": {"path": "output/BV1xx.srt"} + } +} +``` + +## Installation + +```bash +cp -r kb-bilibili-content/ ~/skills/ +pip install aiohttp +``` diff --git a/skills/kb-bilibili-content/bilibili_content.py b/skills/kb-bilibili-content/bilibili_content.py new file mode 100644 index 0000000..7d81341 --- /dev/null +++ b/skills/kb-bilibili-content/bilibili_content.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Bilibili Content Downloader - B站内容获取工具 + +功能: +- download_video(bvid, quality=480) - 下载低分辨率视频 +- extract_subtitle(bvid) - 提取字幕 +- extract_article(cvid) - 获取专栏全文 +- 命令行参数支持 +- JSON 输出 +""" + +import argparse +import asyncio +import json +import os +import re +import sys +from datetime import datetime +from pathlib import Path +from typing import Optional, Dict, Any, List + +import aiohttp + + +class BilibiliContent: + """B站内容获取器""" + + def __init__(self, output_dir: str = "./output"): + self.output_dir = Path(output_dir) + self.output_dir.mkdir(parents=True, exist_ok=True) + + def _get_headers(self) -> Dict[str, str]: + """获取请求头""" + return { + 'Referer': 'https://www.bilibili.com/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'application/json, text/plain, */*', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', + } + + async def _make_request(self, session, url: str, params: Dict = None) -> Optional[Dict]: + """发送API请求""" + try: + import aiohttp + async with session.get(url, params=params, headers=self._get_headers(), timeout=aiohttp.ClientTimeout(total=30)) as response: + if response.status == 200: + return await response.json() + return None + except Exception as e: + print(f"请求错误: {e}") + return None + + async def download_video(self, bvid: str, quality: int = 480) -> Dict[str, Any]: + """ + 下载B站视频 + + Args: + bvid: B站视频BV号 (如: BV1xx411c7mD) + quality: 视频质量 (默认480) + - 127: 8K + - 126:杜比 + - 125:HDR + - 120:4K + - 116:1080P60 + - 112:1080P+ + - 80:1080P + - 74:720P60 + - 64:720P + - 32:480P + - 16:360P + + Returns: + Dict: 包含状态、消息和文件路径的结果 + """ + result = { + "success": False, + "bvid": bvid, + "message": "", + "filepath": None, + "metadata": {} + } + + async with aiohttp.ClientSession() as session: + # 1. 获取视频信息 + print(f"正在获取视频信息: {bvid}") + api_url = "https://api.bilibili.com/x/web-interface/view" + data = await self._make_request(session, api_url, {'bvid': bvid}) + + if not data or data.get('code') != 0: + result["message"] = f"获取视频信息失败: {data.get('message', '未知错误')}" + return result + + video_data = data.get('data', {}) + title = video_data.get('title', '未知标题') + aid = video_data.get('aid') + cid = video_data.get('cid') + duration = video_data.get('duration', 0) + author = video_data.get('owner', {}).get('name', '未知作者') + + result["metadata"] = { + "title": title, + "author": author, + "aid": aid, + "cid": cid, + "duration": duration, + "bvid": bvid + } + + print(f"视频标题: {title}") + print(f"作者: {author}") + print(f"时长: {duration}秒") + + # 2. 获取视频流地址 + print(f"\n正在获取视频流...") + + # 选择合适的质量 + quality_map = {127: 127, 126: 126, 125: 125, 120: 120, 116: 116, 112: 112, 80: 80, 74: 74, 64: 64, 32: 32, 16: 16} + target_quality = quality_map.get(quality, 32) # 默认480P + + playurl_api = "https://api.bilibili.com/x/player/playurl" + params = { + 'bvid': bvid, + 'cid': cid, + 'qn': target_quality, + 'otype': 'json', + 'fnver': 0, + 'fnval': 16, # 支持DASH格式 + } + + playurl_data = await self._make_request(session, playurl_api, params) + + if not playurl_data or playurl_data.get('code') != 0: + result["message"] = f"获取视频流失败: {playurl_data.get('message', '未知错误')}" + return result + + play_data = playurl_data.get('data', {}) + + # 获取视频和音频流 + dash_data = play_data.get('dash') + if dash_data: + video_list = dash_data.get('video', []) + audio_list = dash_data.get('audio', []) + + if video_list: + # 选择质量最高的视频流 + video_stream = video_list[0] + video_url = video_stream.get('base_url') or video_stream.get('baseUrl') + video_size = video_list[0].get('size', 0) + + # 获取音频流 + audio_stream = audio_list[0] if audio_list else None + audio_url = audio_stream.get('base_url') or audio_stream.get('baseUrl') if audio_stream else None + + print(f"找到视频流: {video_url[:100]}...") + + # 3. 下载视频流 + print(f"\n正在下载视频...") + + # 下载视频 + async with session.get(video_url, headers=self._get_headers()) as resp: + if resp.status == 200: + video_content = await resp.read() + print(f"视频下载完成: {len(video_content) / 1024 / 1024:.2f}MB") + else: + result["message"] = f"视频下载失败: HTTP {resp.status}" + return result + + # 下载音频 + audio_content = None + if audio_url: + print(f"正在下载音频...") + async with session.get(audio_url, headers=self._get_headers()) as resp: + if resp.status == 200: + audio_content = await resp.read() + print(f"音频下载完成: {len(audio_content) / 1024 / 1024:.2f}MB") + + # 4. 合并音视频 + print(f"\n正在合并音视频...") + + safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50] + safe_title = re.sub(r'[<>:"/\\|?*]', '', safe_title) + + output_path = self.output_dir / f"{safe_title}_{bvid}" + + # 保存文件 + if video_content: + video_path = output_path.with_suffix('.mp4') + with open(video_path, 'wb') as f: + f.write(video_content) + + result["filepath"] = str(video_path) + result["success"] = True + result["message"] = f"视频已保存: {video_path}" + print(f"✅ 下载完成: {video_path}") + else: + result["message"] = "未找到视频流" + else: + # 回退到FLV格式 + durl = play_data.get('durl', []) + if durl: + video_url = durl[0].get('url') + print(f"使用FLV格式: {video_url[:100]}...") + + async with session.get(video_url, headers=self._get_headers()) as resp: + if resp.status == 200: + video_content = await resp.read() + + safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50] + safe_title = re.sub(r'[<>:"/\\|?*]', '', safe_title) + + output_path = self.output_dir / f"{safe_title}_{bvid}.flv" + with open(output_path, 'wb') as f: + f.write(video_content) + + result["filepath"] = str(output_path) + result["success"] = True + result["message"] = f"视频已保存: {output_path}" + print(f"✅ 下载完成: {output_path}") + else: + result["message"] = f"视频下载失败: HTTP {resp.status}" + else: + result["message"] = "未找到可用的视频流" + + return result + + async def extract_subtitle(self, bvid: str) -> Dict[str, Any]: + """ + 提取B站视频字幕 + + Args: + bvid: B站视频BV号 + + Returns: + Dict: 包含字幕内容的JSON结果 + """ + result = { + "success": False, + "bvid": bvid, + "subtitles": [], + "message": "" + } + + async with aiohttp.ClientSession() as session: + # 获取视频信息 + api_url = "https://api.bilibili.com/x/web-interface/view" + data = await self._make_request(session, api_url, {'bvid': bvid}) + + if not data or data.get('code') != 0: + result["message"] = f"获取视频信息失败: {data.get('message', '未知错误')}" + return result + + cid = data.get('data', {}).get('cid') + + # 获取字幕列表 + subtitle_api = "https://api.bilibili.com/x/player/pagelist" + page_data = await self._make_request(session, subtitle_api, {'bvid': bvid}) + + # 获取字幕详情 + subtitle_list = [] + if page_data and page_data.get('code') == 0: + cid = page_data.get('data', [{}])[0].get('cid', cid) + + subtitle_url = f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}" + subtitle_data = await self._make_request(session, subtitle_url) + + if subtitle_data and subtitle_data.get('code') == 0: + subtitles = subtitle_data.get('data', {}).get('subtitle', {}).get('list', []) + + for sub in subtitles: + sub_title = sub.get('lan_doc', '未知语言') + sub_url = sub.get('subtitle_url') + + if sub_url: + # 下载字幕文件 + async with session.get(sub_url) as resp: + if resp.status == 200: + sub_content = await resp.json() + subtitle_list.append({ + "language": sub.get('lan', 'unknown'), + "title": sub_title, + "content": sub_content + }) + + result["subtitles"] = subtitle_list + result["success"] = len(subtitle_list) > 0 + result["message"] = f"找到 {len(subtitle_list)} 个字幕" if subtitle_list else "未找到字幕" + + # 保存字幕到文件 + if subtitle_list: + output_path = self.output_dir / f"{bvid}_subtitles.json" + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(result, f, ensure_ascii=False, indent=2) + result["filepath"] = str(output_path) + print(f"✅ 字幕已保存: {output_path}") + + return result + + async def extract_article(self, cvid: str) -> Dict[str, Any]: + """ + 获取B站专栏全文 + + Args: + cvid: B站专栏CV号 + + Returns: + Dict: 包含专栏内容的JSON结果 + """ + result = { + "success": False, + "cvid": cvid, + "title": "", + "content": "", + "message": "" + } + + async with aiohttp.ClientSession() as session: + # 获取专栏信息 + api_url = "https://api.bilibili.com/x/article/viewinfo" + data = await self._make_request(session, api_url, {'id': cvid}) + + if not data or data.get('code') != 0: + result["message"] = f"获取专栏信息失败: {data.get('message', '未知错误')}" + return result + + article_data = data.get('data', {}) + title = article_data.get('title', '') + content = article_data.get('content', '') + + # 获取作者信息 + author = article_data.get('author', {}).get('name', '未知作者') + publish_time = article_data.get('publish_time', 0) + view = article_data.get('stats', {}).get('view', 0) + like = article_data.get('stats', {}).get('like', 0) + + result["title"] = title + result["content"] = content + result["author"] = author + result["publish_time"] = datetime.fromtimestamp(publish_time).strftime('%Y-%m-%d %H:%M:%S') if publish_time else '' + result["stats"] = {"view": view, "like": like} + result["success"] = bool(title and content) + result["message"] = f"成功获取专栏: {title}" if result["success"] else "获取专栏内容失败" + + # 保存到文件 + if result["success"]: + safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:30] + safe_title = re.sub(r'[<>:"/\\|?*]', '', safe_title) + + # 保存JSON + json_path = self.output_dir / f"{safe_title}_{cvid}.json" + with open(json_path, 'w', encoding='utf-8') as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + # 保存Markdown + md_content = f"""# {title} + +> **来源**: [B站专栏](https://www.bilibili.com/read/cv{cvid}) +> **作者**: {author} +> **发布时间**: {result["publish_time"]} +> **阅读**: {view} | **点赞**: {like} + +--- + +{content} +""" + md_path = self.output_dir / f"{safe_title}_{cvid}.md" + with open(md_path, 'w', encoding='utf-8') as f: + f.write(md_content) + + result["json_path"] = str(json_path) + result["md_path"] = str(md_path) + print(f"✅ 专栏已保存: {md_path}") + + return result + + +async def main(): + """命令行入口""" + parser = argparse.ArgumentParser( + description='Bilibili Content Downloader - B站内容获取工具', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + %(prog)s --id "BV1xx411c7mD" --type video # 下载视频 + %(prog)s --id "BV1xx411c7mD" --type video --quality 720 # 下载720P视频 + %(prog)s --id "BV1xx411c7mD" --type subtitle # 提取字幕 + %(prog)s --id "cv43780186" --type article # 获取专栏 + """ + ) + + parser.add_argument('--id', '-i', required=True, help='B站内容ID (BV号或CV号)') + parser.add_argument('--type', '-t', required=True, choices=['video', 'subtitle', 'article'], + help='内容类型: video=视频, subtitle=字幕, article=专栏') + parser.add_argument('--output', '-o', default='./output', help='输出目录 (默认: ./output)') + parser.add_argument('--quality', '-q', type=int, default=480, + help='视频质量 (默认: 480, 可选: 360, 480, 720, 1080)') + parser.add_argument('--json', action='store_true', help='JSON格式输出') + + args = parser.parse_args() + + # 确定内容类型 + is_article = args.id.startswith('cv') + + if args.type == 'article' or is_article: + content_type = 'article' + content_id = args.id.replace('cv', '') + elif args.type == 'subtitle': + content_type = 'subtitle' + content_id = args.id + else: + content_type = 'video' + content_id = args.id + + # 创建获取器 + downloader = BilibiliContent(output_dir=args.output) + + # 执行操作 + if content_type == 'video': + result = await downloader.download_video(content_id, quality=args.quality) + elif content_type == 'subtitle': + result = await downloader.extract_subtitle(content_id) + else: + result = await downloader.extract_article(content_id) + + # 输出结果 + if args.json: + print(json.dumps(result, ensure_ascii=False, indent=2)) + else: + if result["success"]: + print(f"\n✅ {result['message']}") + else: + print(f"\n❌ {result['message']}") + + if result.get("filepath"): + print(f"文件路径: {result['filepath']}") + + return result + + +if __name__ == '__main__': + try: + result = asyncio.run(main()) + sys.exit(0 if result.get("success") else 1) + except KeyboardInterrupt: + print("\n操作已取消") + sys.exit(1) + except Exception as e: + print(f"\n错误: {e}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/skills/kb-bilibili-content/bilibili_extractor.py b/skills/kb-bilibili-content/bilibili_extractor.py new file mode 100644 index 0000000..971d239 --- /dev/null +++ b/skills/kb-bilibili-content/bilibili_extractor.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +B站视频提取脚本 +提取视频信息、统计数据、字幕并生成Markdown笔记 +""" + +import sys +import os +import json +import re +from datetime import datetime + +# 添加skills路径 +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb')) + +from src.extractors.bilibili_extractor import BilibiliExtractor +from src.config import get_config + + +def format_number(num): + """格式化数字""" + if num >= 100000000: + return f"{num / 100000000:.1f}亿" + elif num >= 10000: + return f"{num / 10000:.1f}万" + else: + return f"{num:,}" + + +def generate_markdown(result, bvid): + """生成Markdown笔记""" + metadata = result['metadata'] + stats = result['stats'] + description = result['description'] + subtitle = result['subtitle'] + + # 获取当前时间 + now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + md = f"""# {metadata['title']} + +> **来源**: [B站](https://www.bilibili.com/video/{bvid}) +> **作者**: {metadata['author']} +> **发布时间**: {metadata['pubtime']} +> **提取时间**: {now} +> **时长**: {metadata['duration']} + +## 📊 统计数据 + +| 指标 | 数量 | +|------|------| +| 👀 播放 | {format_number(stats['view'])} | +| 💬 弹幕 | {format_number(stats['danmaku'])} | +| 📝 评论 | {format_number(stats['reply'])} | +| ⭐ 收藏 | {format_number(stats['favorite'])} | +| 🪙 投币 | {format_number(stats['coin'])} | +| 🔗 分享 | {format_number(stats['share'])} | +| 👍 点赞 | {format_number(stats['like'])} | + +## 📝 视频简介 + +{description if description else '(暂无简介)'} + +## 📌 标签 + +{metadata['tags'] if metadata['tags'] else '(无标签)'} + +## 🎬 字幕 + +{subtitle if subtitle else '(无字幕)'} + +--- + +**元数据**: +- BV号: `{metadata['bvid']}` +- AV号: `{metadata['aid']}` +- UP主ID: `{metadata['mid']}` +""" + + return md + + +def main(): + """主函数""" + # BV号 + bvid = "BV1N5foBjE5K" # 从 https://b23.tv/btMGoOi 解析得到 + + print(f"开始提取B站视频: {bvid}") + print("=" * 60) + + # 获取配置 + try: + config = get_config() + except Exception as e: + print(f"⚠️ 获取配置失败，使用默认配置: {e}") + config = None + + # 创建提取器 + extractor = BilibiliExtractor(config=config) + + # 提取视频信息 + try: + result = extractor.extract(bvid) + + print("\n✅ 提取成功！\n") + print("=" * 60) + print("基本信息:") + print("=" * 60) + print(f"标题: {result['metadata']['title']}") + print(f"作者: {result['metadata']['author']}") + print(f"时长: {result['metadata']['duration']}") + print(f"发布: {result['metadata']['pubtime']}") + + print("\n" + "=" * 60) + print("统计数据:") + print("=" * 60) + print(f"播放: {format_number(result['stats']['view'])}") + print(f"弹幕: {format_number(result['stats']['danmaku'])}") + print(f"评论: {format_number(result['stats']['reply'])}") + print(f"收藏: {format_number(result['stats']['favorite'])}") + print(f"投币: {format_number(result['stats']['coin'])}") + print(f"点赞: {format_number(result['stats']['like'])}") + + # 生成 Markdown 笔记 + md_content = generate_markdown(result, bvid) + + # 确保 notes 目录存在 + notes_dir = "/home/admin/.openclaw/workspace/notes" + os.makedirs(notes_dir, exist_ok=True) + + # 生成文件名（使用BV号作为文件名） + filename = f"{bvid}.md" + filepath = os.path.join(notes_dir, filename) + + # 写入文件 + with open(filepath, 'w', encoding='utf-8') as f: + f.write(md_content) + + print("\n" + "=" * 60) + print(f"✅ Markdown笔记已生成: {filepath}") + print("=" * 60) + + # 返回笔记内容 + return md_content + + except Exception as e: + print(f"\n❌ 提取失败: {e}") + import traceback + traceback.print_exc() + return None + + +if __name__ == '__main__': + main() diff --git a/skills/kb-bilibili-content/video_downloader.py b/skills/kb-bilibili-content/video_downloader.py new file mode 100644 index 0000000..f5d25c4 --- /dev/null +++ b/skills/kb-bilibili-content/video_downloader.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +使用旧版 API 获取并下载视频流 +""" +import asyncio +import json +import os +from playwright.async_api import async_playwright + +async def download_video_stream(bvid, output_dir="/tmp/bilibili_downloads"): + """下载视频流""" + url = f"https://www.bilibili.com/video/{bvid}" + + # 创建输出目录 + os.makedirs(output_dir, exist_ok=True) + + async with async_playwright() as playwright: + browser = await playwright.chromium.launch(headless=True) + context = await browser.new_context() + page = await context.new_page() + + print(f"正在下载视频: {url}") + print("=" * 80) + + # 首先获取 aid 和 cid + print("\n1. 获取视频信息...") + try: + api_url = "https://api.bilibili.com/x/web-interface/view" + response = await page.request.get(api_url, params={'bvid': bvid}) + data = await response.json() + + if data.get('code') != 0: + print(f" ❌ 失败: {data.get('message')}") + await browser.close() + return None + + video_data = data.get('data', {}) + title = video_data.get('title') + aid = video_data.get('aid') + cid = video_data.get('cid') + duration = video_data.get('duration') + print(f" 标题: {title}") + print(f" 时长: {duration}秒") + except Exception as e: + print(f" ❌ 异常: {e}") + await browser.close() + return None + + # 获取视频流 URL + print("\n2. 获取视频流 URL...") + try: + playurl_api = "https://api.bilibili.com/x/player/playurl" + params = { + 'bvid': bvid, + 'cid': cid, + 'qn': 80, # 1080P + 'otype': 'json', + 'fnver': 0, + 'fnval': 0, # FLV 格式 + } + + playurl_response = await page.request.get(playurl_api, params=params) + + if not playurl_response.ok: + print(f" ❌ HTTP 错误: {playurl_response.status}") + await browser.close() + return None + + playurl_data = await playurl_response.json() + + if playurl_data.get('code') != 0: + print(f" ❌ API 错误: {playurl_data.get('message')}") + await browser.close() + return None + + play_data = playurl_data.get('data', {}) + + # 检查支持的清晰度 + print(f"\n 支持的清晰度:") + for i, (q, desc) in enumerate(zip(play_data.get('accept_quality', []), play_data.get('accept_description', []))): + current = " (当前)" if q == play_data.get('quality') else "" + print(f" {desc} ({q}){current}") + + # 获取视频 URL + durl = play_data.get('durl', []) + if not durl: + print(f" ❌ 没有找到视频流") + await browser.close() + return None + + print(f"\n 找到 {len(durl)} 个视频分段") + video_urls = [] + for i, segment in enumerate(durl): + segment_url = segment.get('url') + segment_size = segment.get('size', 0) + video_urls.append(segment_url) + print(f" 分段 {i+1}: {segment_size / 1024 / 1024:.2f}MB") + print(f" {segment_url[:100]}...") + + except Exception as e: + print(f" ❌ 异常: {e}") + import traceback + traceback.print_exc() + await browser.close() + return None + + # 下载视频 + print(f"\n3. 开始下载视频...") + all_segments = [] + total_size = 0 + + for i, video_url in enumerate(video_urls, 1): + print(f"\n 下载分段 {i}/{len(video_urls)}...") + + try: + # 添加必要的请求头 + headers = { + 'Referer': 'https://www.bilibili.com/', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + } + + # 下载分段 + segment_response = await page.request.get( + video_url, + headers=headers, + timeout=60000 + ) + + if not segment_response.ok: + print(f" ❌ 下载失败: {segment_response.status}") + continue + + segment_data = await segment_response.body() + segment_size = len(segment_data) + total_size += segment_size + all_segments.append(segment_data) + + print(f" ✅ 下载成功: {segment_size / 1024 / 1024:.2f}MB") + + except Exception as e: + print(f" ❌ 下载异常: {e}") + continue + + if not all_segments: + print(f"\n ❌ 没有成功下载任何分段") + await browser.close() + return None + + # 合并分段 + print(f"\n4. 合并视频分段...") + video_data = b''.join(all_segments) + + # 保存文件 + safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50] + output_path = os.path.join(output_dir, f"{safe_title}.flv") + + with open(output_path, 'wb') as f: + f.write(video_data) + + print(f" ✅ 视频已保存: {output_path}") + print(f" 文件大小: {total_size / 1024 / 1024:.2f}MB") + + await browser.close() + + return output_path + +if __name__ == '__main__': + bvid = "BV1KtFzzqEUG" + result = asyncio.run(download_video_stream(bvid)) + if result: + print(f"\n✅ 下载完成！") + print(f"文件路径: {result}") + else: + print(f"\n❌ 下载失败") diff --git a/skills/kb-bilibili-metadata/SKILL.md b/skills/kb-bilibili-metadata/SKILL.md new file mode 100644 index 0000000..d90d862 --- /dev/null +++ b/skills/kb-bilibili-metadata/SKILL.md @@ -0,0 +1,46 @@ +# KB Bilibili Metadata Skill + +**kb-bilibili-metadata** - Extract Bilibili video/article/up主 metadata. + +## Features + +- Video metadata extraction +- Article (专栏) metadata extraction +- UP主 information extraction +- JSON output format +- Playwright + HTTP API fallback + +## Commands + +```bash +# Video metadata +python bilibili_metadata.py --id "BV1xx411c7mD" --type video + +# Article metadata +python bilibili_metadata.py --id "cv43780186" --type article + +# UP主 info +python bilibili_metadata.py --id "7770724" --type up +``` + +## Output + +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "video", + "id": "BV1xx411c7mD", + "title": "视频标题", + "author": "UP主名字", + "stats": {"view": 100000, "like": 5000} + } +} +``` + +## Installation + +```bash +cp -r kb-bilibili-metadata/ ~/skills/ +``` diff --git a/skills/kb-bilibili-metadata/__init__.py b/skills/kb-bilibili-metadata/__init__.py new file mode 100644 index 0000000..0614e1f --- /dev/null +++ b/skills/kb-bilibili-metadata/__init__.py @@ -0,0 +1,8 @@ +""" +bilibili-metadata 技能包 + +B站元数据提取工具 +""" + +__version__ = "1.0.0" +__author__ = "OpenClaw" diff --git a/skills/kb-bilibili-metadata/bilibili_extractor.py b/skills/kb-bilibili-metadata/bilibili_extractor.py new file mode 100644 index 0000000..21c0e73 --- /dev/null +++ b/skills/kb-bilibili-metadata/bilibili_extractor.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +B站元数据提取器 + +独立版本，不依赖 obsidian-kb 结构 +支持视频、专栏、UP主信息提取 +""" + +import asyncio +import json +import logging +import re +from datetime import datetime +from typing import Dict, Any, Optional +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +# 默认请求头 +DEFAULT_HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Referer': 'https://www.bilibili.com/', + 'Accept': 'application/json, text/plain, */*', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', +} + +try: + from playwright.async_api import async_playwright + PLAYWRIGHT_AVAILABLE = True +except ImportError: + PLAYWRIGHT_AVAILABLE = False + +try: + from undetected_playwright import stealth_async + STEALTH_AVAILABLE = True +except ImportError: + STEALTH_AVAILABLE = False + +try: + import httpx + HTTPX_AVAILABLE = True +except ImportError: + HTTPX_AVAILABLE = False + + +class BilibiliExtractor: + """ + B站元数据提取器 + + 支持： + - 视频信息提取（标题、作者、统计数据、字幕等） + - 专栏文章提取 + - UP主信息提取 + """ + + # 正则模式 + BV_PATTERN = re.compile(r'BV[a-zA-Z0-9]{10}') + CV_PATTERN = re.compile(r'cv(\d+)') + URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/video/(BV[a-zA-Z0-9]{10})') + ARTICLE_URL_PATTERN = re.compile(r'https?://www\.bilibili\.com/read/(cv\d+)') + UP_URL_PATTERN = re.compile(r'https?://space\.bilibili\.com/(\d+)') + + def __init__(self, headless: bool = True, timeout: int = 30): + """ + 初始化提取器 + + Args: + headless: 是否使用无头模式（浏览器） + timeout: 超时时间（秒） + """ + self.headless = headless + self.timeout = timeout * 1000 # 转换为毫秒 + + def extract_video_metadata(self, bvid: str) -> Dict[str, Any]: + """ + 提取视频元数据 + + Args: + bvid: B站视频BV号（如 BV1xx411c7mD） + + Returns: + Dict containing metadata, stats, description, subtitle + """ + # 验证并解析BV号 + bvid = self._resolve_bvid(bvid) + if not bvid: + raise ValueError(f"无效的BV号: {bvid}") + + # 使用异步方法提取 + return asyncio.run(self._extract_video_async(bvid)) + + def extract_article_metadata(self, cvid: str) -> Dict[str, Any]: + """ + 提取专栏元数据 + + Args: + cvid: B站专栏CV号（如 cv43780186） + + Returns: + Dict containing article metadata + """ + # 解析CV号 + cvid = self._resolve_cvid(cvid) + if not cvid: + raise ValueError(f"无效的CV号: {cvid}") + + return asyncio.run(self._extract_article_async(cvid)) + + def extract_up_metadata(self, mid: str) -> Dict[str, Any]: + """ + 提取UP主信息 + + Args: + mid: B站用户ID（如 7770724） + + Returns: + Dict containing UP主信息 + """ + # 解析MID + mid = self._resolve_mid(mid) + if not mid: + raise ValueError(f"无效的MID: {mid}") + + return asyncio.run(self._extract_up_async(mid)) + + def _resolve_bvid(self, source_id: str) -> Optional[str]: + """解析并返回BV号""" + source = source_id.strip() + + # 直接BV号 + if self.BV_PATTERN.fullmatch(source): + return source + + # 完整URL + match = self.URL_PATTERN.match(source) + if match: + return match.group(1) + + return None + + def _resolve_cvid(self, source_id: str) -> Optional[str]: + """解析并返回CV号""" + source = source_id.strip() + + # 直接CV号（带cv前缀） + if source.startswith('cv'): + match = self.CV_PATTERN.match(source) + if match: + return source + + # 纯数字 + if source.isdigit(): + return f"cv{source}" + + # 完整URL + match = self.ARTICLE_URL_PATTERN.match(source) + if match: + return match.group(1) + + return None + + def _resolve_mid(self, source_id: str) -> Optional[str]: + """解析并返回MID""" + source = source_id.strip() + + # 纯数字 + if source.isdigit(): + return source + + # 完整URL + match = self.UP_URL_PATTERN.match(source) + if match: + return match.group(1) + + return None + + async def _extract_video_async(self, bvid: str) -> Dict[str, Any]: + """异步提取视频信息""" + url = f"https://www.bilibili.com/video/{bvid}" + result = { + 'metadata': {}, + 'stats': {}, + 'subtitle': '', + 'description': '' + } + + if not PLAYWRIGHT_AVAILABLE: + # 使用 HTTP API + return await self._extract_video_by_api(bvid) + + async with async_playwright() as playwright: + try: + browser = await playwright.chromium.launch( + headless=self.headless, + args=[ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-blink-features=AutomationControlled', + '--disable-dev-shm-usage', + ] + ) + + context = await browser.new_context( + viewport={'width': 1920, 'height': 1080}, + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + locale='zh-CN', + ) + + if STEALTH_AVAILABLE: + await stealth_async(context) + + page = await context.new_page() + await page.goto(url, wait_until='domcontentloaded', timeout=self.timeout) + await asyncio.sleep(2) + + content = await page.content() + video_data = await self._try_extract_video_methods(page, content, bvid) + + if video_data: + result['metadata'] = { + 'title': video_data.get('title', ''), + 'author': video_data.get('author', ''), + 'bvid': video_data.get('bvid', ''), + 'aid': video_data.get('aid', ''), + 'mid': video_data.get('mid', ''), + 'pubtime': video_data.get('pubtime', ''), + 'duration': video_data.get('duration', ''), + 'url': url, + 'tags': video_data.get('tags', ''), + } + result['stats'] = { + 'view': video_data.get('view', 0), + 'danmaku': video_data.get('danmaku', 0), + 'reply': video_data.get('reply', 0), + 'favorite': video_data.get('favorite', 0), + 'coin': video_data.get('coin', 0), + 'share': video_data.get('share', 0), + 'like': video_data.get('like', 0), + } + result['description'] = video_data.get('desc', '') + + await browser.close() + + except Exception as e: + logger.error(f"视频提取错误: {e}") + # 回退到 API + return await self._extract_video_by_api(bvid) + + return result + + async def _extract_video_by_api(self, bvid: str) -> Dict[str, Any]: + """使用B站API提取视频信息""" + result = { + 'metadata': {}, + 'stats': {}, + 'subtitle': '', + 'description': '' + } + + try: + async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client: + # 获取视频信息 + response = await client.get('https://api.bilibili.com/x/web-interface/view', params={'bvid': bvid}) + if response.status_code == 200: + data = response.json() + if data.get('code') == 0: + video_data = data.get('data', {}) + + result['metadata'] = { + 'title': video_data.get('title', ''), + 'author': video_data.get('owner', {}).get('name', ''), + 'bvid': video_data.get('bvid', ''), + 'aid': video_data.get('aid', ''), + 'mid': video_data.get('owner', {}).get('mid', ''), + 'pubtime': datetime.fromtimestamp(video_data.get('pubdate', 0)).strftime('%Y-%m-%d %H:%M:%S') if video_data.get('pubdate') else '', + 'duration': self._format_duration(video_data.get('duration', 0)), + 'url': f"https://www.bilibili.com/video/{bvid}", + 'tags': ', '.join([t.get('tag_name', '') for t in video_data.get('tag', [])]), + } + + stat = video_data.get('stat', {}) + result['stats'] = { + 'view': stat.get('view', 0), + 'danmaku': stat.get('danmaku', 0), + 'reply': stat.get('reply', 0), + 'favorite': stat.get('favorite', 0), + 'coin': stat.get('coin', 0), + 'share': stat.get('share', 0), + 'like': stat.get('like', 0), + } + result['description'] = video_data.get('desc', '') + + except Exception as e: + logger.error(f"API提取错误: {e}") + + return result + + async def _try_extract_video_methods(self, page, content: str, bvid: str) -> Dict[str, Any]: + """尝试多种方法提取视频信息""" + video_data = {} + + # 方法1: 从 __INITIAL_STATE__ + try: + initial_state = await page.evaluate('() => typeof window.__INITIAL_STATE__ !== "undefined" ? window.__INITIAL_STATE__ : null') + if initial_state: + video_data = self._parse_video_state(initial_state) + if video_data and video_data.get('title'): + return video_data + except: + pass + + # 方法2: 使用 API（更可靠） + return await self._extract_video_by_api(bvid) + + def _parse_video_state(self, data: Dict) -> Dict[str, Any]: + """解析 __INITIAL_STATE__ 数据""" + video_data = {} + try: + video = data.get('videoData', {}) + video_data['title'] = video.get('title', '') + video_data['bvid'] = video.get('bvid', '') + video_data['aid'] = video.get('aid', '') + owner = video.get('owner', {}) + video_data['author'] = owner.get('name', '') + video_data['mid'] = owner.get('mid', '') + stat = video.get('stat', {}) + video_data['view'] = stat.get('view', 0) + video_data['danmaku'] = stat.get('danmaku', 0) + video_data['reply'] = stat.get('reply', 0) + video_data['favorite'] = stat.get('favorite', 0) + video_data['coin'] = stat.get('coin', 0) + video_data['share'] = stat.get('share', 0) + video_data['like'] = stat.get('like', 0) + pubdate = video.get('pubdate', 0) + video_data['pubtime'] = datetime.fromtimestamp(pubdate).strftime('%Y-%m-%d %H:%M:%S') if pubdate else '' + video_data['duration'] = self._format_duration(video.get('duration', 0)) + video_data['desc'] = video.get('desc', '') + video_data['tags'] = ', '.join([t.get('tag_name', '') for t in video.get('tag', [])]) + except: + pass + return video_data + + async def _extract_article_async(self, cvid: str) -> Dict[str, Any]: + """异步提取专栏信息""" + result = {} + + if not HTTPX_AVAILABLE: + return {'error': 'httpx 未安装'} + + try: + async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client: + # 获取专栏信息 + response = await client.get('https://api.bilibili.com/x/article/viewinfo', params={'id': cvid[2:]}) + if response.status_code == 200: + data = response.json() + if data.get('code') == 0: + article = data.get('data', {}) + result = { + 'cvid': cvid, + 'title': article.get('title', ''), + 'author': article.get('author_name', ''), + 'mid': article.get('author_mid', ''), + 'publish_time': datetime.fromtimestamp(article.get('publish_time', 0)).strftime('%Y-%m-%d %H:%M:%S') if article.get('publish_time') else '', + 'view_count': article.get('stats', {}).get('view', 0), + 'like_count': article.get('stats', {}).get('like', 0), + 'coin_count': article.get('stats', {}).get('coin', 0), + 'share_count': article.get('stats', {}).get('share', 0), + 'reply_count': article.get('stats', {}).get('reply', 0), + 'url': f"https://www.bilibili.com/read/{cvid}", + 'summary': article.get('summary', ''), + 'word_count': article.get('words', 0), + } + except Exception as e: + logger.error(f"专栏提取错误: {e}") + result = {'error': str(e)} + + return result + + async def _extract_up_async(self, mid: str) -> Dict[str, Any]: + """异步提取UP主信息""" + result = {} + + if not HTTPX_AVAILABLE: + return {'error': 'httpx 未安装'} + + try: + async with httpx.AsyncClient(timeout=30.0, headers=DEFAULT_HEADERS) as client: + # 获取UP主信息 + response = await client.get('https://api.bilibili.com/x/space/acc/info', params={'mid': mid}) + if response.status_code == 200: + data = response.json() + if data.get('code') == 0: + up = data.get('data', {}) + result = { + 'mid': mid, + 'name': up.get('name', ''), + 'sex': up.get('sex', ''), + 'face': up.get('face', ''), + 'sign': up.get('sign', ''), + 'level': up.get('level', 0), + 'vip_type': up.get('vip', {}).get('type', 0), + 'vip_status': up.get('vip', {}).get('status', 0), + 'following_count': up.get('following', 0), + 'follower_count': up.get('follower', 0), + 'url': f"https://space.bilibili.com/{mid}", + } + + # 获取UP主视频统计 + stats_response = await client.get('https://api.bilibili.com/x/space/navnum', params={'mid': mid}) + if stats_response.status_code == 200: + stats_data = stats_response.json() + if stats_data.get('code') == 0: + result['video_count'] = stats_data.get('data', {}).get('video', 0) + + except Exception as e: + logger.error(f"UP主提取错误: {e}") + result = {'error': str(e)} + + return result + + def _format_duration(self, seconds: int) -> str: + """格式化时长（秒 -> HH:MM:SS）""" + if not seconds: + return '' + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + secs = seconds % 60 + if hours > 0: + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + else: + return f"{minutes:02d}:{secs:02d}" + + +# 测试代码 +if __name__ == '__main__': + extractor = BilibiliExtractor() + + print("=" * 60) + print("B站元数据提取器测试") + print("=" * 60) + + # 测试视频 + print("\n测试视频提取: BV1MnFNz5EDL") + result = extractor.extract_video_metadata('BV1MnFNz5EDL') + print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/skills/kb-bilibili-metadata/bilibili_metadata.py b/skills/kb-bilibili-metadata/bilibili_metadata.py new file mode 100755 index 0000000..7908031 --- /dev/null +++ b/skills/kb-bilibili-metadata/bilibili_metadata.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +B站元数据提取 CLI 工具 + +支持命令行调用，输出 JSON 格式数据 +""" + +import argparse +import json +import sys +from typing import Dict, Any + +from bilibili_extractor import BilibiliExtractor + + +def format_number(num: int) -> str: + """格式化数字""" + if num >= 100000000: + return f"{num / 100000000:.1f}亿" + elif num >= 10000: + return f"{num / 10000:.1f}万" + else: + return f"{num:,}" + + +def extract_video(extractor: BilibiliExtractor, bvid: str, output_file: str = None) -> Dict[str, Any]: + """提取视频元数据""" + result = extractor.extract_video_metadata(bvid) + + # 如果需要输出到文件 + if output_file: + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + return result + + +def extract_article(extractor: BilibiliExtractor, cvid: str, output_file: str = None) -> Dict[str, Any]: + """提取专栏元数据""" + result = extractor.extract_article_metadata(cvid) + + if output_file: + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + return result + + +def extract_up(extractor: BilibiliExtractor, mid: str, output_file: str = None) -> Dict[str, Any]: + """提取UP主信息""" + result = extractor.extract_up_metadata(mid) + + if output_file: + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(result, f, ensure_ascii=False, indent=2) + + return result + + +def main(): + """主函数""" + parser = argparse.ArgumentParser( + description='B站元数据提取工具', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + # 提取视频元数据 + python bilibili_metadata.py --id "BV1xx411c7mD" --type video + + # 提取专栏元数据 + python bilibili_metadata.py --id "cv43780186" --type article + + # 提取UP主信息 + python bilibili_metadata.py --id "7770724" --type up + + # 输出到文件 + python bilibili_metadata.py --id "BV1xx411c7mD" --type video --output result.json + """ + ) + + parser.add_argument( + '--id', + required=True, + help='B站资源ID（BV号、CV号或UP主ID）' + ) + parser.add_argument( + '--type', + required=True, + choices=['video', 'article', 'up'], + help='资源类型' + ) + parser.add_argument( + '--output', + '-o', + help='输出文件路径（JSON格式）' + ) + parser.add_argument( + '--headless', + action='store_true', + default=True, + help='使用无头模式（浏览器，默认开启）' + ) + parser.add_argument( + '--timeout', + type=int, + default=30, + help='超时时间（秒，默认30）' + ) + parser.add_argument( + '--pretty', + action='store_true', + default=True, + help='格式化输出（默认开启）' + ) + + args = parser.parse_args() + + # 创建提取器 + extractor = BilibiliExtractor(headless=args.headless, timeout=args.timeout) + + try: + # 根据类型提取数据 + if args.type == 'video': + result = extract_video(extractor, args.id, args.output) + elif args.type == 'article': + result = extract_article(extractor, args.id, args.output) + elif args.type == 'up': + result = extract_up(extractor, args.id, args.output) + else: + print(f"错误: 不支持的类型 {args.type}") + sys.exit(1) + + # 输出结果 + if not args.output: + if args.pretty: + print(json.dumps(result, ensure_ascii=False, indent=2)) + else: + print(json.dumps(result, ensure_ascii=False)) + + if args.output: + print(f"结果已保存到: {args.output}") + + except Exception as e: + print(f"错误: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/skills/kb-deep-analyzer/SKILL.md b/skills/kb-deep-analyzer/SKILL.md new file mode 100644 index 0000000..74b5d64 --- /dev/null +++ b/skills/kb-deep-analyzer/SKILL.md @@ -0,0 +1,48 @@ +# KB Deep Analyzer Skill + +**kb-deep-analyzer** - AI-powered deep content analysis. + +## Features + +- Multi-type analysis (tutorial, documentation, news) +- Template-based output +- Quality scoring +- Automatic tagging + +## Commands + +```bash +# Analyze content +python deep_analyzer.py --metadata meta.json --content content.json --output analysis.md + +# Specify type +python deep_analyzer.py --metadata meta.json --content content.json --type tutorial --output tutorial.md +``` + +## Output + +```markdown +--- +title: "分析标题" +source: "bilibili" +quality_score: 95 +tags: + - python + - tutorial +--- + +## 一句话总结 + +内容摘要... + +## 核心要点 + +1. 要点1 +2. 要点2 +``` + +## Installation + +```bash +cp -r kb-deep-analyzer/ ~/skills/ +``` diff --git a/skills/kb-deep-analyzer/ai_client.py b/skills/kb-deep-analyzer/ai_client.py new file mode 100644 index 0000000..cee3c73 --- /dev/null +++ b/skills/kb-deep-analyzer/ai_client.py @@ -0,0 +1,514 @@ +"""Obsidian KB - AI客户端 +支持多个 AI 提供商和 fallback 机制 +""" +import os +import json +import requests +from typing import Dict, Any, List, Optional, Tuple +from src.models import ContentType +from src.config import get_config +from src.content_chunker import ContentChunker, get_chunking_strategy, chunk_content + + +class AIClient: + """ + AI客户端 - 支持分段分析长视频内容和 fallback 机制 + """ + + def __init__(self): + self.config = get_config() + self.chunker = ContentChunker() + # 获取 AI 配置 + self._load_ai_config() + + def _load_ai_config(self) -> None: + """加载 AI 配置（主 provider + fallbacks）""" + self.ai_configs = [] + + # 加载主配置 + main_provider = self.config.get('ai.provider', 'openai') + main_config = { + 'provider': main_provider, + 'api_key': self.config.get('ai.api_key'), + 'model': self.config.get('ai.model', 'gpt-4'), + 'base_url': self.config.get('ai.base_url'), + 'is_primary': True + } + self.ai_configs.append(main_config) + + # 加载 fallback 配置 + fallbacks = self.config.get('ai.fallbacks', []) + for fb in fallbacks: + self.ai_configs.append({ + 'provider': fb.get('provider'), + 'api_key': fb.get('api_key'), + 'model': fb.get('model'), + 'base_url': fb.get('base_url'), + 'is_primary': False + }) + + def _call_api_with_fallback(self, prompt: str, max_tokens: int = 1000, response_format: str = 'text') -> Tuple[bool, str]: + """ + 使用 fallback 机制调用 API + + Args: + prompt: 提示词 + max_tokens: 最大 token 数 + response_format: 响应格式 ('text' 或 'json') + + Returns: + (success, response) 元组 + """ + last_error = None + + for i, ai_config in enumerate(self.ai_configs): + try: + if not ai_config.get('api_key'): + print(f"[AIClient] 跳过 {ai_config.get('provider')} - API Key 为空") + continue + + provider_type = ai_config.get('provider') + print(f"[AIClient] 尝试使用 {provider_type} (优先级 {i+1}/{len(self.ai_configs)})...") + + response = self._call_single_provider(ai_config, prompt, max_tokens, response_format) + return True, response + + except Exception as e: + last_error = e + print(f"[AIClient] {provider_type} 调用失败: {e}") + continue + + # 所有 provider 都失败了 + error_msg = f"所有 AI 提供商均失败，最后错误: {last_error}" if last_error else "没有可用的 AI 配置" + print(f"[AIClient] {error_msg}") + return False, self._mock_generate(prompt, max_tokens) # 返回 mock 响应作为兜底 + + def _call_single_provider(self, ai_config: Dict, prompt: str, max_tokens: int, response_format: str) -> str: + """ + 调用单个 AI 提供商 + + Args: + ai_config: AI 配置 + prompt: 提示词 + max_tokens: 最大 token 数 + response_format: 响应格式 + + Returns: + API 响应文本 + """ + provider = ai_config.get('provider') + api_key = ai_config.get('api_key') + model = ai_config.get('model') + base_url = ai_config.get('base_url') + + if provider == 'minimax': + return self._call_minimax(api_key, model, base_url, prompt, max_tokens, response_format) + elif provider == 'zhipu': + return self._call_zhipu(api_key, model, base_url, prompt, max_tokens, response_format) + elif provider == 'alibaba': + return self._call_alibaba(api_key, model, base_url, prompt, max_tokens, response_format) + elif provider == 'openai': + return self._call_openai(api_key, model, base_url, prompt, max_tokens, response_format) + else: + raise ValueError(f"不支持的 provider: {provider}") + + def _call_minimax(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str: + """调用 MiniMax API (Anthropic Messages 兼容接口)""" + url = base_url or "https://api.minimaxi.com/anthropic/v1/messages" + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + payload = { + "model": model, + "max_tokens": max_tokens, + "messages": [ + {"role": "user", "content": prompt} + ] + } + + if response_format == 'json': + payload['messages'][0]['content'] = prompt + "\n\n请直接返回 JSON 格式，不要有其他文字。" + + response = requests.post(url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + data = response.json() + + # MiniMax 返回的是 Anthropic 格式，content 是一个数组 + # 需要找到 type='text' 的元素 + for item in data.get('content', []): + if item.get('type') == 'text': + return item.get('text', '') + + # 如果没有找到 text 类型，返回第一个元素的值（兼容旧格式） + return data.get('content', [{}])[0].get('text', '') + + def _call_zhipu(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str: + """调用智谱 AI API (OpenAI 兼容接口)""" + # 确保 base_url 包含 /chat/completions + if base_url and not base_url.endswith('/chat/completions'): + url = f"{base_url}/chat/completions" + else: + url = base_url or "https://open.bigmodel.cn/api/paas/v4/chat/completions" + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + payload = { + "model": model, + "max_tokens": max_tokens, + "messages": [ + {"role": "user", "content": prompt} + ] + } + + response = requests.post(url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + data = response.json() + return data['choices'][0]['message']['content'] + + def _call_alibaba(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str: + """调用阿里云通义千问 API (OpenAI 兼容接口)""" + url = f"{base_url}/chat/completions" + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + payload = { + "model": model, + "max_tokens": max_tokens, + "messages": [ + {"role": "user", "content": prompt} + ] + } + + response = requests.post(url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + data = response.json() + return data['choices'][0]['message']['content'] + + def _call_openai(self, api_key: str, model: str, base_url: str, prompt: str, max_tokens: int, response_format: str) -> str: + """调用 OpenAI API""" + url = base_url or "https://api.openai.com/v1/chat/completions" + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + payload = { + "model": model, + "max_tokens": max_tokens, + "messages": [ + {"role": "user", "content": prompt} + ] + } + + response = requests.post(url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + data = response.json() + return data['choices'][0]['message']['content'] + + def quick_generate(self, prompt: str, max_tokens: int = 100) -> str: + """快速生成（用于轻量扫描）""" + success, response = self._call_api_with_fallback(prompt, max_tokens, 'text') + return response + + def generate_json(self, prompt: str) -> Dict[str, Any]: + """生成JSON格式响应""" + success, response = self._call_api_with_fallback(prompt, 1000, 'json') + try: + return json.loads(response) + except: + return self._parse_json_from_text(response) + + def _deep_analyze(self, content_text: str, content_type: str, duration: Optional[int] = None) -> Dict[str, Any]: + """ + C5.3: 深度分析入口，根据内容时长决定是否分段分析 + + Args: + content_text: 内容文本 + content_type: 内容类型 + duration: 视频时长（秒） + + Returns: + 分析结果字典 + """ + duration = duration or 0 + + print(f"[AIClient] 深度分析: type={content_type}, duration={duration}s") + + # 长视频（>=30分钟）且是视频类型，使用分段分析 + if duration >= 1800 and content_type == "video": + print(f"[AIClient] 使用分段分析策略") + return self._analyze_in_chunks(content_text, duration) + + # 短视频直接分析 + print(f"[AIClient] 使用直接分析策略") + return self._analyze_direct(content_text) + + def _analyze_in_chunks(self, content_text: str, duration: int) -> Dict[str, Any]: + """ + C5.3: 分段分析长视频 + + Args: + content_text: SRT字幕文本 + duration: 视频时长（秒） + + Returns: + 合并后的分析结果 + """ + # 1. 获取分段策略 + strategy = get_chunking_strategy(duration) + print(f"[AIClient] 分段策略: {strategy['description']}") + + # 2. 执行分片 + chunks = chunk_content(content_text, duration) + print(f"[AIClient] 内容已分为 {len(chunks)} 段") + + # 3. 逐段分析 + chunk_summaries = [] + for i, chunk in enumerate(chunks): + print(f"[AIClient] 分析第 {i+1}/{len(chunks)} 段...") + summary = self._analyze_chunk(chunk.text, chunk.index) + + # 处理时间格式（可能是float秒数或字符串） + start_time = chunk.start_time + end_time = chunk.end_time + if isinstance(start_time, (int, float)): + start_time = self._format_time(start_time) + if isinstance(end_time, (int, float)): + end_time = self._format_time(end_time) + + chunk_summaries.append({ + 'index': chunk.index, + 'start_time': start_time, + 'end_time': end_time, + 'summary': summary + }) + + # 4. 合并所有段摘要 + print(f"[AIClient] 合并 {len(chunk_summaries)} 段分析结果") + final_result = self._merge_summaries(chunk_summaries, "video") + + # 5. 构建完整时间轴 + final_result['timeline'] = self._build_timeline_from_chunks(chunk_summaries) + + return final_result + + def _analyze_chunk(self, chunk_text: str, chunk_index: int) -> str: + """ + C5.3: 分析单个内容分片 + + Args: + chunk_text: 分片文本 + chunk_index: 分片索引 + + Returns: + 分片摘要（限制500字以内） + """ + # 截断文本避免过长 + text_sample = chunk_text[:3000] if len(chunk_text) > 3000 else chunk_text + + prompt = f"""请对以下视频片段进行简要分析（限制在300字以内）： + +片段内容： +{text_sample} + +请提供： +1. 这段的核心主题（50字以内） +2. 关键要点（3-5条） +3. 重要细节或引用 + +请用中文回答，控制在300字以内。""" + + try: + summary = self.quick_generate(prompt, max_tokens=500) + # 确保不超过500字 + if len(summary) > 500: + summary = summary[:497] + "..." + return summary + except Exception as e: + print(f"[AIClient] 分片 {chunk_index} 分析失败: {e}") + return f"第{chunk_index+1}段分析失败" + + def _merge_summaries(self, chunk_summaries: List[Dict], content_type: str) -> Dict[str, Any]: + """ + C5.3: 合并所有段摘要，生成完整分析报告 + + Args: + chunk_summaries: 各段摘要列表 + content_type: 内容类型 + + Returns: + 完整分析结果 + """ + # 构建合并提示 + summaries_text = "\n\n".join([ + f"=== 第{s['index']+1}段 ({s['start_time']}-{s['end_time']}) ===\n{s['summary']}" + for s in chunk_summaries + ]) + + prompt = f"""基于以下各段分析，生成完整的视频分析报告： + +{summaries_text} + +请提供以下格式的JSON： +{{ + "summary": "整体摘要（200字以内）", + "topics": ["主题1", "主题2", "主题3", "主题4"], + "insights": ["深入见解1", "深入见解2", "深入见解3"], + "actions": ["建议行动1", "建议行动2"], + "full_analysis": "详细分析（500字以内）" +}}""" + + try: + result = self.generate_json(prompt) + + # 确保必要字段存在 + if 'topics' not in result: + result['topics'] = ["主题A", "主题B", "主题C"] + if 'insights' not in result: + result['insights'] = ["重要见解1", "重要见解2"] + if 'actions' not in result: + result['actions'] = [] + if 'summary' not in result: + result['summary'] = "长视频分析完成" + if 'full_analysis' not in result: + result['full_analysis'] = summaries_text[:500] + + return result + + except Exception as e: + print(f"[AIClient] 合并摘要失败: {e}") + # 返回基础合并结果 + return { + 'summary': f'视频共分为{len(chunk_summaries)}个部分进行分析', + 'topics': [f"主题{i+1}" for i in range(min(4, len(chunk_summaries)))], + 'insights': ['从分段分析中综合得出'], + 'actions': [], + 'full_analysis': summaries_text[:800] + } + + def _analyze_direct(self, content_text: str) -> Dict[str, Any]: + """直接分析（不分段）""" + # 截断文本 + text_sample = content_text[:5000] if len(content_text) > 5000 else content_text + + prompt = f"""请分析以下内容，提供JSON格式结果： + +{text_sample} + +返回格式： +{{ + "summary": "整体摘要（150字以内）", + "topics": ["主题1", "主题2", "主题3"], + "insights": ["见解1", "见解2"], + "actions": ["行动建议1"], + "timeline": [ + {{"start_time": "00:00", "end_time": "05:00", "title": "第一部分", "summary": "..."}} + ] +}}""" + + try: + return self.generate_json(prompt) + except Exception as e: + print(f"[AIClient] 直接分析失败: {e}") + return { + 'summary': '分析失败', + 'topics': ['无法解析'], + 'insights': ['请检查内容格式'], + 'actions': [], + 'timeline': [] + } + + def _format_time(self, seconds: float) -> str: + """格式化秒数为时间字符串""" + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + secs = int(seconds % 60) + if hours > 0: + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + return f"{minutes:02d}:{secs:02d}" + + def _build_timeline_from_chunks(self, chunk_summaries: List[Dict]) -> List[Dict[str, Any]]: + """从分片摘要构建时间轴""" + timeline = [] + for summary in chunk_summaries: + timeline.append({ + 'index': summary['index'], + 'start_time': summary['start_time'], + 'end_time': summary['end_time'], + 'title': f"第{summary['index']+1}部分", + 'summary': summary['summary'][:200] + "..." if len(summary['summary']) > 200 else summary['summary'] + }) + return timeline + + def _mock_generate(self, prompt: str, max_tokens: int) -> str: + """模拟AI生成（实际实现应调用真实API）""" + # 根据prompt内容返回模拟响应 + prompt_lower = prompt.lower() + + if 'json' in prompt_lower or 'json格式' in prompt: + if '主题' in prompt or 'topic' in prompt_lower: + return json.dumps({ + 'summary': '这是一个关于人工智能和技术发展的内容，涵盖了多个重要主题。', + 'topics': ['人工智能', '机器学习', '未来发展', '技术趋势'], + 'insights': [ + 'AI正在改变我们的工作方式', + '机器学习应用越来越广泛', + '未来技术发展趋势值得关注' + ], + 'actions': ['深入研究相关技术', '关注行业动态'], + 'full_analysis': '详细分析内容...' + }, ensure_ascii=False) + return json.dumps({'result': '模拟JSON响应'}) + + if '摘要' in prompt or 'summary' in prompt_lower: + return '这是一段关于人工智能发展的重要讨论，涵盖了技术趋势和未来应用。' + + if '关键词' in prompt or 'keyword' in prompt_lower: + return '人工智能, 机器学习, 深度学习, 神经网络, 自然语言处理' + + return '模拟AI响应内容' + + def _parse_json_from_text(self, text: str) -> Dict[str, Any]: + """从文本中解析JSON""" + # 尝试提取JSON块 + import re + json_match = re.search(r'\{[\s\S]*\}', text) + if json_match: + try: + return json.loads(json_match.group()) + except: + pass + return {'raw_response': text} + + +class MultiAIClient: + """ + 多AI客户端 - 支持多种AI提供商的客户端 + """ + + def __init__(self): + self.config = get_config() + self.primary_client = AIClient() + self.clients = { + 'openai': self.primary_client, + 'local': AIClient(), # 可以配置为本地模型 + } + + def _deep_analyze(self, content_text: str, content_type: str, duration: Optional[int] = None) -> Dict[str, Any]: + """使用主客户端进行深度分析""" + return self.primary_client._deep_analyze(content_text, content_type, duration) + + def quick_generate(self, prompt: str, max_tokens: int = 100, provider: str = None) -> str: + """快速生成""" + client = self.clients.get(provider, self.primary_client) + return client.quick_generate(prompt, max_tokens) diff --git a/skills/kb-deep-analyzer/deep_analyzer.py b/skills/kb-deep-analyzer/deep_analyzer.py new file mode 100644 index 0000000..9e749d8 --- /dev/null +++ b/skills/kb-deep-analyzer/deep_analyzer.py @@ -0,0 +1,771 @@ +#!/usr/bin/env python3 +"""Deep Analyzer - AI 深度内容分析工具 + +支持根据内容时长选择不同分析策略： +- 短视频 (<10分钟): 轻量分析 +- 长视频 (>=10分钟): 详细时间轴分析 + +Usage: + python deep_analyzer.py --metadata meta.json --content content.json --output analysis.md [--type video] + python deep_analyzer.py --help +""" + +import argparse +import json +import os +import sys +import re +from datetime import datetime +from typing import Dict, Any, List, Optional, Tuple +from dataclasses import dataclass, field, asdict + + +# ============== 数据模型 ============== + +@dataclass +class TimelineEntry: + """时间轴条目""" + index: int = 0 + start_time: str = "" + end_time: str = "" + title: str = "" + summary: str = "" + + +@dataclass +class AnalysisResult: + """分析结果""" + summary: str = "" + main_topics: List[str] = field(default_factory=list) + key_insights: List[str] = field(default_factory=list) + action_items: List[str] = field(default_factory=list) + recommended_tags: List[str] = field(default_factory=list) + full_analysis: str = "" + timeline: List[Dict[str, Any]] = field(default_factory=list) + + +# ============== AI 客户端 ============== + +class SimpleAIClient: + """简化的 AI 客户端 - 支持多提供商 fallback""" + + # 时长阈值（秒） + SHORT_VIDEO_THRESHOLD = 600 # 10分钟 + LONG_VIDEO_THRESHOLD = 1800 # 30分钟 + + def __init__(self, api_key: str = None, provider: str = "mock", model: str = "gpt-4"): + self.api_key = api_key or os.environ.get("AI_API_KEY", "") + self.provider = provider + self.model = model + self.base_url = os.environ.get("AI_BASE_URL", "") + + def generate(self, prompt: str, max_tokens: int = 1000) -> str: + """生成文本""" + if not self.api_key or self.provider == "mock": + return self._mock_generate(prompt, max_tokens) + + try: + if self.provider == "openai": + return self._call_openai(prompt, max_tokens) + elif self.provider == "zhipu": + return self._call_zhipu(prompt, max_tokens) + elif self.provider == "minimax": + return self._call_minimax(prompt, max_tokens) + else: + return self._mock_generate(prompt, max_tokens) + except Exception as e: + print(f"[AI Client] 调用失败: {e}") + return self._mock_generate(prompt, max_tokens) + + def generate_json(self, prompt: str) -> Dict[str, Any]: + """生成 JSON 格式响应""" + response = self.generate(prompt, max_tokens=2000) + try: + return json.loads(response) + except json.JSONDecodeError: + # 尝试从文本中提取 JSON + return self._parse_json_from_text(response) + + def _call_openai(self, prompt: str, max_tokens: int) -> str: + """调用 OpenAI API""" + import requests + url = f"{self.base_url}/chat/completions" if self.base_url else "https://api.openai.com/v1/chat/completions" + + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + payload = { + "model": self.model, + "max_tokens": max_tokens, + "messages": [{"role": "user", "content": prompt}] + } + + response = requests.post(url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + return response.json()['choices'][0]['message']['content'] + + def _call_zhipu(self, prompt: str, max_tokens: int) -> str: + """调用智谱 AI API""" + import requests + base_url = self.base_url or "https://open.bigmodel.cn/api/paas/v4" + url = f"{base_url}/chat/completions" + + headers = {"Authorization": f"Bearer {self.api_key}"} + payload = { + "model": self.model, + "messages": [{"role": "user", "content": prompt}] + } + + response = requests.post(url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + return response.json()['choices'][0]['message']['content'] + + def _call_minimax(self, prompt: str, max_tokens: int) -> str: + """调用 MiniMax API""" + import requests + url = self.base_url or "https://api.minimaxi.com/anthropic/v1/messages" + + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + payload = { + "model": self.model, + "max_tokens": max_tokens, + "messages": [{"role": "user", "content": prompt}] + } + + response = requests.post(url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + data = response.json() + for item in data.get('content', []): + if item.get('type') == 'text': + return item.get('text', '') + return str(data) + + def _mock_generate(self, prompt: str, max_tokens: int) -> str: + """模拟 AI 生成""" + prompt_lower = prompt.lower() + + if 'json' in prompt_lower or '请返回' in prompt: + # 返回模拟 JSON 响应 + if '主题' in prompt or 'topic' in prompt_lower: + return json.dumps({ + 'summary': '这是一个模拟的AI分析结果，包含内容的主要主题和关键见解。', + 'topics': ['主题一', '主题二', '主题三', '主题四'], + 'insights': [ + 'AI正在深刻改变内容消费方式', + '个性化推荐提升了用户体验', + '技术发展带来新的可能性' + ], + 'actions': ['深入研究相关技术', '关注行业动态'], + 'full_analysis': '详细分析内容基于模拟AI生成。' + }, ensure_ascii=False) + if '标签' in prompt or 'tag' in prompt_lower: + return json.dumps({'tags': ['技术', 'AI', '视频分析', '深度学习']}, ensure_ascii=False) + return json.dumps({'result': '模拟响应'}, ensure_ascii=False) + + if '摘要' in prompt or 'summary' in prompt_lower: + return '这是一段关于内容深度分析的重要讨论，涵盖了多个关键主题和见解。' + + return '模拟AI响应内容' + + def _parse_json_from_text(self, text: str) -> Dict[str, Any]: + """从文本中解析 JSON""" + json_match = re.search(r'\{[\s\S]*\}', text) + if json_match: + try: + return json.loads(json_match.group()) + except: + pass + return {'raw_response': text} + + +# ============== 深度分析器 ============== + +class DeepAnalyzer: + """深度内容分析器""" + + SHORT_VIDEO_THRESHOLD = 600 # 10分钟 + LONG_VIDEO_THRESHOLD = 1800 # 30分钟 + + def __init__(self, ai_client: SimpleAIClient = None): + self.ai_client = ai_client or SimpleAIClient() + + def analyze(self, metadata: Dict[str, Any], content: Dict[str, Any], + content_type: str = "video") -> AnalysisResult: + """ + 分析内容 + + Args: + metadata: 元数据字典 + content: 内容数据字典（包含 subtitle, description 等） + content_type: 内容类型 + + Returns: + AnalysisResult 分析结果 + """ + title = metadata.get('title', '未知') + duration = metadata.get('duration', 0) + subtitle = content.get('subtitle', '') + description = content.get('description', '') + + print(f"[DeepAnalyzer] 开始分析: {title}") + print(f"[DeepAnalyzer] 类型: {content_type}, 时长: {duration}秒") + print(f"[DeepAnalyzer] 字幕存在: {bool(subtitle and subtitle.strip())}") + + has_subtitle = bool(subtitle and subtitle.strip()) + + if has_subtitle: + if duration < self.SHORT_VIDEO_THRESHOLD: + print(f"[DeepAnalyzer] 使用轻量分析策略 (时长 < 10分钟)") + result = self._analyze_short_with_subtitle(title, subtitle, duration) + else: + print(f"[DeepAnalyzer] 使用详细时间轴分析策略 (时长 >= 10分钟)") + result = self._analyze_long_with_subtitle(title, subtitle, duration, content_type) + else: + print(f"[DeepAnalyzer] 降级处理（无字幕）") + result = self._analyze_fallback(title, description, metadata) + + return result + + def _analyze_short_with_subtitle(self, title: str, subtitle: str, duration: int) -> AnalysisResult: + """有字幕的短视频轻量分析""" + result = AnalysisResult() + text_sample = self._get_text_sample(subtitle, max_chars=5000) + + analysis = self._analyze_with_ai_short_text(text_sample, title, duration) + result.summary = analysis.get('summary', title) + result.main_topics = analysis.get('topics', []) + result.key_insights = analysis.get('insights', []) + result.action_items = analysis.get('actions', []) + result.timeline = self._generate_simple_timeline_from_duration(title, duration) + + return result + + def _analyze_long_with_subtitle(self, title: str, subtitle: str, duration: int, + content_type: str) -> AnalysisResult: + """有字幕的长视频详细分析""" + result = AnalysisResult() + + if duration >= self.LONG_VIDEO_THRESHOLD and content_type == "video": + # 使用分段分析 + result = self._analyze_in_chunks(title, subtitle, duration) + else: + # 直接分析 + text_sample = self._get_text_sample(subtitle, max_chars=5000) + analysis = self._analyze_with_ai_short_text(text_sample, title, duration) + result.summary = analysis.get('summary', title) + result.main_topics = analysis.get('topics', []) + result.key_insights = analysis.get('insights', []) + result.action_items = analysis.get('actions', []) + result.full_analysis = analysis.get('full_analysis', '') + result.timeline = self._generate_detailed_timeline(title, duration) + + return result + + def _analyze_in_chunks(self, title: str, subtitle: str, duration: int) -> AnalysisResult: + """分段分析长视频""" + result = AnalysisResult() + + # 确定分段数 + if duration < 3600: # <60分钟 + num_segments = 3 + elif duration < 7200: # 60-120分钟 + num_segments = 4 + else: # >120分钟 + num_segments = 6 + + # 生成分段摘要 + segment_duration = duration // num_segments + chunk_summaries = [] + + # 简单按字符数分割字幕 + subtitle_chunks = self._chunk_subtitle(subtitle, num_segments) + + for i, chunk_text in enumerate(subtitle_chunks): + start_sec = i * segment_duration + end_sec = min((i + 1) * segment_duration, duration) + start_time = self._format_time(start_sec) + end_time = self._format_time(end_sec) + + # AI 分析每段 + chunk_summary = self._analyze_chunk(chunk_text, i + 1) + chunk_summaries.append({ + 'index': i, + 'start_time': start_time, + 'end_time': end_time, + 'summary': chunk_summary + }) + + # 合并结果 + result = self._merge_summaries(chunk_summaries, title) + result.timeline = self._build_timeline_from_chunks(chunk_summaries) + + return result + + def _analyze_chunk(self, chunk_text: str, chunk_index: int) -> str: + """分析单个分片""" + text_sample = self._get_text_sample(chunk_text, max_chars=3000) + + prompt = f"""请对以下视频片段进行简要分析（限制在300字以内）： + +片段内容： +{text_sample} + +请用中文简要回答，包括核心主题和关键要点。""" + + try: + return self.ai_client.generate(prompt, max_tokens=500) + except Exception as e: + print(f"[DeepAnalyzer] 分片 {chunk_index} 分析失败: {e}") + return f"第{chunk_index}段内容分析" + + def _merge_summaries(self, chunk_summaries: List[Dict], title: str) -> AnalysisResult: + """合并分段摘要""" + summaries_text = "\n\n".join([ + f"=== 第{s['index']+1}段 ({s['start_time']}-{s['end_time']}) ===\n{s['summary']}" + for s in chunk_summaries + ]) + + prompt = f"""基于以下视频各段分析，生成完整的分析报告： + +标题：{title} + +各段内容： +{summaries_text} + +请返回JSON格式： +{{ + "summary": "整体摘要（150字以内）", + "topics": ["主题1", "主题2", "主题3", "主题4"], + "insights": ["见解1", "见解2", "见解3"], + "actions": ["建议1", "建议2"], + "full_analysis": "详细分析（500字以内）" +}}""" + + try: + analysis = self.ai_client.generate_json(prompt) + result = AnalysisResult() + result.summary = analysis.get('summary', title) + result.main_topics = analysis.get('topics', []) + result.key_insights = analysis.get('insights', []) + result.action_items = analysis.get('actions', []) + result.full_analysis = analysis.get('full_analysis', summaries_text) + return result + except Exception as e: + print(f"[DeepAnalyzer] 合并摘要失败: {e}") + return AnalysisResult( + summary=f"共{len(chunk_summaries)}段分析完成", + main_topics=[f"主题{i+1}" for i in range(min(4, len(chunk_summaries)))], + key_insights=["综合各段分析得出"], + full_analysis=summaries_text[:500] + ) + + def _analyze_fallback(self, title: str, description: str, metadata: Dict) -> AnalysisResult: + """无字幕降级处理""" + result = AnalysisResult() + + if description and description.strip(): + summary = description[:200] + "..." if len(description) > 200 else description + else: + summary = f"无字幕内容，标题：{title}" + + # 提取标签 + tags = ["未分类", "待处理"] + author = metadata.get('author', '') + source = metadata.get('source', '') + if author: + tags.append(f"作者:{author}") + if source: + tags.append(f"来源:{source}") + + result.summary = summary + result.main_topics = tags[:3] + result.recommended_tags = tags + result.key_insights = ["无字幕内容，仅能从描述获取基本信息"] + result.action_items = ["建议补充字幕以便深度分析"] + + return result + + def _analyze_with_ai_short_text(self, text: str, title: str, duration: int) -> Dict[str, Any]: + """AI 短视频分析""" + prompt = f"""请分析以下视频内容，以JSON格式返回： + +标题：{title} +时长：{self._format_time(duration)} + +字幕内容： +{text[:3000]} + +请返回以下格式JSON： +{{ + "summary": "整体摘要（100字以内）", + "topics": ["主题1", "主题2", "主题3"], + "insights": ["见解1", "见解2"], + "actions": ["建议1"] +}}""" + + try: + return self.ai_client.generate_json(prompt) + except Exception as e: + print(f"[DeepAnalyzer] AI分析失败: {e}") + return { + "summary": title, + "topics": ["主题1", "主题2", "主题3"], + "insights": ["无法获取深入见解"], + "actions": [] + } + + def _get_text_sample(self, text: str, max_chars: int) -> str: + """获取文本样本（头尾采样）""" + if len(text) <= max_chars: + return text + + part_size = max_chars // 3 + return ( + text[:part_size] + + "\n\n[...省略中间部分...]\n\n" + + text[-part_size:] + ) + + def _chunk_subtitle(self, subtitle: str, num_chunks: int) -> List[str]: + """简单分割字幕""" + if not subtitle: + return [""] * num_chunks + + chunk_size = len(subtitle) // num_chunks + chunks = [] + + for i in range(num_chunks): + start = i * chunk_size + end = start + chunk_size if i < num_chunks - 1 else len(subtitle) + chunks.append(subtitle[start:end]) + + return chunks + + def _generate_simple_timeline_from_duration(self, title: str, duration: int) -> List[Dict]: + """生成简单时间轴""" + segments = 2 if duration < 300 else 3 + seg_duration = duration // segments + + timeline = [] + for i in range(segments): + start = i * seg_duration + end = min((i + 1) * seg_duration, duration) + timeline.append({ + 'index': i + 1, + 'start_time': self._format_time(start), + 'end_time': self._format_time(end), + 'title': f"段落 {i+1}", + 'summary': f"从{self._format_time(start)}到{self._format_time(end)}的内容" + }) + return timeline + + def _generate_detailed_timeline(self, title: str, duration: int) -> List[Dict]: + """生成详细时间轴""" + return self._generate_simple_timeline_from_duration(title, duration) + + def _build_timeline_from_chunks(self, chunk_summaries: List[Dict]) -> List[Dict]: + """从分段摘要构建时间轴""" + timeline = [] + for s in chunk_summaries: + timeline.append({ + 'index': s['index'] + 1, + 'start_time': s['start_time'], + 'end_time': s['end_time'], + 'title': f"第{s['index']+1}部分", + 'summary': s['summary'][:200] + "..." if len(s['summary']) > 200 else s['summary'] + }) + return timeline + + def _format_time(self, seconds: int) -> str: + """格式化秒数为时间字符串""" + h = seconds // 3600 + m = (seconds % 3600) // 60 + s = seconds % 60 + if h > 0: + return f"{h}:{m:02d}:{s:02d}" + return f"{m}:{s:02d}" + + +# ============== 模板系统 ============== + +class TemplateLoader: + """模板加载器""" + + TEMPLATES = { + "video": """# {{title}} + +## 基本信息 +- **时长**: {{duration}} +- **作者**: {{author}} +- **来源**: {{source}} +- **分析时间**: {{analyze_time}} + +## 摘要 +{{summary}} + +## 主要主题 +{% for topic in topics %} +- {{topic}} +{% endfor %} + +## 关键见解 +{% for insight in insights %} +- {{insight}} +{% endfor %} + +## 建议行动 +{% for action in actions %} +- [ ] {{action}} +{% endfor %} + +{% if timeline %} +## 时间轴 +{% for entry in timeline %} +### {{entry.index}}. {{entry.start_time}} - {{entry.end_time}} +{{entry.title}}: {{entry.summary}} +{% endfor %} +{% endif %} + +{% if tags %} +## 标签 +{% for tag in tags %} +- {{tag}} +{% endfor %} +{% endif %} +""", + "article": """# {{title}} + +## 基本信息 +- **作者**: {{author}} +- **来源**: {{source}} +- **分析时间**: {{analyze_time}} + +## 摘要 +{{summary}} + +## 主要主题 +{% for topic in topics %} +- {{topic}} +{% endfor %} + +## 关键见解 +{% for insight in insights %} +- {{insight}} +{% endfor %} + +## 建议行动 +{% for action in actions %} +- [ ] {{action}} +{% endfor %} + +{% if tags %} +## 标签 +{% for tag in tags %} +- {{tag}} +{% endfor %} +{% endif %} +""" + } + + def load_template(self, template_type: str = "video") -> str: + """加载模板""" + # 优先检查环境变量 + template_path = os.environ.get(f"TEMPLATE_{template_type.upper()}", "") + if template_path and os.path.exists(template_path): + with open(template_path, 'r', encoding='utf-8') as f: + return f.read() + + return self.TEMPLATES.get(template_type, self.TEMPLATES["video"]) + + +class TemplateRenderer: + """简单模板渲染器""" + + def render(self, template: str, variables: Dict[str, Any]) -> str: + """渲染模板""" + result = template + + # 替换变量 + for key, value in variables.items(): + placeholder = f"{{{{{key}}}}}" + result = result.replace(placeholder, str(value)) + + # 渲染 for 循环 (简化版本) + # 匹配 {% for item in list %} ... {% endfor %} + pattern = r'\{%\s*for\s+(\w+)\s+in\s+(\w+)\s*%\}(.*?)\{%\s*endfor\s*%\}' + + def replace_loop(match): + item_name = match.group(1) + list_name = match.group(2) + content = match.group(3) + + items = variables.get(list_name, []) + rendered = [] + for item in items: + if isinstance(item, dict): + # 字典项 + item_content = content + for k, v in item.items(): + item_content = item_content.replace(f"{{{{{item_name}.{k}}}}}", str(v)) + rendered.append(item_content) + else: + # 简单项 + rendered.append(content.replace(f"{{{{{item_name}}}}}", str(item))) + + return '\n'.join(rendered) + + result = re.sub(pattern, replace_loop, result, flags=re.DOTALL) + + # 移除空的 for 循环块 + result = re.sub(r'\{%\s*for\s+\w+\s+in\s+\w+\s*%\}\s*\{%\s*endfor\s*%\}', '', result) + + return result + + +# ============== 工具函数 ============== + +def load_json(file_path: str) -> Dict: + """加载 JSON 文件""" + if not file_path or not os.path.exists(file_path): + return {} + + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) + + +def format_duration(seconds: int) -> str: + """格式化时长""" + h = seconds // 3600 + m = (seconds % 3600) // 60 + s = seconds % 60 + if h > 0: + return f"{h}:{m:02d}:{s:02d}" + return f"{m}:{s:02d}" + + +def result_to_dict(result: AnalysisResult) -> Dict: + """将分析结果转换为字典""" + return { + "summary": result.summary, + "main_topics": result.main_topics, + "key_insights": result.key_insights, + "action_items": result.action_items, + "recommended_tags": result.recommended_tags, + "full_analysis": result.full_analysis, + "timeline": result.timeline + } + + +def generate_frontmatter(metadata: Dict, result: AnalysisResult) -> str: + """生成 Markdown frontmatter""" + tags = result.recommended_tags or result.main_topics[:5] + + fm = ["---", "title: \"" + metadata.get('title', '未命名') + "\"", + f"date: {datetime.now().strftime('%Y-%m-%d')}", + f"duration: {format_duration(metadata.get('duration', 0))}", + f"author: {metadata.get('author', '未知')}"] + + if tags: + fm.append("tags: [" + ", ".join(f'"{t}"' for t in tags) + "]") + + fm.append("---") + return '\n'.join(fm) + + +# ============== 主程序 ============== + +def main(): + parser = argparse.ArgumentParser( + description="Deep Analyzer - AI 深度内容分析工具", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + python deep_analyzer.py --metadata meta.json --content content.json --output analysis.md + python deep_analyzer.py --metadata meta.json --content content.json --type video --api-key xxx + """ + ) + + parser.add_argument("--metadata", "-m", required=True, + help="元数据 JSON 文件路径") + parser.add_argument("--content", "-c", required=True, + help="内容 JSON 文件路径") + parser.add_argument("--output", "-o", required=True, + help="输出 Markdown 文件路径") + parser.add_argument("--type", "-t", default="video", choices=["video", "article"], + help="内容类型 (默认: video)") + parser.add_argument("--api-key", "-k", default=None, + help="AI API Key (或设置 AI_API_KEY 环境变量)") + parser.add_argument("--provider", "-p", default="mock", + choices=["openai", "zhipu", "minimax", "mock"], + help="AI 提供商 (默认: mock)") + parser.add_argument("--model", default="gpt-4", + help="AI 模型 (默认: gpt-4)") + parser.add_argument("--no-frontmatter", action="store_true", + help="不生成 frontmatter") + + args = parser.parse_args() + + # 加载数据 + print(f"[Main] 加载元数据: {args.metadata}") + metadata = load_json(args.metadata) + + print(f"[Main] 加载内容: {args.content}") + content = load_json(args.content) + + # 初始化 + ai_client = SimpleAIClient( + api_key=args.api_key, + provider=args.provider, + model=args.model + ) + analyzer = DeepAnalyzer(ai_client) + + # 执行分析 + print(f"[Main] 开始分析...") + result = analyzer.analyze(metadata, content, args.type) + + # 渲染模板 + template_loader = TemplateLoader() + renderer = TemplateRenderer() + + template = template_loader.load_template(args.type) + + variables = { + "title": metadata.get('title', '未命名'), + "duration": format_duration(metadata.get('duration', 0)), + "author": metadata.get('author', '未知'), + "source": metadata.get('source', ''), + "analyze_time": datetime.now().strftime('%Y-%m-%d %H:%M'), + "summary": result.summary, + "topics": result.main_topics, + "insights": result.key_insights, + "actions": result.action_items, + "tags": result.recommended_tags, + "timeline": result.timeline + } + + markdown_content = renderer.render(template, variables) + + # 添加 frontmatter + if not args.no_frontmatter: + frontmatter = generate_frontmatter(metadata, result) + markdown_content = frontmatter + "\n\n" + markdown_content + + # 写入输出 + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + with open(args.output, 'w', encoding='utf-8') as f: + f.write(markdown_content) + + print(f"[Main] 分析完成！输出: {args.output}") + print(f"[Main] 摘要: {result.summary[:100]}...") + print(f"[Main] 主题数: {len(result.main_topics)}") + + +if __name__ == "__main__": + main() diff --git a/skills/kb-link-parser/SKILL.md b/skills/kb-link-parser/SKILL.md new file mode 100644 index 0000000..b8c97cc --- /dev/null +++ b/skills/kb-link-parser/SKILL.md @@ -0,0 +1,43 @@ +# KB Link Parser Skill + +**kb-link-parser** - Extract and resolve IDs from various platform URLs. + +## Features + +- Multi-platform support: Bilibili, YouTube, Zhihu +- Short link detection: b23.tv and other short links +- Multiple content types: video, article, up主, playlist, collection +- Batch processing from file +- JSON output format + +## Commands + +```bash +# Parse single URL +python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD" + +# Batch processing +python link-parser.py --batch urls.txt --output results.json + +# Output format +python link-parser.py --url "..." --format json +``` + +## Output + +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "video", + "id": "BV1xx411c7mD" + } +} +``` + +## Installation + +```bash +cp -r kb-link-parser/ ~/skills/ +``` diff --git a/skills/kb-link-parser/link-parser.py b/skills/kb-link-parser/link-parser.py new file mode 100644 index 0000000..84981d5 --- /dev/null +++ b/skills/kb-link-parser/link-parser.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +""" +Link Parser - Extract and resolve IDs from various platform URLs. + +Supports: +- Bilibili (video, article, up主, collection, series) +- YouTube (video, channel, playlist) +- Zhihu (question, answer, user, collection) +- Short links (b23.tv, etc.) + +Usage: + python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD" + python link-parser.py --batch urls.txt --output results.json +""" + +import argparse +import json +import re +import sys +from typing import Optional, Dict, Any +from urllib.parse import urlparse + +from link_parser_shortlink import ShortLinkResolver + + +class LinkParser: + """Main link parser class""" + + # Bilibili patterns + BILIBILI_VIDEO_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV[a-zA-Z0-9]+)') + BILIBILI_VIDEO_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)') + BILIBILI_ARTICLE_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/article/([a-zA-Z0-9]+)') + BILIBILI_SPACE_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/(\d+)') + BILIBILI_FAVEPATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/opus/(\d+)') + BILIBILI_SERIES_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/series/([a-zA-Z0-9]+)') + + # YouTube patterns + YOUTUBE_VIDEO_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/)([a-zA-Z0-9_-]{11})') + YOUTUBE_CHANNEL_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/channel/([a-zA-Z0-9_-]+)') + YOUTUBE_USER_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/user/([a-zA-Z0-9_-]+)') + YOUTUBE_PLAYLIST_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/playlist\?list=([a-zA-Z0-9_-]+)') + + # Zhihu patterns + ZHIHAU_QUESTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/(\d+)') + ZHIHAU_ANSWER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)') + ZHIHAU_USER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/people/([a-zA-Z0-9_-]+)') + ZHIHAU_COLUMN_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/column/([a-zA-Z0-9_-]+)') + ZHIHAU_COLLECTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/collection/(\d+)') + + def __init__(self): + self.shortlink_resolver = ShortLinkResolver() + + def resolve_shortlink(self, url: str) -> Optional[str]: + """ + Resolve short link to get final URL. + + Args: + url: Short URL to resolve + + Returns: + Final URL after resolution, or None if not resolvable + """ + # Check if it's a known short link pattern + if self.shortlink_resolver.is_bilibili_short_link(url): + # For b23.tv, we need Playwright to resolve + # Return the short URL as-is with a note + return url + + # Return original URL for other cases + return url + + def extract_bilibili_id(self, url: str) -> Optional[Dict[str, str]]: + """ + Extract Bilibili ID from URL. + + Args: + url: Bilibili URL + + Returns: + Dict with platform, type, and id fields, or None if not matched + """ + # Try video + match = self.BILIBILI_VIDEO_PATTERN.match(url) + if match: + return { + "platform": "bilibili", + "type": "video", + "id": match.group(1) + } + + # Try article + match = self.BILIBILI_ARTICLE_PATTERN.match(url) + if match: + return { + "platform": "bilibili", + "type": "article", + "id": match.group(1) + } + + # Try space (UP主) + match = self.BILIBILI_SPACE_PATTERN.match(url) + if match: + return { + "platform": "bilibili", + "type": "up主", + "id": match.group(1) + } + + # Try collection + match = self.BILIBILI_FAVEPATTERN.match(url) + if match: + return { + "platform": "bilibili", + "type": "collection", + "id": match.group(1) + } + + # Try series + match = self.BILIBILI_SERIES_PATTERN.match(url) + if match: + return { + "platform": "bilibili", + "type": "series", + "id": match.group(1) + } + + return None + + def extract_youtube_id(self, url: str) -> Optional[Dict[str, str]]: + """ + Extract YouTube ID from URL. + + Args: + url: YouTube URL + + Returns: + Dict with platform, type, and id fields, or None if not matched + """ + # Try video + match = self.YOUTUBE_VIDEO_PATTERN.match(url) + if match: + return { + "platform": "youtube", + "type": "video", + "id": match.group(1) + } + + # Try channel + match = self.YOUTUBE_CHANNEL_PATTERN.match(url) + if match: + return { + "platform": "youtube", + "type": "channel", + "id": match.group(1) + } + + # Try user + match = self.YOUTUBE_USER_PATTERN.match(url) + if match: + return { + "platform": "youtube", + "type": "user", + "id": match.group(1) + } + + # Try playlist + match = self.YOUTUBE_PLAYLIST_PATTERN.match(url) + if match: + return { + "platform": "youtube", + "type": "playlist", + "id": match.group(1) + } + + return None + + def extract_zhihu_id(self, url: str) -> Optional[Dict[str, str]]: + """ + Extract Zhihu ID from URL. + + Args: + url: Zhihu URL + + Returns: + Dict with platform, type, and id fields, or None if not matched + """ + # Try question + match = self.ZHIHAU_QUESTION_PATTERN.match(url) + if match: + return { + "platform": "zhihu", + "type": "question", + "id": match.group(1) + } + + # Try answer + match = self.ZHIHAU_ANSWER_PATTERN.match(url) + if match: + return { + "platform": "zhihu", + "type": "answer", + "id": match.group(1) + } + + # Try user + match = self.ZHIHAU_USER_PATTERN.match(url) + if match: + return { + "platform": "zhihu", + "type": "user", + "id": match.group(1) + } + + # Try column + match = self.ZHIHAU_COLUMN_PATTERN.match(url) + if match: + return { + "platform": "zhihu", + "type": "column", + "id": match.group(1) + } + + # Try collection + match = self.ZHIHAU_COLLECTION_PATTERN.match(url) + if match: + return { + "platform": "zhihu", + "type": "collection", + "id": match.group(1) + } + + return None + + def parse(self, url: str) -> Dict[str, Any]: + """ + Parse URL and extract platform-specific ID. + + Args: + url: URL to parse + + Returns: + Dict with platform, type, id, and original_url fields + """ + result = { + "platform": None, + "type": None, + "id": None, + "original_url": url, + "resolved_url": None, + "success": False, + "error": None + } + + # First, resolve short links + resolved = self.resolve_shortlink(url) + if resolved: + result["resolved_url"] = resolved + url = resolved + + # Try each platform + bilibili_result = self.extract_bilibili_id(url) + if bilibili_result: + result.update(bilibili_result) + result["success"] = True + return result + + youtube_result = self.extract_youtube_id(url) + if youtube_result: + result.update(youtube_result) + result["success"] = True + return result + + zhihu_result = self.extract_zhihu_id(url) + if zhihu_result: + result.update(zhihu_result) + result["success"] = True + return result + + # Check for short links that need resolution + short_links = self.shortlink_resolver.detect_short_links(url) + if short_links: + result["error"] = "Short link detected. Please provide the full URL." + return result + + result["error"] = "Unsupported URL format" + return result + + +def format_output(data: Dict[str, Any], output_format: str = "json") -> str: + """Format output based on specified format.""" + if output_format == "json": + return json.dumps(data, ensure_ascii=False, indent=2) + elif output_format == "compact": + return json.dumps(data, ensure_ascii=False, separators=(',', ':')) + elif output_format == "text": + lines = [] + for key, value in data.items(): + if value is not None: + lines.append(f"{key}: {value}") + return "\n".join(lines) + else: + return json.dumps(data, ensure_ascii=False, indent=2) + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Parse and extract IDs from various platform URLs", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s --url "https://www.bilibili.com/video/BV1xx411c7mD" + %(prog)s --url "https://www.youtube.com/watch?v=dQw4w9WgXcQ" + %(prog)s --url "https://www.zhihu.com/question/266309957" + %(prog)s --batch urls.txt --output results.json + """ + ) + + parser.add_argument( + "--url", "-u", + help="Single URL to parse" + ) + + parser.add_argument( + "--batch", "-b", + help="File containing URLs to parse (one per line)" + ) + + parser.add_argument( + "--output", "-o", + help="Output file path (for batch mode)" + ) + + parser.add_argument( + "--format", "-f", + choices=["json", "compact", "text"], + default="json", + help="Output format (default: json)" + ) + + args = parser.parse_args() + + # Check if we're in shortlink help mode + if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']: + parser.print_help() + return 0 + + if not args.url and not args.batch: + parser.print_help() + print("\nError: Please provide --url or --batch") + return 1 + + link_parser = LinkParser() + + if args.url: + # Single URL mode + result = link_parser.parse(args.url) + print(format_output(result, args.format)) + return 0 if result["success"] else 1 + + elif args.batch: + try: + with open(args.batch, 'r', encoding='utf-8') as f: + urls = [line.strip() for line in f if line.strip()] + except Exception as e: + print(f"Error reading file: {e}", file=sys.stderr) + return 1 + + results = [] + for url in urls: + result = link_parser.parse(url) + results.append(result) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + json.dump(results, f, ensure_ascii=False, indent=2) + print(f"Results written to {args.output}") + else: + for result in results: + print(format_output(result, args.format)) + print("-" * 40) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/skills/kb-link-parser/link_parser_shortlink.py b/skills/kb-link-parser/link_parser_shortlink.py new file mode 100644 index 0000000..20739e7 --- /dev/null +++ b/skills/kb-link-parser/link_parser_shortlink.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +""" +Short link resolver for link-parser skill. + +Provides utilities for detecting and resolving short links, particularly +Bilibili short links (b23.tv) which may be blocked by anti-scraping measures. + +Supported short link services: +- Bilibili: b23.tv +- Generic URL shorteners (detection only) +""" + +import re +from typing import Optional, Dict, List, Tuple +from urllib.parse import urlparse + + +class ShortLinkResolver: + """Short link detection and resolution handler""" + + # Pattern for Bilibili short links + BILIBILI_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)') + + # Pattern for other short link services + OTHER_SHORT_PATTERNS = { + 'bit.ly': re.compile(r'https?://bit\.ly/\w+'), + 't.cn': re.compile(r'https?://t\.cn/\w+'), + 'dwz.cn': re.compile(r'https?://dwz\.cn/\w+'), + 'goo.gl': re.compile(r'https?://goo\.gl/\w+'), + 'tinyurl.com': re.compile(r'https?://tinyurl\.com/\w+'), + } + + @staticmethod + def detect_short_links(text: str) -> List[Dict[str, str]]: + """ + Detect short links in text. + + Args: + text: Input text to search + + Returns: + List of detected short links with metadata: + [ + { + 'url': 'https://b23.tv/xxxxx', + 'platform': 'bilibili', + 'is_short': True + }, + ... + ] + """ + detected = [] + + # Check for Bilibili short links + for match in ShortLinkResolver.BILIBILI_SHORT_PATTERN.finditer(text): + detected.append({ + 'url': match.group(0), + 'code': match.group(1), + 'platform': 'bilibili', + 'is_short': True + }) + + # Check for other short link services + for platform, pattern in ShortLinkResolver.OTHER_SHORT_PATTERNS.items(): + for match in pattern.finditer(text): + detected.append({ + 'url': match.group(0), + 'platform': platform, + 'is_short': True + }) + + return detected + + @staticmethod + def is_bilibili_short_link(url: str) -> bool: + """ + Check if URL is a Bilibili short link. + + Args: + url: URL to check + + Returns: + True if URL is a b23.tv short link + """ + return bool(ShortLinkResolver.BILIBILI_SHORT_PATTERN.match(url)) + + @staticmethod + def format_short_link_help(short_link: Optional[str] = None) -> str: + """ + Format help message for short links. + + Args: + short_link: Optional short link URL to mention + + Returns: + Help message + """ + lines = [ + "Detection: Short link", + "" + ] + + if short_link: + lines.append(f"Short link: {short_link}") + lines.append("") + + lines.extend([ + "Due to anti-scraping protection, short links cannot be automatically resolved.", + "Please provide the full URL or the content ID (e.g., BV号 for Bilibili).", + "", + "Full URL format: https://www.bilibili.com/video/BVxxxxx" + ]) + + return "\n".join(lines) + + @staticmethod + def format_short_link_warning(short_links: List[Dict[str, str]]) -> str: + """ + Format warning message for multiple short links. + + Args: + short_links: List of short link metadata + + Returns: + Warning message + """ + if not short_links: + return "" + + lines = [ + "Short links detected:", + "" + ] + + for i, link in enumerate(short_links, 1): + platform = link.get('platform', 'unknown') + url = link['url'] + + if platform == 'bilibili': + lines.append(f" [{i}] Bilibili short link: {url}") + lines.append(" Full URL or BV ID required") + else: + lines.append(f" [{i}] {platform}: {url}") + + lines.append("") + + return "\n".join(lines) + + +class PlaywrightResolver: + """ + Browser-based short link resolver using Playwright. + + This is an optional implementation for environments where Playwright + is available. It can bypass anti-scraping measures by simulating + a real browser. + + Requirements: + pip install playwright + playwright install chromium + """ + + def __init__(self, headless: bool = True, timeout: int = 10000): + """ + Initialize Playwright resolver. + + Args: + headless: Whether to run browser in headless mode + timeout: Page navigation timeout in milliseconds + """ + self.headless = headless + self.timeout = timeout + self._playwright = None + + def _check_playwright_available(self) -> bool: + """Check if Playwright is installed and available""" + try: + from playwright.sync_api import sync_playwright + return True + except ImportError: + return False + + def resolve(self, short_url: str) -> Optional[str]: + """ + Resolve short link using Playwright browser automation. + + Args: + short_url: Short URL to resolve + + Returns: + Final URL after redirects, or None if resolution fails + """ + if not self._check_playwright_available(): + raise ImportError( + "Playwright not installed. Install with: pip install playwright && playwright install chromium" + ) + + from playwright.sync_api import sync_playwright + + try: + with sync_playwright() as p: + # Launch browser + browser = p.chromium.launch(headless=self.headless) + context = browser.new_context( + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + viewport={'width': 1920, 'height': 1080} + ) + page = context.new_page() + + # Navigate to short URL + page.goto(short_url, timeout=self.timeout, wait_until='networkidle') + + # Get final URL + final_url = page.url + + # Close browser + browser.close() + + return final_url + + except Exception as e: + print(f"Playwright resolution failed: {e}") + return None + + def resolve_bilibili_short(self, short_url: str) -> Optional[str]: + """ + Resolve Bilibili short link and extract BV ID. + + Args: + short_url: Bilibili short URL (e.g., https://b23.tv/xxxxx) + + Returns: + Full Bilibili URL with BV ID, or None if resolution fails + """ + final_url = self.resolve(short_url) + if not final_url: + return None + + # Check if final URL contains BV ID + if 'bilibili.com/video/' in final_url: + return final_url + + # Extract BV ID if present in URL + bv_pattern = re.compile(r'(BV[a-zA-Z0-9]{10})') + match = bv_pattern.search(final_url) + if match: + return f"https://www.bilibili.com/video/{match.group(1)}" + + return None + + +def get_resolver() -> ShortLinkResolver: + """ + Get the default resolver instance. + + Returns: + ShortLinkResolver instance + """ + return ShortLinkResolver() + + +def detect_short_links(text: str) -> List[Dict[str, str]]: + """ + Detect short links in text. + + Args: + text: Input text to search + + Returns: + List of detected short links with metadata + """ + resolver = ShortLinkResolver() + return resolver.detect_short_links(text) + + +# CLI interface for testing +def main(): + """CLI for testing the short link resolver.""" + import sys + + if len(sys.argv) < 2: + print("Usage: python3 link_parser_shortlink.py 'text to test'") + print("\nExamples:") + print(' python3 link_parser_shortlink.py "Check this: https://b23.tv/oLBZZfK"') + print(' python3 link_parser_shortlink.py "https://bit.ly/xxxxx"') + sys.exit(1) + + text = ' '.join(sys.argv[1:]) + short_links = detect_short_links(text) + + import json + print(json.dumps(short_links, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/skills/kb-ocr-processor/SKILL.md b/skills/kb-ocr-processor/SKILL.md new file mode 100644 index 0000000..a2ebdc5 --- /dev/null +++ b/skills/kb-ocr-processor/SKILL.md @@ -0,0 +1,43 @@ +# KB OCR Processor Skill + +**kb-ocr-processor** - Extract text from video frames using OCR. + +## Features + +- Batch frame processing +- PaddleOCR/Tesseract support +- Confidence scoring +- Text merging + +## Commands + +```bash +# Process frames +python ocr_processor.py --input ./frames/ --output ocr.json + +# With custom FPS +python ocr_processor.py --input ./frames/ --fps 0.1 --output ocr.json +``` + +## Output + +```json +{ + "success": true, + "data": { + "frames": { + "frame_0001.jpg": { + "texts": [{"text": "字幕", "confidence": 0.95}] + } + }, + "merged_text": "合并的字幕文本..." + } +} +``` + +## Installation + +```bash +cp -r kb-ocr-processor/ ~/skills/ +pip install -r requirements.txt +``` diff --git a/skills/kb-ocr-processor/ocr_processor.py b/skills/kb-ocr-processor/ocr_processor.py new file mode 100644 index 0000000..b793ae6 --- /dev/null +++ b/skills/kb-ocr-processor/ocr_processor.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +""" +OCR Processor - 视频帧文字识别工具 +支持 EasyOCR 和 Tesseract 引擎 +""" + +import argparse +import json +import os +import glob +from pathlib import Path +from typing import List, Dict, Optional +import easyocr +import pytesseract +from PIL import Image + + +class OCRProcessor: + """OCR 文字识别处理器""" + + def __init__(self, engine: str = "easyocr", languages: List[str] = ["ch_sim", "en"]): + """ + 初始化 OCR 处理器 + + Args: + engine: OCR 引擎 ("easyocr" 或 "tesseract") + languages: 语言列表 + """ + self.engine = engine + self.languages = languages + self.reader = None + + if engine == "easyocr": + self.reader = easyocr.Reader(languages, gpu=False) + + def process_image(self, image_path: str) -> Dict: + """ + 处理单张图片进行 OCR 识别 + + Args: + image_path: 图片路径 + + Returns: + 包含 text 和 confidence 的字典 + """ + if not os.path.exists(image_path): + return {"text": "", "confidence": 0.0, "error": "File not found"} + + try: + if self.engine == "easyocr": + return self._process_with_easyocr(image_path) + elif self.engine == "tesseract": + return self._process_with_tesseract(image_path) + else: + return {"text": "", "confidence": 0.0, "error": f"Unknown engine: {self.engine}"} + except Exception as e: + return {"text": "", "confidence": 0.0, "error": str(e)} + + def _process_with_easyocr(self, image_path: str) -> Dict: + """使用 EasyOCR 处理图片""" + results = self.reader.readtext(image_path) + + if not results: + return {"text": "", "confidence": 0.0} + + texts = [] + confidences = [] + + for bbox, text, conf in results: + texts.append(text) + confidences.append(conf) + + merged_text = "\n".join(texts) + avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0 + + return { + "text": merged_text, + "confidence": round(avg_confidence, 4), + "details": [ + {"text": text, "confidence": round(conf, 4)} + for text, conf in zip(texts, confidences) + ] + } + + def _process_with_tesseract(self, image_path: str) -> Dict: + """使用 Tesseract 处理图片""" + image = Image.open(image_path) + text = pytesseract.image_to_string(image, lang="+".join(self.languages)) + + # Tesseract 不直接返回置信度，返回 0.0 + return { + "text": text.strip(), + "confidence": 0.0 + } + + def process_frames(self, input_dir: str, fps: float = 0.1) -> List[Dict]: + """ + 处理目录下所有帧图片 + + Args: + input_dir: 帧图片目录 + fps: 采样帧率 + + Returns: + OCR 结果列表 + """ + if not os.path.isdir(input_dir): + print(f"Error: Directory not found: {input_dir}") + return [] + + # 支持的图片格式 + image_extensions = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.webp"] + image_files = [] + + for ext in image_extensions: + image_files.extend(glob.glob(os.path.join(input_dir, ext))) + image_files.extend(glob.glob(os.path.join(input_dir, "**", ext), recursive=True)) + + # 按文件名排序 + image_files = sorted(image_files) + + if not image_files: + print(f"No images found in {input_dir}") + return [] + + print(f"Found {len(image_files)} images in {input_dir}") + print(f"Processing with {self.engine} engine...") + + results = [] + for i, image_path in enumerate(image_files): + print(f"Processing [{i+1}/{len(image_files)}]: {image_path}") + ocr_result = self.process_image(image_path) + ocr_result["source"] = os.path.basename(image_path) + results.append(ocr_result) + + return results + + def merge_texts(self, results: List[Dict]) -> str: + """ + 合并多个 OCR 结果的文本 + + Args: + results: OCR 结果列表 + + Returns: + 合并后的文本 + """ + texts = [] + for result in results: + if result.get("text"): + texts.append(result["text"]) + return "\n\n---\n\n".join(texts) + + def save_results(self, results: List[Dict], output_path: str, merge: bool = False): + """ + 保存 OCR 结果到 JSON 文件 + + Args: + results: OCR 结果列表 + output_path: 输出文件路径 + merge: 是否同时保存合并后的文本文件 + """ + output_data = { + "engine": self.engine, + "total_frames": len(results), + "results": results + } + + # 保存 JSON + with open(output_path, "w", encoding="utf-8") as f: + json.dump(output_data, f, ensure_ascii=False, indent=2) + + print(f"Results saved to {output_path}") + + # 如果需要，同时保存合并的文本 + if merge: + text_path = output_path.replace(".json", ".txt") + merged_text = self.merge_texts(results) + with open(text_path, "w", encoding="utf-8") as f: + f.write(merged_text) + print(f"Merged text saved to {text_path}") + + +def main(): + """命令行入口""" + parser = argparse.ArgumentParser( + description="OCR 视频帧文字识别工具", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + # 使用 EasyOCR 处理帧目录 + python ocr_processor.py --input ./frames/ --output ./ocr.json + + # 使用 Tesseract 处理 + python ocr_processor.py --input ./frames/ --output ./ocr.json --engine tesseract + + # 指定采样帧率 + python ocr_processor.py --input ./frames/ --output ./ocr.json --fps 0.5 + + # 同时生成合并的文本文件 + python ocr_processor.py --input ./frames/ --output ./ocr.json --merge + """ + ) + + parser.add_argument("--input", "-i", required=True, help="输入帧图片目录") + parser.add_argument("--output", "-o", required=True, help="输出 JSON 文件路径") + parser.add_argument("--fps", "-f", type=float, default=0.1, help="采样帧率 (默认: 0.1)") + parser.add_argument("--engine", "-e", choices=["easyocr", "tesseract"], + default="easyocr", help="OCR 引擎 (默认: easyocr)") + parser.add_argument("--merge", action="store_true", help="同时保存合并的文本文件") + parser.add_argument("--languages", "-l", nargs="+", + default=["ch_sim", "en"], help="语言列表") + + args = parser.parse_args() + + # 初始化处理器 + processor = OCRProcessor(engine=args.engine, languages=args.languages) + + # 处理帧 + results = processor.process_frames(args.input, fps=args.fps) + + if results: + # 保存结果 + processor.save_results(results, args.output, merge=args.merge) + + # 打印摘要 + total_texts = sum(1 for r in results if r.get("text")) + avg_conf = sum(r.get("confidence", 0) for r in results) / len(results) if results else 0 + + print(f"\n=== 摘要 ===") + print(f"处理帧数: {len(results)}") + print(f"识别到文字的帧: {total_texts}") + print(f"平均置信度: {avg_conf:.2%}") + else: + print("未处理到任何结果") + return 1 + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/skills/kb-ocr-processor/requirements.txt b/skills/kb-ocr-processor/requirements.txt new file mode 100644 index 0000000..f3f72c2 --- /dev/null +++ b/skills/kb-ocr-processor/requirements.txt @@ -0,0 +1,3 @@ +easyocr>=1.7.0 +pytesseract>=0.3.10 +Pillow>=9.0.0 diff --git a/skills/kb-workflow/SKILL.md b/skills/kb-workflow/SKILL.md new file mode 100644 index 0000000..4ac3932 --- /dev/null +++ b/skills/kb-workflow/SKILL.md @@ -0,0 +1,285 @@ +--- +name: kb-workflow +description: 知识库工作流编排技能，提供收件箱扫描、内容处理、归档、订阅检测和汇报生成功能。 +metadata: + short-description: 知识库工作流编排 +--- + +# KB Workflow + +## Description + +KB Workflow 是一个知识库工作流编排技能，负责协调和管理知识库内容的完整生命周期： + +**核心功能**： +- 📥 **收件箱扫描** - 扫描四个子目录 (pending, metadata, content, analyzed) +- 🔄 **内容处理** - 调用技能2-4处理内容 +- 📦 **智能归档** - 将处理完成的内容归档到 archive/ +- 🔔 **订阅检测** - 检测订阅源的更新 +- 📊 **汇报生成** - 生成处理统计和推荐 + +## Workflow Stages + +``` +┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌──────────┐ +│ pending │ → │ metadata│ → │ content │ → │ analyzed│ → │ archive/ │ +└─────────┘ └─────────┘ └─────────┘ └─────────┘ └──────────┘ + ↓ ↓ ↓ ↓ + Skill 2 Skill 3 Skill 4 ↓ + (Link解析) (元数据提取) (内容分析) 最终归档 +``` + +## Installation + +KB Workflow 作为 OpenClaw 技能套件的一部分，无需额外安装。 + +确保以下依赖已配置： +- Python 3.8+ +- 知识库目录结构 (knowledge-base/) + +## Directory Structure + +``` +knowledge-base/ +├── pending/ # 待处理的新内容 +├── metadata/ # 已提取元数据 +├── content/ # 已提取内容 +├── analyzed/ # 已完成分析 +├── archive/ # 归档内容 +│ └── YYYY-MM-DD/ # 按日期归档 +└── reports/ # 生成的汇报 +``` + +## Usage + +### 命令行用法 + +```bash +# 查看工作流状态 +python kb_workflow.py --trigger status + +# 手动处理待处理内容 +python kb_workflow.py --trigger manual --scope pending + +# 处理所有新内容（限制5个） +python kb_workflow.py --trigger manual --limit 5 + +# 生成日报 +python kb_workflow.py --trigger scheduled --report + +# 检查订阅更新 +python kb_workflow.py --trigger subscriptions +``` + +### 参数说明 + +| 参数 | 说明 | 默认值 | +|------|------|--------| +| `--trigger` | 触发类型 | `manual` | +| `--scope` | 处理范围 | `all` | +| `--limit` | 处理数量限制 | 无限制 | +| `--report` | 生成汇报 | False | +| `--output` | 输出格式 | `json` | + +### 触发类型 + +- `manual` - 手动触发，立即处理 +- `scheduled` - 定时任务触发 +- `webhook` - Webhook 触发 +- `status` - 仅查看状态 +- `report` - 生成汇报 +- `subscriptions` - 检查订阅 + +### 作用域 + +- `all` - 所有目录 +- `pending` - 仅待处理 +- `metadata` - 仅元数据阶段 +- `content` - 仅内容阶段 +- `analyzed` - 仅已完成分析 +- `archive` - 归档目录 + +## Integration + +### 技能2-4 集成 + +KB Workflow 协调以下子技能： + +1. **Skill 2: Link Parser** - 解析 URL/链接 +2. **Skill 3: Zhihu Metadata** - 提取元数据 +3. **Skill 4: Bilibili Analyzer** - 内容分析 + +### 调用示例 + +```python +from kb_workflow import KBWorkflow + +# 初始化 +workflow = KBWorkflow(base_path="/path/to/knowledge-base") + +# 扫描收件箱 +inbox = workflow.scan_inbox(scope="pending") +print(f"待处理: {len(inbox['pending'])} 项") + +# 处理项目 +for item in inbox['pending'][:5]: + result = workflow.process_item(item) + print(f"处理结果: {result['success']}") + +# 查看状态 +status = workflow.get_status() +print(json.dumps(status, indent=2)) + +# 生成汇报 +report = workflow.generate_report(period="daily") +``` + +## Input/Output Formats + +### Input + +#### 扫描收件箱输入 +```json +{ + "scope": "all", // 或 "pending", "metadata", "content", "analyzed" + "limit": 10 // 可选，限制返回数量 +} +``` + +#### 处理项目输入 +```json +{ + "name": "example.md", + "path": "/path/to/knowledge-base/pending/example.md", + "extension": ".md", + "size": 1024, + "modified": "2026-02-10T12:00:00" +} +``` + +### Output + +#### 状态输出 +```json +{ + "status": "ready", + "timestamp": "2026-02-10T12:00:00", + "inbox": { + "pending": {"count": 5, "path": "/path/pending"}, + "metadata": {"count": 3, "path": "/path/metadata"}, + "content": {"count": 2, "path": "/path/content"}, + "analyzed": {"count": 8, "path": "/path/analyzed"} + }, + "archive": { + "count": 120, + "path": "/path/archive" + }, + "total_items": 18 +} +``` + +#### 处理结果输出 +```json +{ + "item": {"name": "example.md", ...}, + "success": true, + "stages_completed": ["link_parsing", "metadata_extraction", "content_analysis"], + "output_path": "/path/analyzed/example.md", + "error": null +} +``` + +#### 汇报输出 +```json +{ + "generated_at": "2026-02-10T12:00:00", + "period": "daily", + "summary": { + "total_items_inbox": 18, + "total_archived": 120, + "items_by_stage": {...} + }, + "processing_stats": {...}, + "recommendations": [ + "⚠️ 待处理内容较多 (15 项)，建议优先处理", + "📦 已分析内容较多 (25 项)，建议归档" + ] +} +``` + +## Configuration + +KB Workflow 无需额外配置，自动使用以下路径： +- 知识库根目录: `skills/kb-workflow/knowledge-base/` +- 归档目录: `skills/kb-workflow/knowledge-base/archive/` +- 汇报目录: `skills/kb-workflow/knowledge-base/reports/` + +### 自定义路径 + +```python +from kb_workflow import KBWorkflow + +# 使用自定义路径 +workflow = KBWorkflow( + base_path="/custom/knowledge-base", + archive_path="/custom/archive", + output_format="json" +) +``` + +## Workflow Steps + +### 完整处理流程 + +1. **扫描收件箱** + - 扫描 `pending/` 目录获取新内容 + - 扫描其他阶段目录获取待继续处理的内容 + +2. **分类处理** + - 新内容 → 提取链接/URL (Skill 2) + - 待提取元数据 → 提取元数据 (Skill 3) + - 待分析内容 → 深度分析 (Skill 4) + +3. **自动流转** + - 处理完成后自动移动到下一阶段目录 + +4. **归档** + - 分析完成后可归档到 `archive/YYYY-MM-DD/` + +5. **汇报** + - 生成处理统计 + - 提供处理建议 + +### 定时任务示例 + +```bash +# 每天早上9点检查并处理新内容 +0 9 * * * cd /path/to/skills/kb-workflow && python kb_workflow.py --trigger scheduled --scope pending --report + +# 每小时检查订阅更新 +0 * * * * cd /path/to/skills/kb-workflow && python kb_workflow.py --trigger subscriptions +``` + +## Troubleshooting + +### 常见问题 + +**Q: 扫描不到内容？** +A: 确保知识库目录存在且包含正确的子目录 (pending, metadata, content, analyzed) + +**Q: 处理失败？** +A: 检查日志输出，使用 `--verbose` 参数获取详细调试信息 + +**Q: 如何查看处理进度？** +A: 使用 `--trigger status` 查看各阶段的项目数量 + +## Related Skills + +- **obsidian-kb** - Obsidian 知识库管理 +- **link-parser** - 链接解析 +- **zhihu-metadata** - 知乎元数据提取 +- **bilibili-analyzer** - B站视频分析 + +--- + +*KB Workflow v1.0 - Knowledge Base Workflow Orchestration* diff --git a/skills/kb-workflow/kb_handler.py b/skills/kb-workflow/kb_handler.py new file mode 100755 index 0000000..efc3dda --- /dev/null +++ b/skills/kb-workflow/kb_handler.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python3 +""" +Auto-trigger handler for obsidian-kb skill. + +Detects video/article links in user messages and automatically processes them. +Supports Bilibili, YouTube, and Zhihu. +""" + +import re +import sys +import os +from typing import List, Dict, Optional, Tuple +from urllib.parse import urlparse + +# Add path for imports +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'src')) + +# Import short link resolver +from shortlink_resolver import ( + ShortLinkResolver, + detect_and_format, + PlaywrightResolver +) + +# 导入主流程 +try: + from main_pipeline import MainPipeline + MAIN_PIPELINE_AVAILABLE = True +except ImportError: + MAIN_PIPELINE_AVAILABLE = False + +# Pattern definitions for auto-detection +PATTERNS = { + 'bilibili_full': re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV\w+)'), + 'bilibili_short': re.compile(r'https?://b23\.tv/(\w+)'), + 'bilibili_bv': re.compile(r'\b(BV[a-zA-Z0-9]{10})\b'), + 'youtube_full': re.compile(r'https?://(?:www\.)?youtube\.com/watch\?v=([a-zA-Z0-9_-]{11})'), + 'youtube_short': re.compile(r'https?://youtu\.be/([a-zA-Z0-9_-]{11})'), + 'zhihu_article': re.compile(r'https?://zhuanlan\.zhihu\.com/p/(\d+)'), + 'zhihu_answer': re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)'), +} + + +def detect_links(text: str) -> List[Dict[str, str]]: + """ + Detect video/article links in text. + + Args: + text: Input text to search + + Returns: + List of detected links with metadata: + [ + { + 'url': 'https://...', + 'platform': 'bilibili', + 'type': 'video', + 'id': 'BVxxx', + 'is_short': False + }, + ... + ] + """ + detected = [] + seen_ids = set() # Track IDs to avoid duplicates + + for pattern_name, pattern in PATTERNS.items(): + matches = pattern.finditer(text) + + for match in matches: + # Determine platform and type + if 'bilibili' in pattern_name: + platform = 'bilibili' + content_type = 'video' + if 'short' in pattern_name: + is_short = True + url = match.group(0) + content_id = None # Need to resolve short URL + else: + is_short = False + if pattern_name == 'bilibili_bv': + url = f"https://www.bilibili.com/video/{match.group(1)}" + else: + url = match.group(0) + content_id = match.group(1) + + elif 'youtube' in pattern_name: + platform = 'youtube' + content_type = 'video' + is_short = 'short' in pattern_name + url = match.group(0) + content_id = match.group(1) + + elif 'zhihu' in pattern_name: + platform = 'zhihu' + content_type = 'article' + is_short = False + url = match.group(0) + content_id = match.group(1) + + else: + continue + + # Skip duplicates by URL or ID + unique_key = url if (not content_id or is_short) else content_id + if unique_key in seen_ids: + continue + seen_ids.add(unique_key) + + detected.append({ + 'url': url, + 'platform': platform, + 'type': content_type, + 'id': content_id, + 'is_short': is_short, + 'pattern': pattern_name + }) + + return detected + + +def format_preview(detected: List[Dict[str, str]]) -> str: + """ + Format a preview message for detected links. + + Args: + detected: List of detected links + + Returns: + Formatted preview message + """ + if not detected: + return "" + + lines = ["🔍 检测到内容链接:\n"] + + for i, item in enumerate(detected, 1): + platform_emoji = { + 'bilibili': '📺', + 'youtube': '▶️', + 'zhihu': '📝' + }.get(item['platform'], '🔗') + + platform_name = { + 'bilibili': 'B站视频', + 'youtube': 'YouTube视频', + 'zhihu': '知乎文章' + }.get(item['platform'], item['platform']) + + lines.append(f"{platform_emoji} [{i}] {platform_name}") + lines.append(f" URL: {item['url']}") + + if item['is_short']: + lines.append(f" ⚠️ 短链接，需要用户提供完整URL或ID") + elif item['id']: + lines.append(f" ID: {item['id']}") + + lines.append("") + + return "\n".join(lines) + + +def format_short_link_help(short_link: Optional[str] = None) -> str: + """ + Format help message for short links. + + Args: + short_link: Optional short link URL to mention + + Returns: + Help message + """ + return ShortLinkResolver.format_short_link_help(short_link) + + +def get_action_prompt(detected: List[Dict[str, str]]) -> str: + """ + Generate prompt asking user what to do. + + Args: + detected: List of detected links + + Returns: + Action prompt + """ + has_short = any(item['is_short'] for item in detected) + + if has_short: + return f"{format_short_link_help()}\n\n其他链接是否立即处理？(y/n)" + + count = len(detected) + if count == 1: + return "是否添加到收件箱并处理？(y/n)" + else: + return f"是否将上述 {count} 个内容添加到收件箱？(y/n)" + + +def should_auto_process(text: str) -> bool: + """ + Check if user wants to auto-process based on their message. + + Args: + text: User's message + + Returns: + True if user indicates they want immediate processing + """ + auto_keywords = ['立即处理', '处理', 'process', '--process', '马上'] + return any(keyword in text.lower() for keyword in auto_keywords) + + +def process_link(url: str, platform: str, auto_process: bool = False) -> Tuple[bool, str]: + """ + Process a single link (extract metadata, add to inbox). + + Args: + url: Content URL + platform: Platform name (bilibili/youtube/zhihu) + auto_process: Whether to immediately process after adding + + Returns: + (success, message) tuple + """ + try: + # Import here to avoid issues if dependencies not available + from sources.bilibili import BilibiliAdapter + from models import Content, ContentStatus + from storage import Storage + + # Only support Bilibili for now + if platform != 'bilibili': + return False, f"暂不支持 {platform}，请手动添加" + + # Handle short links + if 'b23.tv' in url: + return False, "短链接无法自动处理，请提供完整URL或BV号" + + # Create adapter + adapter = BilibiliAdapter() + + # Extract metadata + content = adapter.extract_content(url) + if not content: + return False, "无法提取视频信息" + + # Get transcript + transcript = adapter.get_transcript(content) + if transcript: + content.transcript = transcript + + # Save to inbox + content.status = ContentStatus.CONFIRMED + storage = Storage() + storage.save_content(content) + + # Build success message + msg = f"✅ 已添加到收件箱\n" + msg += f" 标题: {content.title}\n" + msg += f" UP主: {content.author}\n" + msg += f" 时长: {content.duration}秒\n" + + if auto_process: + # Process immediately + try: + from src.pipeline import ProcessingPipeline + pipeline = ProcessingPipeline() + content_text = content.transcript or content.summary or "" + result = pipeline.process_full(content, content_text) + storage.save_content(result) + + msg += f"\n🔄 处理完成\n" + msg += f" 📄 笔记: {result.output_path or 'N/A'}\n" + msg += f" 🏷️ 标签: {', '.join(result.tags)}" + except Exception as e: + msg += f"\n⚠️ 处理失败: {e}" + + return True, msg + + except ImportError as e: + return False, f"依赖模块未安装: {e}" + except Exception as e: + return False, f"处理失败: {e}" + + +def process_video_full(source_id: str, enable_git_sync: bool = False) -> Tuple[bool, str]: + """ + 使用主流程完整处理视频（新接口） + + 完整流程: 提取 → 处理 → 输出 → 同步 + + Args: + source_id: 视频ID（支持BV号、URL） + enable_git_sync: 是否启用Git同步 + + Returns: + (success, message) tuple + """ + if not MAIN_PIPELINE_AVAILABLE: + return False, "主流程模块未可用，请检查依赖" + + try: + # 导入配置 + from src.config import get_config + + # 获取配置 + config = get_config() + + # 创建主流程 + pipeline = MainPipeline(config=config, enable_git_sync=enable_git_sync) + + # 处理视频 + result = pipeline.process_video(source_id) + + # 返回结果 + if result['success']: + return True, result['message'] + else: + return False, result.get('error', 'Unknown error') + + except Exception as e: + return False, f"处理失败: {e}" + + +def handle_message(text: str) -> Optional[str]: + """ + Main entry point: handle incoming message and return response. + + Args: + text: User's message text + + Returns: + Response message, or None if no action needed + """ + # First, check for short links and provide help + has_short, short_msg = detect_and_format(text) + + # Detect all links (including short ones) + detected = detect_links(text) + + if not detected: + return None + + # If we have short links, show help message first + if has_short: + return short_msg + + # Check if this is a confirmation (y/yes) + text_lower = text.lower().strip() + if text_lower in ('y', 'yes', '是', '好的', 'ok'): + # This would need state management - for now return a message + # about using explicit command + return "请使用 /kb-add 命令添加内容，或重新发送链接" + + # Show preview + preview = format_preview(detected) + prompt = get_action_prompt(detected) + + return f"{preview}\n{prompt}" + + +# CLI interface for testing +def main(): + """CLI for testing the handler.""" + if len(sys.argv) < 2: + print("Usage: python3 handler.py [options]") + print("\nCommands:") + print(" detect 'text' - 检测文本中的内容链接") + print(" process - 使用主流程处理视频") + print(" process --git - 处理视频并同步到Git") + print("\nExamples:") + print(' python3 handler.py detect "https://www.bilibili.com/video/BV1xx411c7mD"') + print(' python3 handler.py process BV1xx411c7mD') + print(' python3 handler.py process BV1xx411c7mD --git') + sys.exit(1) + + command = sys.argv[1].lower() + + if command == 'detect': + # 检测链接 + if len(sys.argv) < 3: + print("Error: Please provide text to detect") + sys.exit(1) + text = ' '.join(sys.argv[2:]) + response = handle_message(text) + if response: + print(response) + else: + print("No content links detected") + + elif command == 'process': + # 使用主流程处理视频 + if len(sys.argv) < 3: + print("Error: Please provide BV number or URL") + sys.exit(1) + + source_id = sys.argv[2] + enable_git = '--git' in sys.argv + + print(f"Processing video: {source_id}") + if enable_git: + print("Git sync: enabled") + else: + print("Git sync: disabled") + print() + + success, message = process_video_full(source_id, enable_git_sync=enable_git) + + if success: + print(message) + sys.exit(0) + else: + print(f"\n❌ Failed: {message}") + sys.exit(1) + + else: + # 兼容旧格式：直接传入文本进行检测 + text = sys.argv[1] + response = handle_message(text) + if response: + print(response) + else: + print("No content links detected") + + +if __name__ == '__main__': + main() diff --git a/skills/kb-workflow/kb_pipeline.py b/skills/kb-workflow/kb_pipeline.py new file mode 100755 index 0000000..c9d9e5a --- /dev/null +++ b/skills/kb-workflow/kb_pipeline.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Obsidian KB - 主流程集成 + +整合提取层、处理层、输出层，实现完整的数据流： +提取(BilibiliExtractor) → 处理(DeepAnalyzer/ContentChunker) → 输出(ObsidianWriter) → 同步(GitSync) +""" + +import os +import sys +import logging +from typing import Dict, Any, Optional +from datetime import datetime +from pathlib import Path + +# 添加项目路径 +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from src.extractors.bilibili_extractor import BilibiliExtractor, ExtractionError +from src.deep_analyzer import DeepAnalyzer +from src.content_chunker import ContentChunker, Chunk +from src.outputs.obsidian_writer import ObsidianWriter +from src.outputs.git_sync import GitSync +from src.models import Content, ContentType, ContentStatus, TimelineEntry +from src.config import get_config + + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler('obsidian_kb.log', encoding='utf-8') + ] +) +logger = logging.getLogger('ObsidianKB') + + +class MainPipeline: + """主流程集成器""" + + def __init__(self, config=None, enable_git_sync: bool = False): + """ + 初始化主流程 + + Args: + config: 配置对象 + enable_git_sync: 是否启用Git同步 + """ + self.config = config or get_config() + self.enable_git_sync = enable_git_sync + + logger.info("初始化主流程...") + logger.info(f"Obsidian vault 路径: {self.config.vault_path}") + logger.info(f"Git 同步: {'启用' if enable_git_sync else '禁用'}") + + # 初始化各层组件 + self.extractor = BilibiliExtractor(config=self.config) + self.analyzer = DeepAnalyzer(ai_client=None) # 暂时不传入ai_client + self.chunker = ContentChunker() + self.writer = ObsidianWriter(config=self.config) + + # 初始化Git同步器（仅在需要时） + self.git_sync = None + if enable_git_sync and self._is_git_enabled(): + self.git_sync = GitSync(config=self.config) + logger.info("Git 同步器已初始化") + + def _is_git_enabled(self) -> bool: + """检查Git同步是否在配置中启用""" + return self.config.get('git.enabled', False) + + def process_video(self, source_id: str) -> Dict[str, Any]: + """ + 处理单个视频的完整流程 + + Args: + source_id: 视频ID（支持BV号、URL） + + Returns: + 处理结果字典，包含： + - success: bool - 是否成功 + - content: Content对象 - 处理的内容 + - message: str - 处理消息 + - error: str - 错误信息（如果失败） + """ + result = { + 'success': False, + 'content': None, + 'message': '', + 'error': '' + } + + try: + logger.info("=" * 80) + logger.info(f"开始处理视频: {source_id}") + logger.info("=" * 80) + + # Step 1: 提取层 - 提取视频信息 + logger.info("\n[Step 1/5] 提取视频信息...") + extractor_data = self._extract_video(source_id) + + # Step 2: 构建Content对象 + logger.info("\n[Step 2/5] 构建内容对象...") + content = self._build_content(extractor_data) + + # Step 3: 处理层 - 深度分析 + logger.info("\n[Step 3/5] 深度分析...") + analysis_result = self._analyze_content(extractor_data) + + # Step 4: 处理层 - 内容分片 + logger.info("\n[Step 4/5] 内容分片...") + chunks = self._chunk_content(extractor_data) + + # 合并分析结果和分片信息到content + self._merge_analysis_to_content(content, analysis_result, chunks) + + # Step 5: 输出层 - 写入Obsidian + logger.info("\n[Step 5/5] 写入Obsidian...") + output_path = self._write_obsidian_note(content) + + # Step 6: 同步层 - Git同步（可选） + if self.enable_git_sync and self.git_sync: + logger.info("\n[Step 6/6] Git同步...") + self._sync_to_git(content) + + # 更新状态 + content.update_status(ContentStatus.GENERATED) + content.output_path = output_path + + # 构建成功结果 + result['success'] = True + result['content'] = content + result['message'] = self._format_success_message(content, chunks) + + logger.info("\n" + "=" * 80) + logger.info(f"✅ 处理完成: {content.title}") + logger.info(f" 📄 笔记路径: {output_path}") + logger.info(f" 📊 字幕条数: {len(chunks) if chunks else 0}") + logger.info("=" * 80 + "\n") + + except ExtractionError as e: + logger.error(f"❌ 提取失败: {e}") + result['error'] = f"提取失败: {e}" + result['message'] = f"提取视频信息失败: {e}" + + except Exception as e: + logger.error(f"❌ 处理失败: {e}", exc_info=True) + result['error'] = str(e) + result['message'] = f"处理过程中发生错误: {e}" + + return result + + def _extract_video(self, source_id: str) -> Dict[str, Any]: + """ + 提取视频信息 + + Args: + source_id: 视频ID或URL + + Returns: + 提取器返回的数据结构 + + Raises: + ExtractionError: 提取失败 + """ + logger.info(f" 正在提取: {source_id}") + + try: + extractor_data = self.extractor.extract(source_id) + logger.info(f" ✅ 提取成功: {extractor_data.get('metadata', {}).get('title', '未知标题')}") + logger.info(f" 作者: {extractor_data.get('metadata', {}).get('author', '未知')}") + logger.info(f" 时长: {extractor_data.get('metadata', {}).get('duration', '0')}") + + # 检查字幕 + subtitle = extractor_data.get('subtitle', '') + if subtitle: + subtitle_lines = subtitle.split('\n') + logger.info(f" 字幕: {len(subtitle_lines)} 条") + else: + logger.warning(f" ⚠️ 无字幕") + + return extractor_data + + except ExtractionError as e: + logger.error(f" ❌ 提取失败: {e}") + raise + except Exception as e: + logger.error(f" ❌ 提取异常: {e}", exc_info=True) + raise ExtractionError(f"提取过程异常: {e}", source_id=source_id) + + def _build_content(self, extractor_data: Dict[str, Any]) -> Content: + """ + 从提取器数据构建Content对象 + + Args: + extractor_data: 提取器返回的数据 + + Returns: + Content对象 + """ + metadata = extractor_data.get('metadata', {}) + + content = Content( + source='bilibili', + source_id=metadata.get('bvid', ''), + url=metadata.get('url', ''), + title=metadata.get('title', ''), + author=metadata.get('author', ''), + description=extractor_data.get('description', ''), + duration=self._parse_duration(metadata.get('duration', '0')), + content_type=ContentType.VIDEO, + transcript=extractor_data.get('subtitle', ''), + status=ContentStatus.SCANNED + ) + + logger.info(f" ✅ Content对象已创建: {content.id}") + return content + + def _parse_duration(self, duration_str: str) -> Optional[int]: + """ + 解析时长字符串为秒数 + + Args: + duration_str: 时长字符串（如 "10:30"） + + Returns: + 秒数 + """ + try: + if ':' in duration_str: + parts = duration_str.split(':') + if len(parts) == 2: + # MM:SS + return int(parts[0]) * 60 + int(parts[1]) + elif len(parts) == 3: + # HH:MM:SS + return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2]) + except: + pass + return None + + def _analyze_content(self, extractor_data: Dict[str, Any]) -> Dict[str, Any]: + """ + 深度分析内容 + + Args: + extractor_data: 提取器返回的数据 + + Returns: + 分析结果字典 + """ + try: + logger.info(" 正在进行深度分析...") + + # 使用DeepAnalyzer进行分析 + # DeepAnalyzer接收extractor_data作为输入 + analysis_result = self.analyzer.analyze(extractor_data) + + # 将AnalysisResult转换为字典以便合并 + result_dict = { + 'main_topics': analysis_result.main_topics, + 'key_insights': analysis_result.key_insights, + 'action_items': analysis_result.action_items, + 'summary': analysis_result.summary, + 'timeline': analysis_result.timeline, + 'recommended_tags': analysis_result.recommended_tags, + 'full_analysis': analysis_result.full_analysis + } + + logger.info(f" ✅ 分析完成: {len(result_dict['main_topics'])} 个主题") + + return result_dict + + except Exception as e: + logger.error(f" ❌ 分析失败: {e}", exc_info=True) + # 返回默认结果，不中断流程 + return { + 'main_topics': ['未分类'], + 'key_insights': [], + 'action_items': [], + 'summary': '分析失败，无法生成摘要', + 'timeline': [], + 'recommended_tags': ['未分类'], + 'full_analysis': '' + } + + def _chunk_content(self, extractor_data: Dict[str, Any]) -> list: + """ + 内容分片 + + Args: + extractor_data: 提取器返回的数据 + + Returns: + Chunk对象列表 + """ + try: + logger.info(" 正在进行内容分片...") + + # 使用ContentChunker进行分片 + chunks = self.chunker.chunk_from_extractor(extractor_data) + + logger.info(f" ✅ 分片完成: {len(chunks)} 段") + + return chunks + + except Exception as e: + logger.error(f" ❌ 分片失败: {e}", exc_info=True) + # 返回空列表，不中断流程 + return [] + + def _merge_analysis_to_content(self, content: Content, analysis_result: Dict[str, Any], chunks: list): + """ + 将分析结果和分片信息合并到Content对象 + + Args: + content: Content对象 + analysis_result: 分析结果字典 + chunks: Chunk对象列表 + """ + # 合并基本信息 + content.summary = analysis_result.get('summary', '') + content.key_points = analysis_result.get('main_topics', []) + content.tags = analysis_result.get('recommended_tags', []) + + # 合并时间轴 + timeline_data = analysis_result.get('timeline', []) + if timeline_data: + content.timeline = [ + TimelineEntry( + timestamp=item.get('start_time', ''), + title=item.get('title', ''), + description=item.get('summary', '') + ) + for item in timeline_data + ] + else: + # 从chunks生成时间轴 + if chunks: + content.timeline = [ + TimelineEntry( + timestamp=chunk.time_range, + title=f"片段 {chunk.index + 1}", + description=f"时长 {int(chunk.duration)}秒" + ) + for chunk in chunks + ] + + logger.info(f" ✅ 数据合并完成: 时间轴 {len(content.timeline)} 条") + + def _write_obsidian_note(self, content: Content) -> Optional[str]: + """ + 写入Obsidian笔记 + + Args: + content: Content对象 + + Returns: + 文件路径，失败返回None + """ + try: + logger.info(f" 正在写入笔记...") + + # 确保输出目录存在 + self.config.vault_path.mkdir(parents=True, exist_ok=True) + + # 调用ObsidianWriter写入 + output_path = self.writer.write(content, self.config) + + if output_path: + logger.info(f" ✅ 笔记已写入: {output_path}") + else: + logger.warning(f" ⚠️ 笔记写入失败") + + return output_path + + except Exception as e: + logger.error(f" ❌ 写入失败: {e}", exc_info=True) + return None + + def _sync_to_git(self, content: Content): + """ + 同步到Git仓库 + + Args: + content: Content对象 + """ + if not self.git_sync: + logger.warning(" ⚠️ Git同步器未初始化，跳过同步") + return + + try: + logger.info(" 正在同步到Git...") + + success = self.git_sync.sync(self.config) + + if success: + logger.info(" ✅ Git同步成功") + else: + logger.warning(" ⚠️ Git同步失败") + + except Exception as e: + logger.error(f" ❌ Git同步异常: {e}", exc_info=True) + # Git同步失败不视为整体失败 + + def _format_success_message(self, content: Content, chunks: list) -> str: + """ + 格式化成功消息 + + Args: + content: Content对象 + chunks: Chunk列表 + + Returns: + 格式化的消息字符串 + """ + lines = [ + "✅ 处理完成！", + "", + f"📺 标题: {content.title}", + f"👤 作者: {content.author}", + f"📄 笔记: {content.output_path or 'N/A'}", + ] + + if content.duration: + lines.append(f"⏱️ 时长: {self._format_duration(content.duration)}") + + if chunks: + lines.append(f"📊 分片: {len(chunks)} 段") + + if content.tags: + lines.append(f"🏷️ 标签: {', '.join(content.tags[:5])}") + + lines.append("") + + return "\n".join(lines) + + def _format_duration(self, seconds: int) -> str: + """格式化时长""" + if not seconds: + return "N/A" + + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + secs = seconds % 60 + + if hours > 0: + return f"{hours}小时{minutes}分钟" + elif minutes > 0: + return f"{minutes}分钟{secs}秒" + else: + return f"{secs}秒" + + +def main(): + """命令行入口""" + import argparse + + parser = argparse.ArgumentParser(description='Obsidian KB - 主流程集成') + parser.add_argument('source_id', help='视频ID（BV号或URL）') + parser.add_argument('--git', action='store_true', help='启用Git同步') + parser.add_argument('--config', help='配置文件路径') + + args = parser.parse_args() + + # 加载配置 + config = get_config(args.config) if args.config else get_config() + + # 创建主流程 + pipeline = MainPipeline(config=config, enable_git_sync=args.git) + + # 处理视频 + result = pipeline.process_video(args.source_id) + + # 输出结果 + print(result['message']) + + if not result['success']: + print(f"\n❌ 失败: {result['error']}") + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/skills/kb-workflow/kb_workflow.py b/skills/kb-workflow/kb_workflow.py new file mode 100644 index 0000000..b6e636d --- /dev/null +++ b/skills/kb-workflow/kb_workflow.py @@ -0,0 +1,674 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +KB Workflow - Knowledge Base Workflow Orchestration + +工作流编排技能，整合以下功能： +- 扫描收件箱目录 (pending, metadata, content, analyzed) +- 调用技能2-4处理内容 +- 归档处理完成的条目 +- 订阅更新检测 +- 生成汇报 + +Usage: + python kb_workflow.py --trigger manual + python kb_workflow.py --trigger scheduled + python kb_workflow.py --status + python kb_workflow.py --report + python kb_workflow.py --scope pending --limit 5 +""" + +import os +import sys +import json +import argparse +import logging +from datetime import datetime, timedelta +from pathlib import Path +from typing import Dict, List, Optional, Any +from enum import Enum + +# Setup logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger('KBWorkflow') + + +class TriggerType(Enum): + MANUAL = "manual" + SCHEDULED = "scheduled" + WEBHOOK = "webhook" + + +class Scope(Enum): + ALL = "all" + PENDING = "pending" + METADATA = "metadata" + CONTENT = "content" + ANALYZED = "analyzed" + ARCHIVE = "archive" + + +class KBWorkflow: + """Knowledge Base Workflow Orchestrator""" + + # Base directories for the inbox structure + INBOX_DIRS = ['pending', 'metadata', 'content', 'analyzed'] + + def __init__(self, + base_path: str = None, + archive_path: str = None, + output_format: str = "json"): + """ + Initialize KB Workflow + + Args: + base_path: Base path for knowledge base (default: workspace/knowledge-base) + archive_path: Path for archived items (default: archive/) + output_format: Output format (json, text) + """ + self.base_path = Path(base_path) if base_path else Path(__file__).parent / "knowledge-base" + self.archive_path = Path(archive_path) if archive_path else self.base_path / "archive" + self.output_format = output_format + + # Ensure directories exist + self._ensure_directories() + + # Initialize subsystems + self._init_subsystems() + + def _ensure_directories(self): + """Create necessary directories""" + # Create main inbox directories + for subdir in self.INBOX_DIRS: + dir_path = self.base_path / subdir + if not dir_path.exists(): + dir_path.mkdir(parents=True, exist_ok=True) + logger.info(f"Created directory: {dir_path}") + + # Create archive directory + if not self.archive_path.exists(): + self.archive_path.mkdir(parents=True, exist_ok=True) + logger.info(f"Created archive directory: {self.archive_path}") + + def _init_subsystems(self): + """Initialize subsystem references""" + # Placeholder for skill integrations + self.subscriptions_file = self.base_path / "subscriptions.json" + self.report_file = self.base_path / "reports" / f"report_{datetime.now().strftime('%Y%m%d')}.json" + + # ==================== Inbox Scanning ==================== + + def scan_inbox(self, scope: str = "all", limit: int = None) -> Dict[str, List[Dict]]: + """ + Scan inbox directories for items + + Args: + scope: Which directories to scan (all, pending, metadata, content, analyzed) + limit: Maximum number of items to return per directory + + Returns: + Dictionary with directory names as keys and item lists as values + """ + result = {} + + if scope == "all": + dirs_to_scan = self.INBOX_DIRS + elif scope in self.INBOX_DIRS: + dirs_to_scan = [scope] + else: + logger.warning(f"Unknown scope: {scope}, scanning all") + dirs_to_scan = self.INBOX_DIRS + + for dir_name in dirs_to_scan: + dir_path = self.base_path / dir_name + items = self._scan_directory(dir_path) + + if limit: + items = items[:limit] + + result[dir_name] = items + logger.info(f"Scanned {dir_name}: {len(items)} items") + + return result + + def _scan_directory(self, path: Path) -> List[Dict]: + """ + Scan a directory for items + + Args: + path: Directory path to scan + + Returns: + List of item dictionaries + """ + items = [] + + if not path.exists(): + return items + + for item_path in path.iterdir(): + if item_path.is_file(): + item_info = { + 'name': item_path.name, + 'path': str(item_path), + 'extension': item_path.suffix, + 'size': item_path.stat().st_size, + 'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat(), + 'type': self._classify_file_type(item_path) + } + items.append(item_info) + elif item_path.is_dir(): + # Treat subdirectories as items too + item_info = { + 'name': item_path.name, + 'path': str(item_path), + 'type': 'directory', + 'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat() + } + items.append(item_info) + + return items + + def _classify_file_type(self, path: Path) -> str: + """Classify file type by extension""" + suffix = path.suffix.lower() + + type_mapping = { + '.md': 'markdown', + '.txt': 'text', + '.json': 'json', + '.yaml': 'yaml', + '.yml': 'yaml', + '.pdf': 'pdf', + '.jpg': 'image', + '.jpeg': 'image', + '.png': 'image', + '.mp4': 'video', + '.mp3': 'audio', + '.zip': 'archive', + '.url': 'link' + } + + return type_mapping.get(suffix, 'unknown') + + def get_status(self) -> Dict[str, Any]: + """ + Get current workflow status + + Returns: + Status dictionary + """ + inbox_status = {} + total_items = 0 + + for dir_name in self.INBOX_DIRS: + dir_path = self.base_path / dir_name + items = self._scan_directory(dir_path) + inbox_status[dir_name] = { + 'count': len(items), + 'path': str(dir_path) + } + total_items += len(items) + + # Check archive + archive_items = self._scan_directory(self.archive_path) + + return { + 'status': 'ready', + 'timestamp': datetime.now().isoformat(), + 'inbox': inbox_status, + 'archive': { + 'count': len(archive_items), + 'path': str(self.archive_path) + }, + 'total_items': total_items + } + + # ==================== Item Processing ==================== + + def process_item(self, item: Dict, target_stage: str = None) -> Dict[str, Any]: + """ + Process a single item through the workflow + + This method calls skills 2-4 to process items: + - Skill 2: Link parsing / URL extraction + - Skill 3: Metadata extraction + - Skill 4: Content analysis + + Args: + item: Item dictionary with path and metadata + target_stage: Target stage to process to + + Returns: + Processing result dictionary + """ + result = { + 'item': item, + 'success': False, + 'stages_completed': [], + 'output_path': None, + 'error': None + } + + try: + item_path = Path(item['path']) + + # Determine processing stage based on current location + current_stage = self._get_stage_from_path(item_path) + + # Skill 2: Link/URL Parsing + if current_stage in ['pending', None] or target_stage == 'metadata': + logger.info(f"Stage 1 → 2: Processing link/URL for {item['name']}") + # TODO: Integrate with link-parser skill + result['stages_completed'].append('link_parsing') + + # Skill 3: Metadata Extraction + if current_stage in ['metadata', 'pending'] or target_stage == 'content': + logger.info(f"Stage 2 → 3: Extracting metadata for {item['name']}") + # TODO: Integrate with zhihu-metadata skill + result['stages_completed'].append('metadata_extraction') + + # Skill 4: Content Analysis + if current_stage in ['content', 'metadata'] or target_stage == 'analyzed': + logger.info(f"Stage 3 → 4: Analyzing content for {item['name']}") + # TODO: Integrate with bilibili-analyzer skill + result['stages_completed'].append('content_analysis') + + # Move to next stage if successful + if result['stages_completed']: + self._move_to_next_stage(item_path, result['stages_completed'][-1]) + result['success'] = True + + except Exception as e: + result['error'] = str(e) + logger.error(f"Processing failed: {e}") + + return result + + def _get_stage_from_path(self, path: Path) -> Optional[str]: + """Determine which stage a file is in based on its path""" + for stage in self.INBOX_DIRS: + if stage in str(path): + return stage + return None + + def _move_to_next_stage(self, item_path: Path, completed_stage: str) -> None: + """Move item to the next stage directory""" + stage_order = ['pending', 'metadata', 'content', 'analyzed'] + + try: + current_idx = stage_order.index(completed_stage) + if current_idx < len(stage_order) - 1: + next_stage = stage_order[current_idx + 1] + next_path = self.base_path / next_stage / item_path.name + + # Move file + if item_path.exists(): + item_path.rename(next_path) + logger.info(f"Moved {item_path.name} → {next_stage}/") + except ValueError: + logger.warning(f"Unknown stage: {completed_stage}") + + def process_all(self, scope: str = "pending", limit: int = None) -> Dict[str, Any]: + """ + Process all items in specified scope + + Args: + scope: Which directories to process + limit: Maximum items to process + + Returns: + Processing summary + """ + summary = { + 'scope': scope, + 'processed': 0, + 'failed': 0, + 'results': [] + } + + inbox = self.scan_inbox(scope=scope) + + for dir_name, items in inbox.items(): + if limit and summary['processed'] >= limit: + break + + for item in items: + if limit and summary['processed'] >= limit: + break + + result = self.process_item(item) + summary['results'].append(result) + + if result['success']: + summary['processed'] += 1 + else: + summary['failed'] += 1 + + return summary + + # ==================== Archiving ==================== + + def archive_item(self, item: Dict, analysis: Dict = None) -> Dict[str, Any]: + """ + Archive a processed item to the archive directory + + Args: + item: Item to archive + analysis: Optional analysis metadata + + Returns: + Archiving result + """ + result = { + 'success': False, + 'original_path': item['path'], + 'archive_path': None, + 'error': None + } + + try: + item_path = Path(item['path']) + + if not item_path.exists(): + result['error'] = f"Item not found: {item['path']}" + return result + + # Create archive subdirectory by date + date_dir = self.archive_path / datetime.now().strftime('%Y-%m-%d') + if not date_dir.exists(): + date_dir.mkdir(parents=True, exist_ok=True) + + # Move item to archive + archive_path = date_dir / item_path.name + item_path.rename(archive_path) + + # Save analysis metadata if provided + if analysis: + meta_path = date_dir / f"{item_path.stem}_meta.json" + with open(meta_path, 'w', encoding='utf-8') as f: + json.dump({ + 'archived_at': datetime.now().isoformat(), + 'analysis': analysis + }, f, ensure_ascii=False, indent=2) + + result['success'] = True + result['archive_path'] = str(archive_path) + logger.info(f"Archived: {item['name']} → {archive_path}") + + except Exception as e: + result['error'] = str(e) + logger.error(f"Archive failed: {e}") + + return result + + # ==================== Subscriptions ==================== + + def check_subscriptions(self) -> Dict[str, Any]: + """ + Check for updates from subscribed sources + + Returns: + Subscription check results + """ + result = { + 'checked_at': datetime.now().isoformat(), + 'sources': {}, + 'new_items': 0, + 'errors': [] + } + + # Load subscription list + subscriptions = self._load_subscriptions() + + for sub in subscriptions: + try: + source_name = sub.get('name', 'unknown') + source_type = sub.get('type', 'rss') + + # Placeholder: Check for new content + logger.info(f"Checking subscription: {source_name}") + + # TODO: Implement actual subscription checking + # - RSS feeds + # - YouTube channel updates + # - Bilibili user updates + # - Custom web hooks + + result['sources'][source_name] = { + 'type': source_type, + 'status': 'checked', + 'new_items': 0 + } + + except Exception as e: + result['errors'].append({ + 'source': sub.get('name', 'unknown'), + 'error': str(e) + }) + + return result + + def _load_subscriptions(self) -> List[Dict]: + """Load subscriptions from file""" + if self.subscriptions_file.exists(): + try: + with open(self.subscriptions_file, 'r', encoding='utf-8') as f: + return json.load(f) + except Exception as e: + logger.warning(f"Failed to load subscriptions: {e}") + + return [] + + # ==================== Reporting ==================== + + def generate_report(self, period: str = "daily") -> Dict[str, Any]: + """ + Generate workflow report + + Args: + period: Report period (daily, weekly, monthly) + + Returns: + Report dictionary + """ + # Calculate date range + now = datetime.now() + + if period == "daily": + start_date = now - timedelta(days=1) + elif period == "weekly": + start_date = now - timedelta(weeks=1) + elif period == "monthly": + start_date = now - timedelta(days=30) + else: + start_date = now - timedelta(days=1) + + # Gather statistics + status = self.get_status() + inbox = self.scan_inbox(scope="all") + + # Build report + report = { + 'generated_at': now.isoformat(), + 'period': period, + 'start_date': start_date.isoformat(), + 'summary': { + 'total_items_inbox': status['total_items'], + 'total_archived': status['archive']['count'], + 'items_by_stage': status['inbox'] + }, + 'processing_stats': { + 'pending': len(inbox.get('pending', [])), + 'metadata': len(inbox.get('metadata', [])), + 'content': len(inbox.get('content', [])), + 'analyzed': len(inbox.get('analyzed', [])) + }, + 'recommendations': self._generate_recommendations(status, inbox) + } + + # Save report + if self.report_file: + self.report_file.parent.mkdir(parents=True, exist_ok=True) + with open(self.report_file, 'w', encoding='utf-8') as f: + json.dump(report, f, ensure_ascii=False, indent=2) + logger.info(f"Report saved: {self.report_file}") + + return report + + def _generate_recommendations(self, status: Dict, inbox: Dict) -> List[str]: + """Generate processing recommendations""" + recommendations = [] + + pending_count = status['inbox'].get('pending', {}).get('count', 0) + analyzed_count = status['inbox'].get('analyzed', {}).get('count', 0) + + if pending_count > 10: + recommendations.append(f"⚠️ 待处理内容较多 ({pending_count} 项)，建议优先处理") + + if analyzed_count > 20: + recommendations.append(f"📦 已分析内容较多 ({analyzed_count} 项)，建议归档") + + if status['total_items'] == 0: + recommendations.append("📭 收件箱为空，暂无待处理内容") + + return recommendations + + # ==================== CLI Interface ==================== + + def run(self, trigger: str = "manual", scope: str = "all", + limit: int = None, report: bool = False): + """ + Main entry point for running the workflow + + Args: + trigger: Trigger type (manual, scheduled) + scope: Processing scope + limit: Maximum items to process + report: Generate report after processing + """ + logger.info(f"KB Workflow started: trigger={trigger}, scope={scope}") + + # Handle different triggers + if trigger == "status": + result = self.get_status() + elif trigger == "report": + result = self.generate_report() + elif trigger == "subscriptions": + result = self.check_subscriptions() + else: + # Process items + result = self.process_all(scope=scope, limit=limit) + + # Generate report if requested + if report: + report_result = self.generate_report() + result['report'] = report_result + + return result + + +def main(): + """CLI entry point""" + parser = argparse.ArgumentParser( + description='KB Workflow - Knowledge Base Workflow Orchestration', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Get workflow status + python kb_workflow.py --trigger status + + # Process pending items manually + python kb_workflow.py --trigger manual --scope pending + + # Process up to 5 items + python kb_workflow.py --trigger manual --limit 5 + + # Generate daily report + python kb_workflow.py --trigger scheduled --report + + # Check subscriptions + python kb_workflow.py --trigger scheduled --scope subscriptions + """ + ) + + parser.add_argument( + '--trigger', + type=str, + default='manual', + choices=['manual', 'scheduled', 'webhook', 'status', 'report', 'subscriptions'], + help='Workflow trigger type (default: manual)' + ) + + parser.add_argument( + '--scope', + type=str, + default='all', + choices=['all', 'pending', 'metadata', 'content', 'analyzed', 'archive'], + help='Processing scope (default: all)' + ) + + parser.add_argument( + '--limit', + type=int, + default=None, + help='Maximum items to process' + ) + + parser.add_argument( + '--report', + action='store_true', + help='Generate report after processing' + ) + + parser.add_argument( + '--output', + type=str, + choices=['json', 'text'], + default='json', + help='Output format (default: json)' + ) + + parser.add_argument( + '--base-path', + type=str, + default=None, + help='Base path for knowledge base' + ) + + parser.add_argument( + '--verbose', + action='store_true', + help='Enable verbose logging' + ) + + args = parser.parse_args() + + if args.verbose: + logging.getLogger().setLevel(logging.DEBUG) + + # Initialize workflow + workflow = KBWorkflow(base_path=args.base_path) + + # Run workflow + result = workflow.run( + trigger=args.trigger, + scope=args.scope, + limit=args.limit, + report=args.report + ) + + # Output result + if args.output == 'json': + print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) + else: + print(result) + + return 0 if result else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/skills/kb-workflow/knowledge-base/reports/report_20260210.json b/skills/kb-workflow/knowledge-base/reports/report_20260210.json new file mode 100644 index 0000000..7790cce --- /dev/null +++ b/skills/kb-workflow/knowledge-base/reports/report_20260210.json @@ -0,0 +1,36 @@ +{ + "generated_at": "2026-02-10T12:13:35.989689", + "period": "daily", + "start_date": "2026-02-09T12:13:35.989689", + "summary": { + "total_items_inbox": 0, + "total_archived": 0, + "items_by_stage": { + "pending": { + "count": 0, + "path": "/home/admin/.openclaw/workspace/skills/kb-workflow/knowledge-base/pending" + }, + "metadata": { + "count": 0, + "path": "/home/admin/.openclaw/workspace/skills/kb-workflow/knowledge-base/metadata" + }, + "content": { + "count": 0, + "path": "/home/admin/.openclaw/workspace/skills/kb-workflow/knowledge-base/content" + }, + "analyzed": { + "count": 0, + "path": "/home/admin/.openclaw/workspace/skills/kb-workflow/knowledge-base/analyzed" + } + } + }, + "processing_stats": { + "pending": 0, + "metadata": 0, + "content": 0, + "analyzed": 0 + }, + "recommendations": [ + "📭 收件箱为空，暂无待处理内容" + ] +} \ No newline at end of file diff --git a/skills/kb-zhihu-metadata/SKILL.md b/skills/kb-zhihu-metadata/SKILL.md new file mode 100644 index 0000000..d4e77d7 --- /dev/null +++ b/skills/kb-zhihu-metadata/SKILL.md @@ -0,0 +1,44 @@ +# KB Zhihu Metadata Skill + +**kb-zhihu-metadata** - Extract Zhihu question/answer/user metadata. + +## Features + +- Question metadata extraction +- Answer metadata extraction +- User information extraction +- JSON output format +- Markdown conversion support + +## Commands + +```bash +# Question metadata +python zhihu_metadata.py --id "266309957" --type question + +# Answer metadata +python zhihu_metadata.py --id "2003427891925509507" --type answer +``` + +## Output + +```json +{ + "success": true, + "data": { + "platform": "zhihu", + "type": "question", + "id": "266309957", + "title": "问题标题", + "author": "作者名字", + "answer_count": 50, + "followers": 1000 + } +} +``` + +## Installation + +```bash +cp -r kb-zhihu-metadata/ ~/skills/ +``` diff --git a/skills/kb-zhihu-metadata/markdown_converter.py b/skills/kb-zhihu-metadata/markdown_converter.py new file mode 100644 index 0000000..c0f912d --- /dev/null +++ b/skills/kb-zhihu-metadata/markdown_converter.py @@ -0,0 +1,17 @@ +from markdownify import MarkdownConverter as _MarkdownConverter + +class MarkdownConverter: + """HTML 转 Markdown 转换器，保留完整格式""" + + def __init__(self): + self.converter = _MarkdownConverter( + heading_style="ATX", + bullets="*,-", + code_blocks=True, + tables=True, + strip=["script", "style", "nav", "footer", "aside"] + ) + + def convert(self, html: str) -> str: + """转换 HTML 为 Markdown""" + return self.converter.convert(html) diff --git a/skills/kb-zhihu-metadata/zhihu_extractor.py b/skills/kb-zhihu-metadata/zhihu_extractor.py new file mode 100644 index 0000000..2baf648 --- /dev/null +++ b/skills/kb-zhihu-metadata/zhihu_extractor.py @@ -0,0 +1,385 @@ +""" +知乎 Playwright 内容提取器 +使用 Playwright 浏览器自动化工具抓取知乎内容 +""" + +import re +import sys +import os +from typing import Dict, Optional, Any +from urllib.parse import urlparse + +# 添加当前目录到 Python 路径以支持绝对导入 +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +try: + from .markdown_converter import MarkdownConverter +except ImportError: + from markdown_converter import MarkdownConverter + + +class PlaywrightBrowser: + """Playwright 浏览器管理器""" + + def __init__(self, headless: bool = True, timeout: int = 30): + self.headless = headless + self.timeout = timeout * 1000 + self.browser = None + self.context = None + self.page = None + + def start(self): + """启动浏览器""" + from playwright.sync_api import sync_playwright + self.playwright = sync_playwright().start() + self.browser = self.playwright.chromium.launch(headless=self.headless) + self.context = self.browser.new_context( + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36' + ) + self.page = self.context.new_page() + self.page.set_default_timeout(self.timeout) + + def close(self): + """关闭浏览器""" + if self.page: + self.page.close() + if self.context: + self.context.close() + if self.browser: + self.browser.close() + if hasattr(self, 'playwright'): + self.playwright.stop() + + def goto(self, url: str, wait_until: str = "networkidle"): + """导航到指定URL""" + return self.page.goto(url, wait_until=wait_until) + + def wait_for_selector(self, selector: str, timeout: int = None): + """等待选择器出现""" + timeout_ms = timeout * 1000 if timeout else self.timeout + return self.page.wait_for_selector(selector, timeout=timeout_ms) + + def query_selector(self, selector: str): + """查询选择器""" + return self.page.query_selector(selector) + + def query_selector_all(self, selector: str): + """查询所有匹配的选择器""" + return self.page.query_selector_all(selector) + + def evaluate(self, script: str): + """执行 JavaScript 代码""" + return self.page.evaluate(script) + + +class ZhihuPlaywrightExtractor: + """知乎内容提取器 - 使用 Playwright""" + + URL_PATTERNS = { + 'answer': r'zhihu\.com/question/\d+/answer/(\d+)', + 'article': r'zhuanlan\.zhihu\.com/p/(\d+)', + 'question': r'zhihu\.com/question/(\d+)(?:$|/)', + } + + def __init__(self, headless: bool = True, timeout: int = 30): + """ + 初始化提取器 + + Args: + headless: 是否使用无头浏览器 + timeout: 超时时间（秒） + """ + self.browser = PlaywrightBrowser(headless=headless, timeout=timeout) + self.markdown_converter = MarkdownConverter() + + def identify_url_type(self, url: str) -> Optional[str]: + """ + 识别 URL 类型 + + Returns: + 'answer', 'article', 'question' 或 None + """ + for url_type, pattern in self.URL_PATTERNS.items(): + if re.search(pattern, url): + return url_type + return None + + def extract_id(self, url: str) -> Optional[str]: + """ + 从 URL 中提取内容 ID + + Returns: + 内容 ID 或 None + """ + url_type = self.identify_url_type(url) + if url_type: + pattern = self.URL_PATTERNS[url_type] + match = re.search(pattern, url) + if match: + return match.group(1) + return None + + def _extract_text_content(self, element) -> str: + """提取元素的文本内容""" + if element is None: + return '' + return element.inner_text() if hasattr(element, 'inner_text') else '' + + def _extract_html_content(self, element) -> str: + """提取元素的 HTML 内容""" + if element is None: + return '' + return element.inner_html() if hasattr(element, 'inner_html') else '' + + def _extract_answer_content(self) -> Dict[str, Any]: + """提取回答内容""" + result = { + 'type': 'answer', + 'title': '', + 'author': '', + 'content': '', + 'content_markdown': '', + 'publish_time': '', + 'vote_up': 0, + } + + # 打印页面标题用于调试 + page_title = self.browser.page.title() + print(f"页面标题: {page_title}") + + # 尝试保存页面快照用于调试 + try: + import os + debug_dir = '/home/admin/.openclaw/workspace/debug' + os.makedirs(debug_dir, exist_ok=True) + self.browser.page.screenshot(path=os.path.join(debug_dir, 'zhihu_screenshot.png')) + with open(os.path.join(debug_dir, 'zhihu_page.html'), 'w', encoding='utf-8') as f: + f.write(self.browser.page.content()) + print(f"页面快照已保存到 {debug_dir}") + except Exception as e: + print(f"保存页面快照时出错: {e}") + + # 尝试多个可能的选择器 + selectors = { + 'title': ['h1.QuestionHeader-title', 'h1', '.QuestionHeader-main'], + 'author': ['.AuthorInfo-name', '.AuthorInfo .Text', '[itemprop="name"]'], + 'content': ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist] article'], + 'time': ['.ContentItem-time', '[itemprop="dateCreated"]'], + 'vote': ['.VoteButton--up .VoteButton-label', '.VoteButton--up'] + } + + for selector in selectors['title']: + title_elem = self.browser.query_selector(selector) + if title_elem: + result['title'] = self._extract_text_content(title_elem) + print(f"找到标题（使用选择器 {selector}）: {result['title'][:50]}...") + break + + for selector in selectors['author']: + author_elem = self.browser.query_selector(selector) + if author_elem: + result['author'] = self._extract_text_content(author_elem) + print(f"找到作者（使用选择器 {selector}）: {result['author']}") + break + + for selector in selectors['content']: + content_elem = self.browser.query_selector(selector) + if content_elem: + html_content = self._extract_html_content(content_elem) + result['content'] = html_content + result['content_markdown'] = self.markdown_converter.convert(html_content) + print(f"找到内容（使用选择器 {selector}），长度: {len(result['content_markdown'])}") + break + + for selector in selectors['time']: + time_elem = self.browser.query_selector(selector) + if time_elem: + result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem) + break + + for selector in selectors['vote']: + vote_elem = self.browser.query_selector(selector) + if vote_elem: + vote_text = self._extract_text_content(vote_elem) + try: + result['vote_up'] = int(vote_text.replace(',', '')) + except (ValueError, AttributeError): + pass + break + + return result + + def _extract_article_content(self) -> Dict[str, Any]: + """提取文章内容""" + result = { + 'type': 'article', + 'title': '', + 'author': '', + 'content': '', + 'content_markdown': '', + 'publish_time': '', + 'vote_up': 0, + } + + # 提取文章标题 + title_elem = self.browser.query_selector('.Post-Title') + if title_elem: + result['title'] = self._extract_text_content(title_elem) + + # 提取作者信息 + author_elem = self.browser.query_selector('.AuthorInfo-name') + if author_elem: + result['author'] = self._extract_text_content(author_elem) + + # 提取文章内容 + content_elem = self.browser.query_selector('.Post-RichText') + if content_elem: + html_content = self._extract_html_content(content_elem) + result['content'] = html_content + result['content_markdown'] = self.markdown_converter.convert(html_content) + + # 提取发布时间 + time_elem = self.browser.query_selector('.ContentItem-time') + if time_elem: + result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem) + + # 提取点赞数 + vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label') + if vote_elem: + vote_text = self._extract_text_content(vote_elem) + try: + result['vote_up'] = int(vote_text.replace(',', '')) + except (ValueError, AttributeError): + pass + + return result + + def _extract_question_content(self) -> Dict[str, Any]: + """提取问题内容（问题页本身）""" + result = { + 'type': 'question', + 'title': '', + 'author': '', + 'content': '', + 'content_markdown': '', + 'publish_time': '', + 'vote_up': 0, + } + + # 提取问题标题 + title_elem = self.browser.query_selector('h1.QuestionHeader-title') + if title_elem: + result['title'] = self._extract_text_content(title_elem) + + # 提取问题详情 + detail_elem = self.browser.query_selector('.QuestionHeader-detail') + if detail_elem: + html_content = self._extract_html_content(detail_elem) + result['content'] = html_content + result['content_markdown'] = self.markdown_converter.convert(html_content) + + # 问题的作者通常是提问者 + author_elem = self.browser.query_selector('.AuthorInfo-name') + if author_elem: + result['author'] = self._extract_text_content(author_elem) + + return result + + def extract(self, url: str) -> Optional[Dict[str, Any]]: + """ + 从 URL 提取知乎内容 + + Args: + url: 知乎 URL + + Returns: + 包含提取内容的字典，结构如下: + { + 'type': 'answer' | 'article' | 'question', + 'title': str, + 'author': str, + 'content': str (HTML), + 'content_markdown': str (Markdown), + 'publish_time': str, + 'vote_up': int, + } + 如果提取失败返回 None + """ + try: + # 识别 URL 类型 + url_type = self.identify_url_type(url) + if not url_type: + print(f"无法识别 URL 类型: {url}") + return None + + print(f"URL 类型: {url_type}") + print(f"启动浏览器...") + + # 启动浏览器 + self.browser.start() + + print(f"访问页面: {url}") + # 访问页面 + self.browser.goto(url, wait_until="domcontentloaded") + + print("等待页面加载...") + import time + time.sleep(3) # 等待 JavaScript 执行 + + # 尝试多个可能的选择器 + if url_type == 'answer': + selectors = ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist]'] + print(f"尝试查找选择器: {selectors}") + result = self._extract_answer_content() + elif url_type == 'article': + selectors = ['.Post-RichText', '.Post-Title'] + print(f"尝试查找选择器: {selectors}") + result = self._extract_article_content() + else: # question + selectors = ['.QuestionHeader-title', 'h1'] + print(f"尝试查找选择器: {selectors}") + result = self._extract_question_content() + + # 添加额外信息 + result['url'] = url + result['content_id'] = self.extract_id(url) + + print(f"提取成功! 标题: {result['title'][:50]}...") + return result + + except Exception as e: + print(f"提取知乎内容时出错: {e}") + import traceback + traceback.print_exc() + return None + finally: + # 关闭浏览器 + self.browser.close() + + def __enter__(self): + """上下文管理器入口""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """上下文管理器出口""" + self.browser.close() + return False + + +# 便捷函数 +def extract_zhihu_content(url: str, headless: bool = True, timeout: int = 30) -> Optional[Dict[str, Any]]: + """ + 便捷函数：提取知乎内容 + + Args: + url: 知乎 URL + headless: 是否使用无头浏览器 + timeout: 超时时间（秒） + + Returns: + 提取的内容字典 + """ + with ZhihuPlaywrightExtractor(headless=headless, timeout=timeout) as extractor: + return extractor.extract(url) diff --git a/skills/kb-zhihu-metadata/zhihu_metadata.py b/skills/kb-zhihu-metadata/zhihu_metadata.py new file mode 100644 index 0000000..d0a5c2c --- /dev/null +++ b/skills/kb-zhihu-metadata/zhihu_metadata.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +""" +知乎元数据获取工具 + +支持获取知乎问题、回答、用户的元数据信息。 +""" + +import argparse +import json +import sys +import os +from typing import Dict, Optional, Any + +# 添加当前目录到 Python 路径 +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.insert(0, current_dir) + +from zhihu_extractor import ZhihuPlaywrightExtractor + + +def extract_question(question_id: str, headless: bool = True) -> Dict[str, Any]: + """ + 获取知乎问题元数据 + + Args: + question_id: 问题 ID (数字字符串) + headless: 是否使用无头浏览器 + + Returns: + 问题元数据字典 + """ + url = f"https://www.zhihu.com/question/{question_id}" + return _extract_metadata(url, "question", headless) + + +def extract_answer(answer_id: str, headless: bool = True) -> Dict[str, Any]: + """ + 获取知乎回答元数据 + + Args: + answer_id: 回答 ID (数字字符串) + headless: 是否使用无头浏览器 + + Returns: + 回答元数据字典 + """ + # 回答 URL 格式: https://www.zhihu.com/question/{question_id}/answer/{answer_id} + # 需要先获取问题ID，这里简化处理，尝试从回答ID构建URL + # 实际上知乎回答URL需要问题ID，这里用模拟URL + url = f"https://www.zhihu.com/question/0/answer/{answer_id}" + return _extract_metadata(url, "answer", headless) + + +def extract_user(url_token: str, headless: bool = True) -> Dict[str, Any]: + """ + 获取知乎用户信息 + + Args: + url_token: 用户 URL token + headless: 是否使用无头浏览器 + + Returns: + 用户信息字典 + """ + url = f"https://www.zhihu.com/people/{url_token}" + return _extract_user_metadata(url, headless) + + +def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, Any]: + """ + 内部函数：提取元数据 + + Args: + url: 知乎 URL + content_type: 内容类型 (question/answer) + headless: 是否使用无头浏览器 + + Returns: + 元数据字典 + """ + extractor = ZhihuPlaywrightExtractor(headless=headless) + result = extractor.extract(url) + + if result: + # 标准化输出格式 + metadata = { + "type": content_type, + "id": result.get("content_id", ""), + "url": result.get("url", url), + "title": result.get("title", ""), + "author": result.get("author", ""), + "publish_time": result.get("publish_time", ""), + "vote_up": result.get("vote_up", 0), + "content_markdown": result.get("content_markdown", ""), + "raw_html": result.get("content", ""), + } + return metadata + else: + return { + "type": content_type, + "error": "Failed to extract content", + "url": url + } + + +def _extract_user_metadata(url: str, headless: bool) -> Dict[str, Any]: + """ + 内部函数：提取用户信息 + + Args: + url: 用户主页 URL + headless: 是否使用无头浏览器 + + Returns: + 用户信息字典 + """ + extractor = ZhihuPlaywrightExtractor(headless=headless) + + try: + extractor.browser.start() + extractor.browser.goto(url, wait_until="networkidle") + + import time + time.sleep(3) + + # 提取用户名 + username = "" + username_elem = extractor.browser.query_selector('.ProfileHeader-name') + if username_elem: + username = username_elem.inner_text() + + # 提取个人简介 + headline = "" + headline_elem = extractor.browser.query_selector('.ProfileHeader-headline') + if headline_elem: + headline = headline_elem.inner_text() + + # 提取简介详情 + description = "" + desc_elem = extractor.browser.query_selector('.ProfileHeader-description') + if desc_elem: + description = desc_elem.inner_text() + + # 提取关注者数 + followers = 0 + follower_elem = extractor.browser.query_selector('.FollowshipCard-followersNum') + if follower_elem: + follower_text = follower_elem.inner_text() + try: + followers = int(follower_text.replace(',', '')) + except ValueError: + pass + + # 提取关注者数 + following = 0 + following_elem = extractor.browser.query_selector('.FollowshipCard-followingsNum') + if following_elem: + following_text = following_elem.inner_text() + try: + following = int(following_text.replace(',', '')) + except ValueError: + pass + + # 提取回答数 + answers = 0 + answers_elem = extractor.browser.query_selector('.Profile-mainColumn .Tabs-item span') + if answers_elem: + answers_text = answers_elem.inner_text() + try: + answers = int(answers_text.replace(',', '')) + except ValueError: + pass + + metadata = { + "type": "user", + "url": url, + "username": username, + "headline": headline, + "description": description, + "followers": followers, + "following": following, + "answers_count": answers, + } + return metadata + + except Exception as e: + return { + "type": "user", + "error": str(e), + "url": url + } + finally: + extractor.browser.close() + + +def main(): + """命令行入口""" + parser = argparse.ArgumentParser( + description="知乎元数据获取工具", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + python zhihu_metadata.py --id "266309957" --type question + python zhihu_metadata.py --id "2003427891925509507" --type answer + python zhihu_metadata.py --id "example-user" --type user + """ + ) + + parser.add_argument( + "--id", "-i", + required=True, + help="知乎内容 ID (问题ID、回答ID或用户URL token)" + ) + + parser.add_argument( + "--type", "-t", + required=True, + choices=["question", "answer", "user"], + help="内容类型" + ) + + parser.add_argument( + "--output", "-o", + default=None, + help="输出文件路径 (默认: stdout)" + ) + + parser.add_argument( + "--headless", + action="store_true", + default=True, + help="使用无头浏览器模式 (默认: True)" + ) + + args = parser.parse_args() + + # 提取元数据 + if args.type == "question": + result = extract_question(args.id, headless=args.headless) + elif args.type == "answer": + result = extract_answer(args.id, headless=args.headless) + else: # user + result = extract_user(args.id, headless=args.headless) + + # 输出结果 + json_output = json.dumps(result, ensure_ascii=False, indent=2) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(json_output) + print(f"结果已保存到: {args.output}") + else: + print(json_output) + + +if __name__ == "__main__": + main() diff --git a/skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md b/skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md new file mode 100644 index 0000000..a5b18d3 --- /dev/null +++ b/skills/obsidian-kb/ARCHITECTURE_FLOWCHART.md @@ -0,0 +1,168 @@ +# 知识库维护系统 - 整体流程图 + +## 1. 用户交互流程（接收链接） +```mermaid +flowchart TD + A[用户发送链接] --> B{技能1: 链接解析} + B --> C{类型判断} + + C -->|单个内容| D[直接解析] + C -->|列表类型| E[询问用户] + + E --> F[处理范围选择] + F --> G[订阅选项选择] + G --> H[写入订阅配置] + + D --> I[写入收件箱 pending/] + H --> I + + I --> J[返回用户: 已收录] +``` + +## 2. 定时任务/手动触发流程 +```mermaid +flowchart TD + A[定时任务触发
或手动触发] --> B[技能5: 工作流编排] + B --> C[扫描收件箱] + + C --> D{pending/ 目录} + D -->|有待处理| E[获取待处理列表] + D -->|无| F[检查订阅更新] + + E --> G[技能2: 获取元数据] + G --> H[更新状态: metadata_ready] + H --> I[技能3: 获取内容] + I --> J[更新状态: content_ready] + J --> K[技能4: 深度分析] + K --> L[归档到 archive/] + L --> M[更新订阅追踪] + M --> N[继续处理下一个] + + F --> O{有订阅?} + O -->|是| P[检测更新] + P --> Q[新增内容加入 pending/] + Q --> R[更新订阅时间] + + O -->|否| S[跳过] + + N --> T{还有待处理?} + T -->|是| E + T -->|否| U[生成汇报文件] + U --> V[完成] +``` + +## 3. 完整主流程（简化版） +```mermaid +flowchart TD + subgraph 用户交互 + A[用户发送链接] --> B[技能1: link-parser] + B --> C{内容类型} + C -->|单个| D[直接处理] + C -->|列表| E[询问用户选项] + E --> F[保存订阅配置] + D --> G[写入收件箱 pending/] + F --> G + end + + subgraph 定时处理 + H[定时/手动触发] --> I[技能5: kb-workflow] + I --> J[扫描收件箱] + J --> K[技能2: metadata-fetcher] + K --> L[技能3: content-fetcher] + L --> M[技能4: deep-analyzer] + M --> N[归档] + N --> O[更新订阅] + O --> P[生成汇报] + end + + G --> H + + style A fill:#e1f5fe + style I fill:#fff3e0 + style K fill:#f3e5f5 + style L fill:#e8f5e9 + style M fill:#fce4ec +``` + +## 4. 各平台技能调用关系 +```mermaid +flowchart LR + subgraph 技能1 + LP[link-parser] + end + + subgraph 技能2[元数据获取] + BM[bilibili-metadata] + YM[ youtube-metadata] + ZM[ zhihu-metadata] + end + + subgraph 技能3[内容获取] + BC[bilibili-content] + OC[ocr-processor] + end + + subgraph 技能4 + DA[deep-analyzer] + end + + LP -->|解析结果| BM + LP -->|解析结果| YM + LP -->|解析结果| ZM + + BM -->|元数据| BC + BC -->|低分视频| OC + OC -->|OCR结果| DA + BM -->|元数据| DA +``` + +## 5. 数据流转 +```mermaid +flowchart LR + subgraph 输入 + URL[用户链接] + end + + subgraph 收件箱 + P[pending/] + M[metadata/] + C[content/] + end + + subgraph 输出 + A[archive/YYYY/MM/] + R[report.json] + end + + URL -->|技能1| P + P -->|技能2| M + M -->|技能3| C + C -->|技能4| A + A -.->|统计| R + + style P fill:#fffde7 + style M fill:#e3f2fd + style C fill:#e8f5e9 + style A fill:#f3e5f5 +``` + +## 6. 状态流转 +```mermaid +stateDiagram-v2 + [*] --> pending: 链接收录 + pending --> metadata: 技能2完成 + metadata --> content: 技能3完成 + content --> analyzed: 技能4完成 + analyzed --> archived: 归档完成 + archived --> [*] + + pending --> skipped: 跳过 + metadata --> failed: 元数据获取失败 + content --> failed: 内容获取失败 + analyzed --> failed: 分析失败 + + failed --> retry: 重试 + retry --> pending + retry --> metadata + retry --> content +``` diff --git a/skills/obsidian-kb/DETAILED_DESIGN.md b/skills/obsidian-kb/DETAILED_DESIGN.md new file mode 100644 index 0000000..944eb69 --- /dev/null +++ b/skills/obsidian-kb/DETAILED_DESIGN.md @@ -0,0 +1,1108 @@ +# 知识库维护系统 - 详细设计 + +> 版本: 1.0.0 +> 状态: 草稿 + +--- + +## 目录 + +1. [技能接口总览](#1-技能接口总览) +2. [技能1-link-parser](#2-技能1-link-parser) +3. [技能2-bilibili-metadata](#3-技能2-bilibili-metadata) +4. [技能2-zhihu-metadata](#4-技能2-zhihu-metadata) +5. [技能3-bilibili-content](#5-技能3-bilibili-content) +6. [技能3-ocr-processor](#6-技能3-ocr-processor) +7. [技能4-deep-analyzer](#7-技能4-deep-analyzer) +8. [技能5-kb-workflow](#8-技能5-kb-workflow) +9. [数据格式定义](#9-数据格式定义) +10. [配置文件规范](#10-配置文件规范) +11. [扩展指南](#11-扩展指南) + +--- + +## 1. 技能接口总览 + +### 1.1 调用约定 + +所有技能遵循以下调用约定： + +```yaml +输入: + - 命令行参数 + - stdin JSON (可选) + - 环境变量 + +输出: + - stdout JSON + - 0: 成功 + - 1: 失败 (错误信息输出到 stderr) + +状态码: + - 0: 成功 + - 1: 一般错误 + - 2: 参数错误 + - 3: 网络错误 + - 4: 权限错误 +``` + +### 1.2 通用错误响应 + +```json +{ + "success": false, + "error": "错误描述", + "error_code": "ERROR_CODE", + "details": {} +} +``` + +--- + +## 2. 技能1: link-parser + +**功能**: 解析链接，提取平台+类型+ID，处理短链跳转 + +### 2.1 命令行接口 + +```bash +# 基本用法 +python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD" + +# 支持短链 +python link-parser.py --url "https://b23.tv/xxxxx" + +# 批量处理 +python link-parser.py --batch input.txt + +# 输出文件 +python link-parser.py --url "..." --output result.json +``` + +### 2.2 输入参数 + +```yaml +参数: + --url, -u # 必需，单个URL + --batch, -b # 可选，批量文件（每行一个URL） + --output, -o # 可选，输出文件路径 + --format # 输出格式: json/yaml (默认: json) +``` + +### 2.3 输出格式 + +**成功**: +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "video", + "id": "BV1xx411c7mD", + "source_url": "https://www.bilibili.com/video/BV1xx411c7mD", + "canonical_url": "https://www.bilibili.com/video/BV1xx411c7mD", + "short_link_resolved": false, + "title": null, + "metadata": {} + }, + "metadata": { + "processed_at": "2024-01-15T10:00:00", + "parser_version": "1.0.0" + } +} +``` + +**列表类型**: +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "up", + "id": "7770724", + "source_url": "https://space.bilibili.com/7770724", + "is_list_type": true, + "list_info": { + "total_items": 156, + "item_type": "video" + } + }, + "metadata": { + "requires_user_input": true, + "input_options": { + "scope": ["subscribe_only", "new_only", "all"], + "subscription": ["daily", "weekly", "custom", "none"] + } + } +} +``` + +### 2.4 支持的链接格式 + +| 类型 | 平台 | 正则示例 | 输出 | +|------|------|----------|------| +| 短链 | B站 | `b23.tv/[a-zA-Z0-9]+` | resolved_url + 解析 | +| 视频 | B站 | `bilibili.com/video/BV[a-zA-Z0-9]+` | `{type: video, id: BV...}` | +| 专栏 | B站 | `bilibili.com/read/cv[0-9]+` | `{type: article, id: cv...}` | +| 动态 | B站 | `bilibili.com/opus/[0-9]+` | `{type: opus, id: ...}` | +| UP主 | B站 | `space.bilibili.com/[0-9]+` | `{type: up, id: ...}` | +| 收藏夹 | B站 | `space.bilibili.com/[0-9]+/favlist?fid=[0-9]+` | `{type: favlist, id: ...}` | +| 合集 | B站 | `space.bilibili.com/[0-9]+/lists/[0-9]+?type=season` | `{type: playlist, id: ...}` | +| YouTube | YouTube | `(youtu.be/[a-zA-Z0-9]+|youtube.com/watch?v=[a-zA-Z0-9]+)` | `{type: video, id: ...}` | +| 知乎 | 知乎 | `zhihu.com/question/[0-9]+` | `{type: question, id: ...}` | +| 知乎 | 知乎 | `zhihu.com/answer/[0-9]+` | `{type: answer, id: ...}` | +| 知乎 | 知乎 | `zhihu.com/people/[a-zA-Z0-9-]+` | `{type: user, id: ...}` | +| 知乎 | 知乎 | `zhihu.com/collection/[0-9]+` | `{type: collection, id: ...}` | + +### 2.5 复用代码 + +```python +# 复用现有代码 +from shortlink_resolver import resolve_shortlink, extract_bilibili_id +``` + +--- + +## 3. 技能2: bilibili-metadata + +**功能**: 反爬获取B站视频/专栏元数据 + +### 3.1 命令行接口 + +```bash +# 基本用法 +python bilibili-metadata.py --id "BV1xx411c7mD" + +# 指定输出目录 +python bilibili-metadata.py --id "BV1xx" --output ./metadata/ + +# 批量处理 +python bilibili-metadata.py --batch ids.txt --output ./ + +# 视频+专栏支持 +python bilibili-metadata.py --id "cv43780186" --type "article" +``` + +### 3.2 输入参数 + +```yaml +参数: + --id, -i # 必需，内容ID + --type, -t # 内容类型: video/article/opus (默认: video) + --output, -o # 输出目录 (默认: ./) + --batch, -b # 批量文件 (每行: id [type]) + --config, -c # 配置文件路径 +``` + +### 3.3 输出格式 + +**视频元数据**: +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "video", + "id": "BV1xx411c7mD", + "source_url": "https://www.bilibili.com/video/BV1xx411c7mD", + "title": "Python完整教程", + "description": "这是一个Python教程...", + "author": { + "id": 7770724, + "name": "UP主名字", + "face": "https://i0.hdslb.com/xxx.jpg" + }, + "duration": 9000, + "duration_text": "2:30:00", + "publish_time": "2024-01-01T12:00:00", + "tags": ["python", "tutorial", "编程"], + "stats": { + "view": 100000, + "like": 5000, + "coin": 2000, + "favorite": 1000, + "share": 500 + }, + "subtitle": { + "has_subtitle": true, + "languages": ["zh-CN"], + "url": "https://api.bilibili.com/..." + }, + "pic": "https://i0.hdslb.com/xxx.jpg", + "quality_score": null + }, + "metadata": { + "fetched_at": "2024-01-15T10:00:00", + "source": "bilibili_api" + } +} +``` + +**专栏文章元数据**: +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "article", + "id": "cv43780186", + "source_url": "https://www.bilibili.com/read/cv43780186", + "title": "文章标题", + "description": "文章摘要...", + "author": { + "id": 123456, + "name": "作者名字", + "face": "https://i0.hdslb.com/xxx.jpg" + }, + "publish_time": "2024-01-01T12:00:00", + "stats": { + "view": 50000, + "like": 1000, + "comment": 200 + }, + "word_count": 5000, + "images_count": 10, + "category": "技术" + }, + "metadata": { + "fetched_at": "2024-01-15T10:00:00", + "source": "bilibili_api" + } +} +``` + +**UP主/收藏夹列表**: +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "up", + "id": "7770724", + "source_url": "https://space.bilibili.com/7770724", + "name": "UP主名字", + "description": "UP主介绍...", + "follower": 100000, + "following": 100, + "archive_count": 156, + "content_stats": { + "total_views": 10000000, + "total_likes": 500000 + }, + "subscription": null + }, + "metadata": { + "fetched_at": "2024-01-15T10:00:00" + } +} +``` + +### 3.4 复用代码 + +```python +# 复用现有代码 +from bilibili_extractor import BilibiliExtractor +``` + +--- + +## 4. 技能2: zhihu-metadata + +**功能**: 获取知乎元数据 + +### 4.1 命令行接口 + +```bash +python zhihu-metadata.py --id "266309957" --type "question" +python zhihu-metadata.py --id "2003427891925509507" --type "answer" +``` + +### 4.2 输入参数 + +```yaml +参数: + --id, -i # 必需，内容ID + --type, -t # 内容类型: question/answer/article/user/collection (默认: question) + --output, -o # 输出目录 + --config, -c # 配置文件路径 +``` + +### 4.3 输出格式 + +**问题元数据**: +```json +{ + "success": true, + "data": { + "platform": "zhihu", + "type": "question", + "id": "266309957", + "source_url": "https://www.zhihu.com/question/266309957", + "title": "问题标题", + "description": "问题描述...", + "author": { + "id": "user-xxx", + "name": "作者名字", + "url_token": "user-xxx" + }, + "answer_count": 50, + "followers": 1000, + "browse_count": 50000, + "created_time": "2024-01-01T12:00:00", + "updated_time": "2024-01-15T10:00:00", + "tags": ["标签1", "标签2"] + }, + "metadata": { + "fetched_at": "2024-01-15T10:00:00" + } +} +``` + +**回答元数据**: +```json +{ + "success": true, + "data": { + "platform": "zhihu", + "type": "answer", + "id": "2003427891925509507", + "source_url": "https://www.zhihu.com/question/266309957/answer/2003427891925509507", + "question_id": "266309957", + "question_title": "问题标题", + "author": { + "id": "user-xxx", + "name": "作者名字", + "headline": "作者简介" + }, + "vote_up": 1000, + "comment_count": 50, + "created_time": "2024-01-01T12:00:00", + "updated_time": "2024-01-15T10:00:00", + "content_summary": "回答摘要..." + }, + "metadata": { + "fetched_at": "2024-01-15T10:00:00" + } +} +``` + +### 4.4 复用代码 + +```python +# 复用现有代码 +from zhihu_extractor import ZhihuDrissionExtractor +``` + +--- + +## 5. 技能3: bilibili-content + +**功能**: 下载低分视频 + 提取字幕/专栏全文 + +### 5.1 命令行接口 + +```bash +# 基本用法（视频） +python bilibili-content.py --id "BV1xx411c7mD" --output ./content/ + +# 视频+字幕 +python bilibili-content.py --id "BV1xx" --output ./ --with-subtitle + +# 专栏全文 +python bilibili-content.py --id "cv43780186" --type article --output ./ + +# 低分辨率（用于OCR） +python bilibili-content.py --id "BV1xx" --quality 480 --output ./ +``` + +### 5.2 输入参数 + +```yaml +参数: + --id, -i # 必需，内容ID + --type, -t # 内容类型: video/article/opus (默认: video) + --output, -o # 输出目录 (默认: ./) + --quality # 视频质量: 360/480/720/1080 (默认: 480) + --with-subtitle # 是否提取字幕 (默认: true) + --subtitle-lang # 字幕语言 (默认: zh-CN) + --format # 输出格式: json/ndjson (默认: json) +``` + +### 5.3 输出格式 + +**视频内容**: +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "video", + "id": "BV1xx411c7mD", + "video_file": { + "path": "content/BV1xx_video.mp4", + "size_bytes": 52428800, + "duration": 9000, + "quality": "480p" + }, + "subtitle": { + "format": "srt", + "path": "content/BV1xx_subtitle.srt", + "language": "zh-CN", + "text_length": 50000, + "lines_count": 1500 + }, + "danmaku": { + "path": "content/BV1xx_danmaku.xml", + "count": 5000 + }, + "frames": { + "output_dir": "content/BV1xx_frames", + "count": 900, + "fps": 0.1 + } + }, + "metadata": { + "fetched_at": "2024-01-15T10:00:00", + "content_quality": "preview" + } +} +``` + +**专栏内容**: +```json +{ + "success": true, + "data": { + "platform": "bilibili", + "type": "article", + "id": "cv43780186", + "content": { + "markdown": "# 文章标题\n\n内容...", + "markdown_path": "content/cv43780186.md", + "html": "...", + "word_count": 5000, + "images": [ + { + "url": "https://...", + "path": "content/images/img_001.jpg", + "description": "图片描述" + } + ] + }, + "author": { + "name": "作者名字" + }, + "publish_time": "2024-01-01T12:00:00" + }, + "metadata": { + "fetched_at": "2024-01-15T10:00:00" + } +} +``` + +### 5.4 复用代码 + +```python +# 复用现有代码 +from video_stream_extractor import VideoStreamExtractor +from bilibili_extractor import BilibiliExtractor +``` + +--- + +## 6. 技能3: ocr-processor + +**功能**: 通用字幕OCR（被技能3调用） + +### 6.1 命令行接口 + +```bash +# 基本用法 +python ocr-processor.py --input ./frames/ --output ./ocr.json + +# 指定引擎 +python ocr-processor.py --input ./frames/ --engine paddleocr --output ./ + +# 只处理部分帧 +python ocr-processor.py --input ./frames/ --start 0 --end 100 --output ./ +``` + +### 6.2 输入参数 + +```yaml +参数: + --input, -i # 必需，帧图片目录或单个图片 + --output, -o # 输出文件 (默认: ocr.json) + --engine # OCR引擎: paddleocr/tesseract (默认: paddleocr) + --region # 字幕区域: bottom/full/auto (默认: bottom) + --fps # 抽帧频率 (默认: 0.1，即每10秒1帧) + --start # 开始帧号 (默认: 0) + --end # 结束帧号 (默认: -1，即全部) + --confidence # 置信度阈值 (默认: 0.7) +``` + +### 6.3 输出格式 + +```json +{ + "success": true, + "data": { + "frames": { + "frame_0001.jpg": { + "texts": [ + { + "text": "字幕文本", + "confidence": 0.95, + "bbox": [x1, y1, x2, y2] + } + ], + "timestamp": 0.0 + } + }, + "merged_text": "合并的字幕文本...", + "statistics": { + "total_frames": 100, + "frames_with_text": 80, + "total_texts": 500, + "unique_texts": 200 + } + }, + "metadata": { + "engine": "paddleocr", + "processed_at": "2024-01-15T10:00:00" + } +} +``` + +### 6.4 复用代码 + +```python +# 复用现有代码 +from ocr_processor import OCRProcessor +``` + +--- + +## 7. 技能4: deep-analyzer + +**功能**: AI深度分析，按内容类型使用不同模板 + +### 7.1 命令行接口 + +```bash +# 基本用法 +python deep-analyzer.py --metadata meta.json --content content.json --output ./analysis.md + +# 指定类型 +python deep-analyzer.py --metadata meta.json --type tutorial --output ./ + +# 使用自定义模板 +python deep-analyzer.py --metadata meta.json --template my_template.md --output ./ + +# 带OCR结果 +python deep-analyzer.py --metadata meta.json --content content.json --ocr ocr.json --output ./ +``` + +### 7.2 输入参数 + +```yaml +参数: + --metadata, -m # 必需，元数据JSON文件 + --content, -c # 必需，内容JSON文件 + --output, -o # 必需，输出Markdown文件 + --type, -t # 内容类型: auto/tutorial/documentation/news (默认: auto) + --template # 自定义模板路径 + --ocr # OCR结果JSON文件 (可选) + --model # AI模型 (默认: 配置中指定) + --tags # 预设标签 (可选) +``` + +### 7.3 模板变量 + +**通用变量**: +```jinja2 +{{ title }} # 标题 +{{ source }} # 来源 (bilibili/zhihu) +{{ url }} # 原始URL +{{ author }} # 作者/UP主 +{{ duration }} # 时长 +{{ date }} # 发布日期 +{{ tags }} # 标签列表 +{{ summary }} # 详细摘要 +{{ key_points }} # 核心要点 +{{ quality_score }} # 质量评分 (1-100) +``` + +**教程视频额外变量**: +```jinja2 +{{ prerequisites }} # 前置知识 +{{ outline }} # 分段大纲 +{{ pitfalls }} # 常见坑点 +{{ practice_advice }} # 实践建议 +``` + +### 7.4 输出格式 + +**教程类输出**: +```markdown +--- +title: "Python完整教程" +source: "bilibili" +url: "https://..." +author: "CodeMaster" +duration: "2:30:00" +date: "2024-01-01T00:00:00" +tags: + - python + - tutorial + - 编程入门 +quality_score: 95 +--- + +## 🎯 一句话总结 + +本教程系统讲解Python基础语法，适合零基础学习者。 + +## 📚 前置知识 + +- 无需编程基础 +- 电脑基本操作能力 + +## 📝 步骤详解 + +### 环境配置 [00:00-15:00] + +首先下载Python... + +![frame_0015: Python官网下载页面](./images/frame_0015.jpg) + +```bash +# 安装Python +pip install python +``` + +### 基础语法 [15:00-45:00] + +... + +## 💡 关键要点 + +1. Python使用缩进定义代码块 +2. 变量不需要声明类型 +3. ... + +## ⚠️ 常见坑点 + +- Python 2 与 3 的区别 +- 缩进问题导致报错 + +## 🤔 实践建议 + +- 跟随视频动手编码 +- 完成课后练习 + +## 📚 相关资源 + +- [Python官网](https://www.python.org/) +- [官方文档](https://docs.python.org/) +``` + +### 7.5 复用代码 + +```python +# 复用现有代码 +from deep_analyzer import DeepAnalyzer +from ai_client import AIClient +``` + +--- + +## 8. 技能5: kb-workflow + +**功能**: 工作流编排 + OpenClaw cron定时 + 手动触发 + 归档管理 + +### 8.1 命令行接口 + +```bash +# 手动触发处理 +python kb-workflow.py --trigger manual + +# 定时任务触发 +python kb-workflow.py --trigger scheduled + +# 只处理待处理列表 +python kb-workflow.py --trigger manual --scope pending + +# 强制刷新订阅 +python kb-workflow.py --trigger manual --refresh-subscriptions + +# 生成汇报 +python kb-workflow.py --report + +# 查看状态 +python kb-workflow.py --status +``` + +### 8.2 输入参数 + +```yaml +参数: + --trigger, -t # 触发方式: manual/scheduled (默认: manual) + --scope # 处理范围: all/pending/metadata/content (默认: all) + --limit # 最大处理数量 (默认: 5) + --refresh-subs # 刷新订阅 + --report # 生成汇报 + --status # 显示状态 + --config, -c # 配置文件路径 +``` + +### 8.3 汇报文件格式 + +```json +{ + "run_id": "run_20240115_030000", + "run_time": "2024-01-15T03:00:00", + "trigger": "scheduled", + "scope": "all", + "processed": 5, + "success": 4, + "failed": 1, + "details": { + "total_inbox": 12, + "pending": 2, + "metadata_ready": 3, + "content_ready": 5, + "analyzed": 2, + "archived": 10 + }, + "items": [ + { + "id": "BV1xx411c7mD", + "type": "video", + "status": "success", + "duration_seconds": 45 + }, + { + "id": "BV2yy522c8nE", + "type": "video", + "status": "failed", + "error": "download timeout" + } + ], + "subscriptions": { + "checked": 3, + "updates_found": 5, + "new_items_added": 2 + }, + "storage": { + "inbox_size_mb": 128, + "archive_size_mb": 512 + }, + "generated_at": "2024-01-15T03:05:00" +} +``` + +### 8.4 工作流步骤 + +```python +def run_workflow(trigger, scope, limit): + # 1. 扫描收件箱 + pending_items = scan_pending() + metadata_items = scan_metadata() + content_items = scan_content() + + # 2. 检查订阅更新 + if trigger == "scheduled": + check_subscriptions() + + # 3. 按优先级处理 + all_items = pending_items + metadata_items + content_items + + for item in all_items[:limit]: + try: + if item.status == "pending": + metadata = run_skill2(item) + update_item(item, metadata, status="metadata_ready") + elif item.status == "metadata_ready": + content = run_skill3(item) + update_item(item, content, status="content_ready") + elif item.status == "content_ready": + analysis = run_skill4(item) + archive_item(item, analysis) + update_item(item, status="archived") + except Exception as e: + record_failure(item, str(e)) + + # 4. 生成汇报 + generate_report() +``` + +### 8.5 复用代码 + +```python +# 复用现有代码 +from pipeline import Pipeline +from git_sync import GitSync +``` + +--- + +## 9. 数据格式定义 + +### 9.1 收件箱目录结构 + +``` +inbox/ +├── pending/ +│ └── bilibili_video_BV1xx_2024-01-15T100000.json +├── metadata/ +│ └── bilibili_video_BV1xx_2024-01-15T100000.json +├── content/ +│ └── bilibili_video_BV1xx_2024-01-15T100000.json +└── analyzed/ + └── bilibili_video_BV1xx_2024-01-15T100000.json +``` + +### 9.2 元数据文件格式 + +```json +{ + "platform": "bilibili", + "type": "video", + "id": "BV1xx411c7mD", + "source_url": "https://www.bilibili.com/video/BV1xx411c7mD", + "status": "pending", + "scope": "all", + "title": "Python完整教程", + "author": {"id": 7770724, "name": "UP主名字"}, + "duration": 9000, + "publish_time": "2024-01-01T12:00:00", + "tags": ["python", "tutorial"], + "stats": {"view": 100000, "like": 5000}, + "subscription": { + "enabled": true, + "scope": "new_only", + "frequency": "daily", + "last_checked": "2024-01-15T00:00:00", + "items_count": 156, + "processed_count": 45 + }, + "created_at": "2024-01-15T00:00:00", + "updated_at": "2024-01-15T10:00:00", + "retry_count": 0 +} +``` + +### 9.3 归档目录结构 + +``` +archive/ +├── 2024/ +│ ├── 01/ +│ │ ├── video/ +│ │ │ ├── bilibili_BV1xx_2024-01-15.md +│ │ │ └── index.json +│ │ ├── article/ +│ │ │ └── index.json +│ │ └── index.json +│ └── index.json +└── index.json +``` + +--- + +## 10. 配置文件规范 + +### 10.1 配置文件位置 + +优先级: +1. `./config.yaml` (当前目录) +2. `~/.config/obsidian-kb/config.yaml` +3. `~/.obsidian-kb/config.yaml` + +### 10.2 完整配置示例 + +```yaml +# AI 配置 +ai: + provider: "openai" + model: "gpt-4" + api_key: "${OPENAI_API_KEY}" + temperature: 0.3 + max_tokens: 8192 + +# 存储配置 +storage: + inbox_dir: "~/work/inbox" + archive_dir: "~/ObsidianVault/knowledge-base" + temp_dir: "/tmp/obsidian-kb" + max_concurrent: 3 + cleanup_after_days: 7 + +# 输出配置 +output: + format: "markdown" + encoding: "utf-8" + include_timestamp: true + include_source: true + +# Git 同步配置 +git: + enabled: true + vault_path: "~/ObsidianVault" + remote: "origin" + branch: "main" + auto_commit: true + +# Bilibili 提取器配置 +bilibili: + enabled: true + cookie: "${BILIBILI_COOKIE}" + use_playwright: true + headless: true + timeout: 30 + subtitle: + enabled: true + language: "zh-CN" + +# 知乎提取器配置 +zhihu: + enabled: true + use_drissionpage: true + cookie_file: "~/.obsidian-kb/zhihu_cookies.json" + +# OCR 配置 +ocr: + engine: "paddleocr" + confidence_threshold: 0.7 + region: "bottom" + fps: 0.1 + max_frames: 500 + +# 工作流配置 +workflow: + schedule: "0 3 * * *" + max_items_per_run: 5 + subscription_check_interval: 24 + +# 汇报配置 +reporting: + enabled: true + output_dir: "~/work/reports" + format: "json" + retention_days: 30 + +# 日志配置 +logging: + level: "INFO" + format: "[%(asctime)s] %(levelname)s - %(message)s" + file: "~/work/logs/obsidian-kb.log" +``` + +--- + +## 11. 扩展指南 + +### 11.1 新增平台步骤 + +以新增「小红书」为例： + +#### 步骤1: 创建链接解析器扩展 + +```python +# link-parser 的 platform_registry 中添加 +PLATFORM_REGISTRY = { + "xiaohongshu": { + "name": "小红书", + "patterns": [ + r"xiaohongshu.com/notes/([a-zA-Z0-9]+)", + ], + "types": ["note", "user", "collection"] + } +} +``` + +#### 步骤2: 创建元数据获取器 + +```bash +# 创建文件: skills/xiaohongshu-metadata/ +``` + +```python +class XiaohongshuMetadata: + def extract(self, url, config): + # 实现获取逻辑 + pass +``` + +#### 步骤3: 更新工作流配置 + +```yaml +xiaohongshu: + enabled: true + cookie: "${XIAOHONGSHU_COOKIE}" +``` + +### 11.2 新增分析模板 + +```jinja2 +# templates/podcast.md +--- +title: "{{ title }}" +source: "{{ source }}" +url: "{{ url }}" +author: "{{ author }}" +duration: "{{ duration }}" +date: "{{ date }}" +tags: +{{- for tag in tags }} + - {{ tag }} +{{- endfor }} +--- + +## 🎧 播客摘要 + +{{ summary }} + +## 💡 核心要点 + +{{ key_points }} +``` + +--- + +## 附录A: 技能清单 + +| # | 技能名 | 主要文件 | 复用代码 | +|---|--------|----------|----------| +| 1 | link-parser | `link-parser.py` | shortlink_resolver.py | +| 2 | bilibili-metadata | `bilibili_metadata.py` | bilibili_extractor.py | +| 2 | zhihu-metadata | `zhihu_metadata.py` | zhihu_extractor.py | +| 3 | bilibili-content | `bilibili_content.py` | video_stream_extractor.py | +| 3 | ocr-processor | `ocr_processor.py` | ocr_processor.py | +| 4 | deep-analyzer | `deep_analyzer.py` | deep_analyzer.py | +| 5 | kb-workflow | `kb_workflow.py` | pipeline.py, git_sync.py | + +--- + +## 附录B: OpenClaw Cron 配置示例 + +```yaml +# 定时任务配置 +schedule: "0 3 * * *" +command: "python /path/to/skills/kb-workflow/kb_workflow.py --trigger scheduled" +``` + +--- + +蒋先生，详细设计文档已完成！ + +**包含内容：** +- ✅ 7个技能的完整接口规范（命令行参数、输入输出格式） +- ✅ 数据格式详细定义（收件箱、归档、汇报） +- ✅ 配置文件规范 +- ✅ 复用代码对照表 +- ✅ 扩展指南（新增平台/模板步骤） + +**下一步：** +可以进入 **write-plan** 阶段，生成具体实现计划吗？ \ No newline at end of file