#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ KB Workflow - Knowledge Base Workflow Orchestration 知识库工作流编排技能。 ## 文件存放规则 ``` knowledge-base/ ├── kb.config.yaml # 配置文件 ├── .gitignore # Git忽略规则 ├── pending/ # 待处理内容 (*.md) ├── metadata/ # 链接解析结果 (*.link.json) ├── content/ │ ├── json/ # 元数据 (*.meta.json) │ ├── downloads/ # 视频文件 (*.mp4/*.flv) - 7天后清理 │ └── subtitles/ # 字幕文件 (*.srt/*.lrc) - 7天后清理 ├── frames/ # 视频截帧 (*.jpg) - 7天后清理 ├── analyzed/ # 分析报告 (*.md) └── archive/ # Git归档目录 ├── YYYY-MM-DD/ # 按日期归档 └── subscriptions/ # 订阅数据 ├── subscriptions.json # 订阅配置 └── processed.json # 已处理项目 ``` ## Git版本管理 配置环境变量: - KB_GIT_ROOT: archive目录路径 - KB_GIT_REMOTE: 远程仓库地址 - KB_GIT_USER: 用户名 - KB_GIT_EMAIL: 邮箱 Usage: python kb_workflow.py --trigger manual --scope pending --limit 5 """ import os import sys import json import argparse import logging import subprocess import shutil from datetime import datetime, timedelta from pathlib import Path from typing import Dict, List, Optional, Any import yaml logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) logger = logging.getLogger('KBWorkflow') class KBWorkflow: """Knowledge Base Workflow Orchestrator""" STAGES = ['pending', 'metadata', 'content', 'analyzed'] # Git配置环境变量名 ENV_GIT_ROOT = 'KB_GIT_ROOT' ENV_GIT_REMOTE = 'KB_GIT_REMOTE' ENV_GIT_USER = 'KB_GIT_USER' ENV_GIT_EMAIL = 'KB_GIT_EMAIL' def __init__(self, base_path: str = None, skills_path: str = None): """ Initialize KB Workflow Args: base_path: Base path for knowledge base skills_path: Path to skills directory """ self.base_path = Path(base_path) if base_path else Path(__file__).resolve().parent.parent / "knowledge-base" self.skills_path = Path(skills_path) if skills_path else Path(__file__).resolve().parent.parent # Git配置 self.git_root = os.environ.get(self.ENV_GIT_ROOT, '') self.git_remote = os.environ.get(self.ENV_GIT_REMOTE, '') self.git_user = os.environ.get(self.ENV_GIT_USER, '') self.git_email = os.environ.get(self.ENV_GIT_EMAIL, '') # 创建目录结构 self._ensure_directories() # 加载配置 self.config = self._load_config() # 检查Git self.git_status = self._check_git() logger.info(f"KB Workflow initialized") logger.info(f"Base: {self.base_path}") logger.info(f"Git: {'✓' if self.git_root else '✗'}") def _ensure_directories(self): """创建目录结构""" # 主目录 for subdir in self.STAGES: dir_path = self.base_path / subdir if not dir_path.exists(): dir_path.mkdir(parents=True, exist_ok=True) # 内容子目录 for subdir in ['json', 'downloads', 'subtitles']: dir_path = self.base_path / 'content' / subdir if not dir_path.exists(): dir_path.mkdir(parents=True, exist_ok=True) # 截帧目录 frames_dir = self.base_path / 'frames' if not frames_dir.exists(): frames_dir.mkdir(parents=True, exist_ok=True) # 归档目录 archive_dir = self.base_path / 'archive' if not archive_dir.exists(): archive_dir.mkdir(parents=True, exist_ok=True) # 订阅目录 subs_dir = archive_dir / 'subscriptions' if not subs_dir.exists(): subs_dir.mkdir(parents=True, exist_ok=True) def _load_config(self) -> Dict: """加载配置文件""" config_path = self.base_path / 'kb.config.yaml' default_config = { 'git': { 'enabled': bool(self.git_root), 'remote': self.git_remote, 'user': self.git_user, 'email': self.git_email }, 'cleanup': { 'video_retention_days': 7, 'frame_retention_days': 7, 'auto_cleanup': True }, 'subscriptions': { 'check_interval_hours': 24 } } if config_path.exists(): try: with open(config_path, 'r', encoding='utf-8') as f: user_config = yaml.safe_load(f) or {} # 合并配置 for section in default_config: if section in user_config: default_config[section].update(user_config[section]) except Exception as e: logger.warning(f"Failed to load config: {e}") return default_config # ==================== Git管理 ==================== def _check_git(self) -> Dict[str, Any]: """检查Git状态""" status = { 'initialized': False, 'remote_configured': False, 'remote_url': '', 'status': 'not configured' } git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive' if not str(git_root): status['status'] = 'not configured - set KB_GIT_ROOT' return status git_dir = git_root / '.git' if not git_dir.exists(): status['status'] = 'not initialized' return status status['initialized'] = True # 检查远程 try: result = subprocess.run( ['git', 'remote', '-v'], cwd=str(git_root), capture_output=True, text=True ) if result.returncode == 0 and result.stdout: status['remote_configured'] = True status['remote_url'] = result.stdout.split()[1] if result.stdout.split() else '' status['status'] = 'ready' else: status['status'] = 'no remote' except Exception as e: status['status'] = f'error: {e}' return status def _git_init(self) -> bool: """初始化Git仓库""" git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive' if not git_root.exists(): git_root.mkdir(parents=True, exist_ok=True) git_dir = git_root / '.git' if git_dir.exists(): logger.info("Git already initialized") return True try: # 初始化 subprocess.run(['git', 'init'], cwd=str(git_root), capture_output=True) # 配置用户 if self.git_user: subprocess.run(['git', 'config', 'user.name', self.git_user], cwd=str(git_root)) if self.git_email: subprocess.run(['git', 'config', 'user.email', self.git_email], cwd=str(git_root)) # 创建.gitignore ignore_content = """# 知识库Git归档 *.tmp *.log """ (git_root / '.gitignore').write_text(ignore_content) # 初始提交 subprocess.run(['git', 'add', '.'], cwd=str(git_root)) commit_msg = f"Initial commit - Knowledge base archive" subprocess.run(['git', 'commit', '-m', commit_msg], cwd=str(git_root)) logger.info(f"Git initialized at {git_root}") return True except Exception as e: logger.error(f"Git init failed: {e}") return False def _git_add_commit(self, files: List[str], message: str) -> bool: """Git add and commit""" git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive' if not (git_root / '.git').exists(): logger.warning("Git not initialized") return False try: # Add for f in files: subprocess.run(['git', 'add', f], cwd=str(git_root), capture_output=True) # Commit result = subprocess.run( ['git', 'commit', '-m', message], cwd=str(git_root), capture_output=True, text=True ) if result.returncode == 0: logger.info(f"Committed: {message}") return True else: logger.warning(f"Commit failed: {result.stderr}") return False except Exception as e: logger.error(f"Git error: {e}") return False def _git_push(self, retries: int = 3) -> Dict[str, Any]: """Git push with retry""" git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive' result = {'success': False, 'message': '', 'retries': 0} if not self.git_remote: result['message'] = 'no remote configured' return result if not (git_root / '.git').exists(): result['message'] = 'git not initialized' return result for i in range(retries): result['retries'] = i + 1 try: proc = subprocess.run( ['git', 'push', 'origin', 'master'], cwd=str(git_root), capture_output=True, text=True, timeout=60 ) if proc.returncode == 0: result['success'] = True result['message'] = 'pushed successfully' return result else: result['message'] = proc.stderr if i < retries - 1: import time time.sleep(2 ** i) # 指数退避 except subprocess.TimeoutExpired: result['message'] = 'timeout' except Exception as e: result['message'] = str(e) logger.warning(f"Push failed after {retries} attempts") return result def _git_status(self) -> Dict: """获取Git状态""" git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive' if not (git_root / '.git').exists(): return {'error': 'not initialized'} try: # 获取远程URL remote = subprocess.run( ['git', 'remote', 'get-url', 'origin'], cwd=str(git_root), capture_output=True, text=True ) remote_url = remote.stdout.strip() if remote.returncode == 0 else '' # 获取提交数 log = subprocess.run( ['git', 'rev-list', '--count', 'HEAD'], cwd=str(git_root), capture_output=True, text=True ) commits = log.stdout.strip() if log.returncode == 0 else '0' return { 'initialized': True, 'remote_url': remote_url, 'commits': commits, 'status': 'ready' if remote_url else 'no remote' } except Exception as e: return {'error': str(e)} # ==================== 订阅管理 ==================== def _load_subscriptions(self) -> Dict: """加载订阅数据""" subs_file = self.base_path / 'archive' / 'subscriptions' / 'subscriptions.json' default_subs = { 'version': '1.0', 'updated_at': datetime.now().isoformat(), 'subscriptions': [], 'processed_items': [] } if subs_file.exists(): try: with open(subs_file, 'r', encoding='utf-8') as f: return json.load(f) except Exception as e: logger.warning(f"Failed to load subscriptions: {e}") return default_subs def _save_subscriptions(self, data: Dict) -> bool: """保存订阅数据""" subs_file = self.base_path / 'archive' / 'subscriptions' / 'subscriptions.json' try: data['updated_at'] = datetime.now().isoformat() with open(subs_file, 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=2) # Git提交 self._git_add_commit( ['subscriptions/subscriptions.json'], f"Update subscriptions - {datetime.now().strftime('%Y-%m-%d %H:%M')}" ) return True except Exception as e: logger.error(f"Failed to save subscriptions: {e}") return False def add_subscription(self, platform: str, subs_type: str, identifier: str, name: str = '') -> Dict: """添加订阅""" subs = self._load_subscriptions() sub_id = f"sub_{len(subs['subscriptions']) + 1:03d}" new_sub = { 'id': sub_id, 'platform': platform, 'type': subs_type, 'identifier': identifier, 'name': name, 'subscribed_at': datetime.now().isoformat(), 'last_checked': None, 'last_update': None, 'status': 'active' } subs['subscriptions'].append(new_sub) self._save_subscriptions(subs) logger.info(f"Added subscription: {sub_id} - {platform}/{subs_type}/{identifier}") return new_sub def mark_item_processed(self, item_id: str, title: str = '', archived: bool = True) -> bool: """标记项目为已处理""" subs = self._load_subscriptions() # 检查是否已存在 existing = [p for p in subs['processed_items'] if p['id'] == item_id] if existing: return True # 已存在,不重复添加 new_item = { 'id': item_id, 'title': title, 'processed_at': datetime.now().isoformat(), 'archived': archived } subs['processed_items'].append(new_item) # Git提交 self._git_add_commit( ['subscriptions/subscriptions.json'], f"Mark processed: {item_id}" ) return self._save_subscriptions(subs) def is_item_processed(self, item_id: str) -> bool: """检查项目是否已处理""" subs = self._load_subscriptions() return any(p['id'] == item_id for p in subs['processed_items']) # ==================== 清理策略 ==================== def cleanup_project(self, base_name: str) -> Dict[str, Any]: """清理单个项目的临时文件""" result = {'cleaned': [], 'errors': []} # 清理视频文件 downloads_dir = self.base_path / 'content' / 'downloads' for f in downloads_dir.glob(f'{base_name}.*'): try: f.unlink() result['cleaned'].append(f"downloads/{f.name}") except Exception as e: result['errors'].append(str(e)) # 清理截帧 frames_dir = self.base_path / 'frames' for f in frames_dir.glob(f'{base_name}_*'): try: f.unlink() result['cleaned'].append(f"frames/{f.name}") except Exception as e: result['errors'].append(str(e)) if result['cleaned']: logger.info(f"Cleaned project {base_name}: {len(result['cleaned'])} files") return result def cleanup_expired(self, days: int = 30) -> Dict[str, Any]: """清理过期临时文件""" result = {'cleaned': [], 'errors': [], 'count': 0} cutoff = datetime.now() - timedelta(days=days) # 清理下载目录 for f in (self.base_path / 'content' / 'downloads').iterdir(): if f.is_file(): mtime = datetime.fromtimestamp(f.stat().st_mtime) if mtime < cutoff: try: f.unlink() result['cleaned'].append(str(f.relative_to(self.base_path))) result['count'] += 1 except Exception as e: result['errors'].append(str(e)) # 清理截帧目录 for f in (self.base_path / 'frames').iterdir(): if f.is_file(): mtime = datetime.fromtimestamp(f.stat().st_mtime) if mtime < cutoff: try: f.unlink() result['cleaned'].append(str(f.relative_to(self.base_path))) result['count'] += 1 except Exception as e: result['errors'].append(str(e)) logger.info(f"Expired cleanup: {result['count']} files") return result # ==================== 归档 ==================== def archive_item(self, base_name: str, report_content: str) -> Dict[str, Any]: """归档项目""" result = { 'success': False, 'archive_path': '', 'git_commit': False, 'git_push': None, 'cleanup': None, 'errors': [] } # 1. 创建归档目录 date_dir = self.base_path / 'archive' / datetime.now().strftime('%Y-%m-%d') date_dir.mkdir(parents=True, exist_ok=True) # 2. 复制分析报告 report_path = date_dir / f"{base_name}.md" with open(report_path, 'w', encoding='utf-8') as f: f.write(report_content) result['archive_path'] = str(report_path) # 3. 复制元数据 meta_src = self.base_path / 'content' / 'json' / f"{base_name}.meta.json" if meta_src.exists(): shutil.copy(meta_src, date_dir / f"{base_name}.meta.json") # 4. Git提交 if self._git_add_commit( [datetime.now().strftime('%Y-%m-%d')], f"Archive: {base_name}" ): result['git_commit'] = True # 5. 推送到远程 push_result = self._git_push() result['git_push'] = push_result # 6. 清理临时文件 result['cleanup'] = self.cleanup_project(base_name) result['success'] = True return result # ==================== 链接解析 ==================== def _extract_link(self, file_path: Path) -> Optional[Dict[str, Any]]: """解析链接""" try: content = file_path.read_text(encoding='utf-8') import re url_pattern = r'https?://[^\s<>"]+' urls = re.findall(url_pattern, content) if not urls: return None url = urls[0] result = { "platform": self._detect_platform(url), "type": self._detect_type(url), "id": self._extract_id(url), "url": url, "original_content": content } logger.info(f"Parsed: {result['platform']}/{result['type']} - {result['id']}") return result except Exception as e: logger.error(f"Link extraction failed: {e}") return None def _detect_platform(self, url: str) -> str: """检测平台""" if 'bilibili.com' in url or 'b23.tv' in url: return 'bilibili' elif 'zhihu.com' in url or 'zhuanlan.zhihu.com' in url: return 'zhihu' return 'unknown' def _detect_type(self, url: str) -> str: """检测类型""" if '/video/' in url or '/zvideo/' in url: return 'video' elif '/read/' in url or '/p/' in url: return 'article' elif 'space.bilibili.com' in url: return 'up主' return 'unknown' def _extract_id(self, url: str) -> str: """提取ID""" import re bv = re.search(r'BV[a-zA-Z0-9]{10}', url) if bv: return bv.group() cv = re.search(r'cv(\d+)', url) if cv: return f"cv{cv.group(1)}" num = re.search(r'/(\d+)', url) if num: return num.group(1) return '' # ==================== 主处理流程 ==================== def _save_json(self, data: Dict, subdir: str, base_name: str, suffix: str) -> Path: """保存JSON""" file_path = self.base_path / subdir / f"{base_name}{suffix}" with open(file_path, 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=2) logger.info(f"Saved: {subdir}/{base_name}{suffix}") return file_path def _move_to_stage(self, file_path: Path, to_stage: str) -> Path: """移动文件""" new_path = self.base_path / to_stage / file_path.name if file_path.exists(): file_path.rename(new_path) logger.info(f"Moved: {file_path.name} → {to_stage}/") return new_path def process_item(self, item: Dict) -> Dict[str, Any]: """处理单个项目""" result = { "item": item, "success": False, "stages_completed": [], "files_created": [], "archive": None, "error": None } file_path = Path(item['path']) base_name = file_path.stem try: logger.info(f"Processing: {file_path.name}") # 阶段1: 链接解析 link_info = self._extract_link(file_path) if not link_info: result["error"] = "无法解析链接" return result result["stages_completed"].append("link_parsing") self._save_json(link_info, 'metadata', base_name, '.link.json') # 阶段2: 元数据提取 logger.info(f"Stage 2: Extracting metadata...") metadata = self._extract_metadata(link_info) if 'error' not in metadata: self._save_json(metadata, 'content/json', base_name, '.meta.json') result["stages_completed"].append("metadata_extraction") # 标记为已处理 item_id = metadata.get('bvid') or metadata.get('id') or base_name self.mark_item_processed(item_id, metadata.get('title', '')) # 移动原文件 self._move_to_stage(file_path, 'metadata') result["success"] = len(result["stages_completed"]) >= 2 logger.info(f"Complete: {len(result['stages_completed'])} stages") except Exception as e: result["error"] = str(e) logger.error(f"Processing failed: {e}") return result def _extract_metadata(self, link_info: Dict) -> Dict: """提取元数据""" platform = link_info.get('platform', '') content_type = link_info.get('type', '') content_id = link_info.get('id', '') metadata = { "platform": platform, "type": content_type, "id": content_id, "url": link_info.get('url', ''), "source_file": link_info.get('source_file', ''), "extracted_at": datetime.now().isoformat() } # 调用相应技能的提取逻辑... # (简化处理) return metadata def process_all(self, scope: str = "pending", limit: int = None) -> Dict[str, Any]: """处理所有项目""" summary = {"processed": 0, "failed": 0, "results": []} inbox = self.scan_inbox(scope=scope) for stage, items in inbox.items(): if limit and summary['processed'] >= limit: break for item in items: if limit and summary['processed'] >= limit: break result = self.process_item(item) summary["results"].append(result) if result["success"]: summary["processed"] += 1 else: summary["failed"] += 1 return summary # ==================== 状态和汇报 ==================== def scan_inbox(self, scope: str = "all") -> Dict[str, List[Dict]]: """扫描收件箱""" result = {} stages = self.STAGES if scope == "all" else ([scope] if scope in self.STAGES else self.STAGES) for stage in stages: dir_path = self.base_path / stage items = [] if dir_path.exists(): for item_path in dir_path.iterdir(): if item_path.is_file(): items.append({ 'name': item_path.name, 'path': str(item_path), 'stage': stage, 'modified': datetime.fromtimestamp( item_path.stat().st_mtime ).isoformat() }) result[stage] = items return result def get_status(self) -> Dict[str, Any]: """获取状态""" inbox = self.scan_inbox(scope="all") # 统计文件 file_counts = {} total_size = 0 for root, dirs, files in os.walk(self.base_path): for f in files: file_path = Path(root) / f ext = file_path.suffix file_counts[ext] = file_counts.get(ext, 0) + 1 total_size += file_path.stat().st_size # Git状态 git_info = self._git_status() # 订阅状态 subs = self._load_subscriptions() return { "status": "ready", "timestamp": datetime.now().isoformat(), "stages": inbox, "total_items": sum(len(items) for items in inbox.values()), "total_size_mb": round(total_size / 1024 / 1024, 2), "file_counts": file_counts, "git": git_info, "subscriptions": { "count": len(subs.get('subscriptions', [])), "processed": len(subs.get('processed_items', [])) }, "directories": { 'pending': '待处理内容', 'metadata': '链接解析结果', 'content/json': '元数据', 'content/downloads': '视频文件(临时)', 'content/subtitles': '字幕文件(临时)', 'frames': '截帧(临时)', 'analyzed': '分析报告', 'archive/': 'Git归档' } } def print_status(self): """打印状态""" status = self.get_status() print("\n" + "="*50) print("KB Workflow Status") print("="*50) print(f"\n📁 阶段统计:") for stage, items in status['stages'].items(): print(f" {stage}: {len(items)} 项") print(f"\n💾 存储: {status['total_size_mb']} MB") print(f"\n🔗 Git:") git = status.get('git', {}) if git.get('initialized'): print(f" ✓ 已初始化") print(f" 远程: {git.get('remote_url', '未配置')}") print(f" 提交: {git.get('commits', 0)} 个") else: print(f" ✗ 未初始化") print(f" 状态: {git.get('status', 'unknown')}") print(f"\n📋 订阅:") subs = status.get('subscriptions', {}) print(f" 订阅数: {subs.get('count', 0)}") print(f" 已处理: {subs.get('processed', 0)}") print(f"\n📚 目录说明:") for dir, desc in status.get('directories', {}).items(): print(f" {dir}: {desc}") # 清理建议 if status['total_size_mb'] > 1000: print(f"\n💡 建议: 存储空间较大 ({status['total_size_mb']} MB)") print(f" 运行清理: python kb_workflow.py --trigger cleanup --days 30") print() # ==================== CLI ==================== def main(): parser = argparse.ArgumentParser( description='KB Workflow - Knowledge Base Workflow Orchestration', formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" 使用示例: # 查看状态 python kb_workflow.py --trigger status # 处理待处理内容 python kb_workflow.py --trigger manual --scope pending --limit 5 # 初始化Git python kb_workflow.py --trigger git-init # 清理过期文件 python kb_workflow.py --trigger cleanup --days 30 # 查看Git状态 python kb_workflow.py --trigger git-status 配置环境变量: export KB_GIT_ROOT="/path/to/archive" # Git仓库目录 export KB_GIT_REMOTE="git@github.com:user/repo.git" # 远程仓库 export KB_GIT_USER="Username" # 用户名 export KB_GIT_EMAIL="user@example.com" # 邮箱 """ ) parser.add_argument('--trigger', type=str, default='status', choices=['status', 'manual', 'cleanup', 'git-init', 'git-status', 'archive'], help='触发类型') parser.add_argument('--scope', type=str, default='pending', choices=['all', 'pending', 'metadata', 'content', 'analyzed'], help='处理范围') parser.add_argument('--limit', type=int, default=None, help='处理数量限制') parser.add_argument('--days', type=int, default=30, help='清理天数') parser.add_argument('--base-path', type=str, default=None, help='知识库基础路径') args = parser.parse_args() workflow = KBWorkflow(base_path=args.base_path) if args.trigger == 'status': workflow.print_status() elif args.trigger == 'manual': result = workflow.process_all(scope=args.scope, limit=args.limit) print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) elif args.trigger == 'cleanup': result = workflow.cleanup_expired(days=args.days) print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) elif args.trigger == 'git-init': success = workflow._git_init() print(f"Git init: {'success' if success else 'failed'}") elif args.trigger == 'git-status': status = workflow._git_status() print(json.dumps(status, ensure_ascii=False, indent=2, default=str)) else: print(f"Unknown trigger: {args.trigger}") return 1 return 0 if __name__ == '__main__': sys.exit(main())