#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ KB Workflow - Knowledge Base Workflow Orchestration 工作流编排技能,整合以下功能: - 扫描收件箱目录 (pending, metadata, content, analyzed) - 调用技能2-4处理内容 - 归档处理完成的条目 - 订阅更新检测 - 生成汇报 Usage: python kb_workflow.py --trigger manual python kb_workflow.py --trigger scheduled python kb_workflow.py --status python kb_workflow.py --report python kb_workflow.py --scope pending --limit 5 """ import os import sys import json import argparse import logging from datetime import datetime, timedelta from pathlib import Path from typing import Dict, List, Optional, Any from enum import Enum # Setup logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) logger = logging.getLogger('KBWorkflow') class TriggerType(Enum): MANUAL = "manual" SCHEDULED = "scheduled" WEBHOOK = "webhook" class Scope(Enum): ALL = "all" PENDING = "pending" METADATA = "metadata" CONTENT = "content" ANALYZED = "analyzed" ARCHIVE = "archive" class KBWorkflow: """Knowledge Base Workflow Orchestrator""" # Base directories for the inbox structure INBOX_DIRS = ['pending', 'metadata', 'content', 'analyzed'] def __init__(self, base_path: str = None, archive_path: str = None, output_format: str = "json"): """ Initialize KB Workflow Args: base_path: Base path for knowledge base (default: workspace/knowledge-base) archive_path: Path for archived items (default: archive/) output_format: Output format (json, text) """ self.base_path = Path(base_path) if base_path else Path(__file__).parent / "knowledge-base" self.archive_path = Path(archive_path) if archive_path else self.base_path / "archive" self.output_format = output_format # Ensure directories exist self._ensure_directories() # Initialize subsystems self._init_subsystems() def _ensure_directories(self): """Create necessary directories""" # Create main inbox directories for subdir in self.INBOX_DIRS: dir_path = self.base_path / subdir if not dir_path.exists(): dir_path.mkdir(parents=True, exist_ok=True) logger.info(f"Created directory: {dir_path}") # Create archive directory if not self.archive_path.exists(): self.archive_path.mkdir(parents=True, exist_ok=True) logger.info(f"Created archive directory: {self.archive_path}") def _init_subsystems(self): """Initialize subsystem references""" # Placeholder for skill integrations self.subscriptions_file = self.base_path / "subscriptions.json" self.report_file = self.base_path / "reports" / f"report_{datetime.now().strftime('%Y%m%d')}.json" # ==================== Inbox Scanning ==================== def scan_inbox(self, scope: str = "all", limit: int = None) -> Dict[str, List[Dict]]: """ Scan inbox directories for items Args: scope: Which directories to scan (all, pending, metadata, content, analyzed) limit: Maximum number of items to return per directory Returns: Dictionary with directory names as keys and item lists as values """ result = {} if scope == "all": dirs_to_scan = self.INBOX_DIRS elif scope in self.INBOX_DIRS: dirs_to_scan = [scope] else: logger.warning(f"Unknown scope: {scope}, scanning all") dirs_to_scan = self.INBOX_DIRS for dir_name in dirs_to_scan: dir_path = self.base_path / dir_name items = self._scan_directory(dir_path) if limit: items = items[:limit] result[dir_name] = items logger.info(f"Scanned {dir_name}: {len(items)} items") return result def _scan_directory(self, path: Path) -> List[Dict]: """ Scan a directory for items Args: path: Directory path to scan Returns: List of item dictionaries """ items = [] if not path.exists(): return items for item_path in path.iterdir(): if item_path.is_file(): item_info = { 'name': item_path.name, 'path': str(item_path), 'extension': item_path.suffix, 'size': item_path.stat().st_size, 'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat(), 'type': self._classify_file_type(item_path) } items.append(item_info) elif item_path.is_dir(): # Treat subdirectories as items too item_info = { 'name': item_path.name, 'path': str(item_path), 'type': 'directory', 'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat() } items.append(item_info) return items def _classify_file_type(self, path: Path) -> str: """Classify file type by extension""" suffix = path.suffix.lower() type_mapping = { '.md': 'markdown', '.txt': 'text', '.json': 'json', '.yaml': 'yaml', '.yml': 'yaml', '.pdf': 'pdf', '.jpg': 'image', '.jpeg': 'image', '.png': 'image', '.mp4': 'video', '.mp3': 'audio', '.zip': 'archive', '.url': 'link' } return type_mapping.get(suffix, 'unknown') def get_status(self) -> Dict[str, Any]: """ Get current workflow status Returns: Status dictionary """ inbox_status = {} total_items = 0 for dir_name in self.INBOX_DIRS: dir_path = self.base_path / dir_name items = self._scan_directory(dir_path) inbox_status[dir_name] = { 'count': len(items), 'path': str(dir_path) } total_items += len(items) # Check archive archive_items = self._scan_directory(self.archive_path) return { 'status': 'ready', 'timestamp': datetime.now().isoformat(), 'inbox': inbox_status, 'archive': { 'count': len(archive_items), 'path': str(self.archive_path) }, 'total_items': total_items } # ==================== Item Processing ==================== def process_item(self, item: Dict, target_stage: str = None) -> Dict[str, Any]: """ Process a single item through the workflow This method calls skills 2-4 to process items: - Skill 2: Link parsing / URL extraction - Skill 3: Metadata extraction - Skill 4: Content analysis Args: item: Item dictionary with path and metadata target_stage: Target stage to process to Returns: Processing result dictionary """ result = { 'item': item, 'success': False, 'stages_completed': [], 'output_path': None, 'error': None } try: item_path = Path(item['path']) # Determine processing stage based on current location current_stage = self._get_stage_from_path(item_path) # Skill 2: Link/URL Parsing if current_stage in ['pending', None] or target_stage == 'metadata': logger.info(f"Stage 1 → 2: Processing link/URL for {item['name']}") # TODO: Integrate with link-parser skill result['stages_completed'].append('link_parsing') # Skill 3: Metadata Extraction if current_stage in ['metadata', 'pending'] or target_stage == 'content': logger.info(f"Stage 2 → 3: Extracting metadata for {item['name']}") # TODO: Integrate with zhihu-metadata skill result['stages_completed'].append('metadata_extraction') # Skill 4: Content Analysis if current_stage in ['content', 'metadata'] or target_stage == 'analyzed': logger.info(f"Stage 3 → 4: Analyzing content for {item['name']}") # TODO: Integrate with bilibili-analyzer skill result['stages_completed'].append('content_analysis') # Move to next stage if successful if result['stages_completed']: self._move_to_next_stage(item_path, result['stages_completed'][-1]) result['success'] = True except Exception as e: result['error'] = str(e) logger.error(f"Processing failed: {e}") return result def _get_stage_from_path(self, path: Path) -> Optional[str]: """Determine which stage a file is in based on its path""" for stage in self.INBOX_DIRS: if stage in str(path): return stage return None def _move_to_next_stage(self, item_path: Path, completed_stage: str) -> None: """Move item to the next stage directory""" stage_order = ['pending', 'metadata', 'content', 'analyzed'] try: current_idx = stage_order.index(completed_stage) if current_idx < len(stage_order) - 1: next_stage = stage_order[current_idx + 1] next_path = self.base_path / next_stage / item_path.name # Move file if item_path.exists(): item_path.rename(next_path) logger.info(f"Moved {item_path.name} → {next_stage}/") except ValueError: logger.warning(f"Unknown stage: {completed_stage}") def process_all(self, scope: str = "pending", limit: int = None) -> Dict[str, Any]: """ Process all items in specified scope Args: scope: Which directories to process limit: Maximum items to process Returns: Processing summary """ summary = { 'scope': scope, 'processed': 0, 'failed': 0, 'results': [] } inbox = self.scan_inbox(scope=scope) for dir_name, items in inbox.items(): if limit and summary['processed'] >= limit: break for item in items: if limit and summary['processed'] >= limit: break result = self.process_item(item) summary['results'].append(result) if result['success']: summary['processed'] += 1 else: summary['failed'] += 1 return summary # ==================== Archiving ==================== def archive_item(self, item: Dict, analysis: Dict = None) -> Dict[str, Any]: """ Archive a processed item to the archive directory Args: item: Item to archive analysis: Optional analysis metadata Returns: Archiving result """ result = { 'success': False, 'original_path': item['path'], 'archive_path': None, 'error': None } try: item_path = Path(item['path']) if not item_path.exists(): result['error'] = f"Item not found: {item['path']}" return result # Create archive subdirectory by date date_dir = self.archive_path / datetime.now().strftime('%Y-%m-%d') if not date_dir.exists(): date_dir.mkdir(parents=True, exist_ok=True) # Move item to archive archive_path = date_dir / item_path.name item_path.rename(archive_path) # Save analysis metadata if provided if analysis: meta_path = date_dir / f"{item_path.stem}_meta.json" with open(meta_path, 'w', encoding='utf-8') as f: json.dump({ 'archived_at': datetime.now().isoformat(), 'analysis': analysis }, f, ensure_ascii=False, indent=2) result['success'] = True result['archive_path'] = str(archive_path) logger.info(f"Archived: {item['name']} → {archive_path}") except Exception as e: result['error'] = str(e) logger.error(f"Archive failed: {e}") return result # ==================== Subscriptions ==================== def check_subscriptions(self) -> Dict[str, Any]: """ Check for updates from subscribed sources Returns: Subscription check results """ result = { 'checked_at': datetime.now().isoformat(), 'sources': {}, 'new_items': 0, 'errors': [] } # Load subscription list subscriptions = self._load_subscriptions() for sub in subscriptions: try: source_name = sub.get('name', 'unknown') source_type = sub.get('type', 'rss') # Placeholder: Check for new content logger.info(f"Checking subscription: {source_name}") # TODO: Implement actual subscription checking # - RSS feeds # - YouTube channel updates # - Bilibili user updates # - Custom web hooks result['sources'][source_name] = { 'type': source_type, 'status': 'checked', 'new_items': 0 } except Exception as e: result['errors'].append({ 'source': sub.get('name', 'unknown'), 'error': str(e) }) return result def _load_subscriptions(self) -> List[Dict]: """Load subscriptions from file""" if self.subscriptions_file.exists(): try: with open(self.subscriptions_file, 'r', encoding='utf-8') as f: return json.load(f) except Exception as e: logger.warning(f"Failed to load subscriptions: {e}") return [] # ==================== Reporting ==================== def generate_report(self, period: str = "daily") -> Dict[str, Any]: """ Generate workflow report Args: period: Report period (daily, weekly, monthly) Returns: Report dictionary """ # Calculate date range now = datetime.now() if period == "daily": start_date = now - timedelta(days=1) elif period == "weekly": start_date = now - timedelta(weeks=1) elif period == "monthly": start_date = now - timedelta(days=30) else: start_date = now - timedelta(days=1) # Gather statistics status = self.get_status() inbox = self.scan_inbox(scope="all") # Build report report = { 'generated_at': now.isoformat(), 'period': period, 'start_date': start_date.isoformat(), 'summary': { 'total_items_inbox': status['total_items'], 'total_archived': status['archive']['count'], 'items_by_stage': status['inbox'] }, 'processing_stats': { 'pending': len(inbox.get('pending', [])), 'metadata': len(inbox.get('metadata', [])), 'content': len(inbox.get('content', [])), 'analyzed': len(inbox.get('analyzed', [])) }, 'recommendations': self._generate_recommendations(status, inbox) } # Save report if self.report_file: self.report_file.parent.mkdir(parents=True, exist_ok=True) with open(self.report_file, 'w', encoding='utf-8') as f: json.dump(report, f, ensure_ascii=False, indent=2) logger.info(f"Report saved: {self.report_file}") return report def _generate_recommendations(self, status: Dict, inbox: Dict) -> List[str]: """Generate processing recommendations""" recommendations = [] pending_count = status['inbox'].get('pending', {}).get('count', 0) analyzed_count = status['inbox'].get('analyzed', {}).get('count', 0) if pending_count > 10: recommendations.append(f"⚠️ 待处理内容较多 ({pending_count} 项),建议优先处理") if analyzed_count > 20: recommendations.append(f"📦 已分析内容较多 ({analyzed_count} 项),建议归档") if status['total_items'] == 0: recommendations.append("📭 收件箱为空,暂无待处理内容") return recommendations # ==================== CLI Interface ==================== def run(self, trigger: str = "manual", scope: str = "all", limit: int = None, report: bool = False): """ Main entry point for running the workflow Args: trigger: Trigger type (manual, scheduled) scope: Processing scope limit: Maximum items to process report: Generate report after processing """ logger.info(f"KB Workflow started: trigger={trigger}, scope={scope}") # Handle different triggers if trigger == "status": result = self.get_status() elif trigger == "report": result = self.generate_report() elif trigger == "subscriptions": result = self.check_subscriptions() else: # Process items result = self.process_all(scope=scope, limit=limit) # Generate report if requested if report: report_result = self.generate_report() result['report'] = report_result return result def main(): """CLI entry point""" parser = argparse.ArgumentParser( description='KB Workflow - Knowledge Base Workflow Orchestration', formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: # Get workflow status python kb_workflow.py --trigger status # Process pending items manually python kb_workflow.py --trigger manual --scope pending # Process up to 5 items python kb_workflow.py --trigger manual --limit 5 # Generate daily report python kb_workflow.py --trigger scheduled --report # Check subscriptions python kb_workflow.py --trigger scheduled --scope subscriptions """ ) parser.add_argument( '--trigger', type=str, default='manual', choices=['manual', 'scheduled', 'webhook', 'status', 'report', 'subscriptions'], help='Workflow trigger type (default: manual)' ) parser.add_argument( '--scope', type=str, default='all', choices=['all', 'pending', 'metadata', 'content', 'analyzed', 'archive'], help='Processing scope (default: all)' ) parser.add_argument( '--limit', type=int, default=None, help='Maximum items to process' ) parser.add_argument( '--report', action='store_true', help='Generate report after processing' ) parser.add_argument( '--output', type=str, choices=['json', 'text'], default='json', help='Output format (default: json)' ) parser.add_argument( '--base-path', type=str, default=None, help='Base path for knowledge base' ) parser.add_argument( '--verbose', action='store_true', help='Enable verbose logging' ) args = parser.parse_args() if args.verbose: logging.getLogger().setLevel(logging.DEBUG) # Initialize workflow workflow = KBWorkflow(base_path=args.base_path) # Run workflow result = workflow.run( trigger=args.trigger, scope=args.scope, limit=args.limit, report=args.report ) # Output result if args.output == 'json': print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) else: print(result) return 0 if result else 1 if __name__ == '__main__': sys.exit(main())