OpenClaw Agent
5507b1d986
- kb-link-parser: Multi-platform URL parser (Bilibili, YouTube, Zhihu) - kb-bilibili-metadata: Bilibili video/article/up主 metadata extractor - kb-zhihu-metadata: Zhihu question/answer/user metadata extractor - kb-bilibili-content: Bilibili video download and content extraction - kb-ocr-processor: OCR text extraction from video frames - kb-deep-analyzer: AI-powered deep content analysis - kb-workflow: Workflow orchestration (inbox, archive, subscriptions, report) Also add: - ARCHITECTURE_FLOWCHART.md: System architecture diagrams - DETAILED_DESIGN.md: Complete design documentation
675 lines
21 KiB
Python
675 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
KB Workflow - Knowledge Base Workflow Orchestration
|
|
|
|
工作流编排技能，整合以下功能：
|
|
- 扫描收件箱目录 (pending, metadata, content, analyzed)
|
|
- 调用技能2-4处理内容
|
|
- 归档处理完成的条目
|
|
- 订阅更新检测
|
|
- 生成汇报
|
|
|
|
Usage:
|
|
python kb_workflow.py --trigger manual
|
|
python kb_workflow.py --trigger scheduled
|
|
python kb_workflow.py --status
|
|
python kb_workflow.py --report
|
|
python kb_workflow.py --scope pending --limit 5
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import argparse
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Any
|
|
from enum import Enum
|
|
|
|
# Setup logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
)
|
|
logger = logging.getLogger('KBWorkflow')
|
|
|
|
|
|
class TriggerType(Enum):
|
|
MANUAL = "manual"
|
|
SCHEDULED = "scheduled"
|
|
WEBHOOK = "webhook"
|
|
|
|
|
|
class Scope(Enum):
|
|
ALL = "all"
|
|
PENDING = "pending"
|
|
METADATA = "metadata"
|
|
CONTENT = "content"
|
|
ANALYZED = "analyzed"
|
|
ARCHIVE = "archive"
|
|
|
|
|
|
class KBWorkflow:
|
|
"""Knowledge Base Workflow Orchestrator"""
|
|
|
|
# Base directories for the inbox structure
|
|
INBOX_DIRS = ['pending', 'metadata', 'content', 'analyzed']
|
|
|
|
def __init__(self,
|
|
base_path: str = None,
|
|
archive_path: str = None,
|
|
output_format: str = "json"):
|
|
"""
|
|
Initialize KB Workflow
|
|
|
|
Args:
|
|
base_path: Base path for knowledge base (default: workspace/knowledge-base)
|
|
archive_path: Path for archived items (default: archive/)
|
|
output_format: Output format (json, text)
|
|
"""
|
|
self.base_path = Path(base_path) if base_path else Path(__file__).parent / "knowledge-base"
|
|
self.archive_path = Path(archive_path) if archive_path else self.base_path / "archive"
|
|
self.output_format = output_format
|
|
|
|
# Ensure directories exist
|
|
self._ensure_directories()
|
|
|
|
# Initialize subsystems
|
|
self._init_subsystems()
|
|
|
|
def _ensure_directories(self):
|
|
"""Create necessary directories"""
|
|
# Create main inbox directories
|
|
for subdir in self.INBOX_DIRS:
|
|
dir_path = self.base_path / subdir
|
|
if not dir_path.exists():
|
|
dir_path.mkdir(parents=True, exist_ok=True)
|
|
logger.info(f"Created directory: {dir_path}")
|
|
|
|
# Create archive directory
|
|
if not self.archive_path.exists():
|
|
self.archive_path.mkdir(parents=True, exist_ok=True)
|
|
logger.info(f"Created archive directory: {self.archive_path}")
|
|
|
|
def _init_subsystems(self):
|
|
"""Initialize subsystem references"""
|
|
# Placeholder for skill integrations
|
|
self.subscriptions_file = self.base_path / "subscriptions.json"
|
|
self.report_file = self.base_path / "reports" / f"report_{datetime.now().strftime('%Y%m%d')}.json"
|
|
|
|
# ==================== Inbox Scanning ====================
|
|
|
|
def scan_inbox(self, scope: str = "all", limit: int = None) -> Dict[str, List[Dict]]:
|
|
"""
|
|
Scan inbox directories for items
|
|
|
|
Args:
|
|
scope: Which directories to scan (all, pending, metadata, content, analyzed)
|
|
limit: Maximum number of items to return per directory
|
|
|
|
Returns:
|
|
Dictionary with directory names as keys and item lists as values
|
|
"""
|
|
result = {}
|
|
|
|
if scope == "all":
|
|
dirs_to_scan = self.INBOX_DIRS
|
|
elif scope in self.INBOX_DIRS:
|
|
dirs_to_scan = [scope]
|
|
else:
|
|
logger.warning(f"Unknown scope: {scope}, scanning all")
|
|
dirs_to_scan = self.INBOX_DIRS
|
|
|
|
for dir_name in dirs_to_scan:
|
|
dir_path = self.base_path / dir_name
|
|
items = self._scan_directory(dir_path)
|
|
|
|
if limit:
|
|
items = items[:limit]
|
|
|
|
result[dir_name] = items
|
|
logger.info(f"Scanned {dir_name}: {len(items)} items")
|
|
|
|
return result
|
|
|
|
def _scan_directory(self, path: Path) -> List[Dict]:
|
|
"""
|
|
Scan a directory for items
|
|
|
|
Args:
|
|
path: Directory path to scan
|
|
|
|
Returns:
|
|
List of item dictionaries
|
|
"""
|
|
items = []
|
|
|
|
if not path.exists():
|
|
return items
|
|
|
|
for item_path in path.iterdir():
|
|
if item_path.is_file():
|
|
item_info = {
|
|
'name': item_path.name,
|
|
'path': str(item_path),
|
|
'extension': item_path.suffix,
|
|
'size': item_path.stat().st_size,
|
|
'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat(),
|
|
'type': self._classify_file_type(item_path)
|
|
}
|
|
items.append(item_info)
|
|
elif item_path.is_dir():
|
|
# Treat subdirectories as items too
|
|
item_info = {
|
|
'name': item_path.name,
|
|
'path': str(item_path),
|
|
'type': 'directory',
|
|
'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat()
|
|
}
|
|
items.append(item_info)
|
|
|
|
return items
|
|
|
|
def _classify_file_type(self, path: Path) -> str:
|
|
"""Classify file type by extension"""
|
|
suffix = path.suffix.lower()
|
|
|
|
type_mapping = {
|
|
'.md': 'markdown',
|
|
'.txt': 'text',
|
|
'.json': 'json',
|
|
'.yaml': 'yaml',
|
|
'.yml': 'yaml',
|
|
'.pdf': 'pdf',
|
|
'.jpg': 'image',
|
|
'.jpeg': 'image',
|
|
'.png': 'image',
|
|
'.mp4': 'video',
|
|
'.mp3': 'audio',
|
|
'.zip': 'archive',
|
|
'.url': 'link'
|
|
}
|
|
|
|
return type_mapping.get(suffix, 'unknown')
|
|
|
|
def get_status(self) -> Dict[str, Any]:
|
|
"""
|
|
Get current workflow status
|
|
|
|
Returns:
|
|
Status dictionary
|
|
"""
|
|
inbox_status = {}
|
|
total_items = 0
|
|
|
|
for dir_name in self.INBOX_DIRS:
|
|
dir_path = self.base_path / dir_name
|
|
items = self._scan_directory(dir_path)
|
|
inbox_status[dir_name] = {
|
|
'count': len(items),
|
|
'path': str(dir_path)
|
|
}
|
|
total_items += len(items)
|
|
|
|
# Check archive
|
|
archive_items = self._scan_directory(self.archive_path)
|
|
|
|
return {
|
|
'status': 'ready',
|
|
'timestamp': datetime.now().isoformat(),
|
|
'inbox': inbox_status,
|
|
'archive': {
|
|
'count': len(archive_items),
|
|
'path': str(self.archive_path)
|
|
},
|
|
'total_items': total_items
|
|
}
|
|
|
|
# ==================== Item Processing ====================
|
|
|
|
def process_item(self, item: Dict, target_stage: str = None) -> Dict[str, Any]:
|
|
"""
|
|
Process a single item through the workflow
|
|
|
|
This method calls skills 2-4 to process items:
|
|
- Skill 2: Link parsing / URL extraction
|
|
- Skill 3: Metadata extraction
|
|
- Skill 4: Content analysis
|
|
|
|
Args:
|
|
item: Item dictionary with path and metadata
|
|
target_stage: Target stage to process to
|
|
|
|
Returns:
|
|
Processing result dictionary
|
|
"""
|
|
result = {
|
|
'item': item,
|
|
'success': False,
|
|
'stages_completed': [],
|
|
'output_path': None,
|
|
'error': None
|
|
}
|
|
|
|
try:
|
|
item_path = Path(item['path'])
|
|
|
|
# Determine processing stage based on current location
|
|
current_stage = self._get_stage_from_path(item_path)
|
|
|
|
# Skill 2: Link/URL Parsing
|
|
if current_stage in ['pending', None] or target_stage == 'metadata':
|
|
logger.info(f"Stage 1 → 2: Processing link/URL for {item['name']}")
|
|
# TODO: Integrate with link-parser skill
|
|
result['stages_completed'].append('link_parsing')
|
|
|
|
# Skill 3: Metadata Extraction
|
|
if current_stage in ['metadata', 'pending'] or target_stage == 'content':
|
|
logger.info(f"Stage 2 → 3: Extracting metadata for {item['name']}")
|
|
# TODO: Integrate with zhihu-metadata skill
|
|
result['stages_completed'].append('metadata_extraction')
|
|
|
|
# Skill 4: Content Analysis
|
|
if current_stage in ['content', 'metadata'] or target_stage == 'analyzed':
|
|
logger.info(f"Stage 3 → 4: Analyzing content for {item['name']}")
|
|
# TODO: Integrate with bilibili-analyzer skill
|
|
result['stages_completed'].append('content_analysis')
|
|
|
|
# Move to next stage if successful
|
|
if result['stages_completed']:
|
|
self._move_to_next_stage(item_path, result['stages_completed'][-1])
|
|
result['success'] = True
|
|
|
|
except Exception as e:
|
|
result['error'] = str(e)
|
|
logger.error(f"Processing failed: {e}")
|
|
|
|
return result
|
|
|
|
def _get_stage_from_path(self, path: Path) -> Optional[str]:
|
|
"""Determine which stage a file is in based on its path"""
|
|
for stage in self.INBOX_DIRS:
|
|
if stage in str(path):
|
|
return stage
|
|
return None
|
|
|
|
def _move_to_next_stage(self, item_path: Path, completed_stage: str) -> None:
|
|
"""Move item to the next stage directory"""
|
|
stage_order = ['pending', 'metadata', 'content', 'analyzed']
|
|
|
|
try:
|
|
current_idx = stage_order.index(completed_stage)
|
|
if current_idx < len(stage_order) - 1:
|
|
next_stage = stage_order[current_idx + 1]
|
|
next_path = self.base_path / next_stage / item_path.name
|
|
|
|
# Move file
|
|
if item_path.exists():
|
|
item_path.rename(next_path)
|
|
logger.info(f"Moved {item_path.name} → {next_stage}/")
|
|
except ValueError:
|
|
logger.warning(f"Unknown stage: {completed_stage}")
|
|
|
|
def process_all(self, scope: str = "pending", limit: int = None) -> Dict[str, Any]:
|
|
"""
|
|
Process all items in specified scope
|
|
|
|
Args:
|
|
scope: Which directories to process
|
|
limit: Maximum items to process
|
|
|
|
Returns:
|
|
Processing summary
|
|
"""
|
|
summary = {
|
|
'scope': scope,
|
|
'processed': 0,
|
|
'failed': 0,
|
|
'results': []
|
|
}
|
|
|
|
inbox = self.scan_inbox(scope=scope)
|
|
|
|
for dir_name, items in inbox.items():
|
|
if limit and summary['processed'] >= limit:
|
|
break
|
|
|
|
for item in items:
|
|
if limit and summary['processed'] >= limit:
|
|
break
|
|
|
|
result = self.process_item(item)
|
|
summary['results'].append(result)
|
|
|
|
if result['success']:
|
|
summary['processed'] += 1
|
|
else:
|
|
summary['failed'] += 1
|
|
|
|
return summary
|
|
|
|
# ==================== Archiving ====================
|
|
|
|
def archive_item(self, item: Dict, analysis: Dict = None) -> Dict[str, Any]:
|
|
"""
|
|
Archive a processed item to the archive directory
|
|
|
|
Args:
|
|
item: Item to archive
|
|
analysis: Optional analysis metadata
|
|
|
|
Returns:
|
|
Archiving result
|
|
"""
|
|
result = {
|
|
'success': False,
|
|
'original_path': item['path'],
|
|
'archive_path': None,
|
|
'error': None
|
|
}
|
|
|
|
try:
|
|
item_path = Path(item['path'])
|
|
|
|
if not item_path.exists():
|
|
result['error'] = f"Item not found: {item['path']}"
|
|
return result
|
|
|
|
# Create archive subdirectory by date
|
|
date_dir = self.archive_path / datetime.now().strftime('%Y-%m-%d')
|
|
if not date_dir.exists():
|
|
date_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Move item to archive
|
|
archive_path = date_dir / item_path.name
|
|
item_path.rename(archive_path)
|
|
|
|
# Save analysis metadata if provided
|
|
if analysis:
|
|
meta_path = date_dir / f"{item_path.stem}_meta.json"
|
|
with open(meta_path, 'w', encoding='utf-8') as f:
|
|
json.dump({
|
|
'archived_at': datetime.now().isoformat(),
|
|
'analysis': analysis
|
|
}, f, ensure_ascii=False, indent=2)
|
|
|
|
result['success'] = True
|
|
result['archive_path'] = str(archive_path)
|
|
logger.info(f"Archived: {item['name']} → {archive_path}")
|
|
|
|
except Exception as e:
|
|
result['error'] = str(e)
|
|
logger.error(f"Archive failed: {e}")
|
|
|
|
return result
|
|
|
|
# ==================== Subscriptions ====================
|
|
|
|
def check_subscriptions(self) -> Dict[str, Any]:
|
|
"""
|
|
Check for updates from subscribed sources
|
|
|
|
Returns:
|
|
Subscription check results
|
|
"""
|
|
result = {
|
|
'checked_at': datetime.now().isoformat(),
|
|
'sources': {},
|
|
'new_items': 0,
|
|
'errors': []
|
|
}
|
|
|
|
# Load subscription list
|
|
subscriptions = self._load_subscriptions()
|
|
|
|
for sub in subscriptions:
|
|
try:
|
|
source_name = sub.get('name', 'unknown')
|
|
source_type = sub.get('type', 'rss')
|
|
|
|
# Placeholder: Check for new content
|
|
logger.info(f"Checking subscription: {source_name}")
|
|
|
|
# TODO: Implement actual subscription checking
|
|
# - RSS feeds
|
|
# - YouTube channel updates
|
|
# - Bilibili user updates
|
|
# - Custom web hooks
|
|
|
|
result['sources'][source_name] = {
|
|
'type': source_type,
|
|
'status': 'checked',
|
|
'new_items': 0
|
|
}
|
|
|
|
except Exception as e:
|
|
result['errors'].append({
|
|
'source': sub.get('name', 'unknown'),
|
|
'error': str(e)
|
|
})
|
|
|
|
return result
|
|
|
|
def _load_subscriptions(self) -> List[Dict]:
|
|
"""Load subscriptions from file"""
|
|
if self.subscriptions_file.exists():
|
|
try:
|
|
with open(self.subscriptions_file, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to load subscriptions: {e}")
|
|
|
|
return []
|
|
|
|
# ==================== Reporting ====================
|
|
|
|
def generate_report(self, period: str = "daily") -> Dict[str, Any]:
|
|
"""
|
|
Generate workflow report
|
|
|
|
Args:
|
|
period: Report period (daily, weekly, monthly)
|
|
|
|
Returns:
|
|
Report dictionary
|
|
"""
|
|
# Calculate date range
|
|
now = datetime.now()
|
|
|
|
if period == "daily":
|
|
start_date = now - timedelta(days=1)
|
|
elif period == "weekly":
|
|
start_date = now - timedelta(weeks=1)
|
|
elif period == "monthly":
|
|
start_date = now - timedelta(days=30)
|
|
else:
|
|
start_date = now - timedelta(days=1)
|
|
|
|
# Gather statistics
|
|
status = self.get_status()
|
|
inbox = self.scan_inbox(scope="all")
|
|
|
|
# Build report
|
|
report = {
|
|
'generated_at': now.isoformat(),
|
|
'period': period,
|
|
'start_date': start_date.isoformat(),
|
|
'summary': {
|
|
'total_items_inbox': status['total_items'],
|
|
'total_archived': status['archive']['count'],
|
|
'items_by_stage': status['inbox']
|
|
},
|
|
'processing_stats': {
|
|
'pending': len(inbox.get('pending', [])),
|
|
'metadata': len(inbox.get('metadata', [])),
|
|
'content': len(inbox.get('content', [])),
|
|
'analyzed': len(inbox.get('analyzed', []))
|
|
},
|
|
'recommendations': self._generate_recommendations(status, inbox)
|
|
}
|
|
|
|
# Save report
|
|
if self.report_file:
|
|
self.report_file.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(self.report_file, 'w', encoding='utf-8') as f:
|
|
json.dump(report, f, ensure_ascii=False, indent=2)
|
|
logger.info(f"Report saved: {self.report_file}")
|
|
|
|
return report
|
|
|
|
def _generate_recommendations(self, status: Dict, inbox: Dict) -> List[str]:
|
|
"""Generate processing recommendations"""
|
|
recommendations = []
|
|
|
|
pending_count = status['inbox'].get('pending', {}).get('count', 0)
|
|
analyzed_count = status['inbox'].get('analyzed', {}).get('count', 0)
|
|
|
|
if pending_count > 10:
|
|
recommendations.append(f"⚠️ 待处理内容较多 ({pending_count} 项)，建议优先处理")
|
|
|
|
if analyzed_count > 20:
|
|
recommendations.append(f"📦 已分析内容较多 ({analyzed_count} 项)，建议归档")
|
|
|
|
if status['total_items'] == 0:
|
|
recommendations.append("📭 收件箱为空，暂无待处理内容")
|
|
|
|
return recommendations
|
|
|
|
# ==================== CLI Interface ====================
|
|
|
|
def run(self, trigger: str = "manual", scope: str = "all",
|
|
limit: int = None, report: bool = False):
|
|
"""
|
|
Main entry point for running the workflow
|
|
|
|
Args:
|
|
trigger: Trigger type (manual, scheduled)
|
|
scope: Processing scope
|
|
limit: Maximum items to process
|
|
report: Generate report after processing
|
|
"""
|
|
logger.info(f"KB Workflow started: trigger={trigger}, scope={scope}")
|
|
|
|
# Handle different triggers
|
|
if trigger == "status":
|
|
result = self.get_status()
|
|
elif trigger == "report":
|
|
result = self.generate_report()
|
|
elif trigger == "subscriptions":
|
|
result = self.check_subscriptions()
|
|
else:
|
|
# Process items
|
|
result = self.process_all(scope=scope, limit=limit)
|
|
|
|
# Generate report if requested
|
|
if report:
|
|
report_result = self.generate_report()
|
|
result['report'] = report_result
|
|
|
|
return result
|
|
|
|
|
|
def main():
|
|
"""CLI entry point"""
|
|
parser = argparse.ArgumentParser(
|
|
description='KB Workflow - Knowledge Base Workflow Orchestration',
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
# Get workflow status
|
|
python kb_workflow.py --trigger status
|
|
|
|
# Process pending items manually
|
|
python kb_workflow.py --trigger manual --scope pending
|
|
|
|
# Process up to 5 items
|
|
python kb_workflow.py --trigger manual --limit 5
|
|
|
|
# Generate daily report
|
|
python kb_workflow.py --trigger scheduled --report
|
|
|
|
# Check subscriptions
|
|
python kb_workflow.py --trigger scheduled --scope subscriptions
|
|
"""
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--trigger',
|
|
type=str,
|
|
default='manual',
|
|
choices=['manual', 'scheduled', 'webhook', 'status', 'report', 'subscriptions'],
|
|
help='Workflow trigger type (default: manual)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--scope',
|
|
type=str,
|
|
default='all',
|
|
choices=['all', 'pending', 'metadata', 'content', 'analyzed', 'archive'],
|
|
help='Processing scope (default: all)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--limit',
|
|
type=int,
|
|
default=None,
|
|
help='Maximum items to process'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--report',
|
|
action='store_true',
|
|
help='Generate report after processing'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--output',
|
|
type=str,
|
|
choices=['json', 'text'],
|
|
default='json',
|
|
help='Output format (default: json)'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--base-path',
|
|
type=str,
|
|
default=None,
|
|
help='Base path for knowledge base'
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--verbose',
|
|
action='store_true',
|
|
help='Enable verbose logging'
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.verbose:
|
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
|
|
# Initialize workflow
|
|
workflow = KBWorkflow(base_path=args.base_path)
|
|
|
|
# Run workflow
|
|
result = workflow.run(
|
|
trigger=args.trigger,
|
|
scope=args.scope,
|
|
limit=args.limit,
|
|
report=args.report
|
|
)
|
|
|
|
# Output result
|
|
if args.output == 'json':
|
|
print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
|
|
else:
|
|
print(result)
|
|
|
|
return 0 if result else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|