feat: Add KB knowledge base series skills

- kb-link-parser: Multi-platform URL parser (Bilibili, YouTube, Zhihu)
- kb-bilibili-metadata: Bilibili video/article/up主 metadata extractor
- kb-zhihu-metadata: Zhihu question/answer/user metadata extractor
- kb-bilibili-content: Bilibili video download and content extraction
- kb-ocr-processor: OCR text extraction from video frames
- kb-deep-analyzer: AI-powered deep content analysis
- kb-workflow: Workflow orchestration (inbox, archive, subscriptions, report)

Also add:
- ARCHITECTURE_FLOWCHART.md: System architecture diagrams
- DETAILED_DESIGN.md: Complete design documentation
This commit is contained in:
OpenClaw Agent
2026-02-10 12:18:52 +08:00
parent 2f5a269866
commit 5507b1d986
28 changed files with 7715 additions and 0 deletions
+674
View File
@@ -0,0 +1,674 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
KB Workflow - Knowledge Base Workflow Orchestration
工作流编排技能,整合以下功能:
- 扫描收件箱目录 (pending, metadata, content, analyzed)
- 调用技能2-4处理内容
- 归档处理完成的条目
- 订阅更新检测
- 生成汇报
Usage:
python kb_workflow.py --trigger manual
python kb_workflow.py --trigger scheduled
python kb_workflow.py --status
python kb_workflow.py --report
python kb_workflow.py --scope pending --limit 5
"""
import os
import sys
import json
import argparse
import logging
from datetime import datetime, timedelta
from pathlib import Path
from typing import Dict, List, Optional, Any
from enum import Enum
# Setup logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger('KBWorkflow')
class TriggerType(Enum):
MANUAL = "manual"
SCHEDULED = "scheduled"
WEBHOOK = "webhook"
class Scope(Enum):
ALL = "all"
PENDING = "pending"
METADATA = "metadata"
CONTENT = "content"
ANALYZED = "analyzed"
ARCHIVE = "archive"
class KBWorkflow:
"""Knowledge Base Workflow Orchestrator"""
# Base directories for the inbox structure
INBOX_DIRS = ['pending', 'metadata', 'content', 'analyzed']
def __init__(self,
base_path: str = None,
archive_path: str = None,
output_format: str = "json"):
"""
Initialize KB Workflow
Args:
base_path: Base path for knowledge base (default: workspace/knowledge-base)
archive_path: Path for archived items (default: archive/)
output_format: Output format (json, text)
"""
self.base_path = Path(base_path) if base_path else Path(__file__).parent / "knowledge-base"
self.archive_path = Path(archive_path) if archive_path else self.base_path / "archive"
self.output_format = output_format
# Ensure directories exist
self._ensure_directories()
# Initialize subsystems
self._init_subsystems()
def _ensure_directories(self):
"""Create necessary directories"""
# Create main inbox directories
for subdir in self.INBOX_DIRS:
dir_path = self.base_path / subdir
if not dir_path.exists():
dir_path.mkdir(parents=True, exist_ok=True)
logger.info(f"Created directory: {dir_path}")
# Create archive directory
if not self.archive_path.exists():
self.archive_path.mkdir(parents=True, exist_ok=True)
logger.info(f"Created archive directory: {self.archive_path}")
def _init_subsystems(self):
"""Initialize subsystem references"""
# Placeholder for skill integrations
self.subscriptions_file = self.base_path / "subscriptions.json"
self.report_file = self.base_path / "reports" / f"report_{datetime.now().strftime('%Y%m%d')}.json"
# ==================== Inbox Scanning ====================
def scan_inbox(self, scope: str = "all", limit: int = None) -> Dict[str, List[Dict]]:
"""
Scan inbox directories for items
Args:
scope: Which directories to scan (all, pending, metadata, content, analyzed)
limit: Maximum number of items to return per directory
Returns:
Dictionary with directory names as keys and item lists as values
"""
result = {}
if scope == "all":
dirs_to_scan = self.INBOX_DIRS
elif scope in self.INBOX_DIRS:
dirs_to_scan = [scope]
else:
logger.warning(f"Unknown scope: {scope}, scanning all")
dirs_to_scan = self.INBOX_DIRS
for dir_name in dirs_to_scan:
dir_path = self.base_path / dir_name
items = self._scan_directory(dir_path)
if limit:
items = items[:limit]
result[dir_name] = items
logger.info(f"Scanned {dir_name}: {len(items)} items")
return result
def _scan_directory(self, path: Path) -> List[Dict]:
"""
Scan a directory for items
Args:
path: Directory path to scan
Returns:
List of item dictionaries
"""
items = []
if not path.exists():
return items
for item_path in path.iterdir():
if item_path.is_file():
item_info = {
'name': item_path.name,
'path': str(item_path),
'extension': item_path.suffix,
'size': item_path.stat().st_size,
'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat(),
'type': self._classify_file_type(item_path)
}
items.append(item_info)
elif item_path.is_dir():
# Treat subdirectories as items too
item_info = {
'name': item_path.name,
'path': str(item_path),
'type': 'directory',
'modified': datetime.fromtimestamp(item_path.stat().st_mtime).isoformat()
}
items.append(item_info)
return items
def _classify_file_type(self, path: Path) -> str:
"""Classify file type by extension"""
suffix = path.suffix.lower()
type_mapping = {
'.md': 'markdown',
'.txt': 'text',
'.json': 'json',
'.yaml': 'yaml',
'.yml': 'yaml',
'.pdf': 'pdf',
'.jpg': 'image',
'.jpeg': 'image',
'.png': 'image',
'.mp4': 'video',
'.mp3': 'audio',
'.zip': 'archive',
'.url': 'link'
}
return type_mapping.get(suffix, 'unknown')
def get_status(self) -> Dict[str, Any]:
"""
Get current workflow status
Returns:
Status dictionary
"""
inbox_status = {}
total_items = 0
for dir_name in self.INBOX_DIRS:
dir_path = self.base_path / dir_name
items = self._scan_directory(dir_path)
inbox_status[dir_name] = {
'count': len(items),
'path': str(dir_path)
}
total_items += len(items)
# Check archive
archive_items = self._scan_directory(self.archive_path)
return {
'status': 'ready',
'timestamp': datetime.now().isoformat(),
'inbox': inbox_status,
'archive': {
'count': len(archive_items),
'path': str(self.archive_path)
},
'total_items': total_items
}
# ==================== Item Processing ====================
def process_item(self, item: Dict, target_stage: str = None) -> Dict[str, Any]:
"""
Process a single item through the workflow
This method calls skills 2-4 to process items:
- Skill 2: Link parsing / URL extraction
- Skill 3: Metadata extraction
- Skill 4: Content analysis
Args:
item: Item dictionary with path and metadata
target_stage: Target stage to process to
Returns:
Processing result dictionary
"""
result = {
'item': item,
'success': False,
'stages_completed': [],
'output_path': None,
'error': None
}
try:
item_path = Path(item['path'])
# Determine processing stage based on current location
current_stage = self._get_stage_from_path(item_path)
# Skill 2: Link/URL Parsing
if current_stage in ['pending', None] or target_stage == 'metadata':
logger.info(f"Stage 1 → 2: Processing link/URL for {item['name']}")
# TODO: Integrate with link-parser skill
result['stages_completed'].append('link_parsing')
# Skill 3: Metadata Extraction
if current_stage in ['metadata', 'pending'] or target_stage == 'content':
logger.info(f"Stage 2 → 3: Extracting metadata for {item['name']}")
# TODO: Integrate with zhihu-metadata skill
result['stages_completed'].append('metadata_extraction')
# Skill 4: Content Analysis
if current_stage in ['content', 'metadata'] or target_stage == 'analyzed':
logger.info(f"Stage 3 → 4: Analyzing content for {item['name']}")
# TODO: Integrate with bilibili-analyzer skill
result['stages_completed'].append('content_analysis')
# Move to next stage if successful
if result['stages_completed']:
self._move_to_next_stage(item_path, result['stages_completed'][-1])
result['success'] = True
except Exception as e:
result['error'] = str(e)
logger.error(f"Processing failed: {e}")
return result
def _get_stage_from_path(self, path: Path) -> Optional[str]:
"""Determine which stage a file is in based on its path"""
for stage in self.INBOX_DIRS:
if stage in str(path):
return stage
return None
def _move_to_next_stage(self, item_path: Path, completed_stage: str) -> None:
"""Move item to the next stage directory"""
stage_order = ['pending', 'metadata', 'content', 'analyzed']
try:
current_idx = stage_order.index(completed_stage)
if current_idx < len(stage_order) - 1:
next_stage = stage_order[current_idx + 1]
next_path = self.base_path / next_stage / item_path.name
# Move file
if item_path.exists():
item_path.rename(next_path)
logger.info(f"Moved {item_path.name} → {next_stage}/")
except ValueError:
logger.warning(f"Unknown stage: {completed_stage}")
def process_all(self, scope: str = "pending", limit: int = None) -> Dict[str, Any]:
"""
Process all items in specified scope
Args:
scope: Which directories to process
limit: Maximum items to process
Returns:
Processing summary
"""
summary = {
'scope': scope,
'processed': 0,
'failed': 0,
'results': []
}
inbox = self.scan_inbox(scope=scope)
for dir_name, items in inbox.items():
if limit and summary['processed'] >= limit:
break
for item in items:
if limit and summary['processed'] >= limit:
break
result = self.process_item(item)
summary['results'].append(result)
if result['success']:
summary['processed'] += 1
else:
summary['failed'] += 1
return summary
# ==================== Archiving ====================
def archive_item(self, item: Dict, analysis: Dict = None) -> Dict[str, Any]:
"""
Archive a processed item to the archive directory
Args:
item: Item to archive
analysis: Optional analysis metadata
Returns:
Archiving result
"""
result = {
'success': False,
'original_path': item['path'],
'archive_path': None,
'error': None
}
try:
item_path = Path(item['path'])
if not item_path.exists():
result['error'] = f"Item not found: {item['path']}"
return result
# Create archive subdirectory by date
date_dir = self.archive_path / datetime.now().strftime('%Y-%m-%d')
if not date_dir.exists():
date_dir.mkdir(parents=True, exist_ok=True)
# Move item to archive
archive_path = date_dir / item_path.name
item_path.rename(archive_path)
# Save analysis metadata if provided
if analysis:
meta_path = date_dir / f"{item_path.stem}_meta.json"
with open(meta_path, 'w', encoding='utf-8') as f:
json.dump({
'archived_at': datetime.now().isoformat(),
'analysis': analysis
}, f, ensure_ascii=False, indent=2)
result['success'] = True
result['archive_path'] = str(archive_path)
logger.info(f"Archived: {item['name']} → {archive_path}")
except Exception as e:
result['error'] = str(e)
logger.error(f"Archive failed: {e}")
return result
# ==================== Subscriptions ====================
def check_subscriptions(self) -> Dict[str, Any]:
"""
Check for updates from subscribed sources
Returns:
Subscription check results
"""
result = {
'checked_at': datetime.now().isoformat(),
'sources': {},
'new_items': 0,
'errors': []
}
# Load subscription list
subscriptions = self._load_subscriptions()
for sub in subscriptions:
try:
source_name = sub.get('name', 'unknown')
source_type = sub.get('type', 'rss')
# Placeholder: Check for new content
logger.info(f"Checking subscription: {source_name}")
# TODO: Implement actual subscription checking
# - RSS feeds
# - YouTube channel updates
# - Bilibili user updates
# - Custom web hooks
result['sources'][source_name] = {
'type': source_type,
'status': 'checked',
'new_items': 0
}
except Exception as e:
result['errors'].append({
'source': sub.get('name', 'unknown'),
'error': str(e)
})
return result
def _load_subscriptions(self) -> List[Dict]:
"""Load subscriptions from file"""
if self.subscriptions_file.exists():
try:
with open(self.subscriptions_file, 'r', encoding='utf-8') as f:
return json.load(f)
except Exception as e:
logger.warning(f"Failed to load subscriptions: {e}")
return []
# ==================== Reporting ====================
def generate_report(self, period: str = "daily") -> Dict[str, Any]:
"""
Generate workflow report
Args:
period: Report period (daily, weekly, monthly)
Returns:
Report dictionary
"""
# Calculate date range
now = datetime.now()
if period == "daily":
start_date = now - timedelta(days=1)
elif period == "weekly":
start_date = now - timedelta(weeks=1)
elif period == "monthly":
start_date = now - timedelta(days=30)
else:
start_date = now - timedelta(days=1)
# Gather statistics
status = self.get_status()
inbox = self.scan_inbox(scope="all")
# Build report
report = {
'generated_at': now.isoformat(),
'period': period,
'start_date': start_date.isoformat(),
'summary': {
'total_items_inbox': status['total_items'],
'total_archived': status['archive']['count'],
'items_by_stage': status['inbox']
},
'processing_stats': {
'pending': len(inbox.get('pending', [])),
'metadata': len(inbox.get('metadata', [])),
'content': len(inbox.get('content', [])),
'analyzed': len(inbox.get('analyzed', []))
},
'recommendations': self._generate_recommendations(status, inbox)
}
# Save report
if self.report_file:
self.report_file.parent.mkdir(parents=True, exist_ok=True)
with open(self.report_file, 'w', encoding='utf-8') as f:
json.dump(report, f, ensure_ascii=False, indent=2)
logger.info(f"Report saved: {self.report_file}")
return report
def _generate_recommendations(self, status: Dict, inbox: Dict) -> List[str]:
"""Generate processing recommendations"""
recommendations = []
pending_count = status['inbox'].get('pending', {}).get('count', 0)
analyzed_count = status['inbox'].get('analyzed', {}).get('count', 0)
if pending_count > 10:
recommendations.append(f"⚠️ 待处理内容较多 ({pending_count} 项),建议优先处理")
if analyzed_count > 20:
recommendations.append(f"📦 已分析内容较多 ({analyzed_count} 项),建议归档")
if status['total_items'] == 0:
recommendations.append("📭 收件箱为空,暂无待处理内容")
return recommendations
# ==================== CLI Interface ====================
def run(self, trigger: str = "manual", scope: str = "all",
limit: int = None, report: bool = False):
"""
Main entry point for running the workflow
Args:
trigger: Trigger type (manual, scheduled)
scope: Processing scope
limit: Maximum items to process
report: Generate report after processing
"""
logger.info(f"KB Workflow started: trigger={trigger}, scope={scope}")
# Handle different triggers
if trigger == "status":
result = self.get_status()
elif trigger == "report":
result = self.generate_report()
elif trigger == "subscriptions":
result = self.check_subscriptions()
else:
# Process items
result = self.process_all(scope=scope, limit=limit)
# Generate report if requested
if report:
report_result = self.generate_report()
result['report'] = report_result
return result
def main():
"""CLI entry point"""
parser = argparse.ArgumentParser(
description='KB Workflow - Knowledge Base Workflow Orchestration',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Get workflow status
python kb_workflow.py --trigger status
# Process pending items manually
python kb_workflow.py --trigger manual --scope pending
# Process up to 5 items
python kb_workflow.py --trigger manual --limit 5
# Generate daily report
python kb_workflow.py --trigger scheduled --report
# Check subscriptions
python kb_workflow.py --trigger scheduled --scope subscriptions
"""
)
parser.add_argument(
'--trigger',
type=str,
default='manual',
choices=['manual', 'scheduled', 'webhook', 'status', 'report', 'subscriptions'],
help='Workflow trigger type (default: manual)'
)
parser.add_argument(
'--scope',
type=str,
default='all',
choices=['all', 'pending', 'metadata', 'content', 'analyzed', 'archive'],
help='Processing scope (default: all)'
)
parser.add_argument(
'--limit',
type=int,
default=None,
help='Maximum items to process'
)
parser.add_argument(
'--report',
action='store_true',
help='Generate report after processing'
)
parser.add_argument(
'--output',
type=str,
choices=['json', 'text'],
default='json',
help='Output format (default: json)'
)
parser.add_argument(
'--base-path',
type=str,
default=None,
help='Base path for knowledge base'
)
parser.add_argument(
'--verbose',
action='store_true',
help='Enable verbose logging'
)
args = parser.parse_args()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
# Initialize workflow
workflow = KBWorkflow(base_path=args.base_path)
# Run workflow
result = workflow.run(
trigger=args.trigger,
scope=args.scope,
limit=args.limit,
report=args.report
)
# Output result
if args.output == 'json':
print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
else:
print(result)
return 0 if result else 1
if __name__ == '__main__':
sys.exit(main())