feat: implement obsidian-kb skill with long video chunking
- Multi-source content aggregation (Bilibili/YouTube/Zhihu) - Smart inbox management with AI quality scoring - Long video chunking analysis (30min+ auto-segmentation) - Topic-based organization - Git auto-sync to Obsidian - Template system with Jinja2 and fallback rendering - CLI commands: subscribe, inbox, process, skip, status, search - Comprehensive test suite with unit and integration tests - Documentation: README.md and SKILL.md Checkpoint 7-8 completed by Agent E
This commit is contained in:
OpenClaw Agent
@@ -0,0 +1,569 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Obsidian KB Knowledge Base CLI Tool
|
||||
|
||||
Command line interface for managing Obsidian knowledge base content.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
try:
|
||||
from src.config import Config
|
||||
from src.storage import Storage
|
||||
from src.pipeline import ProcessingPipeline
|
||||
from src.models import ContentStatus
|
||||
CONFIG_AVAILABLE = True
|
||||
except ImportError as e:
|
||||
print(f"Import warning: {e}", file=sys.stderr)
|
||||
CONFIG_AVAILABLE = False
|
||||
|
||||
|
||||
def print_header(text: str):
|
||||
"""Print formatted header."""
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f" {text}")
|
||||
print(f"{'=' * 50}\n")
|
||||
|
||||
|
||||
def print_table(headers: list, rows: list, max_width: int = 80):
|
||||
"""Print a simple formatted table."""
|
||||
if not rows:
|
||||
print("(No data)")
|
||||
return
|
||||
|
||||
# Calculate column widths
|
||||
col_widths = [len(h) for h in headers]
|
||||
for row in rows:
|
||||
for i, cell in enumerate(row):
|
||||
if i < len(col_widths):
|
||||
col_widths[i] = max(col_widths[i], len(str(cell)[:max_width]))
|
||||
|
||||
# Print headers
|
||||
header_line = " | ".join(h.ljust(w) for h, w in zip(headers, col_widths))
|
||||
print(header_line)
|
||||
print("-" * len(header_line))
|
||||
|
||||
# Print rows
|
||||
for row in rows:
|
||||
row_str = " | ".join(str(cell)[:max_width].ljust(w) for cell, w in zip(row, col_widths))
|
||||
print(row_str)
|
||||
print()
|
||||
|
||||
|
||||
def cmd_subscribe(args: argparse.Namespace):
|
||||
"""Handle subscribe command - save subscription configuration."""
|
||||
print_header("Subscribe Configuration")
|
||||
|
||||
if not CONFIG_AVAILABLE:
|
||||
print("Error: Config module not available. Please check installation.")
|
||||
return 1
|
||||
|
||||
config = Config()
|
||||
|
||||
# Build subscription entry
|
||||
subscription = {
|
||||
'platform': args.platform,
|
||||
'type': args.type,
|
||||
'id': args.id,
|
||||
'name': args.name or f"{args.platform}_{args.type}_{args.id}",
|
||||
'created_at': datetime.now().isoformat()
|
||||
}
|
||||
|
||||
# Load existing subscriptions
|
||||
subscriptions = config.get('subscriptions', [])
|
||||
|
||||
# Check for duplicates
|
||||
for sub in subscriptions:
|
||||
if sub['platform'] == args.platform and sub['id'] == args.id:
|
||||
print(f"⚠️ Subscription already exists: {sub['name']}")
|
||||
print(" Updating configuration...")
|
||||
sub.update(subscription)
|
||||
break
|
||||
else:
|
||||
subscriptions.append(subscription)
|
||||
print(f"✅ Added new subscription: {subscription['name']}")
|
||||
|
||||
# Save configuration
|
||||
config.set('subscriptions', subscriptions)
|
||||
config.save()
|
||||
|
||||
print(f"\n Platform: {args.platform}")
|
||||
print(f" Type: {args.type}")
|
||||
print(f" ID: {args.id}")
|
||||
print(f" Name: {subscription['name']}")
|
||||
print(f"\n💾 Configuration saved to config.yaml")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_inbox(args: argparse.Namespace):
|
||||
"""Handle inbox command - display pending items."""
|
||||
print_header("Inbox Contents")
|
||||
|
||||
if not CONFIG_AVAILABLE:
|
||||
print("Error: Storage module not available.")
|
||||
return 1
|
||||
|
||||
storage = Storage()
|
||||
# Filter for CONFIRMED status (ready to process)
|
||||
contents = storage.list_inbox(status=ContentStatus.CONFIRMED)
|
||||
|
||||
if not contents:
|
||||
print("📭 Inbox is empty!")
|
||||
return 0
|
||||
|
||||
# Sort by the requested criteria
|
||||
if args.sort == 'score':
|
||||
contents.sort(key=lambda x: x.quality_metrics.overall_score if hasattr(x, 'quality_metrics') else 0, reverse=True)
|
||||
else: # date
|
||||
contents.sort(key=lambda x: x.created_at, reverse=True)
|
||||
|
||||
# Limit results
|
||||
contents = contents[:args.limit]
|
||||
|
||||
print(f"Showing {len(contents)} item(s) (sorted by {args.sort}):\n")
|
||||
|
||||
# Format table rows
|
||||
rows = []
|
||||
for idx, content in enumerate(contents, 1):
|
||||
title = content.title[:40] if content.title else 'Untitled'
|
||||
score = content.quality_metrics.overall_score if hasattr(content, 'quality_metrics') else 0
|
||||
summary = content.summary[:50] if content.summary else 'No summary'
|
||||
content_id = content.id
|
||||
|
||||
rows.append([
|
||||
f"{idx:2d}",
|
||||
content_id[:12],
|
||||
f"{score:.1f}",
|
||||
title,
|
||||
summary + "..." if len(content.summary or '') > 50 else summary
|
||||
])
|
||||
|
||||
print_table(
|
||||
["#", "ID", "Score", "Title", "Summary"],
|
||||
rows
|
||||
)
|
||||
|
||||
print(f"\n💡 Use 'process <id>' to process an item, or 'process all' to process all")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_process(args: argparse.Namespace):
|
||||
"""Handle process command - process content items."""
|
||||
print_header("Process Content")
|
||||
|
||||
if not CONFIG_AVAILABLE:
|
||||
print("Error: Pipeline module not available.")
|
||||
return 1
|
||||
|
||||
storage = Storage()
|
||||
|
||||
# Import pipeline components
|
||||
try:
|
||||
from src.pipeline import ProcessingPipeline
|
||||
from src.light_scanner import LightScanner
|
||||
from src.deep_analyzer import DeepAnalyzer
|
||||
from src.config import get_config
|
||||
|
||||
config = get_config()
|
||||
pipeline = ProcessingPipeline()
|
||||
except Exception as e:
|
||||
print(f"❌ Error initializing pipeline: {e}")
|
||||
return 1
|
||||
|
||||
# Determine what to process
|
||||
if args.content_id == 'all':
|
||||
items = storage.list_inbox(status=ContentStatus.CONFIRMED)
|
||||
if not items:
|
||||
print("📭 Inbox is empty! Nothing to process.")
|
||||
return 0
|
||||
|
||||
print(f"Found {len(items)} item(s) to process.\n")
|
||||
|
||||
if not args.yes:
|
||||
confirm = input("Process all items? [Y/n]: ").strip().lower()
|
||||
if confirm and confirm not in ('y', 'yes'):
|
||||
print("❌ Cancelled.")
|
||||
return 0
|
||||
else:
|
||||
item = storage.load_content(args.content_id)
|
||||
if not item:
|
||||
print(f"❌ Content not found: {args.content_id}")
|
||||
return 1
|
||||
items = [item]
|
||||
print(f"Processing single item: {item.title}\n")
|
||||
|
||||
# Process items with progress
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
|
||||
for i, item in enumerate(items, 1):
|
||||
content_id = item.id
|
||||
title = item.title or 'Untitled'
|
||||
|
||||
print(f"[{i}/{len(items)}] Processing: {title[:50]}...")
|
||||
print(f" ID: {content_id}")
|
||||
|
||||
try:
|
||||
# Use the pipeline to process the content
|
||||
# The pipeline expects content_text - we'll use transcript or summary
|
||||
content_text = item.transcript or item.summary or ""
|
||||
|
||||
result = pipeline.process_full(item, content_text)
|
||||
|
||||
# Save the updated content
|
||||
storage.save_content(result)
|
||||
|
||||
print(f" ✅ Success")
|
||||
print(f" 📄 Note: {result.output_path or 'N/A'}")
|
||||
print(f" 🏷️ Tags: {', '.join(result.tags)}")
|
||||
success_count += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
fail_count += 1
|
||||
|
||||
print()
|
||||
|
||||
# Summary
|
||||
print(f"\n📊 Processing Complete")
|
||||
print(f" Successful: {success_count}")
|
||||
print(f" Failed: {fail_count}")
|
||||
print(f" Total: {len(items)}")
|
||||
|
||||
return 0 if fail_count == 0 else 1
|
||||
|
||||
|
||||
def cmd_skip(args: argparse.Namespace):
|
||||
"""Handle skip command - skip/remove an item from inbox."""
|
||||
print_header("Skip Content")
|
||||
|
||||
if not CONFIG_AVAILABLE:
|
||||
print("Error: Storage module not available.")
|
||||
return 1
|
||||
|
||||
storage = Storage()
|
||||
|
||||
# Find the item
|
||||
content = storage.load_content(args.content_id)
|
||||
if not content:
|
||||
print(f"❌ Content not found: {args.content_id}")
|
||||
return 1
|
||||
|
||||
print(f"Skipping: {content.title or 'Untitled'}")
|
||||
print(f"ID: {args.content_id}")
|
||||
|
||||
if args.reason:
|
||||
print(f"Reason: {args.reason}")
|
||||
|
||||
# Mark as skipped
|
||||
content.status = ContentStatus.SKIPPED
|
||||
if args.reason:
|
||||
content.summary = f"[Skipped: {args.reason}] {content.summary or ''}"
|
||||
|
||||
storage.save_content(content)
|
||||
|
||||
print("✅ Item skipped and marked as skipped.")
|
||||
if args.reason:
|
||||
print(f" Reason recorded: {args.reason}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_status(args: argparse.Namespace):
|
||||
"""Handle status command - show system status."""
|
||||
print_header("System Status")
|
||||
|
||||
# Git status
|
||||
print("📁 Git Repository Status")
|
||||
print("-" * 30)
|
||||
|
||||
import subprocess
|
||||
try:
|
||||
# Check if we're in a git repo
|
||||
result = subprocess.run(
|
||||
['git', 'status', '--short'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=os.getcwd()
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
if result.stdout.strip():
|
||||
print("Uncommitted changes:")
|
||||
print(result.stdout)
|
||||
else:
|
||||
print("✅ Working directory clean")
|
||||
|
||||
# Get branch info
|
||||
branch_result = subprocess.run(
|
||||
['git', 'branch', '--show-current'],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
if branch_result.returncode == 0:
|
||||
print(f"Branch: {branch_result.stdout.strip()}")
|
||||
else:
|
||||
print("⚠️ Not a git repository or git not available")
|
||||
except FileNotFoundError:
|
||||
print("⚠️ Git not available")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Git error: {e}")
|
||||
|
||||
print()
|
||||
|
||||
# Inbox statistics
|
||||
print("📥 Inbox Statistics")
|
||||
print("-" * 30)
|
||||
|
||||
if CONFIG_AVAILABLE:
|
||||
storage = Storage()
|
||||
stats = storage.get_stats()
|
||||
|
||||
print(f"Pending (confirmed): {stats.get(ContentStatus.CONFIRMED.value, 0)}")
|
||||
print(f"New: {stats.get(ContentStatus.NEW.value, 0)}")
|
||||
print(f"Scanned: {stats.get(ContentStatus.SCANNED.value, 0)}")
|
||||
print(f"Analyzed: {stats.get(ContentStatus.ANALYZED.value, 0)}")
|
||||
print(f"Generated: {stats.get(ContentStatus.GENERATED.value, 0)}")
|
||||
print(f"Skipped: {stats.get(ContentStatus.SKIPPED.value, 0)}")
|
||||
else:
|
||||
print("⚠️ Storage module not available")
|
||||
|
||||
print()
|
||||
|
||||
# Configuration summary
|
||||
print("⚙️ Configuration")
|
||||
print("-" * 30)
|
||||
|
||||
if CONFIG_AVAILABLE:
|
||||
config = Config()
|
||||
subscriptions = config.get('subscriptions', [])
|
||||
print(f"Active subscriptions: {len(subscriptions)}")
|
||||
|
||||
if subscriptions:
|
||||
for sub in subscriptions:
|
||||
print(f" • {sub.get('name', 'Unnamed')} ({sub.get('platform')})")
|
||||
else:
|
||||
print("⚠️ Config module not available")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_search(args: argparse.Namespace):
|
||||
"""Handle search command - search processed notes."""
|
||||
print_header(f"Search: '{args.query}'")
|
||||
|
||||
if not CONFIG_AVAILABLE:
|
||||
print("Error: Storage module not available.")
|
||||
return 1
|
||||
|
||||
storage = Storage()
|
||||
|
||||
# Perform search across all content
|
||||
results = storage.search(args.query)
|
||||
|
||||
if not results:
|
||||
print("🔍 No results found.")
|
||||
return 0
|
||||
|
||||
# Filter by tags if requested
|
||||
if args.in_tags:
|
||||
query_lower = args.query.lower()
|
||||
results = [r for r in results if any(query_lower in tag.lower() for tag in r.tags)]
|
||||
|
||||
if not results:
|
||||
print("🔍 No results found in tags.")
|
||||
return 0
|
||||
|
||||
print(f"Found {len(results)} result(s):\n")
|
||||
|
||||
# Format results
|
||||
for i, content in enumerate(results, 1):
|
||||
title = content.title or 'Untitled'
|
||||
path = content.output_path or f"inbox/{content.id}.json"
|
||||
tags = content.tags
|
||||
|
||||
print(f"{i}. {title}")
|
||||
print(f" ID: {content.id}")
|
||||
print(f" Status: {content.status.value}")
|
||||
print(f" Path: {path}")
|
||||
if tags:
|
||||
print(f" Tags: {', '.join(tags)}")
|
||||
|
||||
# Show matching content preview
|
||||
if not args.in_tags and content.summary:
|
||||
content_text = content.summary
|
||||
query_lower = args.query.lower()
|
||||
idx = content_text.lower().find(query_lower)
|
||||
if idx >= 0:
|
||||
start = max(0, idx - 30)
|
||||
end = min(len(content_text), idx + len(args.query) + 30)
|
||||
preview = content_text[start:end]
|
||||
if start > 0:
|
||||
preview = "..." + preview
|
||||
if end < len(content_text):
|
||||
preview = preview + "..."
|
||||
print(f" Match: {preview}")
|
||||
|
||||
print()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for CLI."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Obsidian KB 知识库维护工具',
|
||||
prog='obsidian-kb',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s subscribe bilibili fav 12345 --name="Tech Favorites"
|
||||
%(prog)s inbox --limit 10 --sort date
|
||||
%(prog)s process all --yes
|
||||
%(prog)s process abc123
|
||||
%(prog)s skip abc123 --reason="Low quality"
|
||||
%(prog)s status
|
||||
%(prog)s search "python tutorial" --in-tags
|
||||
"""
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(dest='command', help='Available commands')
|
||||
|
||||
# subscribe 子命令
|
||||
subscribe_parser = subparsers.add_parser(
|
||||
'subscribe',
|
||||
aliases=['sub'],
|
||||
help='Subscribe to a content source'
|
||||
)
|
||||
subscribe_parser.add_argument(
|
||||
'platform',
|
||||
choices=['bilibili', 'youtube', 'zhihu'],
|
||||
help='Platform to subscribe to'
|
||||
)
|
||||
subscribe_parser.add_argument(
|
||||
'type',
|
||||
choices=['fav', 'up', 'channel'],
|
||||
help='Subscription type'
|
||||
)
|
||||
subscribe_parser.add_argument(
|
||||
'id',
|
||||
help='Platform-specific ID (user ID, channel ID, etc.)'
|
||||
)
|
||||
subscribe_parser.add_argument(
|
||||
'--name',
|
||||
help='Friendly name for this subscription'
|
||||
)
|
||||
subscribe_parser.set_defaults(func=cmd_subscribe)
|
||||
|
||||
# inbox 子命令
|
||||
inbox_parser = subparsers.add_parser(
|
||||
'inbox',
|
||||
aliases=['i'],
|
||||
help='Show inbox contents'
|
||||
)
|
||||
inbox_parser.add_argument(
|
||||
'--limit',
|
||||
type=int,
|
||||
default=20,
|
||||
help='Maximum number of items to show (default: 20)'
|
||||
)
|
||||
inbox_parser.add_argument(
|
||||
'--sort',
|
||||
choices=['score', 'date'],
|
||||
default='score',
|
||||
help='Sort order (default: score)'
|
||||
)
|
||||
inbox_parser.set_defaults(func=cmd_inbox)
|
||||
|
||||
# process 子命令
|
||||
process_parser = subparsers.add_parser(
|
||||
'process',
|
||||
aliases=['p'],
|
||||
help='Process content items'
|
||||
)
|
||||
process_parser.add_argument(
|
||||
'content_id',
|
||||
nargs='?',
|
||||
default='all',
|
||||
help='Content ID to process, or "all" for all items (default: all)'
|
||||
)
|
||||
process_parser.add_argument(
|
||||
'--yes',
|
||||
action='store_true',
|
||||
help='Auto-confirm without prompting'
|
||||
)
|
||||
process_parser.set_defaults(func=cmd_process)
|
||||
|
||||
# skip 子命令
|
||||
skip_parser = subparsers.add_parser(
|
||||
'skip',
|
||||
aliases=['s'],
|
||||
help='Skip an item (remove from inbox)'
|
||||
)
|
||||
skip_parser.add_argument(
|
||||
'content_id',
|
||||
help='Content ID to skip'
|
||||
)
|
||||
skip_parser.add_argument(
|
||||
'--reason',
|
||||
help='Reason for skipping'
|
||||
)
|
||||
skip_parser.set_defaults(func=cmd_skip)
|
||||
|
||||
# status 子命令
|
||||
status_parser = subparsers.add_parser(
|
||||
'status',
|
||||
aliases=['st'],
|
||||
help='Show system status'
|
||||
)
|
||||
status_parser.set_defaults(func=cmd_status)
|
||||
|
||||
# search 子命令
|
||||
search_parser = subparsers.add_parser(
|
||||
'search',
|
||||
aliases=['q'],
|
||||
help='Search processed notes'
|
||||
)
|
||||
search_parser.add_argument(
|
||||
'query',
|
||||
help='Search query'
|
||||
)
|
||||
search_parser.add_argument(
|
||||
'--in-tags',
|
||||
action='store_true',
|
||||
help='Search only in tags'
|
||||
)
|
||||
search_parser.set_defaults(func=cmd_search)
|
||||
|
||||
# Parse arguments
|
||||
args = parser.parse_args()
|
||||
|
||||
# Execute command
|
||||
if args.command is None:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
try:
|
||||
return args.func(args) or 0
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ Interrupted by user")
|
||||
return 130
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user