f41fb6167d
## Git Version Management - Git repository in archive/ directory - Environment variables for configuration (KB_GIT_ROOT, KB_GIT_REMOTE, etc.) - Auto commit after archive with retry (3 attempts) - Status command shows git status ## Subscription Management - Subscriptions stored in archive/subscriptions/ - Track processed items to avoid duplicates - Git tracked for history ## Automatic Cleanup - Cleanup strategy based on file type and age - video/frame retention: 7 days - metadata/reports: permanent - archive/: Git managed ## New Commands - --trigger git-init: Initialize git repository - --trigger git-status: Show git status - --trigger cleanup: Clean expired temporary files
915 lines
31 KiB
Python
915 lines
31 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
KB Workflow - Knowledge Base Workflow Orchestration
|
|
|
|
知识库工作流编排技能。
|
|
|
|
## 文件存放规则
|
|
|
|
```
|
|
knowledge-base/
|
|
├── kb.config.yaml # 配置文件
|
|
├── .gitignore # Git忽略规则
|
|
├── pending/ # 待处理内容 (*.md)
|
|
├── metadata/ # 链接解析结果 (*.link.json)
|
|
├── content/
|
|
│ ├── json/ # 元数据 (*.meta.json)
|
|
│ ├── downloads/ # 视频文件 (*.mp4/*.flv) - 7天后清理
|
|
│ └── subtitles/ # 字幕文件 (*.srt/*.lrc) - 7天后清理
|
|
├── frames/ # 视频截帧 (*.jpg) - 7天后清理
|
|
├── analyzed/ # 分析报告 (*.md)
|
|
└── archive/ # Git归档目录
|
|
├── YYYY-MM-DD/ # 按日期归档
|
|
└── subscriptions/ # 订阅数据
|
|
├── subscriptions.json # 订阅配置
|
|
└── processed.json # 已处理项目
|
|
```
|
|
|
|
## Git版本管理
|
|
|
|
配置环境变量:
|
|
- KB_GIT_ROOT: archive目录路径
|
|
- KB_GIT_REMOTE: 远程仓库地址
|
|
- KB_GIT_USER: 用户名
|
|
- KB_GIT_EMAIL: 邮箱
|
|
|
|
Usage:
|
|
python kb_workflow.py --trigger manual --scope pending --limit 5
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import argparse
|
|
import logging
|
|
import subprocess
|
|
import shutil
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Any
|
|
import yaml
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
)
|
|
logger = logging.getLogger('KBWorkflow')
|
|
|
|
|
|
class KBWorkflow:
|
|
"""Knowledge Base Workflow Orchestrator"""
|
|
|
|
STAGES = ['pending', 'metadata', 'content', 'analyzed']
|
|
|
|
# Git配置环境变量名
|
|
ENV_GIT_ROOT = 'KB_GIT_ROOT'
|
|
ENV_GIT_REMOTE = 'KB_GIT_REMOTE'
|
|
ENV_GIT_USER = 'KB_GIT_USER'
|
|
ENV_GIT_EMAIL = 'KB_GIT_EMAIL'
|
|
|
|
def __init__(self,
|
|
base_path: str = None,
|
|
skills_path: str = None):
|
|
"""
|
|
Initialize KB Workflow
|
|
|
|
Args:
|
|
base_path: Base path for knowledge base
|
|
skills_path: Path to skills directory
|
|
"""
|
|
self.base_path = Path(base_path) if base_path else Path(__file__).resolve().parent.parent / "knowledge-base"
|
|
self.skills_path = Path(skills_path) if skills_path else Path(__file__).resolve().parent.parent
|
|
|
|
# Git配置
|
|
self.git_root = os.environ.get(self.ENV_GIT_ROOT, '')
|
|
self.git_remote = os.environ.get(self.ENV_GIT_REMOTE, '')
|
|
self.git_user = os.environ.get(self.ENV_GIT_USER, '')
|
|
self.git_email = os.environ.get(self.ENV_GIT_EMAIL, '')
|
|
|
|
# 创建目录结构
|
|
self._ensure_directories()
|
|
|
|
# 加载配置
|
|
self.config = self._load_config()
|
|
|
|
# 检查Git
|
|
self.git_status = self._check_git()
|
|
|
|
logger.info(f"KB Workflow initialized")
|
|
logger.info(f"Base: {self.base_path}")
|
|
logger.info(f"Git: {'✓' if self.git_root else '✗'}")
|
|
|
|
def _ensure_directories(self):
|
|
"""创建目录结构"""
|
|
# 主目录
|
|
for subdir in self.STAGES:
|
|
dir_path = self.base_path / subdir
|
|
if not dir_path.exists():
|
|
dir_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 内容子目录
|
|
for subdir in ['json', 'downloads', 'subtitles']:
|
|
dir_path = self.base_path / 'content' / subdir
|
|
if not dir_path.exists():
|
|
dir_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 截帧目录
|
|
frames_dir = self.base_path / 'frames'
|
|
if not frames_dir.exists():
|
|
frames_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 归档目录
|
|
archive_dir = self.base_path / 'archive'
|
|
if not archive_dir.exists():
|
|
archive_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 订阅目录
|
|
subs_dir = archive_dir / 'subscriptions'
|
|
if not subs_dir.exists():
|
|
subs_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
def _load_config(self) -> Dict:
|
|
"""加载配置文件"""
|
|
config_path = self.base_path / 'kb.config.yaml'
|
|
default_config = {
|
|
'git': {
|
|
'enabled': bool(self.git_root),
|
|
'remote': self.git_remote,
|
|
'user': self.git_user,
|
|
'email': self.git_email
|
|
},
|
|
'cleanup': {
|
|
'video_retention_days': 7,
|
|
'frame_retention_days': 7,
|
|
'auto_cleanup': True
|
|
},
|
|
'subscriptions': {
|
|
'check_interval_hours': 24
|
|
}
|
|
}
|
|
|
|
if config_path.exists():
|
|
try:
|
|
with open(config_path, 'r', encoding='utf-8') as f:
|
|
user_config = yaml.safe_load(f) or {}
|
|
# 合并配置
|
|
for section in default_config:
|
|
if section in user_config:
|
|
default_config[section].update(user_config[section])
|
|
except Exception as e:
|
|
logger.warning(f"Failed to load config: {e}")
|
|
|
|
return default_config
|
|
|
|
# ==================== Git管理 ====================
|
|
|
|
def _check_git(self) -> Dict[str, Any]:
|
|
"""检查Git状态"""
|
|
status = {
|
|
'initialized': False,
|
|
'remote_configured': False,
|
|
'remote_url': '',
|
|
'status': 'not configured'
|
|
}
|
|
|
|
git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive'
|
|
|
|
if not str(git_root):
|
|
status['status'] = 'not configured - set KB_GIT_ROOT'
|
|
return status
|
|
|
|
git_dir = git_root / '.git'
|
|
if not git_dir.exists():
|
|
status['status'] = 'not initialized'
|
|
return status
|
|
|
|
status['initialized'] = True
|
|
|
|
# 检查远程
|
|
try:
|
|
result = subprocess.run(
|
|
['git', 'remote', '-v'],
|
|
cwd=str(git_root),
|
|
capture_output=True, text=True
|
|
)
|
|
if result.returncode == 0 and result.stdout:
|
|
status['remote_configured'] = True
|
|
status['remote_url'] = result.stdout.split()[1] if result.stdout.split() else ''
|
|
status['status'] = 'ready'
|
|
else:
|
|
status['status'] = 'no remote'
|
|
except Exception as e:
|
|
status['status'] = f'error: {e}'
|
|
|
|
return status
|
|
|
|
def _git_init(self) -> bool:
|
|
"""初始化Git仓库"""
|
|
git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive'
|
|
|
|
if not git_root.exists():
|
|
git_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
git_dir = git_root / '.git'
|
|
if git_dir.exists():
|
|
logger.info("Git already initialized")
|
|
return True
|
|
|
|
try:
|
|
# 初始化
|
|
subprocess.run(['git', 'init'], cwd=str(git_root), capture_output=True)
|
|
|
|
# 配置用户
|
|
if self.git_user:
|
|
subprocess.run(['git', 'config', 'user.name', self.git_user], cwd=str(git_root))
|
|
if self.git_email:
|
|
subprocess.run(['git', 'config', 'user.email', self.git_email], cwd=str(git_root))
|
|
|
|
# 创建.gitignore
|
|
ignore_content = """# 知识库Git归档
|
|
*.tmp
|
|
*.log
|
|
"""
|
|
(git_root / '.gitignore').write_text(ignore_content)
|
|
|
|
# 初始提交
|
|
subprocess.run(['git', 'add', '.'], cwd=str(git_root))
|
|
commit_msg = f"Initial commit - Knowledge base archive"
|
|
subprocess.run(['git', 'commit', '-m', commit_msg], cwd=str(git_root))
|
|
|
|
logger.info(f"Git initialized at {git_root}")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"Git init failed: {e}")
|
|
return False
|
|
|
|
def _git_add_commit(self, files: List[str], message: str) -> bool:
|
|
"""Git add and commit"""
|
|
git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive'
|
|
|
|
if not (git_root / '.git').exists():
|
|
logger.warning("Git not initialized")
|
|
return False
|
|
|
|
try:
|
|
# Add
|
|
for f in files:
|
|
subprocess.run(['git', 'add', f], cwd=str(git_root), capture_output=True)
|
|
|
|
# Commit
|
|
result = subprocess.run(
|
|
['git', 'commit', '-m', message],
|
|
cwd=str(git_root),
|
|
capture_output=True, text=True
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
logger.info(f"Committed: {message}")
|
|
return True
|
|
else:
|
|
logger.warning(f"Commit failed: {result.stderr}")
|
|
return False
|
|
|
|
except Exception as e:
|
|
logger.error(f"Git error: {e}")
|
|
return False
|
|
|
|
def _git_push(self, retries: int = 3) -> Dict[str, Any]:
|
|
"""Git push with retry"""
|
|
git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive'
|
|
result = {'success': False, 'message': '', 'retries': 0}
|
|
|
|
if not self.git_remote:
|
|
result['message'] = 'no remote configured'
|
|
return result
|
|
|
|
if not (git_root / '.git').exists():
|
|
result['message'] = 'git not initialized'
|
|
return result
|
|
|
|
for i in range(retries):
|
|
result['retries'] = i + 1
|
|
try:
|
|
proc = subprocess.run(
|
|
['git', 'push', 'origin', 'master'],
|
|
cwd=str(git_root),
|
|
capture_output=True, text=True,
|
|
timeout=60
|
|
)
|
|
|
|
if proc.returncode == 0:
|
|
result['success'] = True
|
|
result['message'] = 'pushed successfully'
|
|
return result
|
|
else:
|
|
result['message'] = proc.stderr
|
|
if i < retries - 1:
|
|
import time
|
|
time.sleep(2 ** i) # 指数退避
|
|
|
|
except subprocess.TimeoutExpired:
|
|
result['message'] = 'timeout'
|
|
except Exception as e:
|
|
result['message'] = str(e)
|
|
|
|
logger.warning(f"Push failed after {retries} attempts")
|
|
return result
|
|
|
|
def _git_status(self) -> Dict:
|
|
"""获取Git状态"""
|
|
git_root = Path(self.git_root) if self.git_root else self.base_path / 'archive'
|
|
|
|
if not (git_root / '.git').exists():
|
|
return {'error': 'not initialized'}
|
|
|
|
try:
|
|
# 获取远程URL
|
|
remote = subprocess.run(
|
|
['git', 'remote', 'get-url', 'origin'],
|
|
cwd=str(git_root), capture_output=True, text=True
|
|
)
|
|
remote_url = remote.stdout.strip() if remote.returncode == 0 else ''
|
|
|
|
# 获取提交数
|
|
log = subprocess.run(
|
|
['git', 'rev-list', '--count', 'HEAD'],
|
|
cwd=str(git_root), capture_output=True, text=True
|
|
)
|
|
commits = log.stdout.strip() if log.returncode == 0 else '0'
|
|
|
|
return {
|
|
'initialized': True,
|
|
'remote_url': remote_url,
|
|
'commits': commits,
|
|
'status': 'ready' if remote_url else 'no remote'
|
|
}
|
|
|
|
except Exception as e:
|
|
return {'error': str(e)}
|
|
|
|
# ==================== 订阅管理 ====================
|
|
|
|
def _load_subscriptions(self) -> Dict:
|
|
"""加载订阅数据"""
|
|
subs_file = self.base_path / 'archive' / 'subscriptions' / 'subscriptions.json'
|
|
|
|
default_subs = {
|
|
'version': '1.0',
|
|
'updated_at': datetime.now().isoformat(),
|
|
'subscriptions': [],
|
|
'processed_items': []
|
|
}
|
|
|
|
if subs_file.exists():
|
|
try:
|
|
with open(subs_file, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to load subscriptions: {e}")
|
|
|
|
return default_subs
|
|
|
|
def _save_subscriptions(self, data: Dict) -> bool:
|
|
"""保存订阅数据"""
|
|
subs_file = self.base_path / 'archive' / 'subscriptions' / 'subscriptions.json'
|
|
|
|
try:
|
|
data['updated_at'] = datetime.now().isoformat()
|
|
with open(subs_file, 'w', encoding='utf-8') as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
|
|
# Git提交
|
|
self._git_add_commit(
|
|
['subscriptions/subscriptions.json'],
|
|
f"Update subscriptions - {datetime.now().strftime('%Y-%m-%d %H:%M')}"
|
|
)
|
|
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"Failed to save subscriptions: {e}")
|
|
return False
|
|
|
|
def add_subscription(self, platform: str, subs_type: str, identifier: str, name: str = '') -> Dict:
|
|
"""添加订阅"""
|
|
subs = self._load_subscriptions()
|
|
|
|
sub_id = f"sub_{len(subs['subscriptions']) + 1:03d}"
|
|
|
|
new_sub = {
|
|
'id': sub_id,
|
|
'platform': platform,
|
|
'type': subs_type,
|
|
'identifier': identifier,
|
|
'name': name,
|
|
'subscribed_at': datetime.now().isoformat(),
|
|
'last_checked': None,
|
|
'last_update': None,
|
|
'status': 'active'
|
|
}
|
|
|
|
subs['subscriptions'].append(new_sub)
|
|
self._save_subscriptions(subs)
|
|
|
|
logger.info(f"Added subscription: {sub_id} - {platform}/{subs_type}/{identifier}")
|
|
return new_sub
|
|
|
|
def mark_item_processed(self, item_id: str, title: str = '', archived: bool = True) -> bool:
|
|
"""标记项目为已处理"""
|
|
subs = self._load_subscriptions()
|
|
|
|
# 检查是否已存在
|
|
existing = [p for p in subs['processed_items'] if p['id'] == item_id]
|
|
if existing:
|
|
return True # 已存在,不重复添加
|
|
|
|
new_item = {
|
|
'id': item_id,
|
|
'title': title,
|
|
'processed_at': datetime.now().isoformat(),
|
|
'archived': archived
|
|
}
|
|
|
|
subs['processed_items'].append(new_item)
|
|
|
|
# Git提交
|
|
self._git_add_commit(
|
|
['subscriptions/subscriptions.json'],
|
|
f"Mark processed: {item_id}"
|
|
)
|
|
|
|
return self._save_subscriptions(subs)
|
|
|
|
def is_item_processed(self, item_id: str) -> bool:
|
|
"""检查项目是否已处理"""
|
|
subs = self._load_subscriptions()
|
|
return any(p['id'] == item_id for p in subs['processed_items'])
|
|
|
|
# ==================== 清理策略 ====================
|
|
|
|
def cleanup_project(self, base_name: str) -> Dict[str, Any]:
|
|
"""清理单个项目的临时文件"""
|
|
result = {'cleaned': [], 'errors': []}
|
|
|
|
# 清理视频文件
|
|
downloads_dir = self.base_path / 'content' / 'downloads'
|
|
for f in downloads_dir.glob(f'{base_name}.*'):
|
|
try:
|
|
f.unlink()
|
|
result['cleaned'].append(f"downloads/{f.name}")
|
|
except Exception as e:
|
|
result['errors'].append(str(e))
|
|
|
|
# 清理截帧
|
|
frames_dir = self.base_path / 'frames'
|
|
for f in frames_dir.glob(f'{base_name}_*'):
|
|
try:
|
|
f.unlink()
|
|
result['cleaned'].append(f"frames/{f.name}")
|
|
except Exception as e:
|
|
result['errors'].append(str(e))
|
|
|
|
if result['cleaned']:
|
|
logger.info(f"Cleaned project {base_name}: {len(result['cleaned'])} files")
|
|
|
|
return result
|
|
|
|
def cleanup_expired(self, days: int = 30) -> Dict[str, Any]:
|
|
"""清理过期临时文件"""
|
|
result = {'cleaned': [], 'errors': [], 'count': 0}
|
|
|
|
cutoff = datetime.now() - timedelta(days=days)
|
|
|
|
# 清理下载目录
|
|
for f in (self.base_path / 'content' / 'downloads').iterdir():
|
|
if f.is_file():
|
|
mtime = datetime.fromtimestamp(f.stat().st_mtime)
|
|
if mtime < cutoff:
|
|
try:
|
|
f.unlink()
|
|
result['cleaned'].append(str(f.relative_to(self.base_path)))
|
|
result['count'] += 1
|
|
except Exception as e:
|
|
result['errors'].append(str(e))
|
|
|
|
# 清理截帧目录
|
|
for f in (self.base_path / 'frames').iterdir():
|
|
if f.is_file():
|
|
mtime = datetime.fromtimestamp(f.stat().st_mtime)
|
|
if mtime < cutoff:
|
|
try:
|
|
f.unlink()
|
|
result['cleaned'].append(str(f.relative_to(self.base_path)))
|
|
result['count'] += 1
|
|
except Exception as e:
|
|
result['errors'].append(str(e))
|
|
|
|
logger.info(f"Expired cleanup: {result['count']} files")
|
|
return result
|
|
|
|
# ==================== 归档 ====================
|
|
|
|
def archive_item(self, base_name: str, report_content: str) -> Dict[str, Any]:
|
|
"""归档项目"""
|
|
result = {
|
|
'success': False,
|
|
'archive_path': '',
|
|
'git_commit': False,
|
|
'git_push': None,
|
|
'cleanup': None,
|
|
'errors': []
|
|
}
|
|
|
|
# 1. 创建归档目录
|
|
date_dir = self.base_path / 'archive' / datetime.now().strftime('%Y-%m-%d')
|
|
date_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# 2. 复制分析报告
|
|
report_path = date_dir / f"{base_name}.md"
|
|
with open(report_path, 'w', encoding='utf-8') as f:
|
|
f.write(report_content)
|
|
result['archive_path'] = str(report_path)
|
|
|
|
# 3. 复制元数据
|
|
meta_src = self.base_path / 'content' / 'json' / f"{base_name}.meta.json"
|
|
if meta_src.exists():
|
|
shutil.copy(meta_src, date_dir / f"{base_name}.meta.json")
|
|
|
|
# 4. Git提交
|
|
if self._git_add_commit(
|
|
[datetime.now().strftime('%Y-%m-%d')],
|
|
f"Archive: {base_name}"
|
|
):
|
|
result['git_commit'] = True
|
|
|
|
# 5. 推送到远程
|
|
push_result = self._git_push()
|
|
result['git_push'] = push_result
|
|
|
|
# 6. 清理临时文件
|
|
result['cleanup'] = self.cleanup_project(base_name)
|
|
|
|
result['success'] = True
|
|
return result
|
|
|
|
# ==================== 链接解析 ====================
|
|
|
|
def _extract_link(self, file_path: Path) -> Optional[Dict[str, Any]]:
|
|
"""解析链接"""
|
|
try:
|
|
content = file_path.read_text(encoding='utf-8')
|
|
|
|
import re
|
|
url_pattern = r'https?://[^\s<>"]+'
|
|
urls = re.findall(url_pattern, content)
|
|
|
|
if not urls:
|
|
return None
|
|
|
|
url = urls[0]
|
|
result = {
|
|
"platform": self._detect_platform(url),
|
|
"type": self._detect_type(url),
|
|
"id": self._extract_id(url),
|
|
"url": url,
|
|
"original_content": content
|
|
}
|
|
|
|
logger.info(f"Parsed: {result['platform']}/{result['type']} - {result['id']}")
|
|
return result
|
|
|
|
except Exception as e:
|
|
logger.error(f"Link extraction failed: {e}")
|
|
return None
|
|
|
|
def _detect_platform(self, url: str) -> str:
|
|
"""检测平台"""
|
|
if 'bilibili.com' in url or 'b23.tv' in url:
|
|
return 'bilibili'
|
|
elif 'zhihu.com' in url or 'zhuanlan.zhihu.com' in url:
|
|
return 'zhihu'
|
|
return 'unknown'
|
|
|
|
def _detect_type(self, url: str) -> str:
|
|
"""检测类型"""
|
|
if '/video/' in url or '/zvideo/' in url:
|
|
return 'video'
|
|
elif '/read/' in url or '/p/' in url:
|
|
return 'article'
|
|
elif 'space.bilibili.com' in url:
|
|
return 'up主'
|
|
return 'unknown'
|
|
|
|
def _extract_id(self, url: str) -> str:
|
|
"""提取ID"""
|
|
import re
|
|
bv = re.search(r'BV[a-zA-Z0-9]{10}', url)
|
|
if bv: return bv.group()
|
|
cv = re.search(r'cv(\d+)', url)
|
|
if cv: return f"cv{cv.group(1)}"
|
|
num = re.search(r'/(\d+)', url)
|
|
if num: return num.group(1)
|
|
return ''
|
|
|
|
# ==================== 主处理流程 ====================
|
|
|
|
def _save_json(self, data: Dict, subdir: str, base_name: str, suffix: str) -> Path:
|
|
"""保存JSON"""
|
|
file_path = self.base_path / subdir / f"{base_name}{suffix}"
|
|
with open(file_path, 'w', encoding='utf-8') as f:
|
|
json.dump(data, f, ensure_ascii=False, indent=2)
|
|
logger.info(f"Saved: {subdir}/{base_name}{suffix}")
|
|
return file_path
|
|
|
|
def _move_to_stage(self, file_path: Path, to_stage: str) -> Path:
|
|
"""移动文件"""
|
|
new_path = self.base_path / to_stage / file_path.name
|
|
if file_path.exists():
|
|
file_path.rename(new_path)
|
|
logger.info(f"Moved: {file_path.name} → {to_stage}/")
|
|
return new_path
|
|
|
|
def process_item(self, item: Dict) -> Dict[str, Any]:
|
|
"""处理单个项目"""
|
|
result = {
|
|
"item": item,
|
|
"success": False,
|
|
"stages_completed": [],
|
|
"files_created": [],
|
|
"archive": None,
|
|
"error": None
|
|
}
|
|
|
|
file_path = Path(item['path'])
|
|
base_name = file_path.stem
|
|
|
|
try:
|
|
logger.info(f"Processing: {file_path.name}")
|
|
|
|
# 阶段1: 链接解析
|
|
link_info = self._extract_link(file_path)
|
|
if not link_info:
|
|
result["error"] = "无法解析链接"
|
|
return result
|
|
|
|
result["stages_completed"].append("link_parsing")
|
|
self._save_json(link_info, 'metadata', base_name, '.link.json')
|
|
|
|
# 阶段2: 元数据提取
|
|
logger.info(f"Stage 2: Extracting metadata...")
|
|
metadata = self._extract_metadata(link_info)
|
|
if 'error' not in metadata:
|
|
self._save_json(metadata, 'content/json', base_name, '.meta.json')
|
|
result["stages_completed"].append("metadata_extraction")
|
|
|
|
# 标记为已处理
|
|
item_id = metadata.get('bvid') or metadata.get('id') or base_name
|
|
self.mark_item_processed(item_id, metadata.get('title', ''))
|
|
|
|
# 移动原文件
|
|
self._move_to_stage(file_path, 'metadata')
|
|
|
|
result["success"] = len(result["stages_completed"]) >= 2
|
|
logger.info(f"Complete: {len(result['stages_completed'])} stages")
|
|
|
|
except Exception as e:
|
|
result["error"] = str(e)
|
|
logger.error(f"Processing failed: {e}")
|
|
|
|
return result
|
|
|
|
def _extract_metadata(self, link_info: Dict) -> Dict:
|
|
"""提取元数据"""
|
|
platform = link_info.get('platform', '')
|
|
content_type = link_info.get('type', '')
|
|
content_id = link_info.get('id', '')
|
|
|
|
metadata = {
|
|
"platform": platform,
|
|
"type": content_type,
|
|
"id": content_id,
|
|
"url": link_info.get('url', ''),
|
|
"source_file": link_info.get('source_file', ''),
|
|
"extracted_at": datetime.now().isoformat()
|
|
}
|
|
|
|
# 调用相应技能的提取逻辑...
|
|
# (简化处理)
|
|
return metadata
|
|
|
|
def process_all(self, scope: str = "pending", limit: int = None) -> Dict[str, Any]:
|
|
"""处理所有项目"""
|
|
summary = {"processed": 0, "failed": 0, "results": []}
|
|
|
|
inbox = self.scan_inbox(scope=scope)
|
|
|
|
for stage, items in inbox.items():
|
|
if limit and summary['processed'] >= limit:
|
|
break
|
|
|
|
for item in items:
|
|
if limit and summary['processed'] >= limit:
|
|
break
|
|
|
|
result = self.process_item(item)
|
|
summary["results"].append(result)
|
|
|
|
if result["success"]:
|
|
summary["processed"] += 1
|
|
else:
|
|
summary["failed"] += 1
|
|
|
|
return summary
|
|
|
|
# ==================== 状态和汇报 ====================
|
|
|
|
def scan_inbox(self, scope: str = "all") -> Dict[str, List[Dict]]:
|
|
"""扫描收件箱"""
|
|
result = {}
|
|
|
|
stages = self.STAGES if scope == "all" else ([scope] if scope in self.STAGES else self.STAGES)
|
|
|
|
for stage in stages:
|
|
dir_path = self.base_path / stage
|
|
items = []
|
|
|
|
if dir_path.exists():
|
|
for item_path in dir_path.iterdir():
|
|
if item_path.is_file():
|
|
items.append({
|
|
'name': item_path.name,
|
|
'path': str(item_path),
|
|
'stage': stage,
|
|
'modified': datetime.fromtimestamp(
|
|
item_path.stat().st_mtime
|
|
).isoformat()
|
|
})
|
|
|
|
result[stage] = items
|
|
|
|
return result
|
|
|
|
def get_status(self) -> Dict[str, Any]:
|
|
"""获取状态"""
|
|
inbox = self.scan_inbox(scope="all")
|
|
|
|
# 统计文件
|
|
file_counts = {}
|
|
total_size = 0
|
|
|
|
for root, dirs, files in os.walk(self.base_path):
|
|
for f in files:
|
|
file_path = Path(root) / f
|
|
ext = file_path.suffix
|
|
file_counts[ext] = file_counts.get(ext, 0) + 1
|
|
total_size += file_path.stat().st_size
|
|
|
|
# Git状态
|
|
git_info = self._git_status()
|
|
|
|
# 订阅状态
|
|
subs = self._load_subscriptions()
|
|
|
|
return {
|
|
"status": "ready",
|
|
"timestamp": datetime.now().isoformat(),
|
|
"stages": inbox,
|
|
"total_items": sum(len(items) for items in inbox.values()),
|
|
"total_size_mb": round(total_size / 1024 / 1024, 2),
|
|
"file_counts": file_counts,
|
|
"git": git_info,
|
|
"subscriptions": {
|
|
"count": len(subs.get('subscriptions', [])),
|
|
"processed": len(subs.get('processed_items', []))
|
|
},
|
|
"directories": {
|
|
'pending': '待处理内容',
|
|
'metadata': '链接解析结果',
|
|
'content/json': '元数据',
|
|
'content/downloads': '视频文件(临时)',
|
|
'content/subtitles': '字幕文件(临时)',
|
|
'frames': '截帧(临时)',
|
|
'analyzed': '分析报告',
|
|
'archive/': 'Git归档'
|
|
}
|
|
}
|
|
|
|
def print_status(self):
|
|
"""打印状态"""
|
|
status = self.get_status()
|
|
|
|
print("\n" + "="*50)
|
|
print("KB Workflow Status")
|
|
print("="*50)
|
|
|
|
print(f"\n📁 阶段统计:")
|
|
for stage, items in status['stages'].items():
|
|
print(f" {stage}: {len(items)} 项")
|
|
|
|
print(f"\n💾 存储: {status['total_size_mb']} MB")
|
|
|
|
print(f"\n🔗 Git:")
|
|
git = status.get('git', {})
|
|
if git.get('initialized'):
|
|
print(f" ✓ 已初始化")
|
|
print(f" 远程: {git.get('remote_url', '未配置')}")
|
|
print(f" 提交: {git.get('commits', 0)} 个")
|
|
else:
|
|
print(f" ✗ 未初始化")
|
|
print(f" 状态: {git.get('status', 'unknown')}")
|
|
|
|
print(f"\n📋 订阅:")
|
|
subs = status.get('subscriptions', {})
|
|
print(f" 订阅数: {subs.get('count', 0)}")
|
|
print(f" 已处理: {subs.get('processed', 0)}")
|
|
|
|
print(f"\n📚 目录说明:")
|
|
for dir, desc in status.get('directories', {}).items():
|
|
print(f" {dir}: {desc}")
|
|
|
|
# 清理建议
|
|
if status['total_size_mb'] > 1000:
|
|
print(f"\n💡 建议: 存储空间较大 ({status['total_size_mb']} MB)")
|
|
print(f" 运行清理: python kb_workflow.py --trigger cleanup --days 30")
|
|
|
|
print()
|
|
|
|
|
|
# ==================== CLI ====================
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description='KB Workflow - Knowledge Base Workflow Orchestration',
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
使用示例:
|
|
# 查看状态
|
|
python kb_workflow.py --trigger status
|
|
|
|
# 处理待处理内容
|
|
python kb_workflow.py --trigger manual --scope pending --limit 5
|
|
|
|
# 初始化Git
|
|
python kb_workflow.py --trigger git-init
|
|
|
|
# 清理过期文件
|
|
python kb_workflow.py --trigger cleanup --days 30
|
|
|
|
# 查看Git状态
|
|
python kb_workflow.py --trigger git-status
|
|
|
|
配置环境变量:
|
|
export KB_GIT_ROOT="/path/to/archive" # Git仓库目录
|
|
export KB_GIT_REMOTE="git@github.com:user/repo.git" # 远程仓库
|
|
export KB_GIT_USER="Username" # 用户名
|
|
export KB_GIT_EMAIL="user@example.com" # 邮箱
|
|
"""
|
|
)
|
|
|
|
parser.add_argument('--trigger', type=str, default='status',
|
|
choices=['status', 'manual', 'cleanup', 'git-init', 'git-status', 'archive'],
|
|
help='触发类型')
|
|
parser.add_argument('--scope', type=str, default='pending',
|
|
choices=['all', 'pending', 'metadata', 'content', 'analyzed'],
|
|
help='处理范围')
|
|
parser.add_argument('--limit', type=int, default=None,
|
|
help='处理数量限制')
|
|
parser.add_argument('--days', type=int, default=30,
|
|
help='清理天数')
|
|
parser.add_argument('--base-path', type=str, default=None,
|
|
help='知识库基础路径')
|
|
|
|
args = parser.parse_args()
|
|
|
|
workflow = KBWorkflow(base_path=args.base_path)
|
|
|
|
if args.trigger == 'status':
|
|
workflow.print_status()
|
|
|
|
elif args.trigger == 'manual':
|
|
result = workflow.process_all(scope=args.scope, limit=args.limit)
|
|
print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
|
|
|
|
elif args.trigger == 'cleanup':
|
|
result = workflow.cleanup_expired(days=args.days)
|
|
print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
|
|
|
|
elif args.trigger == 'git-init':
|
|
success = workflow._git_init()
|
|
print(f"Git init: {'success' if success else 'failed'}")
|
|
|
|
elif args.trigger == 'git-status':
|
|
status = workflow._git_status()
|
|
print(json.dumps(status, ensure_ascii=False, indent=2, default=str))
|
|
|
|
else:
|
|
print(f"Unknown trigger: {args.trigger}")
|
|
return 1
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|