自动同步: 2026-03-15 02:10
This commit is contained in:
+128
-354
@@ -5,7 +5,7 @@ KB Workflow - Knowledge Base Workflow Orchestrator
|
||||
|
||||
知识库工作流辅助工具函数。
|
||||
|
||||
此模块提供查重、文件操作、Git 管理、状态管理等工具函数，
|
||||
此模块提供查重、文件操作、Git 管理等工具函数，
|
||||
供主代理在执行 SKILL.md 中定义的 workflow 时调用。
|
||||
"""
|
||||
|
||||
@@ -18,46 +18,7 @@ import shutil
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any, Tuple
|
||||
|
||||
|
||||
# ==================== 状态机定义 ====================
|
||||
|
||||
# 工作流步骤顺序
|
||||
WORKFLOW_STEPS = [
|
||||
"init",
|
||||
"link_parse",
|
||||
"metadata",
|
||||
"content",
|
||||
"whisper", # 可选步骤
|
||||
"analyze",
|
||||
"archive",
|
||||
"done"
|
||||
]
|
||||
|
||||
# 每个步骤的必需输出字段
|
||||
STEP_OUTPUT_FIELDS = {
|
||||
"init": [],
|
||||
"link_parse": ["platform", "content_type", "content_id"],
|
||||
"metadata": ["title"],
|
||||
"content": ["filepath"],
|
||||
"whisper": ["output_files"],
|
||||
"analyze": ["output_file"],
|
||||
"archive": ["archive_path"],
|
||||
"done": []
|
||||
}
|
||||
|
||||
# 步骤描述（用于汇报）
|
||||
STEP_DESCRIPTIONS = {
|
||||
"init": "初始化",
|
||||
"link_parse": "链接解析",
|
||||
"metadata": "元数据提取",
|
||||
"content": "内容下载",
|
||||
"whisper": "语音识别",
|
||||
"analyze": "深度分析",
|
||||
"archive": "归档",
|
||||
"done": "完成"
|
||||
}
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
|
||||
# ==================== 子技能映射 ====================
|
||||
@@ -332,249 +293,6 @@ def archive_and_git(
|
||||
return result
|
||||
|
||||
|
||||
# ==================== 状态管理 ====================
|
||||
|
||||
def get_state_file_path(url: str, base_path: Path = None) -> Path:
|
||||
"""获取状态文件路径"""
|
||||
if base_path is None:
|
||||
base_path = get_knowledge_base_path()
|
||||
pending_dir = base_path / "pending"
|
||||
pending_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 从 URL 提取标识符
|
||||
# 尝试提取 BV 号或其他 ID
|
||||
bv_match = re.search(r'BV[a-zA-Z0-9]+', url)
|
||||
if bv_match:
|
||||
content_id = bv_match.group()
|
||||
platform = "bilibili"
|
||||
else:
|
||||
# 尝试提取知乎 ID
|
||||
zhihu_match = re.search(r'zhihu\.com/(?:question|zvideo|p)/(\d+)', url)
|
||||
if zhihu_match:
|
||||
content_id = zhihu_match.group(1)
|
||||
platform = "zhihu"
|
||||
else:
|
||||
# 使用 URL hash 作为标识
|
||||
import hashlib
|
||||
content_id = hashlib.md5(url.encode()).hexdigest()[:12]
|
||||
platform = "unknown"
|
||||
|
||||
return pending_dir / f"{platform}_{content_id}.state.json"
|
||||
|
||||
|
||||
def init_state(url: str, base_path: Path = None) -> Dict[str, Any]:
|
||||
"""初始化工作流状态"""
|
||||
state_file = get_state_file_path(url, base_path)
|
||||
|
||||
# 如果状态文件已存在，返回现有状态
|
||||
if state_file.exists():
|
||||
with open(state_file, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
# 创建新状态
|
||||
now = datetime.now().isoformat()
|
||||
state = {
|
||||
"url": url,
|
||||
"platform": None,
|
||||
"content_id": None,
|
||||
"content_type": None,
|
||||
"current_step": "init",
|
||||
"completed_steps": [],
|
||||
"step_results": {},
|
||||
"errors": [],
|
||||
"created_at": now,
|
||||
"updated_at": now
|
||||
}
|
||||
|
||||
with open(state_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return state
|
||||
|
||||
|
||||
def get_state(url: str, base_path: Path = None) -> Dict[str, Any]:
|
||||
"""获取当前工作流状态"""
|
||||
state_file = get_state_file_path(url, base_path)
|
||||
|
||||
if not state_file.exists():
|
||||
return {
|
||||
"success": False,
|
||||
"error": "状态文件不存在",
|
||||
"url": url,
|
||||
"state_file": str(state_file)
|
||||
}
|
||||
|
||||
with open(state_file, 'r', encoding='utf-8') as f:
|
||||
state = json.load(f)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"state": state,
|
||||
"state_file": str(state_file)
|
||||
}
|
||||
|
||||
|
||||
def advance_step(
|
||||
url: str,
|
||||
step_name: str,
|
||||
result: Dict[str, Any] = None,
|
||||
base_path: Path = None
|
||||
) -> Dict[str, Any]:
|
||||
"""推进到下一步并记录结果"""
|
||||
state_file = get_state_file_path(url, base_path)
|
||||
|
||||
if not state_file.exists():
|
||||
return {
|
||||
"success": False,
|
||||
"error": "状态文件不存在，请先执行 --init-state"
|
||||
}
|
||||
|
||||
with open(state_file, 'r', encoding='utf-8') as f:
|
||||
state = json.load(f)
|
||||
|
||||
# 验证步骤名称
|
||||
if step_name not in WORKFLOW_STEPS:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"无效的步骤名称: {step_name}",
|
||||
"valid_steps": WORKFLOW_STEPS
|
||||
}
|
||||
|
||||
# 获取当前步骤索引
|
||||
try:
|
||||
current_idx = WORKFLOW_STEPS.index(state["current_step"])
|
||||
new_idx = WORKFLOW_STEPS.index(step_name)
|
||||
except ValueError:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"步骤索引错误"
|
||||
}
|
||||
|
||||
# 更新状态
|
||||
if step_name != state["current_step"]:
|
||||
# 标记当前步骤为完成
|
||||
if state["current_step"] not in state["completed_steps"]:
|
||||
state["completed_steps"].append(state["current_step"])
|
||||
|
||||
state["current_step"] = step_name
|
||||
state["updated_at"] = datetime.now().isoformat()
|
||||
|
||||
# 记录步骤结果
|
||||
if result:
|
||||
state["step_results"][step_name] = result
|
||||
|
||||
# 提取平台和 ID 信息（从 link_parse 步骤）
|
||||
if step_name == "link_parse" and result.get("success"):
|
||||
state["platform"] = result.get("platform")
|
||||
state["content_id"] = result.get("content_id")
|
||||
state["content_type"] = result.get("type")
|
||||
|
||||
# 保存状态
|
||||
with open(state_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"current_step": step_name,
|
||||
"completed_steps": state["completed_steps"],
|
||||
"state_file": str(state_file)
|
||||
}
|
||||
|
||||
|
||||
def record_error(
|
||||
url: str,
|
||||
error_msg: str,
|
||||
step: str = None,
|
||||
base_path: Path = None
|
||||
) -> Dict[str, Any]:
|
||||
"""记录错误"""
|
||||
state_file = get_state_file_path(url, base_path)
|
||||
|
||||
if not state_file.exists():
|
||||
return {
|
||||
"success": False,
|
||||
"error": "状态文件不存在，请先执行 --init-state"
|
||||
}
|
||||
|
||||
with open(state_file, 'r', encoding='utf-8') as f:
|
||||
state = json.load(f)
|
||||
|
||||
# 记录错误
|
||||
error_entry = {
|
||||
"step": step or state["current_step"],
|
||||
"error": error_msg,
|
||||
"timestamp": datetime.now().isoformat()
|
||||
}
|
||||
state["errors"].append(error_entry)
|
||||
state["updated_at"] = datetime.now().isoformat()
|
||||
|
||||
# 保存状态
|
||||
with open(state_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(state, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"error_recorded": error_entry,
|
||||
"total_errors": len(state["errors"])
|
||||
}
|
||||
|
||||
|
||||
def get_resume_info(url: str, base_path: Path = None) -> Dict[str, Any]:
|
||||
"""获取恢复执行的指令"""
|
||||
state_file = get_state_file_path(url, base_path)
|
||||
|
||||
if not state_file.exists():
|
||||
return {
|
||||
"success": False,
|
||||
"error": "状态文件不存在",
|
||||
"suggestion": "请使用 --init-state 初始化工作流"
|
||||
}
|
||||
|
||||
with open(state_file, 'r', encoding='utf-8') as f:
|
||||
state = json.load(f)
|
||||
|
||||
current_step = state["current_step"]
|
||||
step_desc = STEP_DESCRIPTIONS.get(current_step, current_step)
|
||||
|
||||
# 检查是否有错误
|
||||
if state["errors"]:
|
||||
last_error = state["errors"][-1]
|
||||
return {
|
||||
"success": True,
|
||||
"status": "error",
|
||||
"current_step": current_step,
|
||||
"step_description": step_desc,
|
||||
"last_error": last_error,
|
||||
"completed_steps": state["completed_steps"],
|
||||
"suggestion": f"请先解决 {last_error['step']} 步骤的错误: {last_error['error']}"
|
||||
}
|
||||
|
||||
# 检查是否已完成
|
||||
if current_step == "done":
|
||||
return {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"current_step": "done",
|
||||
"step_description": "已完成",
|
||||
"completed_steps": state["completed_steps"],
|
||||
"step_results": state["step_results"]
|
||||
}
|
||||
|
||||
# 返回下一步指令
|
||||
return {
|
||||
"success": True,
|
||||
"status": "in_progress",
|
||||
"current_step": current_step,
|
||||
"step_description": step_desc,
|
||||
"completed_steps": state["completed_steps"],
|
||||
"next_action": f"继续执行步骤: {step_desc}",
|
||||
"platform": state.get("platform"),
|
||||
"content_id": state.get("content_id"),
|
||||
"content_type": state.get("content_type"),
|
||||
"step_results": state.get("step_results", {})
|
||||
}
|
||||
|
||||
|
||||
def generate_summary(url: str, stages: List[Dict], final_file: str, archive_result: Dict = None) -> str:
|
||||
"""生成处理完成汇报"""
|
||||
# 提取视频标题（如果有）
|
||||
@@ -607,6 +325,83 @@ def generate_summary(url: str, stages: List[Dict], final_file: str, archive_resu
|
||||
return summary
|
||||
|
||||
|
||||
# ==================== CLI Process Command ====================
|
||||
|
||||
def process_command(
|
||||
url: str,
|
||||
output_dir: Optional[str] = None,
|
||||
skip_completed: bool = True
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute full pipeline for a URL
|
||||
|
||||
Args:
|
||||
url: URL to process
|
||||
output_dir: Optional output directory for content
|
||||
skip_completed: Skip already completed steps (default True)
|
||||
|
||||
Returns:
|
||||
Dictionary with success status, content_dir, and results
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
# Add project root to path for kb_pipeline imports
|
||||
project_root = Path(__file__).parent.parent
|
||||
if str(project_root) not in sys.path:
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
try:
|
||||
from kb_pipeline.pipeline import Pipeline
|
||||
from kb_pipeline.processors import ParseProcessor, DownloadProcessor, ASRProcessor, AnalyzeProcessor
|
||||
from kb_pipeline.schemas import PipelineConfig
|
||||
|
||||
async def run_pipeline():
|
||||
# Determine output directory
|
||||
out_dir = Path(output_dir) if output_dir else Path.cwd()
|
||||
|
||||
# Create processors
|
||||
processors = [
|
||||
ParseProcessor(),
|
||||
DownloadProcessor(),
|
||||
ASRProcessor(),
|
||||
AnalyzeProcessor(),
|
||||
]
|
||||
|
||||
# Create pipeline config
|
||||
config = PipelineConfig(
|
||||
steps=["parse", "download", "asr", "analyze"],
|
||||
output_dir=out_dir,
|
||||
skip_completed=skip_completed,
|
||||
)
|
||||
|
||||
# Create and run pipeline
|
||||
pipeline = Pipeline(
|
||||
processors=processors,
|
||||
config=config,
|
||||
output_dir=out_dir,
|
||||
)
|
||||
|
||||
result = await pipeline.run(url)
|
||||
return result
|
||||
|
||||
# Run async pipeline
|
||||
result = asyncio.run(run_pipeline())
|
||||
|
||||
return {
|
||||
"success": result.success,
|
||||
"error": result.error_message,
|
||||
"content_dir": str(result.content_dir) if result.content_dir else None,
|
||||
"results": result.results,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"content_dir": None,
|
||||
"results": {},
|
||||
}
|
||||
|
||||
|
||||
# ==================== CLI ====================
|
||||
|
||||
def main():
|
||||
@@ -618,117 +413,96 @@ def main():
|
||||
使用示例:
|
||||
# 查重
|
||||
python kb_workflow.py --check --url "https://www.bilibili.com/video/BVxxx"
|
||||
|
||||
|
||||
# 归档
|
||||
python kb_workflow.py --archive-file "/path/to/analyzed.md"
|
||||
|
||||
# 状态管理
|
||||
python kb_workflow.py --init-state --url "https://..."
|
||||
python kb_workflow.py --get-state --url "https://..."
|
||||
python kb_workflow.py --advance-step --url "https://..." --step "metadata" --result '{"success":true}'
|
||||
python kb_workflow.py --record-error --url "https://..." --error-msg "403 Forbidden"
|
||||
python kb_workflow.py --resume --url "https://..."
|
||||
python kb_workflow.py --archive "/path/to/analyzed.md"
|
||||
|
||||
# 处理URL (通过完整管道)
|
||||
python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
|
||||
"""
|
||||
)
|
||||
|
||||
# 基础参数
|
||||
parser.add_argument('--url', type=str, help='URL 地址')
|
||||
parser.add_argument('--base-path', type=str, help='知识库根目录')
|
||||
|
||||
# 查重和归档
|
||||
|
||||
# Create subparsers for commands
|
||||
subparsers = parser.add_subparsers(dest='command', help='Available commands')
|
||||
|
||||
# Process subcommand
|
||||
process_parser = subparsers.add_parser('process', help='Process URL through full pipeline')
|
||||
process_parser.add_argument('url', type=str, help='URL to process')
|
||||
process_parser.add_argument('--output-dir', '-o', type=str, help='Output directory for content')
|
||||
process_parser.add_argument('--skip-completed', action='store_true', default=True,
|
||||
help='Skip completed steps (default: True)')
|
||||
process_parser.add_argument('--no-skip', action='store_false', dest='skip_completed',
|
||||
help='Re-run all steps')
|
||||
|
||||
# Legacy arguments (for backward compatibility)
|
||||
parser.add_argument('--check', action='store_true', help='查重')
|
||||
parser.add_argument('--url', type=str, help='要检查的URL')
|
||||
parser.add_argument('--archive-file', type=str, help='归档的文件路径')
|
||||
|
||||
# 目录路径查询
|
||||
parser.add_argument('--pending', action='store_true', help='获取 URL 解析目录')
|
||||
parser.add_argument('--base-path', type=str, help='获取知识库根目录')
|
||||
|
||||
parser.add_argument('--pending', action='store_true', help='获取url解析目录')
|
||||
parser.add_argument('--metadata', action='store_true', help='获取元数据目录')
|
||||
parser.add_argument('--content', action='store_true', help='获取内容目录')
|
||||
parser.add_argument('--content_downloads', action='store_true', help='获取内容下载目录')
|
||||
parser.add_argument('--content_downloads', action='store_true', help='获取主内容下载目录')
|
||||
parser.add_argument('--analyzed', action='store_true', help='获取深度分析目录')
|
||||
parser.add_argument('--archive_dir', action='store_true', help='获取归档目录')
|
||||
parser.add_argument('--archive_subscriptions', action='store_true', help='获取订阅信息目录')
|
||||
parser.add_argument('--extract_platform', type=str, help='从文件名提取平台名称')
|
||||
|
||||
# 状态管理
|
||||
parser.add_argument('--init-state', action='store_true', help='初始化工作流状态')
|
||||
parser.add_argument('--get-state', action='store_true', help='获取当前状态')
|
||||
parser.add_argument('--advance-step', action='store_true', help='推进到下一步')
|
||||
parser.add_argument('--step', type=str, help='步骤名称')
|
||||
parser.add_argument('--result', type=str, help='步骤结果（JSON 字符串）')
|
||||
parser.add_argument('--record-error', action='store_true', help='记录错误')
|
||||
parser.add_argument('--error-msg', type=str, help='错误信息')
|
||||
parser.add_argument('--resume', action='store_true', help='获取恢复执行信息')
|
||||
|
||||
parser.add_argument('--extract_platform', type=str, help='要提取平台名称的文件名')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
base_path = Path(args.base_path) if args.base_path else None
|
||||
dirs = ensure_directories(base_path)
|
||||
|
||||
# 查重
|
||||
|
||||
# Handle process subcommand
|
||||
if args.command == 'process':
|
||||
result = process_command(
|
||||
url=args.url,
|
||||
output_dir=args.output_dir,
|
||||
skip_completed=args.skip_completed
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
# Legacy argument handling
|
||||
if args.check and args.url:
|
||||
result = check_duplicate(base_path, args.url)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
# 归档
|
||||
|
||||
elif args.archive_file:
|
||||
result = archive_and_git(args.archive_file, base_path)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
# 目录路径查询
|
||||
|
||||
elif args.pending:
|
||||
print(json.dumps({"path": str(dirs["pending"])}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.metadata:
|
||||
print(json.dumps({"path": str(dirs["metadata"])}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.content:
|
||||
print(json.dumps({"path": str(dirs["content"])}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.content_downloads:
|
||||
print(json.dumps({"path": str(dirs["content_downloads"])}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.analyzed:
|
||||
print(json.dumps({"path": str(dirs["analyzed"])}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.archive_dir:
|
||||
print(json.dumps({"path": str(get_archive_path())}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.archive_subscriptions:
|
||||
print(json.dumps({"path": str(dirs["archive_subscriptions"])}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.extract_platform:
|
||||
platform = extract_platform(args.extract_platform)
|
||||
print(json.dumps({"platform": platform}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.base_path:
|
||||
print(json.dumps({"path": str(dirs["base"])}, ensure_ascii=False, indent=2))
|
||||
|
||||
# 状态管理
|
||||
elif args.init_state and args.url:
|
||||
result = init_state(args.url, base_path)
|
||||
print(json.dumps({"success": True, "state": result}, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.get_state and args.url:
|
||||
result = get_state(args.url, base_path)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.advance_step and args.url and args.step:
|
||||
result_data = None
|
||||
if args.result:
|
||||
try:
|
||||
result_data = json.loads(args.result)
|
||||
except json.JSONDecodeError:
|
||||
result_data = {"raw": args.result}
|
||||
result = advance_step(args.url, args.step, result_data, base_path)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.record_error and args.url and args.error_msg:
|
||||
result = record_error(args.url, args.error_msg, args.step, base_path)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
elif args.resume and args.url:
|
||||
result = get_resume_info(args.url, base_path)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user