feat: integrate bilibili-extractor into obsidian-kb
- Add extractors layer with BilibiliExtractor (Playwright stealth mode) - Add outputs layer with ObsidianWriter and GitSync - Implement subtitle/no-subtitle graceful degradation - Update config with output and bilibili extraction settings - Complete integration pipeline: extract → process → output → sync - Add comprehensive tests (100% pass rate) - Update documentation (SKILL.md, README.md, requirements.txt)
This commit is contained in:
OpenClaw Agent
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Obsidian KB - 主流程集成验证脚本
|
||||
|
||||
验证代码结构和集成逻辑，不执行实际提取操作
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
# 添加项目路径
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
def verify_file_exists(filepath, description):
|
||||
"""验证文件是否存在"""
|
||||
path = Path(filepath)
|
||||
if path.exists():
|
||||
print(f"✅ {description}: {filepath}")
|
||||
return True
|
||||
else:
|
||||
print(f"❌ {description} 不存在: {filepath}")
|
||||
return False
|
||||
|
||||
|
||||
def verify_imports(filepath, required_imports):
|
||||
"""验证文件中是否包含指定的导入"""
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
tree = ast.parse(content)
|
||||
|
||||
found_imports = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
found_imports.add(alias.name)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module:
|
||||
for alias in node.names:
|
||||
found_imports.add(f"{node.module}.{alias.name}")
|
||||
|
||||
all_found = True
|
||||
for imp in required_imports:
|
||||
if any(imp in found for found in found_imports):
|
||||
print(f" ✅ 导入: {imp}")
|
||||
else:
|
||||
print(f" ❌ 缺少导入: {imp}")
|
||||
all_found = False
|
||||
|
||||
return all_found
|
||||
|
||||
|
||||
def verify_methods(filepath, required_methods):
|
||||
"""验证文件中是否包含指定的方法"""
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
tree = ast.parse(content)
|
||||
|
||||
found_methods = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.FunctionDef):
|
||||
found_methods.add(node.name)
|
||||
elif isinstance(node, ast.ClassDef):
|
||||
for item in node.body:
|
||||
if isinstance(item, ast.FunctionDef):
|
||||
found_methods.add(f"{node.name}.{item.name}")
|
||||
|
||||
all_found = True
|
||||
for method in required_methods:
|
||||
if method in found_methods or any(method in found for found in found_methods):
|
||||
print(f" ✅ 方法: {method}")
|
||||
else:
|
||||
print(f" ❌ 缺少方法: {method}")
|
||||
all_found = False
|
||||
|
||||
return all_found
|
||||
|
||||
|
||||
def verify_integration():
|
||||
"""验证主流程集成"""
|
||||
print("\n" + "=" * 80)
|
||||
print("Obsidian KB - 主流程集成验证")
|
||||
print("=" * 80 + "\n")
|
||||
|
||||
results = {}
|
||||
|
||||
# 验证文件存在
|
||||
print("1. 验证文件存在性")
|
||||
print("-" * 80)
|
||||
files = [
|
||||
('main_pipeline.py', '主流程集成文件'),
|
||||
('test_integration.py', '集成测试脚本'),
|
||||
('handler.py', 'Handler入口'),
|
||||
('src/extractors/bilibili_extractor.py', '提取层'),
|
||||
('src/deep_analyzer.py', '处理层-分析器'),
|
||||
('src/content_chunker.py', '处理层-分片器'),
|
||||
('src/outputs/obsidian_writer.py', '输出层-写入器'),
|
||||
('src/outputs/git_sync.py', '输出层-同步器'),
|
||||
]
|
||||
|
||||
all_files_exist = True
|
||||
for filepath, description in files:
|
||||
if not verify_file_exists(filepath, description):
|
||||
all_files_exist = False
|
||||
results['files_exist'] = all_files_exist
|
||||
|
||||
# 验证主流程导入
|
||||
print("\n2. 验证主流程导入 (main_pipeline.py)")
|
||||
print("-" * 80)
|
||||
required_imports = [
|
||||
'extractors.bilibili_extractor',
|
||||
'deep_analyzer',
|
||||
'content_chunker',
|
||||
'outputs.obsidian_writer',
|
||||
'outputs.git_sync',
|
||||
]
|
||||
results['main_pipeline_imports'] = verify_imports('main_pipeline.py', required_imports)
|
||||
|
||||
# 验证主流程方法
|
||||
print("\n3. 验证主流程方法 (main_pipeline.py)")
|
||||
print("-" * 80)
|
||||
required_methods = [
|
||||
'MainPipeline.__init__',
|
||||
'MainPipeline.process_video',
|
||||
'MainPipeline._extract_video',
|
||||
'MainPipeline._analyze_content',
|
||||
'MainPipeline._chunk_content',
|
||||
'MainPipeline._write_obsidian_note',
|
||||
'MainPipeline._sync_to_git',
|
||||
]
|
||||
results['main_pipeline_methods'] = verify_methods('main_pipeline.py', required_methods)
|
||||
|
||||
# 验证测试脚本导入
|
||||
print("\n4. 验证测试脚本导入 (test_integration.py)")
|
||||
print("-" * 80)
|
||||
required_imports = [
|
||||
'main_pipeline',
|
||||
'tempfile',
|
||||
'shutil',
|
||||
]
|
||||
results['test_imports'] = verify_imports('test_integration.py', required_imports)
|
||||
|
||||
# 验证测试脚本方法
|
||||
print("\n5. 验证测试脚本方法 (test_integration.py)")
|
||||
print("-" * 80)
|
||||
required_methods = [
|
||||
'test_extract_only',
|
||||
'test_full_pipeline',
|
||||
'test_error_handling',
|
||||
'cleanup_test_env',
|
||||
]
|
||||
results['test_methods'] = verify_methods('test_integration.py', required_methods)
|
||||
|
||||
# 验证 handler.py 集成
|
||||
print("\n6. 验证 Handler 集成 (handler.py)")
|
||||
print("-" * 80)
|
||||
required_methods = [
|
||||
'process_video_full',
|
||||
]
|
||||
results['handler_integration'] = verify_methods('handler.py', required_methods)
|
||||
|
||||
# 验证错误处理
|
||||
print("\n7. 验证错误处理 (main_pipeline.py)")
|
||||
print("-" * 80)
|
||||
with open('main_pipeline.py', 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
error_handling_checks = [
|
||||
('try:', 'try 块'),
|
||||
('except', 'except 块'),
|
||||
('ExtractionError', 'ExtractionError 捕获'),
|
||||
('logger.error', '日志错误记录'),
|
||||
('logger.info', '日志信息记录'),
|
||||
]
|
||||
|
||||
error_handling_ok = True
|
||||
for check, description in error_handling_checks:
|
||||
if check in content:
|
||||
print(f" ✅ {description}")
|
||||
else:
|
||||
print(f" ❌ 缺少 {description}")
|
||||
error_handling_ok = False
|
||||
results['error_handling'] = error_handling_ok
|
||||
|
||||
# 验证日志配置
|
||||
print("\n8. 验证日志配置 (main_pipeline.py)")
|
||||
print("-" * 80)
|
||||
logging_checks = [
|
||||
('import logging', 'logging 模块导入'),
|
||||
('logging.basicConfig', 'logging 基本配置'),
|
||||
('FileHandler', '文件日志处理器'),
|
||||
('obsidian_kb.log', '日志文件'),
|
||||
]
|
||||
|
||||
logging_ok = True
|
||||
for check, description in logging_checks:
|
||||
if check in content:
|
||||
print(f" ✅ {description}")
|
||||
else:
|
||||
print(f" ❌ 缺少 {description}")
|
||||
logging_ok = False
|
||||
results['logging'] = logging_ok
|
||||
|
||||
# 汇总结果
|
||||
print("\n" + "=" * 80)
|
||||
print("验证结果汇总")
|
||||
print("=" * 80 + "\n")
|
||||
|
||||
checks = [
|
||||
('文件存在性', 'files_exist'),
|
||||
('主流程导入', 'main_pipeline_imports'),
|
||||
('主流程方法', 'main_pipeline_methods'),
|
||||
('测试脚本导入', 'test_imports'),
|
||||
('测试脚本方法', 'test_methods'),
|
||||
('Handler 集成', 'handler_integration'),
|
||||
('错误处理', 'error_handling'),
|
||||
('日志配置', 'logging'),
|
||||
]
|
||||
|
||||
for description, key in checks:
|
||||
status = '✅ 通过' if results.get(key, False) else '❌ 失败'
|
||||
print(f" {description:20s} {status}")
|
||||
|
||||
all_passed = all(results.values())
|
||||
print(f"\n 总体结果: {'✅ 全部通过' if all_passed else '❌ 有验证失败'}")
|
||||
|
||||
return 0 if all_passed else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(verify_integration())
|
||||
Reference in New Issue
Block a user