feat: implement obsidian-kb skill with long video chunking

- Multi-source content aggregation (Bilibili/YouTube/Zhihu)
- Smart inbox management with AI quality scoring
- Long video chunking analysis (30min+ auto-segmentation)
- Topic-based organization
- Git auto-sync to Obsidian
- Template system with Jinja2 and fallback rendering
- CLI commands: subscribe, inbox, process, skip, status, search
- Comprehensive test suite with unit and integration tests
- Documentation: README.md and SKILL.md

Checkpoint 7-8 completed by Agent E
This commit is contained in:
OpenClaw Agent
2026-02-06 18:15:30 +08:00
commit f26d4d8f7f
37 changed files with 6414 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""Tests for Obsidian KB Skill."""
+180
View File
@@ -0,0 +1,180 @@
"""
Bilibili Adapter 测试
"""
import sys
import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
from datetime import datetime, timedelta
from sources import SourceRegistry, BilibiliAdapter
from sources.bilibili import BilibiliAdapter as BiliAdapter
def test_registry():
"""测试注册表"""
print("=" * 50)
print("测试1: SourceRegistry 注册表")
print("=" * 50)
adapters = SourceRegistry.list()
print(f"已注册适配器: {adapters}")
adapter_class = SourceRegistry.get('bilibili')
assert adapter_class is not None, "bilibili适配器未注册"
print(f"✓ bilibili适配器类: {adapter_class}")
print()
def test_adapter_init():
"""测试适配器初始化"""
print("=" * 50)
print("测试2: 适配器初始化")
print("=" * 50)
adapter = BiliAdapter()
assert adapter is not None
print("✓ BilibiliAdapter实例化成功")
print()
def test_extract_bvid():
"""测试BV号提取"""
print("=" * 50)
print("测试3: BV号提取")
print("=" * 50)
adapter = BiliAdapter()
test_urls = [
("https://www.bilibili.com/video/BV1GJ411x7h7", "BV1GJ411x7h7"),
("https://b23.tv/BV1GJ411x7h7", "BV1GJ411x7h7"),
("BV1GJ411x7h7", "BV1GJ411x7h7"),
("https://www.bilibili.com/video/bv1gj411x7h7", "bv1gj411x7h7"),
]
for url, expected in test_urls:
result = adapter._extract_bvid_from_url(url)
if result and result.upper() == expected.upper():
print(f"✓ {url} -> {result}")
else:
print(f"✗ {url} -> {result} (期望: {expected})")
print()
def test_fetch_content():
"""测试获取视频内容(真实API调用)"""
print("=" * 50)
print("测试4: 从URL提取视频内容")
print("=" * 50)
adapter = BiliAdapter()
# 使用一个公开的视频链接测试
test_url = "https://www.bilibili.com/video/BV1GJ411x7h7"
try:
content = adapter.extract_content(test_url)
if content:
print(f"✓ 成功提取视频信息:")
print(f" - ID: {content.id}")
print(f" - 标题: {content.title[:50]}...")
print(f" - 作者: {content.author}")
print(f" - 时长: {content.duration}秒")
print(f" - URL: {content.url}")
else:
print(f"✗ 无法提取视频内容")
except Exception as e:
print(f"✗ 提取失败: {e}")
print()
def test_get_transcript():
"""测试获取字幕(真实API调用)"""
print("=" * 50)
print("测试5: 获取视频字幕")
print("=" * 50)
adapter = BiliAdapter()
# 找一个有字幕的视频
# BV1GJ411x7h7 是经典测试视频
test_url = "https://www.bilibili.com/video/BV1GJ411x7h7"
try:
content = adapter.extract_content(test_url)
if content:
transcript = adapter.get_transcript(content)
if transcript:
lines = transcript.split('\n')
print(f"✓ 成功获取字幕:")
print(f" - 总行数: {len(lines)}")
print(f" - 前200字符:\n{transcript[:200]}...")
else:
print(f"ℹ 该视频无字幕或字幕获取失败")
else:
print(f"✗ 无法获取视频内容")
except Exception as e:
print(f"✗ 获取字幕失败: {e}")
print()
def test_parse_duration():
"""测试时长解析"""
print("=" * 50)
print("测试6: 时长解析")
print("=" * 50)
adapter = BiliAdapter()
test_cases = [
("5:30", 330),
("1:23:45", 5025),
("0:45", 45),
("10:00", 600),
]
for duration_str, expected in test_cases:
result = adapter._parse_duration(duration_str)
if result == expected:
print(f"✓ {duration_str} -> {result}秒")
else:
print(f"✗ {duration_str} -> {result}秒 (期望: {expected})")
print()
def test_timestamp_conversion():
"""测试时间戳转换"""
print("=" * 50)
print("测试7: 时间戳转换")
print("=" * 50)
adapter = BiliAdapter()
# 测试秒级时间戳
ts = 1704067200 # 2024-01-01 00:00:00
dt = adapter._timestamp_to_datetime(ts)
print(f"✓ 秒级时间戳 {ts} -> {dt}")
# 测试毫秒级时间戳
ts_ms = 1704067200000
dt = adapter._timestamp_to_datetime(ts_ms)
print(f"✓ 毫秒级时间戳 {ts_ms} -> {dt}")
print()
if __name__ == "__main__":
print("\n" + "=" * 60)
print("Bilibili Adapter 测试套件")
print("=" * 60 + "\n")
test_registry()
test_adapter_init()
test_extract_bvid()
test_parse_duration()
test_timestamp_conversion()
test_fetch_content()
test_get_transcript()
print("\n" + "=" * 60)
print("测试完成")
print("=" * 60)
+149
View File
@@ -0,0 +1,149 @@
"""
Unit tests for content chunking module.
"""
import unittest
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / 'src'))
from content_chunker import ContentChunker, Chunk
class TestContentChunker(unittest.TestCase):
"""Test ContentChunker class."""
def setUp(self):
"""Set up test fixtures."""
self.chunker = ContentChunker(chunk_duration_seconds=900)
def test_chunker_initialization(self):
"""Test chunker initialization."""
self.assertEqual(self.chunker.chunk_duration, 900)
self.assertEqual(self.chunker.max_chunk_duration, 1800)
def test_parse_srt(self):
"""Test SRT parsing."""
srt_content = """1
00:00:01,000 --> 00:00:05,000
First subtitle
2
00:00:06,000 --> 00:00:10,000
Second subtitle
3
00:00:11,000 --> 00:00:15,000
Third subtitle"""
subtitles = self.chunker._parse_srt(srt_content)
self.assertEqual(len(subtitles), 3)
self.assertEqual(subtitles[0], (1.0, 5.0, "First subtitle"))
self.assertEqual(subtitles[1], (6.0, 10.0, "Second subtitle"))
self.assertEqual(subtitles[2], (11.0, 15.0, "Third subtitle"))
def test_time_to_seconds(self):
"""Test time conversion."""
self.assertEqual(self.chunker._time_to_seconds("00:00:30,000"), 30.0)
self.assertEqual(self.chunker._time_to_seconds("00:01:30,000"), 90.0)
self.assertEqual(self.chunker._time_to_seconds("01:00:00,000"), 3600.0)
self.assertEqual(self.chunker._time_to_seconds("01:30:45,500"), 5445.5)
def test_chunk_by_duration_short_content(self):
"""Test chunking short content (single chunk)."""
srt_content = """1
00:00:00,000 --> 00:00:10,000
This is the first line
2
00:00:10,000 --> 00:00:20,000
This is the second line"""
chunks = self.chunker.chunk_by_duration(srt_content, target_duration=900)
self.assertEqual(len(chunks), 1)
self.assertEqual(chunks[0].index, 0)
self.assertEqual(chunks[0].start_time, 0.0)
self.assertEqual(chunks[0].end_time, 20.0)
def test_chunk_by_duration_long_content(self):
"""Test chunking long content (multiple chunks)."""
# Create SRT content that spans 25 minutes
srt_lines = []
for i in range(25):
start = i * 60
end = (i + 1) * 60
srt_lines.append(f"{i+1}")
srt_lines.append(f"00:{i:02d}:00,000 --> 00:{i+1:02d}:00,000")
srt_lines.append(f"Minute {i+1} content")
srt_lines.append("")
srt_content = "\n".join(srt_lines)
# Use 10-minute chunks
chunks = self.chunker.chunk_by_duration(srt_content, target_duration=600)
self.assertGreater(len(chunks), 1)
# First chunk should start at 0
self.assertEqual(chunks[0].start_time, 0.0)
def test_chunk_time_range(self):
"""Test chunk time range formatting."""
chunk = Chunk(index=0, start_time=125.0, end_time=245.0, text="Test")
self.assertEqual(chunk.time_range, "02:05-04:05")
self.assertEqual(chunk.duration, 120.0)
def test_chunk_time_range_with_hours(self):
"""Test chunk time range with hours."""
chunk = Chunk(index=0, start_time=3665.0, end_time=7265.0, text="Test")
self.assertEqual(chunk.time_range, "01:01:05-02:01:05")
def test_get_chunking_strategy_short(self):
"""Test chunking strategy for short content."""
strategy = self.chunker.get_chunking_strategy(300) # 5 minutes
self.assertEqual(strategy['strategy'], 'direct')
self.assertEqual(strategy['chunk_count'], 1)
def test_get_chunking_strategy_medium(self):
"""Test chunking strategy for medium content."""
strategy = self.chunker.get_chunking_strategy(1200) # 20 minutes
self.assertEqual(strategy['strategy'], 'truncate')
self.assertEqual(strategy['chunk_count'], 1)
def test_get_chunking_strategy_long(self):
"""Test chunking strategy for long content."""
strategy = self.chunker.get_chunking_strategy(2400) # 40 minutes
self.assertEqual(strategy['strategy'], 'two_part')
self.assertEqual(strategy['chunk_count'], 2)
def test_get_chunking_strategy_very_long(self):
"""Test chunking strategy for very long content."""
strategy = self.chunker.get_chunking_strategy(7200) # 2 hours
self.assertEqual(strategy['strategy'], 'multi_part')
self.assertGreater(strategy['chunk_count'], 2)
class TestChunk(unittest.TestCase):
"""Test Chunk dataclass."""
def test_chunk_properties(self):
chunk = Chunk(
index=1,
start_time=100.0,
end_time=250.0,
text="Test content"
)
self.assertEqual(chunk.index, 1)
self.assertEqual(chunk.duration, 150.0)
if __name__ == '__main__':
unittest.main()
+144
View File
@@ -0,0 +1,144 @@
"""
Unit tests for config module.
"""
import unittest
import tempfile
import os
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).parent.parent / 'src'))
from config import Config
class TestConfig(unittest.TestCase):
"""Test Config class."""
def setUp(self):
"""Set up test fixtures."""
self.temp_dir = tempfile.mkdtemp()
self.config_path = Path(self.temp_dir) / "config.yaml"
# Save and clean up environment variables
self.env_backup = {}
for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']:
if key in os.environ:
self.env_backup[key] = os.environ[key]
del os.environ[key]
def tearDown(self):
"""Restore environment variables after each test."""
# Restore original environment variables
for key, value in self.env_backup.items():
os.environ[key] = value
# Clean up any that weren't backed up
for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']:
if key in os.environ and key not in self.env_backup:
del os.environ[key]
def test_default_config(self):
"""Test default configuration."""
config = Config(config_path=str(self.config_path), load_env=False)
self.assertEqual(config.get('ai.provider'), 'openai')
self.assertEqual(config.get('ai.model'), 'gpt-4')
self.assertTrue(config.get('chunking.enabled'))
self.assertEqual(config.get('chunking.chunk_duration_seconds'), 900)
def test_get_with_default(self):
"""Test get with default value."""
# Ensure clean environment
for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']:
if key in os.environ:
del os.environ[key]
config = Config(config_path=str(self.config_path), load_env=False)
# Existing key
self.assertEqual(config.get('ai.model'), 'gpt-4')
# Non-existing key with default
self.assertEqual(config.get('nonexistent.key', 'default'), 'default')
# Non-existing key without default
self.assertIsNone(config.get('nonexistent.key'))
def test_set_value(self):
"""Test setting configuration value."""
config = Config(config_path=str(self.config_path), load_env=False)
config.set('ai.model', 'gpt-3.5-turbo')
self.assertEqual(config.get('ai.model'), 'gpt-3.5-turbo')
config.set('custom.new_key', 'new_value')
self.assertEqual(config.get('custom.new_key'), 'new_value')
def test_save_and_load(self):
"""Test saving and loading configuration."""
# Create and save config
config1 = Config(config_path=str(self.config_path), load_env=False)
config1.set('ai.model', 'gpt-3.5-turbo')
config1.set('custom.test', 'value')
config1.save()
# Load config
config2 = Config(config_path=str(self.config_path), load_env=False)
self.assertEqual(config2.get('ai.model'), 'gpt-3.5-turbo')
self.assertEqual(config2.get('custom.test'), 'value')
def test_properties(self):
"""Test config properties."""
# Ensure clean environment
for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']:
if key in os.environ:
del os.environ[key]
config = Config(config_path=str(self.config_path), load_env=False)
self.assertIsNone(config.ai_api_key)
self.assertEqual(config.ai_model, 'gpt-4')
self.assertTrue(config.chunking_enabled)
self.assertEqual(config.chunk_duration, 900)
self.assertEqual(config.long_video_threshold, 1800)
def test_vault_path(self):
"""Test vault path property."""
config = Config(config_path=str(self.config_path), load_env=False)
path = config.vault_path
self.assertTrue(isinstance(path, Path))
def test_env_override_api_key(self):
"""Test environment variable override for API key."""
# Clean up environment first
for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']:
if key in os.environ:
del os.environ[key]
os.environ['OPENAI_API_KEY'] = 'test_api_key_123'
config = Config(config_path=str(self.config_path))
self.assertEqual(config.ai_api_key, 'test_api_key_123')
# Clean up
del os.environ['OPENAI_API_KEY']
def test_env_override_model(self):
"""Test environment variable override for model."""
# Clean up environment first
for key in ['OPENAI_API_KEY', 'AI_MODEL', 'OBSIDIAN_VAULT_PATH']:
if key in os.environ:
del os.environ[key]
os.environ['AI_MODEL'] = 'custom-model'
config = Config(config_path=str(self.config_path))
self.assertEqual(config.ai_model, 'custom-model')
# Clean up
del os.environ['AI_MODEL']
if __name__ == '__main__':
unittest.main()
+187
View File
@@ -0,0 +1,187 @@
"""
Unit tests for models module.
"""
import unittest
import json
from datetime import datetime
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / 'src'))
from models import Content, ContentType, ContentStatus, ContentAnalysis
from content_chunker import Chunk
class TestContentType(unittest.TestCase):
"""Test ContentType enum."""
def test_content_type_values(self):
self.assertEqual(ContentType.VIDEO.value, 'video')
self.assertEqual(ContentType.ARTICLE.value, 'article')
self.assertEqual(ContentType.AUDIO.value, 'audio')
class TestContentStatus(unittest.TestCase):
"""Test ContentStatus enum."""
def test_content_status_values(self):
self.assertEqual(ContentStatus.NEW.value, 'new')
self.assertEqual(ContentStatus.SCANNING.value, 'scanning')
self.assertEqual(ContentStatus.SCANNED.value, 'scanned')
self.assertEqual(ContentStatus.ANALYZING.value, 'analyzing')
self.assertEqual(ContentStatus.GENERATED.value, 'completed')
self.assertEqual(ContentStatus.FAILED.value, 'failed')
self.assertEqual(ContentStatus.SKIPPED.value, 'skipped')
class TestContent(unittest.TestCase):
"""Test Content model."""
def test_content_creation(self):
"""Test basic content creation."""
content = Content(
id="test_001",
source="bilibili",
title="Test Video",
url="https://bilibili.com/video/test"
)
self.assertEqual(content.id, "test_001")
self.assertEqual(content.source, "bilibili")
self.assertEqual(content.title, "Test Video")
self.assertEqual(content.url, "https://bilibili.com/video/test")
self.assertEqual(content.content_type, ContentType.VIDEO)
self.assertEqual(content.status, ContentStatus.NEW)
def test_content_with_all_fields(self):
"""Test content creation with all fields."""
analysis = ContentAnalysis(
one_liner="Test summary",
summary="Detailed summary",
key_points=["Point 1", "Point 2"],
tags=["test", "video"],
quality_score=85
)
content = Content(
id="test_002",
source="youtube",
title="Full Test Video",
url="https://youtube.com/watch?v=test",
author="Test Author",
duration=3600,
content_type=ContentType.VIDEO,
status=ContentStatus.GENERATED,
tags=["python", "tutorial"],
transcript="Test transcript",
analysis=analysis
)
self.assertEqual(content.author, "Test Author")
self.assertEqual(content.duration, 3600)
self.assertEqual(content.tags, ["python", "tutorial"])
self.assertEqual(content.analysis.quality_score, 85)
self.assertEqual(content.analysis.key_points, ["Point 1", "Point 2"])
def test_content_to_dict(self):
"""Test serialization to dictionary."""
content = Content(
id="test_003",
source="zhihu",
title="Test Article",
url="https://zhihu.com/question/test"
)
data = content.to_dict()
self.assertEqual(data['id'], "test_003")
self.assertEqual(data['source'], "zhihu")
self.assertEqual(data['content_type'], 'video')
self.assertEqual(data['status'], 'new')
def test_content_from_dict(self):
"""Test deserialization from dictionary."""
data = {
'id': 'test_004',
'source': 'bilibili',
'title': 'From Dict',
'url': 'https://example.com',
'content_type': 'article',
'status': 'scanned',
'tags': ['test', 'article'],
'duration': 600
}
content = Content.from_dict(data)
self.assertEqual(content.id, 'test_004')
self.assertEqual(content.content_type, ContentType.ARTICLE)
self.assertEqual(content.status, ContentStatus.SCANNED)
self.assertEqual(content.duration, 600)
def test_content_json_roundtrip(self):
"""Test JSON serialization roundtrip."""
original = Content(
id="test_005",
source="bilibili",
title="JSON Test",
url="https://example.com",
tags=["test", "json"]
)
json_str = original.to_json()
restored = Content.from_json(json_str)
self.assertEqual(restored.id, original.id)
self.assertEqual(restored.title, original.title)
self.assertEqual(restored.tags, original.tags)
self.assertEqual(restored.content_type, original.content_type)
class TestChunk(unittest.TestCase):
"""Test Chunk model."""
def test_chunk_creation(self):
chunk = Chunk(
index=0,
start_time=0.0,
end_time=300.0,
text="Test transcript"
)
self.assertEqual(chunk.index, 0)
self.assertEqual(chunk.start_time, 0.0)
self.assertEqual(chunk.end_time, 300.0)
class TestContentAnalysis(unittest.TestCase):
"""Test ContentAnalysis model."""
def test_analysis_defaults(self):
analysis = ContentAnalysis()
self.assertEqual(analysis.one_liner, "")
self.assertEqual(analysis.quality_score, 0)
self.assertEqual(analysis.key_points, [])
self.assertEqual(analysis.tags, [])
def test_analysis_full(self):
analysis = ContentAnalysis(
one_liner="Quick summary",
summary="Full summary",
key_points=["Point A", "Point B", "Point C"],
tags=["python", "tutorial", "advanced"],
quality_score=90,
prerequisites=["Basic Python"],
outline=[{"title": "Part 1", "summary": "Intro"}],
pitfalls=["Don't forget X"],
practice_advice=["Try Y first"]
)
self.assertEqual(analysis.quality_score, 90)
self.assertEqual(len(analysis.key_points), 3)
self.assertEqual(len(analysis.outline), 1)
if __name__ == '__main__':
unittest.main()
+331
View File
@@ -0,0 +1,331 @@
"""
Integration tests for the complete Obsidian KB pipeline.
Tests the full workflow: subscribe → fetch → analyze → generate → sync
"""
import unittest
import tempfile
import json
from pathlib import Path
from datetime import datetime
import sys
sys.path.insert(0, str(Path(__file__).parent.parent / 'src'))
from models import Content, ContentType, ContentStatus, ContentAnalysis
from storage import Storage
from template_renderer import TemplateRenderer
from config import Config
from content_chunker import ContentChunker
class TestPipeline(unittest.TestCase):
"""Integration tests for the complete pipeline."""
def setUp(self):
"""Set up test environment."""
self.temp_dir = tempfile.mkdtemp()
self.data_dir = Path(self.temp_dir) / "data"
self.vault_dir = Path(self.temp_dir) / "vault"
self.vault_dir.mkdir(parents=True, exist_ok=True)
# Initialize components
self.storage = Storage(base_path=str(self.data_dir))
self.config = Config(config_path=str(self.temp_dir / "config.yaml"))
self.config.set('git.vault_path', str(self.vault_dir))
templates_dir = Path(__file__).parent.parent / 'src' / 'templates'
self.renderer = TemplateRenderer(str(templates_dir))
self.chunker = ContentChunker()
def test_full_pipeline_short_video(self):
"""Test complete pipeline with short video."""
# 1. Create content (simulating subscription/fetch)
content = Content(
id="bvid_BV1test001",
source="bilibili",
title="Python Tips in 5 Minutes",
url="https://bilibili.com/video/BV1test001",
author="CodeMaster",
duration=300, # 5 minutes
content_type=ContentType.VIDEO,
status=ContentStatus.NEW,
tags=["python", "programming"]
)
# 2. Add to inbox
self.storage.add_to_inbox(content)
inbox = self.storage.get_inbox()
self.assertEqual(len(inbox), 1)
# 3. Simulate light scanning
content.status = ContentStatus.SCANNED
content.analysis = ContentAnalysis(
one_liner="Quick Python productivity tips",
summary="This video covers 5 essential Python tips for beginners.",
key_points=["Use list comprehensions", "Leverage built-in functions"],
tags=["python", "productivity"],
quality_score=85
)
self.storage.update_inbox_item(content)
# 4. Generate note
output_path = self.vault_dir / "inbox" / "python_tips.md"
result_path = self.renderer.render_to_file(content, str(output_path))
# 5. Verify output
self.assertTrue(Path(result_path).exists())
note_content = Path(result_path).read_text(encoding='utf-8')
# Verify content
self.assertIn("Python Tips in 5 Minutes", note_content)
self.assertIn("Quick Python productivity tips", note_content)
self.assertIn("#python", note_content)
# 6. Update status to completed
content.status = ContentStatus.GENERATED
content.note_path = str(result_path)
self.storage.update_inbox_item(content)
# 7. Verify final state
completed_item = self.storage.get_item_by_id(content.id)
self.assertEqual(completed_item.status, ContentStatus.GENERATED)
self.assertIsNotNone(completed_item.note_path)
def test_pipeline_long_video_with_chunking(self):
"""Test pipeline with long video requiring chunking."""
# 1. Create long video content
content = Content(
id="bvid_BV2long001",
source="bilibili",
title="Complete Python Course",
url="https://bilibili.com/video/BV2long001",
author="PythonAcademy",
duration=7200, # 2 hours
content_type=ContentType.VIDEO,
status=ContentStatus.NEW,
tags=["python", "tutorial", "course"]
)
# 2. Add to inbox
self.storage.add_to_inbox(content)
# 3. Determine chunking strategy
strategy = self.chunker.get_chunking_strategy(content.duration)
self.assertEqual(strategy['strategy'], 'multi_part')
self.assertGreater(strategy['chunk_count'], 2)
# 4. Simulate chunking analysis
# Create mock chunks
chunks = [
{"index": 0, "start": 0, "end": 1800, "summary": "Introduction to Python"},
{"index": 1, "start": 1800, "end": 3600, "summary": "Data structures"},
{"index": 2, "start": 3600, "end": 5400, "summary": "Functions and classes"},
{"index": 3, "start": 5400, "end": 7200, "summary": "Advanced topics"},
]
# 5. Simulate deep analysis with chunks
content.status = ContentStatus.GENERATED
content.analysis = ContentAnalysis(
one_liner="Comprehensive Python course from basics to advanced",
summary="A 2-hour course covering Python fundamentals and advanced concepts.",
key_points=["Python basics", "Data structures", "OOP", "Advanced patterns"],
tags=["python", "course", "programming"],
quality_score=95,
prerequisites=["Basic programming knowledge"],
outline=[
{"title": "Introduction", "time_range": "00:00-30:00", "summary": "Python basics"},
{"title": "Data Structures", "time_range": "30:00-60:00", "summary": "Lists, dicts, sets"},
{"title": "OOP", "time_range": "60:00-90:00", "summary": "Classes and objects"},
{"title": "Advanced", "time_range": "90:00-120:00", "summary": "Decorators, generators"},
],
pitfalls=["Python 2 vs 3 confusion", "Mutable default arguments"],
practice_advice=["Code along with examples", "Build projects"]
)
# 6. Generate tutorial note
output_path = self.vault_dir / "topics" / "programming" / "python_course.md"
result_path = self.renderer.render_to_file(content, str(output_path))
# 7. Verify tutorial template was used
note_content = Path(result_path).read_text(encoding='utf-8')
self.assertIn("Complete Python Course", note_content)
self.assertIn("Comprehensive Python course", note_content)
self.assertIn("前置知识", note_content) # Prerequisites section
self.assertIn("步骤详解", note_content) # Steps section
self.assertIn("常见坑点", note_content) # Pitfalls section
self.assertIn("实践建议", note_content) # Practice advice section
def test_pipeline_article(self):
"""Test pipeline with article content."""
# 1. Create article content
content = Content(
id="zhihu_article_001",
source="zhihu",
title="Understanding Neural Networks",
url="https://zhihu.com/question/123456",
author="AIResearcher",
content_type=ContentType.ARTICLE,
status=ContentStatus.NEW,
tags=["ai", "machine-learning"]
)
# 2. Add to inbox
self.storage.add_to_inbox(content)
# 3. Simulate analysis
content.status = ContentStatus.GENERATED
content.analysis = ContentAnalysis(
one_liner="Comprehensive guide to neural network fundamentals",
summary="Deep dive into how neural networks work.",
key_points=["Perceptrons", "Backpropagation", "Activation functions"],
tags=["ai", "neural-networks", "deep-learning"],
quality_score=90
)
# 4. Generate note
output_path = self.vault_dir / "inbox" / "neural_networks.md"
result_path = self.renderer.render_to_file(content, str(output_path))
# 5. Verify article template
note_content = Path(result_path).read_text(encoding='utf-8')
self.assertIn("Understanding Neural Networks", note_content)
self.assertIn("原文链接", note_content) # Original link section
def test_inbox_to_topic_workflow(self):
"""Test moving content from inbox to topics."""
# 1. Add content to inbox
content = Content(
id="bvid_BV3topic01",
source="bilibili",
title="React Tutorial",
url="https://bilibili.com/video/BV3topic01",
author="FrontendDev",
duration=1800,
tags=["react", "javascript", "frontend"]
)
content.analysis = ContentAnalysis(
one_liner="React fundamentals",
quality_score=88
)
self.storage.add_to_inbox(content)
# 2. Process and complete
content.status = ContentStatus.GENERATED
self.storage.update_inbox_item(content)
# 3. Move to topic (simulate)
topic_path = self.vault_dir / "topics" / "frontend" / "react_tutorial.md"
topic_path.parent.mkdir(parents=True, exist_ok=True)
result_path = self.renderer.render_to_file(content, str(topic_path))
content.note_path = str(result_path)
self.storage.update_inbox_item(content)
# 4. Verify
self.assertTrue(Path(result_path).exists())
self.assertIn("topics/frontend", content.note_path)
def test_skip_content(self):
"""Test skipping content."""
# 1. Add content
content = Content(
id="bvid_BV4skip001",
source="bilibili",
title="Low Quality Video",
url="https://bilibili.com/video/BV4skip001"
)
content.analysis = ContentAnalysis(quality_score=30) # Low quality
self.storage.add_to_inbox(content)
# 2. Skip it
content.status = ContentStatus.SKIPPED
self.storage.update_inbox_item(content)
# 3. Verify not in pending
pending = self.storage.get_pending_items()
skipped_ids = [c.id for c in self.storage.get_inbox() if c.status == ContentStatus.SKIPPED]
self.assertIn(content.id, skipped_ids)
self.assertNotIn(content, pending)
def test_batch_processing(self):
"""Test processing multiple items."""
# Add multiple items
contents = [
Content(id=f"batch_{i}", source="bilibili", title=f"Video {i}", url=f"https://example.com/{i}")
for i in range(5)
]
for content in contents:
self.storage.add_to_inbox(content)
# Process all pending
pending = self.storage.get_pending_items()
self.assertEqual(len(pending), 5)
for content in pending:
content.status = ContentStatus.GENERATED
content.analysis = ContentAnalysis(
one_liner=f"Summary for {content.title}",
quality_score=80
)
self.storage.update_inbox_item(content)
# Verify all processed
final_pending = self.storage.get_pending_items()
self.assertEqual(len(final_pending), 0)
class TestEndToEnd(unittest.TestCase):
"""End-to-end integration tests."""
def test_config_storage_renderer_integration(self):
"""Test integration of config, storage, and renderer."""
with tempfile.TemporaryDirectory() as tmpdir:
# Setup
data_dir = Path(tmpdir) / "data"
vault_dir = Path(tmpdir) / "vault"
config_path = Path(tmpdir) / "config.yaml"
config = Config(config_path=str(config_path))
config.set('git.vault_path', str(vault_dir))
config.set('chunking.chunk_duration_seconds', 600)
config.save()
storage = Storage(data_dir=str(data_dir))
templates_dir = Path(__file__).parent.parent / 'src' / 'templates'
renderer = TemplateRenderer(str(templates_dir))
# Create and process content
content = Content(
id="e2e_test_001",
source="bilibili",
title="E2E Test Video",
url="https://example.com/e2e",
duration=1200
)
content.analysis = ContentAnalysis(
one_liner="End to end test",
key_points=["Point 1", "Point 2"]
)
# Full workflow
storage.add_to_inbox(content)
output_path = Path(config.vault_path) / "e2e_output.md"
renderer.render_to_file(content, str(output_path))
content.status = ContentStatus.GENERATED
content.note_path = str(output_path)
storage.update_inbox_item(content)
# Verify
self.assertTrue(output_path.exists())
self.assertEqual(storage.get_item_by_id(content.id).status, ContentStatus.GENERATED)
if __name__ == '__main__':
unittest.main()
+146
View File
@@ -0,0 +1,146 @@
"""
Unit tests for storage module.
"""
import unittest
import tempfile
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).parent.parent / 'src'))
from storage import Storage
from models import Content, ContentType, ContentStatus
class TestStorage(unittest.TestCase):
"""Test Storage class."""
def setUp(self):
"""Set up test fixtures."""
self.temp_dir = tempfile.mkdtemp()
self.storage = Storage(base_path=self.temp_dir)
def test_initialization(self):
"""Test storage initialization."""
self.assertTrue(Path(self.temp_dir).exists())
self.assertTrue(Path(self.temp_dir / "topics").exists())
def test_get_empty_inbox(self):
"""Test getting empty inbox."""
inbox = self.storage.get_inbox()
self.assertEqual(inbox, [])
def test_add_to_inbox(self):
"""Test adding content to inbox."""
content = Content(
id="test_001",
source="bilibili",
title="Test Video",
url="https://example.com"
)
self.storage.add_to_inbox(content)
inbox = self.storage.get_inbox()
self.assertEqual(len(inbox), 1)
self.assertEqual(inbox[0].id, "test_001")
def test_add_duplicate(self):
"""Test adding duplicate content."""
content = Content(
id="test_002",
source="bilibili",
title="Test",
url="https://example.com"
)
self.storage.add_to_inbox(content)
self.storage.add_to_inbox(content) # Duplicate
inbox = self.storage.get_inbox()
self.assertEqual(len(inbox), 1)
def test_update_inbox_item(self):
"""Test updating inbox item."""
content = Content(
id="test_003",
source="bilibili",
title="Original Title",
url="https://example.com"
)
self.storage.add_to_inbox(content)
# Update content
content.title = "Updated Title"
content.status = ContentStatus.COMPLETED
self.storage.update_inbox_item(content)
# Retrieve and verify
inbox = self.storage.get_inbox()
self.assertEqual(inbox[0].title, "Updated Title")
self.assertEqual(inbox[0].status, ContentStatus.COMPLETED)
def test_remove_from_inbox(self):
"""Test removing content from inbox."""
content = Content(
id="test_004",
source="bilibili",
title="To Remove",
url="https://example.com"
)
self.storage.add_to_inbox(content)
self.storage.remove_from_inbox("test_004")
inbox = self.storage.get_inbox()
self.assertEqual(len(inbox), 0)
def test_get_pending_items(self):
"""Test getting pending items."""
# Add new item
new_content = Content(
id="test_005",
source="bilibili",
title="New",
url="https://example.com",
status=ContentStatus.NEW
)
self.storage.add_to_inbox(new_content)
# Add completed item
completed_content = Content(
id="test_006",
source="bilibili",
title="Completed",
url="https://example.com",
status=ContentStatus.COMPLETED
)
self.storage.add_to_inbox(completed_content)
pending = self.storage.get_pending_items()
self.assertEqual(len(pending), 1)
self.assertEqual(pending[0].id, "test_005")
def test_get_item_by_id(self):
"""Test getting item by ID."""
content = Content(
id="test_007",
source="bilibili",
title="Find Me",
url="https://example.com"
)
self.storage.add_to_inbox(content)
found = self.storage.get_item_by_id("test_007")
not_found = self.storage.get_item_by_id("nonexistent")
self.assertIsNotNone(found)
self.assertEqual(found.title, "Find Me")
self.assertIsNone(not_found)
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,257 @@
"""
Unit tests for template renderer module.
"""
import unittest
import tempfile
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).parent.parent / 'src'))
from template_renderer import TemplateRenderer, render_content
from models import Content, ContentType, ContentStatus, ContentAnalysis
class TestTemplateRenderer(unittest.TestCase):
"""Test TemplateRenderer class."""
def setUp(self):
"""Set up test fixtures."""
# Use the actual templates directory
self.templates_dir = Path(__file__).parent.parent / 'src' / 'templates'
self.renderer = TemplateRenderer(str(self.templates_dir))
def test_initialization(self):
"""Test renderer initialization."""
self.assertTrue(self.renderer.templates_dir.exists())
def test_get_template_name_video(self):
"""Test template selection for video content."""
content = Content(
id="test_001",
source="bilibili",
title="Test Video",
url="https://example.com",
duration=600, # 10 minutes
content_type=ContentType.VIDEO
)
template_name = self.renderer._get_template_name(content)
self.assertEqual(template_name, 'light_video.md')
def test_get_template_name_long_video(self):
"""Test template selection for long video."""
content = Content(
id="test_002",
source="bilibili",
title="Long Tutorial",
url="https://example.com",
duration=3600, # 60 minutes
content_type=ContentType.VIDEO
)
template_name = self.renderer._get_template_name(content)
self.assertEqual(template_name, 'tutorial_video.md')
def test_get_template_name_article(self):
"""Test template selection for article."""
content = Content(
id="test_003",
source="zhihu",
title="Test Article",
url="https://example.com",
content_type=ContentType.ARTICLE
)
template_name = self.renderer._get_template_name(content)
self.assertEqual(template_name, 'article.md')
def test_prepare_context(self):
"""Test context preparation."""
content = Content(
id="test_004",
source="bilibili",
title="Test Title",
url="https://example.com",
author="Test Author",
duration=300,
tags=["python", "tutorial"]
)
context = self.renderer._prepare_context(content)
self.assertEqual(context['title'], "Test Title")
self.assertEqual(context['source'], "bilibili")
self.assertEqual(context['author'], "Test Author")
self.assertEqual(context['tags'], ["python", "tutorial"])
def test_prepare_context_with_analysis(self):
"""Test context preparation with analysis data."""
analysis = ContentAnalysis(
one_liner="Quick summary",
prerequisites=["Basic Python"],
outline=[{"title": "Part 1", "summary": "Introduction"}],
pitfalls=["Common mistake"],
practice_advice=["Try this"]
)
content = Content(
id="test_005",
source="bilibili",
title="Test",
url="https://example.com",
analysis=analysis
)
context = self.renderer._prepare_context(content)
self.assertEqual(context['prerequisites'], ["Basic Python"])
self.assertEqual(context['outline'], [{"title": "Part 1", "summary": "Introduction"}])
def test_render_light_video(self):
"""Test rendering light video template."""
content = Content(
id="test_006",
source="bilibili",
title="Quick Tips",
url="https://example.com",
author="Creator",
duration=300,
tags=["tips", "productivity"]
)
content.analysis = ContentAnalysis(
one_liner="Great productivity tips",
summary="This video covers essential tips.",
key_points=["Point 1", "Point 2"],
quality_score=85
)
result = self.renderer.render(content)
self.assertIn("Quick Tips", result)
self.assertIn("Great productivity tips", result)
self.assertIn("Point 1", result)
self.assertIn("#tips", result)
def test_render_tutorial_video(self):
"""Test rendering tutorial video template."""
content = Content(
id="test_007",
source="bilibili",
title="Python Tutorial",
url="https://example.com",
author="PythonMaster",
duration=3600,
tags=["python", "programming"]
)
content.analysis = ContentAnalysis(
one_liner="Complete Python guide",
prerequisites=["Basic programming"],
outline=[{
"title": "Introduction",
"time_range": "00:00-05:00",
"summary": "Getting started"
}],
key_points=["Install Python", "Write first script"],
pitfalls=["Don't use Python 2"],
practice_advice=["Practice daily"],
quality_score=95
)
result = self.renderer.render(content)
self.assertIn("Python Tutorial", result)
self.assertIn("Complete Python guide", result)
self.assertIn("Basic programming", result)
def test_render_article(self):
"""Test rendering article template."""
content = Content(
id="test_008",
source="zhihu",
title="Deep Dive Article",
url="https://zhihu.com/article/test",
author="Writer",
content_type=ContentType.ARTICLE,
tags=["technology", "ai"]
)
content.analysis = ContentAnalysis(
one_liner="Comprehensive analysis",
summary="Detailed article content.",
key_points=["Insight 1", "Insight 2"],
quality_score=90
)
result = self.renderer.render(content)
self.assertIn("Deep Dive Article", result)
self.assertIn("Comprehensive analysis", result)
def test_render_to_file(self):
"""Test rendering to file."""
with tempfile.TemporaryDirectory() as tmpdir:
content = Content(
id="test_009",
source="bilibili",
title="File Test",
url="https://example.com",
duration=600
)
content.analysis = ContentAnalysis(
one_liner="Test one-liner",
key_points=["Point A"]
)
output_path = Path(tmpdir) / "output.md"
result_path = self.renderer.render_to_file(content, str(output_path))
self.assertTrue(Path(result_path).exists())
content_text = Path(result_path).read_text(encoding='utf-8')
self.assertIn("File Test", content_text)
def test_simple_render_fallback(self):
"""Test simple rendering fallback (no Jinja2)."""
template = "Title: {{ title }}, Author: {{ author }}"
context = {'title': 'Test', 'author': 'User'}
result = self.renderer._simple_render(template, context)
self.assertIn("Title: Test", result)
self.assertIn("Author: User", result)
def test_simple_render_with_list(self):
"""Test simple rendering with list values."""
template = "Tags: {{ tags }}"
context = {'tags': ['python', 'coding']}
result = self.renderer._simple_render(template, context)
self.assertIn("Tags: python, coding", result)
class TestRenderContentFunction(unittest.TestCase):
"""Test render_content convenience function."""
def test_render_content(self):
"""Test convenience function."""
templates_dir = Path(__file__).parent.parent / 'src' / 'templates'
content = Content(
id="test_010",
source="bilibili",
title="Convenience Test",
url="https://example.com",
duration=300
)
content.analysis = ContentAnalysis(
one_liner="Simple test",
key_points=["Test point"]
)
result = render_content(content, str(templates_dir))
self.assertIn("Convenience Test", result)
self.assertIn("Simple test", result)
if __name__ == '__main__':
unittest.main()