feat(download): add video download capability for videos without official subtitles

- Add download_video() to BilibiliAdapter using yt-dlp
- Add bilibili_cookie config to Settings
- Update DownloadProcessor to call video download for bilibili videos
- Add idempotency check to ASRProcessor for existing subtitle files
- Fix indentation error in asr.py

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-16 00:06:31 +08:00
parent 07a9db85f7
commit 0b215dabe7
5 changed files with 221 additions and 13 deletions
+14 -7
View File
@@ -10,6 +10,7 @@ Provides content extraction for Zhihu platform:
from typing import Dict, Any, Optional
from datetime import datetime
import re
import asyncio
from kb_pipeline.platforms.base import PlatformAdapter
from kb_pipeline.errors import AuthError
@@ -146,12 +147,13 @@ class ZhihuAdapter(PlatformAdapter):
return False
async def fetch_metadata(self, content_id: str, content_type: str) -> Dict[str, Any]:
async def fetch_metadata(self, content_id: str, content_type: str, source_url: str = None) -> Dict[str, Any]:
"""Fetch metadata for the content.
Args:
content_id: Answer ID or article ID
content_type: 'answer' or 'zhuanlan'
source_url: Original source URL (optional, used for answers to get question ID)
Returns:
Dictionary containing metadata (title, author, publish_time, etc.)
@@ -169,15 +171,19 @@ class ZhihuAdapter(PlatformAdapter):
# Build URL based on content type
if content_type == "answer":
# For answers, we need the question ID too, but extractor handles this
url = f"https://www.zhihu.com/question/0/answer/{content_id}"
# For answers, try to use source_url first, then fallback to constructed URL
if source_url:
url = source_url
else:
# Fallback: use answer ID only (may not work for all cases)
url = f"https://www.zhihu.com/question/0/answer/{content_id}"
elif content_type == "zhuanlan":
url = f"https://zhuanlan.zhihu.com/p/{content_id}"
else:
raise ValueError(f"Unsupported content type: {content_type}")
# Extract content
result = self._extract_with_playwright(url)
# Extract content in a separate thread to avoid blocking asyncio
result = await asyncio.to_thread(self._extract_with_playwright, url)
# Check for login page
if self._detect_login_page(result):
@@ -198,12 +204,13 @@ class ZhihuAdapter(PlatformAdapter):
return result
async def fetch_content(self, content_id: str, content_type: str) -> str:
async def fetch_content(self, content_id: str, content_type: str, source_url: str = None) -> str:
"""Fetch content in Markdown format.
Args:
content_id: Answer ID or article ID
content_type: 'answer' or 'zhuanlan'
source_url: Original source URL (optional, used for answers)
Returns:
Markdown-formatted content with YAML frontmatter
@@ -212,7 +219,7 @@ class ZhihuAdapter(PlatformAdapter):
AuthError: If cookie is missing or expired
"""
# Get metadata (which also validates auth)
metadata = await self.fetch_metadata(content_id, content_type)
metadata = await self.fetch_metadata(content_id, content_type, source_url=source_url)
# Get content markdown
content_markdown = metadata.get("content_markdown", "") or metadata.get("content", "")