feat(download): add video download capability for videos without official subtitles
- Add download_video() to BilibiliAdapter using yt-dlp - Add bilibili_cookie config to Settings - Update DownloadProcessor to call video download for bilibili videos - Add idempotency check to ASRProcessor for existing subtitle files - Fix indentation error in asr.py Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,7 @@ Provides content extraction for Zhihu platform:
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
import re
|
||||
import asyncio
|
||||
|
||||
from kb_pipeline.platforms.base import PlatformAdapter
|
||||
from kb_pipeline.errors import AuthError
|
||||
@@ -146,12 +147,13 @@ class ZhihuAdapter(PlatformAdapter):
|
||||
|
||||
return False
|
||||
|
||||
async def fetch_metadata(self, content_id: str, content_type: str) -> Dict[str, Any]:
|
||||
async def fetch_metadata(self, content_id: str, content_type: str, source_url: str = None) -> Dict[str, Any]:
|
||||
"""Fetch metadata for the content.
|
||||
|
||||
Args:
|
||||
content_id: Answer ID or article ID
|
||||
content_type: 'answer' or 'zhuanlan'
|
||||
source_url: Original source URL (optional, used for answers to get question ID)
|
||||
|
||||
Returns:
|
||||
Dictionary containing metadata (title, author, publish_time, etc.)
|
||||
@@ -169,15 +171,19 @@ class ZhihuAdapter(PlatformAdapter):
|
||||
|
||||
# Build URL based on content type
|
||||
if content_type == "answer":
|
||||
# For answers, we need the question ID too, but extractor handles this
|
||||
url = f"https://www.zhihu.com/question/0/answer/{content_id}"
|
||||
# For answers, try to use source_url first, then fallback to constructed URL
|
||||
if source_url:
|
||||
url = source_url
|
||||
else:
|
||||
# Fallback: use answer ID only (may not work for all cases)
|
||||
url = f"https://www.zhihu.com/question/0/answer/{content_id}"
|
||||
elif content_type == "zhuanlan":
|
||||
url = f"https://zhuanlan.zhihu.com/p/{content_id}"
|
||||
else:
|
||||
raise ValueError(f"Unsupported content type: {content_type}")
|
||||
|
||||
# Extract content
|
||||
result = self._extract_with_playwright(url)
|
||||
# Extract content in a separate thread to avoid blocking asyncio
|
||||
result = await asyncio.to_thread(self._extract_with_playwright, url)
|
||||
|
||||
# Check for login page
|
||||
if self._detect_login_page(result):
|
||||
@@ -198,12 +204,13 @@ class ZhihuAdapter(PlatformAdapter):
|
||||
|
||||
return result
|
||||
|
||||
async def fetch_content(self, content_id: str, content_type: str) -> str:
|
||||
async def fetch_content(self, content_id: str, content_type: str, source_url: str = None) -> str:
|
||||
"""Fetch content in Markdown format.
|
||||
|
||||
Args:
|
||||
content_id: Answer ID or article ID
|
||||
content_type: 'answer' or 'zhuanlan'
|
||||
source_url: Original source URL (optional, used for answers)
|
||||
|
||||
Returns:
|
||||
Markdown-formatted content with YAML frontmatter
|
||||
@@ -212,7 +219,7 @@ class ZhihuAdapter(PlatformAdapter):
|
||||
AuthError: If cookie is missing or expired
|
||||
"""
|
||||
# Get metadata (which also validates auth)
|
||||
metadata = await self.fetch_metadata(content_id, content_type)
|
||||
metadata = await self.fetch_metadata(content_id, content_type, source_url=source_url)
|
||||
|
||||
# Get content markdown
|
||||
content_markdown = metadata.get("content_markdown", "") or metadata.get("content", "")
|
||||
|
||||
Reference in New Issue
Block a user