feat(06-02): move processors and platforms to kb_workflow.pipeline
- Move all processor implementations (parse, download, asr, analyze, archive) - Move all platform adapters (base, bilibili, zhihu) - Update all internal imports to use kb_workflow.pipeline prefix Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
"""Platform Adapters for KB Pipeline
|
||||
|
||||
This module provides platform-specific adapters for content extraction
|
||||
from various platforms (Bilibili, Zhihu, etc.).
|
||||
|
||||
Each adapter implements the PlatformAdapter interface:
|
||||
- get_content_type: Identify content type from URL
|
||||
- fetch_metadata: Fetch content metadata
|
||||
- fetch_content: Fetch content body in Markdown format
|
||||
- get_output_filename: Generate output filename
|
||||
"""
|
||||
|
||||
from kb_workflow.pipeline.platforms.base import PlatformAdapter
|
||||
from kb_workflow.pipeline.platforms.bilibili import BilibiliAdapter
|
||||
from kb_workflow.pipeline.platforms.zhihu import ZhihuAdapter
|
||||
|
||||
__all__ = [
|
||||
"PlatformAdapter",
|
||||
"BilibiliAdapter",
|
||||
"ZhihuAdapter",
|
||||
]
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Platform Adapter Base Class
|
||||
|
||||
Provides an abstract base class for platform-specific content adapters.
|
||||
Each platform (Bilibili, Zhihu, etc.) should implement a concrete adapter
|
||||
that handles metadata extraction and content fetching.
|
||||
|
||||
Purpose: Define a unified interface for multi-platform content extraction
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
class PlatformAdapter(ABC):
|
||||
"""Abstract base class for platform-specific content adapters
|
||||
|
||||
A platform adapter encapsulates the logic for:
|
||||
- Identifying content type from URL (video, article, answer, etc.)
|
||||
- Fetching metadata for a given content ID
|
||||
- Fetching content body in Markdown format
|
||||
- Generating output filenames
|
||||
|
||||
Subclasses must implement all abstract methods to provide
|
||||
platform-specific functionality.
|
||||
|
||||
Example:
|
||||
class BilibiliAdapter(PlatformAdapter):
|
||||
def get_content_type(self, url: str) -> str:
|
||||
if '/video/' in url:
|
||||
return 'video'
|
||||
elif '/read/' in url:
|
||||
return 'article'
|
||||
return 'unknown'
|
||||
|
||||
async def fetch_metadata(self, content_id: str, content_type: str) -> Dict[str, Any]:
|
||||
# Platform-specific implementation
|
||||
...
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_content_type(self, url: str) -> str:
|
||||
"""Identify content type from URL
|
||||
|
||||
Args:
|
||||
url: The content URL to analyze
|
||||
|
||||
Returns:
|
||||
Content type string (e.g., "video", "article", "answer", "zhuanlan")
|
||||
|
||||
Example:
|
||||
adapter.get_content_type("https://www.bilibili.com/video/BV1xx...")
|
||||
-> "video"
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def fetch_metadata(self, content_id: str, content_type: str) -> Dict[str, Any]:
|
||||
"""Fetch metadata for the given content
|
||||
|
||||
Args:
|
||||
content_id: Platform-specific content identifier (e.g., BV号, CV号)
|
||||
content_type: Type of content ("video", "article", etc.)
|
||||
|
||||
Returns:
|
||||
Dictionary containing metadata fields. Structure varies by platform
|
||||
and content type, but typically includes:
|
||||
- title: Content title
|
||||
- author: Author name
|
||||
- url: Original content URL
|
||||
- Platform-specific fields
|
||||
|
||||
Raises:
|
||||
NetworkError: If network request fails
|
||||
AuthError: If authentication fails or quota exceeded
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def fetch_content(self, content_id: str, content_type: str) -> str:
|
||||
"""Fetch content body in Markdown format
|
||||
|
||||
Args:
|
||||
content_id: Platform-specific content identifier
|
||||
content_type: Type of content
|
||||
|
||||
Returns:
|
||||
Markdown-formatted content string with YAML frontmatter:
|
||||
---
|
||||
title: Article Title
|
||||
author: Author Name
|
||||
source_url: https://...
|
||||
platform: bilibili
|
||||
content_type: article
|
||||
fetched_at: 2024-01-15T10:30:00
|
||||
---
|
||||
|
||||
# Article Content...
|
||||
|
||||
Note:
|
||||
For video content, this may return an empty string or
|
||||
placeholder text, as video content extraction is handled
|
||||
separately (e.g., ASR transcription).
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str:
|
||||
"""Generate output filename for content
|
||||
|
||||
Args:
|
||||
content_id: Platform-specific content identifier
|
||||
content_type: Type of content
|
||||
suffix: Filename suffix (e.g., "metadata.json", "content.md")
|
||||
|
||||
Returns:
|
||||
Filename string without directory path
|
||||
|
||||
Example:
|
||||
adapter.get_output_filename("BV1xx...", "video", "metadata.json")
|
||||
-> "bilibili_BV1xx..._metadata.json"
|
||||
"""
|
||||
pass
|
||||
@@ -0,0 +1,485 @@
|
||||
"""Bilibili Platform Adapter
|
||||
|
||||
Implements the PlatformAdapter interface for Bilibili platform content.
|
||||
Supports video and article metadata extraction and content fetching.
|
||||
|
||||
Key features:
|
||||
- Video metadata extraction (BV号)
|
||||
- Article metadata extraction (CV号)
|
||||
- Video download using yt-dlp (for ASR/Whisper)
|
||||
- Automatic retry on network errors (3 attempts with exponential backoff)
|
||||
- State recording for failed steps
|
||||
- Fallback from API to Playwright for 403 errors (handled by BilibiliExtractor)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from kb_workflow.pipeline.platforms.base import PlatformAdapter
|
||||
from kb_workflow.pipeline.errors import NetworkError, AuthError
|
||||
from kb_workflow.pipeline.retry import retry_on_network_error
|
||||
from kb_workflow.pipeline.state import WorkflowStateManager
|
||||
from kb_workflow.metadata.bilibili_extractor import BilibiliExtractor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BilibiliAdapter(PlatformAdapter):
|
||||
"""Bilibili platform adapter for video and article content
|
||||
|
||||
Implements the PlatformAdapter interface to provide:
|
||||
- Content type identification from URL
|
||||
- Video metadata extraction
|
||||
- Article metadata extraction
|
||||
- Article content fetching in Markdown format
|
||||
- Automatic error retry and state recording
|
||||
|
||||
Attributes:
|
||||
PLATFORM: Platform identifier string ("bilibili")
|
||||
_extractor: BilibiliExtractor instance for actual extraction
|
||||
_state_manager: Optional WorkflowStateManager for recording step status
|
||||
|
||||
Example:
|
||||
adapter = BilibiliAdapter()
|
||||
content_type = adapter.get_content_type("https://www.bilibili.com/video/BV1xx...")
|
||||
# -> "video"
|
||||
|
||||
metadata = await adapter.fetch_metadata("BV1xx...", "video")
|
||||
# -> {"metadata": {...}, "stats": {...}, ...}
|
||||
"""
|
||||
|
||||
PLATFORM = "bilibili"
|
||||
|
||||
# URL patterns for content type identification
|
||||
VIDEO_PATTERN = re.compile(r'bilibili\.com/video/(BV[a-zA-Z0-9]+)')
|
||||
ARTICLE_PATTERN = re.compile(r'bilibili\.com/read/(cv\d+)')
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
headless: bool = True,
|
||||
timeout: int = 30,
|
||||
state_manager: Optional[WorkflowStateManager] = None
|
||||
):
|
||||
"""Initialize BilibiliAdapter
|
||||
|
||||
Args:
|
||||
headless: Whether to use headless mode for Playwright (default True)
|
||||
timeout: Request timeout in seconds (default 30)
|
||||
state_manager: Optional WorkflowStateManager for recording step status
|
||||
"""
|
||||
self._extractor = BilibiliExtractor(headless=headless, timeout=timeout)
|
||||
self._state_manager = state_manager
|
||||
|
||||
def get_content_type(self, url: str) -> str:
|
||||
"""Identify content type from URL
|
||||
|
||||
Args:
|
||||
url: Bilibili URL to analyze
|
||||
|
||||
Returns:
|
||||
Content type string: "video", "article", or "unknown"
|
||||
|
||||
Example:
|
||||
adapter.get_content_type("https://www.bilibili.com/video/BV1xx...")
|
||||
-> "video"
|
||||
"""
|
||||
if self.VIDEO_PATTERN.search(url):
|
||||
return "video"
|
||||
if self.ARTICLE_PATTERN.search(url):
|
||||
return "article"
|
||||
return "unknown"
|
||||
|
||||
async def fetch_metadata(
|
||||
self,
|
||||
content_id: str,
|
||||
content_type: str
|
||||
) -> Dict[str, Any]:
|
||||
"""Fetch metadata for the given content
|
||||
|
||||
Args:
|
||||
content_id: BV号 for videos, CV号 for articles
|
||||
content_type: "video" or "article"
|
||||
|
||||
Returns:
|
||||
Dictionary containing metadata. Structure varies by content type:
|
||||
- Video: {metadata: {...}, stats: {...}, description: str}
|
||||
- Article: {cvid, title, author, publish_time, ...}
|
||||
|
||||
Raises:
|
||||
NetworkError: If network request fails after 3 retries
|
||||
AuthError: If authentication fails (403/quota exceeded)
|
||||
ValueError: If content_type is not supported
|
||||
"""
|
||||
try:
|
||||
return await self._fetch_metadata_with_retry(content_id, content_type)
|
||||
except (NetworkError, AuthError) as e:
|
||||
# Record failure to state manager only after all retries exhausted
|
||||
self._record_step_failed("metadata", str(e))
|
||||
raise
|
||||
except Exception as e:
|
||||
# Check if this is an API error that should be treated as network error
|
||||
if self._is_network_related_error(e):
|
||||
network_error = NetworkError(
|
||||
step_name="metadata",
|
||||
input_summary=f"content_id={content_id}, type={content_type}",
|
||||
cause=e
|
||||
)
|
||||
self._record_step_failed("metadata", str(network_error))
|
||||
raise network_error
|
||||
raise
|
||||
|
||||
@retry_on_network_error
|
||||
async def _fetch_metadata_with_retry(
|
||||
self,
|
||||
content_id: str,
|
||||
content_type: str
|
||||
) -> Dict[str, Any]:
|
||||
"""Internal method with retry logic for metadata fetching
|
||||
|
||||
Args:
|
||||
content_id: BV号 for videos, CV号 for articles
|
||||
content_type: "video" or "article"
|
||||
|
||||
Returns:
|
||||
Dictionary containing metadata
|
||||
|
||||
Raises:
|
||||
NetworkError: If network request fails
|
||||
AuthError: If authentication fails
|
||||
ValueError: If content_type is not supported
|
||||
"""
|
||||
if content_type == "video":
|
||||
# Call async method directly to avoid nested asyncio.run()
|
||||
return await self._extractor._extract_video_async(content_id)
|
||||
elif content_type == "article":
|
||||
# Call async method directly to avoid nested asyncio.run()
|
||||
return await self._extractor._extract_article_async(content_id)
|
||||
else:
|
||||
raise ValueError(f"Unsupported content type: {content_type}")
|
||||
|
||||
def _is_network_related_error(self, error: Exception) -> bool:
|
||||
"""Check if an error is network-related and should trigger retry
|
||||
|
||||
Args:
|
||||
error: Exception to check
|
||||
|
||||
Returns:
|
||||
True if error is network-related
|
||||
"""
|
||||
error_str = str(error).lower()
|
||||
network_indicators = [
|
||||
'timeout', 'connection', 'network', '503', '502', '500',
|
||||
'connection refused', 'connection reset', 'eof',
|
||||
]
|
||||
return any(indicator in error_str for indicator in network_indicators)
|
||||
|
||||
async def fetch_content(
|
||||
self,
|
||||
content_id: str,
|
||||
content_type: str
|
||||
) -> str:
|
||||
"""Fetch content body in Markdown format
|
||||
|
||||
For videos, this returns an empty string as video content
|
||||
extraction is handled separately (ASR transcription).
|
||||
|
||||
For articles, this fetches the article content and formats
|
||||
it with YAML frontmatter.
|
||||
|
||||
Args:
|
||||
content_id: CV号 for articles
|
||||
content_type: "video" or "article"
|
||||
|
||||
Returns:
|
||||
Markdown-formatted content string with YAML frontmatter,
|
||||
or empty string for videos
|
||||
"""
|
||||
if content_type == "video":
|
||||
# Video content is handled by ASR in Phase 3
|
||||
return ""
|
||||
|
||||
if content_type == "article":
|
||||
# Fetch article metadata first
|
||||
metadata = await self.fetch_metadata(content_id, "article")
|
||||
|
||||
# Fetch article content (requires additional implementation)
|
||||
content = await self._fetch_article_content(content_id)
|
||||
|
||||
# Format with YAML frontmatter
|
||||
return self._format_content_markdown(metadata, content)
|
||||
|
||||
raise ValueError(f"Unsupported content type: {content_type}")
|
||||
|
||||
async def _fetch_article_content(self, cvid: str) -> str:
|
||||
"""Fetch article body content
|
||||
|
||||
Args:
|
||||
cvid: Article CV号
|
||||
|
||||
Returns:
|
||||
Raw article content string
|
||||
|
||||
Note:
|
||||
This method currently returns a placeholder.
|
||||
Full implementation would use additional API calls
|
||||
or Playwright to fetch the article body.
|
||||
"""
|
||||
# TODO: Implement full article content fetching
|
||||
# This would require additional API endpoints or Playwright
|
||||
logger.warning(f"Article content fetching not fully implemented for {cvid}")
|
||||
return ""
|
||||
|
||||
def _format_content_markdown(
|
||||
self,
|
||||
metadata: Dict[str, Any],
|
||||
content: str,
|
||||
content_type: str = "article"
|
||||
) -> str:
|
||||
"""Format content with YAML frontmatter
|
||||
|
||||
Args:
|
||||
metadata: Article metadata dictionary
|
||||
content: Article body content
|
||||
content_type: Content type string
|
||||
|
||||
Returns:
|
||||
Markdown string with YAML frontmatter
|
||||
"""
|
||||
import yaml
|
||||
from datetime import datetime
|
||||
|
||||
frontmatter = {
|
||||
"title": metadata.get("title", ""),
|
||||
"author": metadata.get("author", ""),
|
||||
"source_url": metadata.get("url", ""),
|
||||
"platform": self.PLATFORM,
|
||||
"content_type": content_type,
|
||||
"fetched_at": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
# Add additional metadata if available
|
||||
if "cvid" in metadata:
|
||||
frontmatter["cvid"] = metadata["cvid"]
|
||||
if "mid" in metadata:
|
||||
frontmatter["author_id"] = metadata["mid"]
|
||||
if "word_count" in metadata:
|
||||
frontmatter["word_count"] = metadata["word_count"]
|
||||
|
||||
yaml_str = yaml.dump(frontmatter, allow_unicode=True, sort_keys=False)
|
||||
return f"---\n{yaml_str}---\n\n{content}"
|
||||
|
||||
def get_output_filename(
|
||||
self,
|
||||
content_id: str,
|
||||
content_type: str,
|
||||
suffix: str
|
||||
) -> str:
|
||||
"""Generate output filename for content
|
||||
|
||||
Args:
|
||||
content_id: BV号 or CV号
|
||||
content_type: "video" or "article"
|
||||
suffix: Filename suffix (e.g., "metadata.json", "content.md")
|
||||
|
||||
Returns:
|
||||
Filename string in format "bilibili_{content_id}_{suffix}"
|
||||
|
||||
Example:
|
||||
adapter.get_output_filename("BV1xx...", "video", "metadata.json")
|
||||
-> "bilibili_BV1xx..._metadata.json"
|
||||
"""
|
||||
return f"{self.PLATFORM}_{content_id}_{suffix}"
|
||||
|
||||
def _record_step_failed(self, step_name: str, error_message: str) -> None:
|
||||
"""Record step failure to state manager
|
||||
|
||||
Args:
|
||||
step_name: Name of the failed step
|
||||
error_message: Error message to record
|
||||
"""
|
||||
if self._state_manager is not None:
|
||||
self._state_manager.mark_step_failed(step_name, error_message)
|
||||
logger.warning(f"Step '{step_name}' failed: {error_message}")
|
||||
|
||||
async def download_video(
|
||||
self,
|
||||
bvid: str,
|
||||
output_dir: Path,
|
||||
quality: int = 32
|
||||
) -> Dict[str, Any]:
|
||||
"""Download Bilibili video using yt-dlp.
|
||||
|
||||
Downloads only audio (sufficient for Whisper ASR) to save bandwidth.
|
||||
|
||||
Args:
|
||||
bvid: Bilibili video ID (BV号)
|
||||
output_dir: Directory to save the video
|
||||
quality: Video quality (16=360P, 32=480P, 64=720P, 80=1080P)
|
||||
|
||||
Returns:
|
||||
Dictionary with:
|
||||
- success: bool
|
||||
- filepath: str or None
|
||||
- message: str
|
||||
"""
|
||||
result = {"success": False, "filepath": None, "message": ""}
|
||||
|
||||
try:
|
||||
from kb_workflow.pipeline.config import get_settings
|
||||
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
# Get cookie from settings or environment
|
||||
settings = get_settings()
|
||||
cookie = settings.bilibili_cookie.get_secret_value() or os.environ.get('BILIBILI_COOKIE', '')
|
||||
|
||||
# Create cookie file if needed
|
||||
cookie_file = None
|
||||
if cookie:
|
||||
cookie_file = self._create_cookie_file(cookie)
|
||||
|
||||
try:
|
||||
# Get video title for filename
|
||||
title = await self._get_video_title(url, cookie_file)
|
||||
safe_title = self._sanitize_filename(title, bvid)
|
||||
|
||||
# Build yt-dlp command - download best audio only for ASR
|
||||
# This saves bandwidth since Whisper only needs audio
|
||||
output_template = str(output_dir / f"{safe_title}.%(ext)s")
|
||||
|
||||
# Use best audio format for ASR
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"-f", "bestaudio/best",
|
||||
"-o", output_template,
|
||||
"--no-download-archive",
|
||||
"-q",
|
||||
url
|
||||
]
|
||||
|
||||
if cookie_file:
|
||||
cmd.extend(["--cookies", cookie_file])
|
||||
|
||||
logger.info(f"Downloading video: {bvid}")
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
|
||||
# Find downloaded file
|
||||
downloaded_files = list(output_dir.glob(f"{safe_title}.*"))
|
||||
video_extensions = {'.mp4', '.mkv', '.webm', '.mp3', '.m4a', '.opus', '.wav'}
|
||||
downloaded_files = [f for f in downloaded_files if f.suffix.lower() in video_extensions]
|
||||
|
||||
if downloaded_files:
|
||||
video_file = downloaded_files[0]
|
||||
result["success"] = True
|
||||
result["filepath"] = str(video_file)
|
||||
result["message"] = f"Downloaded: {video_file.name}"
|
||||
logger.info(f"Video downloaded: {video_file}")
|
||||
else:
|
||||
# Try to find any new media file in output_dir
|
||||
all_media = list(output_dir.glob("*.*"))
|
||||
all_media = [f for f in all_media if f.suffix.lower() in video_extensions]
|
||||
if all_media:
|
||||
# Get the most recently modified file
|
||||
video_file = max(all_media, key=lambda f: f.stat().st_mtime)
|
||||
result["success"] = True
|
||||
result["filepath"] = str(video_file)
|
||||
result["message"] = f"Downloaded: {video_file.name}"
|
||||
logger.info(f"Video downloaded (fallback): {video_file}")
|
||||
else:
|
||||
result["message"] = "No downloaded file found"
|
||||
logger.warning(f"No downloaded file found for {bvid}")
|
||||
|
||||
finally:
|
||||
# Clean up cookie file
|
||||
if cookie_file and os.path.exists(cookie_file):
|
||||
os.remove(cookie_file)
|
||||
|
||||
except Exception as e:
|
||||
result["message"] = f"Download error: {str(e)}"
|
||||
logger.error(f"Video download failed for {bvid}: {e}")
|
||||
|
||||
return result
|
||||
|
||||
async def _get_video_title(self, url: str, cookie_file: Optional[str] = None) -> str:
|
||||
"""Get video title using yt-dlp.
|
||||
|
||||
Args:
|
||||
url: Bilibili video URL
|
||||
cookie_file: Optional path to cookie file
|
||||
|
||||
Returns:
|
||||
Video title or fallback name
|
||||
"""
|
||||
try:
|
||||
cmd = ["yt-dlp", "--get-title", url]
|
||||
if cookie_file:
|
||||
cmd.extend(["--cookies", cookie_file])
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
|
||||
title = stdout.decode('utf-8').strip()
|
||||
if title and 'error' not in title.lower():
|
||||
return title
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to get video title: {e}")
|
||||
|
||||
return "video"
|
||||
|
||||
def _sanitize_filename(self, title: str, bvid: str) -> str:
|
||||
"""Generate standardized filename for video download.
|
||||
|
||||
Args:
|
||||
title: Video title (unused, kept for API compatibility)
|
||||
bvid: Video BV号
|
||||
|
||||
Returns:
|
||||
Standardized filename in format "bilibili_{bvid}"
|
||||
"""
|
||||
# Use standardized naming convention: platform_bvid
|
||||
return f"bilibili_{bvid}"
|
||||
|
||||
def _create_cookie_file(self, cookie: str) -> str:
|
||||
"""Create Netscape format cookie file for yt-dlp.
|
||||
|
||||
Args:
|
||||
cookie: Cookie string (e.g., "SESSDATA=xxx; bili_jct=xxx")
|
||||
|
||||
Returns:
|
||||
Path to temporary cookie file
|
||||
"""
|
||||
# Parse cookie string
|
||||
cookies = {}
|
||||
for item in cookie.split(';'):
|
||||
item = item.strip()
|
||||
if '=' in item:
|
||||
key, value = item.split('=', 1)
|
||||
cookies[key.strip()] = value.strip()
|
||||
|
||||
# Create Netscape format
|
||||
cookie_content = "# Netscape HTTP Cookie File\n"
|
||||
for name, value in cookies.items():
|
||||
cookie_content += f".bilibili.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n"
|
||||
|
||||
# Write to temp file
|
||||
fd, path = tempfile.mkstemp(suffix='.txt')
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
f.write(cookie_content)
|
||||
|
||||
return path
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Zhihu Platform Adapter
|
||||
|
||||
Provides content extraction for Zhihu platform:
|
||||
- Answer extraction from question/answer URLs
|
||||
- Zhuanlan (article) extraction from zhuanlan URLs
|
||||
- Cookie-based authentication
|
||||
- Login page detection for expired cookies
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
import re
|
||||
import asyncio
|
||||
|
||||
from kb_workflow.pipeline.platforms.base import PlatformAdapter
|
||||
from kb_workflow.pipeline.errors import AuthError
|
||||
from kb_workflow.pipeline.config import get_settings
|
||||
from kb_workflow.metadata.zhihu_extractor import ZhihuPlaywrightExtractor
|
||||
|
||||
|
||||
class ZhihuAdapter(PlatformAdapter):
|
||||
"""Zhihu platform adapter for answer and zhuanlan content extraction.
|
||||
|
||||
Uses ZhihuPlaywrightExtractor (from kb-zhihu-metadata) for content extraction.
|
||||
Requires KB_ZHIHU_COOKIE environment variable for authentication.
|
||||
|
||||
Example:
|
||||
adapter = ZhihuAdapter()
|
||||
content_type = adapter.get_content_type("https://www.zhihu.com/question/123/answer/456")
|
||||
# -> "answer"
|
||||
|
||||
metadata = await adapter.fetch_metadata("456", "answer")
|
||||
content = await adapter.fetch_content("456", "answer")
|
||||
"""
|
||||
|
||||
PLATFORM = "zhihu"
|
||||
|
||||
# URL patterns for content type detection
|
||||
ANSWER_PATTERN = re.compile(r'zhihu\.com/question/\d+/answer/(\d+)')
|
||||
ZHUANLAN_PATTERN = re.compile(r'zhuanlan\.zhihu\.com/p/([\w\d]+)')
|
||||
QUESTION_PATTERN = re.compile(r'zhihu\.com/question/(\d+)(?!/answer)')
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||
"""Initialize ZhihuAdapter.
|
||||
|
||||
Args:
|
||||
headless: Run browser in headless mode (default: True)
|
||||
timeout: Page load timeout in seconds (default: 30)
|
||||
"""
|
||||
self.headless = headless
|
||||
self.timeout = timeout
|
||||
self._cookie: Optional[str] = None
|
||||
self._extractor = None
|
||||
|
||||
@property
|
||||
def cookie(self) -> str:
|
||||
"""Get Zhihu cookie from settings.
|
||||
|
||||
Returns:
|
||||
Cookie string from KB_ZHIHU_COOKIE environment variable.
|
||||
Returns empty string if not configured.
|
||||
"""
|
||||
if self._cookie is None:
|
||||
settings = get_settings()
|
||||
self._cookie = settings.zhihu_cookie.get_secret_value()
|
||||
return self._cookie
|
||||
|
||||
def get_content_type(self, url: str) -> str:
|
||||
"""Identify content type from URL.
|
||||
|
||||
Args:
|
||||
url: The content URL to analyze
|
||||
|
||||
Returns:
|
||||
Content type: 'answer', 'zhuanlan', or 'unknown'
|
||||
"""
|
||||
if self.ANSWER_PATTERN.search(url):
|
||||
return "answer"
|
||||
elif self.ZHUANLAN_PATTERN.search(url):
|
||||
return "zhuanlan"
|
||||
return "unknown"
|
||||
|
||||
def _get_extractor(self):
|
||||
"""Get or create ZhihuPlaywrightExtractor instance.
|
||||
|
||||
Returns:
|
||||
ZhihuPlaywrightExtractor instance
|
||||
"""
|
||||
if self._extractor is None:
|
||||
self._extractor = ZhihuPlaywrightExtractor(
|
||||
headless=self.headless,
|
||||
timeout=self.timeout,
|
||||
cookie=self.cookie
|
||||
)
|
||||
return self._extractor
|
||||
|
||||
def _extract_with_playwright(self, url: str) -> Optional[Dict[str, Any]]:
|
||||
"""Extract content using Playwright-based extractor.
|
||||
|
||||
Args:
|
||||
url: Full URL to extract content from
|
||||
|
||||
Returns:
|
||||
Extraction result dictionary or None if extraction failed
|
||||
"""
|
||||
extractor = self._get_extractor()
|
||||
return extractor.extract(url)
|
||||
|
||||
def _detect_login_page(self, result: Dict[str, Any]) -> bool:
|
||||
"""Detect if the result indicates a login page redirect.
|
||||
|
||||
This happens when the cookie has expired or is invalid.
|
||||
|
||||
Args:
|
||||
result: Extraction result dictionary
|
||||
|
||||
Returns:
|
||||
True if login page is detected, False otherwise
|
||||
"""
|
||||
if not result:
|
||||
return False
|
||||
|
||||
title = result.get("title", "")
|
||||
content = result.get("content", "")
|
||||
content_markdown = result.get("content_markdown", "")
|
||||
|
||||
# Check for login-related indicators
|
||||
login_indicators = ["登录", "登陆", "sign in", "login"]
|
||||
title_lower = title.lower()
|
||||
|
||||
# If title contains login keywords
|
||||
if any(indicator in title_lower for indicator in login_indicators):
|
||||
return True
|
||||
|
||||
# If content is suspiciously short (login page has minimal content)
|
||||
total_content = content + content_markdown
|
||||
if len(total_content.strip()) < 100:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
async def fetch_metadata(self, content_id: str, content_type: str, source_url: str = None) -> Dict[str, Any]:
|
||||
"""Fetch metadata for the content.
|
||||
|
||||
Args:
|
||||
content_id: Answer ID or article ID
|
||||
content_type: 'answer' or 'zhuanlan'
|
||||
source_url: Original source URL (optional, used for answers to get question ID)
|
||||
|
||||
Returns:
|
||||
Dictionary containing metadata (title, author, publish_time, etc.)
|
||||
|
||||
Raises:
|
||||
AuthError: If cookie is missing or expired
|
||||
"""
|
||||
# Check for missing cookie
|
||||
if not self.cookie:
|
||||
raise AuthError(
|
||||
step_name="zhihu_metadata",
|
||||
input_summary=f"content_id={content_id}, type={content_type}",
|
||||
auth_type="zhihu_cookie",
|
||||
)
|
||||
|
||||
# Build URL based on content type
|
||||
if content_type == "answer":
|
||||
# For answers, try to use source_url first, then fallback to constructed URL
|
||||
if source_url:
|
||||
url = source_url
|
||||
else:
|
||||
# Fallback: use answer ID only (may not work for all cases)
|
||||
url = f"https://www.zhihu.com/question/0/answer/{content_id}"
|
||||
elif content_type == "zhuanlan":
|
||||
url = f"https://zhuanlan.zhihu.com/p/{content_id}"
|
||||
else:
|
||||
raise ValueError(f"Unsupported content type: {content_type}")
|
||||
|
||||
# Extract content in a separate thread to avoid blocking asyncio
|
||||
result = await asyncio.to_thread(self._extract_with_playwright, url)
|
||||
|
||||
# Check for login page
|
||||
if self._detect_login_page(result):
|
||||
raise AuthError(
|
||||
step_name="zhihu_metadata",
|
||||
input_summary=f"content_id={content_id}, type={content_type}",
|
||||
auth_type="zhihu_cookie",
|
||||
cause=Exception("Cookie may have expired, please update KB_ZHIHU_COOKIE")
|
||||
)
|
||||
|
||||
if not result:
|
||||
raise AuthError(
|
||||
step_name="zhihu_metadata",
|
||||
input_summary=f"content_id={content_id}, type={content_type}",
|
||||
auth_type="zhihu_cookie",
|
||||
cause=Exception("Failed to extract content, check if content exists")
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
async def fetch_content(self, content_id: str, content_type: str, source_url: str = None) -> str:
|
||||
"""Fetch content in Markdown format.
|
||||
|
||||
Args:
|
||||
content_id: Answer ID or article ID
|
||||
content_type: 'answer' or 'zhuanlan'
|
||||
source_url: Original source URL (optional, used for answers)
|
||||
|
||||
Returns:
|
||||
Markdown-formatted content with YAML frontmatter
|
||||
|
||||
Raises:
|
||||
AuthError: If cookie is missing or expired
|
||||
"""
|
||||
# Get metadata (which also validates auth)
|
||||
metadata = await self.fetch_metadata(content_id, content_type, source_url=source_url)
|
||||
|
||||
# Get content markdown
|
||||
content_markdown = metadata.get("content_markdown", "") or metadata.get("content", "")
|
||||
|
||||
# Format with YAML frontmatter
|
||||
return self._format_with_frontmatter(metadata, content_markdown, content_type)
|
||||
|
||||
def _format_with_frontmatter(
|
||||
self,
|
||||
metadata: Dict[str, Any],
|
||||
content: str,
|
||||
content_type: str
|
||||
) -> str:
|
||||
"""Format content with YAML frontmatter.
|
||||
|
||||
Args:
|
||||
metadata: Content metadata dictionary
|
||||
content: Markdown content
|
||||
content_type: 'answer' or 'zhuanlan'
|
||||
|
||||
Returns:
|
||||
Formatted string with YAML frontmatter and content
|
||||
"""
|
||||
frontmatter_lines = [
|
||||
"---",
|
||||
f"title: {metadata.get('title', '')}",
|
||||
f"author: {metadata.get('author', '')}",
|
||||
f"source_url: {metadata.get('url', '')}",
|
||||
f"platform: {self.PLATFORM}",
|
||||
f"content_type: {content_type}",
|
||||
f"fetched_at: {datetime.now().isoformat()}",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
|
||||
return "\n".join(frontmatter_lines) + content
|
||||
|
||||
def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str:
|
||||
"""Generate output filename for the content.
|
||||
|
||||
Args:
|
||||
content_id: Answer ID or article ID
|
||||
content_type: 'answer' or 'zhuanlan'
|
||||
suffix: File suffix (e.g., 'metadata.json', 'content.md')
|
||||
|
||||
Returns:
|
||||
Filename string (e.g., 'zhihu_789012_metadata.json')
|
||||
"""
|
||||
return f"zhihu_{content_id}_{suffix}"
|
||||
Reference in New Issue
Block a user