"""Zhihu Platform Adapter Provides content extraction for Zhihu platform: - Answer extraction from question/answer URLs - Zhuanlan (article) extraction from zhuanlan URLs - Cookie-based authentication with auto-login - Comment crawling and formatting - Idempotent execution Phase 09 - Pipeline Integration: - Uses ZhihuCrawler instead of ZhihuPlaywrightExtractor - Comments included by default and appended to content - Cookie expiration triggers QR code login """ from typing import Dict, Any, Optional, List from datetime import datetime from pathlib import Path import re import asyncio import html2text from kb_workflow.pipeline.platforms.base import PlatformAdapter from kb_workflow.pipeline.errors import AuthError from kb_workflow.pipeline.config import get_settings from kb_workflow.pipeline.idempotency import IdempotencyChecker from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler from kb_workflow.crawler.models import ( CrawlResult, AnswerData, ArticleData, CommentData, ) from kb_workflow.auth.zhihu_login import ZhihuLogin, ZhihuLoginResult class ZhihuAdapter(PlatformAdapter): """Zhihu platform adapter for answer and zhuanlan content extraction. Uses ZhihuCrawler for content extraction with: - Comment crawling enabled by default - Cookie expiration auto-detection and QR code login - Idempotent execution via IdempotencyChecker Example: adapter = ZhihuAdapter() content_type = adapter.get_content_type("https://www.zhihu.com/question/123/answer/456") # -> "answer" metadata = await adapter.fetch_metadata("456", "answer") content = await adapter.fetch_content("456", "answer") """ PLATFORM = "zhihu" # URL patterns for content type detection ANSWER_PATTERN = re.compile(r'zhihu\.com/question/\d+/answer/(\d+)') ZHUANLAN_PATTERN = re.compile(r'zhuanlan\.zhihu\.com/p/([\w\d]+)') QUESTION_PATTERN = re.compile(r'zhihu\.com/question/(\d+)(?!/answer)') def __init__( self, headless: bool = True, timeout: int = 30, output_dir: Optional[Path] = None, idempotency_checker: Optional[IdempotencyChecker] = None ): """Initialize ZhihuAdapter. Args: headless: Run browser in headless mode (default: True) timeout: Page load timeout in seconds (default: 30) output_dir: Directory for output files (for idempotency checks) idempotency_checker: Optional IdempotencyChecker instance """ self.headless = headless self.timeout = timeout self._cookie: Optional[str] = None self._crawler: Optional[ZhihuCrawler] = None self._idempotency_checker = idempotency_checker or IdempotencyChecker() self._output_dir = output_dir self._html_to_text = html2text.HTML2Text() self._html_to_text.ignore_links = False self._html_to_text.body_width = 0 @property def cookie(self) -> str: """Get Zhihu cookie from settings. Returns: Cookie string from KB_ZHIHU_COOKIE environment variable. Returns empty string if not configured. """ if self._cookie is None: settings = get_settings() self._cookie = settings.zhihu_cookie.get_secret_value() return self._cookie def get_content_type(self, url: str) -> str: """Identify content type from URL. Args: url: The content URL to analyze Returns: Content type: 'answer', 'zhuanlan', or 'unknown' """ if self.ANSWER_PATTERN.search(url): return "answer" elif self.ZHUANLAN_PATTERN.search(url): return "zhuanlan" return "unknown" def _get_crawler(self) -> ZhihuCrawler: """Get or create ZhihuCrawler instance. Returns: ZhihuCrawler instance """ if self._crawler is None: self._crawler = ZhihuCrawler( headless=self.headless, timeout=self.timeout ) return self._crawler def _crawl_with_crawler( self, url: str, content_type: str, include_comments: bool = True ) -> CrawlResult: """Crawl content using ZhihuCrawler. Args: url: Full URL to crawl content_type: Type of content ('answer' or 'zhuanlan') include_comments: Whether to include comments (default: True) Returns: CrawlResult from the crawler """ crawler = self._get_crawler() if content_type == "answer": return crawler.crawl_answer(url, include_comments=include_comments) elif content_type == "zhuanlan": # For articles, we need to crawl comments separately result = crawler.crawl_article(url) if result.success and include_comments: # Fetch comments for articles from kb_workflow.crawler.models import ArticleData if isinstance(result.data, ArticleData): comments_result = crawler.crawl_comments( content_type="articles", content_id=result.data.id, include_comments=True ) result.comments = comments_result.comments return result else: return CrawlResult( success=False, error=f"Unsupported content type: {content_type}" ) def _is_cookie_expired(self, result: CrawlResult) -> bool: """Check if the result indicates cookie expiration. Args: result: CrawlResult from crawler Returns: True if cookie appears to be expired """ if not result.success: error_msg = result.error or "" expired_indicators = [ "cookie expired", "unauthorized", "forbidden", "需要登录", "请先登录", "登录失效", ] return any(indicator in error_msg.lower() for indicator in expired_indicators) return False def _detect_login_page(self, result: CrawlResult) -> bool: """Detect if the result indicates a login page redirect. This happens when the cookie has expired or is invalid. Args: result: CrawlResult from crawler Returns: True if login page is detected, False otherwise """ if not result.success: error_msg = result.error or "" login_indicators = ["登录", "登陆", "sign in", "login"] return any(indicator in error_msg.lower() for indicator in login_indicators) return False def _trigger_qrcode_login(self) -> ZhihuLoginResult: """Trigger QR code login flow. Returns: ZhihuLoginResult with success status and cookie """ # Use intermediate path from settings or default settings = get_settings() intermediate_path = getattr(settings, 'intermediate_path', '/tmp') login = ZhihuLogin( intermediate_path=str(intermediate_path), headless=False, # Show browser for QR code display timeout=120 # 2 minutes to scan ) return login.login_by_qrcode() def _convert_crawl_result_to_metadata( self, result: CrawlResult, content_type: str, source_url: str ) -> Dict[str, Any]: """Convert CrawlResult to metadata dictionary. Args: result: CrawlResult from crawler content_type: Type of content source_url: Original source URL Returns: Metadata dictionary """ data = result.data if isinstance(data, AnswerData): return { "type": "answer", "title": data.question_title, "author": data.metadata.author.name if data.metadata.author else "", "author_url_token": data.metadata.author.url_token if data.metadata.author else None, "content": data.content, "content_markdown": self._html_to_text.handle(data.content), "excerpt": data.excerpt, "question_id": data.question_id, "answer_id": data.id, "voteup_count": data.metadata.voteup_count, "comment_count": data.metadata.comment_count, "created_time": data.metadata.created_time, "updated_time": data.metadata.updated_time, "url": source_url, "has_comments": bool(result.comments), "comment_count_actual": len(result.comments) if result.comments else 0, } elif isinstance(data, ArticleData): return { "type": "zhuanlan", "title": data.title, "author": data.metadata.author.name if data.metadata.author else "", "author_url_token": data.metadata.author.url_token if data.metadata.author else None, "content": data.content, "content_markdown": self._html_to_text.handle(data.content), "excerpt": data.excerpt, "article_id": data.id, "voteup_count": data.metadata.voteup_count, "comment_count": data.metadata.comment_count, "created_time": data.metadata.created_time, "updated_time": data.metadata.updated_time, "url": source_url, "has_comments": bool(result.comments), "comment_count_actual": len(result.comments) if result.comments else 0, } else: return { "type": content_type, "title": "", "author": "", "content": "", "content_markdown": "", "url": source_url, "error": "Unknown data type", } def _format_comments_section(self, comments: List[CommentData]) -> str: """Format comments as Markdown. Args: comments: List of CommentData objects Returns: Markdown-formatted comment section """ if not comments: return "" lines = [] for i, comment in enumerate(comments, 1): author_name = comment.author.name if comment.author else "Anonymous" like_str = f"({comment.like_count} likes)" if comment.like_count > 0 else "" lines.append(f"{i}. **@{author_name}** {like_str}: {comment.content}") return "\n".join(lines) def _format_with_frontmatter( self, metadata: Dict[str, Any], content: str, content_type: str, comments: Optional[List[CommentData]] = None ) -> str: """Format content with YAML frontmatter. Args: metadata: Content metadata dictionary content: Markdown content content_type: 'answer' or 'zhuanlan' comments: Optional list of comments to append Returns: Formatted string with YAML frontmatter and content """ frontmatter_lines = [ "---", f"title: {metadata.get('title', '')}", f"author: {metadata.get('author', '')}", f"source_url: {metadata.get('url', '')}", f"platform: {self.PLATFORM}", f"content_type: {content_type}", f"fetched_at: {datetime.now().isoformat()}", "---", "", ] result = "\n".join(frontmatter_lines) + content # Append comments section if present if comments: comments_md = self._format_comments_section(comments) if comments_md: result += "\n\n---\n\n## 热门评论\n\n" + comments_md return result def _load_cached_metadata(self, metadata_file: Path) -> Optional[Dict[str, Any]]: """Load cached metadata from file. Args: metadata_file: Path to metadata JSON file Returns: Metadata dictionary or None if file doesn't exist or is invalid """ import json try: if metadata_file.exists(): return json.loads(metadata_file.read_text()) except (json.JSONDecodeError, IOError): pass return None async def fetch_metadata( self, content_id: str, content_type: str, source_url: str = None ) -> Dict[str, Any]: """Fetch metadata for the content. Args: content_id: Answer ID or article ID content_type: 'answer' or 'zhuanlan' source_url: Original source URL (optional, used for answers to get question ID) Returns: Dictionary containing metadata (title, author, publish_time, etc.) Raises: AuthError: If cookie is missing or expired """ # Check for missing cookie if not self.cookie: raise AuthError( step_name="zhihu_metadata", input_summary=f"content_id={content_id}, type={content_type}", auth_type="zhihu_cookie", ) # Idempotency check: if output_dir is set and content already processed, return cached result if self._output_dir: exists, existing_files = self._idempotency_checker.check_exists( self._output_dir, content_id, self.PLATFORM ) if exists: # Load and return cached metadata metadata_file = existing_files[0] if existing_files else None if metadata_file and metadata_file.suffix == ".json": cached = self._load_cached_metadata(metadata_file) if cached: return cached # Build URL based on content type if content_type == "answer": url = source_url or f"https://www.zhihu.com/question/0/answer/{content_id}" elif content_type == "zhuanlan": url = f"https://zhuanlan.zhihu.com/p/{content_id}" else: raise ValueError(f"Unsupported content type: {content_type}") # Crawl content in a separate thread to avoid blocking asyncio result = await asyncio.to_thread( self._crawl_with_crawler, url, content_type, True # include_comments=True by default ) # Check for login page / expired cookie if self._detect_login_page(result) or self._is_cookie_expired(result): # Try QR code login login_result = self._trigger_qrcode_login() if login_result.success: # Update cookie and retry self._cookie = login_result.cookie_str # Retry crawl with new cookie result = await asyncio.to_thread( self._crawl_with_crawler, url, content_type, True ) else: raise AuthError( step_name="zhihu_metadata", input_summary=f"content_id={content_id}, type={content_type}", auth_type="zhihu_cookie", cause=Exception(f"Login failed: {login_result.error_message}") ) if not result.success: raise AuthError( step_name="zhihu_metadata", input_summary=f"content_id={content_id}, type={content_type}", auth_type="zhihu_cookie", cause=Exception(result.error or "Failed to extract content") ) # Convert to metadata dictionary return self._convert_crawl_result_to_metadata(result, content_type, url) async def fetch_content( self, content_id: str, content_type: str, source_url: str = None ) -> str: """Fetch content in Markdown format. Args: content_id: Answer ID or article ID content_type: 'answer' or 'zhuanlan' source_url: Original source URL (optional, used for answers) Returns: Markdown-formatted content with YAML frontmatter and comments Raises: AuthError: If cookie is missing or expired """ # Build URL for crawling if content_type == "answer": url = source_url or f"https://www.zhihu.com/question/0/answer/{content_id}" elif content_type == "zhuanlan": url = f"https://zhuanlan.zhihu.com/p/{content_id}" else: raise ValueError(f"Unsupported content type: {content_type}") # Crawl content with comments result = await asyncio.to_thread( self._crawl_with_crawler, url, content_type, True ) # Handle cookie expiration if self._detect_login_page(result) or self._is_cookie_expired(result): login_result = self._trigger_qrcode_login() if login_result.success: self._cookie = login_result.cookie_str result = await asyncio.to_thread( self._crawl_with_crawler, url, content_type, True ) else: raise AuthError( step_name="zhihu_content", input_summary=f"content_id={content_id}, type={content_type}", auth_type="zhihu_cookie", cause=Exception(f"Login failed: {login_result.error_message}") ) if not result.success: raise AuthError( step_name="zhihu_content", input_summary=f"content_id={content_id}, type={content_type}", auth_type="zhihu_cookie", cause=Exception(result.error or "Failed to extract content") ) # Convert to metadata metadata = self._convert_crawl_result_to_metadata(result, content_type, url) # Get content markdown content_markdown = metadata.get("content_markdown", "") or metadata.get("content", "") # Format with YAML frontmatter and comments return self._format_with_frontmatter( metadata, content_markdown, content_type, result.comments ) def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str: """Generate output filename for the content. Args: content_id: Answer ID or article ID content_type: 'answer' or 'zhuanlan' suffix: File suffix (e.g., 'metadata.json', 'content.md') Returns: Filename string (e.g., 'zhihu_789012_metadata.json') """ return f"zhihu_{content_id}_{suffix}"