starmen1265
76f06c330b
- Replace ZhihuPlaywrightExtractor with ZhihuCrawler - Add comment crawling with include_comments=True by default - Implement cookie expiration detection and QR code login trigger - Add idempotency check via IdempotencyChecker - Add comment formatting with "## 热门评论" section - Convert HTML content to Markdown using html2text Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
544 lines
19 KiB
Python
544 lines
19 KiB
Python
"""Zhihu Platform Adapter
|
|
|
|
Provides content extraction for Zhihu platform:
|
|
- Answer extraction from question/answer URLs
|
|
- Zhuanlan (article) extraction from zhuanlan URLs
|
|
- Cookie-based authentication with auto-login
|
|
- Comment crawling and formatting
|
|
- Idempotent execution
|
|
|
|
Phase 09 - Pipeline Integration:
|
|
- Uses ZhihuCrawler instead of ZhihuPlaywrightExtractor
|
|
- Comments included by default and appended to content
|
|
- Cookie expiration triggers QR code login
|
|
"""
|
|
|
|
from typing import Dict, Any, Optional, List
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
import re
|
|
import asyncio
|
|
import html2text
|
|
|
|
from kb_workflow.pipeline.platforms.base import PlatformAdapter
|
|
from kb_workflow.pipeline.errors import AuthError
|
|
from kb_workflow.pipeline.config import get_settings
|
|
from kb_workflow.pipeline.idempotency import IdempotencyChecker
|
|
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
|
|
from kb_workflow.crawler.models import (
|
|
CrawlResult,
|
|
AnswerData,
|
|
ArticleData,
|
|
CommentData,
|
|
)
|
|
from kb_workflow.auth.zhihu_login import ZhihuLogin, ZhihuLoginResult
|
|
|
|
|
|
class ZhihuAdapter(PlatformAdapter):
|
|
"""Zhihu platform adapter for answer and zhuanlan content extraction.
|
|
|
|
Uses ZhihuCrawler for content extraction with:
|
|
- Comment crawling enabled by default
|
|
- Cookie expiration auto-detection and QR code login
|
|
- Idempotent execution via IdempotencyChecker
|
|
|
|
Example:
|
|
adapter = ZhihuAdapter()
|
|
content_type = adapter.get_content_type("https://www.zhihu.com/question/123/answer/456")
|
|
# -> "answer"
|
|
|
|
metadata = await adapter.fetch_metadata("456", "answer")
|
|
content = await adapter.fetch_content("456", "answer")
|
|
"""
|
|
|
|
PLATFORM = "zhihu"
|
|
|
|
# URL patterns for content type detection
|
|
ANSWER_PATTERN = re.compile(r'zhihu\.com/question/\d+/answer/(\d+)')
|
|
ZHUANLAN_PATTERN = re.compile(r'zhuanlan\.zhihu\.com/p/([\w\d]+)')
|
|
QUESTION_PATTERN = re.compile(r'zhihu\.com/question/(\d+)(?!/answer)')
|
|
|
|
def __init__(
|
|
self,
|
|
headless: bool = True,
|
|
timeout: int = 30,
|
|
output_dir: Optional[Path] = None,
|
|
idempotency_checker: Optional[IdempotencyChecker] = None
|
|
):
|
|
"""Initialize ZhihuAdapter.
|
|
|
|
Args:
|
|
headless: Run browser in headless mode (default: True)
|
|
timeout: Page load timeout in seconds (default: 30)
|
|
output_dir: Directory for output files (for idempotency checks)
|
|
idempotency_checker: Optional IdempotencyChecker instance
|
|
"""
|
|
self.headless = headless
|
|
self.timeout = timeout
|
|
self._cookie: Optional[str] = None
|
|
self._crawler: Optional[ZhihuCrawler] = None
|
|
self._idempotency_checker = idempotency_checker or IdempotencyChecker()
|
|
self._output_dir = output_dir
|
|
self._html_to_text = html2text.HTML2Text()
|
|
self._html_to_text.ignore_links = False
|
|
self._html_to_text.body_width = 0
|
|
|
|
@property
|
|
def cookie(self) -> str:
|
|
"""Get Zhihu cookie from settings.
|
|
|
|
Returns:
|
|
Cookie string from KB_ZHIHU_COOKIE environment variable.
|
|
Returns empty string if not configured.
|
|
"""
|
|
if self._cookie is None:
|
|
settings = get_settings()
|
|
self._cookie = settings.zhihu_cookie.get_secret_value()
|
|
return self._cookie
|
|
|
|
def get_content_type(self, url: str) -> str:
|
|
"""Identify content type from URL.
|
|
|
|
Args:
|
|
url: The content URL to analyze
|
|
|
|
Returns:
|
|
Content type: 'answer', 'zhuanlan', or 'unknown'
|
|
"""
|
|
if self.ANSWER_PATTERN.search(url):
|
|
return "answer"
|
|
elif self.ZHUANLAN_PATTERN.search(url):
|
|
return "zhuanlan"
|
|
return "unknown"
|
|
|
|
def _get_crawler(self) -> ZhihuCrawler:
|
|
"""Get or create ZhihuCrawler instance.
|
|
|
|
Returns:
|
|
ZhihuCrawler instance
|
|
"""
|
|
if self._crawler is None:
|
|
self._crawler = ZhihuCrawler(
|
|
headless=self.headless,
|
|
timeout=self.timeout
|
|
)
|
|
return self._crawler
|
|
|
|
def _crawl_with_crawler(
|
|
self,
|
|
url: str,
|
|
content_type: str,
|
|
include_comments: bool = True
|
|
) -> CrawlResult:
|
|
"""Crawl content using ZhihuCrawler.
|
|
|
|
Args:
|
|
url: Full URL to crawl
|
|
content_type: Type of content ('answer' or 'zhuanlan')
|
|
include_comments: Whether to include comments (default: True)
|
|
|
|
Returns:
|
|
CrawlResult from the crawler
|
|
"""
|
|
crawler = self._get_crawler()
|
|
|
|
if content_type == "answer":
|
|
return crawler.crawl_answer(url, include_comments=include_comments)
|
|
elif content_type == "zhuanlan":
|
|
# For articles, we need to crawl comments separately
|
|
result = crawler.crawl_article(url)
|
|
if result.success and include_comments:
|
|
# Fetch comments for articles
|
|
from kb_workflow.crawler.models import ArticleData
|
|
if isinstance(result.data, ArticleData):
|
|
comments_result = crawler.crawl_comments(
|
|
content_type="articles",
|
|
content_id=result.data.id,
|
|
include_comments=True
|
|
)
|
|
result.comments = comments_result.comments
|
|
return result
|
|
else:
|
|
return CrawlResult(
|
|
success=False,
|
|
error=f"Unsupported content type: {content_type}"
|
|
)
|
|
|
|
def _is_cookie_expired(self, result: CrawlResult) -> bool:
|
|
"""Check if the result indicates cookie expiration.
|
|
|
|
Args:
|
|
result: CrawlResult from crawler
|
|
|
|
Returns:
|
|
True if cookie appears to be expired
|
|
"""
|
|
if not result.success:
|
|
error_msg = result.error or ""
|
|
expired_indicators = [
|
|
"cookie expired",
|
|
"unauthorized",
|
|
"forbidden",
|
|
"需要登录",
|
|
"请先登录",
|
|
"登录失效",
|
|
]
|
|
return any(indicator in error_msg.lower() for indicator in expired_indicators)
|
|
return False
|
|
|
|
def _detect_login_page(self, result: CrawlResult) -> bool:
|
|
"""Detect if the result indicates a login page redirect.
|
|
|
|
This happens when the cookie has expired or is invalid.
|
|
|
|
Args:
|
|
result: CrawlResult from crawler
|
|
|
|
Returns:
|
|
True if login page is detected, False otherwise
|
|
"""
|
|
if not result.success:
|
|
error_msg = result.error or ""
|
|
login_indicators = ["登录", "登陆", "sign in", "login"]
|
|
return any(indicator in error_msg.lower() for indicator in login_indicators)
|
|
return False
|
|
|
|
def _trigger_qrcode_login(self) -> ZhihuLoginResult:
|
|
"""Trigger QR code login flow.
|
|
|
|
Returns:
|
|
ZhihuLoginResult with success status and cookie
|
|
"""
|
|
# Use intermediate path from settings or default
|
|
settings = get_settings()
|
|
intermediate_path = getattr(settings, 'intermediate_path', '/tmp')
|
|
|
|
login = ZhihuLogin(
|
|
intermediate_path=str(intermediate_path),
|
|
headless=False, # Show browser for QR code display
|
|
timeout=120 # 2 minutes to scan
|
|
)
|
|
return login.login_by_qrcode()
|
|
|
|
def _convert_crawl_result_to_metadata(
|
|
self,
|
|
result: CrawlResult,
|
|
content_type: str,
|
|
source_url: str
|
|
) -> Dict[str, Any]:
|
|
"""Convert CrawlResult to metadata dictionary.
|
|
|
|
Args:
|
|
result: CrawlResult from crawler
|
|
content_type: Type of content
|
|
source_url: Original source URL
|
|
|
|
Returns:
|
|
Metadata dictionary
|
|
"""
|
|
data = result.data
|
|
|
|
if isinstance(data, AnswerData):
|
|
return {
|
|
"type": "answer",
|
|
"title": data.question_title,
|
|
"author": data.metadata.author.name if data.metadata.author else "",
|
|
"author_url_token": data.metadata.author.url_token if data.metadata.author else None,
|
|
"content": data.content,
|
|
"content_markdown": self._html_to_text.handle(data.content),
|
|
"excerpt": data.excerpt,
|
|
"question_id": data.question_id,
|
|
"answer_id": data.id,
|
|
"voteup_count": data.metadata.voteup_count,
|
|
"comment_count": data.metadata.comment_count,
|
|
"created_time": data.metadata.created_time,
|
|
"updated_time": data.metadata.updated_time,
|
|
"url": source_url,
|
|
"has_comments": bool(result.comments),
|
|
"comment_count_actual": len(result.comments) if result.comments else 0,
|
|
}
|
|
elif isinstance(data, ArticleData):
|
|
return {
|
|
"type": "zhuanlan",
|
|
"title": data.title,
|
|
"author": data.metadata.author.name if data.metadata.author else "",
|
|
"author_url_token": data.metadata.author.url_token if data.metadata.author else None,
|
|
"content": data.content,
|
|
"content_markdown": self._html_to_text.handle(data.content),
|
|
"excerpt": data.excerpt,
|
|
"article_id": data.id,
|
|
"voteup_count": data.metadata.voteup_count,
|
|
"comment_count": data.metadata.comment_count,
|
|
"created_time": data.metadata.created_time,
|
|
"updated_time": data.metadata.updated_time,
|
|
"url": source_url,
|
|
"has_comments": bool(result.comments),
|
|
"comment_count_actual": len(result.comments) if result.comments else 0,
|
|
}
|
|
else:
|
|
return {
|
|
"type": content_type,
|
|
"title": "",
|
|
"author": "",
|
|
"content": "",
|
|
"content_markdown": "",
|
|
"url": source_url,
|
|
"error": "Unknown data type",
|
|
}
|
|
|
|
def _format_comments_section(self, comments: List[CommentData]) -> str:
|
|
"""Format comments as Markdown.
|
|
|
|
Args:
|
|
comments: List of CommentData objects
|
|
|
|
Returns:
|
|
Markdown-formatted comment section
|
|
"""
|
|
if not comments:
|
|
return ""
|
|
|
|
lines = []
|
|
for i, comment in enumerate(comments, 1):
|
|
author_name = comment.author.name if comment.author else "Anonymous"
|
|
like_str = f"({comment.like_count} likes)" if comment.like_count > 0 else ""
|
|
lines.append(f"{i}. **@{author_name}** {like_str}: {comment.content}")
|
|
|
|
return "\n".join(lines)
|
|
|
|
def _format_with_frontmatter(
|
|
self,
|
|
metadata: Dict[str, Any],
|
|
content: str,
|
|
content_type: str,
|
|
comments: Optional[List[CommentData]] = None
|
|
) -> str:
|
|
"""Format content with YAML frontmatter.
|
|
|
|
Args:
|
|
metadata: Content metadata dictionary
|
|
content: Markdown content
|
|
content_type: 'answer' or 'zhuanlan'
|
|
comments: Optional list of comments to append
|
|
|
|
Returns:
|
|
Formatted string with YAML frontmatter and content
|
|
"""
|
|
frontmatter_lines = [
|
|
"---",
|
|
f"title: {metadata.get('title', '')}",
|
|
f"author: {metadata.get('author', '')}",
|
|
f"source_url: {metadata.get('url', '')}",
|
|
f"platform: {self.PLATFORM}",
|
|
f"content_type: {content_type}",
|
|
f"fetched_at: {datetime.now().isoformat()}",
|
|
"---",
|
|
"",
|
|
]
|
|
|
|
result = "\n".join(frontmatter_lines) + content
|
|
|
|
# Append comments section if present
|
|
if comments:
|
|
comments_md = self._format_comments_section(comments)
|
|
if comments_md:
|
|
result += "\n\n---\n\n## 热门评论\n\n" + comments_md
|
|
|
|
return result
|
|
|
|
def _load_cached_metadata(self, metadata_file: Path) -> Optional[Dict[str, Any]]:
|
|
"""Load cached metadata from file.
|
|
|
|
Args:
|
|
metadata_file: Path to metadata JSON file
|
|
|
|
Returns:
|
|
Metadata dictionary or None if file doesn't exist or is invalid
|
|
"""
|
|
import json
|
|
try:
|
|
if metadata_file.exists():
|
|
return json.loads(metadata_file.read_text())
|
|
except (json.JSONDecodeError, IOError):
|
|
pass
|
|
return None
|
|
|
|
async def fetch_metadata(
|
|
self,
|
|
content_id: str,
|
|
content_type: str,
|
|
source_url: str = None
|
|
) -> Dict[str, Any]:
|
|
"""Fetch metadata for the content.
|
|
|
|
Args:
|
|
content_id: Answer ID or article ID
|
|
content_type: 'answer' or 'zhuanlan'
|
|
source_url: Original source URL (optional, used for answers to get question ID)
|
|
|
|
Returns:
|
|
Dictionary containing metadata (title, author, publish_time, etc.)
|
|
|
|
Raises:
|
|
AuthError: If cookie is missing or expired
|
|
"""
|
|
# Check for missing cookie
|
|
if not self.cookie:
|
|
raise AuthError(
|
|
step_name="zhihu_metadata",
|
|
input_summary=f"content_id={content_id}, type={content_type}",
|
|
auth_type="zhihu_cookie",
|
|
)
|
|
|
|
# Idempotency check: if output_dir is set and content already processed, return cached result
|
|
if self._output_dir:
|
|
exists, existing_files = self._idempotency_checker.check_exists(
|
|
self._output_dir, content_id, self.PLATFORM
|
|
)
|
|
if exists:
|
|
# Load and return cached metadata
|
|
metadata_file = existing_files[0] if existing_files else None
|
|
if metadata_file and metadata_file.suffix == ".json":
|
|
cached = self._load_cached_metadata(metadata_file)
|
|
if cached:
|
|
return cached
|
|
|
|
# Build URL based on content type
|
|
if content_type == "answer":
|
|
url = source_url or f"https://www.zhihu.com/question/0/answer/{content_id}"
|
|
elif content_type == "zhuanlan":
|
|
url = f"https://zhuanlan.zhihu.com/p/{content_id}"
|
|
else:
|
|
raise ValueError(f"Unsupported content type: {content_type}")
|
|
|
|
# Crawl content in a separate thread to avoid blocking asyncio
|
|
result = await asyncio.to_thread(
|
|
self._crawl_with_crawler,
|
|
url,
|
|
content_type,
|
|
True # include_comments=True by default
|
|
)
|
|
|
|
# Check for login page / expired cookie
|
|
if self._detect_login_page(result) or self._is_cookie_expired(result):
|
|
# Try QR code login
|
|
login_result = self._trigger_qrcode_login()
|
|
|
|
if login_result.success:
|
|
# Update cookie and retry
|
|
self._cookie = login_result.cookie_str
|
|
# Retry crawl with new cookie
|
|
result = await asyncio.to_thread(
|
|
self._crawl_with_crawler,
|
|
url,
|
|
content_type,
|
|
True
|
|
)
|
|
else:
|
|
raise AuthError(
|
|
step_name="zhihu_metadata",
|
|
input_summary=f"content_id={content_id}, type={content_type}",
|
|
auth_type="zhihu_cookie",
|
|
cause=Exception(f"Login failed: {login_result.error_message}")
|
|
)
|
|
|
|
if not result.success:
|
|
raise AuthError(
|
|
step_name="zhihu_metadata",
|
|
input_summary=f"content_id={content_id}, type={content_type}",
|
|
auth_type="zhihu_cookie",
|
|
cause=Exception(result.error or "Failed to extract content")
|
|
)
|
|
|
|
# Convert to metadata dictionary
|
|
return self._convert_crawl_result_to_metadata(result, content_type, url)
|
|
|
|
async def fetch_content(
|
|
self,
|
|
content_id: str,
|
|
content_type: str,
|
|
source_url: str = None
|
|
) -> str:
|
|
"""Fetch content in Markdown format.
|
|
|
|
Args:
|
|
content_id: Answer ID or article ID
|
|
content_type: 'answer' or 'zhuanlan'
|
|
source_url: Original source URL (optional, used for answers)
|
|
|
|
Returns:
|
|
Markdown-formatted content with YAML frontmatter and comments
|
|
|
|
Raises:
|
|
AuthError: If cookie is missing or expired
|
|
"""
|
|
# Build URL for crawling
|
|
if content_type == "answer":
|
|
url = source_url or f"https://www.zhihu.com/question/0/answer/{content_id}"
|
|
elif content_type == "zhuanlan":
|
|
url = f"https://zhuanlan.zhihu.com/p/{content_id}"
|
|
else:
|
|
raise ValueError(f"Unsupported content type: {content_type}")
|
|
|
|
# Crawl content with comments
|
|
result = await asyncio.to_thread(
|
|
self._crawl_with_crawler,
|
|
url,
|
|
content_type,
|
|
True
|
|
)
|
|
|
|
# Handle cookie expiration
|
|
if self._detect_login_page(result) or self._is_cookie_expired(result):
|
|
login_result = self._trigger_qrcode_login()
|
|
if login_result.success:
|
|
self._cookie = login_result.cookie_str
|
|
result = await asyncio.to_thread(
|
|
self._crawl_with_crawler,
|
|
url,
|
|
content_type,
|
|
True
|
|
)
|
|
else:
|
|
raise AuthError(
|
|
step_name="zhihu_content",
|
|
input_summary=f"content_id={content_id}, type={content_type}",
|
|
auth_type="zhihu_cookie",
|
|
cause=Exception(f"Login failed: {login_result.error_message}")
|
|
)
|
|
|
|
if not result.success:
|
|
raise AuthError(
|
|
step_name="zhihu_content",
|
|
input_summary=f"content_id={content_id}, type={content_type}",
|
|
auth_type="zhihu_cookie",
|
|
cause=Exception(result.error or "Failed to extract content")
|
|
)
|
|
|
|
# Convert to metadata
|
|
metadata = self._convert_crawl_result_to_metadata(result, content_type, url)
|
|
|
|
# Get content markdown
|
|
content_markdown = metadata.get("content_markdown", "") or metadata.get("content", "")
|
|
|
|
# Format with YAML frontmatter and comments
|
|
return self._format_with_frontmatter(
|
|
metadata,
|
|
content_markdown,
|
|
content_type,
|
|
result.comments
|
|
)
|
|
|
|
def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str:
|
|
"""Generate output filename for the content.
|
|
|
|
Args:
|
|
content_id: Answer ID or article ID
|
|
content_type: 'answer' or 'zhuanlan'
|
|
suffix: File suffix (e.g., 'metadata.json', 'content.md')
|
|
|
|
Returns:
|
|
Filename string (e.g., 'zhihu_789012_metadata.json')
|
|
"""
|
|
return f"zhihu_{content_id}_{suffix}"
|