自动同步: 2026-03-18 02:10
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
"""Zhihu Crawler Module
|
||||
|
||||
Provides content crawling capabilities for Zhihu using Playwright API interception.
|
||||
"""
|
||||
|
||||
from .models import (
|
||||
AuthorInfo,
|
||||
ContentMetadata,
|
||||
AnswerData,
|
||||
ArticleData,
|
||||
CommentAuthor,
|
||||
CommentData,
|
||||
CommentsResult,
|
||||
CrawlResult,
|
||||
)
|
||||
from .config import (
|
||||
ProxyConfig,
|
||||
CrawlerConfig,
|
||||
get_crawler_config,
|
||||
reset_crawler_config,
|
||||
)
|
||||
from .zhihu_crawler import ZhihuCrawler, ZhihuApiInterceptor
|
||||
|
||||
__all__ = [
|
||||
"AuthorInfo",
|
||||
"ContentMetadata",
|
||||
"AnswerData",
|
||||
"ArticleData",
|
||||
"CommentAuthor",
|
||||
"CommentData",
|
||||
"CommentsResult",
|
||||
"CrawlResult",
|
||||
"ProxyConfig",
|
||||
"CrawlerConfig",
|
||||
"get_crawler_config",
|
||||
"reset_crawler_config",
|
||||
"ZhihuCrawler",
|
||||
"ZhihuApiInterceptor",
|
||||
]
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Crawler configuration
|
||||
|
||||
Handles proxy settings, rate limiting, and retry configuration.
|
||||
"""
|
||||
|
||||
import os
|
||||
from urllib.parse import urlparse
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ProxyConfig(BaseModel):
|
||||
"""Proxy configuration for crawler requests.
|
||||
|
||||
Attributes:
|
||||
server: Proxy server address (e.g., http://127.0.0.1:7890)
|
||||
username: Optional proxy username for authentication
|
||||
password: Optional proxy password for authentication
|
||||
"""
|
||||
|
||||
server: str = Field(..., description="Proxy server address")
|
||||
username: Optional[str] = Field(None, description="Proxy username")
|
||||
password: Optional[str] = Field(None, description="Proxy password")
|
||||
|
||||
def to_playwright_format(self) -> Dict[str, Any]:
|
||||
"""Convert to Playwright proxy format.
|
||||
|
||||
Returns:
|
||||
Dictionary compatible with browser.new_context(proxy=...)
|
||||
"""
|
||||
config = {"server": self.server}
|
||||
if self.username:
|
||||
config["username"] = self.username
|
||||
if self.password:
|
||||
config["password"] = self.password
|
||||
return config
|
||||
|
||||
|
||||
class CrawlerConfig(BaseModel):
|
||||
"""Crawler configuration.
|
||||
|
||||
Handles proxy settings, rate limiting, and retry behavior.
|
||||
|
||||
Attributes:
|
||||
proxy: Optional proxy configuration
|
||||
proxy_required: Whether proxy is mandatory (raises error if not configured)
|
||||
min_delay: Minimum delay between requests (seconds)
|
||||
max_delay: Maximum delay between requests (seconds)
|
||||
max_retries: Maximum number of retry attempts
|
||||
retry_min_wait: Minimum wait time between retries (seconds)
|
||||
retry_max_wait: Maximum wait time between retries (seconds)
|
||||
"""
|
||||
|
||||
# Proxy settings
|
||||
proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration")
|
||||
proxy_required: bool = Field(default=False, description="Whether proxy is mandatory (False for 旁路由 auto-proxy)")
|
||||
|
||||
# Rate limiting
|
||||
min_delay: float = Field(default=1.0, description="Minimum delay between requests (seconds)")
|
||||
max_delay: float = Field(default=3.0, description="Maximum delay between requests (seconds)")
|
||||
|
||||
# Retry settings
|
||||
max_retries: int = Field(default=3, description="Maximum retry attempts")
|
||||
retry_min_wait: float = Field(default=2.0, description="Minimum retry wait (seconds)")
|
||||
retry_max_wait: float = Field(default=10.0, description="Maximum retry wait (seconds)")
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "CrawlerConfig":
|
||||
"""Load configuration from environment variables.
|
||||
|
||||
Reads HTTP_PROXY or HTTPS_PROXY for proxy settings.
|
||||
|
||||
Returns:
|
||||
CrawlerConfig instance with proxy settings from environment
|
||||
"""
|
||||
proxy = cls._parse_proxy_env()
|
||||
return cls(proxy=proxy)
|
||||
|
||||
@staticmethod
|
||||
def _parse_proxy_env() -> Optional[ProxyConfig]:
|
||||
"""Parse proxy configuration from environment variables.
|
||||
|
||||
Checks HTTPS_PROXY first, then HTTP_PROXY.
|
||||
|
||||
Returns:
|
||||
ProxyConfig if proxy URL found, None otherwise
|
||||
"""
|
||||
proxy_url = os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY")
|
||||
|
||||
if not proxy_url:
|
||||
return None
|
||||
|
||||
parsed = urlparse(proxy_url)
|
||||
server = f"{parsed.scheme}://{parsed.hostname}"
|
||||
|
||||
if parsed.port:
|
||||
server += f":{parsed.port}"
|
||||
|
||||
return ProxyConfig(
|
||||
server=server,
|
||||
username=parsed.username,
|
||||
password=parsed.password
|
||||
)
|
||||
|
||||
def validate_proxy(self) -> None:
|
||||
"""Validate proxy configuration.
|
||||
|
||||
Raises:
|
||||
ValueError: If proxy_required=True but no proxy is configured
|
||||
"""
|
||||
if self.proxy_required and not self.proxy:
|
||||
raise ValueError(
|
||||
"Proxy configuration missing! Please set HTTP_PROXY or HTTPS_PROXY environment variable.\n"
|
||||
"Example: export HTTPS_PROXY=http://127.0.0.1:7890"
|
||||
)
|
||||
|
||||
|
||||
# Module-level cached config
|
||||
_crawler_config: Optional[CrawlerConfig] = None
|
||||
|
||||
|
||||
def get_crawler_config() -> CrawlerConfig:
|
||||
"""Get crawler configuration singleton.
|
||||
|
||||
Returns:
|
||||
CrawlerConfig instance loaded from environment variables
|
||||
"""
|
||||
global _crawler_config
|
||||
if _crawler_config is None:
|
||||
_crawler_config = CrawlerConfig.from_env()
|
||||
return _crawler_config
|
||||
|
||||
|
||||
def reset_crawler_config() -> None:
|
||||
"""Reset the cached crawler configuration.
|
||||
|
||||
Useful when environment variables change during runtime.
|
||||
"""
|
||||
global _crawler_config
|
||||
_crawler_config = None
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Zhihu Crawler Data Models
|
||||
|
||||
Pydantic models for structured data from Zhihu API responses.
|
||||
"""
|
||||
|
||||
from typing import Optional, Union, List
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class AuthorInfo(BaseModel):
|
||||
"""Author information from Zhihu content.
|
||||
|
||||
Attributes:
|
||||
name: Author display name
|
||||
url_token: Author URL token (e.g., 'zhang-san-12')
|
||||
headline: Author bio/headline
|
||||
"""
|
||||
name: str = Field(..., description="Author display name")
|
||||
url_token: Optional[str] = Field(None, description="Author URL token")
|
||||
headline: Optional[str] = Field(None, description="Author bio/headline")
|
||||
|
||||
|
||||
class ContentMetadata(BaseModel):
|
||||
"""Metadata for Zhihu content (answers, articles, etc).
|
||||
|
||||
Attributes:
|
||||
author: Author information
|
||||
voteup_count: Number of upvotes/likes
|
||||
comment_count: Number of comments
|
||||
created_time: Unix timestamp of creation
|
||||
updated_time: Unix timestamp of last update
|
||||
"""
|
||||
author: Optional[AuthorInfo] = Field(None, description="Author info")
|
||||
voteup_count: int = Field(default=0, description="Upvote count")
|
||||
comment_count: int = Field(default=0, description="Comment count")
|
||||
created_time: Optional[int] = Field(None, description="Creation timestamp")
|
||||
updated_time: Optional[int] = Field(None, description="Update timestamp")
|
||||
|
||||
|
||||
class AnswerData(BaseModel):
|
||||
"""Data model for Zhihu answer.
|
||||
|
||||
Attributes:
|
||||
id: Answer ID
|
||||
question_id: Question ID this answer belongs to
|
||||
question_title: Title of the question
|
||||
content: Answer content in HTML format
|
||||
excerpt: Plain text excerpt/summary
|
||||
metadata: Additional metadata (author, votes, etc.)
|
||||
"""
|
||||
id: int = Field(..., description="Answer ID")
|
||||
question_id: int = Field(..., description="Question ID")
|
||||
question_title: str = Field(..., description="Question title")
|
||||
content: str = Field(default="", description="Answer content (HTML)")
|
||||
excerpt: str = Field(default="", description="Plain text excerpt")
|
||||
metadata: ContentMetadata = Field(
|
||||
default_factory=ContentMetadata,
|
||||
description="Content metadata"
|
||||
)
|
||||
|
||||
|
||||
class ArticleData(BaseModel):
|
||||
"""Data model for Zhihu zhuanlan article.
|
||||
|
||||
Attributes:
|
||||
id: Article ID
|
||||
title: Article title
|
||||
content: Article content in HTML format
|
||||
excerpt: Plain text excerpt/summary
|
||||
metadata: Additional metadata (author, votes, etc.)
|
||||
"""
|
||||
id: int = Field(..., description="Article ID")
|
||||
title: str = Field(..., description="Article title")
|
||||
content: str = Field(default="", description="Article content (HTML)")
|
||||
excerpt: str = Field(default="", description="Plain text excerpt")
|
||||
metadata: ContentMetadata = Field(
|
||||
default_factory=ContentMetadata,
|
||||
description="Content metadata"
|
||||
)
|
||||
|
||||
|
||||
class CommentAuthor(BaseModel):
|
||||
"""Comment author information.
|
||||
|
||||
Attributes:
|
||||
name: Author display name
|
||||
url_token: Author URL token (e.g., 'zhang-san-12')
|
||||
"""
|
||||
name: str = Field(..., description="Author display name")
|
||||
url_token: Optional[str] = Field(None, description="Author URL token")
|
||||
|
||||
|
||||
class CommentData(BaseModel):
|
||||
"""Data model for Zhihu comment.
|
||||
|
||||
Attributes:
|
||||
id: Comment ID
|
||||
content: Comment content in plain text
|
||||
author: Comment author information
|
||||
created_time: Unix timestamp of creation
|
||||
like_count: Number of likes/upvotes
|
||||
child_count: Number of child/reply comments
|
||||
is_author: Whether the commenter is the content author
|
||||
"""
|
||||
id: int = Field(..., description="Comment ID")
|
||||
content: str = Field(default="", description="Comment content")
|
||||
author: Optional[CommentAuthor] = Field(None, description="Comment author")
|
||||
created_time: Optional[int] = Field(None, description="Creation timestamp")
|
||||
like_count: int = Field(default=0, description="Like/upvote count")
|
||||
child_count: int = Field(default=0, description="Number of child comments")
|
||||
is_author: bool = Field(default=False, description="Whether commenter is content author")
|
||||
|
||||
|
||||
class CommentsResult(BaseModel):
|
||||
"""Result of comment crawling operation.
|
||||
|
||||
Attributes:
|
||||
content_type: Type of content ("answers" or "articles")
|
||||
content_id: ID of the content
|
||||
comments: List of comments
|
||||
total_count: Total number of comments (may exceed returned count)
|
||||
"""
|
||||
content_type: str = Field(..., description="Content type: answers or articles")
|
||||
content_id: int = Field(..., description="Content ID")
|
||||
comments: List[CommentData] = Field(
|
||||
default_factory=list,
|
||||
description="List of comments"
|
||||
)
|
||||
total_count: int = Field(default=0, description="Total comment count")
|
||||
|
||||
|
||||
class CrawlResult(BaseModel):
|
||||
"""Result of a crawl operation.
|
||||
|
||||
Attributes:
|
||||
success: Whether the crawl was successful
|
||||
content_type: Type of content ("answer" or "article")
|
||||
data: Extracted content data if successful
|
||||
comments: Optional list of comments (when include_comments=True)
|
||||
error: Error message if failed
|
||||
"""
|
||||
success: bool = Field(..., description="Whether crawl succeeded")
|
||||
content_type: str = Field(default="", description="Content type: answer or article")
|
||||
data: Optional[Union[AnswerData, ArticleData]] = Field(
|
||||
None,
|
||||
description="Extracted content data"
|
||||
)
|
||||
comments: Optional[List[CommentData]] = Field(
|
||||
None,
|
||||
description="Comments if include_comments was True"
|
||||
)
|
||||
error: Optional[str] = Field(None, description="Error message if failed")
|
||||
@@ -0,0 +1,594 @@
|
||||
"""Zhihu Content Crawler
|
||||
|
||||
Implements content crawling using Playwright API interception.
|
||||
Captures structured JSON data from Zhihu API responses.
|
||||
|
||||
Supports:
|
||||
- Mandatory proxy configuration
|
||||
- Random request delay
|
||||
- Retry with exponential backoff
|
||||
- Automatic x-zse-96 signature handling via browser
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from playwright.sync_api import Route, Response, sync_playwright
|
||||
from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
|
||||
|
||||
from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser
|
||||
# NOTE: get_settings is imported lazily in _get_cookie() to avoid circular import
|
||||
# Circular: zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
|
||||
from .models import (
|
||||
AnswerData,
|
||||
ArticleData,
|
||||
ContentMetadata,
|
||||
AuthorInfo,
|
||||
CommentData,
|
||||
CommentAuthor,
|
||||
CommentsResult,
|
||||
CrawlResult,
|
||||
)
|
||||
from .config import CrawlerConfig, get_crawler_config
|
||||
|
||||
|
||||
class ZhihuApiInterceptor:
|
||||
"""API Interceptor for capturing Zhihu API responses.
|
||||
|
||||
Intercepts network requests and captures JSON responses from
|
||||
Zhihu API endpoints (/api/v4/answers/, /api/v4/articles/, etc).
|
||||
The x-zse-96 signature is automatically handled by the browser via route.fetch().
|
||||
|
||||
Attributes:
|
||||
captured_data: Dictionary storing captured API response data
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.captured_data: Dict[str, Any] = {}
|
||||
|
||||
def handle_route(self, route: Route) -> None:
|
||||
"""Handle intercepted route/request.
|
||||
|
||||
The route.fetch() call automatically includes browser-generated
|
||||
x-zse-96 signature headers, No manual signature handling needed.
|
||||
|
||||
Args:
|
||||
route: Playwright Route object for the intercepted request
|
||||
"""
|
||||
url = route.request.url
|
||||
|
||||
# Intercept answer API
|
||||
if "/api/v4/answers/" in url:
|
||||
response = route.fetch()
|
||||
self.captured_data["answer"] = response.json()
|
||||
route.fulfill(response=response)
|
||||
return
|
||||
|
||||
# Intercept article API (zhuanlan)
|
||||
if "/api/v4/articles/" in url or "zhuanlan" in url:
|
||||
response = route.fetch()
|
||||
self.captured_data["article"] = response.json()
|
||||
route.fulfill(response=response)
|
||||
return
|
||||
|
||||
route.continue_()
|
||||
|
||||
|
||||
class ZhihuCrawler:
|
||||
"""Zhihu Content Crawler.
|
||||
|
||||
Crawls Zhihu answers and articles using Playwright API interception.
|
||||
Includes anti-scraping measures:
|
||||
- Mandatory proxy configuration (configurable via proxy_required)
|
||||
- Random delay between requests (1-3 seconds by default)
|
||||
- Retry with exponential backoff (3 attempts by default)
|
||||
- Automatic x-zse-96 signature handling via browser
|
||||
|
||||
Attributes:
|
||||
ANSWER_PATTERN: Regex pattern for answer URLs
|
||||
ARTICLE_PATTERN: Regex pattern for zhuanlan article URLs
|
||||
COMMENT_API_TEMPLATE: Template for comment API URL
|
||||
"""
|
||||
|
||||
ANSWER_PATTERN = r"zhihu\.com/question/(\d+)/answer/(\d+)"
|
||||
ARTICLE_PATTERN = r"zhuanlan\.zhihu\.com/p/(\d+)"
|
||||
COMMENT_API_TEMPLATE = (
|
||||
"https://www.zhihu.com/api/v4/comment_v5/"
|
||||
"{content_type}/{content_id}/comments"
|
||||
"?order=hot&limit=20"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
headless: bool = True,
|
||||
timeout: int = 30,
|
||||
config: Optional[CrawlerConfig] = None
|
||||
):
|
||||
"""Initialize ZhihuCrawler.
|
||||
|
||||
Args:
|
||||
headless: Whether to run browser in headless mode
|
||||
timeout: Request timeout in seconds
|
||||
config: Optional CrawlerConfig instance (uses environment if not provided)
|
||||
"""
|
||||
self.headless = headless
|
||||
self.timeout = timeout
|
||||
self.config = config or get_crawler_config()
|
||||
self.browser: Optional[PlaywrightBrowser] = None
|
||||
self.interceptor: Optional[ZhihuApiInterceptor] = None
|
||||
# Internal browser management for proxy support
|
||||
self._playwright = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._page = None
|
||||
|
||||
def _get_cookie(self) -> str:
|
||||
"""Get Cookie from configuration.
|
||||
|
||||
Returns:
|
||||
Cookie string from settings
|
||||
"""
|
||||
# Lazy import to avoid circular dependency:
|
||||
# zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
|
||||
from kb_workflow.pipeline.config import get_settings
|
||||
settings = get_settings()
|
||||
return settings.zhihu_cookie.get_secret_value()
|
||||
|
||||
def _apply_random_delay(self) -> None:
|
||||
"""Apply random delay between requests.
|
||||
|
||||
Uses min_delay and max_delay from configuration.
|
||||
"""
|
||||
delay = random.uniform(self.config.min_delay, self.config.max_delay)
|
||||
time.sleep(delay)
|
||||
|
||||
def _start_browser_with_proxy(self, cookie: str) -> None:
|
||||
"""Start browser with proxy configuration.
|
||||
|
||||
Validates proxy if required and applies to browser context.
|
||||
|
||||
Args:
|
||||
cookie: Cookie string for authentication
|
||||
|
||||
Raises:
|
||||
ValueError: If proxy_required=True but no proxy is configured
|
||||
"""
|
||||
# Validate proxy configuration
|
||||
self.config.validate_proxy()
|
||||
|
||||
self._playwright = sync_playwright().start()
|
||||
self._browser = self._playwright.chromium.launch(headless=self.headless)
|
||||
|
||||
# Build context options
|
||||
context_options = {
|
||||
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
# Apply proxy if configured
|
||||
if self.config.proxy:
|
||||
context_options["proxy"] = self.config.proxy.to_playwright_format()
|
||||
|
||||
# Block Service Worker to prevent interference with API interception
|
||||
context_options["service_workers"] = "block"
|
||||
|
||||
self._context = self._browser.new_context(**context_options)
|
||||
|
||||
# Add cookies if provided
|
||||
if cookie:
|
||||
self._add_cookies(cookie)
|
||||
|
||||
self._page = self._context.new_page()
|
||||
self._page.set_default_timeout(self.timeout * 1000)
|
||||
|
||||
def _add_cookies(self, cookie_str: str) -> None:
|
||||
"""Add cookies to browser context.
|
||||
|
||||
Args:
|
||||
cookie_str: Cookie string in "name=value; name2=value2" format
|
||||
"""
|
||||
cookies = []
|
||||
for item in cookie_str.split(';'):
|
||||
item = item.strip()
|
||||
if '=' in item:
|
||||
name, value = item.split('=', 1)
|
||||
cookies.append({
|
||||
'name': name.strip(),
|
||||
'value': value.strip(),
|
||||
'domain': '.zhihu.com',
|
||||
'path': '/',
|
||||
'secure': True,
|
||||
'httpOnly': False,
|
||||
'sameSite': 'Lax'
|
||||
})
|
||||
self._context.add_cookies(cookies)
|
||||
|
||||
def _close_browser(self) -> None:
|
||||
"""Close browser and cleanup resources."""
|
||||
if self._page:
|
||||
self._page.close()
|
||||
if self._context:
|
||||
self._context.close()
|
||||
if self._browser:
|
||||
self._browser.close()
|
||||
if self._playwright:
|
||||
self._playwright.stop()
|
||||
|
||||
def _crawl_with_retry(self, crawl_func, *args, **kwargs) -> CrawlResult:
|
||||
"""Execute crawl with retry logic.
|
||||
|
||||
Applies random delay before each attempt and retries on failure
|
||||
with exponential backoff.
|
||||
|
||||
Args:
|
||||
crawl_func: Crawl function to execute
|
||||
*args: Positional arguments for crawl_func
|
||||
**kwargs: Keyword arguments for crawl_func
|
||||
|
||||
Returns:
|
||||
CrawlResult from crawl_func or error result after retries exhausted
|
||||
"""
|
||||
try:
|
||||
for attempt in Retrying(
|
||||
stop=stop_after_attempt(self.config.max_retries),
|
||||
wait=wait_exponential(
|
||||
multiplier=1,
|
||||
min=self.config.retry_min_wait,
|
||||
max=self.config.retry_max_wait
|
||||
),
|
||||
reraise=True
|
||||
):
|
||||
with attempt:
|
||||
# Apply random delay before each attempt
|
||||
self._apply_random_delay()
|
||||
return crawl_func(*args, **kwargs)
|
||||
except RetryError as e:
|
||||
return CrawlResult(
|
||||
success=False,
|
||||
error=f"Failed after {self.config.max_retries} retries: {str(e)}"
|
||||
)
|
||||
except Exception as e:
|
||||
return CrawlResult(success=False, error=str(e))
|
||||
|
||||
def _parse_author(self, data: Optional[Dict]) -> Optional[AuthorInfo]:
|
||||
"""Parse author information from API response.
|
||||
|
||||
Args:
|
||||
data: Author data dictionary from API response
|
||||
|
||||
Returns:
|
||||
AuthorInfo object or None if data is empty/None
|
||||
"""
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
# Empty dict should return AuthorInfo with empty name
|
||||
return AuthorInfo(
|
||||
name=data.get("name", ""),
|
||||
url_token=data.get("url_token"),
|
||||
headline=data.get("headline")
|
||||
)
|
||||
|
||||
def _parse_metadata(self, data: Dict) -> ContentMetadata:
|
||||
"""Parse content metadata from API response.
|
||||
|
||||
Args:
|
||||
data: API response data dictionary
|
||||
|
||||
Returns:
|
||||
ContentMetadata object
|
||||
"""
|
||||
return ContentMetadata(
|
||||
author=self._parse_author(data.get("author")),
|
||||
voteup_count=data.get("voteup_count", 0),
|
||||
comment_count=data.get("comment_count", 0),
|
||||
created_time=data.get("created_time"),
|
||||
updated_time=data.get("updated_time")
|
||||
)
|
||||
|
||||
def _parse_answer_data(
|
||||
self,
|
||||
data: Dict,
|
||||
answer_id: int,
|
||||
question_id: int
|
||||
) -> AnswerData:
|
||||
"""Parse answer data from API response.
|
||||
|
||||
Args:
|
||||
data: API response data dictionary
|
||||
answer_id: Answer ID
|
||||
question_id: Question ID
|
||||
|
||||
Returns:
|
||||
AnswerData object
|
||||
"""
|
||||
question = data.get("question", {})
|
||||
return AnswerData(
|
||||
id=answer_id,
|
||||
question_id=question_id,
|
||||
question_title=question.get("title", ""),
|
||||
content=data.get("content", ""),
|
||||
excerpt=data.get("excerpt", ""),
|
||||
metadata=self._parse_metadata(data)
|
||||
)
|
||||
|
||||
def _parse_article_data(self, data: Dict, article_id: int) -> ArticleData:
|
||||
"""Parse article data from API response.
|
||||
|
||||
Args:
|
||||
data: API response data dictionary
|
||||
article_id: Article ID
|
||||
|
||||
Returns:
|
||||
ArticleData object
|
||||
"""
|
||||
return ArticleData(
|
||||
id=article_id,
|
||||
title=data.get("title", ""),
|
||||
content=data.get("content", ""),
|
||||
excerpt=data.get("excerpt", ""),
|
||||
metadata=self._parse_metadata(data)
|
||||
)
|
||||
|
||||
def crawl_comments(
|
||||
self,
|
||||
content_type: str,
|
||||
content_id: int,
|
||||
include_comments: bool = True
|
||||
) -> CommentsResult:
|
||||
"""Crawl hot comments for a Zhihu content.
|
||||
|
||||
Fetches first-level comments using the hot sorting API.
|
||||
Does not expand nested replies.
|
||||
|
||||
Args:
|
||||
content_type: Content type ("answers" or "articles")
|
||||
content_id: Content ID
|
||||
include_comments: Whether to fetch comments (default True)
|
||||
|
||||
Returns:
|
||||
CommentsResult with list of CommentData
|
||||
"""
|
||||
if not include_comments:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
try:
|
||||
self._start_browser_with_proxy(self._get_cookie())
|
||||
|
||||
# Build comment API URL
|
||||
api_url = self.COMMENT_API_TEMPLATE.format(
|
||||
content_type=content_type,
|
||||
content_id=content_id
|
||||
)
|
||||
|
||||
# Use Playwright's request API to directly call the API
|
||||
response = self._page.request.get(api_url)
|
||||
|
||||
if response.status != 200:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
comments = []
|
||||
|
||||
for item in data.get("data", []):
|
||||
author_data = item.get("author", {})
|
||||
comment = CommentData(
|
||||
id=item.get("id", 0),
|
||||
content=item.get("content", ""),
|
||||
author=CommentAuthor(
|
||||
name=author_data.get("name", ""),
|
||||
url_token=author_data.get("url_token")
|
||||
) if author_data else None,
|
||||
created_time=item.get("created_time"),
|
||||
like_count=item.get("like_count", 0),
|
||||
child_count=item.get("child_comment_count", 0),
|
||||
is_author=item.get("is_author", False)
|
||||
)
|
||||
comments.append(comment)
|
||||
|
||||
# Get total count from paging
|
||||
total_count = data.get("paging", {}).get("totals", len(comments))
|
||||
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=comments,
|
||||
total_count=total_count
|
||||
)
|
||||
|
||||
except Exception:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
finally:
|
||||
self._close_browser()
|
||||
|
||||
def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult:
|
||||
"""Crawl Zhihu answer with retry support.
|
||||
|
||||
Args:
|
||||
url: Zhihu answer URL (e.g., https://www.zhihu.com/question/123/answer/456)
|
||||
include_comments: Whether to include hot comments (default False)
|
||||
|
||||
Returns:
|
||||
CrawlResult with AnswerData if successful, error message otherwise.
|
||||
If include_comments=True and successful, comments field will be populated.
|
||||
"""
|
||||
return self._crawl_with_retry(
|
||||
self._crawl_answer_impl, url, include_comments
|
||||
)
|
||||
|
||||
def _crawl_answer_impl(self, url: str, include_comments: bool = False) -> CrawlResult:
|
||||
"""Implementation of crawl_answer without retry wrapper.
|
||||
|
||||
Args:
|
||||
url: Zhihu answer URL
|
||||
include_comments: Whether to include hot comments
|
||||
|
||||
Returns:
|
||||
CrawlResult
|
||||
"""
|
||||
match = re.search(self.ANSWER_PATTERN, url)
|
||||
if not match:
|
||||
return CrawlResult(success=False, error="Invalid answer URL format")
|
||||
|
||||
question_id, answer_id = int(match.group(1)), int(match.group(2))
|
||||
|
||||
try:
|
||||
self._start_browser_with_proxy(self._get_cookie())
|
||||
self.interceptor = ZhihuApiInterceptor()
|
||||
|
||||
# Set up route interception - x-zse-96 handled by browser
|
||||
self._page.route("**/api/v4/**", self.interceptor.handle_route)
|
||||
|
||||
# Visit page to trigger API requests
|
||||
self._page.goto(url, wait_until="networkidle")
|
||||
|
||||
# Wait for data capture
|
||||
time.sleep(2)
|
||||
|
||||
if "answer" not in self.interceptor.captured_data:
|
||||
return CrawlResult(
|
||||
success=False,
|
||||
error="Failed to capture answer data from API"
|
||||
)
|
||||
|
||||
answer_data = self.interceptor.captured_data["answer"]
|
||||
result_data = self._parse_answer_data(
|
||||
answer_data,
|
||||
answer_id=answer_id,
|
||||
question_id=question_id
|
||||
)
|
||||
|
||||
result = CrawlResult(
|
||||
success=True,
|
||||
content_type="answer",
|
||||
data=result_data
|
||||
)
|
||||
|
||||
# Fetch comments if requested
|
||||
if include_comments:
|
||||
comments_result = self.crawl_comments(
|
||||
content_type="answers",
|
||||
content_id=answer_id,
|
||||
include_comments=True
|
||||
)
|
||||
result.comments = comments_result.comments
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
return CrawlResult(success=False, error=str(e))
|
||||
|
||||
finally:
|
||||
self._close_browser()
|
||||
|
||||
def crawl_article(self, url: str) -> CrawlResult:
|
||||
"""Crawl Zhihu zhuanlan article with retry support.
|
||||
|
||||
Args:
|
||||
url: Zhihu zhuanlan URL (e.g., https://zhuanlan.zhihu.com/p/123)
|
||||
|
||||
Returns:
|
||||
CrawlResult with ArticleData if successful, error message otherwise
|
||||
"""
|
||||
return self._crawl_with_retry(self._crawl_article_impl, url)
|
||||
|
||||
def _crawl_article_impl(self, url: str) -> CrawlResult:
|
||||
"""Implementation of crawl_article without retry wrapper.
|
||||
|
||||
Args:
|
||||
url: Zhihu zhuanlan URL
|
||||
|
||||
Returns:
|
||||
CrawlResult
|
||||
"""
|
||||
match = re.search(self.ARTICLE_PATTERN, url)
|
||||
if not match:
|
||||
return CrawlResult(success=False, error="Invalid article URL format")
|
||||
|
||||
article_id = int(match.group(1))
|
||||
|
||||
try:
|
||||
self._start_browser_with_proxy(self._get_cookie())
|
||||
self.interceptor = ZhihuApiInterceptor()
|
||||
|
||||
self._page.route("**/api/v4/**", self.interceptor.handle_route)
|
||||
|
||||
self._page.goto(url, wait_until="networkidle")
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
if "article" not in self.interceptor.captured_data:
|
||||
# Fallback: DOM parsing
|
||||
return self._crawl_article_from_dom(url, article_id)
|
||||
|
||||
article_data = self.interceptor.captured_data["article"]
|
||||
result_data = self._parse_article_data(article_data, article_id)
|
||||
|
||||
return CrawlResult(
|
||||
success=True,
|
||||
content_type="article",
|
||||
data=result_data
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return CrawlResult(success=False, error=str(e))
|
||||
|
||||
finally:
|
||||
self._close_browser()
|
||||
|
||||
def _crawl_article_from_dom(self, url: str, article_id: int) -> CrawlResult:
|
||||
"""Crawl article using DOM parsing as fallback.
|
||||
|
||||
Used when API interception fails for zhuanlan articles.
|
||||
|
||||
Args:
|
||||
url: Article URL
|
||||
article_id: Article ID
|
||||
|
||||
Returns:
|
||||
CrawlResult with ArticleData from DOM extraction
|
||||
"""
|
||||
from kb_workflow.metadata.zhihu_extractor import ZhihuPlaywrightExtractor
|
||||
|
||||
extractor = ZhihuPlaywrightExtractor(
|
||||
headless=self.headless,
|
||||
timeout=self.timeout,
|
||||
cookie=self._get_cookie()
|
||||
)
|
||||
result = extractor.extract(url)
|
||||
|
||||
if not result:
|
||||
return CrawlResult(
|
||||
success=False,
|
||||
error="Failed to extract article from DOM"
|
||||
)
|
||||
|
||||
result_data = ArticleData(
|
||||
id=article_id,
|
||||
title=result.get("title", ""),
|
||||
content=result.get("content", ""),
|
||||
excerpt=result.get("content_markdown", "")[:200],
|
||||
metadata=ContentMetadata(
|
||||
author=AuthorInfo(name=result.get("author", "")),
|
||||
voteup_count=result.get("vote_up", 0)
|
||||
)
|
||||
)
|
||||
|
||||
return CrawlResult(success=True, content_type="article", data=result_data)
|
||||
Reference in New Issue
Block a user