自动同步: 2026-03-18 02:10
This commit is contained in:
@@ -0,0 +1,594 @@
|
||||
"""Zhihu Content Crawler
|
||||
|
||||
Implements content crawling using Playwright API interception.
|
||||
Captures structured JSON data from Zhihu API responses.
|
||||
|
||||
Supports:
|
||||
- Mandatory proxy configuration
|
||||
- Random request delay
|
||||
- Retry with exponential backoff
|
||||
- Automatic x-zse-96 signature handling via browser
|
||||
"""
|
||||
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from playwright.sync_api import Route, Response, sync_playwright
|
||||
from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
|
||||
|
||||
from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser
|
||||
# NOTE: get_settings is imported lazily in _get_cookie() to avoid circular import
|
||||
# Circular: zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
|
||||
from .models import (
|
||||
AnswerData,
|
||||
ArticleData,
|
||||
ContentMetadata,
|
||||
AuthorInfo,
|
||||
CommentData,
|
||||
CommentAuthor,
|
||||
CommentsResult,
|
||||
CrawlResult,
|
||||
)
|
||||
from .config import CrawlerConfig, get_crawler_config
|
||||
|
||||
|
||||
class ZhihuApiInterceptor:
|
||||
"""API Interceptor for capturing Zhihu API responses.
|
||||
|
||||
Intercepts network requests and captures JSON responses from
|
||||
Zhihu API endpoints (/api/v4/answers/, /api/v4/articles/, etc).
|
||||
The x-zse-96 signature is automatically handled by the browser via route.fetch().
|
||||
|
||||
Attributes:
|
||||
captured_data: Dictionary storing captured API response data
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.captured_data: Dict[str, Any] = {}
|
||||
|
||||
def handle_route(self, route: Route) -> None:
|
||||
"""Handle intercepted route/request.
|
||||
|
||||
The route.fetch() call automatically includes browser-generated
|
||||
x-zse-96 signature headers, No manual signature handling needed.
|
||||
|
||||
Args:
|
||||
route: Playwright Route object for the intercepted request
|
||||
"""
|
||||
url = route.request.url
|
||||
|
||||
# Intercept answer API
|
||||
if "/api/v4/answers/" in url:
|
||||
response = route.fetch()
|
||||
self.captured_data["answer"] = response.json()
|
||||
route.fulfill(response=response)
|
||||
return
|
||||
|
||||
# Intercept article API (zhuanlan)
|
||||
if "/api/v4/articles/" in url or "zhuanlan" in url:
|
||||
response = route.fetch()
|
||||
self.captured_data["article"] = response.json()
|
||||
route.fulfill(response=response)
|
||||
return
|
||||
|
||||
route.continue_()
|
||||
|
||||
|
||||
class ZhihuCrawler:
|
||||
"""Zhihu Content Crawler.
|
||||
|
||||
Crawls Zhihu answers and articles using Playwright API interception.
|
||||
Includes anti-scraping measures:
|
||||
- Mandatory proxy configuration (configurable via proxy_required)
|
||||
- Random delay between requests (1-3 seconds by default)
|
||||
- Retry with exponential backoff (3 attempts by default)
|
||||
- Automatic x-zse-96 signature handling via browser
|
||||
|
||||
Attributes:
|
||||
ANSWER_PATTERN: Regex pattern for answer URLs
|
||||
ARTICLE_PATTERN: Regex pattern for zhuanlan article URLs
|
||||
COMMENT_API_TEMPLATE: Template for comment API URL
|
||||
"""
|
||||
|
||||
ANSWER_PATTERN = r"zhihu\.com/question/(\d+)/answer/(\d+)"
|
||||
ARTICLE_PATTERN = r"zhuanlan\.zhihu\.com/p/(\d+)"
|
||||
COMMENT_API_TEMPLATE = (
|
||||
"https://www.zhihu.com/api/v4/comment_v5/"
|
||||
"{content_type}/{content_id}/comments"
|
||||
"?order=hot&limit=20"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
headless: bool = True,
|
||||
timeout: int = 30,
|
||||
config: Optional[CrawlerConfig] = None
|
||||
):
|
||||
"""Initialize ZhihuCrawler.
|
||||
|
||||
Args:
|
||||
headless: Whether to run browser in headless mode
|
||||
timeout: Request timeout in seconds
|
||||
config: Optional CrawlerConfig instance (uses environment if not provided)
|
||||
"""
|
||||
self.headless = headless
|
||||
self.timeout = timeout
|
||||
self.config = config or get_crawler_config()
|
||||
self.browser: Optional[PlaywrightBrowser] = None
|
||||
self.interceptor: Optional[ZhihuApiInterceptor] = None
|
||||
# Internal browser management for proxy support
|
||||
self._playwright = None
|
||||
self._browser = None
|
||||
self._context = None
|
||||
self._page = None
|
||||
|
||||
def _get_cookie(self) -> str:
|
||||
"""Get Cookie from configuration.
|
||||
|
||||
Returns:
|
||||
Cookie string from settings
|
||||
"""
|
||||
# Lazy import to avoid circular dependency:
|
||||
# zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
|
||||
from kb_workflow.pipeline.config import get_settings
|
||||
settings = get_settings()
|
||||
return settings.zhihu_cookie.get_secret_value()
|
||||
|
||||
def _apply_random_delay(self) -> None:
|
||||
"""Apply random delay between requests.
|
||||
|
||||
Uses min_delay and max_delay from configuration.
|
||||
"""
|
||||
delay = random.uniform(self.config.min_delay, self.config.max_delay)
|
||||
time.sleep(delay)
|
||||
|
||||
def _start_browser_with_proxy(self, cookie: str) -> None:
|
||||
"""Start browser with proxy configuration.
|
||||
|
||||
Validates proxy if required and applies to browser context.
|
||||
|
||||
Args:
|
||||
cookie: Cookie string for authentication
|
||||
|
||||
Raises:
|
||||
ValueError: If proxy_required=True but no proxy is configured
|
||||
"""
|
||||
# Validate proxy configuration
|
||||
self.config.validate_proxy()
|
||||
|
||||
self._playwright = sync_playwright().start()
|
||||
self._browser = self._playwright.chromium.launch(headless=self.headless)
|
||||
|
||||
# Build context options
|
||||
context_options = {
|
||||
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||||
}
|
||||
|
||||
# Apply proxy if configured
|
||||
if self.config.proxy:
|
||||
context_options["proxy"] = self.config.proxy.to_playwright_format()
|
||||
|
||||
# Block Service Worker to prevent interference with API interception
|
||||
context_options["service_workers"] = "block"
|
||||
|
||||
self._context = self._browser.new_context(**context_options)
|
||||
|
||||
# Add cookies if provided
|
||||
if cookie:
|
||||
self._add_cookies(cookie)
|
||||
|
||||
self._page = self._context.new_page()
|
||||
self._page.set_default_timeout(self.timeout * 1000)
|
||||
|
||||
def _add_cookies(self, cookie_str: str) -> None:
|
||||
"""Add cookies to browser context.
|
||||
|
||||
Args:
|
||||
cookie_str: Cookie string in "name=value; name2=value2" format
|
||||
"""
|
||||
cookies = []
|
||||
for item in cookie_str.split(';'):
|
||||
item = item.strip()
|
||||
if '=' in item:
|
||||
name, value = item.split('=', 1)
|
||||
cookies.append({
|
||||
'name': name.strip(),
|
||||
'value': value.strip(),
|
||||
'domain': '.zhihu.com',
|
||||
'path': '/',
|
||||
'secure': True,
|
||||
'httpOnly': False,
|
||||
'sameSite': 'Lax'
|
||||
})
|
||||
self._context.add_cookies(cookies)
|
||||
|
||||
def _close_browser(self) -> None:
|
||||
"""Close browser and cleanup resources."""
|
||||
if self._page:
|
||||
self._page.close()
|
||||
if self._context:
|
||||
self._context.close()
|
||||
if self._browser:
|
||||
self._browser.close()
|
||||
if self._playwright:
|
||||
self._playwright.stop()
|
||||
|
||||
def _crawl_with_retry(self, crawl_func, *args, **kwargs) -> CrawlResult:
|
||||
"""Execute crawl with retry logic.
|
||||
|
||||
Applies random delay before each attempt and retries on failure
|
||||
with exponential backoff.
|
||||
|
||||
Args:
|
||||
crawl_func: Crawl function to execute
|
||||
*args: Positional arguments for crawl_func
|
||||
**kwargs: Keyword arguments for crawl_func
|
||||
|
||||
Returns:
|
||||
CrawlResult from crawl_func or error result after retries exhausted
|
||||
"""
|
||||
try:
|
||||
for attempt in Retrying(
|
||||
stop=stop_after_attempt(self.config.max_retries),
|
||||
wait=wait_exponential(
|
||||
multiplier=1,
|
||||
min=self.config.retry_min_wait,
|
||||
max=self.config.retry_max_wait
|
||||
),
|
||||
reraise=True
|
||||
):
|
||||
with attempt:
|
||||
# Apply random delay before each attempt
|
||||
self._apply_random_delay()
|
||||
return crawl_func(*args, **kwargs)
|
||||
except RetryError as e:
|
||||
return CrawlResult(
|
||||
success=False,
|
||||
error=f"Failed after {self.config.max_retries} retries: {str(e)}"
|
||||
)
|
||||
except Exception as e:
|
||||
return CrawlResult(success=False, error=str(e))
|
||||
|
||||
def _parse_author(self, data: Optional[Dict]) -> Optional[AuthorInfo]:
|
||||
"""Parse author information from API response.
|
||||
|
||||
Args:
|
||||
data: Author data dictionary from API response
|
||||
|
||||
Returns:
|
||||
AuthorInfo object or None if data is empty/None
|
||||
"""
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
# Empty dict should return AuthorInfo with empty name
|
||||
return AuthorInfo(
|
||||
name=data.get("name", ""),
|
||||
url_token=data.get("url_token"),
|
||||
headline=data.get("headline")
|
||||
)
|
||||
|
||||
def _parse_metadata(self, data: Dict) -> ContentMetadata:
|
||||
"""Parse content metadata from API response.
|
||||
|
||||
Args:
|
||||
data: API response data dictionary
|
||||
|
||||
Returns:
|
||||
ContentMetadata object
|
||||
"""
|
||||
return ContentMetadata(
|
||||
author=self._parse_author(data.get("author")),
|
||||
voteup_count=data.get("voteup_count", 0),
|
||||
comment_count=data.get("comment_count", 0),
|
||||
created_time=data.get("created_time"),
|
||||
updated_time=data.get("updated_time")
|
||||
)
|
||||
|
||||
def _parse_answer_data(
|
||||
self,
|
||||
data: Dict,
|
||||
answer_id: int,
|
||||
question_id: int
|
||||
) -> AnswerData:
|
||||
"""Parse answer data from API response.
|
||||
|
||||
Args:
|
||||
data: API response data dictionary
|
||||
answer_id: Answer ID
|
||||
question_id: Question ID
|
||||
|
||||
Returns:
|
||||
AnswerData object
|
||||
"""
|
||||
question = data.get("question", {})
|
||||
return AnswerData(
|
||||
id=answer_id,
|
||||
question_id=question_id,
|
||||
question_title=question.get("title", ""),
|
||||
content=data.get("content", ""),
|
||||
excerpt=data.get("excerpt", ""),
|
||||
metadata=self._parse_metadata(data)
|
||||
)
|
||||
|
||||
def _parse_article_data(self, data: Dict, article_id: int) -> ArticleData:
|
||||
"""Parse article data from API response.
|
||||
|
||||
Args:
|
||||
data: API response data dictionary
|
||||
article_id: Article ID
|
||||
|
||||
Returns:
|
||||
ArticleData object
|
||||
"""
|
||||
return ArticleData(
|
||||
id=article_id,
|
||||
title=data.get("title", ""),
|
||||
content=data.get("content", ""),
|
||||
excerpt=data.get("excerpt", ""),
|
||||
metadata=self._parse_metadata(data)
|
||||
)
|
||||
|
||||
def crawl_comments(
|
||||
self,
|
||||
content_type: str,
|
||||
content_id: int,
|
||||
include_comments: bool = True
|
||||
) -> CommentsResult:
|
||||
"""Crawl hot comments for a Zhihu content.
|
||||
|
||||
Fetches first-level comments using the hot sorting API.
|
||||
Does not expand nested replies.
|
||||
|
||||
Args:
|
||||
content_type: Content type ("answers" or "articles")
|
||||
content_id: Content ID
|
||||
include_comments: Whether to fetch comments (default True)
|
||||
|
||||
Returns:
|
||||
CommentsResult with list of CommentData
|
||||
"""
|
||||
if not include_comments:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
try:
|
||||
self._start_browser_with_proxy(self._get_cookie())
|
||||
|
||||
# Build comment API URL
|
||||
api_url = self.COMMENT_API_TEMPLATE.format(
|
||||
content_type=content_type,
|
||||
content_id=content_id
|
||||
)
|
||||
|
||||
# Use Playwright's request API to directly call the API
|
||||
response = self._page.request.get(api_url)
|
||||
|
||||
if response.status != 200:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
comments = []
|
||||
|
||||
for item in data.get("data", []):
|
||||
author_data = item.get("author", {})
|
||||
comment = CommentData(
|
||||
id=item.get("id", 0),
|
||||
content=item.get("content", ""),
|
||||
author=CommentAuthor(
|
||||
name=author_data.get("name", ""),
|
||||
url_token=author_data.get("url_token")
|
||||
) if author_data else None,
|
||||
created_time=item.get("created_time"),
|
||||
like_count=item.get("like_count", 0),
|
||||
child_count=item.get("child_comment_count", 0),
|
||||
is_author=item.get("is_author", False)
|
||||
)
|
||||
comments.append(comment)
|
||||
|
||||
# Get total count from paging
|
||||
total_count = data.get("paging", {}).get("totals", len(comments))
|
||||
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=comments,
|
||||
total_count=total_count
|
||||
)
|
||||
|
||||
except Exception:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
finally:
|
||||
self._close_browser()
|
||||
|
||||
def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult:
|
||||
"""Crawl Zhihu answer with retry support.
|
||||
|
||||
Args:
|
||||
url: Zhihu answer URL (e.g., https://www.zhihu.com/question/123/answer/456)
|
||||
include_comments: Whether to include hot comments (default False)
|
||||
|
||||
Returns:
|
||||
CrawlResult with AnswerData if successful, error message otherwise.
|
||||
If include_comments=True and successful, comments field will be populated.
|
||||
"""
|
||||
return self._crawl_with_retry(
|
||||
self._crawl_answer_impl, url, include_comments
|
||||
)
|
||||
|
||||
def _crawl_answer_impl(self, url: str, include_comments: bool = False) -> CrawlResult:
|
||||
"""Implementation of crawl_answer without retry wrapper.
|
||||
|
||||
Args:
|
||||
url: Zhihu answer URL
|
||||
include_comments: Whether to include hot comments
|
||||
|
||||
Returns:
|
||||
CrawlResult
|
||||
"""
|
||||
match = re.search(self.ANSWER_PATTERN, url)
|
||||
if not match:
|
||||
return CrawlResult(success=False, error="Invalid answer URL format")
|
||||
|
||||
question_id, answer_id = int(match.group(1)), int(match.group(2))
|
||||
|
||||
try:
|
||||
self._start_browser_with_proxy(self._get_cookie())
|
||||
self.interceptor = ZhihuApiInterceptor()
|
||||
|
||||
# Set up route interception - x-zse-96 handled by browser
|
||||
self._page.route("**/api/v4/**", self.interceptor.handle_route)
|
||||
|
||||
# Visit page to trigger API requests
|
||||
self._page.goto(url, wait_until="networkidle")
|
||||
|
||||
# Wait for data capture
|
||||
time.sleep(2)
|
||||
|
||||
if "answer" not in self.interceptor.captured_data:
|
||||
return CrawlResult(
|
||||
success=False,
|
||||
error="Failed to capture answer data from API"
|
||||
)
|
||||
|
||||
answer_data = self.interceptor.captured_data["answer"]
|
||||
result_data = self._parse_answer_data(
|
||||
answer_data,
|
||||
answer_id=answer_id,
|
||||
question_id=question_id
|
||||
)
|
||||
|
||||
result = CrawlResult(
|
||||
success=True,
|
||||
content_type="answer",
|
||||
data=result_data
|
||||
)
|
||||
|
||||
# Fetch comments if requested
|
||||
if include_comments:
|
||||
comments_result = self.crawl_comments(
|
||||
content_type="answers",
|
||||
content_id=answer_id,
|
||||
include_comments=True
|
||||
)
|
||||
result.comments = comments_result.comments
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
return CrawlResult(success=False, error=str(e))
|
||||
|
||||
finally:
|
||||
self._close_browser()
|
||||
|
||||
def crawl_article(self, url: str) -> CrawlResult:
|
||||
"""Crawl Zhihu zhuanlan article with retry support.
|
||||
|
||||
Args:
|
||||
url: Zhihu zhuanlan URL (e.g., https://zhuanlan.zhihu.com/p/123)
|
||||
|
||||
Returns:
|
||||
CrawlResult with ArticleData if successful, error message otherwise
|
||||
"""
|
||||
return self._crawl_with_retry(self._crawl_article_impl, url)
|
||||
|
||||
def _crawl_article_impl(self, url: str) -> CrawlResult:
|
||||
"""Implementation of crawl_article without retry wrapper.
|
||||
|
||||
Args:
|
||||
url: Zhihu zhuanlan URL
|
||||
|
||||
Returns:
|
||||
CrawlResult
|
||||
"""
|
||||
match = re.search(self.ARTICLE_PATTERN, url)
|
||||
if not match:
|
||||
return CrawlResult(success=False, error="Invalid article URL format")
|
||||
|
||||
article_id = int(match.group(1))
|
||||
|
||||
try:
|
||||
self._start_browser_with_proxy(self._get_cookie())
|
||||
self.interceptor = ZhihuApiInterceptor()
|
||||
|
||||
self._page.route("**/api/v4/**", self.interceptor.handle_route)
|
||||
|
||||
self._page.goto(url, wait_until="networkidle")
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
if "article" not in self.interceptor.captured_data:
|
||||
# Fallback: DOM parsing
|
||||
return self._crawl_article_from_dom(url, article_id)
|
||||
|
||||
article_data = self.interceptor.captured_data["article"]
|
||||
result_data = self._parse_article_data(article_data, article_id)
|
||||
|
||||
return CrawlResult(
|
||||
success=True,
|
||||
content_type="article",
|
||||
data=result_data
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return CrawlResult(success=False, error=str(e))
|
||||
|
||||
finally:
|
||||
self._close_browser()
|
||||
|
||||
def _crawl_article_from_dom(self, url: str, article_id: int) -> CrawlResult:
|
||||
"""Crawl article using DOM parsing as fallback.
|
||||
|
||||
Used when API interception fails for zhuanlan articles.
|
||||
|
||||
Args:
|
||||
url: Article URL
|
||||
article_id: Article ID
|
||||
|
||||
Returns:
|
||||
CrawlResult with ArticleData from DOM extraction
|
||||
"""
|
||||
from kb_workflow.metadata.zhihu_extractor import ZhihuPlaywrightExtractor
|
||||
|
||||
extractor = ZhihuPlaywrightExtractor(
|
||||
headless=self.headless,
|
||||
timeout=self.timeout,
|
||||
cookie=self._get_cookie()
|
||||
)
|
||||
result = extractor.extract(url)
|
||||
|
||||
if not result:
|
||||
return CrawlResult(
|
||||
success=False,
|
||||
error="Failed to extract article from DOM"
|
||||
)
|
||||
|
||||
result_data = ArticleData(
|
||||
id=article_id,
|
||||
title=result.get("title", ""),
|
||||
content=result.get("content", ""),
|
||||
excerpt=result.get("content_markdown", "")[:200],
|
||||
metadata=ContentMetadata(
|
||||
author=AuthorInfo(name=result.get("author", "")),
|
||||
voteup_count=result.get("vote_up", 0)
|
||||
)
|
||||
)
|
||||
|
||||
return CrawlResult(success=True, content_type="article", data=result_data)
|
||||
Reference in New Issue
Block a user