starmen1265/kb-skills
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(08-03): integrate proxy, delay, and retry into ZhihuCrawler

Browse Source 
- Add CrawlerConfig with ProxyConfig, delay, and retry settings
- Integrate proxy configuration via _start_browser_with_proxy()
- Add random delay via _apply_random_delay()
- Add retry logic via _crawl_with_retry using tenacity
- Add ZhihuApiInterceptor tests for x-zse-96 handling
- All tests pass
This commit is contained in:
starmen1265
2026-03-17 22:14:47 +08:00
parent 6a4f13b2f6
commit 9e1452d2a9
3 changed files with 434 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kb_workflow/crawler/zhihu_crawler.py
+187 -37
View File
@@ -2,13 +2,21 @@
Implements content crawling using Playwright API interception.
Captures structured JSON data from Zhihu API responses.
Supports:
- Mandatory proxy configuration
- Random request delay
- Retry with exponential backoff
- Automatic x-zse-96 signature handling via browser
"""
import random
import re
import time
from typing import Optional, Dict, Any
from playwright.sync_api import Route, Response
from playwright.sync_api import Route, Response, sync_playwright
from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser
from kb_workflow.pipeline.config import get_settings
@@ -22,13 +30,15 @@ from .models import (
CommentsResult,
CrawlResult,
)
from .config import CrawlerConfig, get_crawler_config
class ZhihuApiInterceptor:
"""API Interceptor for capturing Zhihu API responses.
Intercepts network requests and captures JSON responses from
Zhihu API endpoints (/api/v4/answers/, /api/v4/articles/, etc).
Zhihu API endpoints (/api/v4/answers/, /api/v4/articles/, etc).
The x-zse-96 signature is automatically handled by the browser via route.fetch().
Attributes:
captured_data: Dictionary storing captured API response data
@@ -40,6 +50,9 @@ class ZhihuApiInterceptor:
def handle_route(self, route: Route) -> None:
"""Handle intercepted route/request.
The route.fetch() call automatically includes browser-generated
x-zse-96 signature headers, No manual signature handling needed.
Args:
route: Playwright Route object for the intercepted request
"""
@@ -66,7 +79,11 @@ class ZhihuCrawler:
"""Zhihu Content Crawler.
Crawls Zhihu answers and articles using Playwright API interception.
Reuses Cookie management from Phase 7 auth module.
Includes anti-scraping measures:
- Mandatory proxy configuration (configurable via proxy_required)
- Random delay between requests (1-3 seconds by default)
- Retry with exponential backoff (3 attempts by default)
- Automatic x-zse-96 signature handling via browser
Attributes:
ANSWER_PATTERN: Regex pattern for answer URLs
@@ -82,17 +99,29 @@ class ZhihuCrawler:
"?order=hot&limit=20"
)
def __init__(self, headless: bool = True, timeout: int = 30):
def __init__(
self,
headless: bool = True,
timeout: int = 30,
config: Optional[CrawlerConfig] = None
):
"""Initialize ZhihuCrawler.
Args:
headless: Whether to run browser in headless mode
timeout: Request timeout in seconds
config: Optional CrawlerConfig instance (uses environment if not provided)
"""
self.headless = headless
self.timeout = timeout
self.config = config or get_crawler_config()
self.browser: Optional[PlaywrightBrowser] = None
self.interceptor: Optional[ZhihuApiInterceptor] = None
# Internal browser management for proxy support
self._playwright = None
self._browser = None
self._context = None
self._page = None
def _get_cookie(self) -> str:
"""Get Cookie from configuration.
@@ -103,6 +132,121 @@ class ZhihuCrawler:
settings = get_settings()
return settings.zhihu_cookie.get_secret_value()
def _apply_random_delay(self) -> None:
"""Apply random delay between requests.
Uses min_delay and max_delay from configuration.
"""
delay = random.uniform(self.config.min_delay, self.config.max_delay)
time.sleep(delay)
def _start_browser_with_proxy(self, cookie: str) -> None:
"""Start browser with proxy configuration.
Validates proxy if required and applies to browser context.
Args:
cookie: Cookie string for authentication
Raises:
ValueError: If proxy_required=True but no proxy is configured
"""
# Validate proxy configuration
self.config.validate_proxy()
self._playwright = sync_playwright().start()
self._browser = self._playwright.chromium.launch(headless=self.headless)
# Build context options
context_options = {
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
# Apply proxy if configured
if self.config.proxy:
context_options["proxy"] = self.config.proxy.to_playwright_format()
# Block Service Worker to prevent interference with API interception
context_options["service_workers"] = "block"
self._context = self._browser.new_context(**context_options)
# Add cookies if provided
if cookie:
self._add_cookies(cookie)
self._page = self._context.new_page()
self._page.set_default_timeout(self.timeout * 1000)
def _add_cookies(self, cookie_str: str) -> None:
"""Add cookies to browser context.
Args:
cookie_str: Cookie string in "name=value; name2=value2" format
"""
cookies = []
for item in cookie_str.split(';'):
item = item.strip()
if '=' in item:
name, value = item.split('=', 1)
cookies.append({
'name': name.strip(),
'value': value.strip(),
'domain': '.zhihu.com',
'path': '/',
'secure': True,
'httpOnly': False,
'sameSite': 'Lax'
})
self._context.add_cookies(cookies)
def _close_browser(self) -> None:
"""Close browser and cleanup resources."""
if self._page:
self._page.close()
if self._context:
self._context.close()
if self._browser:
self._browser.close()
if self._playwright:
self._playwright.stop()
def _crawl_with_retry(self, crawl_func, *args, **kwargs) -> CrawlResult:
"""Execute crawl with retry logic.
Applies random delay before each attempt and retries on failure
with exponential backoff.
Args:
crawl_func: Crawl function to execute
*args: Positional arguments for crawl_func
**kwargs: Keyword arguments for crawl_func
Returns:
CrawlResult from crawl_func or error result after retries exhausted
"""
try:
for attempt in Retrying(
stop=stop_after_attempt(self.config.max_retries),
wait=wait_exponential(
multiplier=1,
min=self.config.retry_min_wait,
max=self.config.retry_max_wait
),
reraise=True
):
with attempt:
# Apply random delay before each attempt
self._apply_random_delay()
return crawl_func(*args, **kwargs)
except RetryError as e:
return CrawlResult(
success=False,
error=f"Failed after {self.config.max_retries} retries: {str(e)}"
)
except Exception as e:
return CrawlResult(success=False, error=str(e))
def _parse_author(self, data: Optional[Dict]) -> Optional[AuthorInfo]:
"""Parse author information from API response.
@@ -211,11 +355,7 @@ class ZhihuCrawler:
)
try:
self.browser = PlaywrightBrowser(
headless=self.headless,
timeout=self.timeout
)
self.browser.start(cookie=self._get_cookie())
self._start_browser_with_proxy(self._get_cookie())
# Build comment API URL
api_url = self.COMMENT_API_TEMPLATE.format(
@@ -224,7 +364,7 @@ class ZhihuCrawler:
)
# Use Playwright's request API to directly call the API
response = self.browser.page.request.get(api_url)
response = self._page.request.get(api_url)
if response.status != 200:
return CommentsResult(
@@ -272,11 +412,10 @@ class ZhihuCrawler:
)
finally:
if self.browser:
self.browser.close()
self._close_browser()
def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult:
"""Crawl Zhihu answer.
"""Crawl Zhihu answer with retry support.
Args:
url: Zhihu answer URL (e.g., https://www.zhihu.com/question/123/answer/456)
@@ -286,6 +425,20 @@ class ZhihuCrawler:
CrawlResult with AnswerData if successful, error message otherwise.
If include_comments=True and successful, comments field will be populated.
"""
return self._crawl_with_retry(
self._crawl_answer_impl, url, include_comments
)
def _crawl_answer_impl(self, url: str, include_comments: bool = False) -> CrawlResult:
"""Implementation of crawl_answer without retry wrapper.
Args:
url: Zhihu answer URL
include_comments: Whether to include hot comments
Returns:
CrawlResult
"""
match = re.search(self.ANSWER_PATTERN, url)
if not match:
return CrawlResult(success=False, error="Invalid answer URL format")
@@ -293,18 +446,14 @@ class ZhihuCrawler:
question_id, answer_id = int(match.group(1)), int(match.group(2))
try:
self.browser = PlaywrightBrowser(
headless=self.headless,
timeout=self.timeout
)
self.browser.start(cookie=self._get_cookie())
self._start_browser_with_proxy(self._get_cookie())
self.interceptor = ZhihuApiInterceptor()
# Set up route interception
self.browser.page.route("**/api/v4/**", self.interceptor.handle_route)
# Set up route interception - x-zse-96 handled by browser
self._page.route("**/api/v4/**", self.interceptor.handle_route)
# Visit page to trigger API requests
self.browser.goto(url, wait_until="networkidle")
self._page.goto(url, wait_until="networkidle")
# Wait for data capture
time.sleep(2)
@@ -330,10 +479,6 @@ class ZhihuCrawler:
# Fetch comments if requested
if include_comments:
# Need to close current browser first as crawl_comments creates its own
self.browser.close()
self.browser = None
comments_result = self.crawl_comments(
content_type="answers",
content_id=answer_id,
@@ -347,11 +492,10 @@ class ZhihuCrawler:
return CrawlResult(success=False, error=str(e))
finally:
if self.browser:
self.browser.close()
self._close_browser()
def crawl_article(self, url: str) -> CrawlResult:
"""Crawl Zhihu zhuanlan article.
"""Crawl Zhihu zhuanlan article with retry support.
Args:
url: Zhihu zhuanlan URL (e.g., https://zhuanlan.zhihu.com/p/123)
@@ -359,6 +503,17 @@ class ZhihuCrawler:
Returns:
CrawlResult with ArticleData if successful, error message otherwise
"""
return self._crawl_with_retry(self._crawl_article_impl, url)
def _crawl_article_impl(self, url: str) -> CrawlResult:
"""Implementation of crawl_article without retry wrapper.
Args:
url: Zhihu zhuanlan URL
Returns:
CrawlResult
"""
match = re.search(self.ARTICLE_PATTERN, url)
if not match:
return CrawlResult(success=False, error="Invalid article URL format")
@@ -366,16 +521,12 @@ class ZhihuCrawler:
article_id = int(match.group(1))
try:
self.browser = PlaywrightBrowser(
headless=self.headless,
timeout=self.timeout
)
self.browser.start(cookie=self._get_cookie())
self._start_browser_with_proxy(self._get_cookie())
self.interceptor = ZhihuApiInterceptor()
self.browser.page.route("**/api/v4/**", self.interceptor.handle_route)
self._page.route("**/api/v4/**", self.interceptor.handle_route)
self.browser.goto(url, wait_until="networkidle")
self._page.goto(url, wait_until="networkidle")
time.sleep(2)
@@ -396,8 +547,7 @@ class ZhihuCrawler:
return CrawlResult(success=False, error=str(e))
finally:
if self.browser:
self.browser.close()
self._close_browser()
def _crawl_article_from_dom(self, url: str, article_id: int) -> CrawlResult:
"""Crawl article using DOM parsing as fallback.