feat(11-02): integrate stealth fingerprint masking into ZhihuCrawler
- Add stealth_config parameter to ZhihuCrawler.__init__ - Add --disable-blink-features=AutomationControlled to launch args - Call apply_stealth_sync after browser context creation - Maintain backward compatibility (stealth enabled by default) - Add 8 integration tests covering stealth behavior and compatibility Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -35,6 +35,7 @@ from .models import (
|
||||
)
|
||||
from .config import CrawlerConfig, get_crawler_config
|
||||
from .proxy_pool import ProxyPool, ProxyEntry
|
||||
from .stealth import apply_stealth_sync, StealthConfig
|
||||
|
||||
|
||||
class ZhihuApiInterceptor:
|
||||
@@ -108,7 +109,8 @@ class ZhihuCrawler:
|
||||
headless: bool = True,
|
||||
timeout: int = 30,
|
||||
config: Optional[CrawlerConfig] = None,
|
||||
proxy_pool: Optional[ProxyPool] = None
|
||||
proxy_pool: Optional[ProxyPool] = None,
|
||||
stealth_config: Optional[StealthConfig] = None
|
||||
):
|
||||
"""Initialize ZhihuCrawler.
|
||||
|
||||
@@ -117,11 +119,13 @@ class ZhihuCrawler:
|
||||
timeout: Request timeout in seconds
|
||||
config: Optional CrawlerConfig instance (uses environment if not provided)
|
||||
proxy_pool: Optional ProxyPool instance for multi-proxy rotation
|
||||
stealth_config: Optional StealthConfig instance (uses defaults if not provided)
|
||||
"""
|
||||
self.headless = headless
|
||||
self.timeout = timeout
|
||||
self.config = config or get_crawler_config()
|
||||
self.proxy_pool = proxy_pool
|
||||
self._stealth_config = stealth_config
|
||||
self._current_proxy: Optional[ProxyEntry] = None
|
||||
self.browser: Optional[PlaywrightBrowser] = None
|
||||
self.interceptor: Optional[ZhihuApiInterceptor] = None
|
||||
@@ -155,6 +159,7 @@ class ZhihuCrawler:
|
||||
"""Start browser with proxy configuration.
|
||||
|
||||
Validates proxy if required and applies to browser context.
|
||||
Applies stealth fingerprint masking after context creation.
|
||||
Prioritizes proxy_pool over config.proxy for multi-proxy support.
|
||||
|
||||
Args:
|
||||
@@ -166,8 +171,16 @@ class ZhihuCrawler:
|
||||
# Validate proxy configuration
|
||||
self.config.validate_proxy()
|
||||
|
||||
# Build launch args for stealth (add --disable-blink-features)
|
||||
launch_args = []
|
||||
if self._stealth_config is None or self._stealth_config.stealth_enabled:
|
||||
launch_args.append('--disable-blink-features=AutomationControlled')
|
||||
|
||||
self._playwright = sync_playwright().start()
|
||||
self._browser = self._playwright.chromium.launch(headless=self.headless)
|
||||
self._browser = self._playwright.chromium.launch(
|
||||
headless=self.headless,
|
||||
args=launch_args if launch_args else None
|
||||
)
|
||||
|
||||
# Build context options
|
||||
context_options = {
|
||||
@@ -190,6 +203,9 @@ class ZhihuCrawler:
|
||||
|
||||
self._context = self._browser.new_context(**context_options)
|
||||
|
||||
# Apply stealth fingerprint masking
|
||||
apply_stealth_sync(self._context, self._stealth_config)
|
||||
|
||||
# Add cookies if provided
|
||||
if cookie:
|
||||
self._add_cookies(cookie)
|
||||
|
||||
Reference in New Issue
Block a user