feat(11-02): integrate stealth fingerprint masking into ZhihuCrawler
- Add stealth_config parameter to ZhihuCrawler.__init__ - Add --disable-blink-features=AutomationControlled to launch args - Call apply_stealth_sync after browser context creation - Maintain backward compatibility (stealth enabled by default) - Add 8 integration tests covering stealth behavior and compatibility Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -35,6 +35,7 @@ from .models import (
|
|||||||
)
|
)
|
||||||
from .config import CrawlerConfig, get_crawler_config
|
from .config import CrawlerConfig, get_crawler_config
|
||||||
from .proxy_pool import ProxyPool, ProxyEntry
|
from .proxy_pool import ProxyPool, ProxyEntry
|
||||||
|
from .stealth import apply_stealth_sync, StealthConfig
|
||||||
|
|
||||||
|
|
||||||
class ZhihuApiInterceptor:
|
class ZhihuApiInterceptor:
|
||||||
@@ -108,7 +109,8 @@ class ZhihuCrawler:
|
|||||||
headless: bool = True,
|
headless: bool = True,
|
||||||
timeout: int = 30,
|
timeout: int = 30,
|
||||||
config: Optional[CrawlerConfig] = None,
|
config: Optional[CrawlerConfig] = None,
|
||||||
proxy_pool: Optional[ProxyPool] = None
|
proxy_pool: Optional[ProxyPool] = None,
|
||||||
|
stealth_config: Optional[StealthConfig] = None
|
||||||
):
|
):
|
||||||
"""Initialize ZhihuCrawler.
|
"""Initialize ZhihuCrawler.
|
||||||
|
|
||||||
@@ -117,11 +119,13 @@ class ZhihuCrawler:
|
|||||||
timeout: Request timeout in seconds
|
timeout: Request timeout in seconds
|
||||||
config: Optional CrawlerConfig instance (uses environment if not provided)
|
config: Optional CrawlerConfig instance (uses environment if not provided)
|
||||||
proxy_pool: Optional ProxyPool instance for multi-proxy rotation
|
proxy_pool: Optional ProxyPool instance for multi-proxy rotation
|
||||||
|
stealth_config: Optional StealthConfig instance (uses defaults if not provided)
|
||||||
"""
|
"""
|
||||||
self.headless = headless
|
self.headless = headless
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.config = config or get_crawler_config()
|
self.config = config or get_crawler_config()
|
||||||
self.proxy_pool = proxy_pool
|
self.proxy_pool = proxy_pool
|
||||||
|
self._stealth_config = stealth_config
|
||||||
self._current_proxy: Optional[ProxyEntry] = None
|
self._current_proxy: Optional[ProxyEntry] = None
|
||||||
self.browser: Optional[PlaywrightBrowser] = None
|
self.browser: Optional[PlaywrightBrowser] = None
|
||||||
self.interceptor: Optional[ZhihuApiInterceptor] = None
|
self.interceptor: Optional[ZhihuApiInterceptor] = None
|
||||||
@@ -155,6 +159,7 @@ class ZhihuCrawler:
|
|||||||
"""Start browser with proxy configuration.
|
"""Start browser with proxy configuration.
|
||||||
|
|
||||||
Validates proxy if required and applies to browser context.
|
Validates proxy if required and applies to browser context.
|
||||||
|
Applies stealth fingerprint masking after context creation.
|
||||||
Prioritizes proxy_pool over config.proxy for multi-proxy support.
|
Prioritizes proxy_pool over config.proxy for multi-proxy support.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -166,8 +171,16 @@ class ZhihuCrawler:
|
|||||||
# Validate proxy configuration
|
# Validate proxy configuration
|
||||||
self.config.validate_proxy()
|
self.config.validate_proxy()
|
||||||
|
|
||||||
|
# Build launch args for stealth (add --disable-blink-features)
|
||||||
|
launch_args = []
|
||||||
|
if self._stealth_config is None or self._stealth_config.stealth_enabled:
|
||||||
|
launch_args.append('--disable-blink-features=AutomationControlled')
|
||||||
|
|
||||||
self._playwright = sync_playwright().start()
|
self._playwright = sync_playwright().start()
|
||||||
self._browser = self._playwright.chromium.launch(headless=self.headless)
|
self._browser = self._playwright.chromium.launch(
|
||||||
|
headless=self.headless,
|
||||||
|
args=launch_args if launch_args else None
|
||||||
|
)
|
||||||
|
|
||||||
# Build context options
|
# Build context options
|
||||||
context_options = {
|
context_options = {
|
||||||
@@ -190,6 +203,9 @@ class ZhihuCrawler:
|
|||||||
|
|
||||||
self._context = self._browser.new_context(**context_options)
|
self._context = self._browser.new_context(**context_options)
|
||||||
|
|
||||||
|
# Apply stealth fingerprint masking
|
||||||
|
apply_stealth_sync(self._context, self._stealth_config)
|
||||||
|
|
||||||
# Add cookies if provided
|
# Add cookies if provided
|
||||||
if cookie:
|
if cookie:
|
||||||
self._add_cookies(cookie)
|
self._add_cookies(cookie)
|
||||||
|
|||||||
Reference in New Issue
Block a user