diff --git a/kb_workflow/crawler/zhihu_crawler.py b/kb_workflow/crawler/zhihu_crawler.py index 77b029b..c8adf81 100644 --- a/kb_workflow/crawler/zhihu_crawler.py +++ b/kb_workflow/crawler/zhihu_crawler.py @@ -35,6 +35,7 @@ from .models import ( ) from .config import CrawlerConfig, get_crawler_config from .proxy_pool import ProxyPool, ProxyEntry +from .stealth import apply_stealth_sync, StealthConfig class ZhihuApiInterceptor: @@ -108,7 +109,8 @@ class ZhihuCrawler: headless: bool = True, timeout: int = 30, config: Optional[CrawlerConfig] = None, - proxy_pool: Optional[ProxyPool] = None + proxy_pool: Optional[ProxyPool] = None, + stealth_config: Optional[StealthConfig] = None ): """Initialize ZhihuCrawler. @@ -117,11 +119,13 @@ class ZhihuCrawler: timeout: Request timeout in seconds config: Optional CrawlerConfig instance (uses environment if not provided) proxy_pool: Optional ProxyPool instance for multi-proxy rotation + stealth_config: Optional StealthConfig instance (uses defaults if not provided) """ self.headless = headless self.timeout = timeout self.config = config or get_crawler_config() self.proxy_pool = proxy_pool + self._stealth_config = stealth_config self._current_proxy: Optional[ProxyEntry] = None self.browser: Optional[PlaywrightBrowser] = None self.interceptor: Optional[ZhihuApiInterceptor] = None @@ -155,6 +159,7 @@ class ZhihuCrawler: """Start browser with proxy configuration. Validates proxy if required and applies to browser context. + Applies stealth fingerprint masking after context creation. Prioritizes proxy_pool over config.proxy for multi-proxy support. Args: @@ -166,8 +171,16 @@ class ZhihuCrawler: # Validate proxy configuration self.config.validate_proxy() + # Build launch args for stealth (add --disable-blink-features) + launch_args = [] + if self._stealth_config is None or self._stealth_config.stealth_enabled: + launch_args.append('--disable-blink-features=AutomationControlled') + self._playwright = sync_playwright().start() - self._browser = self._playwright.chromium.launch(headless=self.headless) + self._browser = self._playwright.chromium.launch( + headless=self.headless, + args=launch_args if launch_args else None + ) # Build context options context_options = { @@ -190,6 +203,9 @@ class ZhihuCrawler: self._context = self._browser.new_context(**context_options) + # Apply stealth fingerprint masking + apply_stealth_sync(self._context, self._stealth_config) + # Add cookies if provided if cookie: self._add_cookies(cookie)