自动同步: 2026-03-19 02:10

skills/kb_workflow/crawler/zhihu_crawler.py

@@ -8,12 +8,14 @@ Supports:
 - Random request delay
 - Retry with exponential backoff
 - Automatic x-zse-96 signature handling via browser
+- ProxyPool integration for multi-proxy rotation
 """
 
 import random
 import re
 import time
 from typing import Optional, Dict, Any
+from urllib.parse import urlparse
 
 from playwright.sync_api import Route, Response, sync_playwright
 from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
@@ -32,6 +34,8 @@ from .models import (
     CrawlResult,
 )
 from .config import CrawlerConfig, get_crawler_config
+from .proxy_pool import ProxyPool, ProxyEntry
+from .stealth import apply_stealth_sync, StealthConfig
 
 
 class ZhihuApiInterceptor:
@@ -104,7 +108,9 @@ class ZhihuCrawler:
         self,
         headless: bool = True,
         timeout: int = 30,
-        config: Optional[CrawlerConfig] = None
+        config: Optional[CrawlerConfig] = None,
+        proxy_pool: Optional[ProxyPool] = None,
+        stealth_config: Optional[StealthConfig] = None
     ):
         """Initialize ZhihuCrawler.
 
@@ -112,10 +118,15 @@ class ZhihuCrawler:
             headless: Whether to run browser in headless mode
             timeout: Request timeout in seconds
             config: Optional CrawlerConfig instance (uses environment if not provided)
+            proxy_pool: Optional ProxyPool instance for multi-proxy rotation
+            stealth_config: Optional StealthConfig instance (uses defaults if not provided)
         """
         self.headless = headless
         self.timeout = timeout
         self.config = config or get_crawler_config()
+        self.proxy_pool = proxy_pool
+        self._stealth_config = stealth_config
+        self._current_proxy: Optional[ProxyEntry] = None
         self.browser: Optional[PlaywrightBrowser] = None
         self.interceptor: Optional[ZhihuApiInterceptor] = None
         # Internal browser management for proxy support
@@ -148,6 +159,8 @@ class ZhihuCrawler:
         """Start browser with proxy configuration.
 
         Validates proxy if required and applies to browser context.
+        Applies stealth fingerprint masking after context creation.
+        Prioritizes proxy_pool over config.proxy for multi-proxy support.
 
         Args:
             cookie: Cookie string for authentication
@@ -158,16 +171,31 @@ class ZhihuCrawler:
         # Validate proxy configuration
         self.config.validate_proxy()
 
+        # Build launch args for stealth (add --disable-blink-features)
+        launch_args = []
+        if self._stealth_config is None or self._stealth_config.stealth_enabled:
+            launch_args.append('--disable-blink-features=AutomationControlled')
+
         self._playwright = sync_playwright().start()
-        self._browser = self._playwright.chromium.launch(headless=self.headless)
+        self._browser = self._playwright.chromium.launch(
+            headless=self.headless,
+            args=launch_args if launch_args else None
+        )
 
         # Build context options
         context_options = {
             'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
         }
 
-        # Apply proxy if configured
-        if self.config.proxy:
+        # Prioritize proxy pool over single proxy config
+        if self.proxy_pool and self.proxy_pool.has_proxies():
+            proxy_entry = self.proxy_pool.get_next_proxy()
+            if proxy_entry:
+                proxy_config = self._parse_proxy_url(proxy_entry.url)
+                context_options["proxy"] = proxy_config
+                self._current_proxy = proxy_entry
+        elif self.config.proxy:
+            # Backward compatibility: use single proxy configuration
             context_options["proxy"] = self.config.proxy.to_playwright_format()
 
         # Block Service Worker to prevent interference with API interception
@@ -175,6 +203,9 @@ class ZhihuCrawler:
 
         self._context = self._browser.new_context(**context_options)
 
+        # Apply stealth fingerprint masking
+        apply_stealth_sync(self._context, self._stealth_config)
+
         # Add cookies if provided
         if cookie:
             self._add_cookies(cookie)
@@ -182,6 +213,40 @@ class ZhihuCrawler:
         self._page = self._context.new_page()
         self._page.set_default_timeout(self.timeout * 1000)
 
+    @staticmethod
+    def _parse_proxy_url(url: str) -> Dict[str, Any]:
+        """Parse proxy URL into Playwright format.
+
+        Args:
+            url: Proxy URL (e.g., http://user:pass@proxy.example.com:8080)
+
+        Returns:
+            Dictionary with server, username, password for Playwright
+        """
+        parsed = urlparse(url)
+        server = f"{parsed.scheme}://{parsed.hostname}"
+        if parsed.port:
+            server += f":{parsed.port}"
+        config: Dict[str, Any] = {"server": server}
+        if parsed.username:
+            config["username"] = parsed.username
+        if parsed.password:
+            config["password"] = parsed.password
+        return config
+
+    def _on_request_success(self) -> None:
+        """Notify proxy pool of successful request."""
+        if self._current_proxy:
+            self._current_proxy.record_success()
+
+    def _on_request_failure(self) -> None:
+        """Notify proxy pool of failed request."""
+        if self._current_proxy:
+            self._current_proxy.record_failure(
+                max_fail_count=self.proxy_pool.config.max_fail_count
+                if self.proxy_pool else 3
+            )
+
     def _add_cookies(self, cookie_str: str) -> None:
         """Add cookies to browser context.