fix/crawler-proxy-config #32

Merged
starmen1265 merged 18 commits from fix/crawler-proxy-config into main 2026-03-19 00:07:33 +08:00
Showing only changes of commit 08cbe24d08 - Show all commits
+18 -2
View File
@@ -35,6 +35,7 @@ from .models import (
) )
from .config import CrawlerConfig, get_crawler_config from .config import CrawlerConfig, get_crawler_config
from .proxy_pool import ProxyPool, ProxyEntry from .proxy_pool import ProxyPool, ProxyEntry
from .stealth import apply_stealth_sync, StealthConfig
class ZhihuApiInterceptor: class ZhihuApiInterceptor:
@@ -108,7 +109,8 @@ class ZhihuCrawler:
headless: bool = True, headless: bool = True,
timeout: int = 30, timeout: int = 30,
config: Optional[CrawlerConfig] = None, config: Optional[CrawlerConfig] = None,
proxy_pool: Optional[ProxyPool] = None proxy_pool: Optional[ProxyPool] = None,
stealth_config: Optional[StealthConfig] = None
): ):
"""Initialize ZhihuCrawler. """Initialize ZhihuCrawler.
@@ -117,11 +119,13 @@ class ZhihuCrawler:
timeout: Request timeout in seconds timeout: Request timeout in seconds
config: Optional CrawlerConfig instance (uses environment if not provided) config: Optional CrawlerConfig instance (uses environment if not provided)
proxy_pool: Optional ProxyPool instance for multi-proxy rotation proxy_pool: Optional ProxyPool instance for multi-proxy rotation
stealth_config: Optional StealthConfig instance (uses defaults if not provided)
""" """
self.headless = headless self.headless = headless
self.timeout = timeout self.timeout = timeout
self.config = config or get_crawler_config() self.config = config or get_crawler_config()
self.proxy_pool = proxy_pool self.proxy_pool = proxy_pool
self._stealth_config = stealth_config
self._current_proxy: Optional[ProxyEntry] = None self._current_proxy: Optional[ProxyEntry] = None
self.browser: Optional[PlaywrightBrowser] = None self.browser: Optional[PlaywrightBrowser] = None
self.interceptor: Optional[ZhihuApiInterceptor] = None self.interceptor: Optional[ZhihuApiInterceptor] = None
@@ -155,6 +159,7 @@ class ZhihuCrawler:
"""Start browser with proxy configuration. """Start browser with proxy configuration.
Validates proxy if required and applies to browser context. Validates proxy if required and applies to browser context.
Applies stealth fingerprint masking after context creation.
Prioritizes proxy_pool over config.proxy for multi-proxy support. Prioritizes proxy_pool over config.proxy for multi-proxy support.
Args: Args:
@@ -166,8 +171,16 @@ class ZhihuCrawler:
# Validate proxy configuration # Validate proxy configuration
self.config.validate_proxy() self.config.validate_proxy()
# Build launch args for stealth (add --disable-blink-features)
launch_args = []
if self._stealth_config is None or self._stealth_config.stealth_enabled:
launch_args.append('--disable-blink-features=AutomationControlled')
self._playwright = sync_playwright().start() self._playwright = sync_playwright().start()
self._browser = self._playwright.chromium.launch(headless=self.headless) self._browser = self._playwright.chromium.launch(
headless=self.headless,
args=launch_args if launch_args else None
)
# Build context options # Build context options
context_options = { context_options = {
@@ -190,6 +203,9 @@ class ZhihuCrawler:
self._context = self._browser.new_context(**context_options) self._context = self._browser.new_context(**context_options)
# Apply stealth fingerprint masking
apply_stealth_sync(self._context, self._stealth_config)
# Add cookies if provided # Add cookies if provided
if cookie: if cookie:
self._add_cookies(cookie) self._add_cookies(cookie)