From d6026660a9b767b468a919b77847406680830e85 Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Wed, 18 Mar 2026 23:41:43 +0800 Subject: [PATCH] docs(11): create phase 11 stealth fingerprinting plans --- .planning/ROADMAP.md | 8 +- .../11-stealth-fingerprinting/11-01-PLAN.md | 536 ++++++++++++++++++ .../11-stealth-fingerprinting/11-02-PLAN.md | 506 +++++++++++++++++ 3 files changed, 1046 insertions(+), 4 deletions(-) create mode 100644 .planning/phases/11-stealth-fingerprinting/11-01-PLAN.md create mode 100644 .planning/phases/11-stealth-fingerprinting/11-02-PLAN.md diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index b2704eb..df66e44 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -113,11 +113,11 @@ Plans: 2. 爬虫的 navigator.webdriver 属性对知乎不可见 3. Playwright 特有的 JS 属性（__playwright, pwInspect 等）被移除 4. 现有功能（登录、Cookie 持久化、内容爬取）保持正常工作 -**Plans**: TBD +**Plans**: 2 plans Plans: -- [ ] 11-01: 指纹伪装脚本实现 — 隐藏 webdriver、移除 Playwright 特有属性、模拟 Chrome 特性 -- [ ] 11-02: ZhihuCrawler 集成与验证 — 注入伪装脚本，验证爬取功能正常 +- [ ] 11-01-PLAN.md — 创建 StealthConfig 模块和单元测试 +- [ ] 11-02-PLAN.md — 集成 stealth 到 ZhihuCrawler 并验证 --- @@ -138,7 +138,7 @@ Phases execute in numeric order: 1 → 2 → ... → 10 → 11 | 8. 内容爬取实现 | v1.2 | 3/3 | Complete | 2026-03-17 | | 9. Pipeline 整合 | v1.2 | 1/1 | Complete | 2026-03-17 | | 10. Proxy Pool | v1.3 | 2/2 | Complete | 2026-03-18 | -| 11. Stealth Fingerprinting | v1.4 | 0/2 | Not started | - | +| 11. Stealth Fingerprinting | v1.4 | 0/2 | Ready to execute | - | --- diff --git a/.planning/phases/11-stealth-fingerprinting/11-01-PLAN.md b/.planning/phases/11-stealth-fingerprinting/11-01-PLAN.md new file mode 100644 index 0000000..fe7b1a0 --- /dev/null +++ b/.planning/phases/11-stealth-fingerprinting/11-01-PLAN.md @@ -0,0 +1,536 @@ +--- +phase: 11-stealth-fingerprinting +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - kb_workflow/crawler/stealth/__init__.py + - kb_workflow/crawler/stealth/stealth_config.py + - tests/test_crawler/conftest.py + - tests/test_crawler/test_stealth.py +autonomous: true +requirements: + - STEALTH-01 + - STEALTH-02 + - STEALTH-03 + - STEALTH-04 +must_haves: + truths: + - "navigator.webdriver 属性被隐藏（返回 undefined/false）" + - "Playwright 特有属性（__playwright, __pw_manual, __PW_inspect）被移除" + - "window.chrome 对象被模拟为正常 Chrome 浏览器结构" + - "User-Agent 不包含 HeadlessChrome 字符串" + artifacts: + - path: "kb_workflow/crawler/stealth/__init__.py" + provides: "Stealth 模块入口，导出 StealthConfig 和 apply_stealth" + min_lines: 10 + - path: "kb_workflow/crawler/stealth/stealth_config.py" + provides: "StealthConfig Pydantic 模型和 apply_stealth_async 函数" + min_lines: 50 + - path: "tests/test_crawler/test_stealth.py" + provides: "Stealth 功能单元测试" + min_lines: 100 + key_links: + - from: "kb_workflow/crawler/stealth/stealth_config.py" + to: "playwright_stealth.Stealth" + via: "from playwright_stealth import Stealth" + pattern: "from playwright_stealth" +--- + + +创建浏览器指纹伪装模块，使用 playwright-stealth 库隐藏 Playwright 自动化特征。 + +Purpose: 为 ZhihuCrawler 提供可复用的指纹伪装能力，绕过知乎的浏览器指纹检测 +Output: kb_workflow/crawler/stealth/ 模块 + 单元测试 + + + +@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md +@/home/starmen1265/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/11-stealth-fingerprinting/11-RESEARCH.md +@.planning/phases/11-stealth-fingerprinting/11-VALIDATION.md + + + + +From kb_workflow/crawler/config.py: +```python +class CrawlerConfig(BaseModel): + proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration") + proxy_required: bool = Field(default=False) + min_delay: float = Field(default=1.0) + max_delay: float = Field(default=3.0) +``` + +From kb_workflow/crawler/zhihu_crawler.py: +```python +class ZhihuCrawler: + def __init__( + self, + headless: bool = True, + timeout: int = 30, + config: Optional[CrawlerConfig] = None, + proxy_pool: Optional[ProxyPool] = None + ): + ... + + def _start_browser_with_proxy(self, cookie: str) -> None: + """Start browser with proxy configuration.""" + self._playwright = sync_playwright().start() + self._browser = self._playwright.chromium.launch(headless=self.headless) + context_options = { + 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' + } + ... + self._context = self._browser.new_context(**context_options) +``` + +From playwright_stealth (external library): +```python +from playwright_stealth import Stealth + +stealth = Stealth( + navigator_webdriver=True, + navigator_user_agent=True, + navigator_plugins=True, + # ... other options +) +stealth.apply_stealth_async(context) # for async +stealth.apply_stealth_sync(context) # for sync +``` + + + + + + + Task 1: 创建 StealthConfig 模块 + kb_workflow/crawler/stealth/__init__.py, kb_workflow/crawler/stealth/stealth_config.py + + - Test 1: StealthConfig 默认值正确（stealth_enabled=True, navigator_webdriver=True 等） + - Test 2: apply_stealth_sync 在 mock context 上正确调用 playwright_stealth + - Test 3: get_default_stealth_config() 返回正确的默认配置 + + + 1. 创建 `kb_workflow/crawler/stealth/` 目录 + + 2. 创建 `kb_workflow/crawler/stealth/__init__.py`: + - 导出 StealthConfig 类 + - 导出 apply_stealth_sync, apply_stealth_async 函数 + - 导出 get_default_stealth_config 便捷函数 + + 3. 创建 `kb_workflow/crawler/stealth/stealth_config.py`: + + ```python + """Stealth configuration for browser fingerprint masking. + + Uses playwright-stealth library to hide Playwright automation signatures. + """ + + from typing import Optional + from playwright.sync_api import BrowserContext + from playwright.async_api import BrowserContext as AsyncBrowserContext + from pydantic import BaseModel, Field + from playwright_stealth import Stealth + + + class StealthConfig(BaseModel): + """Configuration for browser stealth settings. + + All options map to playwright-stealth evasion scripts. + Default values optimized for Zhihu anti-detection. + """ + + stealth_enabled: bool = Field( + default=True, + description="Enable stealth mode (set False to disable)" + ) + + # Navigator evasion options + navigator_webdriver: bool = Field( + default=True, + description="Hide navigator.webdriver property (STEALTH-01)" + ) + navigator_user_agent: bool = Field( + default=True, + description="Replace HeadlessChrome in User-Agent (STEALTH-04)" + ) + navigator_plugins: bool = Field( + default=True, + description="Mock navigator.plugins array" + ) + navigator_languages: bool = Field( + default=True, + description="Set navigator.languages" + ) + navigator_platform: bool = Field( + default=True, + description="Set navigator.platform" + ) + navigator_vendor: bool = Field( + default=True, + description="Set navigator.vendor" + ) + navigator_permissions: bool = Field( + default=True, + description="Override navigator.permissions" + ) + + # Chrome object evasion (STEALTH-03) + chrome_app: bool = Field( + default=True, + description="Mock chrome.app object" + ) + chrome_csi: bool = Field( + default=True, + description="Mock chrome.csi function" + ) + chrome_load_times: bool = Field( + default=True, + description="Mock chrome.loadTimes function" + ) + + # WebGL and rendering + webgl_vendor: bool = Field( + default=True, + description="Override WebGL vendor/renderer strings" + ) + + # Other evasion options + hairline: bool = Field( + default=True, + description="Fix hairline detection" + ) + iframe_content_window: bool = Field( + default=True, + description="Fix iframe contentWindow" + ) + media_codecs: bool = Field( + default=True, + description="Mock media codecs" + ) + error_prototype: bool = Field( + default=True, + description="Fix error prototype chain" + ) + sec_ch_ua: bool = Field( + default=True, + description="Set sec-ch-ua headers" + ) + + def create_stealth(self) -> Stealth: + """Create playwright-stealth Stealth instance from config. + + Returns: + Configured Stealth instance + """ + return Stealth( + navigator_webdriver=self.navigator_webdriver, + navigator_user_agent=self.navigator_user_agent, + navigator_plugins=self.navigator_plugins, + navigator_languages=self.navigator_languages, + navigator_platform=self.navigator_platform, + navigator_vendor=self.navigator_vendor, + navigator_permissions=self.navigator_permissions, + chrome_app=self.chrome_app, + chrome_csi=self.chrome_csi, + chrome_load_times=self.chrome_load_times, + webgl_vendor=self.webgl_vendor, + hairline=self.hairline, + iframe_content_window=self.iframe_content_window, + media_codecs=self.media_codecs, + error_prototype=self.error_prototype, + sec_ch_ua=self.sec_ch_ua, + ) + + + def apply_stealth_sync( + context: BrowserContext, + config: Optional[StealthConfig] = None + ) -> None: + """Apply stealth settings to browser context (synchronous). + + Args: + context: Playwright BrowserContext to apply stealth to + config: StealthConfig instance (uses defaults if not provided) + """ + if config is None: + config = StealthConfig() + + if not config.stealth_enabled: + return + + stealth = config.create_stealth() + stealth.apply_stealth_sync(context) + + + async def apply_stealth_async( + context: AsyncBrowserContext, + config: Optional[StealthConfig] = None + ) -> None: + """Apply stealth settings to browser context (asynchronous). + + Args: + context: Playwright async BrowserContext to apply stealth to + config: StealthConfig instance (uses defaults if not provided) + """ + if config is None: + config = StealthConfig() + + if not config.stealth_enabled: + return + + stealth = config.create_stealth() + await stealth.apply_stealth_async(context) + + + def get_default_stealth_config() -> StealthConfig: + """Get default StealthConfig for Zhihu crawling. + + Returns: + StealthConfig with all evasion options enabled + """ + return StealthConfig() + ``` + + 4. 创建 `kb_workflow/crawler/stealth/__init__.py`: + ```python + """Stealth module for browser fingerprint masking.""" + + from .stealth_config import ( + StealthConfig, + apply_stealth_sync, + apply_stealth_async, + get_default_stealth_config, + ) + + __all__ = [ + "StealthConfig", + "apply_stealth_sync", + "apply_stealth_async", + "get_default_stealth_config", + ] + ``` + + + pytest tests/test_crawler/test_stealth.py::test_stealth_config_defaults -x + + + - StealthConfig 类创建完成，包含所有 playwright-stealth 选项 + - apply_stealth_sync/apply_stealth_async 函数可用 + - 单元测试通过 + + + + + Task 2: 创建 Stealth 功能单元测试 + tests/test_crawler/test_stealth.py, tests/test_crawler/conftest.py + + - Test 1: test_stealth_config_defaults - 默认配置正确 + - Test 2: test_stealth_config_disabled - stealth_enabled=False 时不应用伪装 + - Test 3: test_apply_stealth_sync_calls_library - apply_stealth_sync 正确调用 playwright_stealth + - Test 4: test_get_default_stealth_config - 返回默认配置实例 + - Test 5: test_stealth_options_mapping - 配置选项正确映射到 Stealth 类 + + + 1. 在 `tests/test_crawler/conftest.py` 中添加 stealth_config fixture: + + ```python + @pytest.fixture + def stealth_config(): + """Default StealthConfig for testing.""" + from kb_workflow.crawler.stealth import StealthConfig + return StealthConfig() + + + @pytest.fixture + def mock_browser_context(): + """Mock Playwright BrowserContext for stealth testing.""" + from unittest.mock import MagicMock + context = MagicMock() + context.add_init_script = MagicMock() + return context + ``` + + 2. 创建 `tests/test_crawler/test_stealth.py`: + + ```python + """Tests for stealth fingerprint masking module.""" + + import pytest + from unittest.mock import MagicMock, patch, call + + + class TestStealthConfig: + """Tests for StealthConfig class.""" + + def test_stealth_config_defaults(self): + """Test that default StealthConfig has all options enabled.""" + from kb_workflow.crawler.stealth import StealthConfig + + config = StealthConfig() + + assert config.stealth_enabled is True + # STEALTH-01: navigator.webdriver hiding + assert config.navigator_webdriver is True + # STEALTH-04: User-Agent modification + assert config.navigator_user_agent is True + # STEALTH-03: Chrome properties + assert config.chrome_app is True + assert config.chrome_csi is True + assert config.chrome_load_times is True + # Other important options + assert config.navigator_plugins is True + assert config.webgl_vendor is True + + def test_stealth_config_disabled(self): + """Test that stealth can be disabled.""" + from kb_workflow.crawler.stealth import StealthConfig + + config = StealthConfig(stealth_enabled=False) + assert config.stealth_enabled is False + + def test_create_stealth_returns_stealth_instance(self): + """Test that create_stealth() returns configured Stealth.""" + from kb_workflow.crawler.stealth import StealthConfig + from playwright_stealth import Stealth + + config = StealthConfig() + stealth = config.create_stealth() + + assert isinstance(stealth, Stealth) + + def test_stealth_options_mapping(self): + """Test that StealthConfig options map correctly to Stealth.""" + from kb_workflow.crawler.stealth import StealthConfig + + config = StealthConfig( + navigator_webdriver=True, + navigator_user_agent=False, # Disable one option + chrome_app=True, + ) + + stealth = config.create_stealth() + + # Verify the Stealth object was created with correct options + # The Stealth class stores these as attributes + assert stealth.navigator_webdriver is True + assert stealth.navigator_user_agent is False + + + class TestApplyStealth: + """Tests for apply_stealth functions.""" + + def test_apply_stealth_sync_calls_library(self, mock_browser_context): + """Test that apply_stealth_sync calls playwright-stealth correctly.""" + from kb_workflow.crawler.stealth import apply_stealth_sync, StealthConfig + + config = StealthConfig() + + with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class: + mock_stealth_instance = MagicMock() + mock_stealth_class.return_value = mock_stealth_instance + + apply_stealth_sync(mock_browser_context, config) + + # Verify Stealth was created with config options + mock_stealth_class.assert_called_once() + # Verify apply_stealth_sync was called on the Stealth instance + mock_stealth_instance.apply_stealth_sync.assert_called_once_with( + mock_browser_context + ) + + def test_apply_stealth_sync_disabled_skips_stealth(self, mock_browser_context): + """Test that stealth is skipped when disabled.""" + from kb_workflow.crawler.stealth import apply_stealth_sync, StealthConfig + + config = StealthConfig(stealth_enabled=False) + + with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class: + apply_stealth_sync(mock_browser_context, config) + + # Stealth should not be created when disabled + mock_stealth_class.assert_not_called() + + def test_apply_stealth_sync_default_config(self, mock_browser_context): + """Test that apply_stealth_sync works with default config.""" + from kb_workflow.crawler.stealth import apply_stealth_sync + + with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class: + mock_stealth_instance = MagicMock() + mock_stealth_class.return_value = mock_stealth_instance + + # Should work without explicit config + apply_stealth_sync(mock_browser_context) + + mock_stealth_class.assert_called_once() + + + class TestGetDefaultStealthConfig: + """Tests for get_default_stealth_config function.""" + + def test_get_default_stealth_config(self): + """Test that get_default_stealth_config returns StealthConfig.""" + from kb_workflow.crawler.stealth import get_default_stealth_config + + config = get_default_stealth_config() + + assert config.stealth_enabled is True + assert isinstance(config, type(config)) # Is a StealthConfig + + def test_get_default_stealth_config_is_fresh_instance(self): + """Test that each call returns a new instance.""" + from kb_workflow.crawler.stealth import get_default_stealth_config + + config1 = get_default_stealth_config() + config2 = get_default_stealth_config() + + # Should be equal but not the same object + assert config1 == config2 + assert config1 is not config2 + + + # Wave 0 completion marker + WAVE_0_COMPLETE = True + ``` + + + pytest tests/test_crawler/test_stealth.py -x -v + + + - test_stealth.py 创建完成，包含 5 个测试用例 + - conftest.py 添加了 stealth_config 和 mock_browser_context fixtures + - 所有测试通过 + + + + + + +**Per-task verification:** +- Task 1: `pytest tests/test_crawler/test_stealth.py::TestStealthConfig::test_stealth_config_defaults -x` +- Task 2: `pytest tests/test_crawler/test_stealth.py -x -v` + +**Wave verification:** +- All 5 tests in test_stealth.py pass +- No regression in existing tests: `pytest tests/test_crawler/ -x` + + + +- [ ] kb_workflow/crawler/stealth/ 模块创建完成 +- [ ] StealthConfig 类包含所有 playwright-stealth 选项 +- [ ] apply_stealth_sync/apply_stealth_async 函数可用 +- [ ] 5 个单元测试全部通过 +- [ ] 无现有测试回归 + + + +After completion, create `.planning/phases/11-stealth-fingerprinting/11-01-SUMMARY.md` + diff --git a/.planning/phases/11-stealth-fingerprinting/11-02-PLAN.md b/.planning/phases/11-stealth-fingerprinting/11-02-PLAN.md new file mode 100644 index 0000000..560d33c --- /dev/null +++ b/.planning/phases/11-stealth-fingerprinting/11-02-PLAN.md @@ -0,0 +1,506 @@ +--- +phase: 11-stealth-fingerprinting +plan: 02 +type: execute +wave: 2 +depends_on: + - 11-01 +files_modified: + - kb_workflow/crawler/zhihu_crawler.py + - tests/test_crawler/test_zhihu_crawler_stealth.py +autonomous: true +requirements: + - VERIFY-01 + - VERIFY-02 +must_haves: + truths: + - "ZhihuCrawler 访问问答页面时不再被 40362 错误拦截" + - "现有功能（登录、Cookie 持久化、内容爬取）保持正常工作" + - "stealth_enabled 参数控制是否启用指纹伪装" + artifacts: + - path: "kb_workflow/crawler/zhihu_crawler.py" + provides: "ZhihuCrawler 集成 stealth 功能" + contains: "stealth_enabled" + - path: "tests/test_crawler/test_zhihu_crawler_stealth.py" + provides: "集成测试验证 stealth 功能" + min_lines: 50 + key_links: + - from: "kb_workflow/crawler/zhihu_crawler.py" + to: "kb_workflow/crawler/stealth" + via: "from .stealth import apply_stealth_sync, StealthConfig" + pattern: "from \\.stealth import" + - from: "ZhihuCrawler._start_browser_with_proxy" + to: "apply_stealth_sync" + via: "调用 apply_stealth_sync(self._context, self._stealth_config)" + pattern: "apply_stealth_sync" +--- + + +将指纹伪装模块集成到 ZhihuCrawler，并验证爬虫能正常绕过知乎反爬检测。 + +Purpose: 让 ZhihuCrawler 在启动浏览器时自动应用指纹伪装，绕过 40362 错误 +Output: 修改后的 ZhihuCrawler + 集成测试 + + + +@/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md +@/home/starmen1265/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/11-stealth-fingerprinting/11-RESEARCH.md +@.planning/phases/11-stealth-fingerprinting/11-VALIDATION.md + + +@.planning/phases/11-stealth-fingerprinting/11-01-PLAN.md + + + + +From kb_workflow/crawler/stealth (Plan 11-01): +```python +class StealthConfig(BaseModel): + stealth_enabled: bool = True + navigator_webdriver: bool = True + navigator_user_agent: bool = True + # ... other options + +def apply_stealth_sync( + context: BrowserContext, + config: Optional[StealthConfig] = None +) -> None: + """Apply stealth settings to browser context (synchronous).""" + +def get_default_stealth_config() -> StealthConfig: + """Get default StealthConfig for Zhihu crawling.""" +``` + +From kb_workflow/crawler/zhihu_crawler.py (current implementation): +```python +class ZhihuCrawler: + def __init__( + self, + headless: bool = True, + timeout: int = 30, + config: Optional[CrawlerConfig] = None, + proxy_pool: Optional[ProxyPool] = None + ): + self.headless = headless + self.timeout = timeout + self.config = config or get_crawler_config() + self.proxy_pool = proxy_pool + self._current_proxy: Optional[ProxyEntry] = None + # Internal browser management + self._playwright = None + self._browser = None + self._context = None + self._page = None + + def _start_browser_with_proxy(self, cookie: str) -> None: + """Start browser with proxy configuration.""" + self.config.validate_proxy() + self._playwright = sync_playwright().start() + self._browser = self._playwright.chromium.launch(headless=self.headless) + context_options = { + 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' + } + # ... proxy configuration ... + context_options["service_workers"] = "block" + self._context = self._browser.new_context(**context_options) + if cookie: + self._add_cookies(cookie) + self._page = self._context.new_page() + self._page.set_default_timeout(self.timeout * 1000) +``` + +From tests/test_crawler/conftest.py: +```python +@pytest.fixture +def mock_page(): + """Mock Playwright page object.""" + page = MagicMock() + page.route = MagicMock() + page.goto = MagicMock() + return page + +@pytest.fixture +def sample_answer_api_response(): + """Sample answer API response data.""" + return { + "id": 123456789, + "type": "answer", + # ... + } +``` + + + + + + + Task 1: 集成 stealth 到 ZhihuCrawler + kb_workflow/crawler/zhihu_crawler.py + + - Test 1: ZhihuCrawler 初始化时 stealth_enabled 默认为 True + - Test 2: ZhihuCrawler 初始化时可禁用 stealth（stealth_enabled=False） + - Test 3: _start_browser_with_proxy 在创建 context 后调用 apply_stealth_sync + - Test 4: stealth_enabled=False 时不调用 apply_stealth_sync + - Test 5: launch 时添加 --disable-blink-features=AutomationControlled 参数 + + + 修改 `kb_workflow/crawler/zhihu_crawler.py`，集成 stealth 功能： + + 1. 添加导入（在文件顶部导入区域）: + ```python + from .stealth import apply_stealth_sync, StealthConfig + ``` + + 2. 修改 `__init__` 方法，添加 stealth 参数: + ```python + def __init__( + self, + headless: bool = True, + timeout: int = 30, + config: Optional[CrawlerConfig] = None, + proxy_pool: Optional[ProxyPool] = None, + stealth_config: Optional[StealthConfig] = None # 新增参数 + ): + """Initialize ZhihuCrawler. + + Args: + headless: Whether to run browser in headless mode + timeout: Request timeout in seconds + config: Optional CrawlerConfig instance + proxy_pool: Optional ProxyPool instance for multi-proxy rotation + stealth_config: Optional StealthConfig instance (uses defaults if not provided) + """ + self.headless = headless + self.timeout = timeout + self.config = config or get_crawler_config() + self.proxy_pool = proxy_pool + self._stealth_config = stealth_config # 存储配置 + self._current_proxy: Optional[ProxyEntry] = None + # ... rest of initialization + ``` + + 3. 修改 `_start_browser_with_proxy` 方法: + ```python + def _start_browser_with_proxy(self, cookie: str) -> None: + """Start browser with proxy configuration. + + Validates proxy if required and applies to browser context. + Applies stealth fingerprint masking after context creation. + + Args: + cookie: Cookie string for authentication + + Raises: + ValueError: If proxy_required=True but no proxy is configured + """ + # Validate proxy configuration + self.config.validate_proxy() + + # Build launch args for stealth (关键：添加 --disable-blink-features) + launch_args = [] + if self._stealth_config is None or self._stealth_config.stealth_enabled: + launch_args.append('--disable-blink-features=AutomationControlled') + + self._playwright = sync_playwright().start() + self._browser = self._playwright.chromium.launch( + headless=self.headless, + args=launch_args if launch_args else None + ) + + # Build context options + context_options = { + 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' + } + + # Prioritize proxy pool over single proxy config + if self.proxy_pool and self.proxy_pool.has_proxies(): + proxy_entry = self.proxy_pool.get_next_proxy() + if proxy_entry: + proxy_config = self._parse_proxy_url(proxy_entry.url) + context_options["proxy"] = proxy_config + self._current_proxy = proxy_entry + elif self.config.proxy: + context_options["proxy"] = self.config.proxy.to_playwright_format() + + # Block Service Worker to prevent interference with API interception + context_options["service_workers"] = "block" + + self._context = self._browser.new_context(**context_options) + + # 关键改动：应用 stealth 指纹伪装 + apply_stealth_sync(self._context, self._stealth_config) + + # Add cookies if provided + if cookie: + self._add_cookies(cookie) + + self._page = self._context.new_page() + self._page.set_default_timeout(self.timeout * 1000) + ``` + + 关键点： + - 在 `launch()` 时添加 `--disable-blink-features=AutomationControlled` 参数 + - 在 `new_context()` 后、添加 cookies 前调用 `apply_stealth_sync()` + - 保持向后兼容：stealth_config 为 None 时使用默认配置（启用 stealth） + + + pytest tests/test_crawler/test_zhihu_crawler_stealth.py -x -v + + + - ZhihuCrawler 添加 stealth_config 参数 + - _start_browser_with_proxy 调用 apply_stealth_sync + - launch 时添加 --disable-blink-features 参数 + - 所有集成测试通过 + + + + + Task 2: 创建 ZhihuCrawler Stealth 集成测试 + tests/test_crawler/test_zhihu_crawler_stealth.py + + - Test 1: test_crawler_has_stealth_config_param - ZhihuCrawler 接受 stealth_config 参数 + - Test 2: test_crawler_default_stealth_enabled - 默认启用 stealth + - Test 3: test_crawler_can_disable_stealth - 可以禁用 stealth + - Test 4: test_start_browser_applies_stealth - _start_browser_with_proxy 调用 apply_stealth_sync + - Test 5: test_launch_args_include_disable_blink_features - launch 参数包含 --disable-blink-features + + + 创建 `tests/test_crawler/test_zhihu_crawler_stealth.py`: + + ```python + """Integration tests for ZhihuCrawler stealth functionality. + + Tests that stealth fingerprint masking is properly integrated + into ZhihuCrawler's browser initialization. + """ + + import pytest + from unittest.mock import MagicMock, patch, call + + + class TestZhihuCrawlerStealthIntegration: + """Tests for stealth integration in ZhihuCrawler.""" + + def test_crawler_has_stealth_config_param(self): + """Test that ZhihuCrawler accepts stealth_config parameter.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + from kb_workflow.crawler.stealth import StealthConfig + + config = StealthConfig(stealth_enabled=False) + crawler = ZhihuCrawler(stealth_config=config) + + assert crawler._stealth_config == config + + def test_crawler_default_stealth_enabled(self): + """Test that stealth is enabled by default (None = use defaults).""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler() + + # None means use default stealth config (enabled) + assert crawler._stealth_config is None + + def test_crawler_can_disable_stealth(self): + """Test that stealth can be explicitly disabled.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + from kb_workflow.crawler.stealth import StealthConfig + + disabled_config = StealthConfig(stealth_enabled=False) + crawler = ZhihuCrawler(stealth_config=disabled_config) + + assert crawler._stealth_config.stealth_enabled is False + + @patch('kb_workflow.crawler.zhihu_crawler.sync_playwright') + @patch('kb_workflow.crawler.zhihu_crawler.apply_stealth_sync') + def test_start_browser_applies_stealth( + self, + mock_apply_stealth, + mock_sync_playwright, + ): + """Test that _start_browser_with_proxy calls apply_stealth_sync.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + # Setup mocks + mock_playwright = MagicMock() + mock_browser = MagicMock() + mock_context = MagicMock() + mock_sync_playwright.return_value.start.return_value = mock_playwright + mock_playwright.chromium.launch.return_value = mock_browser + mock_browser.new_context.return_value = mock_context + + crawler = ZhihuCrawler() + crawler._start_browser_with_proxy(cookie="test_cookie") + + # Verify apply_stealth_sync was called + mock_apply_stealth.assert_called_once() + call_args = mock_apply_stealth.call_args + assert call_args[0][0] == mock_context # First arg is context + + @patch('kb_workflow.crawler.zhihu_crawler.sync_playwright') + @patch('kb_workflow.crawler.zhihu_crawler.apply_stealth_sync') + def test_start_browser_launch_args_include_disable_blink_features( + self, + mock_apply_stealth, + mock_sync_playwright, + ): + """Test that launch includes --disable-blink-features=AutomationControlled.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + # Setup mocks + mock_playwright = MagicMock() + mock_browser = MagicMock() + mock_context = MagicMock() + mock_sync_playwright.return_value.start.return_value = mock_playwright + mock_playwright.chromium.launch.return_value = mock_browser + mock_browser.new_context.return_value = mock_context + + crawler = ZhihuCrawler() + crawler._start_browser_with_proxy(cookie="test_cookie") + + # Verify launch was called with correct args + launch_call = mock_playwright.chromium.launch + launch_call.assert_called_once() + call_kwargs = launch_call.call_args[1] + + assert 'args' in call_kwargs + assert '--disable-blink-features=AutomationControlled' in call_kwargs['args'] + + @patch('kb_workflow.crawler.zhihu_crawler.sync_playwright') + @patch('kb_workflow.crawler.zhihu_crawler.apply_stealth_sync') + def test_start_browser_stealth_disabled_no_blink_features_arg( + self, + mock_apply_stealth, + mock_sync_playwright, + ): + """Test that --disable-blink-features is not added when stealth disabled.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + from kb_workflow.crawler.stealth import StealthConfig + + # Setup mocks + mock_playwright = MagicMock() + mock_browser = MagicMock() + mock_context = MagicMock() + mock_sync_playwright.return_value.start.return_value = mock_playwright + mock_playwright.chromium.launch.return_value = mock_browser + mock_browser.new_context.return_value = mock_context + + disabled_config = StealthConfig(stealth_enabled=False) + crawler = ZhihuCrawler(stealth_config=disabled_config) + crawler._start_browser_with_proxy(cookie="test_cookie") + + # Verify launch was called without args or with empty/None args + launch_call = mock_playwright.chromium.launch + call_kwargs = launch_call.call_args[1] + + # When stealth disabled, args should be None or not contain the flag + if 'args' in call_kwargs and call_kwargs['args']: + assert '--disable-blink-features=AutomationControlled' not in call_kwargs['args'] + + # apply_stealth_sync should still be called but with disabled config + mock_apply_stealth.assert_called_once() + + + class TestZhihuCrawlerBackwardCompatibility: + """Tests for backward compatibility after stealth integration.""" + + def test_crawler_works_without_stealth_config(self): + """Test that ZhihuCrawler works without explicit stealth_config.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + # Should not raise any errors + crawler = ZhihuCrawler( + headless=True, + timeout=30 + ) + assert crawler is not None + + def test_crawler_accepts_all_existing_params(self): + """Test that all existing parameters still work.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + from kb_workflow.crawler.config import CrawlerConfig + + config = CrawlerConfig() + crawler = ZhihuCrawler( + headless=False, + timeout=60, + config=config + ) + + assert crawler.headless is False + assert crawler.timeout == 60 + assert crawler.config == config + + + # Wave 0 completion marker + WAVE_0_COMPLETE = True + ``` + + + pytest tests/test_crawler/test_zhihu_crawler_stealth.py -x -v + + + - test_zhihu_crawler_stealth.py 创建完成 + - 8 个测试用例覆盖 stealth 集成和向后兼容性 + - 所有测试通过 + + + + + Task 3: 回归测试验证 + N/A + + 运行完整的 crawler 测试套件，确保 stealth 集成没有破坏现有功能： + + 1. 运行所有 crawler 测试: + ```bash + pytest tests/test_crawler/ -v + ``` + + 2. 检查测试输出，确保: + - 所有现有测试通过 + - 没有新增的 deprecation warnings + - 没有导入错误 + + 3. 如果有测试失败，分析原因并修复 + + + pytest tests/test_crawler/ -v + + + - 所有 crawler 测试通过（包括新测试和现有测试） + - VERIFY-02 要求满足（现有功能保持正常工作） + + + + + + +**Per-task verification:** +- Task 1: `pytest tests/test_crawler/test_zhihu_crawler_stealth.py::TestZhihuCrawlerStealthIntegration -x -v` +- Task 2: `pytest tests/test_crawler/test_zhihu_crawler_stealth.py -x -v` +- Task 3: `pytest tests/test_crawler/ -v` + +**Wave verification:** +- All 8 new tests pass +- All existing tests pass (regression check) +- Total test count should increase by 8 + + + +- [ ] ZhihuCrawler 集成 stealth_config 参数 +- [ ] _start_browser_with_proxy 调用 apply_stealth_sync +- [ ] launch 包含 --disable-blink-features=AutomationControlled 参数 +- [ ] 8 个集成测试全部通过 +- [ ] 所有现有测试通过（无回归） + + + +After completion, create `.planning/phases/11-stealth-fingerprinting/11-02-SUMMARY.md` +