19 KiB
19 KiB
phase, plan, type, wave, depends_on, files_modified, autonomous, requirements, must_haves
| phase | plan | type | wave | depends_on | files_modified | autonomous | requirements | must_haves | ||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 11-stealth-fingerprinting | 01 | execute | 1 |
|
true |
|
|
Purpose: 为 ZhihuCrawler 提供可复用的指纹伪装能力，绕过知乎的浏览器指纹检测 Output: kb_workflow/crawler/stealth/ 模块 + 单元测试
<execution_context> @/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md @/home/starmen1265/.claude/get-shit-done/templates/summary.md </execution_context>
@.planning/PROJECT.md @.planning/ROADMAP.md @.planning/STATE.md @.planning/phases/11-stealth-fingerprinting/11-RESEARCH.md @.planning/phases/11-stealth-fingerprinting/11-VALIDATION.mdFrom kb_workflow/crawler/config.py:
class CrawlerConfig(BaseModel):
proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration")
proxy_required: bool = Field(default=False)
min_delay: float = Field(default=1.0)
max_delay: float = Field(default=3.0)
From kb_workflow/crawler/zhihu_crawler.py:
class ZhihuCrawler:
def __init__(
self,
headless: bool = True,
timeout: int = 30,
config: Optional[CrawlerConfig] = None,
proxy_pool: Optional[ProxyPool] = None
):
...
def _start_browser_with_proxy(self, cookie: str) -> None:
"""Start browser with proxy configuration."""
self._playwright = sync_playwright().start()
self._browser = self._playwright.chromium.launch(headless=self.headless)
context_options = {
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
...
self._context = self._browser.new_context(**context_options)
From playwright_stealth (external library):
from playwright_stealth import Stealth
stealth = Stealth(
navigator_webdriver=True,
navigator_user_agent=True,
navigator_plugins=True,
# ... other options
)
stealth.apply_stealth_async(context) # for async
stealth.apply_stealth_sync(context) # for sync
2. 创建 `kb_workflow/crawler/stealth/__init__.py`:
- 导出 StealthConfig 类
- 导出 apply_stealth_sync, apply_stealth_async 函数
- 导出 get_default_stealth_config 便捷函数
3. 创建 `kb_workflow/crawler/stealth/stealth_config.py`:
```python
"""Stealth configuration for browser fingerprint masking.
Uses playwright-stealth library to hide Playwright automation signatures.
"""
from typing import Optional
from playwright.sync_api import BrowserContext
from playwright.async_api import BrowserContext as AsyncBrowserContext
from pydantic import BaseModel, Field
from playwright_stealth import Stealth
class StealthConfig(BaseModel):
"""Configuration for browser stealth settings.
All options map to playwright-stealth evasion scripts.
Default values optimized for Zhihu anti-detection.
"""
stealth_enabled: bool = Field(
default=True,
description="Enable stealth mode (set False to disable)"
)
# Navigator evasion options
navigator_webdriver: bool = Field(
default=True,
description="Hide navigator.webdriver property (STEALTH-01)"
)
navigator_user_agent: bool = Field(
default=True,
description="Replace HeadlessChrome in User-Agent (STEALTH-04)"
)
navigator_plugins: bool = Field(
default=True,
description="Mock navigator.plugins array"
)
navigator_languages: bool = Field(
default=True,
description="Set navigator.languages"
)
navigator_platform: bool = Field(
default=True,
description="Set navigator.platform"
)
navigator_vendor: bool = Field(
default=True,
description="Set navigator.vendor"
)
navigator_permissions: bool = Field(
default=True,
description="Override navigator.permissions"
)
# Chrome object evasion (STEALTH-03)
chrome_app: bool = Field(
default=True,
description="Mock chrome.app object"
)
chrome_csi: bool = Field(
default=True,
description="Mock chrome.csi function"
)
chrome_load_times: bool = Field(
default=True,
description="Mock chrome.loadTimes function"
)
# WebGL and rendering
webgl_vendor: bool = Field(
default=True,
description="Override WebGL vendor/renderer strings"
)
# Other evasion options
hairline: bool = Field(
default=True,
description="Fix hairline detection"
)
iframe_content_window: bool = Field(
default=True,
description="Fix iframe contentWindow"
)
media_codecs: bool = Field(
default=True,
description="Mock media codecs"
)
error_prototype: bool = Field(
default=True,
description="Fix error prototype chain"
)
sec_ch_ua: bool = Field(
default=True,
description="Set sec-ch-ua headers"
)
def create_stealth(self) -> Stealth:
"""Create playwright-stealth Stealth instance from config.
Returns:
Configured Stealth instance
"""
return Stealth(
navigator_webdriver=self.navigator_webdriver,
navigator_user_agent=self.navigator_user_agent,
navigator_plugins=self.navigator_plugins,
navigator_languages=self.navigator_languages,
navigator_platform=self.navigator_platform,
navigator_vendor=self.navigator_vendor,
navigator_permissions=self.navigator_permissions,
chrome_app=self.chrome_app,
chrome_csi=self.chrome_csi,
chrome_load_times=self.chrome_load_times,
webgl_vendor=self.webgl_vendor,
hairline=self.hairline,
iframe_content_window=self.iframe_content_window,
media_codecs=self.media_codecs,
error_prototype=self.error_prototype,
sec_ch_ua=self.sec_ch_ua,
)
def apply_stealth_sync(
context: BrowserContext,
config: Optional[StealthConfig] = None
) -> None:
"""Apply stealth settings to browser context (synchronous).
Args:
context: Playwright BrowserContext to apply stealth to
config: StealthConfig instance (uses defaults if not provided)
"""
if config is None:
config = StealthConfig()
if not config.stealth_enabled:
return
stealth = config.create_stealth()
stealth.apply_stealth_sync(context)
async def apply_stealth_async(
context: AsyncBrowserContext,
config: Optional[StealthConfig] = None
) -> None:
"""Apply stealth settings to browser context (asynchronous).
Args:
context: Playwright async BrowserContext to apply stealth to
config: StealthConfig instance (uses defaults if not provided)
"""
if config is None:
config = StealthConfig()
if not config.stealth_enabled:
return
stealth = config.create_stealth()
await stealth.apply_stealth_async(context)
def get_default_stealth_config() -> StealthConfig:
"""Get default StealthConfig for Zhihu crawling.
Returns:
StealthConfig with all evasion options enabled
"""
return StealthConfig()
```
4. 创建 `kb_workflow/crawler/stealth/__init__.py`:
```python
"""Stealth module for browser fingerprint masking."""
from .stealth_config import (
StealthConfig,
apply_stealth_sync,
apply_stealth_async,
get_default_stealth_config,
)
__all__ = [
"StealthConfig",
"apply_stealth_sync",
"apply_stealth_async",
"get_default_stealth_config",
]
```
```python
@pytest.fixture
def stealth_config():
"""Default StealthConfig for testing."""
from kb_workflow.crawler.stealth import StealthConfig
return StealthConfig()
@pytest.fixture
def mock_browser_context():
"""Mock Playwright BrowserContext for stealth testing."""
from unittest.mock import MagicMock
context = MagicMock()
context.add_init_script = MagicMock()
return context
```
2. 创建 `tests/test_crawler/test_stealth.py`:
```python
"""Tests for stealth fingerprint masking module."""
import pytest
from unittest.mock import MagicMock, patch, call
class TestStealthConfig:
"""Tests for StealthConfig class."""
def test_stealth_config_defaults(self):
"""Test that default StealthConfig has all options enabled."""
from kb_workflow.crawler.stealth import StealthConfig
config = StealthConfig()
assert config.stealth_enabled is True
# STEALTH-01: navigator.webdriver hiding
assert config.navigator_webdriver is True
# STEALTH-04: User-Agent modification
assert config.navigator_user_agent is True
# STEALTH-03: Chrome properties
assert config.chrome_app is True
assert config.chrome_csi is True
assert config.chrome_load_times is True
# Other important options
assert config.navigator_plugins is True
assert config.webgl_vendor is True
def test_stealth_config_disabled(self):
"""Test that stealth can be disabled."""
from kb_workflow.crawler.stealth import StealthConfig
config = StealthConfig(stealth_enabled=False)
assert config.stealth_enabled is False
def test_create_stealth_returns_stealth_instance(self):
"""Test that create_stealth() returns configured Stealth."""
from kb_workflow.crawler.stealth import StealthConfig
from playwright_stealth import Stealth
config = StealthConfig()
stealth = config.create_stealth()
assert isinstance(stealth, Stealth)
def test_stealth_options_mapping(self):
"""Test that StealthConfig options map correctly to Stealth."""
from kb_workflow.crawler.stealth import StealthConfig
config = StealthConfig(
navigator_webdriver=True,
navigator_user_agent=False, # Disable one option
chrome_app=True,
)
stealth = config.create_stealth()
# Verify the Stealth object was created with correct options
# The Stealth class stores these as attributes
assert stealth.navigator_webdriver is True
assert stealth.navigator_user_agent is False
class TestApplyStealth:
"""Tests for apply_stealth functions."""
def test_apply_stealth_sync_calls_library(self, mock_browser_context):
"""Test that apply_stealth_sync calls playwright-stealth correctly."""
from kb_workflow.crawler.stealth import apply_stealth_sync, StealthConfig
config = StealthConfig()
with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class:
mock_stealth_instance = MagicMock()
mock_stealth_class.return_value = mock_stealth_instance
apply_stealth_sync(mock_browser_context, config)
# Verify Stealth was created with config options
mock_stealth_class.assert_called_once()
# Verify apply_stealth_sync was called on the Stealth instance
mock_stealth_instance.apply_stealth_sync.assert_called_once_with(
mock_browser_context
)
def test_apply_stealth_sync_disabled_skips_stealth(self, mock_browser_context):
"""Test that stealth is skipped when disabled."""
from kb_workflow.crawler.stealth import apply_stealth_sync, StealthConfig
config = StealthConfig(stealth_enabled=False)
with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class:
apply_stealth_sync(mock_browser_context, config)
# Stealth should not be created when disabled
mock_stealth_class.assert_not_called()
def test_apply_stealth_sync_default_config(self, mock_browser_context):
"""Test that apply_stealth_sync works with default config."""
from kb_workflow.crawler.stealth import apply_stealth_sync
with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class:
mock_stealth_instance = MagicMock()
mock_stealth_class.return_value = mock_stealth_instance
# Should work without explicit config
apply_stealth_sync(mock_browser_context)
mock_stealth_class.assert_called_once()
class TestGetDefaultStealthConfig:
"""Tests for get_default_stealth_config function."""
def test_get_default_stealth_config(self):
"""Test that get_default_stealth_config returns StealthConfig."""
from kb_workflow.crawler.stealth import get_default_stealth_config
config = get_default_stealth_config()
assert config.stealth_enabled is True
assert isinstance(config, type(config)) # Is a StealthConfig
def test_get_default_stealth_config_is_fresh_instance(self):
"""Test that each call returns a new instance."""
from kb_workflow.crawler.stealth import get_default_stealth_config
config1 = get_default_stealth_config()
config2 = get_default_stealth_config()
# Should be equal but not the same object
assert config1 == config2
assert config1 is not config2
# Wave 0 completion marker
WAVE_0_COMPLETE = True
```
Wave verification:
- All 5 tests in test_stealth.py pass
- No regression in existing tests:
pytest tests/test_crawler/ -x
<success_criteria>
- kb_workflow/crawler/stealth/ 模块创建完成
- StealthConfig 类包含所有 playwright-stealth 选项
- apply_stealth_sync/apply_stealth_async 函数可用
- 5 个单元测试全部通过
- 无现有测试回归 </success_criteria>