18 KiB
18 KiB
phase, plan, type, wave, depends_on, files_modified, autonomous, requirements, must_haves
| phase | plan | type | wave | depends_on | files_modified | autonomous | requirements | must_haves | ||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 11-stealth-fingerprinting | 02 | execute | 2 |
|
|
true |
|
|
Purpose: 让 ZhihuCrawler 在启动浏览器时自动应用指纹伪装，绕过 40362 错误 Output: 修改后的 ZhihuCrawler + 集成测试
<execution_context> @/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md @/home/starmen1265/.claude/get-shit-done/templates/summary.md </execution_context>
@.planning/PROJECT.md @.planning/ROADMAP.md @.planning/STATE.md @.planning/phases/11-stealth-fingerprinting/11-RESEARCH.md @.planning/phases/11-stealth-fingerprinting/11-VALIDATION.md@.planning/phases/11-stealth-fingerprinting/11-01-PLAN.md
From kb_workflow/crawler/stealth (Plan 11-01):
class StealthConfig(BaseModel):
stealth_enabled: bool = True
navigator_webdriver: bool = True
navigator_user_agent: bool = True
# ... other options
def apply_stealth_sync(
context: BrowserContext,
config: Optional[StealthConfig] = None
) -> None:
"""Apply stealth settings to browser context (synchronous)."""
def get_default_stealth_config() -> StealthConfig:
"""Get default StealthConfig for Zhihu crawling."""
From kb_workflow/crawler/zhihu_crawler.py (current implementation):
class ZhihuCrawler:
def __init__(
self,
headless: bool = True,
timeout: int = 30,
config: Optional[CrawlerConfig] = None,
proxy_pool: Optional[ProxyPool] = None
):
self.headless = headless
self.timeout = timeout
self.config = config or get_crawler_config()
self.proxy_pool = proxy_pool
self._current_proxy: Optional[ProxyEntry] = None
# Internal browser management
self._playwright = None
self._browser = None
self._context = None
self._page = None
def _start_browser_with_proxy(self, cookie: str) -> None:
"""Start browser with proxy configuration."""
self.config.validate_proxy()
self._playwright = sync_playwright().start()
self._browser = self._playwright.chromium.launch(headless=self.headless)
context_options = {
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
# ... proxy configuration ...
context_options["service_workers"] = "block"
self._context = self._browser.new_context(**context_options)
if cookie:
self._add_cookies(cookie)
self._page = self._context.new_page()
self._page.set_default_timeout(self.timeout * 1000)
From tests/test_crawler/conftest.py:
@pytest.fixture
def mock_page():
"""Mock Playwright page object."""
page = MagicMock()
page.route = MagicMock()
page.goto = MagicMock()
return page
@pytest.fixture
def sample_answer_api_response():
"""Sample answer API response data."""
return {
"id": 123456789,
"type": "answer",
# ...
}
1. 添加导入（在文件顶部导入区域）:
```python
from .stealth import apply_stealth_sync, StealthConfig
```
2. 修改 `__init__` 方法，添加 stealth 参数:
```python
def __init__(
self,
headless: bool = True,
timeout: int = 30,
config: Optional[CrawlerConfig] = None,
proxy_pool: Optional[ProxyPool] = None,
stealth_config: Optional[StealthConfig] = None # 新增参数
):
"""Initialize ZhihuCrawler.
Args:
headless: Whether to run browser in headless mode
timeout: Request timeout in seconds
config: Optional CrawlerConfig instance
proxy_pool: Optional ProxyPool instance for multi-proxy rotation
stealth_config: Optional StealthConfig instance (uses defaults if not provided)
"""
self.headless = headless
self.timeout = timeout
self.config = config or get_crawler_config()
self.proxy_pool = proxy_pool
self._stealth_config = stealth_config # 存储配置
self._current_proxy: Optional[ProxyEntry] = None
# ... rest of initialization
```
3. 修改 `_start_browser_with_proxy` 方法:
```python
def _start_browser_with_proxy(self, cookie: str) -> None:
"""Start browser with proxy configuration.
Validates proxy if required and applies to browser context.
Applies stealth fingerprint masking after context creation.
Args:
cookie: Cookie string for authentication
Raises:
ValueError: If proxy_required=True but no proxy is configured
"""
# Validate proxy configuration
self.config.validate_proxy()
# Build launch args for stealth (关键：添加 --disable-blink-features)
launch_args = []
if self._stealth_config is None or self._stealth_config.stealth_enabled:
launch_args.append('--disable-blink-features=AutomationControlled')
self._playwright = sync_playwright().start()
self._browser = self._playwright.chromium.launch(
headless=self.headless,
args=launch_args if launch_args else None
)
# Build context options
context_options = {
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
# Prioritize proxy pool over single proxy config
if self.proxy_pool and self.proxy_pool.has_proxies():
proxy_entry = self.proxy_pool.get_next_proxy()
if proxy_entry:
proxy_config = self._parse_proxy_url(proxy_entry.url)
context_options["proxy"] = proxy_config
self._current_proxy = proxy_entry
elif self.config.proxy:
context_options["proxy"] = self.config.proxy.to_playwright_format()
# Block Service Worker to prevent interference with API interception
context_options["service_workers"] = "block"
self._context = self._browser.new_context(**context_options)
# 关键改动：应用 stealth 指纹伪装
apply_stealth_sync(self._context, self._stealth_config)
# Add cookies if provided
if cookie:
self._add_cookies(cookie)
self._page = self._context.new_page()
self._page.set_default_timeout(self.timeout * 1000)
```
关键点：
- 在 `launch()` 时添加 `--disable-blink-features=AutomationControlled` 参数
- 在 `new_context()` 后、添加 cookies 前调用 `apply_stealth_sync()`
- 保持向后兼容：stealth_config 为 None 时使用默认配置（启用 stealth）
```python
"""Integration tests for ZhihuCrawler stealth functionality.
Tests that stealth fingerprint masking is properly integrated
into ZhihuCrawler's browser initialization.
"""
import pytest
from unittest.mock import MagicMock, patch, call
class TestZhihuCrawlerStealthIntegration:
"""Tests for stealth integration in ZhihuCrawler."""
def test_crawler_has_stealth_config_param(self):
"""Test that ZhihuCrawler accepts stealth_config parameter."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
from kb_workflow.crawler.stealth import StealthConfig
config = StealthConfig(stealth_enabled=False)
crawler = ZhihuCrawler(stealth_config=config)
assert crawler._stealth_config == config
def test_crawler_default_stealth_enabled(self):
"""Test that stealth is enabled by default (None = use defaults)."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
crawler = ZhihuCrawler()
# None means use default stealth config (enabled)
assert crawler._stealth_config is None
def test_crawler_can_disable_stealth(self):
"""Test that stealth can be explicitly disabled."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
from kb_workflow.crawler.stealth import StealthConfig
disabled_config = StealthConfig(stealth_enabled=False)
crawler = ZhihuCrawler(stealth_config=disabled_config)
assert crawler._stealth_config.stealth_enabled is False
@patch('kb_workflow.crawler.zhihu_crawler.sync_playwright')
@patch('kb_workflow.crawler.zhihu_crawler.apply_stealth_sync')
def test_start_browser_applies_stealth(
self,
mock_apply_stealth,
mock_sync_playwright,
):
"""Test that _start_browser_with_proxy calls apply_stealth_sync."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
# Setup mocks
mock_playwright = MagicMock()
mock_browser = MagicMock()
mock_context = MagicMock()
mock_sync_playwright.return_value.start.return_value = mock_playwright
mock_playwright.chromium.launch.return_value = mock_browser
mock_browser.new_context.return_value = mock_context
crawler = ZhihuCrawler()
crawler._start_browser_with_proxy(cookie="test_cookie")
# Verify apply_stealth_sync was called
mock_apply_stealth.assert_called_once()
call_args = mock_apply_stealth.call_args
assert call_args[0][0] == mock_context # First arg is context
@patch('kb_workflow.crawler.zhihu_crawler.sync_playwright')
@patch('kb_workflow.crawler.zhihu_crawler.apply_stealth_sync')
def test_start_browser_launch_args_include_disable_blink_features(
self,
mock_apply_stealth,
mock_sync_playwright,
):
"""Test that launch includes --disable-blink-features=AutomationControlled."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
# Setup mocks
mock_playwright = MagicMock()
mock_browser = MagicMock()
mock_context = MagicMock()
mock_sync_playwright.return_value.start.return_value = mock_playwright
mock_playwright.chromium.launch.return_value = mock_browser
mock_browser.new_context.return_value = mock_context
crawler = ZhihuCrawler()
crawler._start_browser_with_proxy(cookie="test_cookie")
# Verify launch was called with correct args
launch_call = mock_playwright.chromium.launch
launch_call.assert_called_once()
call_kwargs = launch_call.call_args[1]
assert 'args' in call_kwargs
assert '--disable-blink-features=AutomationControlled' in call_kwargs['args']
@patch('kb_workflow.crawler.zhihu_crawler.sync_playwright')
@patch('kb_workflow.crawler.zhihu_crawler.apply_stealth_sync')
def test_start_browser_stealth_disabled_no_blink_features_arg(
self,
mock_apply_stealth,
mock_sync_playwright,
):
"""Test that --disable-blink-features is not added when stealth disabled."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
from kb_workflow.crawler.stealth import StealthConfig
# Setup mocks
mock_playwright = MagicMock()
mock_browser = MagicMock()
mock_context = MagicMock()
mock_sync_playwright.return_value.start.return_value = mock_playwright
mock_playwright.chromium.launch.return_value = mock_browser
mock_browser.new_context.return_value = mock_context
disabled_config = StealthConfig(stealth_enabled=False)
crawler = ZhihuCrawler(stealth_config=disabled_config)
crawler._start_browser_with_proxy(cookie="test_cookie")
# Verify launch was called without args or with empty/None args
launch_call = mock_playwright.chromium.launch
call_kwargs = launch_call.call_args[1]
# When stealth disabled, args should be None or not contain the flag
if 'args' in call_kwargs and call_kwargs['args']:
assert '--disable-blink-features=AutomationControlled' not in call_kwargs['args']
# apply_stealth_sync should still be called but with disabled config
mock_apply_stealth.assert_called_once()
class TestZhihuCrawlerBackwardCompatibility:
"""Tests for backward compatibility after stealth integration."""
def test_crawler_works_without_stealth_config(self):
"""Test that ZhihuCrawler works without explicit stealth_config."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
# Should not raise any errors
crawler = ZhihuCrawler(
headless=True,
timeout=30
)
assert crawler is not None
def test_crawler_accepts_all_existing_params(self):
"""Test that all existing parameters still work."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
from kb_workflow.crawler.config import CrawlerConfig
config = CrawlerConfig()
crawler = ZhihuCrawler(
headless=False,
timeout=60,
config=config
)
assert crawler.headless is False
assert crawler.timeout == 60
assert crawler.config == config
# Wave 0 completion marker
WAVE_0_COMPLETE = True
```
1. 运行所有 crawler 测试:
```bash
pytest tests/test_crawler/ -v
```
2. 检查测试输出，确保:
- 所有现有测试通过
- 没有新增的 deprecation warnings
- 没有导入错误
3. 如果有测试失败，分析原因并修复
Wave verification:
- All 8 new tests pass
- All existing tests pass (regression check)
- Total test count should increase by 8
<success_criteria>
- ZhihuCrawler 集成 stealth_config 参数
- _start_browser_with_proxy 调用 apply_stealth_sync
- launch 包含 --disable-blink-features=AutomationControlled 参数
- 8 个集成测试全部通过
- 所有现有测试通过（无回归） </success_criteria>