feat(11-01): create stealth fingerprint masking module
- Add StealthConfig Pydantic model with all playwright-stealth options - Add apply_stealth_sync/apply_stealth_async functions for browser context - Add get_default_stealth_config convenience function - Add 9 unit tests covering config defaults, stealth application, and disabled mode - Update conftest.py with stealth_config and mock_browser_context fixtures Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,15 @@
|
|||||||
|
"""Stealth module for browser fingerprint masking."""
|
||||||
|
|
||||||
|
from .stealth_config import (
|
||||||
|
StealthConfig,
|
||||||
|
apply_stealth_sync,
|
||||||
|
apply_stealth_async,
|
||||||
|
get_default_stealth_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"StealthConfig",
|
||||||
|
"apply_stealth_sync",
|
||||||
|
"apply_stealth_async",
|
||||||
|
"get_default_stealth_config",
|
||||||
|
]
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
"""Stealth configuration for browser fingerprint masking.
|
||||||
|
|
||||||
|
Uses playwright-stealth library to hide Playwright automation signatures.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Optional
|
||||||
|
from playwright.sync_api import BrowserContext
|
||||||
|
from playwright.async_api import BrowserContext as AsyncBrowserContext
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from playwright_stealth import Stealth
|
||||||
|
|
||||||
|
|
||||||
|
class StealthConfig(BaseModel):
|
||||||
|
"""Configuration for browser stealth settings.
|
||||||
|
|
||||||
|
All options map to playwright-stealth evasion scripts.
|
||||||
|
Default values optimized for Zhihu anti-detection.
|
||||||
|
"""
|
||||||
|
|
||||||
|
stealth_enabled: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Enable stealth mode (set False to disable)"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Navigator evasion options
|
||||||
|
navigator_webdriver: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Hide navigator.webdriver property (STEALTH-01)"
|
||||||
|
)
|
||||||
|
navigator_user_agent: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Replace HeadlessChrome in User-Agent (STEALTH-04)"
|
||||||
|
)
|
||||||
|
navigator_plugins: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Mock navigator.plugins array"
|
||||||
|
)
|
||||||
|
navigator_languages: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Set navigator.languages"
|
||||||
|
)
|
||||||
|
navigator_platform: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Set navigator.platform"
|
||||||
|
)
|
||||||
|
navigator_vendor: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Set navigator.vendor"
|
||||||
|
)
|
||||||
|
navigator_permissions: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Override navigator.permissions"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Chrome object evasion (STEALTH-03)
|
||||||
|
chrome_app: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Mock chrome.app object"
|
||||||
|
)
|
||||||
|
chrome_csi: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Mock chrome.csi function"
|
||||||
|
)
|
||||||
|
chrome_load_times: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Mock chrome.loadTimes function"
|
||||||
|
)
|
||||||
|
|
||||||
|
# WebGL and rendering
|
||||||
|
webgl_vendor: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Override WebGL vendor/renderer strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Other evasion options
|
||||||
|
hairline: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Fix hairline detection"
|
||||||
|
)
|
||||||
|
iframe_content_window: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Fix iframe contentWindow"
|
||||||
|
)
|
||||||
|
media_codecs: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Mock media codecs"
|
||||||
|
)
|
||||||
|
error_prototype: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Fix error prototype chain"
|
||||||
|
)
|
||||||
|
sec_ch_ua: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Set sec-ch-ua headers"
|
||||||
|
)
|
||||||
|
|
||||||
|
def create_stealth(self) -> Stealth:
|
||||||
|
"""Create playwright-stealth Stealth instance from config.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Configured Stealth instance
|
||||||
|
"""
|
||||||
|
return Stealth(
|
||||||
|
navigator_webdriver=self.navigator_webdriver,
|
||||||
|
navigator_user_agent=self.navigator_user_agent,
|
||||||
|
navigator_plugins=self.navigator_plugins,
|
||||||
|
navigator_languages=self.navigator_languages,
|
||||||
|
navigator_platform=self.navigator_platform,
|
||||||
|
navigator_vendor=self.navigator_vendor,
|
||||||
|
navigator_permissions=self.navigator_permissions,
|
||||||
|
chrome_app=self.chrome_app,
|
||||||
|
chrome_csi=self.chrome_csi,
|
||||||
|
chrome_load_times=self.chrome_load_times,
|
||||||
|
webgl_vendor=self.webgl_vendor,
|
||||||
|
hairline=self.hairline,
|
||||||
|
iframe_content_window=self.iframe_content_window,
|
||||||
|
media_codecs=self.media_codecs,
|
||||||
|
error_prototype=self.error_prototype,
|
||||||
|
sec_ch_ua=self.sec_ch_ua,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_stealth_sync(
|
||||||
|
context: BrowserContext,
|
||||||
|
config: Optional[StealthConfig] = None
|
||||||
|
) -> None:
|
||||||
|
"""Apply stealth settings to browser context (synchronous).
|
||||||
|
|
||||||
|
Args:
|
||||||
|
context: Playwright BrowserContext to apply stealth to
|
||||||
|
config: StealthConfig instance (uses defaults if not provided)
|
||||||
|
"""
|
||||||
|
if config is None:
|
||||||
|
config = StealthConfig()
|
||||||
|
|
||||||
|
if not config.stealth_enabled:
|
||||||
|
return
|
||||||
|
|
||||||
|
stealth = config.create_stealth()
|
||||||
|
stealth.apply_stealth_sync(context)
|
||||||
|
|
||||||
|
|
||||||
|
async def apply_stealth_async(
|
||||||
|
context: AsyncBrowserContext,
|
||||||
|
config: Optional[StealthConfig] = None
|
||||||
|
) -> None:
|
||||||
|
"""Apply stealth settings to browser context (asynchronous).
|
||||||
|
|
||||||
|
Args:
|
||||||
|
context: Playwright async BrowserContext to apply stealth to
|
||||||
|
config: StealthConfig instance (uses defaults if not provided)
|
||||||
|
"""
|
||||||
|
if config is None:
|
||||||
|
config = StealthConfig()
|
||||||
|
|
||||||
|
if not config.stealth_enabled:
|
||||||
|
return
|
||||||
|
|
||||||
|
stealth = config.create_stealth()
|
||||||
|
await stealth.apply_stealth_async(context)
|
||||||
|
|
||||||
|
|
||||||
|
def get_default_stealth_config() -> StealthConfig:
|
||||||
|
"""Get default StealthConfig for Zhihu crawling.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
StealthConfig with all evasion options enabled
|
||||||
|
"""
|
||||||
|
return StealthConfig()
|
||||||
@@ -102,3 +102,18 @@ def sample_article_url():
|
|||||||
def sample_invalid_url():
|
def sample_invalid_url():
|
||||||
"""Sample invalid URL for testing error handling."""
|
"""Sample invalid URL for testing error handling."""
|
||||||
return "https://www.example.com/some-page"
|
return "https://www.example.com/some-page"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def stealth_config():
|
||||||
|
"""Default StealthConfig for testing."""
|
||||||
|
from kb_workflow.crawler.stealth import StealthConfig
|
||||||
|
return StealthConfig()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_browser_context():
|
||||||
|
"""Mock Playwright BrowserContext for stealth testing."""
|
||||||
|
context = MagicMock()
|
||||||
|
context.add_init_script = MagicMock()
|
||||||
|
return context
|
||||||
|
|||||||
@@ -0,0 +1,137 @@
|
|||||||
|
"""Tests for stealth fingerprint masking module."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from unittest.mock import MagicMock, patch, call
|
||||||
|
|
||||||
|
|
||||||
|
class TestStealthConfig:
|
||||||
|
"""Tests for StealthConfig class."""
|
||||||
|
|
||||||
|
def test_stealth_config_defaults(self):
|
||||||
|
"""Test that default StealthConfig has all options enabled."""
|
||||||
|
from kb_workflow.crawler.stealth import StealthConfig
|
||||||
|
|
||||||
|
config = StealthConfig()
|
||||||
|
|
||||||
|
assert config.stealth_enabled is True
|
||||||
|
# STEALTH-01: navigator.webdriver hiding
|
||||||
|
assert config.navigator_webdriver is True
|
||||||
|
# STEALTH-04: User-Agent modification
|
||||||
|
assert config.navigator_user_agent is True
|
||||||
|
# STEALTH-03: Chrome properties
|
||||||
|
assert config.chrome_app is True
|
||||||
|
assert config.chrome_csi is True
|
||||||
|
assert config.chrome_load_times is True
|
||||||
|
# Other important options
|
||||||
|
assert config.navigator_plugins is True
|
||||||
|
assert config.webgl_vendor is True
|
||||||
|
|
||||||
|
def test_stealth_config_disabled(self):
|
||||||
|
"""Test that stealth can be disabled."""
|
||||||
|
from kb_workflow.crawler.stealth import StealthConfig
|
||||||
|
|
||||||
|
config = StealthConfig(stealth_enabled=False)
|
||||||
|
assert config.stealth_enabled is False
|
||||||
|
|
||||||
|
def test_create_stealth_returns_stealth_instance(self):
|
||||||
|
"""Test that create_stealth() returns configured Stealth."""
|
||||||
|
from kb_workflow.crawler.stealth import StealthConfig
|
||||||
|
from playwright_stealth import Stealth
|
||||||
|
|
||||||
|
config = StealthConfig()
|
||||||
|
stealth = config.create_stealth()
|
||||||
|
|
||||||
|
assert isinstance(stealth, Stealth)
|
||||||
|
|
||||||
|
def test_stealth_options_mapping(self):
|
||||||
|
"""Test that StealthConfig options map correctly to Stealth."""
|
||||||
|
from kb_workflow.crawler.stealth import StealthConfig
|
||||||
|
|
||||||
|
config = StealthConfig(
|
||||||
|
navigator_webdriver=True,
|
||||||
|
navigator_user_agent=False, # Disable one option
|
||||||
|
chrome_app=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
stealth = config.create_stealth()
|
||||||
|
|
||||||
|
# Verify the Stealth object was created with correct options
|
||||||
|
# The Stealth class stores these as attributes
|
||||||
|
assert stealth.navigator_webdriver is True
|
||||||
|
assert stealth.navigator_user_agent is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestApplyStealth:
|
||||||
|
"""Tests for apply_stealth functions."""
|
||||||
|
|
||||||
|
def test_apply_stealth_sync_calls_library(self, mock_browser_context):
|
||||||
|
"""Test that apply_stealth_sync calls playwright-stealth correctly."""
|
||||||
|
from kb_workflow.crawler.stealth import apply_stealth_sync, StealthConfig
|
||||||
|
|
||||||
|
config = StealthConfig()
|
||||||
|
|
||||||
|
with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class:
|
||||||
|
mock_stealth_instance = MagicMock()
|
||||||
|
mock_stealth_class.return_value = mock_stealth_instance
|
||||||
|
|
||||||
|
apply_stealth_sync(mock_browser_context, config)
|
||||||
|
|
||||||
|
# Verify Stealth was created with config options
|
||||||
|
mock_stealth_class.assert_called_once()
|
||||||
|
# Verify apply_stealth_sync was called on the Stealth instance
|
||||||
|
mock_stealth_instance.apply_stealth_sync.assert_called_once_with(
|
||||||
|
mock_browser_context
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_apply_stealth_sync_disabled_skips_stealth(self, mock_browser_context):
|
||||||
|
"""Test that stealth is skipped when disabled."""
|
||||||
|
from kb_workflow.crawler.stealth import apply_stealth_sync, StealthConfig
|
||||||
|
|
||||||
|
config = StealthConfig(stealth_enabled=False)
|
||||||
|
|
||||||
|
with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class:
|
||||||
|
apply_stealth_sync(mock_browser_context, config)
|
||||||
|
|
||||||
|
# Stealth should not be created when disabled
|
||||||
|
mock_stealth_class.assert_not_called()
|
||||||
|
|
||||||
|
def test_apply_stealth_sync_default_config(self, mock_browser_context):
|
||||||
|
"""Test that apply_stealth_sync works with default config."""
|
||||||
|
from kb_workflow.crawler.stealth import apply_stealth_sync
|
||||||
|
|
||||||
|
with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class:
|
||||||
|
mock_stealth_instance = MagicMock()
|
||||||
|
mock_stealth_class.return_value = mock_stealth_instance
|
||||||
|
|
||||||
|
# Should work without explicit config
|
||||||
|
apply_stealth_sync(mock_browser_context)
|
||||||
|
|
||||||
|
mock_stealth_class.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetDefaultStealthConfig:
|
||||||
|
"""Tests for get_default_stealth_config function."""
|
||||||
|
|
||||||
|
def test_get_default_stealth_config(self):
|
||||||
|
"""Test that get_default_stealth_config returns StealthConfig."""
|
||||||
|
from kb_workflow.crawler.stealth import get_default_stealth_config
|
||||||
|
|
||||||
|
config = get_default_stealth_config()
|
||||||
|
|
||||||
|
assert config.stealth_enabled is True
|
||||||
|
assert isinstance(config, type(config)) # Is a StealthConfig
|
||||||
|
|
||||||
|
def test_get_default_stealth_config_is_fresh_instance(self):
|
||||||
|
"""Test that each call returns a new instance."""
|
||||||
|
from kb_workflow.crawler.stealth import get_default_stealth_config
|
||||||
|
|
||||||
|
config1 = get_default_stealth_config()
|
||||||
|
config2 = get_default_stealth_config()
|
||||||
|
|
||||||
|
# Should be equal but not the same object
|
||||||
|
assert config1 == config2
|
||||||
|
assert config1 is not config2
|
||||||
|
|
||||||
|
|
||||||
|
# Wave 0 completion marker
|
||||||
|
WAVE_0_COMPLETE = True
|
||||||
Reference in New Issue
Block a user