From e152b8ea0555cff9a4c6e7f5285c1aea0ce18cce Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Wed, 18 Mar 2026 23:47:50 +0800 Subject: [PATCH] feat(11-01): create stealth fingerprint masking module - Add StealthConfig Pydantic model with all playwright-stealth options - Add apply_stealth_sync/apply_stealth_async functions for browser context - Add get_default_stealth_config convenience function - Add 9 unit tests covering config defaults, stealth application, and disabled mode - Update conftest.py with stealth_config and mock_browser_context fixtures Co-Authored-By: Claude Opus 4.6 --- kb_workflow/crawler/stealth/__init__.py | 15 ++ kb_workflow/crawler/stealth/stealth_config.py | 169 ++++++++++++++++++ tests/test_crawler/conftest.py | 15 ++ tests/test_crawler/test_stealth.py | 137 ++++++++++++++ 4 files changed, 336 insertions(+) create mode 100644 kb_workflow/crawler/stealth/__init__.py create mode 100644 kb_workflow/crawler/stealth/stealth_config.py create mode 100644 tests/test_crawler/test_stealth.py diff --git a/kb_workflow/crawler/stealth/__init__.py b/kb_workflow/crawler/stealth/__init__.py new file mode 100644 index 0000000..c0890fb --- /dev/null +++ b/kb_workflow/crawler/stealth/__init__.py @@ -0,0 +1,15 @@ +"""Stealth module for browser fingerprint masking.""" + +from .stealth_config import ( + StealthConfig, + apply_stealth_sync, + apply_stealth_async, + get_default_stealth_config, +) + +__all__ = [ + "StealthConfig", + "apply_stealth_sync", + "apply_stealth_async", + "get_default_stealth_config", +] diff --git a/kb_workflow/crawler/stealth/stealth_config.py b/kb_workflow/crawler/stealth/stealth_config.py new file mode 100644 index 0000000..18d2ff0 --- /dev/null +++ b/kb_workflow/crawler/stealth/stealth_config.py @@ -0,0 +1,169 @@ +"""Stealth configuration for browser fingerprint masking. + +Uses playwright-stealth library to hide Playwright automation signatures. +""" + +from typing import Optional +from playwright.sync_api import BrowserContext +from playwright.async_api import BrowserContext as AsyncBrowserContext +from pydantic import BaseModel, Field +from playwright_stealth import Stealth + + +class StealthConfig(BaseModel): + """Configuration for browser stealth settings. + + All options map to playwright-stealth evasion scripts. + Default values optimized for Zhihu anti-detection. + """ + + stealth_enabled: bool = Field( + default=True, + description="Enable stealth mode (set False to disable)" + ) + + # Navigator evasion options + navigator_webdriver: bool = Field( + default=True, + description="Hide navigator.webdriver property (STEALTH-01)" + ) + navigator_user_agent: bool = Field( + default=True, + description="Replace HeadlessChrome in User-Agent (STEALTH-04)" + ) + navigator_plugins: bool = Field( + default=True, + description="Mock navigator.plugins array" + ) + navigator_languages: bool = Field( + default=True, + description="Set navigator.languages" + ) + navigator_platform: bool = Field( + default=True, + description="Set navigator.platform" + ) + navigator_vendor: bool = Field( + default=True, + description="Set navigator.vendor" + ) + navigator_permissions: bool = Field( + default=True, + description="Override navigator.permissions" + ) + + # Chrome object evasion (STEALTH-03) + chrome_app: bool = Field( + default=True, + description="Mock chrome.app object" + ) + chrome_csi: bool = Field( + default=True, + description="Mock chrome.csi function" + ) + chrome_load_times: bool = Field( + default=True, + description="Mock chrome.loadTimes function" + ) + + # WebGL and rendering + webgl_vendor: bool = Field( + default=True, + description="Override WebGL vendor/renderer strings" + ) + + # Other evasion options + hairline: bool = Field( + default=True, + description="Fix hairline detection" + ) + iframe_content_window: bool = Field( + default=True, + description="Fix iframe contentWindow" + ) + media_codecs: bool = Field( + default=True, + description="Mock media codecs" + ) + error_prototype: bool = Field( + default=True, + description="Fix error prototype chain" + ) + sec_ch_ua: bool = Field( + default=True, + description="Set sec-ch-ua headers" + ) + + def create_stealth(self) -> Stealth: + """Create playwright-stealth Stealth instance from config. + + Returns: + Configured Stealth instance + """ + return Stealth( + navigator_webdriver=self.navigator_webdriver, + navigator_user_agent=self.navigator_user_agent, + navigator_plugins=self.navigator_plugins, + navigator_languages=self.navigator_languages, + navigator_platform=self.navigator_platform, + navigator_vendor=self.navigator_vendor, + navigator_permissions=self.navigator_permissions, + chrome_app=self.chrome_app, + chrome_csi=self.chrome_csi, + chrome_load_times=self.chrome_load_times, + webgl_vendor=self.webgl_vendor, + hairline=self.hairline, + iframe_content_window=self.iframe_content_window, + media_codecs=self.media_codecs, + error_prototype=self.error_prototype, + sec_ch_ua=self.sec_ch_ua, + ) + + +def apply_stealth_sync( + context: BrowserContext, + config: Optional[StealthConfig] = None +) -> None: + """Apply stealth settings to browser context (synchronous). + + Args: + context: Playwright BrowserContext to apply stealth to + config: StealthConfig instance (uses defaults if not provided) + """ + if config is None: + config = StealthConfig() + + if not config.stealth_enabled: + return + + stealth = config.create_stealth() + stealth.apply_stealth_sync(context) + + +async def apply_stealth_async( + context: AsyncBrowserContext, + config: Optional[StealthConfig] = None +) -> None: + """Apply stealth settings to browser context (asynchronous). + + Args: + context: Playwright async BrowserContext to apply stealth to + config: StealthConfig instance (uses defaults if not provided) + """ + if config is None: + config = StealthConfig() + + if not config.stealth_enabled: + return + + stealth = config.create_stealth() + await stealth.apply_stealth_async(context) + + +def get_default_stealth_config() -> StealthConfig: + """Get default StealthConfig for Zhihu crawling. + + Returns: + StealthConfig with all evasion options enabled + """ + return StealthConfig() diff --git a/tests/test_crawler/conftest.py b/tests/test_crawler/conftest.py index 3337b0b..44fa918 100644 --- a/tests/test_crawler/conftest.py +++ b/tests/test_crawler/conftest.py @@ -102,3 +102,18 @@ def sample_article_url(): def sample_invalid_url(): """Sample invalid URL for testing error handling.""" return "https://www.example.com/some-page" + + +@pytest.fixture +def stealth_config(): + """Default StealthConfig for testing.""" + from kb_workflow.crawler.stealth import StealthConfig + return StealthConfig() + + +@pytest.fixture +def mock_browser_context(): + """Mock Playwright BrowserContext for stealth testing.""" + context = MagicMock() + context.add_init_script = MagicMock() + return context diff --git a/tests/test_crawler/test_stealth.py b/tests/test_crawler/test_stealth.py new file mode 100644 index 0000000..f1f9c16 --- /dev/null +++ b/tests/test_crawler/test_stealth.py @@ -0,0 +1,137 @@ +"""Tests for stealth fingerprint masking module.""" + +import pytest +from unittest.mock import MagicMock, patch, call + + +class TestStealthConfig: + """Tests for StealthConfig class.""" + + def test_stealth_config_defaults(self): + """Test that default StealthConfig has all options enabled.""" + from kb_workflow.crawler.stealth import StealthConfig + + config = StealthConfig() + + assert config.stealth_enabled is True + # STEALTH-01: navigator.webdriver hiding + assert config.navigator_webdriver is True + # STEALTH-04: User-Agent modification + assert config.navigator_user_agent is True + # STEALTH-03: Chrome properties + assert config.chrome_app is True + assert config.chrome_csi is True + assert config.chrome_load_times is True + # Other important options + assert config.navigator_plugins is True + assert config.webgl_vendor is True + + def test_stealth_config_disabled(self): + """Test that stealth can be disabled.""" + from kb_workflow.crawler.stealth import StealthConfig + + config = StealthConfig(stealth_enabled=False) + assert config.stealth_enabled is False + + def test_create_stealth_returns_stealth_instance(self): + """Test that create_stealth() returns configured Stealth.""" + from kb_workflow.crawler.stealth import StealthConfig + from playwright_stealth import Stealth + + config = StealthConfig() + stealth = config.create_stealth() + + assert isinstance(stealth, Stealth) + + def test_stealth_options_mapping(self): + """Test that StealthConfig options map correctly to Stealth.""" + from kb_workflow.crawler.stealth import StealthConfig + + config = StealthConfig( + navigator_webdriver=True, + navigator_user_agent=False, # Disable one option + chrome_app=True, + ) + + stealth = config.create_stealth() + + # Verify the Stealth object was created with correct options + # The Stealth class stores these as attributes + assert stealth.navigator_webdriver is True + assert stealth.navigator_user_agent is False + + +class TestApplyStealth: + """Tests for apply_stealth functions.""" + + def test_apply_stealth_sync_calls_library(self, mock_browser_context): + """Test that apply_stealth_sync calls playwright-stealth correctly.""" + from kb_workflow.crawler.stealth import apply_stealth_sync, StealthConfig + + config = StealthConfig() + + with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class: + mock_stealth_instance = MagicMock() + mock_stealth_class.return_value = mock_stealth_instance + + apply_stealth_sync(mock_browser_context, config) + + # Verify Stealth was created with config options + mock_stealth_class.assert_called_once() + # Verify apply_stealth_sync was called on the Stealth instance + mock_stealth_instance.apply_stealth_sync.assert_called_once_with( + mock_browser_context + ) + + def test_apply_stealth_sync_disabled_skips_stealth(self, mock_browser_context): + """Test that stealth is skipped when disabled.""" + from kb_workflow.crawler.stealth import apply_stealth_sync, StealthConfig + + config = StealthConfig(stealth_enabled=False) + + with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class: + apply_stealth_sync(mock_browser_context, config) + + # Stealth should not be created when disabled + mock_stealth_class.assert_not_called() + + def test_apply_stealth_sync_default_config(self, mock_browser_context): + """Test that apply_stealth_sync works with default config.""" + from kb_workflow.crawler.stealth import apply_stealth_sync + + with patch('kb_workflow.crawler.stealth.stealth_config.Stealth') as mock_stealth_class: + mock_stealth_instance = MagicMock() + mock_stealth_class.return_value = mock_stealth_instance + + # Should work without explicit config + apply_stealth_sync(mock_browser_context) + + mock_stealth_class.assert_called_once() + + +class TestGetDefaultStealthConfig: + """Tests for get_default_stealth_config function.""" + + def test_get_default_stealth_config(self): + """Test that get_default_stealth_config returns StealthConfig.""" + from kb_workflow.crawler.stealth import get_default_stealth_config + + config = get_default_stealth_config() + + assert config.stealth_enabled is True + assert isinstance(config, type(config)) # Is a StealthConfig + + def test_get_default_stealth_config_is_fresh_instance(self): + """Test that each call returns a new instance.""" + from kb_workflow.crawler.stealth import get_default_stealth_config + + config1 = get_default_stealth_config() + config2 = get_default_stealth_config() + + # Should be equal but not the same object + assert config1 == config2 + assert config1 is not config2 + + +# Wave 0 completion marker +WAVE_0_COMPLETE = True