自动同步: 2026-03-19 02:10

2026-03-19 02:10:04 +08:00
parent 4d8f8147b6
commit 264d69ca1e
58 changed files with 1416 additions and 234 deletions
@@ -3,6 +3,6 @@
 知识库工作流管道入口。
 """

-from .kb_workflow import process_command, main
+from .cli import process_command, main

 __all__ = ["process_command", "main"]
@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+KB Workflow 包入口点
+
+支持通过 python -m kb_workflow 直接运行 CLI。
+"""
+
+from .cli import main
+
+if __name__ == '__main__':
+    main()
@@ -9,11 +9,15 @@ KB Workflow - Knowledge Base Workflow Orchestrator
 内部使用 kb_workflow.pipeline 包实现完整的内容处理流程。

 使用方式:
-    python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
-    python kb_workflow.py process "https://..." -o /path/to/output
-    python kb_workflow.py login                    # QR code login
-    python kb_workflow.py login --import-cookie cookies.json
-    python kb_workflow.py login --status
+    python -m kb_workflow process "https://www.bilibili.com/video/BVxxx"
+    python -m kb_workflow process "https://..." -o /path/to/output
+    python -m kb_workflow login                    # QR code login
+    python -m kb_workflow login --import-cookie cookies.json
+    python -m kb_workflow login --status
+
+或者通过包接口调用:
+    from kb_workflow import process_command
+    result = process_command("https://...")
 """

 import argparse
@@ -41,13 +45,13 @@ def process_command(
        Dictionary with success status, content_dir, and results
    """
    try:
-        from kb_workflow.pipeline.config import get_settings
-        from kb_workflow.pipeline.pipeline import Pipeline
-        from kb_workflow.pipeline.processors import (
+        from .pipeline.config import get_settings
+        from .pipeline.pipeline import Pipeline
+        from .pipeline.processors import (
            ASRProcessor, AnalyzeProcessor, ArchiveProcessor,
            DownloadProcessor, ParseProcessor
        )
-        from kb_workflow.pipeline.schemas import PipelineConfig
+        from .pipeline.schemas import PipelineConfig

        async def run_pipeline():
            # Determine output directory - default to intermediate_path
@@ -119,9 +123,9 @@ def login_command(
        Dictionary with login result
    """
    try:
-        from kb_workflow.auth import ZhihuLogin
-        from kb_workflow.auth.cookie_manager import CookieManager, parse_json_cookies
-        from kb_workflow.pipeline.config import get_settings
+        from .auth import ZhihuLogin
+        from .auth.cookie_manager import CookieManager, parse_json_cookies
+        from .pipeline.config import get_settings

        settings = get_settings()
        manager = CookieManager()
@@ -175,22 +179,22 @@ def main():
        epilog='''
 使用示例:
  # 处理URL (通过完整管道)
-  python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
+  python -m kb_workflow process "https://www.bilibili.com/video/BVxxx"

  # 指定输出目录
-  python kb_workflow.py process "https://..." -o /path/to/output
+  python -m kb_workflow process "https://..." -o /path/to/output

  # 强制重新执行（忽略已完成状态）
-  python kb_workflow.py process "https://..." --no-skip
+  python -m kb_workflow process "https://..." --no-skip

  # 知乎登录 (二维码扫码)
-  python kb_workflow.py login
+  python -m kb_workflow login

  # 导入 Cookie
-  python kb_workflow.py login --import-cookie cookies.json
+  python -m kb_workflow login --import-cookie cookies.json

  # 检查 Cookie 状态
-  python kb_workflow.py login --status
+  python -m kb_workflow login --status
        '''
    )

@@ -0,0 +1,226 @@
+"""Proxy pool management for web crawling.
+
+Provides multi-proxy configuration, round-robin rotation, and
+backward compatibility with single proxy environment variables.
+
+Configuration file (~/.kb_proxies.yaml):
+    proxies:
+      - "http://127.0.0.1:7890"
+      - "http://user:pass@proxy.example.com:8080"
+      - "socks5://127.0.0.1:1080"
+    cooldown_seconds: 300
+    max_fail_count: 3
+"""
+
+import os
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+import yaml
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+
+class ProxyEntry(BaseModel):
+    """Single proxy configuration and state.
+
+    Tracks proxy URL, failure count, and availability status.
+
+    Attributes:
+        url: Complete proxy URL including authentication
+        fail_count: Consecutive failure count
+        last_fail_time: Timestamp of last failure
+        is_available: Whether proxy is currently available
+    """
+
+    url: str = Field(..., description="Complete proxy URL including auth")
+    fail_count: int = Field(default=0, description="Consecutive failure count")
+    last_fail_time: Optional[datetime] = Field(
+        default=None, description="Timestamp of last failure"
+    )
+    is_available: bool = Field(default=True, description="Whether proxy is available")
+
+    def record_success(self) -> None:
+        """Record successful request, reset failure count."""
+        self.fail_count = 0
+        self.is_available = True
+
+    def record_failure(self, max_fail_count: int = 3) -> None:
+        """Record failed request.
+
+        Args:
+            max_fail_count: Threshold for marking proxy unavailable
+        """
+        self.fail_count += 1
+        self.last_fail_time = datetime.now()
+        if self.fail_count >= max_fail_count:
+            self.is_available = False
+            logger.warning(f"Proxy marked as unavailable: {self.url[:30]}...")
+
+    def check_recovery(self, cooldown_seconds: int = 300) -> bool:
+        """Check if proxy can recover from cooldown.
+
+        Lazy check - only evaluates when called.
+
+        Args:
+            cooldown_seconds: Minimum time before recovery
+
+        Returns:
+            True if proxy is available or has recovered
+        """
+        if self.is_available:
+            return True
+        if self.last_fail_time is None:
+            return True
+
+        elapsed = (datetime.now() - self.last_fail_time).total_seconds()
+        if elapsed >= cooldown_seconds:
+            self.is_available = True
+            self.fail_count = 0
+            logger.info(f"Proxy recovered after {elapsed:.0f}s cooldown")
+            return True
+        return False
+
+
+class ProxyPoolConfig(BaseModel):
+    """Proxy pool configuration.
+
+    Attributes:
+        proxies: List of proxy URLs
+        cooldown_seconds: Time before failed proxy can recover
+        max_fail_count: Consecutive failures before marking unavailable
+    """
+
+    proxies: list[str] = Field(default_factory=list, description="Proxy URL list")
+    cooldown_seconds: int = Field(default=300, description="Cooldown time in seconds")
+    max_fail_count: int = Field(default=3, description="Max consecutive failures")
+
+
+class ProxyPool:
+    """Proxy pool manager.
+
+    Manages multiple proxies with:
+    - YAML file or environment variable loading
+    - Round-robin rotation
+    - Passive health detection
+    - Cooldown recovery
+
+    Usage:
+        pool = ProxyPool.from_yaml(Path.home() / ".kb_proxies.yaml")
+        if pool.has_proxies():
+            proxy = pool.get_next_proxy()
+    """
+
+    def __init__(self, config: ProxyPoolConfig):
+        """Initialize proxy pool.
+
+        Args:
+            config: Proxy pool configuration
+        """
+        self.config = config
+        self._entries: list[ProxyEntry] = [
+            ProxyEntry(url=url) for url in config.proxies
+        ]
+        self._current_index: int = 0
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> "ProxyPool":
+        """Load proxy pool from YAML file.
+
+        Args:
+            path: Path to YAML configuration file
+
+        Returns:
+            ProxyPool instance (empty if file doesn't exist)
+        """
+        if not path.exists():
+            logger.debug(f"Proxy config file not found: {path}")
+            return cls(ProxyPoolConfig())
+
+        with open(path) as f:
+            data = yaml.safe_load(f) or {}
+
+        proxies = data.get("proxies", [])
+        cooldown_seconds = data.get("cooldown_seconds", 300)
+        max_fail_count = data.get("max_fail_count", 3)
+
+        return cls(
+            ProxyPoolConfig(
+                proxies=proxies,
+                cooldown_seconds=cooldown_seconds,
+                max_fail_count=max_fail_count,
+            )
+        )
+
+    @classmethod
+    def from_env_fallback(cls) -> "ProxyPool":
+        """Load single proxy from environment variable.
+
+        Checks HTTPS_PROXY first, then HTTP_PROXY.
+        Provides backward compatibility with single proxy mode.
+
+        Returns:
+            ProxyPool instance with single proxy or empty
+        """
+        proxy_url = os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY")
+        if proxy_url:
+            logger.debug(f"Using proxy from environment: {proxy_url[:30]}...")
+            return cls(ProxyPoolConfig(proxies=[proxy_url]))
+        return cls(ProxyPoolConfig())
+
+    def get_next_proxy(self) -> Optional[ProxyEntry]:
+        """Get next available proxy using round-robin.
+
+        Implements:
+        1. Lazy recovery check for current proxy
+        2. Skip unavailable proxies
+        3. Fallback to first proxy if all unavailable
+
+        Returns:
+            Next ProxyEntry or None if pool is empty
+        """
+        if not self._entries:
+            return None
+
+        # Single proxy mode: always return the same proxy
+        if len(self._entries) == 1:
+            return self._entries[0]
+
+        # Try to find available proxy
+        checked: set[int] = set()
+        while len(checked) < len(self._entries):
+            entry = self._entries[self._current_index]
+            # Move index before any potential early returns
+            self._current_index = (self._current_index + 1) % len(self._entries)
+            checked.add(self._current_index)
+
+            # Lazy recovery check
+            if entry.check_recovery(self.config.cooldown_seconds):
+                return entry
+
+        # All proxies unavailable, fallback to first proxy
+        logger.warning("All proxies unavailable, falling back to first proxy")
+        return self._entries[0]
+
+    def has_proxies(self) -> bool:
+        """Check if pool has configured proxies.
+
+        Returns:
+            True if pool has at least one proxy
+        """
+        return len(self._entries) > 0
+
+    def get_available_count(self) -> int:
+        """Get count of currently available proxies.
+
+        Returns:
+            Number of available proxies
+        """
+        return sum(
+            1
+            for e in self._entries
+            if e.check_recovery(self.config.cooldown_seconds)
+        )
@@ -0,0 +1,15 @@
+"""Stealth module for browser fingerprint masking."""
+
+from .stealth_config import (
+    StealthConfig,
+    apply_stealth_sync,
+    apply_stealth_async,
+    get_default_stealth_config,
+)
+
+__all__ = [
+    "StealthConfig",
+    "apply_stealth_sync",
+    "apply_stealth_async",
+    "get_default_stealth_config",
+]
@@ -0,0 +1,169 @@
+"""Stealth configuration for browser fingerprint masking.
+
+Uses playwright-stealth library to hide Playwright automation signatures.
+"""
+
+from typing import Optional
+from playwright.sync_api import BrowserContext
+from playwright.async_api import BrowserContext as AsyncBrowserContext
+from pydantic import BaseModel, Field
+from playwright_stealth import Stealth
+
+
+class StealthConfig(BaseModel):
+    """Configuration for browser stealth settings.
+
+    All options map to playwright-stealth evasion scripts.
+    Default values optimized for Zhihu anti-detection.
+    """
+
+    stealth_enabled: bool = Field(
+        default=True,
+        description="Enable stealth mode (set False to disable)"
+    )
+
+    # Navigator evasion options
+    navigator_webdriver: bool = Field(
+        default=True,
+        description="Hide navigator.webdriver property (STEALTH-01)"
+    )
+    navigator_user_agent: bool = Field(
+        default=True,
+        description="Replace HeadlessChrome in User-Agent (STEALTH-04)"
+    )
+    navigator_plugins: bool = Field(
+        default=True,
+        description="Mock navigator.plugins array"
+    )
+    navigator_languages: bool = Field(
+        default=True,
+        description="Set navigator.languages"
+    )
+    navigator_platform: bool = Field(
+        default=True,
+        description="Set navigator.platform"
+    )
+    navigator_vendor: bool = Field(
+        default=True,
+        description="Set navigator.vendor"
+    )
+    navigator_permissions: bool = Field(
+        default=True,
+        description="Override navigator.permissions"
+    )
+
+    # Chrome object evasion (STEALTH-03)
+    chrome_app: bool = Field(
+        default=True,
+        description="Mock chrome.app object"
+    )
+    chrome_csi: bool = Field(
+        default=True,
+        description="Mock chrome.csi function"
+    )
+    chrome_load_times: bool = Field(
+        default=True,
+        description="Mock chrome.loadTimes function"
+    )
+
+    # WebGL and rendering
+    webgl_vendor: bool = Field(
+        default=True,
+        description="Override WebGL vendor/renderer strings"
+    )
+
+    # Other evasion options
+    hairline: bool = Field(
+        default=True,
+        description="Fix hairline detection"
+    )
+    iframe_content_window: bool = Field(
+        default=True,
+        description="Fix iframe contentWindow"
+    )
+    media_codecs: bool = Field(
+        default=True,
+        description="Mock media codecs"
+    )
+    error_prototype: bool = Field(
+        default=True,
+        description="Fix error prototype chain"
+    )
+    sec_ch_ua: bool = Field(
+        default=True,
+        description="Set sec-ch-ua headers"
+    )
+
+    def create_stealth(self) -> Stealth:
+        """Create playwright-stealth Stealth instance from config.
+
+        Returns:
+            Configured Stealth instance
+        """
+        return Stealth(
+            navigator_webdriver=self.navigator_webdriver,
+            navigator_user_agent=self.navigator_user_agent,
+            navigator_plugins=self.navigator_plugins,
+            navigator_languages=self.navigator_languages,
+            navigator_platform=self.navigator_platform,
+            navigator_vendor=self.navigator_vendor,
+            navigator_permissions=self.navigator_permissions,
+            chrome_app=self.chrome_app,
+            chrome_csi=self.chrome_csi,
+            chrome_load_times=self.chrome_load_times,
+            webgl_vendor=self.webgl_vendor,
+            hairline=self.hairline,
+            iframe_content_window=self.iframe_content_window,
+            media_codecs=self.media_codecs,
+            error_prototype=self.error_prototype,
+            sec_ch_ua=self.sec_ch_ua,
+        )
+
+
+def apply_stealth_sync(
+    context: BrowserContext,
+    config: Optional[StealthConfig] = None
+) -> None:
+    """Apply stealth settings to browser context (synchronous).
+
+    Args:
+        context: Playwright BrowserContext to apply stealth to
+        config: StealthConfig instance (uses defaults if not provided)
+    """
+    if config is None:
+        config = StealthConfig()
+
+    if not config.stealth_enabled:
+        return
+
+    stealth = config.create_stealth()
+    stealth.apply_stealth_sync(context)
+
+
+async def apply_stealth_async(
+    context: AsyncBrowserContext,
+    config: Optional[StealthConfig] = None
+) -> None:
+    """Apply stealth settings to browser context (asynchronous).
+
+    Args:
+        context: Playwright async BrowserContext to apply stealth to
+        config: StealthConfig instance (uses defaults if not provided)
+    """
+    if config is None:
+        config = StealthConfig()
+
+    if not config.stealth_enabled:
+        return
+
+    stealth = config.create_stealth()
+    await stealth.apply_stealth_async(context)
+
+
+def get_default_stealth_config() -> StealthConfig:
+    """Get default StealthConfig for Zhihu crawling.
+
+    Returns:
+        StealthConfig with all evasion options enabled
+    """
+    return StealthConfig()
@@ -8,12 +8,14 @@ Supports:
 - Random request delay
 - Retry with exponential backoff
 - Automatic x-zse-96 signature handling via browser
+- ProxyPool integration for multi-proxy rotation
 """

 import random
 import re
 import time
 from typing import Optional, Dict, Any
+from urllib.parse import urlparse

 from playwright.sync_api import Route, Response, sync_playwright
 from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
@@ -32,6 +34,8 @@ from .models import (
    CrawlResult,
 )
 from .config import CrawlerConfig, get_crawler_config
+from .proxy_pool import ProxyPool, ProxyEntry
+from .stealth import apply_stealth_sync, StealthConfig


 class ZhihuApiInterceptor:
@@ -104,7 +108,9 @@ class ZhihuCrawler:
        self,
        headless: bool = True,
        timeout: int = 30,
-        config: Optional[CrawlerConfig] = None
+        config: Optional[CrawlerConfig] = None,
+        proxy_pool: Optional[ProxyPool] = None,
+        stealth_config: Optional[StealthConfig] = None
    ):
        """Initialize ZhihuCrawler.

@@ -112,10 +118,15 @@ class ZhihuCrawler:
            headless: Whether to run browser in headless mode
            timeout: Request timeout in seconds
            config: Optional CrawlerConfig instance (uses environment if not provided)
+            proxy_pool: Optional ProxyPool instance for multi-proxy rotation
+            stealth_config: Optional StealthConfig instance (uses defaults if not provided)
        """
        self.headless = headless
        self.timeout = timeout
        self.config = config or get_crawler_config()
+        self.proxy_pool = proxy_pool
+        self._stealth_config = stealth_config
+        self._current_proxy: Optional[ProxyEntry] = None
        self.browser: Optional[PlaywrightBrowser] = None
        self.interceptor: Optional[ZhihuApiInterceptor] = None
        # Internal browser management for proxy support
@@ -148,6 +159,8 @@ class ZhihuCrawler:
        """Start browser with proxy configuration.

        Validates proxy if required and applies to browser context.
+        Applies stealth fingerprint masking after context creation.
+        Prioritizes proxy_pool over config.proxy for multi-proxy support.

        Args:
            cookie: Cookie string for authentication
@@ -158,16 +171,31 @@ class ZhihuCrawler:
        # Validate proxy configuration
        self.config.validate_proxy()

+        # Build launch args for stealth (add --disable-blink-features)
+        launch_args = []
+        if self._stealth_config is None or self._stealth_config.stealth_enabled:
+            launch_args.append('--disable-blink-features=AutomationControlled')
+
        self._playwright = sync_playwright().start()
-        self._browser = self._playwright.chromium.launch(headless=self.headless)
+        self._browser = self._playwright.chromium.launch(
+            headless=self.headless,
+            args=launch_args if launch_args else None
+        )

        # Build context options
        context_options = {
            'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }

-        # Apply proxy if configured
-        if self.config.proxy:
+        # Prioritize proxy pool over single proxy config
+        if self.proxy_pool and self.proxy_pool.has_proxies():
+            proxy_entry = self.proxy_pool.get_next_proxy()
+            if proxy_entry:
+                proxy_config = self._parse_proxy_url(proxy_entry.url)
+                context_options["proxy"] = proxy_config
+                self._current_proxy = proxy_entry
+        elif self.config.proxy:
+            # Backward compatibility: use single proxy configuration
            context_options["proxy"] = self.config.proxy.to_playwright_format()

        # Block Service Worker to prevent interference with API interception
@@ -175,6 +203,9 @@ class ZhihuCrawler:

        self._context = self._browser.new_context(**context_options)

+        # Apply stealth fingerprint masking
+        apply_stealth_sync(self._context, self._stealth_config)
+
        # Add cookies if provided
        if cookie:
            self._add_cookies(cookie)
@@ -182,6 +213,40 @@ class ZhihuCrawler:
        self._page = self._context.new_page()
        self._page.set_default_timeout(self.timeout * 1000)

+    @staticmethod
+    def _parse_proxy_url(url: str) -> Dict[str, Any]:
+        """Parse proxy URL into Playwright format.
+
+        Args:
+            url: Proxy URL (e.g., http://user:pass@proxy.example.com:8080)
+
+        Returns:
+            Dictionary with server, username, password for Playwright
+        """
+        parsed = urlparse(url)
+        server = f"{parsed.scheme}://{parsed.hostname}"
+        if parsed.port:
+            server += f":{parsed.port}"
+        config: Dict[str, Any] = {"server": server}
+        if parsed.username:
+            config["username"] = parsed.username
+        if parsed.password:
+            config["password"] = parsed.password
+        return config
+
+    def _on_request_success(self) -> None:
+        """Notify proxy pool of successful request."""
+        if self._current_proxy:
+            self._current_proxy.record_success()
+
+    def _on_request_failure(self) -> None:
+        """Notify proxy pool of failed request."""
+        if self._current_proxy:
+            self._current_proxy.record_failure(
+                max_fail_count=self.proxy_pool.config.max_fail_count
+                if self.proxy_pool else 3
+            )
+
    def _add_cookies(self, cookie_str: str) -> None:
        """Add cookies to browser context.