kb-skills/kb_workflow/crawler/zhihu_crawler.py

"""Zhihu Content Crawler

Implements content crawling using Playwright API interception.
Captures structured JSON data from Zhihu API responses.

Supports:
- Mandatory proxy configuration
- Random request delay
- Retry with exponential backoff
- Automatic x-zse-96 signature handling via browser
- ProxyPool integration for multi-proxy rotation
"""

import random
import re
import time
from typing import Optional, Dict, Any
from urllib.parse import urlparse

from playwright.sync_api import Route, Response, sync_playwright
from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError

from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser
# NOTE: get_settings is imported lazily in _get_cookie() to avoid circular import
# Circular: zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
from .models import (
    AnswerData,
    ArticleData,
    ContentMetadata,
    AuthorInfo,
    CommentData,
    CommentAuthor,
    CommentsResult,
    CrawlResult,
)
from .config import CrawlerConfig, get_crawler_config
from .proxy_pool import ProxyPool, ProxyEntry
from .stealth import apply_stealth_sync, StealthConfig


class ZhihuApiInterceptor:
    """API Interceptor for capturing Zhihu API responses.

    Intercepts network requests and captures JSON responses from
        Zhihu API endpoints (/api/v4/answers/, /api/v4/articles/, etc).
        The x-zse-96 signature is automatically handled by the browser via route.fetch().

    Attributes:
        captured_data: Dictionary storing captured API response data
    """

    def __init__(self):
        self.captured_data: Dict[str, Any] = {}

    def handle_route(self, route: Route) -> None:
        """Handle intercepted route/request.

        The route.fetch() call automatically includes browser-generated
        x-zse-96 signature headers, No manual signature handling needed.

        Args:
            route: Playwright Route object for the intercepted request
        """
        url = route.request.url

        # Intercept answer API
        if "/api/v4/answers/" in url:
            response = route.fetch()
            self.captured_data["answer"] = response.json()
            route.fulfill(response=response)
            return

        # Intercept article API (zhuanlan)
        if "/api/v4/articles/" in url or "zhuanlan" in url:
            response = route.fetch()
            self.captured_data["article"] = response.json()
            route.fulfill(response=response)
            return

        route.continue_()


class ZhihuCrawler:
    """Zhihu Content Crawler.

    Crawls Zhihu answers and articles using Playwright API interception.
    Includes anti-scraping measures:
    - Mandatory proxy configuration (configurable via proxy_required)
    - Random delay between requests (1-3 seconds by default)
    - Retry with exponential backoff (3 attempts by default)
    - Automatic x-zse-96 signature handling via browser

    Attributes:
        ANSWER_PATTERN: Regex pattern for answer URLs
        ARTICLE_PATTERN: Regex pattern for zhuanlan article URLs
        COMMENT_API_TEMPLATE: Template for comment API URL
    """

    ANSWER_PATTERN = r"zhihu\.com/question/(\d+)/answer/(\d+)"
    ARTICLE_PATTERN = r"zhuanlan\.zhihu\.com/p/(\d+)"
    COMMENT_API_TEMPLATE = (
        "https://www.zhihu.com/api/v4/comment_v5/"
        "{content_type}/{content_id}/comments"
        "?order=hot&limit=20"
    )

    def __init__(
        self,
        headless: bool = True,
        timeout: int = 30,
        config: Optional[CrawlerConfig] = None,
        proxy_pool: Optional[ProxyPool] = None,
        stealth_config: Optional[StealthConfig] = None
    ):
        """Initialize ZhihuCrawler.

        Args:
            headless: Whether to run browser in headless mode
            timeout: Request timeout in seconds
            config: Optional CrawlerConfig instance (uses environment if not provided)
            proxy_pool: Optional ProxyPool instance for multi-proxy rotation
            stealth_config: Optional StealthConfig instance (uses defaults if not provided)
        """
        self.headless = headless
        self.timeout = timeout
        self.config = config or get_crawler_config()
        self.proxy_pool = proxy_pool
        self._stealth_config = stealth_config
        self._current_proxy: Optional[ProxyEntry] = None
        self.browser: Optional[PlaywrightBrowser] = None
        self.interceptor: Optional[ZhihuApiInterceptor] = None
        # Internal browser management for proxy support
        self._playwright = None
        self._browser = None
        self._context = None
        self._page = None

    def _get_cookie(self) -> str:
        """Get Cookie from configuration.

        Returns:
            Cookie string from settings
        """
        # Lazy import to avoid circular dependency:
        # zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
        from kb_workflow.pipeline.config import get_settings
        settings = get_settings()
        return settings.zhihu_cookie.get_secret_value()

    def _apply_random_delay(self) -> None:
        """Apply random delay between requests.

        Uses min_delay and max_delay from configuration.
        """
        delay = random.uniform(self.config.min_delay, self.config.max_delay)
        time.sleep(delay)

    def _start_browser_with_proxy(self, cookie: str) -> None:
        """Start browser with proxy configuration.

        Validates proxy if required and applies to browser context.
        Applies stealth fingerprint masking after context creation.
        Prioritizes proxy_pool over config.proxy for multi-proxy support.

        Args:
            cookie: Cookie string for authentication

        Raises:
            ValueError: If proxy_required=True but no proxy is configured
        """
        # Validate proxy configuration
        self.config.validate_proxy()

        # Build launch args for stealth (add --disable-blink-features)
        launch_args = []
        if self._stealth_config is None or self._stealth_config.stealth_enabled:
            launch_args.append('--disable-blink-features=AutomationControlled')

        self._playwright = sync_playwright().start()
        self._browser = self._playwright.chromium.launch(
            headless=self.headless,
            args=launch_args if launch_args else None
        )

        # Build context options
        context_options = {
            'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }

        # Prioritize proxy pool over single proxy config
        if self.proxy_pool and self.proxy_pool.has_proxies():
            proxy_entry = self.proxy_pool.get_next_proxy()
            if proxy_entry:
                proxy_config = self._parse_proxy_url(proxy_entry.url)
                context_options["proxy"] = proxy_config
                self._current_proxy = proxy_entry
        elif self.config.proxy:
            # Backward compatibility: use single proxy configuration
            context_options["proxy"] = self.config.proxy.to_playwright_format()

        # Block Service Worker to prevent interference with API interception
        context_options["service_workers"] = "block"

        self._context = self._browser.new_context(**context_options)

        # Apply stealth fingerprint masking
        apply_stealth_sync(self._context, self._stealth_config)

        # Add cookies if provided
        if cookie:
            self._add_cookies(cookie)

        self._page = self._context.new_page()
        self._page.set_default_timeout(self.timeout * 1000)

    @staticmethod
    def _parse_proxy_url(url: str) -> Dict[str, Any]:
        """Parse proxy URL into Playwright format.

        Args:
            url: Proxy URL (e.g., http://user:pass@proxy.example.com:8080)

        Returns:
            Dictionary with server, username, password for Playwright
        """
        parsed = urlparse(url)
        server = f"{parsed.scheme}://{parsed.hostname}"
        if parsed.port:
            server += f":{parsed.port}"
        config: Dict[str, Any] = {"server": server}
        if parsed.username:
            config["username"] = parsed.username
        if parsed.password:
            config["password"] = parsed.password
        return config

    def _on_request_success(self) -> None:
        """Notify proxy pool of successful request."""
        if self._current_proxy:
            self._current_proxy.record_success()

    def _on_request_failure(self) -> None:
        """Notify proxy pool of failed request."""
        if self._current_proxy:
            self._current_proxy.record_failure(
                max_fail_count=self.proxy_pool.config.max_fail_count
                if self.proxy_pool else 3
            )

    def _add_cookies(self, cookie_str: str) -> None:
        """Add cookies to browser context.

        Args:
            cookie_str: Cookie string in "name=value; name2=value2" format
        """
        cookies = []
        for item in cookie_str.split(';'):
            item = item.strip()
            if '=' in item:
                name, value = item.split('=', 1)
                cookies.append({
                    'name': name.strip(),
                    'value': value.strip(),
                    'domain': '.zhihu.com',
                    'path': '/',
                    'secure': True,
                    'httpOnly': False,
                    'sameSite': 'Lax'
                })
        self._context.add_cookies(cookies)

    def _close_browser(self) -> None:
        """Close browser and cleanup resources."""
        if self._page:
            self._page.close()
        if self._context:
            self._context.close()
        if self._browser:
            self._browser.close()
        if self._playwright:
            self._playwright.stop()

    def _crawl_with_retry(self, crawl_func, *args, **kwargs) -> CrawlResult:
        """Execute crawl with retry logic.

        Applies random delay before each attempt and retries on failure
        with exponential backoff.

        Args:
            crawl_func: Crawl function to execute
            *args: Positional arguments for crawl_func
            **kwargs: Keyword arguments for crawl_func

        Returns:
            CrawlResult from crawl_func or error result after retries exhausted
        """
        try:
            for attempt in Retrying(
                stop=stop_after_attempt(self.config.max_retries),
                wait=wait_exponential(
                    multiplier=1,
                    min=self.config.retry_min_wait,
                    max=self.config.retry_max_wait
                ),
                reraise=True
            ):
                with attempt:
                    # Apply random delay before each attempt
                    self._apply_random_delay()
                    return crawl_func(*args, **kwargs)
        except RetryError as e:
            return CrawlResult(
                success=False,
                error=f"Failed after {self.config.max_retries} retries: {str(e)}"
            )
        except Exception as e:
            return CrawlResult(success=False, error=str(e))

    def _parse_author(self, data: Optional[Dict]) -> Optional[AuthorInfo]:
        """Parse author information from API response.

        Args:
            data: Author data dictionary from API response

        Returns:
            AuthorInfo object or None if data is empty/None
        """
        if data is None:
            return None

        # Empty dict should return AuthorInfo with empty name
        return AuthorInfo(
            name=data.get("name", ""),
            url_token=data.get("url_token"),
            headline=data.get("headline")
        )

    def _parse_metadata(self, data: Dict) -> ContentMetadata:
        """Parse content metadata from API response.

        Args:
            data: API response data dictionary

        Returns:
            ContentMetadata object
        """
        return ContentMetadata(
            author=self._parse_author(data.get("author")),
            voteup_count=data.get("voteup_count", 0),
            comment_count=data.get("comment_count", 0),
            created_time=data.get("created_time"),
            updated_time=data.get("updated_time")
        )

    def _parse_answer_data(
        self,
        data: Dict,
        answer_id: int,
        question_id: int
    ) -> AnswerData:
        """Parse answer data from API response.

        Args:
            data: API response data dictionary
            answer_id: Answer ID
            question_id: Question ID

        Returns:
            AnswerData object
        """
        question = data.get("question", {})
        return AnswerData(
            id=answer_id,
            question_id=question_id,
            question_title=question.get("title", ""),
            content=data.get("content", ""),
            excerpt=data.get("excerpt", ""),
            metadata=self._parse_metadata(data)
        )

    def _parse_article_data(self, data: Dict, article_id: int) -> ArticleData:
        """Parse article data from API response.

        Args:
            data: API response data dictionary
            article_id: Article ID

        Returns:
            ArticleData object
        """
        return ArticleData(
            id=article_id,
            title=data.get("title", ""),
            content=data.get("content", ""),
            excerpt=data.get("excerpt", ""),
            metadata=self._parse_metadata(data)
        )

    def crawl_comments(
        self,
        content_type: str,
        content_id: int,
        include_comments: bool = True
    ) -> CommentsResult:
        """Crawl hot comments for a Zhihu content.

        Fetches first-level comments using the hot sorting API.
        Does not expand nested replies.

        Args:
            content_type: Content type ("answers" or "articles")
            content_id: Content ID
            include_comments: Whether to fetch comments (default True)

        Returns:
            CommentsResult with list of CommentData
        """
        if not include_comments:
            return CommentsResult(
                content_type=content_type,
                content_id=content_id,
                comments=[],
                total_count=0
            )

        try:
            self._start_browser_with_proxy(self._get_cookie())

            # Build comment API URL
            api_url = self.COMMENT_API_TEMPLATE.format(
                content_type=content_type,
                content_id=content_id
            )

            # Use Playwright's request API to directly call the API
            response = self._page.request.get(api_url)

            if response.status != 200:
                return CommentsResult(
                    content_type=content_type,
                    content_id=content_id,
                    comments=[],
                    total_count=0
                )

            data = response.json()
            comments = []

            for item in data.get("data", []):
                author_data = item.get("author", {})
                comment = CommentData(
                    id=item.get("id", 0),
                    content=item.get("content", ""),
                    author=CommentAuthor(
                        name=author_data.get("name", ""),
                        url_token=author_data.get("url_token")
                    ) if author_data else None,
                    created_time=item.get("created_time"),
                    like_count=item.get("like_count", 0),
                    child_count=item.get("child_comment_count", 0),
                    is_author=item.get("is_author", False)
                )
                comments.append(comment)

            # Get total count from paging
            total_count = data.get("paging", {}).get("totals", len(comments))

            return CommentsResult(
                content_type=content_type,
                content_id=content_id,
                comments=comments,
                total_count=total_count
            )

        except Exception:
            return CommentsResult(
                content_type=content_type,
                content_id=content_id,
                comments=[],
                total_count=0
            )

        finally:
            self._close_browser()

    def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult:
        """Crawl Zhihu answer with retry support.

        Args:
            url: Zhihu answer URL (e.g., https://www.zhihu.com/question/123/answer/456)
            include_comments: Whether to include hot comments (default False)

        Returns:
            CrawlResult with AnswerData if successful, error message otherwise.
            If include_comments=True and successful, comments field will be populated.
        """
        return self._crawl_with_retry(
            self._crawl_answer_impl, url, include_comments
        )

    def _crawl_answer_impl(self, url: str, include_comments: bool = False) -> CrawlResult:
        """Implementation of crawl_answer without retry wrapper.

        Args:
            url: Zhihu answer URL
            include_comments: Whether to include hot comments

        Returns:
            CrawlResult
        """
        match = re.search(self.ANSWER_PATTERN, url)
        if not match:
            return CrawlResult(success=False, error="Invalid answer URL format")

        question_id, answer_id = int(match.group(1)), int(match.group(2))

        try:
            self._start_browser_with_proxy(self._get_cookie())
            self.interceptor = ZhihuApiInterceptor()

            # Set up route interception - x-zse-96 handled by browser
            self._page.route("**/api/v4/**", self.interceptor.handle_route)

            # Visit page to trigger API requests
            self._page.goto(url, wait_until="networkidle")

            # Wait for data capture
            time.sleep(2)

            if "answer" not in self.interceptor.captured_data:
                return CrawlResult(
                    success=False,
                    error="Failed to capture answer data from API"
                )

            answer_data = self.interceptor.captured_data["answer"]
            result_data = self._parse_answer_data(
                answer_data,
                answer_id=answer_id,
                question_id=question_id
            )

            result = CrawlResult(
                success=True,
                content_type="answer",
                data=result_data
            )

            # Fetch comments if requested
            if include_comments:
                comments_result = self.crawl_comments(
                    content_type="answers",
                    content_id=answer_id,
                    include_comments=True
                )
                result.comments = comments_result.comments

            return result

        except Exception as e:
            return CrawlResult(success=False, error=str(e))

        finally:
            self._close_browser()

    def crawl_article(self, url: str) -> CrawlResult:
        """Crawl Zhihu zhuanlan article with retry support.

        Args:
            url: Zhihu zhuanlan URL (e.g., https://zhuanlan.zhihu.com/p/123)

        Returns:
            CrawlResult with ArticleData if successful, error message otherwise
        """
        return self._crawl_with_retry(self._crawl_article_impl, url)

    def _crawl_article_impl(self, url: str) -> CrawlResult:
        """Implementation of crawl_article without retry wrapper.

        Args:
            url: Zhihu zhuanlan URL

        Returns:
            CrawlResult
        """
        match = re.search(self.ARTICLE_PATTERN, url)
        if not match:
            return CrawlResult(success=False, error="Invalid article URL format")

        article_id = int(match.group(1))

        try:
            self._start_browser_with_proxy(self._get_cookie())
            self.interceptor = ZhihuApiInterceptor()

            self._page.route("**/api/v4/**", self.interceptor.handle_route)

            self._page.goto(url, wait_until="networkidle")

            time.sleep(2)

            if "article" not in self.interceptor.captured_data:
                # Fallback: DOM parsing
                return self._crawl_article_from_dom(url, article_id)

            article_data = self.interceptor.captured_data["article"]
            result_data = self._parse_article_data(article_data, article_id)

            return CrawlResult(
                success=True,
                content_type="article",
                data=result_data
            )

        except Exception as e:
            return CrawlResult(success=False, error=str(e))

        finally:
            self._close_browser()

    def _crawl_article_from_dom(self, url: str, article_id: int) -> CrawlResult:
        """Crawl article using DOM parsing as fallback.

        Used when API interception fails for zhuanlan articles.

        Args:
            url: Article URL
            article_id: Article ID

        Returns:
            CrawlResult with ArticleData from DOM extraction
        """
        from kb_workflow.metadata.zhihu_extractor import ZhihuPlaywrightExtractor

        extractor = ZhihuPlaywrightExtractor(
            headless=self.headless,
            timeout=self.timeout,
            cookie=self._get_cookie()
        )
        result = extractor.extract(url)

        if not result:
            return CrawlResult(
                success=False,
                error="Failed to extract article from DOM"
            )

        result_data = ArticleData(
            id=article_id,
            title=result.get("title", ""),
            content=result.get("content", ""),
            excerpt=result.get("content_markdown", "")[:200],
            metadata=ContentMetadata(
                author=AuthorInfo(name=result.get("author", "")),
                voteup_count=result.get("vote_up", 0)
            )
        )

        return CrawlResult(success=True, content_type="article", data=result_data)