feat(09-01): integrate ZhihuCrawler into ZhihuAdapter

- Replace ZhihuPlaywrightExtractor with ZhihuCrawler
- Add comment crawling with include_comments=True by default
- Implement cookie expiration detection and QR code login trigger
- Add idempotency check via IdempotencyChecker
- Add comment formatting with "## 热门评论" section
- Convert HTML content to Markdown using html2text

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

kb_workflow/pipeline/platforms/zhihu.py

@@ -3,26 +3,44 @@
 Provides content extraction for Zhihu platform:
 - Answer extraction from question/answer URLs
 - Zhuanlan (article) extraction from zhuanlan URLs
-- Cookie-based authentication
-- Login page detection for expired cookies
+- Cookie-based authentication with auto-login
+- Comment crawling and formatting
+- Idempotent execution
+
+Phase 09 - Pipeline Integration:
+- Uses ZhihuCrawler instead of ZhihuPlaywrightExtractor
+- Comments included by default and appended to content
+- Cookie expiration triggers QR code login
 """
 
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List
 from datetime import datetime
+from pathlib import Path
 import re
 import asyncio
+import html2text
 
 from kb_workflow.pipeline.platforms.base import PlatformAdapter
 from kb_workflow.pipeline.errors import AuthError
 from kb_workflow.pipeline.config import get_settings
-from kb_workflow.metadata.zhihu_extractor import ZhihuPlaywrightExtractor
+from kb_workflow.pipeline.idempotency import IdempotencyChecker
+from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
+from kb_workflow.crawler.models import (
+    CrawlResult,
+    AnswerData,
+    ArticleData,
+    CommentData,
+)
+from kb_workflow.auth.zhihu_login import ZhihuLogin, ZhihuLoginResult
 
 
 class ZhihuAdapter(PlatformAdapter):
     """Zhihu platform adapter for answer and zhuanlan content extraction.
 
-    Uses ZhihuPlaywrightExtractor (from kb-zhihu-metadata) for content extraction.
-    Requires KB_ZHIHU_COOKIE environment variable for authentication.
+    Uses ZhihuCrawler for content extraction with:
+    - Comment crawling enabled by default
+    - Cookie expiration auto-detection and QR code login
+    - Idempotent execution via IdempotencyChecker
 
     Example:
         adapter = ZhihuAdapter()
@@ -40,17 +58,30 @@ class ZhihuAdapter(PlatformAdapter):
     ZHUANLAN_PATTERN = re.compile(r'zhuanlan\.zhihu\.com/p/([\w\d]+)')
     QUESTION_PATTERN = re.compile(r'zhihu\.com/question/(\d+)(?!/answer)')
 
-    def __init__(self, headless: bool = True, timeout: int = 30):
+    def __init__(
+        self,
+        headless: bool = True,
+        timeout: int = 30,
+        output_dir: Optional[Path] = None,
+        idempotency_checker: Optional[IdempotencyChecker] = None
+    ):
         """Initialize ZhihuAdapter.
 
         Args:
             headless: Run browser in headless mode (default: True)
             timeout: Page load timeout in seconds (default: 30)
+            output_dir: Directory for output files (for idempotency checks)
+            idempotency_checker: Optional IdempotencyChecker instance
         """
         self.headless = headless
         self.timeout = timeout
         self._cookie: Optional[str] = None
-        self._extractor = None
+        self._crawler: Optional[ZhihuCrawler] = None
+        self._idempotency_checker = idempotency_checker or IdempotencyChecker()
+        self._output_dir = output_dir
+        self._html_to_text = html2text.HTML2Text()
+        self._html_to_text.ignore_links = False
+        self._html_to_text.body_width = 0
 
     @property
     def cookie(self) -> str:
@@ -80,66 +111,264 @@ class ZhihuAdapter(PlatformAdapter):
             return "zhuanlan"
         return "unknown"
 
-    def _get_extractor(self):
-        """Get or create ZhihuPlaywrightExtractor instance.
+    def _get_crawler(self) -> ZhihuCrawler:
+        """Get or create ZhihuCrawler instance.
 
         Returns:
-            ZhihuPlaywrightExtractor instance
+            ZhihuCrawler instance
         """
-        if self._extractor is None:
-            self._extractor = ZhihuPlaywrightExtractor(
+        if self._crawler is None:
+            self._crawler = ZhihuCrawler(
                 headless=self.headless,
-                timeout=self.timeout,
-                cookie=self.cookie
+                timeout=self.timeout
             )
-        return self._extractor
+        return self._crawler
 
-    def _extract_with_playwright(self, url: str) -> Optional[Dict[str, Any]]:
-        """Extract content using Playwright-based extractor.
+    def _crawl_with_crawler(
+        self,
+        url: str,
+        content_type: str,
+        include_comments: bool = True
+    ) -> CrawlResult:
+        """Crawl content using ZhihuCrawler.
 
         Args:
-            url: Full URL to extract content from
+            url: Full URL to crawl
+            content_type: Type of content ('answer' or 'zhuanlan')
+            include_comments: Whether to include comments (default: True)
 
         Returns:
-            Extraction result dictionary or None if extraction failed
+            CrawlResult from the crawler
         """
-        extractor = self._get_extractor()
-        return extractor.extract(url)
+        crawler = self._get_crawler()
 
-    def _detect_login_page(self, result: Dict[str, Any]) -> bool:
+        if content_type == "answer":
+            return crawler.crawl_answer(url, include_comments=include_comments)
+        elif content_type == "zhuanlan":
+            # For articles, we need to crawl comments separately
+            result = crawler.crawl_article(url)
+            if result.success and include_comments:
+                # Fetch comments for articles
+                from kb_workflow.crawler.models import ArticleData
+                if isinstance(result.data, ArticleData):
+                    comments_result = crawler.crawl_comments(
+                        content_type="articles",
+                        content_id=result.data.id,
+                        include_comments=True
+                    )
+                    result.comments = comments_result.comments
+            return result
+        else:
+            return CrawlResult(
+                success=False,
+                error=f"Unsupported content type: {content_type}"
+            )
+
+    def _is_cookie_expired(self, result: CrawlResult) -> bool:
+        """Check if the result indicates cookie expiration.
+
+        Args:
+            result: CrawlResult from crawler
+
+        Returns:
+            True if cookie appears to be expired
+        """
+        if not result.success:
+            error_msg = result.error or ""
+            expired_indicators = [
+                "cookie expired",
+                "unauthorized",
+                "forbidden",
+                "需要登录",
+                "请先登录",
+                "登录失效",
+            ]
+            return any(indicator in error_msg.lower() for indicator in expired_indicators)
+        return False
+
+    def _detect_login_page(self, result: CrawlResult) -> bool:
         """Detect if the result indicates a login page redirect.
 
         This happens when the cookie has expired or is invalid.
 
         Args:
-            result: Extraction result dictionary
+            result: CrawlResult from crawler
 
         Returns:
             True if login page is detected, False otherwise
         """
-        if not result:
-            return False
-
-        title = result.get("title", "")
-        content = result.get("content", "")
-        content_markdown = result.get("content_markdown", "")
-
-        # Check for login-related indicators
-        login_indicators = ["登录", "登陆", "sign in", "login"]
-        title_lower = title.lower()
-
-        # If title contains login keywords
-        if any(indicator in title_lower for indicator in login_indicators):
-            return True
-
-        # If content is suspiciously short (login page has minimal content)
-        total_content = content + content_markdown
-        if len(total_content.strip()) < 100:
-            return True
-
+        if not result.success:
+            error_msg = result.error or ""
+            login_indicators = ["登录", "登陆", "sign in", "login"]
+            return any(indicator in error_msg.lower() for indicator in login_indicators)
         return False
 
-    async def fetch_metadata(self, content_id: str, content_type: str, source_url: str = None) -> Dict[str, Any]:
+    def _trigger_qrcode_login(self) -> ZhihuLoginResult:
+        """Trigger QR code login flow.
+
+        Returns:
+            ZhihuLoginResult with success status and cookie
+        """
+        # Use intermediate path from settings or default
+        settings = get_settings()
+        intermediate_path = getattr(settings, 'intermediate_path', '/tmp')
+
+        login = ZhihuLogin(
+            intermediate_path=str(intermediate_path),
+            headless=False,  # Show browser for QR code display
+            timeout=120  # 2 minutes to scan
+        )
+        return login.login_by_qrcode()
+
+    def _convert_crawl_result_to_metadata(
+        self,
+        result: CrawlResult,
+        content_type: str,
+        source_url: str
+    ) -> Dict[str, Any]:
+        """Convert CrawlResult to metadata dictionary.
+
+        Args:
+            result: CrawlResult from crawler
+            content_type: Type of content
+            source_url: Original source URL
+
+        Returns:
+            Metadata dictionary
+        """
+        data = result.data
+
+        if isinstance(data, AnswerData):
+            return {
+                "type": "answer",
+                "title": data.question_title,
+                "author": data.metadata.author.name if data.metadata.author else "",
+                "author_url_token": data.metadata.author.url_token if data.metadata.author else None,
+                "content": data.content,
+                "content_markdown": self._html_to_text.handle(data.content),
+                "excerpt": data.excerpt,
+                "question_id": data.question_id,
+                "answer_id": data.id,
+                "voteup_count": data.metadata.voteup_count,
+                "comment_count": data.metadata.comment_count,
+                "created_time": data.metadata.created_time,
+                "updated_time": data.metadata.updated_time,
+                "url": source_url,
+                "has_comments": bool(result.comments),
+                "comment_count_actual": len(result.comments) if result.comments else 0,
+            }
+        elif isinstance(data, ArticleData):
+            return {
+                "type": "zhuanlan",
+                "title": data.title,
+                "author": data.metadata.author.name if data.metadata.author else "",
+                "author_url_token": data.metadata.author.url_token if data.metadata.author else None,
+                "content": data.content,
+                "content_markdown": self._html_to_text.handle(data.content),
+                "excerpt": data.excerpt,
+                "article_id": data.id,
+                "voteup_count": data.metadata.voteup_count,
+                "comment_count": data.metadata.comment_count,
+                "created_time": data.metadata.created_time,
+                "updated_time": data.metadata.updated_time,
+                "url": source_url,
+                "has_comments": bool(result.comments),
+                "comment_count_actual": len(result.comments) if result.comments else 0,
+            }
+        else:
+            return {
+                "type": content_type,
+                "title": "",
+                "author": "",
+                "content": "",
+                "content_markdown": "",
+                "url": source_url,
+                "error": "Unknown data type",
+            }
+
+    def _format_comments_section(self, comments: List[CommentData]) -> str:
+        """Format comments as Markdown.
+
+        Args:
+            comments: List of CommentData objects
+
+        Returns:
+            Markdown-formatted comment section
+        """
+        if not comments:
+            return ""
+
+        lines = []
+        for i, comment in enumerate(comments, 1):
+            author_name = comment.author.name if comment.author else "Anonymous"
+            like_str = f"({comment.like_count} likes)" if comment.like_count > 0 else ""
+            lines.append(f"{i}. **@{author_name}** {like_str}: {comment.content}")
+
+        return "\n".join(lines)
+
+    def _format_with_frontmatter(
+        self,
+        metadata: Dict[str, Any],
+        content: str,
+        content_type: str,
+        comments: Optional[List[CommentData]] = None
+    ) -> str:
+        """Format content with YAML frontmatter.
+
+        Args:
+            metadata: Content metadata dictionary
+            content: Markdown content
+            content_type: 'answer' or 'zhuanlan'
+            comments: Optional list of comments to append
+
+        Returns:
+            Formatted string with YAML frontmatter and content
+        """
+        frontmatter_lines = [
+            "---",
+            f"title: {metadata.get('title', '')}",
+            f"author: {metadata.get('author', '')}",
+            f"source_url: {metadata.get('url', '')}",
+            f"platform: {self.PLATFORM}",
+            f"content_type: {content_type}",
+            f"fetched_at: {datetime.now().isoformat()}",
+            "---",
+            "",
+        ]
+
+        result = "\n".join(frontmatter_lines) + content
+
+        # Append comments section if present
+        if comments:
+            comments_md = self._format_comments_section(comments)
+            if comments_md:
+                result += "\n\n---\n\n## 热门评论\n\n" + comments_md
+
+        return result
+
+    def _load_cached_metadata(self, metadata_file: Path) -> Optional[Dict[str, Any]]:
+        """Load cached metadata from file.
+
+        Args:
+            metadata_file: Path to metadata JSON file
+
+        Returns:
+            Metadata dictionary or None if file doesn't exist or is invalid
+        """
+        import json
+        try:
+            if metadata_file.exists():
+                return json.loads(metadata_file.read_text())
+        except (json.JSONDecodeError, IOError):
+            pass
+        return None
+
+    async def fetch_metadata(
+        self,
+        content_id: str,
+        content_type: str,
+        source_url: str = None
+    ) -> Dict[str, Any]:
         """Fetch metadata for the content.
 
         Args:
@@ -161,42 +390,75 @@ class ZhihuAdapter(PlatformAdapter):
                 auth_type="zhihu_cookie",
             )
 
+        # Idempotency check: if output_dir is set and content already processed, return cached result
+        if self._output_dir:
+            exists, existing_files = self._idempotency_checker.check_exists(
+                self._output_dir, content_id, self.PLATFORM
+            )
+            if exists:
+                # Load and return cached metadata
+                metadata_file = existing_files[0] if existing_files else None
+                if metadata_file and metadata_file.suffix == ".json":
+                    cached = self._load_cached_metadata(metadata_file)
+                    if cached:
+                        return cached
+
         # Build URL based on content type
         if content_type == "answer":
-            # For answers, try to use source_url first, then fallback to constructed URL
-            if source_url:
-                url = source_url
-            else:
-                # Fallback: use answer ID only (may not work for all cases)
-                url = f"https://www.zhihu.com/question/0/answer/{content_id}"
+            url = source_url or f"https://www.zhihu.com/question/0/answer/{content_id}"
         elif content_type == "zhuanlan":
             url = f"https://zhuanlan.zhihu.com/p/{content_id}"
         else:
             raise ValueError(f"Unsupported content type: {content_type}")
 
-        # Extract content in a separate thread to avoid blocking asyncio
-        result = await asyncio.to_thread(self._extract_with_playwright, url)
+        # Crawl content in a separate thread to avoid blocking asyncio
+        result = await asyncio.to_thread(
+            self._crawl_with_crawler,
+            url,
+            content_type,
+            True  # include_comments=True by default
+        )
 
-        # Check for login page
-        if self._detect_login_page(result):
+        # Check for login page / expired cookie
+        if self._detect_login_page(result) or self._is_cookie_expired(result):
+            # Try QR code login
+            login_result = self._trigger_qrcode_login()
+
+            if login_result.success:
+                # Update cookie and retry
+                self._cookie = login_result.cookie_str
+                # Retry crawl with new cookie
+                result = await asyncio.to_thread(
+                    self._crawl_with_crawler,
+                    url,
+                    content_type,
+                    True
+                )
+            else:
+                raise AuthError(
+                    step_name="zhihu_metadata",
+                    input_summary=f"content_id={content_id}, type={content_type}",
+                    auth_type="zhihu_cookie",
+                    cause=Exception(f"Login failed: {login_result.error_message}")
+                )
+
+        if not result.success:
             raise AuthError(
                 step_name="zhihu_metadata",
                 input_summary=f"content_id={content_id}, type={content_type}",
                 auth_type="zhihu_cookie",
-                cause=Exception("Cookie may have expired, please update KB_ZHIHU_COOKIE")
+                cause=Exception(result.error or "Failed to extract content")
             )
 
-        if not result:
-            raise AuthError(
-                step_name="zhihu_metadata",
-                input_summary=f"content_id={content_id}, type={content_type}",
-                auth_type="zhihu_cookie",
-                cause=Exception("Failed to extract content, check if content exists")
-            )
+        # Convert to metadata dictionary
+        return self._convert_crawl_result_to_metadata(result, content_type, url)
 
-        return result
-
-    async def fetch_content(self, content_id: str, content_type: str, source_url: str = None) -> str:
+    async def fetch_content(
+        self,
+        content_id: str,
+        content_type: str,
+        source_url: str = None
+    ) -> str:
         """Fetch content in Markdown format.
 
         Args:
@@ -205,49 +467,67 @@ class ZhihuAdapter(PlatformAdapter):
             source_url: Original source URL (optional, used for answers)
 
         Returns:
-            Markdown-formatted content with YAML frontmatter
+            Markdown-formatted content with YAML frontmatter and comments
 
         Raises:
             AuthError: If cookie is missing or expired
         """
-        # Get metadata (which also validates auth)
-        metadata = await self.fetch_metadata(content_id, content_type, source_url=source_url)
+        # Build URL for crawling
+        if content_type == "answer":
+            url = source_url or f"https://www.zhihu.com/question/0/answer/{content_id}"
+        elif content_type == "zhuanlan":
+            url = f"https://zhuanlan.zhihu.com/p/{content_id}"
+        else:
+            raise ValueError(f"Unsupported content type: {content_type}")
+
+        # Crawl content with comments
+        result = await asyncio.to_thread(
+            self._crawl_with_crawler,
+            url,
+            content_type,
+            True
+        )
+
+        # Handle cookie expiration
+        if self._detect_login_page(result) or self._is_cookie_expired(result):
+            login_result = self._trigger_qrcode_login()
+            if login_result.success:
+                self._cookie = login_result.cookie_str
+                result = await asyncio.to_thread(
+                    self._crawl_with_crawler,
+                    url,
+                    content_type,
+                    True
+                )
+            else:
+                raise AuthError(
+                    step_name="zhihu_content",
+                    input_summary=f"content_id={content_id}, type={content_type}",
+                    auth_type="zhihu_cookie",
+                    cause=Exception(f"Login failed: {login_result.error_message}")
+                )
+
+        if not result.success:
+            raise AuthError(
+                step_name="zhihu_content",
+                input_summary=f"content_id={content_id}, type={content_type}",
+                auth_type="zhihu_cookie",
+                cause=Exception(result.error or "Failed to extract content")
+            )
+
+        # Convert to metadata
+        metadata = self._convert_crawl_result_to_metadata(result, content_type, url)
 
         # Get content markdown
         content_markdown = metadata.get("content_markdown", "") or metadata.get("content", "")
 
-        # Format with YAML frontmatter
-        return self._format_with_frontmatter(metadata, content_markdown, content_type)
-
-    def _format_with_frontmatter(
-        self,
-        metadata: Dict[str, Any],
-        content: str,
-        content_type: str
-    ) -> str:
-        """Format content with YAML frontmatter.
-
-        Args:
-            metadata: Content metadata dictionary
-            content: Markdown content
-            content_type: 'answer' or 'zhuanlan'
-
-        Returns:
-            Formatted string with YAML frontmatter and content
-        """
-        frontmatter_lines = [
-            "---",
-            f"title: {metadata.get('title', '')}",
-            f"author: {metadata.get('author', '')}",
-            f"source_url: {metadata.get('url', '')}",
-            f"platform: {self.PLATFORM}",
-            f"content_type: {content_type}",
-            f"fetched_at: {datetime.now().isoformat()}",
-            "---",
-            "",
-        ]
-
-        return "\n".join(frontmatter_lines) + content
+        # Format with YAML frontmatter and comments
+        return self._format_with_frontmatter(
+            metadata,
+            content_markdown,
+            content_type,
+            result.comments
+        )
 
     def get_output_filename(self, content_id: str, content_type: str, suffix: str) -> str:
         """Generate output filename for the content.