feat(08-02): implement crawl_comments method with hot sorting

- Add COMMENT_API_TEMPLATE for comment API URL
- Implement crawl_comments() returning CommentsResult
- Use order=hot for popular comments sorting
- Only fetch first-level comments (no nested replies)
- Add include_comments parameter to crawl_answer()
- Export CommentAuthor, CommentData, CommentsResult in __init__.py

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

kb_workflow/crawler/__init__.py

@@ -8,8 +8,17 @@ from .models import (
     ContentMetadata,
     AnswerData,
     ArticleData,
+    CommentAuthor,
+    CommentData,
+    CommentsResult,
     CrawlResult,
 )
+from .config import (
+    ProxyConfig,
+    CrawlerConfig,
+    get_crawler_config,
+    reset_crawler_config,
+)
 from .zhihu_crawler import ZhihuCrawler, ZhihuApiInterceptor
 
 __all__ = [
@@ -17,7 +26,14 @@ __all__ = [
     "ContentMetadata",
     "AnswerData",
     "ArticleData",
+    "CommentAuthor",
+    "CommentData",
+    "CommentsResult",
     "CrawlResult",
+    "ProxyConfig",
+    "CrawlerConfig",
+    "get_crawler_config",
+    "reset_crawler_config",
     "ZhihuCrawler",
     "ZhihuApiInterceptor",
 ]

kb_workflow/crawler/zhihu_crawler.py

@@ -17,6 +17,9 @@ from .models import (
     ArticleData,
     ContentMetadata,
     AuthorInfo,
+    CommentData,
+    CommentAuthor,
+    CommentsResult,
     CrawlResult,
 )
 
@@ -68,10 +71,16 @@ class ZhihuCrawler:
     Attributes:
         ANSWER_PATTERN: Regex pattern for answer URLs
         ARTICLE_PATTERN: Regex pattern for zhuanlan article URLs
+        COMMENT_API_TEMPLATE: Template for comment API URL
     """
 
     ANSWER_PATTERN = r"zhihu\.com/question/(\d+)/answer/(\d+)"
     ARTICLE_PATTERN = r"zhuanlan\.zhihu\.com/p/(\d+)"
+    COMMENT_API_TEMPLATE = (
+        "https://www.zhihu.com/api/v4/comment_v5/"
+        "{content_type}/{content_id}/comments"
+        "?order=hot&limit=20"
+    )
 
     def __init__(self, headless: bool = True, timeout: int = 30):
         """Initialize ZhihuCrawler.
@@ -174,14 +183,108 @@ class ZhihuCrawler:
             metadata=self._parse_metadata(data)
         )
 
-    def crawl_answer(self, url: str) -> CrawlResult:
+    def crawl_comments(
+        self,
+        content_type: str,
+        content_id: int,
+        include_comments: bool = True
+    ) -> CommentsResult:
+        """Crawl hot comments for a Zhihu content.
+
+        Fetches first-level comments using the hot sorting API.
+        Does not expand nested replies.
+
+        Args:
+            content_type: Content type ("answers" or "articles")
+            content_id: Content ID
+            include_comments: Whether to fetch comments (default True)
+
+        Returns:
+            CommentsResult with list of CommentData
+        """
+        if not include_comments:
+            return CommentsResult(
+                content_type=content_type,
+                content_id=content_id,
+                comments=[],
+                total_count=0
+            )
+
+        try:
+            self.browser = PlaywrightBrowser(
+                headless=self.headless,
+                timeout=self.timeout
+            )
+            self.browser.start(cookie=self._get_cookie())
+
+            # Build comment API URL
+            api_url = self.COMMENT_API_TEMPLATE.format(
+                content_type=content_type,
+                content_id=content_id
+            )
+
+            # Use Playwright's request API to directly call the API
+            response = self.browser.page.request.get(api_url)
+
+            if response.status != 200:
+                return CommentsResult(
+                    content_type=content_type,
+                    content_id=content_id,
+                    comments=[],
+                    total_count=0
+                )
+
+            data = response.json()
+            comments = []
+
+            for item in data.get("data", []):
+                author_data = item.get("author", {})
+                comment = CommentData(
+                    id=item.get("id", 0),
+                    content=item.get("content", ""),
+                    author=CommentAuthor(
+                        name=author_data.get("name", ""),
+                        url_token=author_data.get("url_token")
+                    ) if author_data else None,
+                    created_time=item.get("created_time"),
+                    like_count=item.get("like_count", 0),
+                    child_count=item.get("child_comment_count", 0),
+                    is_author=item.get("is_author", False)
+                )
+                comments.append(comment)
+
+            # Get total count from paging
+            total_count = data.get("paging", {}).get("totals", len(comments))
+
+            return CommentsResult(
+                content_type=content_type,
+                content_id=content_id,
+                comments=comments,
+                total_count=total_count
+            )
+
+        except Exception:
+            return CommentsResult(
+                content_type=content_type,
+                content_id=content_id,
+                comments=[],
+                total_count=0
+            )
+
+        finally:
+            if self.browser:
+                self.browser.close()
+
+    def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult:
         """Crawl Zhihu answer.
 
         Args:
             url: Zhihu answer URL (e.g., https://www.zhihu.com/question/123/answer/456)
+            include_comments: Whether to include hot comments (default False)
 
         Returns:
-            CrawlResult with AnswerData if successful, error message otherwise
+            CrawlResult with AnswerData if successful, error message otherwise.
+            If include_comments=True and successful, comments field will be populated.
         """
         match = re.search(self.ANSWER_PATTERN, url)
         if not match:
@@ -219,12 +322,27 @@ class ZhihuCrawler:
                 question_id=question_id
             )
 
-            return CrawlResult(
+            result = CrawlResult(
                 success=True,
                 content_type="answer",
                 data=result_data
             )
 
+            # Fetch comments if requested
+            if include_comments:
+                # Need to close current browser first as crawl_comments creates its own
+                self.browser.close()
+                self.browser = None
+
+                comments_result = self.crawl_comments(
+                    content_type="answers",
+                    content_id=answer_id,
+                    include_comments=True
+                )
+                result.comments = comments_result.comments
+
+            return result
+
         except Exception as e:
             return CrawlResult(success=False, error=str(e))
 

tests/test_crawler/test_comment_crawler.py

@@ -140,25 +140,17 @@ class TestCommentCrawler:
                 mock_browser = MockBrowser.return_value
                 # Setup for answer crawl
                 mock_browser.page.route = MagicMock()
+                mock_browser.page.request.get.return_value.status = 200
+                mock_browser.page.request.get.return_value.json.return_value = mock_comment_response
                 mock_browser.goto = MagicMock()
+                mock_browser.close = MagicMock()
 
-                # We need to simulate the interceptor capturing answer data
+                # Create interceptor that will capture answer data
-                # For simplicity, patch the internal method
+                with patch('kb_workflow.crawler.zhihu_crawler.ZhihuApiInterceptor') as MockInterceptor:
-                with patch.object(crawler, '_crawl_answer_internal') as mock_internal:
+                    mock_interceptor = MockInterceptor.return_value
-                    from kb_workflow.crawler.models import CrawlResult, AnswerData, ContentMetadata, AuthorInfo
+                    mock_interceptor.captured_data = {"answer": mock_answer_data}
-                    mock_internal.return_value = CrawlResult(
-                        success=True,
-                        content_type="answer",
-                        data=AnswerData(
-                            id=456,
-                            question_id=123,
-                            question_title="Test Question",
-                            content="<p>Content</p>",
-                            excerpt="Excerpt",
-                            metadata=ContentMetadata(author=AuthorInfo(name="Author"))
-                        )
-                    )
 
+                    # Mock crawl_comments to return test data
                     with patch.object(crawler, 'crawl_comments') as mock_comments:
                         mock_comments.return_value = CommentsResult(
                             content_type="answers",
@@ -167,6 +159,20 @@ class TestCommentCrawler:
                             total_count=1
                         )
 
-                        # This test verifies the integration point exists
+                        result = crawler.crawl_answer(
-                        # The actual implementation will call crawl_comments when include_comments=True
+                            "https://www.zhihu.com/question/123/answer/456",
-                        pass
+                            include_comments=True
+                        )
+
+                        # Verify crawl_comments was called with correct params
+                        mock_comments.assert_called_once_with(
+                            content_type="answers",
+                            content_id=456,
+                            include_comments=True
+                        )
+
+                        # Verify result includes comments
+                        assert result.success
+                        assert result.comments is not None
+                        assert len(result.comments) == 1
+                        assert result.comments[0].content == "Test comment"