diff --git a/kb_workflow/crawler/__init__.py b/kb_workflow/crawler/__init__.py index cc4b69a..8389733 100644 --- a/kb_workflow/crawler/__init__.py +++ b/kb_workflow/crawler/__init__.py @@ -8,8 +8,17 @@ from .models import ( ContentMetadata, AnswerData, ArticleData, + CommentAuthor, + CommentData, + CommentsResult, CrawlResult, ) +from .config import ( + ProxyConfig, + CrawlerConfig, + get_crawler_config, + reset_crawler_config, +) from .zhihu_crawler import ZhihuCrawler, ZhihuApiInterceptor __all__ = [ @@ -17,7 +26,14 @@ __all__ = [ "ContentMetadata", "AnswerData", "ArticleData", + "CommentAuthor", + "CommentData", + "CommentsResult", "CrawlResult", + "ProxyConfig", + "CrawlerConfig", + "get_crawler_config", + "reset_crawler_config", "ZhihuCrawler", "ZhihuApiInterceptor", ] diff --git a/kb_workflow/crawler/zhihu_crawler.py b/kb_workflow/crawler/zhihu_crawler.py index f7c9643..c928140 100644 --- a/kb_workflow/crawler/zhihu_crawler.py +++ b/kb_workflow/crawler/zhihu_crawler.py @@ -17,6 +17,9 @@ from .models import ( ArticleData, ContentMetadata, AuthorInfo, + CommentData, + CommentAuthor, + CommentsResult, CrawlResult, ) @@ -68,10 +71,16 @@ class ZhihuCrawler: Attributes: ANSWER_PATTERN: Regex pattern for answer URLs ARTICLE_PATTERN: Regex pattern for zhuanlan article URLs + COMMENT_API_TEMPLATE: Template for comment API URL """ ANSWER_PATTERN = r"zhihu\.com/question/(\d+)/answer/(\d+)" ARTICLE_PATTERN = r"zhuanlan\.zhihu\.com/p/(\d+)" + COMMENT_API_TEMPLATE = ( + "https://www.zhihu.com/api/v4/comment_v5/" + "{content_type}/{content_id}/comments" + "?order=hot&limit=20" + ) def __init__(self, headless: bool = True, timeout: int = 30): """Initialize ZhihuCrawler. @@ -174,14 +183,108 @@ class ZhihuCrawler: metadata=self._parse_metadata(data) ) - def crawl_answer(self, url: str) -> CrawlResult: + def crawl_comments( + self, + content_type: str, + content_id: int, + include_comments: bool = True + ) -> CommentsResult: + """Crawl hot comments for a Zhihu content. + + Fetches first-level comments using the hot sorting API. + Does not expand nested replies. + + Args: + content_type: Content type ("answers" or "articles") + content_id: Content ID + include_comments: Whether to fetch comments (default True) + + Returns: + CommentsResult with list of CommentData + """ + if not include_comments: + return CommentsResult( + content_type=content_type, + content_id=content_id, + comments=[], + total_count=0 + ) + + try: + self.browser = PlaywrightBrowser( + headless=self.headless, + timeout=self.timeout + ) + self.browser.start(cookie=self._get_cookie()) + + # Build comment API URL + api_url = self.COMMENT_API_TEMPLATE.format( + content_type=content_type, + content_id=content_id + ) + + # Use Playwright's request API to directly call the API + response = self.browser.page.request.get(api_url) + + if response.status != 200: + return CommentsResult( + content_type=content_type, + content_id=content_id, + comments=[], + total_count=0 + ) + + data = response.json() + comments = [] + + for item in data.get("data", []): + author_data = item.get("author", {}) + comment = CommentData( + id=item.get("id", 0), + content=item.get("content", ""), + author=CommentAuthor( + name=author_data.get("name", ""), + url_token=author_data.get("url_token") + ) if author_data else None, + created_time=item.get("created_time"), + like_count=item.get("like_count", 0), + child_count=item.get("child_comment_count", 0), + is_author=item.get("is_author", False) + ) + comments.append(comment) + + # Get total count from paging + total_count = data.get("paging", {}).get("totals", len(comments)) + + return CommentsResult( + content_type=content_type, + content_id=content_id, + comments=comments, + total_count=total_count + ) + + except Exception: + return CommentsResult( + content_type=content_type, + content_id=content_id, + comments=[], + total_count=0 + ) + + finally: + if self.browser: + self.browser.close() + + def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult: """Crawl Zhihu answer. Args: url: Zhihu answer URL (e.g., https://www.zhihu.com/question/123/answer/456) + include_comments: Whether to include hot comments (default False) Returns: - CrawlResult with AnswerData if successful, error message otherwise + CrawlResult with AnswerData if successful, error message otherwise. + If include_comments=True and successful, comments field will be populated. """ match = re.search(self.ANSWER_PATTERN, url) if not match: @@ -219,12 +322,27 @@ class ZhihuCrawler: question_id=question_id ) - return CrawlResult( + result = CrawlResult( success=True, content_type="answer", data=result_data ) + # Fetch comments if requested + if include_comments: + # Need to close current browser first as crawl_comments creates its own + self.browser.close() + self.browser = None + + comments_result = self.crawl_comments( + content_type="answers", + content_id=answer_id, + include_comments=True + ) + result.comments = comments_result.comments + + return result + except Exception as e: return CrawlResult(success=False, error=str(e)) diff --git a/tests/test_crawler/test_comment_crawler.py b/tests/test_crawler/test_comment_crawler.py index ef23d98..51d41ac 100644 --- a/tests/test_crawler/test_comment_crawler.py +++ b/tests/test_crawler/test_comment_crawler.py @@ -140,25 +140,17 @@ class TestCommentCrawler: mock_browser = MockBrowser.return_value # Setup for answer crawl mock_browser.page.route = MagicMock() + mock_browser.page.request.get.return_value.status = 200 + mock_browser.page.request.get.return_value.json.return_value = mock_comment_response mock_browser.goto = MagicMock() + mock_browser.close = MagicMock() - # We need to simulate the interceptor capturing answer data - # For simplicity, patch the internal method - with patch.object(crawler, '_crawl_answer_internal') as mock_internal: - from kb_workflow.crawler.models import CrawlResult, AnswerData, ContentMetadata, AuthorInfo - mock_internal.return_value = CrawlResult( - success=True, - content_type="answer", - data=AnswerData( - id=456, - question_id=123, - question_title="Test Question", - content="

Content

", - excerpt="Excerpt", - metadata=ContentMetadata(author=AuthorInfo(name="Author")) - ) - ) + # Create interceptor that will capture answer data + with patch('kb_workflow.crawler.zhihu_crawler.ZhihuApiInterceptor') as MockInterceptor: + mock_interceptor = MockInterceptor.return_value + mock_interceptor.captured_data = {"answer": mock_answer_data} + # Mock crawl_comments to return test data with patch.object(crawler, 'crawl_comments') as mock_comments: mock_comments.return_value = CommentsResult( content_type="answers", @@ -167,6 +159,20 @@ class TestCommentCrawler: total_count=1 ) - # This test verifies the integration point exists - # The actual implementation will call crawl_comments when include_comments=True - pass + result = crawler.crawl_answer( + "https://www.zhihu.com/question/123/answer/456", + include_comments=True + ) + + # Verify crawl_comments was called with correct params + mock_comments.assert_called_once_with( + content_type="answers", + content_id=456, + include_comments=True + ) + + # Verify result includes comments + assert result.success + assert result.comments is not None + assert len(result.comments) == 1 + assert result.comments[0].content == "Test comment"