"""Tests for Zhihu comment crawling functionality""" import pytest from unittest.mock import Mock, patch, MagicMock from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler from kb_workflow.crawler.models import CommentData, CommentsResult from kb_workflow.crawler.config import CrawlerConfig class TestCommentCrawler: """Tests for comment crawling functionality.""" @pytest.fixture def crawler(self): """Create ZhihuCrawler instance for testing.""" config = CrawlerConfig(proxy_required=False) return ZhihuCrawler(headless=True, config=config) @pytest.fixture def mock_comment_response(self): """Sample comment API response data. Mimics the structure of /api/v4/comment_v5/{type}/{id}/comments response. """ return { "data": [ { "id": 111, "content": "This is a hot comment", "author": {"name": "User A", "url_token": "user-a"}, "created_time": 1234567890, "like_count": 100, "child_comment_count": 5, "is_author": False }, { "id": 222, "content": "Author reply", "author": {"name": "Author", "url_token": "author"}, "created_time": 1234567900, "like_count": 50, "child_comment_count": 0, "is_author": True } ], "paging": {"totals": 100} } def test_crawl_comments_returns_comments_result(self, crawler, mock_comment_response): """crawl_comments returns CommentsResult.""" with patch.object(crawler, '_get_cookie', return_value="test_cookie"): with patch.object(crawler, '_start_browser_with_proxy'): # Mock _page.request.get mock_response = MagicMock() mock_response.status = 200 mock_response.json.return_value = mock_comment_response crawler._page = MagicMock() crawler._page.request.get.return_value = mock_response result = crawler.crawl_comments("answers", 12345) assert isinstance(result, CommentsResult) assert result.content_type == "answers" assert result.content_id == 12345 def test_comments_are_hot_order(self, crawler, mock_comment_response): """Comments use hot (popular) sorting.""" with patch.object(crawler, '_get_cookie', return_value="test_cookie"): with patch.object(crawler, '_start_browser_with_proxy'): mock_response = MagicMock() mock_response.status = 200 mock_response.json.return_value = mock_comment_response crawler._page = MagicMock() crawler._page.request.get.return_value = mock_response crawler.crawl_comments("answers", 12345) # Verify API URL contains order=hot call_args = crawler._page.request.get.call_args assert "order=hot" in call_args[0][0] def test_comment_has_required_fields(self, crawler, mock_comment_response): """Each comment contains required fields.""" with patch.object(crawler, '_get_cookie', return_value="test_cookie"): with patch.object(crawler, '_start_browser_with_proxy'): mock_response = MagicMock() mock_response.status = 200 mock_response.json.return_value = mock_comment_response crawler._page = MagicMock() crawler._page.request.get.return_value = mock_response result = crawler.crawl_comments("answers", 12345) assert len(result.comments) == 2 comment = result.comments[0] assert comment.content == "This is a hot comment" assert comment.author.name == "User A" assert comment.like_count == 100 def test_no_nested_replies(self, crawler, mock_comment_response): """Do not expand nested replies - only first-level comments.""" with patch.object(crawler, '_get_cookie', return_value="test_cookie"): with patch.object(crawler, '_start_browser_with_proxy'): mock_response = MagicMock() mock_response.status = 200 mock_response.json.return_value = mock_comment_response crawler._page = MagicMock() crawler._page.request.get.return_value = mock_response result = crawler.crawl_comments("answers", 12345) # Only return first-level comments, not nested replies assert len(result.comments) == 2 # child_count records number of child comments but does not expand assert result.comments[0].child_count == 5 def test_include_comments_parameter(self, crawler): """include_comments=False does not fetch comments.""" result = crawler.crawl_comments("answers", 12345, include_comments=False) assert result.comments == [] assert result.total_count == 0 def test_include_comments_in_crawl_answer(self, crawler): """crawl_answer with include_comments=True fetches comments.""" mock_answer_data = { "id": 456, "question": {"id": 123, "title": "Test Question"}, "content": "

Answer content

", "excerpt": "Answer excerpt", "author": {"name": "Author"}, "voteup_count": 10, "comment_count": 5 } with patch.object(crawler, '_get_cookie', return_value="test_cookie"): with patch.object(crawler, '_start_browser_with_proxy'): with patch.object(crawler, '_apply_random_delay'): # Mock ZhihuApiInterceptor from kb_workflow.crawler.zhihu_crawler import ZhihuApiInterceptor mock_interceptor = MagicMock(spec=ZhihuApiInterceptor) mock_interceptor.captured_data = {"answer": mock_answer_data} with patch('kb_workflow.crawler.zhihu_crawler.ZhihuApiInterceptor', return_value=mock_interceptor): # Mock page operations crawler._page = MagicMock() crawler._page.goto = MagicMock() crawler._page.route = MagicMock() crawler._page.wait_for_timeout = MagicMock() # Mock crawl_comments with patch.object(crawler, 'crawl_comments') as mock_comments: mock_comments.return_value = CommentsResult( content_type="answers", content_id=456, comments=[CommentData(id=1, content="Test comment")], total_count=1 ) with patch.object(crawler, '_close_browser'): result = crawler.crawl_answer( "https://www.zhihu.com/question/123/answer/456", include_comments=True ) # Verify crawl_comments was called with correct params mock_comments.assert_called_once_with( content_type="answers", content_id=456, include_comments=True ) # Verify result includes comments assert result.success assert result.comments is not None assert len(result.comments) == 1 assert result.comments[0].content == "Test comment"