kb-skills/tests/test_crawler/test_comment_crawler.py

"""Tests for Zhihu comment crawling functionality"""

import pytest
from unittest.mock import Mock, patch, MagicMock

from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
from kb_workflow.crawler.models import CommentData, CommentsResult
from kb_workflow.crawler.config import CrawlerConfig


class TestCommentCrawler:
    """Tests for comment crawling functionality."""

    @pytest.fixture
    def crawler(self):
        """Create ZhihuCrawler instance for testing."""
        config = CrawlerConfig(proxy_required=False)
        return ZhihuCrawler(headless=True, config=config)

    @pytest.fixture
    def mock_comment_response(self):
        """Sample comment API response data.

        Mimics the structure of /api/v4/comment_v5/{type}/{id}/comments response.
        """
        return {
            "data": [
                {
                    "id": 111,
                    "content": "This is a hot comment",
                    "author": {"name": "User A", "url_token": "user-a"},
                    "created_time": 1234567890,
                    "like_count": 100,
                    "child_comment_count": 5,
                    "is_author": False
                },
                {
                    "id": 222,
                    "content": "Author reply",
                    "author": {"name": "Author", "url_token": "author"},
                    "created_time": 1234567900,
                    "like_count": 50,
                    "child_comment_count": 0,
                    "is_author": True
                }
            ],
            "paging": {"totals": 100}
        }

    def test_crawl_comments_returns_comments_result(self, crawler, mock_comment_response):
        """crawl_comments returns CommentsResult."""
        with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
            with patch.object(crawler, '_start_browser_with_proxy'):
                # Mock _page.request.get
                mock_response = MagicMock()
                mock_response.status = 200
                mock_response.json.return_value = mock_comment_response

                crawler._page = MagicMock()
                crawler._page.request.get.return_value = mock_response

                result = crawler.crawl_comments("answers", 12345)

                assert isinstance(result, CommentsResult)
                assert result.content_type == "answers"
                assert result.content_id == 12345

    def test_comments_are_hot_order(self, crawler, mock_comment_response):
        """Comments use hot (popular) sorting."""
        with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
            with patch.object(crawler, '_start_browser_with_proxy'):
                mock_response = MagicMock()
                mock_response.status = 200
                mock_response.json.return_value = mock_comment_response

                crawler._page = MagicMock()
                crawler._page.request.get.return_value = mock_response

                crawler.crawl_comments("answers", 12345)

                # Verify API URL contains order=hot
                call_args = crawler._page.request.get.call_args
                assert "order=hot" in call_args[0][0]

    def test_comment_has_required_fields(self, crawler, mock_comment_response):
        """Each comment contains required fields."""
        with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
            with patch.object(crawler, '_start_browser_with_proxy'):
                mock_response = MagicMock()
                mock_response.status = 200
                mock_response.json.return_value = mock_comment_response

                crawler._page = MagicMock()
                crawler._page.request.get.return_value = mock_response

                result = crawler.crawl_comments("answers", 12345)

                assert len(result.comments) == 2
                comment = result.comments[0]
                assert comment.content == "This is a hot comment"
                assert comment.author.name == "User A"
                assert comment.like_count == 100

    def test_no_nested_replies(self, crawler, mock_comment_response):
        """Do not expand nested replies - only first-level comments."""
        with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
            with patch.object(crawler, '_start_browser_with_proxy'):
                mock_response = MagicMock()
                mock_response.status = 200
                mock_response.json.return_value = mock_comment_response

                crawler._page = MagicMock()
                crawler._page.request.get.return_value = mock_response

                result = crawler.crawl_comments("answers", 12345)

                # Only return first-level comments, not nested replies
                assert len(result.comments) == 2
                # child_count records number of child comments but does not expand
                assert result.comments[0].child_count == 5

    def test_include_comments_parameter(self, crawler):
        """include_comments=False does not fetch comments."""
        result = crawler.crawl_comments("answers", 12345, include_comments=False)

        assert result.comments == []
        assert result.total_count == 0

    def test_include_comments_in_crawl_answer(self, crawler):
        """crawl_answer with include_comments=True fetches comments."""
        mock_answer_data = {
            "id": 456,
            "question": {"id": 123, "title": "Test Question"},
            "content": "<p>Answer content</p>",
            "excerpt": "Answer excerpt",
            "author": {"name": "Author"},
            "voteup_count": 10,
            "comment_count": 5
        }

        with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
            with patch.object(crawler, '_start_browser_with_proxy'):
                with patch.object(crawler, '_apply_random_delay'):
                    # Mock ZhihuApiInterceptor
                    from kb_workflow.crawler.zhihu_crawler import ZhihuApiInterceptor
                    mock_interceptor = MagicMock(spec=ZhihuApiInterceptor)
                    mock_interceptor.captured_data = {"answer": mock_answer_data}

                    with patch('kb_workflow.crawler.zhihu_crawler.ZhihuApiInterceptor', return_value=mock_interceptor):
                        # Mock page operations
                        crawler._page = MagicMock()
                        crawler._page.goto = MagicMock()
                        crawler._page.route = MagicMock()
                        crawler._page.wait_for_timeout = MagicMock()

                        # Mock crawl_comments
                        with patch.object(crawler, 'crawl_comments') as mock_comments:
                            mock_comments.return_value = CommentsResult(
                                content_type="answers",
                                content_id=456,
                                comments=[CommentData(id=1, content="Test comment")],
                                total_count=1
                            )

                            with patch.object(crawler, '_close_browser'):
                                result = crawler.crawl_answer(
                                    "https://www.zhihu.com/question/123/answer/456",
                                    include_comments=True
                                )

                                # Verify crawl_comments was called with correct params
                                mock_comments.assert_called_once_with(
                                    content_type="answers",
                                    content_id=456,
                                    include_comments=True
                                )

                                # Verify result includes comments
                                assert result.success
                                assert result.comments is not None
                                assert len(result.comments) == 1
                                assert result.comments[0].content == "Test comment"