diff --git a/tests/test_crawler/test_answer_crawler.py b/tests/test_crawler/test_answer_crawler.py new file mode 100644 index 0000000..2793e35 --- /dev/null +++ b/tests/test_crawler/test_answer_crawler.py @@ -0,0 +1,93 @@ +"""Tests for Zhihu Answer Crawler""" + +import pytest +from unittest.mock import MagicMock, patch, PropertyMock + +from kb_workflow.crawler.models import AnswerData, CrawlResult + + +class TestCrawlAnswer: + """Tests for crawl_answer() method.""" + + def test_crawl_answer_returns_crawl_result(self, mock_browser, sample_answer_url): + """Test that crawl_answer returns a CrawlResult object.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + with patch.object(ZhihuCrawler, '_get_cookie', return_value='test_cookie'): + with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as mock_pb: + mock_pb_instance = MagicMock() + mock_pb.return_value = mock_pb_instance + mock_pb_instance.page = MagicMock() + mock_pb_instance.page.route = MagicMock() + + crawler = ZhihuCrawler(headless=True, timeout=10) + result = crawler.crawl_answer(sample_answer_url) + + assert isinstance(result, CrawlResult) + + def test_crawl_answer_invalid_url_returns_failure(self): + """Test that invalid URL returns CrawlResult with success=False.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + result = crawler.crawl_answer("https://www.example.com/invalid") + + assert result.success is False + assert result.error is not None + assert "Invalid" in result.error + + def test_crawl_answer_extracts_question_title( + self, sample_answer_url, sample_answer_api_response + ): + """Test that crawl_answer extracts question title from API response.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler, ZhihuApiInterceptor + + # Create interceptor with captured data + interceptor = ZhihuApiInterceptor() + interceptor.captured_data["answer"] = sample_answer_api_response + + crawler = ZhihuCrawler(headless=True, timeout=10) + + # Test the parse method directly + result_data = crawler._parse_answer_data( + sample_answer_api_response, + answer_id=123456789, + question_id=987654321 + ) + + assert result_data.question_title == "What is the best way to learn programming?" + + +class TestAnswerDataExtraction: + """Tests for answer data extraction.""" + + def test_parse_answer_data_contains_content(self, sample_answer_api_response): + """Test that parsed answer contains content.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + result = crawler._parse_answer_data( + sample_answer_api_response, + answer_id=123456789, + question_id=987654321 + ) + + assert isinstance(result, AnswerData) + assert result.content == "

This is a sample answer content in HTML format.

" + assert result.excerpt == "This is a sample answer content..." + + def test_parse_answer_data_contains_metadata(self, sample_answer_api_response): + """Test that parsed answer contains metadata.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + result = crawler._parse_answer_data( + sample_answer_api_response, + answer_id=123456789, + question_id=987654321 + ) + + assert result.metadata.voteup_count == 1000 + assert result.metadata.comment_count == 50 + assert result.metadata.author is not None + assert result.metadata.author.name == "John Doe" diff --git a/tests/test_crawler/test_article_crawler.py b/tests/test_crawler/test_article_crawler.py new file mode 100644 index 0000000..eda45db --- /dev/null +++ b/tests/test_crawler/test_article_crawler.py @@ -0,0 +1,66 @@ +"""Tests for Zhihu Article Crawler""" + +import pytest +from unittest.mock import MagicMock, patch + +from kb_workflow.crawler.models import ArticleData, CrawlResult + + +class TestCrawlArticle: + """Tests for crawl_article() method.""" + + def test_crawl_article_invalid_url_returns_failure(self): + """Test that invalid URL returns CrawlResult with success=False.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + result = crawler.crawl_article("https://www.example.com/invalid") + + assert result.success is False + assert result.error is not None + assert "Invalid" in result.error + + def test_crawl_article_extracts_title( + self, sample_article_api_response + ): + """Test that crawl_article extracts title from API response.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + result = crawler._parse_article_data( + sample_article_api_response, + article_id=11223344 + ) + + assert isinstance(result, ArticleData) + assert result.title == "How to Build a Web Scraper" + + def test_crawl_article_extracts_content( + self, sample_article_api_response + ): + """Test that crawl_article extracts content from API response.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + result = crawler._parse_article_data( + sample_article_api_response, + article_id=11223344 + ) + + assert result.content == "

This article explains how to build a web scraper...

" + assert result.excerpt == "This article explains how to build a web scraper..." + + def test_crawl_article_contains_metadata(self, sample_article_api_response): + """Test that parsed article contains metadata.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + result = crawler._parse_article_data( + sample_article_api_response, + article_id=11223344 + ) + + assert result.metadata.voteup_count == 500 + assert result.metadata.comment_count == 25 + assert result.metadata.author is not None + assert result.metadata.author.name == "Jane Smith" diff --git a/tests/test_crawler/test_metadata_extraction.py b/tests/test_crawler/test_metadata_extraction.py new file mode 100644 index 0000000..007577d --- /dev/null +++ b/tests/test_crawler/test_metadata_extraction.py @@ -0,0 +1,109 @@ +"""Tests for metadata extraction from Zhihu API responses""" + +import pytest + +from kb_workflow.crawler.models import AuthorInfo, ContentMetadata + + +class TestAuthorExtraction: + """Tests for author information extraction.""" + + def test_parse_author_with_full_data(self): + """Test parsing author with all fields present.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + author_data = { + "name": "Test User", + "url_token": "test-user-123", + "headline": "Test headline" + } + + result = crawler._parse_author(author_data) + + assert isinstance(result, AuthorInfo) + assert result.name == "Test User" + assert result.url_token == "test-user-123" + assert result.headline == "Test headline" + + def test_parse_author_with_minimal_data(self): + """Test parsing author with only required fields.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + author_data = {"name": "Minimal User"} + + result = crawler._parse_author(author_data) + + assert result.name == "Minimal User" + assert result.url_token is None + assert result.headline is None + + def test_parse_author_with_empty_data(self): + """Test parsing author with empty/None data.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + + result = crawler._parse_author(None) + assert result is None + + result = crawler._parse_author({}) + assert result.name == "" + + +class TestMetadataExtraction: + """Tests for content metadata extraction.""" + + def test_parse_metadata_with_full_data(self): + """Test parsing metadata with all fields present.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + data = { + "author": {"name": "Test Author"}, + "voteup_count": 100, + "comment_count": 20, + "created_time": 1700000000, + "updated_time": 1700100000 + } + + result = crawler._parse_metadata(data) + + assert isinstance(result, ContentMetadata) + assert result.voteup_count == 100 + assert result.comment_count == 20 + assert result.created_time == 1700000000 + assert result.updated_time == 1700100000 + assert result.author is not None + assert result.author.name == "Test Author" + + def test_parse_metadata_with_defaults(self): + """Test parsing metadata with missing optional fields.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + data = {} + + result = crawler._parse_metadata(data) + + assert result.voteup_count == 0 + assert result.comment_count == 0 + assert result.created_time is None + assert result.updated_time is None + assert result.author is None + + def test_parse_metadata_with_zero_values(self): + """Test parsing metadata when counts are zero.""" + from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler + + crawler = ZhihuCrawler(headless=True, timeout=10) + data = { + "voteup_count": 0, + "comment_count": 0 + } + + result = crawler._parse_metadata(data) + + assert result.voteup_count == 0 + assert result.comment_count == 0