feat(08-01): create crawler data models and test scaffolding
- Add Pydantic models: AnswerData, ArticleData, ContentMetadata, AuthorInfo, CrawlResult - Create test fixtures in conftest.py with mock browser and sample API responses - Set up crawler module with __init__.py exports Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Tests for Zhihu Crawler Module"""
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Shared fixtures for Zhihu Crawler tests"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, AsyncMock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_page():
|
||||
"""Mock Playwright page object.
|
||||
|
||||
Provides a mock page with route method for API interception testing.
|
||||
"""
|
||||
page = MagicMock()
|
||||
page.route = MagicMock()
|
||||
page.goto = MagicMock()
|
||||
page.wait_for_selector = MagicMock()
|
||||
page.query_selector = MagicMock()
|
||||
page.close = MagicMock()
|
||||
return page
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_browser():
|
||||
"""Mock PlaywrightBrowser instance.
|
||||
|
||||
Provides a mock browser with page attribute and lifecycle methods.
|
||||
"""
|
||||
browser = MagicMock()
|
||||
browser.page = MagicMock()
|
||||
browser.start = MagicMock()
|
||||
browser.close = MagicMock()
|
||||
browser.goto = MagicMock()
|
||||
return browser
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_answer_api_response():
|
||||
"""Sample answer API response data.
|
||||
|
||||
Mimics the structure of /api/v4/answers/{id} response.
|
||||
"""
|
||||
return {
|
||||
"id": 123456789,
|
||||
"type": "answer",
|
||||
"question": {
|
||||
"id": 987654321,
|
||||
"title": "What is the best way to learn programming?",
|
||||
"type": "question"
|
||||
},
|
||||
"content": "<p>This is a sample answer content in HTML format.</p>",
|
||||
"excerpt": "This is a sample answer content...",
|
||||
"author": {
|
||||
"id": "user123",
|
||||
"name": "John Doe",
|
||||
"url_token": "john-doe-123",
|
||||
"headline": "Software Engineer"
|
||||
},
|
||||
"voteup_count": 1000,
|
||||
"comment_count": 50,
|
||||
"created_time": 1700000000,
|
||||
"updated_time": 1700100000
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_article_api_response():
|
||||
"""Sample article API response data.
|
||||
|
||||
Mimics the structure of zhuanlan article API response.
|
||||
"""
|
||||
return {
|
||||
"id": 11223344,
|
||||
"title": "How to Build a Web Scraper",
|
||||
"content": "<p>This article explains how to build a web scraper...</p>",
|
||||
"excerpt": "This article explains how to build a web scraper...",
|
||||
"author": {
|
||||
"id": "author456",
|
||||
"name": "Jane Smith",
|
||||
"url_token": "jane-smith-456",
|
||||
"headline": "Tech Writer"
|
||||
},
|
||||
"voteup_count": 500,
|
||||
"comment_count": 25,
|
||||
"created_time": 1699900000,
|
||||
"updated_time": 1700000000
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_answer_url():
|
||||
"""Sample Zhihu answer URL."""
|
||||
return "https://www.zhihu.com/question/987654321/answer/123456789"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_article_url():
|
||||
"""Sample Zhihu zhuanlan article URL."""
|
||||
return "https://zhuanlan.zhihu.com/p/11223344"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_invalid_url():
|
||||
"""Sample invalid URL for testing error handling."""
|
||||
return "https://www.example.com/some-page"
|
||||
Reference in New Issue
Block a user