diff --git a/kb_workflow/crawler/__init__.py b/kb_workflow/crawler/__init__.py new file mode 100644 index 0000000..4b5a571 --- /dev/null +++ b/kb_workflow/crawler/__init__.py @@ -0,0 +1,20 @@ +"""Zhihu Crawler Module + +Provides content crawling capabilities for Zhihu using Playwright API interception. +""" + +from .models import ( + AuthorInfo, + ContentMetadata, + AnswerData, + ArticleData, + CrawlResult, +) + +__all__ = [ + "AuthorInfo", + "ContentMetadata", + "AnswerData", + "ArticleData", + "CrawlResult", +] diff --git a/kb_workflow/crawler/models.py b/kb_workflow/crawler/models.py new file mode 100644 index 0000000..884242f --- /dev/null +++ b/kb_workflow/crawler/models.py @@ -0,0 +1,97 @@ +"""Zhihu Crawler Data Models + +Pydantic models for structured data from Zhihu API responses. +""" + +from typing import Optional, Union +from pydantic import BaseModel, Field + + +class AuthorInfo(BaseModel): + """Author information from Zhihu content. + + Attributes: + name: Author display name + url_token: Author URL token (e.g., 'zhang-san-12') + headline: Author bio/headline + """ + name: str = Field(..., description="Author display name") + url_token: Optional[str] = Field(None, description="Author URL token") + headline: Optional[str] = Field(None, description="Author bio/headline") + + +class ContentMetadata(BaseModel): + """Metadata for Zhihu content (answers, articles, etc). + + Attributes: + author: Author information + voteup_count: Number of upvotes/likes + comment_count: Number of comments + created_time: Unix timestamp of creation + updated_time: Unix timestamp of last update + """ + author: Optional[AuthorInfo] = Field(None, description="Author info") + voteup_count: int = Field(default=0, description="Upvote count") + comment_count: int = Field(default=0, description="Comment count") + created_time: Optional[int] = Field(None, description="Creation timestamp") + updated_time: Optional[int] = Field(None, description="Update timestamp") + + +class AnswerData(BaseModel): + """Data model for Zhihu answer. + + Attributes: + id: Answer ID + question_id: Question ID this answer belongs to + question_title: Title of the question + content: Answer content in HTML format + excerpt: Plain text excerpt/summary + metadata: Additional metadata (author, votes, etc.) + """ + id: int = Field(..., description="Answer ID") + question_id: int = Field(..., description="Question ID") + question_title: str = Field(..., description="Question title") + content: str = Field(default="", description="Answer content (HTML)") + excerpt: str = Field(default="", description="Plain text excerpt") + metadata: ContentMetadata = Field( + default_factory=ContentMetadata, + description="Content metadata" + ) + + +class ArticleData(BaseModel): + """Data model for Zhihu zhuanlan article. + + Attributes: + id: Article ID + title: Article title + content: Article content in HTML format + excerpt: Plain text excerpt/summary + metadata: Additional metadata (author, votes, etc.) + """ + id: int = Field(..., description="Article ID") + title: str = Field(..., description="Article title") + content: str = Field(default="", description="Article content (HTML)") + excerpt: str = Field(default="", description="Plain text excerpt") + metadata: ContentMetadata = Field( + default_factory=ContentMetadata, + description="Content metadata" + ) + + +class CrawlResult(BaseModel): + """Result of a crawl operation. + + Attributes: + success: Whether the crawl was successful + content_type: Type of content ("answer" or "article") + data: Extracted content data if successful + error: Error message if failed + """ + success: bool = Field(..., description="Whether crawl succeeded") + content_type: str = Field(default="", description="Content type: answer or article") + data: Optional[Union[AnswerData, ArticleData]] = Field( + None, + description="Extracted content data" + ) + error: Optional[str] = Field(None, description="Error message if failed") diff --git a/tests/test_crawler/__init__.py b/tests/test_crawler/__init__.py new file mode 100644 index 0000000..1e63172 --- /dev/null +++ b/tests/test_crawler/__init__.py @@ -0,0 +1 @@ +"""Tests for Zhihu Crawler Module""" diff --git a/tests/test_crawler/conftest.py b/tests/test_crawler/conftest.py new file mode 100644 index 0000000..3337b0b --- /dev/null +++ b/tests/test_crawler/conftest.py @@ -0,0 +1,104 @@ +"""Shared fixtures for Zhihu Crawler tests""" + +import pytest +from unittest.mock import MagicMock, AsyncMock + + +@pytest.fixture +def mock_page(): + """Mock Playwright page object. + + Provides a mock page with route method for API interception testing. + """ + page = MagicMock() + page.route = MagicMock() + page.goto = MagicMock() + page.wait_for_selector = MagicMock() + page.query_selector = MagicMock() + page.close = MagicMock() + return page + + +@pytest.fixture +def mock_browser(): + """Mock PlaywrightBrowser instance. + + Provides a mock browser with page attribute and lifecycle methods. + """ + browser = MagicMock() + browser.page = MagicMock() + browser.start = MagicMock() + browser.close = MagicMock() + browser.goto = MagicMock() + return browser + + +@pytest.fixture +def sample_answer_api_response(): + """Sample answer API response data. + + Mimics the structure of /api/v4/answers/{id} response. + """ + return { + "id": 123456789, + "type": "answer", + "question": { + "id": 987654321, + "title": "What is the best way to learn programming?", + "type": "question" + }, + "content": "
This is a sample answer content in HTML format.
", + "excerpt": "This is a sample answer content...", + "author": { + "id": "user123", + "name": "John Doe", + "url_token": "john-doe-123", + "headline": "Software Engineer" + }, + "voteup_count": 1000, + "comment_count": 50, + "created_time": 1700000000, + "updated_time": 1700100000 + } + + +@pytest.fixture +def sample_article_api_response(): + """Sample article API response data. + + Mimics the structure of zhuanlan article API response. + """ + return { + "id": 11223344, + "title": "How to Build a Web Scraper", + "content": "This article explains how to build a web scraper...
", + "excerpt": "This article explains how to build a web scraper...", + "author": { + "id": "author456", + "name": "Jane Smith", + "url_token": "jane-smith-456", + "headline": "Tech Writer" + }, + "voteup_count": 500, + "comment_count": 25, + "created_time": 1699900000, + "updated_time": 1700000000 + } + + +@pytest.fixture +def sample_answer_url(): + """Sample Zhihu answer URL.""" + return "https://www.zhihu.com/question/987654321/answer/123456789" + + +@pytest.fixture +def sample_article_url(): + """Sample Zhihu zhuanlan article URL.""" + return "https://zhuanlan.zhihu.com/p/11223344" + + +@pytest.fixture +def sample_invalid_url(): + """Sample invalid URL for testing error handling.""" + return "https://www.example.com/some-page"