feat(08-01): create crawler data models and test scaffolding
- Add Pydantic models: AnswerData, ArticleData, ContentMetadata, AuthorInfo, CrawlResult - Create test fixtures in conftest.py with mock browser and sample API responses - Set up crawler module with __init__.py exports Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
"""Zhihu Crawler Module
|
||||
|
||||
Provides content crawling capabilities for Zhihu using Playwright API interception.
|
||||
"""
|
||||
|
||||
from .models import (
|
||||
AuthorInfo,
|
||||
ContentMetadata,
|
||||
AnswerData,
|
||||
ArticleData,
|
||||
CrawlResult,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AuthorInfo",
|
||||
"ContentMetadata",
|
||||
"AnswerData",
|
||||
"ArticleData",
|
||||
"CrawlResult",
|
||||
]
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Zhihu Crawler Data Models
|
||||
|
||||
Pydantic models for structured data from Zhihu API responses.
|
||||
"""
|
||||
|
||||
from typing import Optional, Union
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class AuthorInfo(BaseModel):
|
||||
"""Author information from Zhihu content.
|
||||
|
||||
Attributes:
|
||||
name: Author display name
|
||||
url_token: Author URL token (e.g., 'zhang-san-12')
|
||||
headline: Author bio/headline
|
||||
"""
|
||||
name: str = Field(..., description="Author display name")
|
||||
url_token: Optional[str] = Field(None, description="Author URL token")
|
||||
headline: Optional[str] = Field(None, description="Author bio/headline")
|
||||
|
||||
|
||||
class ContentMetadata(BaseModel):
|
||||
"""Metadata for Zhihu content (answers, articles, etc).
|
||||
|
||||
Attributes:
|
||||
author: Author information
|
||||
voteup_count: Number of upvotes/likes
|
||||
comment_count: Number of comments
|
||||
created_time: Unix timestamp of creation
|
||||
updated_time: Unix timestamp of last update
|
||||
"""
|
||||
author: Optional[AuthorInfo] = Field(None, description="Author info")
|
||||
voteup_count: int = Field(default=0, description="Upvote count")
|
||||
comment_count: int = Field(default=0, description="Comment count")
|
||||
created_time: Optional[int] = Field(None, description="Creation timestamp")
|
||||
updated_time: Optional[int] = Field(None, description="Update timestamp")
|
||||
|
||||
|
||||
class AnswerData(BaseModel):
|
||||
"""Data model for Zhihu answer.
|
||||
|
||||
Attributes:
|
||||
id: Answer ID
|
||||
question_id: Question ID this answer belongs to
|
||||
question_title: Title of the question
|
||||
content: Answer content in HTML format
|
||||
excerpt: Plain text excerpt/summary
|
||||
metadata: Additional metadata (author, votes, etc.)
|
||||
"""
|
||||
id: int = Field(..., description="Answer ID")
|
||||
question_id: int = Field(..., description="Question ID")
|
||||
question_title: str = Field(..., description="Question title")
|
||||
content: str = Field(default="", description="Answer content (HTML)")
|
||||
excerpt: str = Field(default="", description="Plain text excerpt")
|
||||
metadata: ContentMetadata = Field(
|
||||
default_factory=ContentMetadata,
|
||||
description="Content metadata"
|
||||
)
|
||||
|
||||
|
||||
class ArticleData(BaseModel):
|
||||
"""Data model for Zhihu zhuanlan article.
|
||||
|
||||
Attributes:
|
||||
id: Article ID
|
||||
title: Article title
|
||||
content: Article content in HTML format
|
||||
excerpt: Plain text excerpt/summary
|
||||
metadata: Additional metadata (author, votes, etc.)
|
||||
"""
|
||||
id: int = Field(..., description="Article ID")
|
||||
title: str = Field(..., description="Article title")
|
||||
content: str = Field(default="", description="Article content (HTML)")
|
||||
excerpt: str = Field(default="", description="Plain text excerpt")
|
||||
metadata: ContentMetadata = Field(
|
||||
default_factory=ContentMetadata,
|
||||
description="Content metadata"
|
||||
)
|
||||
|
||||
|
||||
class CrawlResult(BaseModel):
|
||||
"""Result of a crawl operation.
|
||||
|
||||
Attributes:
|
||||
success: Whether the crawl was successful
|
||||
content_type: Type of content ("answer" or "article")
|
||||
data: Extracted content data if successful
|
||||
error: Error message if failed
|
||||
"""
|
||||
success: bool = Field(..., description="Whether crawl succeeded")
|
||||
content_type: str = Field(default="", description="Content type: answer or article")
|
||||
data: Optional[Union[AnswerData, ArticleData]] = Field(
|
||||
None,
|
||||
description="Extracted content data"
|
||||
)
|
||||
error: Optional[str] = Field(None, description="Error message if failed")
|
||||
Reference in New Issue
Block a user