starmen1265
a102a75b28
- Add Pydantic models: AnswerData, ArticleData, ContentMetadata, AuthorInfo, CrawlResult - Create test fixtures in conftest.py with mock browser and sample API responses - Set up crawler module with __init__.py exports Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
98 lines
3.4 KiB
Python
98 lines
3.4 KiB
Python
"""Zhihu Crawler Data Models
|
|
|
|
Pydantic models for structured data from Zhihu API responses.
|
|
"""
|
|
|
|
from typing import Optional, Union
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class AuthorInfo(BaseModel):
|
|
"""Author information from Zhihu content.
|
|
|
|
Attributes:
|
|
name: Author display name
|
|
url_token: Author URL token (e.g., 'zhang-san-12')
|
|
headline: Author bio/headline
|
|
"""
|
|
name: str = Field(..., description="Author display name")
|
|
url_token: Optional[str] = Field(None, description="Author URL token")
|
|
headline: Optional[str] = Field(None, description="Author bio/headline")
|
|
|
|
|
|
class ContentMetadata(BaseModel):
|
|
"""Metadata for Zhihu content (answers, articles, etc).
|
|
|
|
Attributes:
|
|
author: Author information
|
|
voteup_count: Number of upvotes/likes
|
|
comment_count: Number of comments
|
|
created_time: Unix timestamp of creation
|
|
updated_time: Unix timestamp of last update
|
|
"""
|
|
author: Optional[AuthorInfo] = Field(None, description="Author info")
|
|
voteup_count: int = Field(default=0, description="Upvote count")
|
|
comment_count: int = Field(default=0, description="Comment count")
|
|
created_time: Optional[int] = Field(None, description="Creation timestamp")
|
|
updated_time: Optional[int] = Field(None, description="Update timestamp")
|
|
|
|
|
|
class AnswerData(BaseModel):
|
|
"""Data model for Zhihu answer.
|
|
|
|
Attributes:
|
|
id: Answer ID
|
|
question_id: Question ID this answer belongs to
|
|
question_title: Title of the question
|
|
content: Answer content in HTML format
|
|
excerpt: Plain text excerpt/summary
|
|
metadata: Additional metadata (author, votes, etc.)
|
|
"""
|
|
id: int = Field(..., description="Answer ID")
|
|
question_id: int = Field(..., description="Question ID")
|
|
question_title: str = Field(..., description="Question title")
|
|
content: str = Field(default="", description="Answer content (HTML)")
|
|
excerpt: str = Field(default="", description="Plain text excerpt")
|
|
metadata: ContentMetadata = Field(
|
|
default_factory=ContentMetadata,
|
|
description="Content metadata"
|
|
)
|
|
|
|
|
|
class ArticleData(BaseModel):
|
|
"""Data model for Zhihu zhuanlan article.
|
|
|
|
Attributes:
|
|
id: Article ID
|
|
title: Article title
|
|
content: Article content in HTML format
|
|
excerpt: Plain text excerpt/summary
|
|
metadata: Additional metadata (author, votes, etc.)
|
|
"""
|
|
id: int = Field(..., description="Article ID")
|
|
title: str = Field(..., description="Article title")
|
|
content: str = Field(default="", description="Article content (HTML)")
|
|
excerpt: str = Field(default="", description="Plain text excerpt")
|
|
metadata: ContentMetadata = Field(
|
|
default_factory=ContentMetadata,
|
|
description="Content metadata"
|
|
)
|
|
|
|
|
|
class CrawlResult(BaseModel):
|
|
"""Result of a crawl operation.
|
|
|
|
Attributes:
|
|
success: Whether the crawl was successful
|
|
content_type: Type of content ("answer" or "article")
|
|
data: Extracted content data if successful
|
|
error: Error message if failed
|
|
"""
|
|
success: bool = Field(..., description="Whether crawl succeeded")
|
|
content_type: str = Field(default="", description="Content type: answer or article")
|
|
data: Optional[Union[AnswerData, ArticleData]] = Field(
|
|
None,
|
|
description="Extracted content data"
|
|
)
|
|
error: Optional[str] = Field(None, description="Error message if failed")
|