14 KiB
14 KiB
phase, plan, type, wave, depends_on, files_modified, autonomous, requirements, user_setup, must_haves
| phase | plan | type | wave | depends_on | files_modified | autonomous | requirements | user_setup | must_haves | ||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 08-crawler | 02 | execute | 1 |
|
true |
|
|
Purpose: 提供可选的评论爬取能力，丰富内容元数据 Output: crawl_comments() 方法 + CommentData 模型 + 测试
<execution_context> @/home/starmen1265/.claude/get-shit-done/workflows/execute-plan.md @/home/starmen1265/.claude/get-shit-done/templates/summary.md </execution_context>
@.planning/PROJECT.md @.planning/ROADMAP.md @.planning/STATE.md @.planning/phases/08-crawler/08-CONTEXT.md @.planning/phases/08-crawler/08-RESEARCH.md评论 API 格式:
https://www.zhihu.com/api/v4/comment_v5/{content_type}/{content_id}/comments
?order=hot&limit=20
参数说明:
- content_type: "answers" (回答) 或 "articles" (文章)
- content_id: 内容 ID
- order: "hot" (热门) 或 "ts" (最新)
- limit: 每页数量，默认 20
响应结构 (预期):
{
"data": [
{
"id": 12345,
"content": "评论内容",
"author": {"name": "用户名"},
"created_time": 1234567890,
"like_count": 10,
"child_comment_count": 2
}
],
"paging": {...}
}
添加到 kb_workflow/crawler/models.py:
class CommentAuthor(BaseModel):
"""评论作者信息"""
name: str = Field(..., description="作者名称")
url_token: Optional[str] = Field(None, description="作者 URL token")
class CommentData(BaseModel):
"""评论数据"""
id: int
content: str = Field(..., description="评论内容")
author: Optional[CommentAuthor] = None
created_time: Optional[int] = Field(None, description="创建时间戳")
like_count: int = Field(default=0, description="点赞数")
child_count: int = Field(default=0, description="子评论数")
is_author: bool = Field(default=False, description="是否为作者回复")
class CommentsResult(BaseModel):
"""评论爬取结果"""
content_type: str # "answers" or "articles"
content_id: int
comments: List[CommentData] = Field(default_factory=list)
total_count: int = Field(default=0, description="总评论数（可能超过实际返回）")
同时更新 CrawlResult 以支持包含评论:
class CrawlResult(BaseModel):
"""爬取结果"""
success: bool
content_type: str # "answer" or "article"
data: Optional[AnswerData | ArticleData] = None
comments: Optional[List[CommentData]] = None # 新增
error: Optional[str] = None
1. 在 ZhihuCrawler 类中添加方法:
# 在 kb_workflow/crawler/zhihu_crawler.py 中添加
from .models import CommentData, CommentsResult, CommentAuthor
class ZhihuCrawler:
# ... 现有代码 ...
COMMENT_API_TEMPLATE = (
"https://www.zhihu.com/api/v4/comment_v5/"
"{content_type}/{content_id}/comments"
"?order=hot&limit=20"
)
def crawl_comments(
self,
content_type: str, # "answers" or "articles"
content_id: int,
include_comments: bool = True
) -> CommentsResult:
"""爬取热门评论
Args:
content_type: 内容类型 ("answers" 或 "articles")
content_id: 内容 ID
include_comments: 是否包含评论（保留参数兼容性）
Returns:
CommentsResult 包含评论列表
"""
if not include_comments:
return CommentsResult(
content_type=content_type,
content_id=content_id,
comments=[],
total_count=0
)
try:
self.browser = PlaywrightBrowser(headless=self.headless, timeout=self.timeout)
self.browser.start(cookie=self._get_cookie())
# 构建评论 API URL
api_url = self.COMMENT_API_TEMPLATE.format(
content_type=content_type,
content_id=content_id
)
# 使用 Playwright 的 request API 直接调用
response = self.browser.page.request.get(api_url)
if response.status != 200:
return CommentsResult(
content_type=content_type,
content_id=content_id,
comments=[],
total_count=0
)
data = response.json()
comments = []
for item in data.get("data", []):
author_data = item.get("author", {})
comment = CommentData(
id=item.get("id", 0),
content=item.get("content", ""),
author=CommentAuthor(
name=author_data.get("name", ""),
url_token=author_data.get("url_token")
) if author_data else None,
created_time=item.get("created_time"),
like_count=item.get("like_count", 0),
child_count=item.get("child_comment_count", 0),
is_author=item.get("is_author", False)
)
comments.append(comment)
# 获取总数
total_count = data.get("paging", {}).get("totals", len(comments))
return CommentsResult(
content_type=content_type,
content_id=content_id,
comments=comments,
total_count=total_count
)
except Exception as e:
return CommentsResult(
content_type=content_type,
content_id=content_id,
comments=[],
total_count=0
)
finally:
if self.browser:
self.browser.close()
def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult:
"""爬取知乎回答（支持可选评论）
Args:
url: 回答 URL
include_comments: 是否包含热门评论
Returns:
CrawlResult 包含回答数据和可选评论
"""
# 原有逻辑...
result = self._crawl_answer_original(url)
if result.success and include_comments:
# 爬取评论
comments_result = self.crawl_comments(
content_type="answers",
content_id=result.data.id,
include_comments=True
)
result.comments = comments_result.comments
return result
2. 创建 tests/test_crawler/test_comment_crawler.py:
import pytest
from unittest.mock import Mock, patch, MagicMock
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
from kb_workflow.crawler.models import CommentData, CommentsResult
class TestCommentCrawler:
@pytest.fixture
def crawler(self):
return ZhihuCrawler(headless=True)
@pytest.fixture
def mock_comment_response(self):
return {
"data": [
{
"id": 111,
"content": "这是一条热门评论",
"author": {"name": "用户A", "url_token": "user-a"},
"created_time": 1234567890,
"like_count": 100,
"child_comment_count": 5,
"is_author": False
},
{
"id": 222,
"content": "作者回复",
"author": {"name": "作者", "url_token": "author"},
"created_time": 1234567900,
"like_count": 50,
"child_comment_count": 0,
"is_author": True
}
],
"paging": {"totals": 100}
}
def test_crawl_comments_returns_comments_result(self, crawler, mock_comment_response):
"""crawl_comments 返回 CommentsResult"""
with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser:
mock_browser = MockBrowser.return_value
mock_response = MagicMock()
mock_response.status = 200
mock_response.json.return_value = mock_comment_response
mock_browser.page.request.get.return_value = mock_response
result = crawler.crawl_comments("answers", 12345)
assert isinstance(result, CommentsResult)
assert result.content_type == "answers"
assert result.content_id == 12345
def test_comments_are_hot_order(self, crawler, mock_comment_response):
"""评论使用热门排序"""
with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser:
mock_browser = MockBrowser.return_value
mock_response = MagicMock()
mock_response.status = 200
mock_response.json.return_value = mock_comment_response
mock_browser.page.request.get.return_value = mock_response
crawler.crawl_comments("answers", 12345)
# 验证 API URL 包含 order=hot
call_args = mock_browser.page.request.get.call_args
assert "order=hot" in call_args[0][0]
def test_comment_has_required_fields(self, crawler, mock_comment_response):
"""每条评论包含必要字段"""
with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser:
mock_browser = MockBrowser.return_value
mock_response = MagicMock()
mock_response.status = 200
mock_response.json.return_value = mock_comment_response
mock_browser.page.request.get.return_value = mock_response
result = crawler.crawl_comments("answers", 12345)
assert len(result.comments) == 2
comment = result.comments[0]
assert comment.content == "这是一条热门评论"
assert comment.author.name == "用户A"
assert comment.like_count == 100
def test_no_nested_replies(self, crawler, mock_comment_response):
"""不展开嵌套回复"""
with patch.object(crawler, '_get_cookie', return_value="test_cookie"):
with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser:
mock_browser = MockBrowser.return_value
mock_response = MagicMock()
mock_response.status = 200
mock_response.json.return_value = mock_comment_response
mock_browser.page.request.get.return_value = mock_response
result = crawler.crawl_comments("answers", 12345)
# 只返回第一层评论，不包含子评论
assert len(result.comments) == 2
# child_count 记录子评论数但不展开
assert result.comments[0].child_count == 5
def test_include_comments_parameter(self, crawler):
"""include_comments=False 不爬取评论"""
result = crawler.crawl_comments("answers", 12345, include_comments=False)
assert result.comments == []
assert result.total_count == 0
集成验证:
from kb_workflow.crawler import ZhihuCrawler
crawler = ZhihuCrawler()
result = crawler.crawl_answer("https://www.zhihu.com/question/123/answer/456", include_comments=True)
assert result.comments is not None
<success_criteria>
- CommentData 模型定义完成
- crawl_comments() 方法实现完成
- 评论使用热门排序 (order=hot)
- 只爬取第一层评论，不展开嵌套
- include_comments 参数控制评论爬取
- 所有测试通过 </success_criteria>