feat(08-02): implement crawl_comments method with hot sorting
- Add COMMENT_API_TEMPLATE for comment API URL - Implement crawl_comments() returning CommentsResult - Use order=hot for popular comments sorting - Only fetch first-level comments (no nested replies) - Add include_comments parameter to crawl_answer() - Export CommentAuthor, CommentData, CommentsResult in __init__.py Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -8,8 +8,17 @@ from .models import (
|
||||
ContentMetadata,
|
||||
AnswerData,
|
||||
ArticleData,
|
||||
CommentAuthor,
|
||||
CommentData,
|
||||
CommentsResult,
|
||||
CrawlResult,
|
||||
)
|
||||
from .config import (
|
||||
ProxyConfig,
|
||||
CrawlerConfig,
|
||||
get_crawler_config,
|
||||
reset_crawler_config,
|
||||
)
|
||||
from .zhihu_crawler import ZhihuCrawler, ZhihuApiInterceptor
|
||||
|
||||
__all__ = [
|
||||
@@ -17,7 +26,14 @@ __all__ = [
|
||||
"ContentMetadata",
|
||||
"AnswerData",
|
||||
"ArticleData",
|
||||
"CommentAuthor",
|
||||
"CommentData",
|
||||
"CommentsResult",
|
||||
"CrawlResult",
|
||||
"ProxyConfig",
|
||||
"CrawlerConfig",
|
||||
"get_crawler_config",
|
||||
"reset_crawler_config",
|
||||
"ZhihuCrawler",
|
||||
"ZhihuApiInterceptor",
|
||||
]
|
||||
|
||||
@@ -17,6 +17,9 @@ from .models import (
|
||||
ArticleData,
|
||||
ContentMetadata,
|
||||
AuthorInfo,
|
||||
CommentData,
|
||||
CommentAuthor,
|
||||
CommentsResult,
|
||||
CrawlResult,
|
||||
)
|
||||
|
||||
@@ -68,10 +71,16 @@ class ZhihuCrawler:
|
||||
Attributes:
|
||||
ANSWER_PATTERN: Regex pattern for answer URLs
|
||||
ARTICLE_PATTERN: Regex pattern for zhuanlan article URLs
|
||||
COMMENT_API_TEMPLATE: Template for comment API URL
|
||||
"""
|
||||
|
||||
ANSWER_PATTERN = r"zhihu\.com/question/(\d+)/answer/(\d+)"
|
||||
ARTICLE_PATTERN = r"zhuanlan\.zhihu\.com/p/(\d+)"
|
||||
COMMENT_API_TEMPLATE = (
|
||||
"https://www.zhihu.com/api/v4/comment_v5/"
|
||||
"{content_type}/{content_id}/comments"
|
||||
"?order=hot&limit=20"
|
||||
)
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||
"""Initialize ZhihuCrawler.
|
||||
@@ -174,14 +183,108 @@ class ZhihuCrawler:
|
||||
metadata=self._parse_metadata(data)
|
||||
)
|
||||
|
||||
def crawl_answer(self, url: str) -> CrawlResult:
|
||||
def crawl_comments(
|
||||
self,
|
||||
content_type: str,
|
||||
content_id: int,
|
||||
include_comments: bool = True
|
||||
) -> CommentsResult:
|
||||
"""Crawl hot comments for a Zhihu content.
|
||||
|
||||
Fetches first-level comments using the hot sorting API.
|
||||
Does not expand nested replies.
|
||||
|
||||
Args:
|
||||
content_type: Content type ("answers" or "articles")
|
||||
content_id: Content ID
|
||||
include_comments: Whether to fetch comments (default True)
|
||||
|
||||
Returns:
|
||||
CommentsResult with list of CommentData
|
||||
"""
|
||||
if not include_comments:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
try:
|
||||
self.browser = PlaywrightBrowser(
|
||||
headless=self.headless,
|
||||
timeout=self.timeout
|
||||
)
|
||||
self.browser.start(cookie=self._get_cookie())
|
||||
|
||||
# Build comment API URL
|
||||
api_url = self.COMMENT_API_TEMPLATE.format(
|
||||
content_type=content_type,
|
||||
content_id=content_id
|
||||
)
|
||||
|
||||
# Use Playwright's request API to directly call the API
|
||||
response = self.browser.page.request.get(api_url)
|
||||
|
||||
if response.status != 200:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
data = response.json()
|
||||
comments = []
|
||||
|
||||
for item in data.get("data", []):
|
||||
author_data = item.get("author", {})
|
||||
comment = CommentData(
|
||||
id=item.get("id", 0),
|
||||
content=item.get("content", ""),
|
||||
author=CommentAuthor(
|
||||
name=author_data.get("name", ""),
|
||||
url_token=author_data.get("url_token")
|
||||
) if author_data else None,
|
||||
created_time=item.get("created_time"),
|
||||
like_count=item.get("like_count", 0),
|
||||
child_count=item.get("child_comment_count", 0),
|
||||
is_author=item.get("is_author", False)
|
||||
)
|
||||
comments.append(comment)
|
||||
|
||||
# Get total count from paging
|
||||
total_count = data.get("paging", {}).get("totals", len(comments))
|
||||
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=comments,
|
||||
total_count=total_count
|
||||
)
|
||||
|
||||
except Exception:
|
||||
return CommentsResult(
|
||||
content_type=content_type,
|
||||
content_id=content_id,
|
||||
comments=[],
|
||||
total_count=0
|
||||
)
|
||||
|
||||
finally:
|
||||
if self.browser:
|
||||
self.browser.close()
|
||||
|
||||
def crawl_answer(self, url: str, include_comments: bool = False) -> CrawlResult:
|
||||
"""Crawl Zhihu answer.
|
||||
|
||||
Args:
|
||||
url: Zhihu answer URL (e.g., https://www.zhihu.com/question/123/answer/456)
|
||||
include_comments: Whether to include hot comments (default False)
|
||||
|
||||
Returns:
|
||||
CrawlResult with AnswerData if successful, error message otherwise
|
||||
CrawlResult with AnswerData if successful, error message otherwise.
|
||||
If include_comments=True and successful, comments field will be populated.
|
||||
"""
|
||||
match = re.search(self.ANSWER_PATTERN, url)
|
||||
if not match:
|
||||
@@ -219,12 +322,27 @@ class ZhihuCrawler:
|
||||
question_id=question_id
|
||||
)
|
||||
|
||||
return CrawlResult(
|
||||
result = CrawlResult(
|
||||
success=True,
|
||||
content_type="answer",
|
||||
data=result_data
|
||||
)
|
||||
|
||||
# Fetch comments if requested
|
||||
if include_comments:
|
||||
# Need to close current browser first as crawl_comments creates its own
|
||||
self.browser.close()
|
||||
self.browser = None
|
||||
|
||||
comments_result = self.crawl_comments(
|
||||
content_type="answers",
|
||||
content_id=answer_id,
|
||||
include_comments=True
|
||||
)
|
||||
result.comments = comments_result.comments
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
return CrawlResult(success=False, error=str(e))
|
||||
|
||||
|
||||
@@ -140,25 +140,17 @@ class TestCommentCrawler:
|
||||
mock_browser = MockBrowser.return_value
|
||||
# Setup for answer crawl
|
||||
mock_browser.page.route = MagicMock()
|
||||
mock_browser.page.request.get.return_value.status = 200
|
||||
mock_browser.page.request.get.return_value.json.return_value = mock_comment_response
|
||||
mock_browser.goto = MagicMock()
|
||||
mock_browser.close = MagicMock()
|
||||
|
||||
# We need to simulate the interceptor capturing answer data
|
||||
# For simplicity, patch the internal method
|
||||
with patch.object(crawler, '_crawl_answer_internal') as mock_internal:
|
||||
from kb_workflow.crawler.models import CrawlResult, AnswerData, ContentMetadata, AuthorInfo
|
||||
mock_internal.return_value = CrawlResult(
|
||||
success=True,
|
||||
content_type="answer",
|
||||
data=AnswerData(
|
||||
id=456,
|
||||
question_id=123,
|
||||
question_title="Test Question",
|
||||
content="<p>Content</p>",
|
||||
excerpt="Excerpt",
|
||||
metadata=ContentMetadata(author=AuthorInfo(name="Author"))
|
||||
)
|
||||
)
|
||||
# Create interceptor that will capture answer data
|
||||
with patch('kb_workflow.crawler.zhihu_crawler.ZhihuApiInterceptor') as MockInterceptor:
|
||||
mock_interceptor = MockInterceptor.return_value
|
||||
mock_interceptor.captured_data = {"answer": mock_answer_data}
|
||||
|
||||
# Mock crawl_comments to return test data
|
||||
with patch.object(crawler, 'crawl_comments') as mock_comments:
|
||||
mock_comments.return_value = CommentsResult(
|
||||
content_type="answers",
|
||||
@@ -167,6 +159,20 @@ class TestCommentCrawler:
|
||||
total_count=1
|
||||
)
|
||||
|
||||
# This test verifies the integration point exists
|
||||
# The actual implementation will call crawl_comments when include_comments=True
|
||||
pass
|
||||
result = crawler.crawl_answer(
|
||||
"https://www.zhihu.com/question/123/answer/456",
|
||||
include_comments=True
|
||||
)
|
||||
|
||||
# Verify crawl_comments was called with correct params
|
||||
mock_comments.assert_called_once_with(
|
||||
content_type="answers",
|
||||
content_id=456,
|
||||
include_comments=True
|
||||
)
|
||||
|
||||
# Verify result includes comments
|
||||
assert result.success
|
||||
assert result.comments is not None
|
||||
assert len(result.comments) == 1
|
||||
assert result.comments[0].content == "Test comment"
|
||||
|
||||
Reference in New Issue
Block a user