feat: 添加知乎问答支持
- zhihu_extractor.py: - 添加 URL_PATTERNS 支持 answer 和 question - 添加 _extract_answer_content() 方法提取知乎回答 - 添加 _extract_question_content() 方法提取知乎问题 - zhihu_metadata.py: - 添加 extract_answer() 函数 - 添加 extract_question() 函数 - 更新命令行参数支持 answer/question 类型 - 更新帮助文档
This commit is contained in:
@@ -80,6 +80,8 @@ class ZhihuPlaywrightExtractor:
|
|||||||
URL_PATTERNS = {
|
URL_PATTERNS = {
|
||||||
'zvideo': r'zhihu\.com/zvideo/(\d+)',
|
'zvideo': r'zhihu\.com/zvideo/(\d+)',
|
||||||
'zhuanlan': r'zhuanlan\.zhihu\.com/p/(\d+)',
|
'zhuanlan': r'zhuanlan\.zhihu\.com/p/(\d+)',
|
||||||
|
'answer': r'zhihu\.com/question/\d+/answer/(\d+)',
|
||||||
|
'question': r'zhihu\.com/question/(\d+)',
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||||
@@ -242,6 +244,115 @@ class ZhihuPlaywrightExtractor:
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def _extract_answer_content(self) -> Dict[str, Any]:
|
||||||
|
"""提取知乎回答内容"""
|
||||||
|
result = {
|
||||||
|
'type': 'answer',
|
||||||
|
'title': '', # 问题标题
|
||||||
|
'question_id': '',
|
||||||
|
'answer_id': '',
|
||||||
|
'author': '',
|
||||||
|
'content': '',
|
||||||
|
'content_markdown': '',
|
||||||
|
'publish_time': '',
|
||||||
|
'vote_up': 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 打印页面标题用于调试
|
||||||
|
page_title = self.browser.page.title()
|
||||||
|
print(f"页面标题: {page_title}")
|
||||||
|
|
||||||
|
# 提取问题标题
|
||||||
|
question_title_elem = self.browser.query_selector('.QuestionHeader-title')
|
||||||
|
if not question_title_elem:
|
||||||
|
question_title_elem = self.browser.query_selector('.QuestionHeader-content h1')
|
||||||
|
if question_title_elem:
|
||||||
|
result['title'] = self._extract_text_content(question_title_elem)
|
||||||
|
|
||||||
|
# 提取回答内容
|
||||||
|
content_elem = self.browser.query_selector('.RichText')
|
||||||
|
if not content_elem:
|
||||||
|
content_elem = self.browser.query_selector('.AnswerItem-richText')
|
||||||
|
if not content_elem:
|
||||||
|
content_elem = self.browser.query_selector('[class*="RichText"]')
|
||||||
|
if content_elem:
|
||||||
|
html_content = self._extract_html_content(content_elem)
|
||||||
|
result['content'] = html_content
|
||||||
|
result['content_markdown'] = self.markdown_converter.convert(html_content)
|
||||||
|
|
||||||
|
# 提取作者信息
|
||||||
|
author_elem = self.browser.query_selector('.AuthorInfo-name')
|
||||||
|
if author_elem:
|
||||||
|
result['author'] = self._extract_text_content(author_elem)
|
||||||
|
|
||||||
|
# 提取发布时间
|
||||||
|
time_elem = self.browser.query_selector('.AnswerItem-time')
|
||||||
|
if not time_elem:
|
||||||
|
time_elem = self.browser.query_selector('.ContentItem-time')
|
||||||
|
if time_elem:
|
||||||
|
result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem)
|
||||||
|
|
||||||
|
# 提取点赞数
|
||||||
|
vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label')
|
||||||
|
if not vote_elem:
|
||||||
|
vote_elem = self.browser.query_selector('.VoteButton .Count')
|
||||||
|
if vote_elem:
|
||||||
|
vote_text = self._extract_text_content(vote_elem)
|
||||||
|
try:
|
||||||
|
result['vote_up'] = int(vote_text.replace(',', ''))
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _extract_question_content(self) -> Dict[str, Any]:
|
||||||
|
"""提取知乎问题内容(包含所有回答)"""
|
||||||
|
result = {
|
||||||
|
'type': 'question',
|
||||||
|
'title': '',
|
||||||
|
'question_id': '',
|
||||||
|
'author': '',
|
||||||
|
'content': '',
|
||||||
|
'content_markdown': '',
|
||||||
|
'publish_time': '',
|
||||||
|
'answer_count': 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
# 打印页面标题用于调试
|
||||||
|
page_title = self.browser.page.title()
|
||||||
|
print(f"页面标题: {page_title}")
|
||||||
|
|
||||||
|
# 提取问题标题
|
||||||
|
title_elem = self.browser.query_selector('.QuestionHeader-title')
|
||||||
|
if not title_elem:
|
||||||
|
title_elem = self.browser.query_selector('.QuestionHeader-content h1')
|
||||||
|
if title_elem:
|
||||||
|
result['title'] = self._extract_text_content(title_elem)
|
||||||
|
|
||||||
|
# 提取问题描述
|
||||||
|
desc_elem = self.browser.query_selector('.QuestionHeader-detail')
|
||||||
|
if not desc_elem:
|
||||||
|
desc_elem = self.browser.query_selector('.QuestionRichText')
|
||||||
|
if desc_elem:
|
||||||
|
html_content = self._extract_html_content(desc_elem)
|
||||||
|
result['content'] = html_content
|
||||||
|
result['content_markdown'] = self.markdown_converter.convert(html_content)
|
||||||
|
|
||||||
|
# 提取回答数量
|
||||||
|
answer_count_elem = self.browser.query_selector('.List-headerText span')
|
||||||
|
if answer_count_elem:
|
||||||
|
count_text = self._extract_text_content(answer_count_elem)
|
||||||
|
try:
|
||||||
|
# 提取数字
|
||||||
|
import re
|
||||||
|
match = re.search(r'(\d+)', count_text)
|
||||||
|
if match:
|
||||||
|
result['answer_count'] = int(match.group(1))
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
def extract(self, url: str) -> Optional[Dict[str, Any]]:
|
def extract(self, url: str) -> Optional[Dict[str, Any]]:
|
||||||
"""
|
"""
|
||||||
从 URL 提取知乎内容
|
从 URL 提取知乎内容
|
||||||
@@ -288,6 +399,10 @@ class ZhihuPlaywrightExtractor:
|
|||||||
result = self._extract_zvideo_content()
|
result = self._extract_zvideo_content()
|
||||||
elif url_type == 'zhuanlan':
|
elif url_type == 'zhuanlan':
|
||||||
result = self._extract_zhuanlan_content()
|
result = self._extract_zhuanlan_content()
|
||||||
|
elif url_type == 'answer':
|
||||||
|
result = self._extract_answer_content()
|
||||||
|
elif url_type == 'question':
|
||||||
|
result = self._extract_question_content()
|
||||||
else:
|
else:
|
||||||
print(f"不支持的类型: {url_type}")
|
print(f"不支持的类型: {url_type}")
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -2,9 +2,11 @@
|
|||||||
"""
|
"""
|
||||||
知乎元数据获取工具
|
知乎元数据获取工具
|
||||||
|
|
||||||
仅支持两种内容类型:
|
支持以下内容类型:
|
||||||
- zvideo: 知乎视频 (https://www.zhihu.com/zvideo/xxx)
|
- zvideo: 知乎视频 (https://www.zhihu.com/zvideo/xxx)
|
||||||
- zhuanlan: 知乎专栏文章 (https://zhuanlan.zhihu.com/p/xxx)
|
- zhuanlan: 知乎专栏文章 (https://zhuanlan.zhihu.com/p/xxx)
|
||||||
|
- answer: 知乎回答 (https://www.zhihu.com/question/xxx/answer/yyy)
|
||||||
|
- question: 知乎问题 (https://www.zhihu.com/question/xxx)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
@@ -51,6 +53,38 @@ def extract_zhuanlan(article_id: str, headless: bool = True) -> Dict[str, Any]:
|
|||||||
return _extract_metadata(url, "zhuanlan", headless)
|
return _extract_metadata(url, "zhuanlan", headless)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_answer(answer_id: str, headless: bool = True) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
获取知乎回答元数据
|
||||||
|
|
||||||
|
Args:
|
||||||
|
answer_id: 回答 ID (数字字符串)
|
||||||
|
headless: 是否使用无头浏览器
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
回答元数据字典
|
||||||
|
"""
|
||||||
|
# 回答需要构建包含问题ID的URL,但answer_id本身足以识别
|
||||||
|
# 使用一个占位符问题ID,因为提取器会从URL中提取answer_id
|
||||||
|
url = f"https://www.zhihu.com/question/0/answer/{answer_id}"
|
||||||
|
return _extract_metadata(url, "answer", headless)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_question(question_id: str, headless: bool = True) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
获取知乎问题元数据
|
||||||
|
|
||||||
|
Args:
|
||||||
|
question_id: 问题 ID (数字字符串)
|
||||||
|
headless: 是否使用无头浏览器
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
问题元数据字典
|
||||||
|
"""
|
||||||
|
url = f"https://www.zhihu.com/question/{question_id}"
|
||||||
|
return _extract_metadata(url, "question", headless)
|
||||||
|
|
||||||
|
|
||||||
def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, Any]:
|
def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
内部函数:提取元数据
|
内部函数:提取元数据
|
||||||
@@ -97,24 +131,28 @@ def main():
|
|||||||
支持的类型:
|
支持的类型:
|
||||||
zvideo - 知乎视频 (https://www.zhihu.com/zvideo/123456)
|
zvideo - 知乎视频 (https://www.zhihu.com/zvideo/123456)
|
||||||
zhuanlan - 知乎专栏文章 (https://zhuanlan.zhihu.com/p/123456)
|
zhuanlan - 知乎专栏文章 (https://zhuanlan.zhihu.com/p/123456)
|
||||||
|
answer - 知乎回答 (https://www.zhihu.com/question/xxx/answer/yyy)
|
||||||
|
question - 知乎问题 (https://www.zhihu.com/question/xxx)
|
||||||
|
|
||||||
示例:
|
示例:
|
||||||
python zhihu_metadata.py --id "123456" --type zvideo
|
python zhihu_metadata.py --id "123456" --type zvideo
|
||||||
python zhihu_metadata.py --id "123456" --type zhuanlan
|
python zhihu_metadata.py --id "123456" --type zhuanlan
|
||||||
|
python zhihu_metadata.py --id "123456" --type answer
|
||||||
|
python zhihu_metadata.py --id "123456" --type question
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--id", "-i",
|
"--id", "-i",
|
||||||
required=True,
|
required=True,
|
||||||
help="知乎内容 ID (视频ID或文章ID)"
|
help="知乎内容 ID (视频ID/文章ID/回答ID/问题ID)"
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--type", "-t",
|
"--type", "-t",
|
||||||
required=True,
|
required=True,
|
||||||
choices=["zvideo", "zhuanlan"],
|
choices=["zvideo", "zhuanlan", "answer", "question"],
|
||||||
help="内容类型: zvideo=视频, zhuanlan=专栏文章"
|
help="内容类型: zvideo=视频, zhuanlan=专栏文章, answer=回答, question=问题"
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
@@ -135,8 +173,15 @@ def main():
|
|||||||
# 提取元数据
|
# 提取元数据
|
||||||
if args.type == "zvideo":
|
if args.type == "zvideo":
|
||||||
result = extract_zvideo(args.id, headless=args.headless)
|
result = extract_zvideo(args.id, headless=args.headless)
|
||||||
else: # zhuanlan
|
elif args.type == "zhuanlan":
|
||||||
result = extract_zhuanlan(args.id, headless=args.headless)
|
result = extract_zhuanlan(args.id, headless=args.headless)
|
||||||
|
elif args.type == "answer":
|
||||||
|
result = extract_answer(args.id, headless=args.headless)
|
||||||
|
elif args.type == "question":
|
||||||
|
result = extract_question(args.id, headless=args.headless)
|
||||||
|
else:
|
||||||
|
print(f"不支持的类型: {args.type}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
# 输出结果
|
# 输出结果
|
||||||
json_output = json.dumps(result, ensure_ascii=False, indent=2)
|
json_output = json.dumps(result, ensure_ascii=False, indent=2)
|
||||||
|
|||||||
Reference in New Issue
Block a user