From 56907e31f53955b7b164d5cf04e955acb57865d0 Mon Sep 17 00:00:00 2001 From: OpenClaw Agent Date: Tue, 10 Feb 2026 14:46:39 +0800 Subject: [PATCH] refactor(kb-zhihu-metadata): Simplify to only support zvideo and zhuanlan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove question, answer, user types (not needed for knowledge base) - Keep only zvideo (知乎视频) and zhuanlan (知乎专栏文章) - Update URL patterns and extraction logic - Simplified CLI interface --- skills/kb-zhihu-metadata/zhihu_extractor.py | 166 +++++++------------- skills/kb-zhihu-metadata/zhihu_metadata.py | 163 ++++--------------- 2 files changed, 87 insertions(+), 242 deletions(-) diff --git a/skills/kb-zhihu-metadata/zhihu_extractor.py b/skills/kb-zhihu-metadata/zhihu_extractor.py index 2baf648..f917f79 100644 --- a/skills/kb-zhihu-metadata/zhihu_extractor.py +++ b/skills/kb-zhihu-metadata/zhihu_extractor.py @@ -78,9 +78,8 @@ class ZhihuPlaywrightExtractor: """知乎内容提取器 - 使用 Playwright""" URL_PATTERNS = { - 'answer': r'zhihu\.com/question/\d+/answer/(\d+)', - 'article': r'zhuanlan\.zhihu\.com/p/(\d+)', - 'question': r'zhihu\.com/question/(\d+)(?:$|/)', + 'zvideo': r'zhihu\.com/zvideo/(\d+)', + 'zhuanlan': r'zhuanlan\.zhihu\.com/p/(\d+)', } def __init__(self, headless: bool = True, timeout: int = 30): @@ -133,10 +132,10 @@ class ZhihuPlaywrightExtractor: return '' return element.inner_html() if hasattr(element, 'inner_html') else '' - def _extract_answer_content(self) -> Dict[str, Any]: - """提取回答内容""" + def _extract_zvideo_content(self) -> Dict[str, Any]: + """提取知乎视频内容""" result = { - 'type': 'answer', + 'type': 'zvideo', 'title': '', 'author': '', 'content': '', @@ -149,72 +148,49 @@ class ZhihuPlaywrightExtractor: page_title = self.browser.page.title() print(f"页面标题: {page_title}") - # 尝试保存页面快照用于调试 - try: - import os - debug_dir = '/home/admin/.openclaw/workspace/debug' - os.makedirs(debug_dir, exist_ok=True) - self.browser.page.screenshot(path=os.path.join(debug_dir, 'zhihu_screenshot.png')) - with open(os.path.join(debug_dir, 'zhihu_page.html'), 'w', encoding='utf-8') as f: - f.write(self.browser.page.content()) - print(f"页面快照已保存到 {debug_dir}") - except Exception as e: - print(f"保存页面快照时出错: {e}") - - # 尝试多个可能的选择器 - selectors = { - 'title': ['h1.QuestionHeader-title', 'h1', '.QuestionHeader-main'], - 'author': ['.AuthorInfo-name', '.AuthorInfo .Text', '[itemprop="name"]'], - 'content': ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist] article'], - 'time': ['.ContentItem-time', '[itemprop="dateCreated"]'], - 'vote': ['.VoteButton--up .VoteButton-label', '.VoteButton--up'] - } - - for selector in selectors['title']: - title_elem = self.browser.query_selector(selector) + # 提取视频标题 + title_elem = self.browser.query_selector('.VideoPlaylistInfo-title') + if title_elem: + result['title'] = self._extract_text_content(title_elem) + + # 备用：尝试其他选择器 + if not result['title']: + title_elem = self.browser.query_selector('h1') if title_elem: result['title'] = self._extract_text_content(title_elem) - print(f"找到标题（使用选择器 {selector}）: {result['title'][:50]}...") - break - for selector in selectors['author']: - author_elem = self.browser.query_selector(selector) - if author_elem: - result['author'] = self._extract_text_content(author_elem) - print(f"找到作者（使用选择器 {selector}）: {result['author']}") - break + # 提取作者信息 + author_elem = self.browser.query_selector('.AuthorInfo-name') + if author_elem: + result['author'] = self._extract_text_content(author_elem) - for selector in selectors['content']: - content_elem = self.browser.query_selector(selector) - if content_elem: - html_content = self._extract_html_content(content_elem) - result['content'] = html_content - result['content_markdown'] = self.markdown_converter.convert(html_content) - print(f"找到内容（使用选择器 {selector}），长度: {len(result['content_markdown'])}") - break + # 提取视频描述 + desc_elem = self.browser.query_selector('.Video-desc') + if desc_elem: + html_content = self._extract_html_content(desc_elem) + result['content'] = html_content + result['content_markdown'] = self.markdown_converter.convert(html_content) - for selector in selectors['time']: - time_elem = self.browser.query_selector(selector) - if time_elem: - result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem) - break + # 提取发布时间 + time_elem = self.browser.query_selector('.Video-meta time') + if time_elem: + result['publish_time'] = time_elem.get_attribute('datetime') or self._extract_text_content(time_elem) - for selector in selectors['vote']: - vote_elem = self.browser.query_selector(selector) - if vote_elem: - vote_text = self._extract_text_content(vote_elem) - try: - result['vote_up'] = int(vote_text.replace(',', '')) - except (ValueError, AttributeError): - pass - break + # 提取点赞数 + vote_elem = self.browser.query_selector('.VideoLikeButton-count') + if vote_elem: + vote_text = self._extract_text_content(vote_elem) + try: + result['vote_up'] = int(vote_text.replace(',', '')) + except (ValueError, AttributeError): + pass return result - def _extract_article_content(self) -> Dict[str, Any]: - """提取文章内容""" + def _extract_zhuanlan_content(self) -> Dict[str, Any]: + """提取知乎专栏文章内容""" result = { - 'type': 'article', + 'type': 'zhuanlan', 'title': '', 'author': '', 'content': '', @@ -223,11 +199,21 @@ class ZhihuPlaywrightExtractor: 'vote_up': 0, } + # 打印页面标题用于调试 + page_title = self.browser.page.title() + print(f"页面标题: {page_title}") + # 提取文章标题 title_elem = self.browser.query_selector('.Post-Title') if title_elem: result['title'] = self._extract_text_content(title_elem) + # 备用 + if not result['title']: + title_elem = self.browser.query_selector('h1') + if title_elem: + result['title'] = self._extract_text_content(title_elem) + # 提取作者信息 author_elem = self.browser.query_selector('.AuthorInfo-name') if author_elem: @@ -256,37 +242,6 @@ class ZhihuPlaywrightExtractor: return result - def _extract_question_content(self) -> Dict[str, Any]: - """提取问题内容（问题页本身）""" - result = { - 'type': 'question', - 'title': '', - 'author': '', - 'content': '', - 'content_markdown': '', - 'publish_time': '', - 'vote_up': 0, - } - - # 提取问题标题 - title_elem = self.browser.query_selector('h1.QuestionHeader-title') - if title_elem: - result['title'] = self._extract_text_content(title_elem) - - # 提取问题详情 - detail_elem = self.browser.query_selector('.QuestionHeader-detail') - if detail_elem: - html_content = self._extract_html_content(detail_elem) - result['content'] = html_content - result['content_markdown'] = self.markdown_converter.convert(html_content) - - # 问题的作者通常是提问者 - author_elem = self.browser.query_selector('.AuthorInfo-name') - if author_elem: - result['author'] = self._extract_text_content(author_elem) - - return result - def extract(self, url: str) -> Optional[Dict[str, Any]]: """ 从 URL 提取知乎内容 @@ -297,7 +252,7 @@ class ZhihuPlaywrightExtractor: Returns: 包含提取内容的字典，结构如下: { - 'type': 'answer' | 'article' | 'question', + 'type': 'zvideo' | 'zhuanlan', 'title': str, 'author': str, 'content': str (HTML), @@ -328,25 +283,20 @@ class ZhihuPlaywrightExtractor: import time time.sleep(3) # 等待 JavaScript 执行 - # 尝试多个可能的选择器 - if url_type == 'answer': - selectors = ['.RichContent-inner', '.RichContent', '.List-itemText', '[data-zop-feedlist]'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_answer_content() - elif url_type == 'article': - selectors = ['.Post-RichText', '.Post-Title'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_article_content() - else: # question - selectors = ['.QuestionHeader-title', 'h1'] - print(f"尝试查找选择器: {selectors}") - result = self._extract_question_content() + # 根据类型提取内容 + if url_type == 'zvideo': + result = self._extract_zvideo_content() + elif url_type == 'zhuanlan': + result = self._extract_zhuanlan_content() + else: + print(f"不支持的类型: {url_type}") + return None # 添加额外信息 result['url'] = url result['content_id'] = self.extract_id(url) - print(f"提取成功! 标题: {result['title'][:50]}...") + print(f"提取成功! 标题: {result.get('title', '')[:50]}...") return result except Exception as e: diff --git a/skills/kb-zhihu-metadata/zhihu_metadata.py b/skills/kb-zhihu-metadata/zhihu_metadata.py index d0a5c2c..fcff2b1 100644 --- a/skills/kb-zhihu-metadata/zhihu_metadata.py +++ b/skills/kb-zhihu-metadata/zhihu_metadata.py @@ -2,7 +2,9 @@ """ 知乎元数据获取工具 -支持获取知乎问题、回答、用户的元数据信息。 +仅支持两种内容类型： +- zvideo: 知乎视频 (https://www.zhihu.com/zvideo/xxx) +- zhuanlan: 知乎专栏文章 (https://zhuanlan.zhihu.com/p/xxx) """ import argparse @@ -19,52 +21,34 @@ if current_dir not in sys.path: from zhihu_extractor import ZhihuPlaywrightExtractor -def extract_question(question_id: str, headless: bool = True) -> Dict[str, Any]: +def extract_zvideo(video_id: str, headless: bool = True) -> Dict[str, Any]: """ - 获取知乎问题元数据 + 获取知乎视频元数据 Args: - question_id: 问题 ID (数字字符串) + video_id: 视频 ID (数字字符串) headless: 是否使用无头浏览器 Returns: - 问题元数据字典 + 视频元数据字典 """ - url = f"https://www.zhihu.com/question/{question_id}" - return _extract_metadata(url, "question", headless) + url = f"https://www.zhihu.com/zvideo/{video_id}" + return _extract_metadata(url, "zvideo", headless) -def extract_answer(answer_id: str, headless: bool = True) -> Dict[str, Any]: +def extract_zhuanlan(article_id: str, headless: bool = True) -> Dict[str, Any]: """ - 获取知乎回答元数据 + 获取知乎专栏文章元数据 Args: - answer_id: 回答 ID (数字字符串) + article_id: 文章 ID (数字字符串) headless: 是否使用无头浏览器 Returns: - 回答元数据字典 + 文章元数据字典 """ - # 回答 URL 格式: https://www.zhihu.com/question/{question_id}/answer/{answer_id} - # 需要先获取问题ID，这里简化处理，尝试从回答ID构建URL - # 实际上知乎回答URL需要问题ID，这里用模拟URL - url = f"https://www.zhihu.com/question/0/answer/{answer_id}" - return _extract_metadata(url, "answer", headless) - - -def extract_user(url_token: str, headless: bool = True) -> Dict[str, Any]: - """ - 获取知乎用户信息 - - Args: - url_token: 用户 URL token - headless: 是否使用无头浏览器 - - Returns: - 用户信息字典 - """ - url = f"https://www.zhihu.com/people/{url_token}" - return _extract_user_metadata(url, headless) + url = f"https://zhuanlan.zhihu.com/p/{article_id}" + return _extract_metadata(url, "zhuanlan", headless) def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, Any]: @@ -73,7 +57,7 @@ def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, Args: url: 知乎 URL - content_type: 内容类型 (question/answer) + content_type: 内容类型 (zvideo/zhuanlan) headless: 是否使用无头浏览器 Returns: @@ -104,120 +88,33 @@ def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, } -def _extract_user_metadata(url: str, headless: bool) -> Dict[str, Any]: - """ - 内部函数：提取用户信息 - - Args: - url: 用户主页 URL - headless: 是否使用无头浏览器 - - Returns: - 用户信息字典 - """ - extractor = ZhihuPlaywrightExtractor(headless=headless) - - try: - extractor.browser.start() - extractor.browser.goto(url, wait_until="networkidle") - - import time - time.sleep(3) - - # 提取用户名 - username = "" - username_elem = extractor.browser.query_selector('.ProfileHeader-name') - if username_elem: - username = username_elem.inner_text() - - # 提取个人简介 - headline = "" - headline_elem = extractor.browser.query_selector('.ProfileHeader-headline') - if headline_elem: - headline = headline_elem.inner_text() - - # 提取简介详情 - description = "" - desc_elem = extractor.browser.query_selector('.ProfileHeader-description') - if desc_elem: - description = desc_elem.inner_text() - - # 提取关注者数 - followers = 0 - follower_elem = extractor.browser.query_selector('.FollowshipCard-followersNum') - if follower_elem: - follower_text = follower_elem.inner_text() - try: - followers = int(follower_text.replace(',', '')) - except ValueError: - pass - - # 提取关注者数 - following = 0 - following_elem = extractor.browser.query_selector('.FollowshipCard-followingsNum') - if following_elem: - following_text = following_elem.inner_text() - try: - following = int(following_text.replace(',', '')) - except ValueError: - pass - - # 提取回答数 - answers = 0 - answers_elem = extractor.browser.query_selector('.Profile-mainColumn .Tabs-item span') - if answers_elem: - answers_text = answers_elem.inner_text() - try: - answers = int(answers_text.replace(',', '')) - except ValueError: - pass - - metadata = { - "type": "user", - "url": url, - "username": username, - "headline": headline, - "description": description, - "followers": followers, - "following": following, - "answers_count": answers, - } - return metadata - - except Exception as e: - return { - "type": "user", - "error": str(e), - "url": url - } - finally: - extractor.browser.close() - - def main(): """命令行入口""" parser = argparse.ArgumentParser( description="知乎元数据获取工具", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" +支持的类型: + zvideo - 知乎视频 (https://www.zhihu.com/zvideo/123456) + zhuanlan - 知乎专栏文章 (https://zhuanlan.zhihu.com/p/123456) + 示例: - python zhihu_metadata.py --id "266309957" --type question - python zhihu_metadata.py --id "2003427891925509507" --type answer - python zhihu_metadata.py --id "example-user" --type user + python zhihu_metadata.py --id "123456" --type zvideo + python zhihu_metadata.py --id "123456" --type zhuanlan """ ) parser.add_argument( "--id", "-i", required=True, - help="知乎内容 ID (问题ID、回答ID或用户URL token)" + help="知乎内容 ID (视频ID或文章ID)" ) parser.add_argument( "--type", "-t", required=True, - choices=["question", "answer", "user"], - help="内容类型" + choices=["zvideo", "zhuanlan"], + help="内容类型: zvideo=视频, zhuanlan=专栏文章" ) parser.add_argument( @@ -236,12 +133,10 @@ def main(): args = parser.parse_args() # 提取元数据 - if args.type == "question": - result = extract_question(args.id, headless=args.headless) - elif args.type == "answer": - result = extract_answer(args.id, headless=args.headless) - else: # user - result = extract_user(args.id, headless=args.headless) + if args.type == "zvideo": + result = extract_zvideo(args.id, headless=args.headless) + else: # zhuanlan + result = extract_zhuanlan(args.id, headless=args.headless) # 输出结果 json_output = json.dumps(result, ensure_ascii=False, indent=2)