添加技能模块
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
# KB Zhihu Metadata Skill
|
||||
|
||||
**kb-zhihu-metadata** - Extract Zhihu question/answer/user metadata.
|
||||
|
||||
## Features
|
||||
|
||||
- Question metadata extraction
|
||||
- Answer metadata extraction
|
||||
- User information extraction
|
||||
- JSON output format
|
||||
- Markdown conversion support
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Question metadata
|
||||
python zhihu_metadata.py --id "266309957" --type question
|
||||
|
||||
# Answer metadata
|
||||
python zhihu_metadata.py --id "2003427891925509507" --type answer
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
"platform": "zhihu",
|
||||
"type": "question",
|
||||
"id": "266309957",
|
||||
"title": "问题标题",
|
||||
"author": "作者名字",
|
||||
"answer_count": 50,
|
||||
"followers": 1000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
cp -r kb-zhihu-metadata/ ~/skills/
|
||||
```
|
||||
@@ -0,0 +1,17 @@
|
||||
from markdownify import MarkdownConverter as _MarkdownConverter
|
||||
|
||||
class MarkdownConverter:
|
||||
"""HTML 转 Markdown 转换器，保留完整格式"""
|
||||
|
||||
def __init__(self):
|
||||
self.converter = _MarkdownConverter(
|
||||
heading_style="ATX",
|
||||
bullets="*,-",
|
||||
code_blocks=True,
|
||||
tables=True,
|
||||
strip=["script", "style", "nav", "footer", "aside"]
|
||||
)
|
||||
|
||||
def convert(self, html: str) -> str:
|
||||
"""转换 HTML 为 Markdown"""
|
||||
return self.converter.convert(html)
|
||||
@@ -0,0 +1,335 @@
|
||||
"""
|
||||
知乎 Playwright 内容提取器
|
||||
使用 Playwright 浏览器自动化工具抓取知乎内容
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Optional, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# 添加当前目录到 Python 路径以支持绝对导入
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
if current_dir not in sys.path:
|
||||
sys.path.insert(0, current_dir)
|
||||
|
||||
try:
|
||||
from .markdown_converter import MarkdownConverter
|
||||
except ImportError:
|
||||
from markdown_converter import MarkdownConverter
|
||||
|
||||
|
||||
class PlaywrightBrowser:
|
||||
"""Playwright 浏览器管理器"""
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||
self.headless = headless
|
||||
self.timeout = timeout * 1000
|
||||
self.browser = None
|
||||
self.context = None
|
||||
self.page = None
|
||||
|
||||
def start(self):
|
||||
"""启动浏览器"""
|
||||
from playwright.sync_api import sync_playwright
|
||||
self.playwright = sync_playwright().start()
|
||||
self.browser = self.playwright.chromium.launch(headless=self.headless)
|
||||
self.context = self.browser.new_context(
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
|
||||
)
|
||||
self.page = self.context.new_page()
|
||||
self.page.set_default_timeout(self.timeout)
|
||||
|
||||
def close(self):
|
||||
"""关闭浏览器"""
|
||||
if self.page:
|
||||
self.page.close()
|
||||
if self.context:
|
||||
self.context.close()
|
||||
if self.browser:
|
||||
self.browser.close()
|
||||
if hasattr(self, 'playwright'):
|
||||
self.playwright.stop()
|
||||
|
||||
def goto(self, url: str, wait_until: str = "networkidle"):
|
||||
"""导航到指定URL"""
|
||||
return self.page.goto(url, wait_until=wait_until)
|
||||
|
||||
def wait_for_selector(self, selector: str, timeout: int = None):
|
||||
"""等待选择器出现"""
|
||||
timeout_ms = timeout * 1000 if timeout else self.timeout
|
||||
return self.page.wait_for_selector(selector, timeout=timeout_ms)
|
||||
|
||||
def query_selector(self, selector: str):
|
||||
"""查询选择器"""
|
||||
return self.page.query_selector(selector)
|
||||
|
||||
def query_selector_all(self, selector: str):
|
||||
"""查询所有匹配的选择器"""
|
||||
return self.page.query_selector_all(selector)
|
||||
|
||||
def evaluate(self, script: str):
|
||||
"""执行 JavaScript 代码"""
|
||||
return self.page.evaluate(script)
|
||||
|
||||
|
||||
class ZhihuPlaywrightExtractor:
|
||||
"""知乎内容提取器 - 使用 Playwright"""
|
||||
|
||||
URL_PATTERNS = {
|
||||
'zvideo': r'zhihu\.com/zvideo/(\d+)',
|
||||
'zhuanlan': r'zhuanlan\.zhihu\.com/p/(\d+)',
|
||||
}
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 30):
|
||||
"""
|
||||
初始化提取器
|
||||
|
||||
Args:
|
||||
headless: 是否使用无头浏览器
|
||||
timeout: 超时时间（秒）
|
||||
"""
|
||||
self.browser = PlaywrightBrowser(headless=headless, timeout=timeout)
|
||||
self.markdown_converter = MarkdownConverter()
|
||||
|
||||
def identify_url_type(self, url: str) -> Optional[str]:
|
||||
"""
|
||||
识别 URL 类型
|
||||
|
||||
Returns:
|
||||
'answer', 'article', 'question' 或 None
|
||||
"""
|
||||
for url_type, pattern in self.URL_PATTERNS.items():
|
||||
if re.search(pattern, url):
|
||||
return url_type
|
||||
return None
|
||||
|
||||
def extract_id(self, url: str) -> Optional[str]:
|
||||
"""
|
||||
从 URL 中提取内容 ID
|
||||
|
||||
Returns:
|
||||
内容 ID 或 None
|
||||
"""
|
||||
url_type = self.identify_url_type(url)
|
||||
if url_type:
|
||||
pattern = self.URL_PATTERNS[url_type]
|
||||
match = re.search(pattern, url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
def _extract_text_content(self, element) -> str:
|
||||
"""提取元素的文本内容"""
|
||||
if element is None:
|
||||
return ''
|
||||
return element.inner_text() if hasattr(element, 'inner_text') else ''
|
||||
|
||||
def _extract_html_content(self, element) -> str:
|
||||
"""提取元素的 HTML 内容"""
|
||||
if element is None:
|
||||
return ''
|
||||
return element.inner_html() if hasattr(element, 'inner_html') else ''
|
||||
|
||||
def _extract_zvideo_content(self) -> Dict[str, Any]:
|
||||
"""提取知乎视频内容"""
|
||||
result = {
|
||||
'type': 'zvideo',
|
||||
'title': '',
|
||||
'author': '',
|
||||
'content': '',
|
||||
'content_markdown': '',
|
||||
'publish_time': '',
|
||||
'vote_up': 0,
|
||||
}
|
||||
|
||||
# 打印页面标题用于调试
|
||||
page_title = self.browser.page.title()
|
||||
print(f"页面标题: {page_title}")
|
||||
|
||||
# 提取视频标题
|
||||
title_elem = self.browser.query_selector('.VideoPlaylistInfo-title')
|
||||
if title_elem:
|
||||
result['title'] = self._extract_text_content(title_elem)
|
||||
|
||||
# 备用：尝试其他选择器
|
||||
if not result['title']:
|
||||
title_elem = self.browser.query_selector('h1')
|
||||
if title_elem:
|
||||
result['title'] = self._extract_text_content(title_elem)
|
||||
|
||||
# 提取作者信息
|
||||
author_elem = self.browser.query_selector('.AuthorInfo-name')
|
||||
if author_elem:
|
||||
result['author'] = self._extract_text_content(author_elem)
|
||||
|
||||
# 提取视频描述
|
||||
desc_elem = self.browser.query_selector('.Video-desc')
|
||||
if desc_elem:
|
||||
html_content = self._extract_html_content(desc_elem)
|
||||
result['content'] = html_content
|
||||
result['content_markdown'] = self.markdown_converter.convert(html_content)
|
||||
|
||||
# 提取发布时间
|
||||
time_elem = self.browser.query_selector('.Video-meta time')
|
||||
if time_elem:
|
||||
result['publish_time'] = time_elem.get_attribute('datetime') or self._extract_text_content(time_elem)
|
||||
|
||||
# 提取点赞数
|
||||
vote_elem = self.browser.query_selector('.VideoLikeButton-count')
|
||||
if vote_elem:
|
||||
vote_text = self._extract_text_content(vote_elem)
|
||||
try:
|
||||
result['vote_up'] = int(vote_text.replace(',', ''))
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _extract_zhuanlan_content(self) -> Dict[str, Any]:
|
||||
"""提取知乎专栏文章内容"""
|
||||
result = {
|
||||
'type': 'zhuanlan',
|
||||
'title': '',
|
||||
'author': '',
|
||||
'content': '',
|
||||
'content_markdown': '',
|
||||
'publish_time': '',
|
||||
'vote_up': 0,
|
||||
}
|
||||
|
||||
# 打印页面标题用于调试
|
||||
page_title = self.browser.page.title()
|
||||
print(f"页面标题: {page_title}")
|
||||
|
||||
# 提取文章标题
|
||||
title_elem = self.browser.query_selector('.Post-Title')
|
||||
if title_elem:
|
||||
result['title'] = self._extract_text_content(title_elem)
|
||||
|
||||
# 备用
|
||||
if not result['title']:
|
||||
title_elem = self.browser.query_selector('h1')
|
||||
if title_elem:
|
||||
result['title'] = self._extract_text_content(title_elem)
|
||||
|
||||
# 提取作者信息
|
||||
author_elem = self.browser.query_selector('.AuthorInfo-name')
|
||||
if author_elem:
|
||||
result['author'] = self._extract_text_content(author_elem)
|
||||
|
||||
# 提取文章内容
|
||||
content_elem = self.browser.query_selector('.Post-RichText')
|
||||
if content_elem:
|
||||
html_content = self._extract_html_content(content_elem)
|
||||
result['content'] = html_content
|
||||
result['content_markdown'] = self.markdown_converter.convert(html_content)
|
||||
|
||||
# 提取发布时间
|
||||
time_elem = self.browser.query_selector('.ContentItem-time')
|
||||
if time_elem:
|
||||
result['publish_time'] = time_elem.get_attribute('data-tooltip') or self._extract_text_content(time_elem)
|
||||
|
||||
# 提取点赞数
|
||||
vote_elem = self.browser.query_selector('.VoteButton--up .VoteButton-label')
|
||||
if vote_elem:
|
||||
vote_text = self._extract_text_content(vote_elem)
|
||||
try:
|
||||
result['vote_up'] = int(vote_text.replace(',', ''))
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def extract(self, url: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
从 URL 提取知乎内容
|
||||
|
||||
Args:
|
||||
url: 知乎 URL
|
||||
|
||||
Returns:
|
||||
包含提取内容的字典，结构如下:
|
||||
{
|
||||
'type': 'zvideo' | 'zhuanlan',
|
||||
'title': str,
|
||||
'author': str,
|
||||
'content': str (HTML),
|
||||
'content_markdown': str (Markdown),
|
||||
'publish_time': str,
|
||||
'vote_up': int,
|
||||
}
|
||||
如果提取失败返回 None
|
||||
"""
|
||||
try:
|
||||
# 识别 URL 类型
|
||||
url_type = self.identify_url_type(url)
|
||||
if not url_type:
|
||||
print(f"无法识别 URL 类型: {url}")
|
||||
return None
|
||||
|
||||
print(f"URL 类型: {url_type}")
|
||||
print(f"启动浏览器...")
|
||||
|
||||
# 启动浏览器
|
||||
self.browser.start()
|
||||
|
||||
print(f"访问页面: {url}")
|
||||
# 访问页面
|
||||
self.browser.goto(url, wait_until="domcontentloaded")
|
||||
|
||||
print("等待页面加载...")
|
||||
import time
|
||||
time.sleep(3) # 等待 JavaScript 执行
|
||||
|
||||
# 根据类型提取内容
|
||||
if url_type == 'zvideo':
|
||||
result = self._extract_zvideo_content()
|
||||
elif url_type == 'zhuanlan':
|
||||
result = self._extract_zhuanlan_content()
|
||||
else:
|
||||
print(f"不支持的类型: {url_type}")
|
||||
return None
|
||||
|
||||
# 添加额外信息
|
||||
result['url'] = url
|
||||
result['content_id'] = self.extract_id(url)
|
||||
|
||||
print(f"提取成功! 标题: {result.get('title', '')[:50]}...")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"提取知乎内容时出错: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return None
|
||||
finally:
|
||||
# 关闭浏览器
|
||||
self.browser.close()
|
||||
|
||||
def __enter__(self):
|
||||
"""上下文管理器入口"""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""上下文管理器出口"""
|
||||
self.browser.close()
|
||||
return False
|
||||
|
||||
|
||||
# 便捷函数
|
||||
def extract_zhihu_content(url: str, headless: bool = True, timeout: int = 30) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
便捷函数：提取知乎内容
|
||||
|
||||
Args:
|
||||
url: 知乎 URL
|
||||
headless: 是否使用无头浏览器
|
||||
timeout: 超时时间（秒）
|
||||
|
||||
Returns:
|
||||
提取的内容字典
|
||||
"""
|
||||
with ZhihuPlaywrightExtractor(headless=headless, timeout=timeout) as extractor:
|
||||
return extractor.extract(url)
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
知乎元数据获取工具
|
||||
|
||||
仅支持两种内容类型：
|
||||
- zvideo: 知乎视频 (https://www.zhihu.com/zvideo/xxx)
|
||||
- zhuanlan: 知乎专栏文章 (https://zhuanlan.zhihu.com/p/xxx)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Optional, Any
|
||||
|
||||
# 添加当前目录到 Python 路径
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
if current_dir not in sys.path:
|
||||
sys.path.insert(0, current_dir)
|
||||
|
||||
from zhihu_extractor import ZhihuPlaywrightExtractor
|
||||
|
||||
|
||||
def extract_zvideo(video_id: str, headless: bool = True) -> Dict[str, Any]:
|
||||
"""
|
||||
获取知乎视频元数据
|
||||
|
||||
Args:
|
||||
video_id: 视频 ID (数字字符串)
|
||||
headless: 是否使用无头浏览器
|
||||
|
||||
Returns:
|
||||
视频元数据字典
|
||||
"""
|
||||
url = f"https://www.zhihu.com/zvideo/{video_id}"
|
||||
return _extract_metadata(url, "zvideo", headless)
|
||||
|
||||
|
||||
def extract_zhuanlan(article_id: str, headless: bool = True) -> Dict[str, Any]:
|
||||
"""
|
||||
获取知乎专栏文章元数据
|
||||
|
||||
Args:
|
||||
article_id: 文章 ID (数字字符串)
|
||||
headless: 是否使用无头浏览器
|
||||
|
||||
Returns:
|
||||
文章元数据字典
|
||||
"""
|
||||
url = f"https://zhuanlan.zhihu.com/p/{article_id}"
|
||||
return _extract_metadata(url, "zhuanlan", headless)
|
||||
|
||||
|
||||
def _extract_metadata(url: str, content_type: str, headless: bool) -> Dict[str, Any]:
|
||||
"""
|
||||
内部函数：提取元数据
|
||||
|
||||
Args:
|
||||
url: 知乎 URL
|
||||
content_type: 内容类型 (zvideo/zhuanlan)
|
||||
headless: 是否使用无头浏览器
|
||||
|
||||
Returns:
|
||||
元数据字典
|
||||
"""
|
||||
extractor = ZhihuPlaywrightExtractor(headless=headless)
|
||||
result = extractor.extract(url)
|
||||
|
||||
if result:
|
||||
# 标准化输出格式
|
||||
metadata = {
|
||||
"type": content_type,
|
||||
"id": result.get("content_id", ""),
|
||||
"url": result.get("url", url),
|
||||
"title": result.get("title", ""),
|
||||
"author": result.get("author", ""),
|
||||
"publish_time": result.get("publish_time", ""),
|
||||
"vote_up": result.get("vote_up", 0),
|
||||
"content_markdown": result.get("content_markdown", ""),
|
||||
"raw_html": result.get("content", ""),
|
||||
}
|
||||
return metadata
|
||||
else:
|
||||
return {
|
||||
"type": content_type,
|
||||
"error": "Failed to extract content",
|
||||
"url": url
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""命令行入口"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="知乎元数据获取工具",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
支持的类型:
|
||||
zvideo - 知乎视频 (https://www.zhihu.com/zvideo/123456)
|
||||
zhuanlan - 知乎专栏文章 (https://zhuanlan.zhihu.com/p/123456)
|
||||
|
||||
示例:
|
||||
python zhihu_metadata.py --id "123456" --type zvideo
|
||||
python zhihu_metadata.py --id "123456" --type zhuanlan
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--id", "-i",
|
||||
required=True,
|
||||
help="知乎内容 ID (视频ID或文章ID)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--type", "-t",
|
||||
required=True,
|
||||
choices=["zvideo", "zhuanlan"],
|
||||
help="内容类型: zvideo=视频, zhuanlan=专栏文章"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--output", "-o",
|
||||
default=None,
|
||||
help="输出文件路径 (默认: stdout)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--headless",
|
||||
action="store_true",
|
||||
default=True,
|
||||
help="使用无头浏览器模式 (默认: True)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 提取元数据
|
||||
if args.type == "zvideo":
|
||||
result = extract_zvideo(args.id, headless=args.headless)
|
||||
else: # zhuanlan
|
||||
result = extract_zhuanlan(args.id, headless=args.headless)
|
||||
|
||||
# 输出结果
|
||||
json_output = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(json_output)
|
||||
print(f"结果已保存到: {args.output}")
|
||||
else:
|
||||
print(json_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user