feat: 添加 MediaCrawler 整合支持

- kb-zhihu-metadata: 新增 zhihu_article_crawler.py 支持爬取知乎专栏文章
- kb-zhihu-content: 添加 article-from-media 类型支持从 MediaCrawler 数据读取
- 更新 SKILL.md 文档说明新功能
This commit is contained in:
2026-02-27 17:42:21 +08:00
parent 85797d0a89
commit 193d2b375d
4 changed files with 423 additions and 23 deletions
+122 -4
View File
@@ -5,6 +5,7 @@
支持:
- zvideo: 知乎视频下载
- article-from-media: 从 MediaCrawler 数据读取知乎文章
- 从页面提取视频播放地址
"""
@@ -13,6 +14,7 @@ import asyncio
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
@@ -21,6 +23,10 @@ from pathlib import Path
from typing import Optional, Dict, Any, List
# MediaCrawler 目录
MEDIACRAWLER_DIR = Path.home() / "work" / "MediaCrawler"
class ZhihuContent:
"""知乎内容获取器"""
@@ -146,13 +152,113 @@ class ZhihuContent:
return result
def read_article_from_mediacrawler(self, input_file: str = None, output_dir: str = None) -> Dict[str, Any]:
"""
从 MediaCrawler 爬取的数据中读取知乎文章内容
Args:
input_file: MediaCrawler 输出文件路径,不指定则自动查找最新
output_dir: 输出目录
Returns:
处理结果字典
"""
# 确定输出目录
if output_dir:
self.output_dir = Path(output_dir)
self.output_dir.mkdir(parents=True, exist_ok=True)
# 查找数据文件
data_dir = MEDIACRAWLER_DIR / "data" / "zhihu" / "json"
if input_file:
# 使用指定的文件
json_file = Path(input_file)
if not json_file.exists():
return {"success": False, "message": f"文件不存在: {input_file}"}
else:
# 查找最新的数据文件
json_files = sorted(data_dir.glob("detail_contents_*.json"), reverse=True)
if not json_files:
return {"success": False, "message": "未找到 MediaCrawler 爬取的数据,请先运行爬虫"}
json_file = json_files[0]
try:
with open(json_file, encoding="utf-8") as f:
data = json.load(f)
if not data:
return {"success": False, "message": "数据文件为空"}
# 提取第一篇文章
item = data[0]
# 准备数据
title = item.get("title", "无标题")
content = item.get("content_text", "")
url = item.get("content_url", "")
author = item.get("user_nickname", "未知")
created_ts = item.get("created_time", 0)
created = datetime.fromtimestamp(created_ts).strftime("%Y-%m-%d %H:%M:%S") if created_ts else ""
voteup = item.get("voteup_count", 0)
comments = item.get("comment_count", 0)
# 生成文件名
safe_title = "".join(c for c in title if c.isalnum() or c in " -_")[:50]
filename = f"zhihu_article_{safe_title}.md"
output_path = self.output_dir / filename
# 写入 Markdown 文件
markdown_content = f"""---
title: "{title}"
author: "{author}"
url: "{url}"
created: "{created}"
voteup: {voteup}
comments: {comments}
platform: zhihu
type: article
---
# {title}
**作者**: {author}
**来源**: [知乎]({url})
**发布时间**: {created}
**点赞**: {voteup}
**评论**: {comments}
---
{content}
"""
with open(output_path, "w", encoding="utf-8") as f:
f.write(markdown_content)
return {
"success": True,
"type": "article",
"title": title,
"filename": filename,
"filepath": str(output_path),
"content_length": len(content),
"message": f"文章内容已导出: {title}"
}
except Exception as e:
return {"success": False, "message": f"读取失败: {str(e)}"}
async def main():
parser = argparse.ArgumentParser(description='知乎内容获取工具')
parser.add_argument('--id', '-i', required=True, help='知乎视频ID')
parser.add_argument('--type', '-t', required=True, choices=['video', 'subtitle'], help='内容类型')
parser.add_argument('--id', '-i', help='知乎视频ID (用于 video/subtitle 类型)')
parser.add_argument('--type', '-t', required=True,
choices=['video', 'subtitle', 'article-from-media'],
help='内容类型: video=视频下载, subtitle=字幕提取, article-from-media=从MediaCrawler读取文章')
parser.add_argument('--output', '-o', default='./output', help='输出目录')
parser.add_argument('--quality', '-q', type=int, default=720, help='视频质量')
parser.add_argument('--input', '-in', help='MediaCrawler 数据文件路径 (用于 article-from-media 类型)')
args = parser.parse_args()
@@ -161,8 +267,16 @@ async def main():
if args.type == 'video':
result = await downloader.download_video(args.id, quality=args.quality)
else:
elif args.type == 'subtitle':
result = await downloader.extract_subtitle(args.id)
elif args.type == 'article-from-media':
# 同步调用,不需要 asyncio
result = downloader.read_article_from_mediacrawler(
input_file=args.input,
output_dir=args.output
)
else:
result = {"success": False, "message": f"不支持的类型: {args.type}"}
# 统一输出格式
if result.get("success"):
@@ -170,10 +284,14 @@ async def main():
"success": True,
"type": args.type,
"has_subtitle": result.get("has_subtitle", False),
"filename": Path(result.get('filepath', '')).name if result.get('filepath') else None,
"filename": result.get('filename') or (Path(result.get('filepath', '')).name if result.get('filepath') else None),
"filepath": str(Path(result.get('filepath', '')).resolve()) if result.get('filepath') else None,
"message": result.get('message', '处理成功')
}
if result.get('title'):
output['title'] = result['title']
if result.get('content_length'):
output['content_length'] = result['content_length']
else:
output = {
"success": False,