Merge pull request 'fix: 修复知乎专栏文章爬取问题' (#3) from fix/ci-deploy-reset into main
Deploy kb-skills / deploy (push) Successful in 2s
Deploy kb-skills / deploy (push) Successful in 2s
Reviewed-on: http://192.168.1.22:8418/starmen1265/kb-skills/pulls/3
This commit was merged in pull request #3.
This commit is contained in:
@@ -222,8 +222,13 @@ python3 /path/to/bilibili_content.py --id "43780186" --type article --output [co
|
||||
|
||||
**知乎文章/问答**:
|
||||
```bash
|
||||
# 知乎问答/专栏提取(需要对应的 content 技能支持)
|
||||
python3 /path/to/zhihu_content.py --id "[ID]" --type [question|answer|zhuanlan] --output [content_dir]
|
||||
# 知乎专栏文章(推荐方式:自动爬取)
|
||||
python3 /path/to/zhihu_content.py --id "2013919514258072086" --type zhuanlan --output [content_dir]
|
||||
# 或使用完整 URL
|
||||
python3 /path/to/zhihu_content.py --id "https://zhuanlan.zhihu.com/p/2013919514258072086" --type zhuanlan --output [content_dir]
|
||||
|
||||
# 如果已有 MediaCrawler 数据,可直接读取
|
||||
python3 /path/to/zhihu_content.py --type article-from-media --input [json文件路径] --output [content_dir]
|
||||
```
|
||||
|
||||
3. 解析输出 JSON,提取文字内容用于深度分析
|
||||
|
||||
@@ -254,8 +254,8 @@ async def main():
|
||||
parser = argparse.ArgumentParser(description='知乎内容获取工具')
|
||||
parser.add_argument('--id', '-i', help='知乎视频ID (用于 video/subtitle 类型)')
|
||||
parser.add_argument('--type', '-t', required=True,
|
||||
choices=['video', 'subtitle', 'article-from-media'],
|
||||
help='内容类型: video=视频下载, subtitle=字幕提取, article-from-media=从MediaCrawler读取文章')
|
||||
choices=['video', 'subtitle', 'article-from-media', 'zhuanlan', 'article'],
|
||||
help='内容类型: video=视频下载, subtitle=字幕提取, article-from-media=从MediaCrawler读取文章, zhuanlan/article=爬取知乎专栏文章')
|
||||
parser.add_argument('--output', '-o', default='./output', help='输出目录')
|
||||
parser.add_argument('--quality', '-q', type=int, default=720, help='视频质量')
|
||||
parser.add_argument('--input', '-in', help='MediaCrawler 数据文件路径 (用于 article-from-media 类型)')
|
||||
@@ -275,6 +275,56 @@ async def main():
|
||||
input_file=args.input,
|
||||
output_dir=args.output
|
||||
)
|
||||
elif args.type in ('zhuanlan', 'article'):
|
||||
# 知乎专栏文章:先爬取再读取
|
||||
# 需要从 --id 或构造完整 URL
|
||||
if args.id:
|
||||
# 如果传入的是完整 URL
|
||||
if args.id.startswith('http'):
|
||||
article_url = args.id
|
||||
else:
|
||||
# 如果只传入 ID,构造 URL
|
||||
article_url = f"https://zhuanlan.zhihu.com/p/{args.id}"
|
||||
else:
|
||||
result = {"success": False, "message": "zhuanlan/article 类型需要 --id 参数(文章ID或完整URL)"}
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return result
|
||||
|
||||
# 调用 zhihu_article_crawler.py 爬取
|
||||
crawler_script = Path(__file__).parent.parent / "kb-zhihu-metadata" / "zhihu_article_crawler.py"
|
||||
if not crawler_script.exists():
|
||||
# 尝试另一种路径
|
||||
crawler_script = Path.home() / ".openclaw" / "agents" / "knowledge-base" / "workspace" / "skills" / "kb-zhihu-metadata" / "zhihu_article_crawler.py"
|
||||
|
||||
if crawler_script.exists():
|
||||
import subprocess
|
||||
cmd = ["python3", str(crawler_script), "--url", article_url, "--output", args.output]
|
||||
print(f"调用爬虫: {' '.join(cmd)}")
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
if proc.returncode != 0:
|
||||
result = {"success": False, "message": f"爬取失败: {proc.stderr[:200]}"}
|
||||
else:
|
||||
# 解析爬虫输出获取文件路径
|
||||
try:
|
||||
crawler_output = json.loads(proc.stdout.split("--- JSON 输出 ---")[-1].strip())
|
||||
if crawler_output.get("success"):
|
||||
# 直接使用爬虫输出的文件
|
||||
result = {
|
||||
"success": True,
|
||||
"type": "article",
|
||||
"title": crawler_output.get("title"),
|
||||
"filename": Path(crawler_output["filepath"]).name,
|
||||
"filepath": crawler_output["filepath"],
|
||||
"content_length": crawler_output.get("content_length"),
|
||||
"message": f"知乎专栏文章已爬取: {crawler_output.get('title')}"
|
||||
}
|
||||
else:
|
||||
result = {"success": False, "message": crawler_output.get("error", "爬取失败")}
|
||||
except Exception as e:
|
||||
result = {"success": False, "message": f"解析爬虫输出失败: {str(e)}"}
|
||||
else:
|
||||
result = {"success": False, "message": f"未找到 zhihu_article_crawler.py: {crawler_script}"}
|
||||
else:
|
||||
result = {"success": False, "message": f"不支持的类型: {args.type}"}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user