refactor(06-01): delete original metadata directories and update tests
- Remove kb-bilibili-metadata/ and kb-zhihu-metadata/ directories - Create kb_workflow/__init__.py to export main and process_command - Update test_cli/test_process_command.py imports for new package structure Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,46 +0,0 @@
|
|||||||
# KB Bilibili Metadata Skill
|
|
||||||
|
|
||||||
**kb-bilibili-metadata** - Extract Bilibili video/article/up主 metadata.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- Video metadata extraction
|
|
||||||
- Article (专栏) metadata extraction
|
|
||||||
- UP主 information extraction
|
|
||||||
- JSON output format
|
|
||||||
- Playwright + HTTP API fallback
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Video metadata
|
|
||||||
python bilibili_metadata.py --id "BV1xx411c7mD" --type video
|
|
||||||
|
|
||||||
# Article metadata
|
|
||||||
python bilibili_metadata.py --id "cv43780186" --type article
|
|
||||||
|
|
||||||
# UP主 info
|
|
||||||
python bilibili_metadata.py --id "7770724" --type up
|
|
||||||
```
|
|
||||||
|
|
||||||
## Output
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"data": {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "video",
|
|
||||||
"id": "BV1xx411c7mD",
|
|
||||||
"title": "视频标题",
|
|
||||||
"author": "UP主名字",
|
|
||||||
"stats": {"view": 100000, "like": 5000}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp -r kb-bilibili-metadata/ ~/skills/
|
|
||||||
```
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
"""
|
|
||||||
bilibili-metadata 技能包
|
|
||||||
|
|
||||||
B站元数据提取工具
|
|
||||||
"""
|
|
||||||
|
|
||||||
__version__ = "1.0.0"
|
|
||||||
__author__ = "OpenClaw"
|
|
||||||
Binary file not shown.
@@ -1,101 +0,0 @@
|
|||||||
# KB Zhihu Metadata Skill
|
|
||||||
|
|
||||||
**kb-zhihu-metadata** - Extract Zhihu question/answer/user metadata and crawl articles.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- Question metadata extraction
|
|
||||||
- Answer metadata extraction
|
|
||||||
- User information extraction
|
|
||||||
- **Article crawling using MediaCrawler** (NEW!)
|
|
||||||
- JSON output format
|
|
||||||
- Markdown conversion support
|
|
||||||
- **Cookie 支持**:可配置 ZHIHU_COOKIE 访问需要登录的内容
|
|
||||||
|
|
||||||
## 环境变量
|
|
||||||
|
|
||||||
| 变量名 | 用途 | 必需 |
|
|
||||||
|--------|------|------|
|
|
||||||
| `ZHIHU_COOKIE` | 知乎登录 Cookie | 推荐(访问专栏等需要) |
|
|
||||||
|
|
||||||
**获取 ZHIHU_COOKIE**:
|
|
||||||
1. 登录知乎网页版
|
|
||||||
2. 打开开发者工具 (F12) → Application → Cookies → zhihu.com
|
|
||||||
3. 复制 Cookie 字符串
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 添加到 ~/.kb_env
|
|
||||||
echo 'export ZHIHU_COOKIE="你的知乎Cookie"' >> ~/.kb_env
|
|
||||||
source ~/.kb_env
|
|
||||||
```
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Question metadata
|
|
||||||
python zhihu_metadata.py --id "266309957" --type question
|
|
||||||
|
|
||||||
# Answer metadata
|
|
||||||
python zhihu_metadata.py --id "2003427891925509507" --type answer
|
|
||||||
|
|
||||||
# 专栏文章爬取 (使用 MediaCrawler) ⚡ NEW!
|
|
||||||
python zhihu_article_crawler.py --url "https://zhuanlan.zhihu.com/p/1894054073625461813" --output "~/work/knowledge-base/data/zhihu/"
|
|
||||||
```
|
|
||||||
|
|
||||||
## MediaCrawler 爬取文章
|
|
||||||
|
|
||||||
**新功能**:使用 MediaCrawler 爬取知乎专栏文章
|
|
||||||
|
|
||||||
### 环境要求
|
|
||||||
|
|
||||||
MediaCrawler 已安装在 `~/work/MediaCrawler`
|
|
||||||
|
|
||||||
### 使用方法
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 爬取单篇文章
|
|
||||||
python zhihu_article_crawler.py --url "https://zhuanlan.zhihu.com/p/XXXXX"
|
|
||||||
|
|
||||||
# 指定输出目录
|
|
||||||
python zhihu_article_crawler.py --url "https://zhuanlan.zhihu.com/p/XXXXX" --output "/path/to/output/"
|
|
||||||
|
|
||||||
# 输出示例
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"type": "article",
|
|
||||||
"title": "如何高效的爬取微信公众号",
|
|
||||||
"filepath": "/home/xxx/work/knowledge-base/data/zhihu/zhihu_article_如何高效的爬取微信公众号.md",
|
|
||||||
"content_length": 4563,
|
|
||||||
"message": "爬取成功: 如何高效的爬取微信公众号"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 工作原理
|
|
||||||
|
|
||||||
1. 调用 MediaCrawler 爬取文章
|
|
||||||
2. 从 JSON 输出中提取内容
|
|
||||||
3. 转换为 Markdown 格式(带 frontmatter)
|
|
||||||
4. 保存到指定目录
|
|
||||||
|
|
||||||
## Output
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"data": {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "question",
|
|
||||||
"id": "266309957",
|
|
||||||
"title": "问题标题",
|
|
||||||
"author": "作者名字",
|
|
||||||
"answer_count": 50,
|
|
||||||
"followers": 1000
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp -r kb-zhihu-metadata/ ~/skills/
|
|
||||||
```
|
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
"""KB Workflow - Knowledge Base Workflow Orchestrator
|
||||||
|
|
||||||
|
知识库工作流管道入口。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .kb_workflow import process_command, main
|
||||||
|
|
||||||
|
__all__ = ["process_command", "main"]
|
||||||
@@ -12,16 +12,13 @@ import sys
|
|||||||
from unittest.mock import patch, MagicMock, AsyncMock
|
from unittest.mock import patch, MagicMock, AsyncMock
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# Add kb-workflow to path for imports
|
|
||||||
sys.path.insert(0, '/home/starmen1265/work/kb-skills/kb-workflow')
|
|
||||||
|
|
||||||
|
|
||||||
class TestProcessCommandHelp:
|
class TestProcessCommandHelp:
|
||||||
"""Test 1: `kb-workflow process --help` shows usage"""
|
"""Test 1: `kb-workflow process --help` shows usage"""
|
||||||
|
|
||||||
def test_process_help_shows_usage(self, capsys):
|
def test_process_help_shows_usage(self, capsys):
|
||||||
"""`kb-workflow process --help` shows usage information"""
|
"""`kb-workflow process --help` shows usage information"""
|
||||||
import kb_workflow
|
from kb_workflow import kb_workflow
|
||||||
|
|
||||||
# Mock sys.argv
|
# Mock sys.argv
|
||||||
with patch.object(sys, 'argv', ['kb-workflow', 'process', '--help']):
|
with patch.object(sys, 'argv', ['kb-workflow', 'process', '--help']):
|
||||||
@@ -40,7 +37,7 @@ class TestProcessCommandCallsPipeline:
|
|||||||
|
|
||||||
def test_process_url_calls_pipeline_run(self, capsys):
|
def test_process_url_calls_pipeline_run(self, capsys):
|
||||||
"""`kb-workflow process <url>` calls Pipeline.run()"""
|
"""`kb-workflow process <url>` calls Pipeline.run()"""
|
||||||
import kb_workflow
|
from kb_workflow import kb_workflow
|
||||||
|
|
||||||
with patch.object(kb_workflow, 'process_command') as mock_process:
|
with patch.object(kb_workflow, 'process_command') as mock_process:
|
||||||
mock_process.return_value = {"success": True, "content_dir": "/tmp/content"}
|
mock_process.return_value = {"success": True, "content_dir": "/tmp/content"}
|
||||||
@@ -55,7 +52,7 @@ class TestProcessCommandCallsPipeline:
|
|||||||
|
|
||||||
def test_process_url_with_output_dir(self, capsys):
|
def test_process_url_with_output_dir(self, capsys):
|
||||||
"""`kb-workflow process <url> --output-dir /custom` passes output_dir"""
|
"""`kb-workflow process <url> --output-dir /custom` passes output_dir"""
|
||||||
import kb_workflow
|
from kb_workflow import kb_workflow
|
||||||
|
|
||||||
with patch.object(kb_workflow, 'process_command') as mock_process:
|
with patch.object(kb_workflow, 'process_command') as mock_process:
|
||||||
mock_process.return_value = {"success": True, "content_dir": "/custom"}
|
mock_process.return_value = {"success": True, "content_dir": "/custom"}
|
||||||
@@ -76,7 +73,7 @@ class TestProcessCommandInvalidUrl:
|
|||||||
|
|
||||||
def test_process_invalid_url_returns_error(self, capsys):
|
def test_process_invalid_url_returns_error(self, capsys):
|
||||||
"""`kb-workflow process <invalid-url>` returns appropriate error"""
|
"""`kb-workflow process <invalid-url>` returns appropriate error"""
|
||||||
import kb_workflow
|
from kb_workflow import kb_workflow
|
||||||
|
|
||||||
with patch.object(kb_workflow, 'process_command') as mock_process:
|
with patch.object(kb_workflow, 'process_command') as mock_process:
|
||||||
mock_process.return_value = {
|
mock_process.return_value = {
|
||||||
@@ -96,7 +93,7 @@ class TestProcessCommandFlags:
|
|||||||
|
|
||||||
def test_output_dir_flag_sets_custom_directory(self):
|
def test_output_dir_flag_sets_custom_directory(self):
|
||||||
"""--output-dir flag sets custom output directory"""
|
"""--output-dir flag sets custom output directory"""
|
||||||
import kb_workflow
|
from kb_workflow import kb_workflow
|
||||||
|
|
||||||
with patch.object(kb_workflow, 'process_command') as mock_process:
|
with patch.object(kb_workflow, 'process_command') as mock_process:
|
||||||
mock_process.return_value = {"success": True}
|
mock_process.return_value = {"success": True}
|
||||||
@@ -113,7 +110,7 @@ class TestProcessCommandFlags:
|
|||||||
|
|
||||||
def test_skip_completed_default_true(self):
|
def test_skip_completed_default_true(self):
|
||||||
"""skip_completed defaults to True"""
|
"""skip_completed defaults to True"""
|
||||||
import kb_workflow
|
from kb_workflow import kb_workflow
|
||||||
|
|
||||||
with patch.object(kb_workflow, 'process_command') as mock_process:
|
with patch.object(kb_workflow, 'process_command') as mock_process:
|
||||||
mock_process.return_value = {"success": True}
|
mock_process.return_value = {"success": True}
|
||||||
@@ -129,7 +126,7 @@ class TestProcessCommandFlags:
|
|||||||
|
|
||||||
def test_no_skip_flag_disables_skip_completed(self):
|
def test_no_skip_flag_disables_skip_completed(self):
|
||||||
"""--no-skip flag disables skip_completed"""
|
"""--no-skip flag disables skip_completed"""
|
||||||
import kb_workflow
|
from kb_workflow import kb_workflow
|
||||||
|
|
||||||
with patch.object(kb_workflow, 'process_command') as mock_process:
|
with patch.object(kb_workflow, 'process_command') as mock_process:
|
||||||
mock_process.return_value = {"success": True}
|
mock_process.return_value = {"success": True}
|
||||||
|
|||||||
Reference in New Issue
Block a user