starmen1265/openclaw
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

自动同步: 2026-03-17 02:10

Browse Source
This commit is contained in:
starmen1265
2026-03-17 02:10:06 +08:00
parent aa29721d99
commit e9e9df0297
58 changed files with 913 additions and 5961 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agents/dev-assist/workspace/AGENTS.md
+8 -3
View File
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
### 用户的工作区
- 路径：~/work/zq
- 这个工作区目录通过rclone与Nas同步
- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`）
  - 所有操作实时同步到NAS，无需手动sync
  - 管理命令：`systemctl --user status|restart|stop rclone-zq`
  - 日志：`~/.cache/rclone/zq-mount.log`
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
  - 该目录已通过 rclone mount 挂载，直接读写挂载点即可
  - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新
  - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
agents/haonan/sessions/sessions.json
+127 -146
View File
@@ -1,7 +1,7 @@
{
"agent:haonan:main": {
"sessionId": "40e80fbb-07c8-440d-a1d4-1e876b86b82d",
"updatedAt": 1773508722585,
"sessionId": "e4845dbe-7d4d-41b6-adb5-b8a739e6520d",
"updatedAt": 1773620936802,
"systemSent": true,
"skillsSnapshot": {
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
@@ -59,11 +59,11 @@
"to": "user:ou_be83febac9ee2afaa343c642bc697c48",
"accountId": "feishu-pm"
},
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/40e80fbb-07c8-440d-a1d4-1e876b86b82d.jsonl",
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/e4845dbe-7d4d-41b6-adb5-b8a739e6520d.jsonl",
"systemPromptReport": {
"source": "run",
"generatedAt": 1773508545805,
"sessionId": "40e80fbb-07c8-440d-a1d4-1e876b86b82d",
"generatedAt": 1773620921082,
"sessionId": "e4845dbe-7d4d-41b6-adb5-b8a739e6520d",
"sessionKey": "agent:haonan:main",
"provider": "zhipu",
"model": "glm-5",
@@ -75,8 +75,8 @@
"sandboxed": false
},
"systemPrompt": {
"chars": 29493,
"projectContextChars": 16491,
"chars": 28195,
"projectContextChars": 15193,
"nonProjectContextChars": 13002
},
"injectedWorkspaceFiles": [
@@ -144,18 +144,6 @@
"injectedChars": 1168,
"truncated": false
},
{
"path": "SELF_IMPROVEMENT_REMINDER.md",
"rawChars": 616,
"injectedChars": 616,
"truncated": false
},
{
"path": "SELF_IMPROVEMENT_REMINDER.md",
"rawChars": 616,
"injectedChars": 616,
"truncated": false
},
{
"path": "SELF_IMPROVEMENT_REMINDER.md",
"rawChars": 616,
@@ -391,12 +379,12 @@
"totalTokensFresh": true,
"lastHeartbeatText": "已在飞书向蒋先生发送日历同步提醒！✅",
"lastHeartbeatSentAt": 1772924419184,
"cacheRead": 29760,
"cacheRead": 5376,
"cacheWrite": 0,
"contextTokens": 200000,
"inputTokens": 13411,
"outputTokens": 1288,
"totalTokens": 30249
"inputTokens": 12014,
"outputTokens": 510,
"totalTokens": 17390
},
"agent:haonan:wecom:dm:zhangqing": {
"origin": {
@@ -727,8 +715,8 @@
"memoryFlushCompactionCount": 0
},
"agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc": {
"sessionId": "a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda",
"updatedAt": 1773597600006,
"sessionId": "3941961e-62e9-42a2-9ec7-0847ef6734fb",
"updatedAt": 1773684000005,
"systemSent": true,
"model": "glm-5",
"contextTokens": 200000,
@@ -769,11 +757,11 @@
"version": 0
},
"modelProvider": "zhipu",
"inputTokens": 11356,
"outputTokens": 531,
"totalTokens": 17361,
"inputTokens": 10481,
"outputTokens": 227,
"totalTokens": 16606,
"totalTokensFresh": true,
"cacheRead": 90112,
"cacheRead": 22592,
"cacheWrite": 0
},
"agent:haonan:cron:f03b184f-5f1d-4093-ae1b-fc9fe0b59eef": {
@@ -819,9 +807,9 @@
"version": 0
},
"modelProvider": "zhipu",
"inputTokens": 153698,
"outputTokens": 6902,
"totalTokens": 33385,
"inputTokens": 41440,
"outputTokens": 3044,
"totalTokens": 25129,
"deliveryContext": {
"channel": "webchat"
},
@@ -830,7 +818,7 @@
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl",
"abortedLastRun": false,
"totalTokensFresh": true,
"cacheRead": 384894,
"cacheRead": 90423,
"cacheWrite": 0
},
"agent:haonan:cron:3245fca7-045b-4977-8fab-020f6df2e453": {
@@ -873,8 +861,8 @@
"cacheWrite": 0
},
"agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc": {
"sessionId": "69fcc73b-3185-4ad9-aacd-e44989b770d9",
"updatedAt": 1773598200007,
"sessionId": "5432f300-44b1-496e-b169-19a4593e5352",
"updatedAt": 1773684600005,
"systemSent": true,
"label": "Cron: openclaw-git-sync",
"skillsSnapshot": {
@@ -922,113 +910,6 @@
"cacheRead": 32768,
"cacheWrite": 0
},
"agent:haonan:cron:f03b184f-5f1d-4093-ae1b-fc9fe0b59eef:run:ed66991b-9963-45f0-b7c1-c2b2655e0463": {
"sessionId": "ed66991b-9963-45f0-b7c1-c2b2655e0463",
"updatedAt": 1773512400007,
"systemSent": true,
"model": "glm-4.7-FlashX",
"contextTokens": 200000,
"label": "Cron: capability-evolver-daily",
"skillsSnapshot": {
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
"skills": [
{
"name": "capability-evolver"
},
{
"name": "clawlist"
}
],
"skillFilter": [
"clawlist",
"capability-evolver",
"self-improving-agent"
],
"resolvedSkills": [
{
"name": "capability-evolver",
"description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.",
"filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md",
"baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver",
"source": "openclaw-managed",
"disableModelInvocation": false
},
{
"name": "clawlist",
"description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.",
"filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md",
"baseDir": "/home/starmen1265/.openclaw/skills/clawlist",
"source": "openclaw-managed",
"disableModelInvocation": false
}
],
"version": 0
},
"modelProvider": "zhipu",
"inputTokens": 153698,
"outputTokens": 6902,
"totalTokens": 33385,
"deliveryContext": {
"channel": "webchat"
},
"lastChannel": "webchat",
"channel": "webchat",
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl",
"abortedLastRun": false,
"totalTokensFresh": true,
"cacheRead": 384894,
"cacheWrite": 0
},
"agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc:run:a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda": {
"sessionId": "a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda",
"updatedAt": 1773597600006,
"systemSent": true,
"model": "glm-5",
"contextTokens": 200000,
"label": "Cron: backup-openclaw-json",
"skillsSnapshot": {
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
"skills": [
{
"name": "capability-evolver"
},
{
"name": "clawlist"
}
],
"skillFilter": [
"clawlist",
"capability-evolver",
"self-improving-agent"
],
"resolvedSkills": [
{
"name": "capability-evolver",
"description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.",
"filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md",
"baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver",
"source": "openclaw-managed",
"disableModelInvocation": false
},
{
"name": "clawlist",
"description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.",
"filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md",
"baseDir": "/home/starmen1265/.openclaw/skills/clawlist",
"source": "openclaw-managed",
"disableModelInvocation": false
}
],
"version": 0
},
"modelProvider": "zhipu",
"inputTokens": 11356,
"outputTokens": 531,
"totalTokens": 17361,
"totalTokensFresh": true,
"cacheRead": 90112,
"cacheWrite": 0
},
"agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc:run:69fcc73b-3185-4ad9-aacd-e44989b770d9": {
"sessionId": "69fcc73b-3185-4ad9-aacd-e44989b770d9",
"updatedAt": 1773598200007,
@@ -1122,9 +1003,9 @@
"version": 0
},
"modelProvider": "zhipu",
"inputTokens": 153698,
"outputTokens": 6902,
"totalTokens": 33385,
"inputTokens": 41440,
"outputTokens": 3044,
"totalTokens": 25129,
"deliveryContext": {
"channel": "webchat"
},
@@ -1133,7 +1014,107 @@
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl",
"abortedLastRun": false,
"totalTokensFresh": true,
"cacheRead": 384894,
"cacheRead": 90423,
"cacheWrite": 0
},
"agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc:run:3941961e-62e9-42a2-9ec7-0847ef6734fb": {
"sessionId": "3941961e-62e9-42a2-9ec7-0847ef6734fb",
"updatedAt": 1773684000005,
"systemSent": true,
"model": "glm-5",
"contextTokens": 200000,
"label": "Cron: backup-openclaw-json",
"skillsSnapshot": {
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
"skills": [
{
"name": "capability-evolver"
},
{
"name": "clawlist"
}
],
"skillFilter": [
"clawlist",
"capability-evolver",
"self-improving-agent"
],
"resolvedSkills": [
{
"name": "capability-evolver",
"description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.",
"filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md",
"baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver",
"source": "openclaw-managed",
"disableModelInvocation": false
},
{
"name": "clawlist",
"description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.",
"filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md",
"baseDir": "/home/starmen1265/.openclaw/skills/clawlist",
"source": "openclaw-managed",
"disableModelInvocation": false
}
],
"version": 0
},
"modelProvider": "zhipu",
"inputTokens": 10481,
"outputTokens": 227,
"totalTokens": 16606,
"totalTokensFresh": true,
"cacheRead": 22592,
"cacheWrite": 0
},
"agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc:run:5432f300-44b1-496e-b169-19a4593e5352": {
"sessionId": "5432f300-44b1-496e-b169-19a4593e5352",
"updatedAt": 1773684600005,
"systemSent": true,
"label": "Cron: openclaw-git-sync",
"skillsSnapshot": {
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
"skills": [
{
"name": "capability-evolver"
},
{
"name": "clawlist"
}
],
"skillFilter": [
"clawlist",
"capability-evolver",
"self-improving-agent"
],
"resolvedSkills": [
{
"name": "capability-evolver",
"description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.",
"filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md",
"baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver",
"source": "openclaw-managed",
"disableModelInvocation": false
},
{
"name": "clawlist",
"description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.",
"filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md",
"baseDir": "/home/starmen1265/.openclaw/skills/clawlist",
"source": "openclaw-managed",
"disableModelInvocation": false
}
],
"version": 0
},
"modelProvider": "zhipu",
"model": "glm-5",
"contextTokens": 200000,
"inputTokens": 579,
"outputTokens": 363,
"totalTokens": 16886,
"totalTokensFresh": true,
"cacheRead": 32768,
"cacheWrite": 0
}
}
agents/haonan/workspace/AGENTS.md
+8 -3
View File
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
### 用户的工作区
- 路径：~/work/zq
- 这个工作区目录通过rclone与Nas同步
- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`）
  - 所有操作实时同步到NAS，无需手动sync
  - 管理命令：`systemctl --user status|restart|stop rclone-zq`
  - 日志：`~/.cache/rclone/zq-mount.log`
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
  - 该目录已通过 rclone mount 挂载，直接读写挂载点即可
  - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新
  - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
agents/knowledge-base/workspace/.learnings/ERRORS.md
+33 -32
View File
@@ -1,39 +1,40 @@
# ERRORS.md - 错误记录
## 2026-03-10: kb-workflow 流程执行不完整
## 2026-03-16: rclone sync 覆盖了用户更新
### 发生了什么
用户请求"添加到知识库 + 链接"，我启动了 kb-workflow 技能，但两次漏掉步骤：
1. 第一次：只做了链接解析、元数据提取、内容提取，漏掉了深度分析
2. 第二次：深度分析完成后，漏掉了归档步骤
**问题描述**：
- `~/work/zq` 已通过 `rclone mount` 挂载，实时同步到 NAS
- 我在 18:25 执行了 `rclone sync ~/work/zq/ "nas-webdav:..."`（直接操作远程）
- 这绕过了挂载点缓存，覆盖了用户 18:30 在 NAS 上的更新
### 根本原因
1. **没有建立步骤检查清单** - SKILL.md 明确列出了8个步骤，但我没有逐项核对
2. **并行子代理导致认知偏差** - 启动3个子代理处理后以为工作结束了
3. **过早汇报完成** - 在步骤4完成后就向用户汇报，没有继续后续步骤
4. **忘记归档是必须步骤** - 即使分析完成了，归档才是最终目标
**根本原因**：
- AGENTS.md 中没有明确禁止对已挂载目录使用 rclone sync
- 误以为需要手动同步
### 正确做法
```
1. 执行技能前，先列出完整步骤清单
2. 每完成一步，标记状态 [x]
3. 全部完成后，再次核对清单
4. 全部 ✅ 才能向用户汇报完成
```
**教训**：
- **禁止对已挂载目录（~/work/zq）使用 rclone sync**
- 直接读写挂载点即可，挂载点会自动同步
### kb-workflow 完整步骤清单（必须全部执行）
| 步骤 | 内容 | 状态 |
|------|------|------|
| 1 | 环境变量检查 | [ ] |
| 2 | 链接解析 (kb-link-parser) | [ ] |
| 3 | 元数据提取 (kb-xxx-metadata) | [ ] |
| 4 | 内容下载/提取 (kb-xxx-content) | [ ] |
| 5 | 语音识别（视频才需要）| [ ] 或 [跳过] |
| 6 | 深度分析 (kb-deep-analyzer) | [ ] |
| 7 | 归档 (kb_workflow.py --archive-file) | [ ] |
| 8 | NAS 同步 + 汇报完成 | [ ] |
**已修正**：
- 更新了 AGENTS.md，添加了明确的禁止规则
### 如何避免再犯
- 每次执行 kb-workflow，必须使用上面的清单逐项核对
- 最后一步"归档"没完成，不算完成
- 用户问"为什么没XX"时，立即检查是否漏步骤
---
## 2026-03-16: kb-workflow 技能缺少归档步骤
**问题描述**：
- SKILL.md 文档描述了 5 步流程（parse → download → asr → analyze → archive）
- 实际 `kb_workflow.py` 只实现了 4 步，缺少 archive 步骤
- `KB_ARCHIVE_PATH` 环境变量被定义但未被使用
**影响**：
- 处理后的文件留在 workspace 而非归档到工作区
- 需要手动移动文件
**临时解决方案**：
- 手动执行 `cp` 到归档目录（直接操作挂载点 ~/work/zq，不用 rclone sync）
**根本解决方案**（需修改技能代码）：
1. 在 pipeline 中添加 ArchiveProcessor
2. 读取 `KB_ARCHIVE_PATH` 环境变量
3. 按日期创建子目录并移动文件
agents/knowledge-base/workspace/AGENTS.md
+8 -3
View File
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
### 用户的工作区
- 路径：~/work/zq
- 这个工作区目录通过rclone与Nas同步
- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`）
- 所有操作实时同步到NAS，无需手动sync
- 管理命令：`systemctl --user status|restart|stop rclone-zq`
- 日志：`~/.cache/rclone/zq-mount.log`
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
- 该目录已通过 rclone mount 挂载，直接读写挂载点即可
- 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新
- 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
agents/knowledge-base/workspace/memory/2026-03-16-0549.md
+70
View File
@@ -0,0 +1,70 @@
# Session: 2026-03-16 05:49:59 UTC
- **Session Key**: agent:knowledge-base:main
- **Session ID**: ad89af23-d276-4fd4-bd7c-e1a06ed88000
- **Source**: feishu
## Conversation Summary
user: Conversation info (untrusted metadata):
```json
{
"message_id": "om_x100b544d24d77ca4c4da0769c820e25",
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
"sender": "张青",
"timestamp": "Mon 2026-03-16 13:35 GMT+8"
}
```
Sender (untrusted metadata):
```json
{
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
"id": "ou_9351c287f42069fa30366ac641d24df7",
"name": "张青"
}
```
[message_id: om_x100b544d24d77ca4c4da0769c820e25]
张青: 现在哪里能找到南京90年代到2000年出头的城市样貌照片或者视频
assistant: 我来帮你搜索一下南京90年代到2000年代初的城市照片和视频资料。
assistant: 蒋先生，根据搜索结果，我为您找到了几个可以找到南京90年代到2000年代初城市样貌照片和视频的渠道：
## 📱 视频平台
### 抖音账号
- **"南京老照片"** - 这是一个专注南京历史文化的账号，发布了大量珍贵的历史影像
- 近期爆款：《2000年的南京大厂区》
- 《九十年代末的三牌楼大街和娄子巷》
- 拥有10万粉丝，持续输出高质量内容
### B站（哔哩哔哩）
- 搜索关键词："南京 90年代"、"南京 老照片"
- 有相关纪录片和用户上传的老视频
## 📸 照片资源
### 网站文章
1. **我苏网** - 《40年南京变化有多大？看这些"老照片"就懂了》
- 包含1990年代的玄武湖公园等场景
2. **搜狐网** - 《寻找消失的南京！40张老照片》
- 有90年代河西地区、7路公交车等珍贵影像
3. **游侠客旅行** - 《2025南京古今大对比，超百张珍贵老照片》
4. **南京市地方志办公室官网**
- 多篇关于南京高校、城市变迁的老照片文章
## 🎬 纪录片
- **《百年南京》** - 央视纪录片，包含南京历史变迁内容
## 💡 建议搜索方式
1. **抖音**：搜索"南京老照片"账号
2. **B站**：搜索"南京 90年代 老视频"
3. **百度图片**：搜索"南京 90年代 街景"
4. **微博**：搜索相关话题
需要我帮您进一步查看某个具体资源的详细内容吗？
agents/knowledge-base/workspace/memory/2026-03-16-0856.md
+226
View File
@@ -0,0 +1,226 @@
# Session: 2026-03-16 08:56:28 UTC
- **Session Key**: agent:knowledge-base:main
- **Session ID**: 9db16f34-09e8-4f14-810a-c210bf13a275
- **Source**: feishu
## Conversation Summary
assistant: 短链接格式不被识别。让我先获取真实URL：
assistant: 正在处理中，让我检查进度：
assistant: 处理进展：
- ✅ 解析成功
- ✅ 下载成功（已获取音频）
- ✅ 语音识别完成
- ❌ 深度分析失败：`AI_API_KEY 未设置`
让我检查环境变量配置：
assistant: 环境变量文件存在。问题是脚本执行时没有加载。让我手动加载后重新运行分析步骤：
assistant: ✅ 处理成功！现在执行归档步骤：
assistant: ---
✅ **知识库添加完成！**
**标题**：2-3月宏观经济运行状态复盘与推演 + 2026年政府工作报告解读
**作者**：阿周的Eureka | **时长**：07:16 | **日期**：2026-03-08
**处理结果**：
- ✅ 解析链接
- ✅ 下载音频
- ✅ 语音识别
- ✅ AI深度分析
- ✅ 归档到知识库
- ✅ 同步到NAS
**归档位置**：`~/work/zq/30.资源/AI分析/2026-03-16/bilibili_BV16cNMzpExh.md`
**核心内容摘要**：
视频复盘了2026年2-3月宏观经济数据，解读了政府工作报告。主要观点：
1. **通缩压力缓解**：猪肉价格触底回升，PPI环比改善
2. **M1增速回升**：源于定期存款集中到期，非内生扩张
3. **社融创新高**：主要靠政府债券融资，企业/居民意愿仍低
4. **房地产筑底**：成交量回暖是刚需入场，政策转向"稳预期"
5. **外部风险**：伊朗局势可能带来输入性通胀
user: Conversation info (untrusted metadata):
```json
{
"message_id": "om_x100b544e420910a4c14cf91d01d5da0",
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
"sender": "张青",
"timestamp": "Mon 2026-03-16 14:18 GMT+8"
}
```
Sender (untrusted metadata):
```json
{
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
"id": "ou_9351c287f42069fa30366ac641d24df7",
"name": "张青"
}
```
[message_id: om_x100b544e420910a4c14cf91d01d5da0]
张青: 工作区现在同步方式有问题，只要修改之前没有进行rclone操作，就会导致本地旧版的文件同步到Nas上。
user: Note: The previous agent run was aborted by the user. Resume carefully or ask for clarification.
Conversation info (untrusted metadata):
```json
{
"message_id": "om_x100b544e41da6c94c37ec270f11b68e",
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
"sender": "张青",
"timestamp": "Mon 2026-03-16 14:19 GMT+8"
}
```
Sender (untrusted metadata):
```json
{
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
"id": "ou_9351c287f42069fa30366ac641d24df7",
"name": "张青"
}
```
[message_id: om_x100b544e41da6c94c37ec270f11b68e]
张青: 工作区现在同步方式有问题，只要修改之前没有进行rclone操作，就会导致本地旧版的文件同步到Nas上。看看有没有更实时的同步方案。比如SynologyNas官方的linux部署方案
assistant: 明白问题所在。当前的双向 `rclone sync` 模式存在风险：如果本地有旧文件，会反向覆盖NAS上的新版本。
让我搜索 Synology NAS 官方的 Linux 同步方案：
assistant: 找到了几种方案。让我整理对比：
---
## 工作区实时同步方案对比
### 当前问题
`rclone sync` 是单向覆盖式同步，如果本地有旧文件，会把NAS新版本覆盖掉。
### 可选方案
| 方案 | 类型 | 实时性 | 优点 | 缺点 |
|------|------|--------|------|------|
| **Synology Drive Client** | 官方客户端 | ✅ 实时双向 | 官方支持、冲突处理、版本控制 | 需要GUI或有headless限制 |
| **rclone mount** | 文件系统挂载 | ✅ 实时读写 | 轻量、直接操作NAS文件 | 依赖网络稳定性 |
| **Syncthing** | P2P同步 | ✅ 实时双向 | 开源、无中心服务器 | 需要在NAS上安装 |
---
### 方案1：Synology Drive Client（官方推荐）
您的系统是 Ubuntu，支持安装：
```bash
# 下载 deb 包（从 Synology 官网或 NAS 套件中心获取）
wget https://global.download.synology.com/download/Package/spk/SynologyDriveClient/3.5.0-16084/Synology%20Drive%20Client-3.5.0-16084-x86_64.deb
# 安装
sudo dpkg -i "Synology Drive Client-3.5.0-16084-x86_64.deb"
sudo apt -f install # 修复依赖
```
**优点**：
- 真正的双向实时同步
- 自动处理冲突
- 支持版本历史
- 离线编辑后自动同步
**注意**：需要确认NAS上已安装 **Synology Drive Server** 套件。
---
### 方案2：rclone mount（轻量替代）
直接把NAS挂载为本地目录，所有操作实时写入：
```bash
# 挂载
rclone mount nas-webdav:/home/Drive/工作区/张青/工作区/ ~/work/zq/ --vfs-cache-mode full --daemon
# 卸载
fusermount -u ~/work/zq/
```
**优点**：
- 无需额外安装
- 操作即同步，不会有"旧版本覆盖"问题
- 配合 `--vfs-cache-mode full` 提升性能
**缺点**：
- 网络断开时无法操作
- 大文件操作可能较慢
---
### 建议
蒋先生，您倾向哪种方案？
1. **Synology Drive Client** - 功能最完整，需要先确认NAS上是否已安装 Drive Server 套件
2. **rclone mount** - 最快上手，我可以帮您配置自动挂载
3. **Syncthing** - 需要在NAS上额外安装
我可以帮您检查NAS上的配置，或者直接配置 rclone mount 方案？
user: Conversation info (untrusted metadata):
```json
{
"message_id": "om_x100b544e744c5080c2c7031b6254cec",
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
"sender": "张青",
"timestamp": "Mon 2026-03-16 14:31 GMT+8"
}
```
Sender (untrusted metadata):
```json
{
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
"id": "ou_9351c287f42069fa30366ac641d24df7",
"name": "张青"
}
```
[message_id: om_x100b544e744c5080c2c7031b6254cec]
张青: 2。rclone mount
assistant: 好的，配置 rclone mount 方案。
assistant: rclone 已配置好。现在创建 systemd 自动挂载服务：
assistant: ✅ **rclone mount 配置完成！**
**挂载状态**：
- 路径：`~/work/zq/` → NAS WebDAV
- 类型：实时读写（操作即同步）
- 缓存：启用（10G 本地缓存，提升性能）
- 自动启动：✅ 开机自动挂载
**常用命令**：
```bash
# 查看状态
systemctl --user status rclone-zq
# 手动重启
systemctl --user restart rclone-zq
# 查看日志
tail -f ~/.cache/rclone/zq-mount.log
```
现在更新 AGENTS.md，移除旧的 sync 步骤：
assistant: ✅ **配置完成！**
**变更摘要**：
| 项目 | 旧方式 | 新方式 |
|------|--------|--------|
| 同步模式 | 手动 `rclone sync` | 实时挂载 `rclone mount` |
| 操作前 | 需先从NAS拉取 | 直接操作，实时写入 |
| 操作后 | 需推送到NAS | 自动同步 |
| 覆盖风险 | ❌ 本地旧文件可能覆盖NAS | ✅ 直接读写NAS，无覆盖风险 |
**服务状态**：
- 服务名：`rclone-zq.service`
- 状态：运行中（开机自启）
- 挂载点：`~/work/zq/` ↔ NAS WebDAV
现在对 `~/work/zq/` 的任何操作都会实时同步到 NAS，不再需要手动 sync 了。
agents/knowledge-base/workspace/memory/2026-03-16-0928.md
+53
View File
@@ -0,0 +1,53 @@
# Session: 2026-03-16 09:28:44 UTC
- **Session Key**: agent:knowledge-base:main
- **Session ID**: fa3cb5a4-5a5b-4d65-96e9-0b17095e5445
- **Source**: feishu
## Conversation Summary
assistant: ✅ New session started · model: zhipu/glm-5
user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning.
assistant: 📚 蒋先生您好，我是达康书记。今天是周一，下午4点56分。有什么工作或生活事务需要我协助处理吗？
user: Conversation info (untrusted metadata):
```json
{
"message_id": "om_x100b54482e7f9480c39f23560aab683",
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
"sender": "张青",
"timestamp": "Mon 2026-03-16 16:57 GMT+8"
}
```
Sender (untrusted metadata):
```json
{
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
"id": "ou_9351c287f42069fa30366ac641d24df7",
"name": "张青"
}
```
[message_id: om_x100b54482e7f9480c39f23560aab683]
张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库
assistant: ❌ **处理失败 - 环境依赖缺失**
蒋先生，在执行"添加到知识库"任务时遇到了意料之外的问题：
**问题详情：**
- kb-workflow 技能缺少必要的 Python 包：`kb_zhihu_metadata`
- 该包未安装在虚拟环境中，也未在 kb-skills 仓库中找到源代码
- 这是一个知乎内容下载所需的依赖包
**已尝试的解决方法：**
1. ✅ 环境变量检查通过
2. ✅ 找到正确的 kb-workflow 执行位置
3. ❌ 缺少 kb_zhihu_metadata 包导致下载步骤失败
**建议解决方案：**
需要安装或获取 kb-zhihu-metadata 包。这可能需要：
- 从 PyPI 安装（如果已发布）
- 从其他仓库获取源代码并安装
- 联系技能维护者获取安装说明
根据工作规范，我无法通过修改代码或配置来解决这个问题。需要您协助处理这个环境依赖问题。
agents/knowledge-base/workspace/memory/2026-03-16-0933.md
+34
View File
@@ -0,0 +1,34 @@
# Session: 2026-03-16 09:33:25 UTC
- **Session Key**: agent:knowledge-base:main
- **Session ID**: bf3272a6-1140-4b72-815a-2c8c4e769629
- **Source**: feishu
## Conversation Summary
assistant: ✅ New session started · model: zhipu/glm-5
user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning.
assistant: 📚 **达康书记为您服务，蒋先生。**
新会话已就绪，有什么需要我协助的？无论是工作区管理、知识库整理，还是日历提醒、邮件处理，随时吩咐。
user: Conversation info (untrusted metadata):
```json
{
"message_id": "om_x100b544898f834b0c2998c7ae1dd83b",
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
"sender": "张青",
"timestamp": "Mon 2026-03-16 17:29 GMT+8"
}
```
Sender (untrusted metadata):
```json
{
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
"id": "ou_9351c287f42069fa30366ac641d24df7",
"name": "张青"
}
```
[message_id: om_x100b544898f834b0c2998c7ae1dd83b]
张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库
agents/knowledge-base/workspace/memory/2026-03-16-1021.md
+48
View File
@@ -0,0 +1,48 @@
# Session: 2026-03-16 10:21:10 UTC
- **Session Key**: agent:knowledge-base:main
- **Session ID**: e5fb9409-8568-482c-9e4d-0dc39679edf3
- **Source**: feishu
## Conversation Summary
assistant: ✅ New session started · model: zhipu/glm-5
user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning.
assistant: 📚 蒋先生您好，我是达康书记，您的私人秘书。请问今天有什么需要我协助您处理的？
user: Conversation info (untrusted metadata):
```json
{
"message_id": "om_x100b5448be14cd10c314363dea64b82",
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
"sender": "张青",
"timestamp": "Mon 2026-03-16 17:36 GMT+8"
}
```
Sender (untrusted metadata):
```json
{
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
"id": "ou_9351c287f42069fa30366ac641d24df7",
"name": "张青"
}
```
[message_id: om_x100b5448be14cd10c314363dea64b82]
张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库
assistant: ✅ **已添加到知识库**
**标题**：怎么成为一个 ai agent 工程师？
**来源**：知乎回答
**作者**：啦啦啦啦
**内容摘要**：
这份内容针对希望转型成为 AI Agent 工程师的人群，提出了一套以"商业项目落地"为核心的敏捷转型方案，核心包含四个步骤：
1. 掌握大模型的基础调用与提示词控制
2. 赋予模型使用工具的能力
3. 解决工程化难题（上下文管理、稳定性、日志监控）
4. 将项目经验转化为高含金量的简历话术
**归档位置**：`~/work/zq/30.资源/AI分析/2026-03-16/zhihu_2003232153480226496.md`
已同步至 NAS。
agents/knowledge-base/workspace/~/work/knowledge-base/data/zhihu/zhihu_article_VS Code 1111 炸了自动驾驶Autopilot模式上线这是要干掉 Cursor.md
-22
View File
File diff suppressed because one or more lines are too long
agents/thinker/workspace/AGENTS.md
+8 -3
View File
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
### 用户的工作区
- 路径：~/work/zq
- 这个工作区目录通过rclone与Nas同步
- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`）
  - 所有操作实时同步到NAS，无需手动sync
  - 管理命令：`systemctl --user status|restart|stop rclone-zq`
  - 日志：`~/.cache/rclone/zq-mount.log`
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
  - 该目录已通过 rclone mount 挂载，直接读写挂载点即可
  - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新
  - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
agents/ui-designer/sessions/sessions.json
+23 -11
View File
@@ -1,7 +1,7 @@
{
"agent:ui-designer:main": {
"sessionId": "fe685776-cc8c-489c-b409-e461e3d3cbfa",
"updatedAt": 1773564045652,
"sessionId": "2790af05-ac44-4b18-82fa-23471948d4d1",
"updatedAt": 1773677918741,
"systemSent": true,
"abortedLastRun": false,
"chatType": "direct",
@@ -22,7 +22,7 @@
"to": "user:ou_9847ce84b960736c14554730b41e4957",
"accountId": "feishu-art"
},
"sessionFile": "/home/starmen1265/.openclaw/agents/ui-designer/sessions/fe685776-cc8c-489c-b409-e461e3d3cbfa.jsonl",
"sessionFile": "/home/starmen1265/.openclaw/agents/ui-designer/sessions/2790af05-ac44-4b18-82fa-23471948d4d1.jsonl",
"compactionCount": 0,
"skillsSnapshot": {
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n</available_skills>",
@@ -51,8 +51,8 @@
"model": "glm-5",
"systemPromptReport": {
"source": "run",
"generatedAt": 1773564010472,
"sessionId": "fe685776-cc8c-489c-b409-e461e3d3cbfa",
"generatedAt": 1773677890931,
"sessionId": "2790af05-ac44-4b18-82fa-23471948d4d1",
"sessionKey": "agent:ui-designer:main",
"provider": "zhipu",
"model": "glm-5",
@@ -64,8 +64,8 @@
"sandboxed": false
},
"systemPrompt": {
"chars": 22552,
"projectContextChars": 10067,
"chars": 23850,
"projectContextChars": 11365,
"nonProjectContextChars": 12485
},
"injectedWorkspaceFiles": [
@@ -169,6 +169,18 @@
"injectedChars": 616,
"truncated": false
},
{
"path": "SELF_IMPROVEMENT_REMINDER.md",
"rawChars": 616,
"injectedChars": 616,
"truncated": false
},
{
"path": "SELF_IMPROVEMENT_REMINDER.md",
"rawChars": 616,
"injectedChars": 616,
"truncated": false
},
{
"path": "SELF_IMPROVEMENT_REMINDER.md",
"rawChars": 616,
@@ -396,13 +408,13 @@
]
}
},
"cacheRead": 15232,
"cacheRead": 15552,
"cacheWrite": 0,
"totalTokensFresh": true,
"contextTokens": 200000,
"inputTokens": 13122,
"outputTokens": 1320,
"totalTokens": 28354
"inputTokens": 2711,
"outputTokens": 882,
"totalTokens": 18263
},
"agent:ui-designer:cron:78b64c2f-7f91-47ff-9250-d13f546eb21f": {
"sessionId": "03e44b46-c707-4a9f-bf3d-1818aafdcde2",
agents/ui-designer/workspace/AGENTS.md
+8 -3
View File
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
### 用户的工作区
- 路径：~/work/zq
- 这个工作区目录通过rclone与Nas同步
- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`）
  - 所有操作实时同步到NAS，无需手动sync
  - 管理命令：`systemctl --user status|restart|stop rclone-zq`
  - 日志：`~/.cache/rclone/zq-mount.log`
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
  - 该目录已通过 rclone mount 挂载，直接读写挂载点即可
  - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新
  - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
agents/ui-designer/workspace/memory/2026-03-17.md
+36
View File
@@ -0,0 +1,36 @@
# 2026-03-17 财务咨询记录
## 用户信息
- 用户：张青 (ou_9847ce84b960736c14554730b41e4957)
---
## 资产配置优化咨询
### 用户背景
- 总资产：190万
- 当前配置：大部分固定收益理财，年化 2%
- 目标：整体年化收益提升到 8%
### 给出的方案
| 方案 | 股票占比 | 股票金额 | 股票目标收益率 | 风险等级 |
|------|---------|---------|--------------|---------|
| 保守型 | 30% | 57万 | 22% | 中高 |
| 平衡型 ⭐ | 50% | 95万 | 14% | 中 |
| 进取型 | 70% | 133万 | 10.6% | 中低（但敞口大）|
### 推荐方案
**平衡型（50% 股票配置）**
- 固定收益：95万（2%）
- 股票投资：95万（目标 14%）
- 理由：风险可控、目标现实、心理压力适中
### 股票投资建议
1. 指数基金打底（60%）：沪深300、中证500
2. 优质个股精选（30%）：高股息蓝筹、行业龙头
3. 灵活仓位（10%）：机会加仓
---
*记录时间：2026-03-17 00:13*
cron/jobs.json
+31 -28
View File
@@ -7,7 +7,7 @@
"name": "backup-openclaw-json",
"enabled": true,
"createdAtMs": 1772261581796,
"updatedAtMs": 1773597621815,
"updatedAtMs": 1773684008663,
"schedule": {
"kind": "cron",
"expr": "0 2 * * *",
@@ -23,10 +23,10 @@
"mode": "announce"
},
"state": {
"nextRunAtMs": 1773684000000,
"lastRunAtMs": 1773597600004,
"nextRunAtMs": 1773770400000,
"lastRunAtMs": 1773684000004,
"lastStatus": "ok",
"lastDurationMs": 21811,
"lastDurationMs": 8659,
"consecutiveErrors": 0,
"lastRunStatus": "ok",
"lastDelivered": true,
@@ -59,7 +59,8 @@
"consecutiveErrors": 0,
"lastRunStatus": "ok",
"lastDeliveryStatus": "delivered",
"lastDelivered": true
"lastDelivered": true,
"runningAtMs": 1773684600003
},
"delivery": {
"mode": "announce"
@@ -71,7 +72,7 @@
"name": "capability-evolver-daily",
"enabled": true,
"createdAtMs": 1772550743528,
"updatedAtMs": 1773513000030,
"updatedAtMs": 1773599129019,
"schedule": {
"kind": "cron",
"expr": "20 2 * * *",
@@ -92,14 +93,16 @@
"to": "feishu-pm"
},
"state": {
"nextRunAtMs": 1773598800000,
"lastRunAtMs": 1773512400004,
"nextRunAtMs": 1773685200000,
"lastRunAtMs": 1773598800006,
"lastStatus": "error",
"lastDurationMs": 600026,
"consecutiveErrors": 2,
"lastDurationMs": 329013,
"consecutiveErrors": 3,
"lastRunStatus": "error",
"lastDeliveryStatus": "unknown",
"runningAtMs": 1773598800004
"lastDeliveryStatus": "not-delivered",
"lastError": "⚠️ ✉️ Message failed",
"lastDelivered": false,
"lastDeliveryError": "⚠️ ✉️ Message failed"
}
},
{
@@ -109,7 +112,7 @@
"description": "每天凌晨2:30整理昨天的已完成任务到每日回顾",
"enabled": true,
"createdAtMs": 1741484880000,
"updatedAtMs": 1773513010858,
"updatedAtMs": 1773599416835,
"schedule": {
"kind": "cron",
"expr": "30 2 * * *",
@@ -127,11 +130,11 @@
"channel": "last"
},
"state": {
"nextRunAtMs": 1773599400000,
"lastRunAtMs": 1773513002054,
"nextRunAtMs": 1773685800000,
"lastRunAtMs": 1773599400005,
"lastRunStatus": "ok",
"lastStatus": "ok",
"lastDurationMs": 8804,
"lastDurationMs": 16830,
"lastDelivered": true,
"lastDeliveryStatus": "delivered",
"consecutiveErrors": 0
@@ -143,7 +146,7 @@
"name": "zq_workspace_sync",
"enabled": true,
"createdAtMs": 1772551807455,
"updatedAtMs": 1773513600005,
"updatedAtMs": 1773600000005,
"schedule": {
"expr": "40 2 * * *",
"kind": "cron",
@@ -156,10 +159,10 @@
"text": "执行 zq 工作区同步：bash ~/.openclaw/cron/scripts/zq_workspace_sync.sh"
},
"state": {
"nextRunAtMs": 1773600000000,
"lastRunAtMs": 1773513600005,
"nextRunAtMs": 1773686400000,
"lastRunAtMs": 1773600000004,
"lastStatus": "skipped",
"lastDurationMs": 0,
"lastDurationMs": 1,
"consecutiveErrors": 0,
"lastRunStatus": "skipped",
"lastDeliveryStatus": "not-requested",
@@ -173,7 +176,7 @@
"description": "每天早上8点整理与各个 AI 代理的对话记录",
"enabled": true,
"createdAtMs": 1772776369645,
"updatedAtMs": 1773532816520,
"updatedAtMs": 1773619224324,
"schedule": {
"kind": "cron",
"expr": "0 8 * * *",
@@ -191,11 +194,11 @@
"channel": "last"
},
"state": {
"nextRunAtMs": 1773619200000,
"lastRunAtMs": 1773532800006,
"nextRunAtMs": 1773705600000,
"lastRunAtMs": 1773619200004,
"lastRunStatus": "ok",
"lastStatus": "ok",
"lastDurationMs": 16514,
"lastDurationMs": 24320,
"lastDelivered": true,
"lastDeliveryStatus": "delivered",
"consecutiveErrors": 0
@@ -208,7 +211,7 @@
"description": "每天7点同步提醒到CalDAV日历",
"enabled": true,
"createdAtMs": 1772791528528,
"updatedAtMs": 1773529221353,
"updatedAtMs": 1773615616422,
"schedule": {
"kind": "cron",
"expr": "0 7 * * *",
@@ -226,11 +229,11 @@
"channel": "last"
},
"state": {
"nextRunAtMs": 1773615600000,
"lastRunAtMs": 1773529200004,
"nextRunAtMs": 1773702000000,
"lastRunAtMs": 1773615600034,
"lastRunStatus": "ok",
"lastStatus": "ok",
"lastDurationMs": 21349,
"lastDurationMs": 16388,
"lastDeliveryStatus": "delivered",
"consecutiveErrors": 0,
"lastDelivered": true
cron/jobs.json.bak
+29 -27
View File
@@ -7,7 +7,7 @@
"name": "backup-openclaw-json",
"enabled": true,
"createdAtMs": 1772261581796,
"updatedAtMs": 1773597621815,
"updatedAtMs": 1773684008663,
"schedule": {
"kind": "cron",
"expr": "0 2 * * *",
@@ -23,10 +23,10 @@
"mode": "announce"
},
"state": {
"nextRunAtMs": 1773684000000,
"lastRunAtMs": 1773597600004,
"nextRunAtMs": 1773770400000,
"lastRunAtMs": 1773684000004,
"lastStatus": "ok",
"lastDurationMs": 21811,
"lastDurationMs": 8659,
"consecutiveErrors": 0,
"lastRunStatus": "ok",
"lastDelivered": true,
@@ -71,7 +71,7 @@
"name": "capability-evolver-daily",
"enabled": true,
"createdAtMs": 1772550743528,
"updatedAtMs": 1773513000030,
"updatedAtMs": 1773599129019,
"schedule": {
"kind": "cron",
"expr": "20 2 * * *",
@@ -92,14 +92,16 @@
"to": "feishu-pm"
},
"state": {
"nextRunAtMs": 1773598800000,
"lastRunAtMs": 1773512400004,
"nextRunAtMs": 1773685200000,
"lastRunAtMs": 1773598800006,
"lastStatus": "error",
"lastDurationMs": 600026,
"consecutiveErrors": 2,
"lastDurationMs": 329013,
"consecutiveErrors": 3,
"lastRunStatus": "error",
"lastDeliveryStatus": "unknown",
"lastError": "cron: job execution timed out"
"lastDeliveryStatus": "not-delivered",
"lastError": "⚠️ ✉️ Message failed",
"lastDelivered": false,
"lastDeliveryError": "⚠️ ✉️ Message failed"
}
},
{
@@ -109,7 +111,7 @@
"description": "每天凌晨2:30整理昨天的已完成任务到每日回顾",
"enabled": true,
"createdAtMs": 1741484880000,
"updatedAtMs": 1773513010858,
"updatedAtMs": 1773599416835,
"schedule": {
"kind": "cron",
"expr": "30 2 * * *",
@@ -127,11 +129,11 @@
"channel": "last"
},
"state": {
"nextRunAtMs": 1773599400000,
"lastRunAtMs": 1773513002054,
"nextRunAtMs": 1773685800000,
"lastRunAtMs": 1773599400005,
"lastRunStatus": "ok",
"lastStatus": "ok",
"lastDurationMs": 8804,
"lastDurationMs": 16830,
"lastDelivered": true,
"lastDeliveryStatus": "delivered",
"consecutiveErrors": 0
@@ -143,7 +145,7 @@
"name": "zq_workspace_sync",
"enabled": true,
"createdAtMs": 1772551807455,
"updatedAtMs": 1773513600005,
"updatedAtMs": 1773600000005,
"schedule": {
"expr": "40 2 * * *",
"kind": "cron",
@@ -156,10 +158,10 @@
"text": "执行 zq 工作区同步：bash ~/.openclaw/cron/scripts/zq_workspace_sync.sh"
},
"state": {
"nextRunAtMs": 1773600000000,
"lastRunAtMs": 1773513600005,
"nextRunAtMs": 1773686400000,
"lastRunAtMs": 1773600000004,
"lastStatus": "skipped",
"lastDurationMs": 0,
"lastDurationMs": 1,
"consecutiveErrors": 0,
"lastRunStatus": "skipped",
"lastDeliveryStatus": "not-requested",
@@ -173,7 +175,7 @@
"description": "每天早上8点整理与各个 AI 代理的对话记录",
"enabled": true,
"createdAtMs": 1772776369645,
"updatedAtMs": 1773532816520,
"updatedAtMs": 1773619224324,
"schedule": {
"kind": "cron",
"expr": "0 8 * * *",
@@ -191,11 +193,11 @@
"channel": "last"
},
"state": {
"nextRunAtMs": 1773619200000,
"lastRunAtMs": 1773532800006,
"nextRunAtMs": 1773705600000,
"lastRunAtMs": 1773619200004,
"lastRunStatus": "ok",
"lastStatus": "ok",
"lastDurationMs": 16514,
"lastDurationMs": 24320,
"lastDelivered": true,
"lastDeliveryStatus": "delivered",
"consecutiveErrors": 0
@@ -208,7 +210,7 @@
"description": "每天7点同步提醒到CalDAV日历",
"enabled": true,
"createdAtMs": 1772791528528,
"updatedAtMs": 1773529221353,
"updatedAtMs": 1773615616422,
"schedule": {
"kind": "cron",
"expr": "0 7 * * *",
@@ -226,11 +228,11 @@
"channel": "last"
},
"state": {
"nextRunAtMs": 1773615600000,
"lastRunAtMs": 1773529200004,
"nextRunAtMs": 1773702000000,
"lastRunAtMs": 1773615600034,
"lastRunStatus": "ok",
"lastStatus": "ok",
"lastDurationMs": 21349,
"lastDurationMs": 16388,
"lastDeliveryStatus": "delivered",
"consecutiveErrors": 0,
"lastDelivered": true
delivery-queue/4b56a737-d726-4c48-8b10-493f0e9df335.json
+21
View File
@@ -0,0 +1,21 @@
{
"id": "4b56a737-d726-4c48-8b10-493f0e9df335",
"enqueuedAt": 1773599128798,
"channel": "feishu",
"to": "feishu-pm",
"accountId": "feishu-pm",
"payloads": [
{
"text": "## 🧬 Capability Evolver 每日报告\n\n**时间**: 2026年 03月 16日 星期一 02:20:22 CST\n**周期**: #0011\n\n---\n\n### 📊 变更概览\n- **意图**: REPAIR (修复)\n- **状态**: SUCCESS\n- **变更范围**: 4 文件修改，2 文件新增\n\n---\n\n### 📝 Git 提交记录\n\n1. **Auto backup before evolver run**\n - 4 files changed, 70 insertions(+), 64 deletions(-)\n\n2. **Evolver auto evolution** \n - 6 files changed, 550 insertions(+), 13 deletions(-)\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 📂 变更文件\n\n**修改文件**:\n- `memory/evolution/evolution_solidify_state.json`\n- `memory/evolution/evolution_state.json`\n- `memory/evolution/memory_graph_state.json`\n- `memory/evolution/personality_state.json`\n\n**新增文件**:\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 🎯 变更原因\n命中信号:\n- memory_missing\n- session_logs_missing\n\n系统检测到记忆缺失和会话日志缺失，触发修复模式，更新了进化状态和记忆图谱。"
}
],
"threadId": null,
"mirror": {
"sessionKey": "agent:haonan:main",
"agentId": "haonan",
"text": "## 🧬 Capability Evolver 每日报告\n\n**时间**: 2026年 03月 16日 星期一 02:20:22 CST\n**周期**: #0011\n\n---\n\n### 📊 变更概览\n- **意图**: REPAIR (修复)\n- **状态**: SUCCESS\n- **变更范围**: 4 文件修改，2 文件新增\n\n---\n\n### 📝 Git 提交记录\n\n1. **Auto backup before evolver run**\n - 4 files changed, 70 insertions(+), 64 deletions(-)\n\n2. **Evolver auto evolution** \n - 6 files changed, 550 insertions(+), 13 deletions(-)\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 📂 变更文件\n\n**修改文件**:\n- `memory/evolution/evolution_solidify_state.json`\n- `memory/evolution/evolution_state.json`\n- `memory/evolution/memory_graph_state.json`\n- `memory/evolution/personality_state.json`\n\n**新增文件**:\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 🎯 变更原因\n命中信号:\n- memory_missing\n- session_logs_missing\n\n系统检测到记忆缺失和会话日志缺失，触发修复模式，更新了进化状态和记忆图谱。"
},
"retryCount": 1,
"lastAttemptAt": 1773599129011,
"lastError": "Request failed with status code 400"
}
feishu/dedup/feishu-art.json
+2 -7
View File
@@ -1,9 +1,4 @@
{
"om_x100b546dc09400acb4a5d0af66ca3c9": 1773508838286,
"om_x100b546dde1c34a4b49a888912755ac": 1773508877713,
"om_x100b545236d06ca0b100846cfa37e9d": 1773561730503,
"om_x100b5452d99e18a4b32ab9bdc02463e": 1773562229918,
"om_x100b5452972138a8b213d14996ffdbd": 1773563295479,
"om_x100b5452a60eaca0b10319a26ed640f": 1773563532727,
"om_x100b5453485b4ca0b37c3413c41bba3": 1773564010102
"om_x100b54b68a9658a8c3597587eaea165": 1773677638200,
"om_x100b54b69ae988acc4e308675a1599b": 1773677890863
}
feishu/dedup/feishu-pm.json
+1 -4
View File
@@ -1,6 +1,3 @@
{
"om_x100b546d172f3c68c2cea708421b99a": 1773507998614,
"om_x100b546d2b32cca0c329a33c339cfb5": 1773508192506,
"om_x100b546d32de24a8b316c197e60d6e4": 1773508545727,
"om_x100b546dc9d61ca0b33d802bc7615e0": 1773508722267
"om_x100b5440a54d84acb3898a37411f37b": 1773620920592
}
feishu/dedup/feishu-worker.json
+32 -30
View File
@@ -1,32 +1,34 @@
{
"om_x100b546ddd8b14a0b3ddda14ce768c5": 1773508916933,
"om_x100b546dd78c8ca0b31b0314f910115": 1773509012676,
"om_x100b546def8ac8a4b38d921a2bfc8b8": 1773509140955,
"om_x100b546dedb654a4b3ce916a129facf": 1773509176311,
"om_x100b546de975dca0b27e2f24aad0d8c": 1773509244102,
"om_x100b546de53a94a0b38693fbd7e385c": 1773509311993,
"om_x100b546dffaa3cb0b35aef2747ed228": 1773509398933,
"om_x100b546df49c10a0b106d597fd348a3": 1773509541678,
"om_x100b546d16e6e4a0c42bef93bca2224": 1773511894148,
"om_x100b546d144668a0c3583fa56a2597d": 1773511924836,
"om_x100b546d112090acc3d096e37b38dc9": 1773512010988,
"om_x100b545c390898a4c379f319d607993": 1773569917108,
"om_x100b545c365b94a0c44e709df0f5422": 1773569929804,
"om_x100b545cce0a04a0c4a26c0203c2b13": 1773570060994,
"om_x100b545ccf6e6ca0c4571d12c405d8f": 1773570074717,
"om_x100b545cccde84acc4d8fa9b0f0c852": 1773570081700,
"om_x100b545ccab08ca0c12b5e6c715a1aa": 1773570120321,
"om_x100b545cca14e8a4c4dcc592a9c425d": 1773570126138,
"om_x100b545c9fd988a4b4bcc03ec7b3368": 1773571345806,
"om_x100b5458d25a5ca0c2cf04550bf07cc": 1773586890090,
"om_x100b5458d055dca4c4dd20469577549": 1773586922136,
"om_x100b54588d4de4a0c14df6ea4b56be0": 1773587512581,
"om_x100b54599445b4a0c4def897c8a2700": 1773591976996,
"om_x100b545992d568a0c3511684eae15a5": 1773592002087,
"om_x100b5459a51f60a4c11dbfc6068d690": 1773592253777,
"om_x100b5459be53f0a0c42f8fe338d2510": 1773592330439,
"om_x100b5459be2d7ca4c32ee845e40c605": 1773592334472,
"om_x100b5459bd4168a0c36ca5cf1588b13": 1773592377222,
"om_x100b5459bae828acc347e53e7e5de8d": 1773592386779,
"om_x100b5459bb6404a0c4ade83756a546a": 1773592411047
"om_x100b544d24d77ca4c4da0769c820e25": 1773639330290,
"om_x100b544deea46100c29a100e8840656": 1773640199255,
"om_x100b544dfc78d090c1266487d499834": 1773640492047,
"om_x100b544df47a2c80c36bffb914f76a1": 1773640620036,
"om_x100b544e420910a4c14cf91d01d5da0": 1773641933039,
"om_x100b544e43f1d4acc2bcd0631ad4df1": 1773641940383,
"om_x100b544e41da6c94c37ec270f11b68e": 1773641970288,
"om_x100b544e744c5080c2c7031b6254cec": 1773642664923,
"om_x100b54481574e8bcc311e8d58fb4c75": 1773651388362,
"om_x100b54482e7f9480c39f23560aab683": 1773651467804,
"om_x100b54489a79ecbcc142ca87d3f064b": 1773653324009,
"om_x100b544898f834b0c2998c7ae1dd83b": 1773653348152,
"om_x100b5448aba858acc4c8be1f8439827": 1773653591178,
"om_x100b5448ab152c80c2a6860a990a363": 1773653598227,
"om_x100b5448a8d4f0a0c32f5cc81fda1b3": 1773653602205,
"om_x100b5448a887a4a0c3d2cacc2350d0f": 1773653605264,
"om_x100b5448be14cd10c314363dea64b82": 1773653774367,
"om_x100b5449d79ebca8c3e54daed500100": 1773656469908,
"om_x100b5449d129e0acc3831f0b28c0dca": 1773656574965,
"om_x100b5449e4e594a0c4504c3f849787d": 1773656739113,
"om_x100b54499819dc84c31ee62a28e622a": 1773657453822,
"om_x100b544994dbc8a0c37f7406c922a18": 1773657506038,
"om_x100b5449953c8ca0c10d2892ab60482": 1773657535728,
"om_x100b5449a63174b4c2c98d2bc1f7fcc": 1773657744382,
"om_x100b544a62d8fc80c4e14309469ce27": 1773658818051,
"om_x100b544a7b1c14a0c49f7000df992cc": 1773658973700,
"om_x100b544a775f1080c3d624954a8460e": 1773659033826,
"om_x100b54b47a79e8a8b37ac4b2ddad147": 1773667147854,
"om_x100b54b760e4b0a0c445e4e92b151e5": 1773679331046,
"om_x100b54b709932ca8b32f9afaf823ea9": 1773679734328,
"om_x100b54b7042de0acb108eaf062898fd": 1773679790410,
"om_x100b54b7174820b0b2678b67a63e045": 1773680024819
}
identity/device-auth.json
+1 -1
View File
@@ -10,7 +10,7 @@
"operator.approvals",
"operator.pairing"
],
"updatedAtMs": 1773598217705
"updatedAtMs": 1773684008206
}
}
}
logs/commands.log
+5
View File
@@ -45,3 +45,8 @@
{"timestamp":"2026-03-15T16:26:17.361Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
{"timestamp":"2026-03-15T16:32:14.537Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
{"timestamp":"2026-03-15T16:33:06.821Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
{"timestamp":"2026-03-16T05:49:59.593Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
{"timestamp":"2026-03-16T08:56:28.934Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
{"timestamp":"2026-03-16T09:28:44.393Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
{"timestamp":"2026-03-16T09:33:25.309Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
{"timestamp":"2026-03-16T10:21:10.561Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
skills/kb-bilibili-content/SKILL.md
-91
View File
@@ -1,91 +0,0 @@
# KB Bilibili Content Skill
**kb-bilibili-content** - Download Bilibili videos and extract content.
## Features
- Video download using **yt-dlp** (recommended)
- Subtitle extraction
- Article content extraction
- Multiple quality options (360P/480P/720P/1080P)
- Supports `BILIBILI_COOKIE` environment variable
## Environment Variables
| Variable | Required | Description |
|----------|----------|-------------|
| `BILIBILI_COOKIE` | No | B站登录 Cookie (for member-only videos) |
## Commands
```bash
# Download video (720P default)
python bilibili_content.py --id "BV1xx411c7mD" --type video --output "/path/to/downloads/"
# Download video (1080P)
python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 80 --output "/path/to/downloads/"
# Download video (480P)
python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 32 --output "/path/to/downloads/"
# Extract subtitle
python bilibili_content.py --id "BV1xx411c7mD" --type subtitle --output "/path/to/subtitles/"
# Get article content
python bilibili_content.py --id "cv43780186" --type article --output "/path/to/article.md"
```
## Output
```json
{
"success": true,
"type": "video",
"filename": "video_title.mp4",
"filepath": "/path/to/downloads/video_title.mp4",
"message": "下载完成: video_title.mp4"
}
```
## Cookie Setup
### Get Cookie from Browser
1. Login to bilibili.com
2. Press F12 → Network → Refresh page
3. Copy Cookie from request headers
### Set Environment Variable
```bash
# Add to ~/.bashrc for persistence
export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx; DedeUserID=xxx; DedeUserID__ckMd5=xxx"
# Or set temporarily
export BILIBILI_COOKIE="your_cookie_here"
```
## Installation
```bash
cp -r kb-bilibili-content/ ~/skills/
pip install aiohttp playwright
playwright install chromium
```
## How It Works
Video download uses **yt-dlp** with Netscape-format cookie file:
1. Parses `BILIBILI_COOKIE` environment variable
2. Creates temporary Netscape-format cookie file
3. Runs `yt-dlp --cookies <file> -f <format> -o <output>`
4. Cleans up temporary file
## Quality Options
| Quality | ID | Resolution | Notes |
|---------|-----|------------|-------|
| 16 | 360P | 640x360 | Fastest |
| 32 | 480P | 852x480 | Default for low bandwidth |
| 64 | 720P | 1280x720 | **Recommended** |
| 80 | 1080P | 1920x1080 | Best quality |
Default quality: **64 (720P)**
skills/kb-bilibili-content/__pycache__/bilibili_content.cpython-312.pyc
BIN
View File
Binary file not shown.
skills/kb-bilibili-content/bilibili_content.py
-419
View File
@@ -1,419 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Bilibili Content Downloader - B站内容获取工具
功能:
- download_video(bvid, quality=480) - 下载视频（使用 yt-dlp）
- extract_subtitle(bvid) - 提取字幕
- extract_article(cvid) - 获取专栏全文（使用Playwright）
- 命令行参数支持
- JSON 输出
"""
import argparse
import asyncio
import json
import os
import re
import subprocess
import sys
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Optional, Dict, Any, List
class BilibiliContent:
"""B站内容获取器"""
def __init__(self, output_dir: str = "./output", cookie: str = None):
self.output_dir = Path(output_dir)
self.output_dir.mkdir(parents=True, exist_ok=True)
self.cookie = cookie
def _create_cookie_file(self) -> str:
"""创建 Netscape 格式的 Cookie 文件"""
if not self.cookie:
return None
# 解析 Cookie 字符串
cookies = {}
for item in self.cookie.split(';'):
item = item.strip()
if '=' in item:
key, value = item.split('=', 1)
cookies[key.strip()] = value.strip()
# 创建 Netscape 格式的 Cookie 文件
cookie_content = """# Netscape HTTP Cookie File
# Generated by bilibili_content.py
"""
for name, value in cookies.items():
cookie_content += f".bilibili.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n"
# 写入临时文件
fd, path = tempfile.mkstemp(suffix='.txt')
with os.fdopen(fd, 'w') as f:
f.write(cookie_content)
return path
async def download_video(self, bvid: str, quality: int = 64) -> Dict[str, Any]:
"""下载B站视频（使用 yt-dlp）"""
result = {"success": False, "bvid": bvid, "message": "", "filepath": None, "metadata": {}}
try:
# 先获取视频信息
print(f"获取视频信息: {bvid}")
cookie_file = self._create_cookie_file()
url = f"https://www.bilibili.com/video/{bvid}"
try:
# 获取视频标题（用于文件名）
if cookie_file:
cmd = ["yt-dlp", "--cookies", cookie_file, "--get-title", url]
else:
cmd = ["yt-dlp", "--get-title", url]
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
title = stdout.decode('utf-8').strip()
if not title or 'error' in title.lower():
# 尝试备用方法
title = f"video_{bvid}"
except Exception as e:
title = f"video_{bvid}"
# 构建 yt-dlp 命令
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_', '丨')).strip()[:50]
# 选择视频格式
if quality == 80:
# 1080P
format_str = "30112+30280"
elif quality == 64:
# 720P
format_str = "30064+30280"
elif quality == 32:
# 480P
format_str = "30032+30280"
else:
# 360P
format_str = "30016+30280"
print(f"下载视频: {title}")
print(f"格式: {format_str}")
output_template = str(self.output_dir / f"{safe_title}.%(ext)s")
cmd = [
"yt-dlp",
"-f", format_str,
"-o", output_template,
"--no-download-archive",
"-q",
url
]
if cookie_file:
cmd.extend(["--cookies", cookie_file])
print(f"执行命令: {' '.join(cmd[:4])}...")
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
# 查找下载的文件
downloaded_files = list(self.output_dir.glob(f"{safe_title}.*"))
if downloaded_files:
video_file = downloaded_files[0]
result["success"] = True
result["filepath"] = str(video_file)
result["message"] = f"下载完成: {video_file.name}"
result["metadata"] = {"title": title, "bvid": bvid}
print(f"✅ {result['message']}")
else:
result["message"] = "未找到下载的文件"
print(f"❌ {result['message']}")
# 清理临时 Cookie 文件
if cookie_file and os.path.exists(cookie_file):
os.remove(cookie_file)
except Exception as e:
result["message"] = f"错误: {str(e)}"
print(f"❌ {result['message']}")
return result
async def extract_subtitle(self, bvid: str) -> Dict[str, Any]:
"""
提取字幕
注意：没有找到官方字幕是正常的！很多视频没有自动生成的字幕。
返回 success=True 即使没有字幕，让工作流继续执行后续步骤（如 Whisper）。
"""
result = {"success": False, "bvid": bvid, "subtitles": [], "has_subtitle": False, "message": ""}
try:
import aiohttp
api_url = f"https://api.bilibili.com/x/player/pagelist?bvid={bvid}"
async with aiohttp.ClientSession() as session:
async with session.get(api_url) as resp:
data = await resp.json()
pages = data.get('data', [])
if not pages:
result["message"] = "未找到分P（这是正常的）"
# 仍然返回 success=True，让工作流继续
result["success"] = True
return result
cid = pages[0].get('cid', '')
async with aiohttp.ClientSession() as session:
async with session.get(f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}") as resp:
sub_data = await resp.json()
subtitles = sub_data.get('data', {}).get('subtitle', {}).get('list', [])
if not subtitles:
result["message"] = "没有官方字幕（这是正常的），将使用 Whisper 进行语音识别"
# 仍然返回 success=True，让工作流继续
result["success"] = True
return result
result["success"] = True
result["has_subtitle"] = True
result["subtitles"] = [{"lan": s.get('lan'), "title": s.get('lan_doc')} for s in subtitles]
result["message"] = f"✅ 找到 {len(subtitles)} 个字幕，将使用官方字幕进行深度分析"
print(result["message"])
except Exception as e:
result["message"] = f"获取字幕出错（这是正常的），将使用 Whisper: {str(e)}"
# 出错时也返回 success=True，让工作流继续
result["success"] = True
return result
async def download_subtitle(self, bvid: str) -> Dict[str, Any]:
"""
下载字幕并转换为 SRT 格式
如果没有官方字幕，返回 success=True 但 has_subtitle=False，
让工作流继续使用 Whisper 进行语音识别。
"""
result = {"success": False, "bvid": bvid, "filepath": None, "has_subtitle": False, "message": ""}
try:
import aiohttp
# 获取 CID
api_url = f"https://api.bilibili.com/x/player/pagelist?bvid={bvid}"
async with aiohttp.ClientSession() as session:
async with session.get(api_url) as resp:
data = await resp.json()
pages = data.get('data', [])
if not pages:
result["message"] = "未找到分P，将使用 Whisper 进行语音识别"
result["success"] = True
return result
cid = pages[0].get('cid', '')
# 获取字幕列表
async with aiohttp.ClientSession() as session:
async with session.get(f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}") as resp:
sub_data = await resp.json()
subtitles = sub_data.get('data', {}).get('subtitle', {}).get('list', [])
if not subtitles:
result["message"] = "没有官方字幕，将使用 Whisper 进行语音识别"
result["success"] = True
return result
# 选择中文字幕，如果没有则选第一个
zh_sub = None
for s in subtitles:
if s.get('lan') in ('ai-zh', 'zh-CN', 'zh'):
zh_sub = s
break
if not zh_sub:
zh_sub = subtitles[0]
# 下载字幕文件
sub_url = zh_sub.get('subtitle_url', '')
if not sub_url:
result["message"] = "字幕URL为空，将使用 Whisper"
result["success"] = True
return result
if sub_url.startswith('//'):
sub_url = 'https:' + sub_url
async with aiohttp.ClientSession() as session:
async with session.get(sub_url) as resp:
sub_content = await resp.json()
# 转换为 SRT 格式
body = sub_content.get('body', [])
if not body:
result["message"] = "字幕内容为空，将使用 Whisper"
result["success"] = True
return result
def format_srt_time(t: float) -> str:
"""将秒数转换为 SRT 时间格式"""
h = int(t // 3600)
m = int((t % 3600) // 60)
s = int(t % 60)
ms = int((t - int(t)) * 1000)
return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
srt_lines = []
for i, line in enumerate(body, 1):
start = line.get('from', 0)
end = line.get('to', 0)
content = line.get('content', '')
srt_lines.extend([str(i), f"{format_srt_time(start)} --> {format_srt_time(end)}", content, ""])
srt_content = "\n".join(srt_lines)
# 保存 SRT 文件
output_path = self.output_dir / f"{bvid}.srt"
with open(output_path, 'w', encoding='utf-8') as f:
f.write(srt_content)
result["success"] = True
result["has_subtitle"] = True
result["filepath"] = str(output_path)
result["message"] = f"✅ 官方字幕已下载: {output_path.name} ({len(body)}条)"
print(result["message"])
except Exception as e:
result["message"] = f"下载字幕出错: {str(e)}，将使用 Whisper"
result["success"] = True # 仍然返回成功，让工作流继续
return result
async def extract_article(self, cvid: str) -> Dict[str, Any]:
"""获取专栏全文（使用Playwright异步API）"""
result = {"success": False, "cvid": cvid, "title": "", "content": "", "message": ""}
try:
from playwright.async_api import async_playwright
cvid_clean = cvid.replace('cv', '')
url = f"https://www.bilibili.com/read/cv{cvid_clean}"
print(f"获取专栏: {url}")
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
page = await browser.new_page()
await page.goto(url, wait_until="networkidle")
await page.wait_for_timeout(3000)
# 提取标题
title_elem = await page.query_selector('h1')
if title_elem:
result["title"] = (await title_elem.inner_text()).strip()
print(f"标题: {result['title'][:50]}...")
# 提取正文内容
content_elem = await page.query_selector('.read-article-holder')
if content_elem:
html_content = await content_elem.inner_html()
text_content = await content_elem.inner_text()
result["content"] = text_content.strip()
result["html_content"] = html_content.strip()
result["success"] = bool(result["title"] and result["content"])
print(f"内容长度: {len(result['content'])} 字符")
else:
result["message"] = "未找到专栏内容"
await browser.close()
if result["success"]:
result["message"] = f"✅ 获取成功: {result['title'][:30]}..."
else:
result["message"] = f"获取失败: {result.get('message', '未知错误')}"
except Exception as e:
result["message"] = f"错误: {str(e)}"
print(f"错误: {e}")
return result
async def main():
parser = argparse.ArgumentParser(description='Bilibili Content Downloader')
parser.add_argument('--id', '-i', required=True, help='B站内容ID')
parser.add_argument('--type', '-t', required=True, choices=['video', 'subtitle', 'subtitle-download', 'article'], help='内容类型')
parser.add_argument('--output', '-o', default='./output', help='输出目录')
parser.add_argument('--quality', '-q', type=int, default=64, help='视频质量: 16=360P, 32=480P, 64=720P, 80=1080P')
args = parser.parse_args()
cookie = os.environ.get('BILIBILI_COOKIE')
downloader = BilibiliContent(output_dir=args.output, cookie=cookie)
if args.type == 'video':
result = await downloader.download_video(args.id, quality=args.quality)
elif args.type == 'subtitle':
result = await downloader.extract_subtitle(args.id)
elif args.type == 'subtitle-download':
# 下载并转换字幕为 SRT 格式
result = await downloader.download_subtitle(args.id)
else:
result = await downloader.extract_article(args.id.replace('cv', ''))
# 统一输出格式
if result.get("success"):
output = {
"success": True,
"type": args.type,
"has_subtitle": result.get("has_subtitle", False),
"filename": Path(result.get('filepath', '')).name if result.get('filepath') else None,
"filepath": str(Path(result.get('filepath', '')).resolve()) if result.get('filepath') else None,
"message": result.get('message', '处理成功')
}
else:
output = {
"success": False,
"error": result.get('message', '处理失败'),
"detail": f"ID: {args.id}, 类型: {args.type}"
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return result
if __name__ == '__main__':
try:
result = asyncio.run(main())
sys.exit(0 if result.get("success") else 1)
except KeyboardInterrupt:
print(json.dumps({"success": False, "error": "用户取消"}, ensure_ascii=False, indent=2))
sys.exit(1)
except Exception as e:
print(json.dumps({"success": False, "error": str(e)}, ensure_ascii=False, indent=2))
sys.exit(1)
skills/kb-bilibili-content/bilibili_extractor.py
-155
View File
@@ -1,155 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
B站视频提取脚本
提取视频信息、统计数据、字幕并生成Markdown笔记
"""
import sys
import os
import json
import re
from datetime import datetime
# 添加skills路径
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb'))
from src.extractors.bilibili_extractor import BilibiliExtractor
from src.config import get_config
def format_number(num):
"""格式化数字"""
if num >= 100000000:
return f"{num / 100000000:.1f}亿"
elif num >= 10000:
return f"{num / 10000:.1f}万"
else:
return f"{num:,}"
def generate_markdown(result, bvid):
"""生成Markdown笔记"""
metadata = result['metadata']
stats = result['stats']
description = result['description']
subtitle = result['subtitle']
# 获取当前时间
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
md = f"""# {metadata['title']}
> **来源**: [B站](https://www.bilibili.com/video/{bvid})
> **作者**: {metadata['author']}
> **发布时间**: {metadata['pubtime']}
> **提取时间**: {now}
> **时长**: {metadata['duration']}
## 📊 统计数据
| 指标 | 数量 |
|------|------|
| 👀 播放 | {format_number(stats['view'])} |
| 💬 弹幕 | {format_number(stats['danmaku'])} |
| 📝 评论 | {format_number(stats['reply'])} |
| ⭐ 收藏 | {format_number(stats['favorite'])} |
| 🪙 投币 | {format_number(stats['coin'])} |
| 🔗 分享 | {format_number(stats['share'])} |
| 👍 点赞 | {format_number(stats['like'])} |
## 📝 视频简介
{description if description else '(暂无简介)'}
## 📌 标签
{metadata['tags'] if metadata['tags'] else '(无标签)'}
## 🎬 字幕
{subtitle if subtitle else '(无字幕)'}
---
**元数据**:
- BV号: `{metadata['bvid']}`
- AV号: `{metadata['aid']}`
- UP主ID: `{metadata['mid']}`
"""
return md
def main():
"""主函数"""
# BV号
bvid = "BV1N5foBjE5K" # 从 https://b23.tv/btMGoOi 解析得到
print(f"开始提取B站视频: {bvid}")
print("=" * 60)
# 获取配置
try:
config = get_config()
except Exception as e:
print(f"⚠️ 获取配置失败，使用默认配置: {e}")
config = None
# 创建提取器
extractor = BilibiliExtractor(config=config)
# 提取视频信息
try:
result = extractor.extract(bvid)
print("\n✅ 提取成功！\n")
print("=" * 60)
print("基本信息:")
print("=" * 60)
print(f"标题: {result['metadata']['title']}")
print(f"作者: {result['metadata']['author']}")
print(f"时长: {result['metadata']['duration']}")
print(f"发布: {result['metadata']['pubtime']}")
print("\n" + "=" * 60)
print("统计数据:")
print("=" * 60)
print(f"播放: {format_number(result['stats']['view'])}")
print(f"弹幕: {format_number(result['stats']['danmaku'])}")
print(f"评论: {format_number(result['stats']['reply'])}")
print(f"收藏: {format_number(result['stats']['favorite'])}")
print(f"投币: {format_number(result['stats']['coin'])}")
print(f"点赞: {format_number(result['stats']['like'])}")
# 生成 Markdown 笔记
md_content = generate_markdown(result, bvid)
# 确保 notes 目录存在
notes_dir = "/home/admin/.openclaw/workspace/notes"
os.makedirs(notes_dir, exist_ok=True)
# 生成文件名（使用BV号作为文件名）
filename = f"{bvid}.md"
filepath = os.path.join(notes_dir, filename)
# 写入文件
with open(filepath, 'w', encoding='utf-8') as f:
f.write(md_content)
print("\n" + "=" * 60)
print(f"✅ Markdown笔记已生成: {filepath}")
print("=" * 60)
# 返回笔记内容
return md_content
except Exception as e:
print(f"\n❌ 提取失败: {e}")
import traceback
traceback.print_exc()
return None
if __name__ == '__main__':
main()
skills/kb-bilibili-content/video_downloader.py
-175
View File
@@ -1,175 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
使用旧版 API 获取并下载视频流
"""
import asyncio
import json
import os
from playwright.async_api import async_playwright
async def download_video_stream(bvid, output_dir="/tmp/bilibili_downloads"):
"""下载视频流"""
url = f"https://www.bilibili.com/video/{bvid}"
# 创建输出目录
os.makedirs(output_dir, exist_ok=True)
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
print(f"正在下载视频: {url}")
print("=" * 80)
# 首先获取 aid 和 cid
print("\n1. 获取视频信息...")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') != 0:
print(f" ❌ 失败: {data.get('message')}")
await browser.close()
return None
video_data = data.get('data', {})
title = video_data.get('title')
aid = video_data.get('aid')
cid = video_data.get('cid')
duration = video_data.get('duration')
print(f" 标题: {title}")
print(f" 时长: {duration}秒")
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
return None
# 获取视频流 URL
print("\n2. 获取视频流 URL...")
try:
playurl_api = "https://api.bilibili.com/x/player/playurl"
params = {
'bvid': bvid,
'cid': cid,
'qn': 80, # 1080P
'otype': 'json',
'fnver': 0,
'fnval': 0, # FLV 格式
}
playurl_response = await page.request.get(playurl_api, params=params)
if not playurl_response.ok:
print(f" ❌ HTTP 错误: {playurl_response.status}")
await browser.close()
return None
playurl_data = await playurl_response.json()
if playurl_data.get('code') != 0:
print(f" ❌ API 错误: {playurl_data.get('message')}")
await browser.close()
return None
play_data = playurl_data.get('data', {})
# 检查支持的清晰度
print(f"\n 支持的清晰度:")
for i, (q, desc) in enumerate(zip(play_data.get('accept_quality', []), play_data.get('accept_description', []))):
current = " (当前)" if q == play_data.get('quality') else ""
print(f" {desc} ({q}){current}")
# 获取视频 URL
durl = play_data.get('durl', [])
if not durl:
print(f" ❌ 没有找到视频流")
await browser.close()
return None
print(f"\n 找到 {len(durl)} 个视频分段")
video_urls = []
for i, segment in enumerate(durl):
segment_url = segment.get('url')
segment_size = segment.get('size', 0)
video_urls.append(segment_url)
print(f" 分段 {i+1}: {segment_size / 1024 / 1024:.2f}MB")
print(f" {segment_url[:100]}...")
except Exception as e:
print(f" ❌ 异常: {e}")
import traceback
traceback.print_exc()
await browser.close()
return None
# 下载视频
print(f"\n3. 开始下载视频...")
all_segments = []
total_size = 0
for i, video_url in enumerate(video_urls, 1):
print(f"\n 下载分段 {i}/{len(video_urls)}...")
try:
# 添加必要的请求头
headers = {
'Referer': 'https://www.bilibili.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
}
# 下载分段
segment_response = await page.request.get(
video_url,
headers=headers,
timeout=60000
)
if not segment_response.ok:
print(f" ❌ 下载失败: {segment_response.status}")
continue
segment_data = await segment_response.body()
segment_size = len(segment_data)
total_size += segment_size
all_segments.append(segment_data)
print(f" ✅ 下载成功: {segment_size / 1024 / 1024:.2f}MB")
except Exception as e:
print(f" ❌ 下载异常: {e}")
continue
if not all_segments:
print(f"\n ❌ 没有成功下载任何分段")
await browser.close()
return None
# 合并分段
print(f"\n4. 合并视频分段...")
video_data = b''.join(all_segments)
# 保存文件
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
output_path = os.path.join(output_dir, f"{safe_title}.flv")
with open(output_path, 'wb') as f:
f.write(video_data)
print(f" ✅ 视频已保存: {output_path}")
print(f" 文件大小: {total_size / 1024 / 1024:.2f}MB")
await browser.close()
return output_path
if __name__ == '__main__':
bvid = "BV1KtFzzqEUG"
result = asyncio.run(download_video_stream(bvid))
if result:
print(f"\n✅ 下载完成！")
print(f"文件路径: {result}")
else:
print(f"\n❌ 下载失败")
skills/kb-deep-analyzer/deep_analyzer.py
+70 -10
View File
@@ -141,14 +141,26 @@ class DeepAnalyzer:
# 标准化元数据
metadata = self._normalize_metadata(metadata)
title = metadata.get('title', '未知')
duration = metadata.get('duration', 0)
author = metadata.get('author', '')
pubtime = metadata.get('pubtime', '')
url = metadata.get('url', '')
bvid = metadata.get('bvid', '')
platform = metadata.get('platform', 'bilibili')
# 推断 platform：优先使用元数据中的值，否则根据 URL 或 content_type 推断
platform = metadata.get('platform', '')
if not platform:
if 'zhihu.com' in url or content_type in ('answer', 'zhuanlan', 'question'):
platform = 'zhihu'
elif 'bilibili.com' in url or bvid:
platform = 'bilibili'
else:
platform = 'bilibili' # 默认值
# 清理 author 字段：移除零宽度空格和前后空白
author = self._clean_author(author)
# 获取内容 - 支持多种字段名
text = content.get('text', '') or content.get('subtitle', '') or content.get('subtitles', '') or content.get('description', '')
@@ -186,11 +198,11 @@ class DeepAnalyzer:
return result
def _build_prompt(self, title: str, duration: int, duration_str: str, author: str,
pubtime: str, url: str, bvid: str, platform: str, text: str,
def _build_prompt(self, title: str, duration: int, duration_str: str, author: str,
pubtime: str, url: str, bvid: str, platform: str, text: str,
content_type: str = "video") -> str:
"""构建分析 prompt，根据内容类型选择不同模板"""
if content_type == "article":
if content_type in ("article", "answer", "zhuanlan"):
return self._build_article_prompt(title, author, pubtime, url, platform, text)
else:
return self._build_video_prompt(title, duration, duration_str, author, pubtime, url, bvid, platform, text)
@@ -422,6 +434,22 @@ URL：{url}
return f"{h}:{m:02d}:{s:02d}"
return f"{m}:{s:02d}"
def _clean_author(self, author: str) -> str:
"""清理 author 字段：移除零宽度空格和前后空白"""
if not author:
return ''
# 移除零宽度空格 (U+200B) 和其他不可见字符
invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u00ad']
cleaned = author
for char in invisible_chars:
cleaned = cleaned.replace(char, '')
# 移除前后空白和换行
cleaned = cleaned.strip()
# 如果包含换行符，只取第一行
if '\n' in cleaned:
cleaned = cleaned.split('\n')[0].strip()
return cleaned
# ============== 工具函数 ==============
@@ -437,19 +465,34 @@ def generate_frontmatter(metadata: Dict, result: AnalysisResult, content_type: s
"""生成 Markdown frontmatter"""
# 提取标签
tags = result.recommended_tags or []
# 格式化日期
pubtime = metadata.get('pubtime', '')
if pubtime:
date_str = str(pubtime)[:10]
else:
date_str = datetime.now().strftime('%Y-%m-%d')
fm = ["---",
# 清理 author 字段
author = metadata.get('author', '未知')
author = _clean_string(author)
# 推断 platform
url = metadata.get('url', '')
platform = metadata.get('platform', '')
if not platform:
if 'zhihu.com' in url or content_type in ('answer', 'zhuanlan', 'question'):
platform = 'zhihu'
elif 'bilibili.com' in url or metadata.get('bvid'):
platform = 'bilibili'
else:
platform = 'bilibili'
fm = ["---",
f"title: \"{metadata.get('title', '未命名')}\"",
f"date: {date_str}",
f"author: {metadata.get('author', '未知')}",
f"platform: {metadata.get('platform', 'bilibili')}"]
f"author: {author}",
f"platform: {platform}"]
# 视频类型添加 duration 和 bvid
if content_type == "video":
@@ -470,6 +513,23 @@ def generate_frontmatter(metadata: Dict, result: AnalysisResult, content_type: s
return '\n'.join(fm)
def _clean_string(text: str) -> str:
"""清理字符串：移除零宽度空格和前后空白"""
if not text:
return ''
# 移除零宽度空格 (U+200B) 和其他不可见字符
invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u00ad']
cleaned = text
for char in invisible_chars:
cleaned = cleaned.replace(char, '')
# 移除前后空白和换行
cleaned = cleaned.strip()
# 如果包含换行符，只取第一行
if '\n' in cleaned:
cleaned = cleaned.split('\n')[0].strip()
return cleaned
# ============== 主程序 ==============
def main():
skills/kb-funasr/.env.example
-2
View File
@@ -1,2 +0,0 @@
# FunASR 模型缓存目录
FUNASR_CACHE_DIR=~/.cache/modelscope/hub
skills/kb-funasr/README.md
-212
View File
@@ -1,212 +0,0 @@
# KB FunASR - 本地语音识别
基于阿里 FunASR 的本地语音识别工具，支持将音视频文件转换为文本和 JSON 格式。
## 功能特性
- 🎯 **本地执行** - 完全离线运行，保护隐私
- 🚀 **多模型支持** - Paraformer、SenseVoice 等模型可选
- 📝 **多格式输出** - JSON、TXT 两种格式
- ⚡ **异步执行** - 支持后台运行，不阻塞主代理
- 🎬 **广泛兼容** - 支持常见音视频格式（MP3、MP4、WAV 等）
- 🇨🇳 **中文优化** - Paraformer 模型针对中文语音优化
## 安装前提
### 1. 安装依赖
```bash
pip install funasr modelscope
```
### 2. 配置环境变量（可选）
在 `~/.clawdbot/clawdbot.json` 中添加：
```json5
{
skills: {
entries: {
"kb-funasr": {
env: {
FUNASR_CACHE_DIR: "~/.cache/modelscope/hub"
}
}
}
}
}
```
或通过命令行配置：
```bash
clawdbot skills config kb-funasr FUNASR_CACHE_DIR "~/.cache/modelscope/hub"
```
## 使用方法
### 基本用法
```bash
# 使用默认 Paraformer 中文模型处理
python funasr.py --input video.mp4
# 后台异步执行
python funasr.py --input video.mp4 --background
```
### 指定模型
```bash
# Paraformer 中文模型（推荐）
python funasr.py --input video.mp4 --model paraformer-zh
# Paraformer 多语言模型
python funasr.py --input video.mp4 --model paraformer
# SenseVoice 多语言模型
python funasr.py --input video.mp4 --model sensevoice
```
### 指定输出目录
```bash
python funasr.py --input video.mp4 --output ./transcripts
```
## 输出文件
对于输入 `video.mp4`，将生成以下文件：
```
output/
├── video.json # 详细结果（含时间戳）
└── video.txt # 纯文本
```
### JSON 格式
```json
{
"success": true,
"input_file": "video.mp4",
"model": "paraformer-zh",
"output_files": {
"json": "video.json",
"txt": "video.txt"
},
"duration_seconds": 120.5,
"segments": [
{
"id": 0,
"start": 0.0,
"end": 3.2,
"text": "这是第一段识别结果"
},
{
"id": 1,
"start": 3.2,
"end": 6.8,
"text": "这是第二段识别结果"
}
]
}
```
## 模型对比
| 模型 | 速度 | 精度 | 适用场景 |
|------|------|------|----------|
| paraformer-zh | ⚡⚡⚡ | ⭐⭐⭐⭐ | 中文语音（推荐） |
| paraformer | ⚡⚡ | ⭐⭐⭐⭐ | 多语言识别 |
| sensevoice | ⚡ | ⭐⭐⭐⭐⭐ | 高精度多语言 |
## 完整示例
### 示例 1: 转录播客
```bash
python funasr.py --input podcast.mp3 --model paraformer-zh --background
```
### 示例 2: 生成视频识别结果
```bash
python funasr.py --input tutorial.mp4 --model paraformer
```
### 示例 3: 批量处理（配合脚本）
```bash
#!/bin/bash
for video in videos/*.mp4; do
python funasr.py --input "$video" --model paraformer-zh
done
```
## 在 Workflow 中使用
在知识库处理流程中，如果视频没有识别结果：
```bash
# 步骤4: 下载视频后
# 步骤5: 使用 FunASR 生成识别结果
python funasr.py --input ${video_path} --model paraformer-zh --background
# 等待 FunASR 完成，然后继续后续步骤
```
## 技术细节
- **引擎**: 阿里 FunASR
- **模型框架**: ModelScope
- **音频采样率**: 自动转换为 16kHz
- **语言**: 根据模型支持不同语言
## 故障排查
### 问题: 模型下载失败
```bash
# 首次运行会自动下载，确保网络稳定
python funasr.py --input test.mp4
# 手动检查缓存目录
ls ~/.cache/modelscope/hub/
```
### 问题: 音频格式不支持
```bash
# 转换音频格式
ffmpeg -i input.avi -acodec libmp3lame output.mp3
python funasr.py --input output.mp3
```
### 问题: 依赖安装失败
```bash
# 使用清华镜像源
pip install funasr modelscope -i https://pypi.tuna.tsinghua.edu.cn/simple
```
## 性能参考
在 CPU 环境下（Intel i7）：
- **paraformer-zh 模型**: ~2x 实时速度（1分钟音频 ~30秒处理）
- **paraformer 模型**: ~1.5x 实时速度（1分钟音频 ~40秒处理）
- **sensevoice 模型**: ~1x 实时速度（1分钟音频 ~1分钟处理）
## 与 Whisper 对比
| 特性 | FunASR | Whisper |
|------|--------|---------|
| 中文识别 | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ |
| 速度 | ⚡⚡⚡ | ⚡⚡ |
| 模型大小 | 较小 | 较大 |
| 多语言支持 | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
## 许可证
本技能使用 MIT 许可证，FunASR 使用 Apache 2.0 许可证。
skills/kb-funasr/SKILL.md
-113
View File
@@ -1,113 +0,0 @@
---
name: kb-funasr
description: 本地语音识别，支持音视频转文字（基于阿里 FunASR）
---
# KB FunASR Skill
**kb-funasr** - 本地语音识别，使用阿里 FunASR 将音视频文件转换为文本和 JSON 格式输出。
## 什么时候使用
当用户需要：
- 将视频/音频文件转换为文字
- 获取带时间戳的详细识别结果
- 需要中文优化的语音识别（Paraformer 模型）
## 环境变量
在使用之前，必须检查环境变量是否已经设置。
| 变量名 | 用途 | 示例 |
|--------|------|------|
| `FUNASR_CACHE_DIR` | FunASR 模型缓存目录 | `~/.cache/modelscope/hub` |
## 使用方法
```bash
# 基本用法（使用 Paraformer 模型）
python funasr.py --input video.mp4
# 指定模型
python funasr.py --input video.mp4 --model paraformer-zh
# 指定输出目录
python funasr.py --input video.mp4 --output ./output
# 异步执行（推荐）
python funasr.py --input video.mp4 --background
```
## 命令参数
| 参数 | 说明 | 默认值 |
|------|------|--------|
| `--input` | 输入音视频文件路径 | 必需 |
| `--model` | FunASR 模型类型 | `paraformer-zh` |
| `--output` | 输出目录 | 文件所在目录 |
| `--background` | 后台异步执行 | False |
## 模型选项
- `paraformer-zh` - Paraformer 中文模型（轻量，中文优化，推荐）
- `paraformer` - Paraformer 多语言模型
- `sensevoice` - SenseVoice 多语言模型
## 输出文件
对于输入 `video.mp4`，将生成：
- `video.json` - 详细结果（包含时间戳、分段信息）
- `video.txt` - 纯文本内容
## 输出格式示例
### JSON 格式
```json
{
"success": true,
"input_file": "video.mp4",
"model": "paraformer-zh",
"output_files": {
"json": "video.json",
"txt": "video.txt"
},
"duration_seconds": 120.5,
"segments": [
{
"id": 0,
"start": 0.0,
"end": 3.2,
"text": "这是第一段识别结果"
}
]
}
```
## Workflow 中的使用
```bash
# 步骤4之后：如果已有音频但无字幕，可以使用 FunASR
python funasr.py --input ${video_path} --model paraformer-zh --background
```
## 安装
```bash
# 1. 安装依赖
pip install funasr modelscope
# 2. 首次运行时会自动下载模型到缓存目录
python funasr.py --input test.mp4
# 3. 复制技能到 skills 目录
cp -r kb-funasr/ ~/skills/
```
## 注意事项
- **异步执行**: 使用 `--background` 参数时，命令会在后台运行，不阻塞主代理
- **模型选择**: paraformer-zh 模型适合中文语音识别，速度较快，精度较高
- **文件格式**: 支持 MP3、MP4、WAV、FLAC、OGG、M4A 等常见格式
- **首次运行**: 首次运行时会自动下载模型文件，需要稳定的网络连接
skills/kb-funasr/__pycache__/funasr.cpython-312.pyc
BIN
View File
Binary file not shown.
skills/kb-funasr/funasr_tool.py
-313
View File
@@ -1,313 +0,0 @@
#!/usr/bin/env python3
"""
KB FunASR - 本地语音识别工具
基于阿里 FunASR 实现音视频转文字
"""
import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from datetime import datetime
from typing import Dict, List, Optional
class FunASRTranscriber:
"""FunASR 语音识别器"""
# 模型映射
MODEL_MAP = {
"paraformer-zh": "damo/speech_paraformer-large-vad_punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
"paraformer": "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
"sensevoice": "FunASR/SenseVoiceSmall"
}
def __init__(self, model: str = "paraformer-zh", cache_dir: Optional[str] = None):
"""初始化 FunASR 识别器
Args:
model: 模型名称
cache_dir: 模型缓存目录
"""
self.model_name = model
self.model_id = self.MODEL_MAP.get(model, self.MODEL_MAP["paraformer-zh"])
self.cache_dir = cache_dir or os.path.expanduser("~/.cache/modelscope/hub")
self._model = None
def _load_model(self):
"""延迟加载模型"""
if self._model is None:
try:
from funasr import AutoModel
print(f"正在加载模型: {self.model_name} ({self.model_id})")
print(f"缓存目录: {self.cache_dir}")
self._model = AutoModel(
model=self.model_id,
disable_pbar=False,
disable_log=False,
)
print("模型加载完成")
except ImportError:
print("错误: 未安装 funasr 库")
print("请运行: pip install funasr modelscope")
sys.exit(1)
def _convert_audio(self, input_path: str) -> str:
"""转换音频格式为 16kHz WAV
Args:
input_path: 输入文件路径
Returns:
转换后的文件路径（如果不需要转换则返回原路径）
"""
# 检查是否已经是 WAV 格式
if input_path.lower().endswith('.wav'):
# 检查采样率
try:
result = subprocess.run(
['ffprobe', '-v', 'error', '-show_entries',
'stream=sample_rate', '-of', 'default=noprint_wrappers=1:nokey=1',
input_path],
capture_output=True, text=True
)
if result.returncode == 0:
sample_rate = int(result.stdout.strip())
if sample_rate == 16000:
return input_path
except Exception:
pass
# 需要转换
output_path = input_path.rsplit('.', 1)[0] + '_converted.wav'
print(f"正在转换音频格式: {input_path} -> {output_path}")
try:
subprocess.run([
'ffmpeg', '-y', '-i', input_path,
'-acodec', 'pcm_s16le',
'-ar', '16000',
'-ac', '1',
output_path
], check=True, capture_output=True)
print("音频转换完成")
return output_path
except subprocess.CalledProcessError as e:
print(f"音频转换失败: {e}")
# 如果转换失败，尝试使用原文件
print("尝试使用原文件继续处理")
return input_path
except FileNotFoundError:
print("警告: 未找到 ffmpeg，跳过音频转换")
return input_path
def transcribe(self, input_path: str) -> Dict:
"""转录音频文件
Args:
input_path: 输入文件路径
Returns:
识别结果字典
"""
input_path = os.path.abspath(input_path)
if not os.path.exists(input_path):
return {
"success": False,
"error": f"文件不存在: {input_path}"
}
print(f"\n开始处理: {input_path}")
print(f"使用模型: {self.model_name}")
# 转换音频格式
audio_path = self._convert_audio(input_path)
# 加载模型
self._load_model()
# 执行识别
print("正在进行语音识别...")
try:
result = self._model.generate(
input=audio_path,
batch_size_s=300,
)
except Exception as e:
return {
"success": False,
"error": f"识别失败: {str(e)}"
}
# 解析结果
if not result or not isinstance(result, list):
return {
"success": False,
"error": "识别结果格式错误"
}
# FunASR 返回格式处理
segments = []
full_text = ""
for seg in result:
if isinstance(seg, dict):
text = seg.get("text", "").strip()
if "timestamp" in seg and seg["timestamp"]:
# 包含时间戳
timestamp = seg["timestamp"]
if len(timestamp) >= 2:
start_time = timestamp[0][0] if timestamp[0] else 0
end_time = timestamp[-1][1] if timestamp[-1] else start_time
else:
start_time = 0
end_time = 0
else:
start_time = 0
end_time = 0
if text:
segments.append({
"id": len(segments),
"start": round(start_time, 2),
"end": round(end_time, 2),
"text": text
})
full_text += text + "\n"
# 清理临时文件
if audio_path != input_path and "_converted.wav" in audio_path:
try:
os.remove(audio_path)
except Exception:
pass
# 计算时长
duration = segments[-1]["end"] if segments else 0
return {
"success": True,
"input_file": os.path.basename(input_path),
"model": self.model_name,
"output_files": {},
"duration_seconds": round(duration, 2),
"segments": segments,
"text": full_text.strip()
}
def save_outputs(self, result: Dict, output_dir: str):
"""保存输出文件
Args:
result: 识别结果
output_dir: 输出目录
"""
if not result.get("success"):
return
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
basename = Path(result["input_file"]).stem
# 保存 JSON
json_path = output_dir / f"{basename}.json"
with open(json_path, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, indent=2)
print(f"已保存: {json_path}")
result["output_files"]["json"] = str(json_path)
# 保存 TXT
txt_path = output_dir / f"{basename}.txt"
with open(txt_path, "w", encoding="utf-8") as f:
f.write(result.get("text", ""))
print(f"已保存: {txt_path}")
result["output_files"]["txt"] = str(txt_path)
def run_as_background(input_path: str, model: str, output_dir: str):
"""后台运行识别
Args:
input_path: 输入文件路径
model: 模型名称
output_dir: 输出目录
"""
# 创建日志文件
log_file = os.path.join(output_dir, f"funasr_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log")
# 重新执行命令（后台模式）
cmd = [
sys.executable, __file__,
"--input", input_path,
"--model", model,
"--output", output_dir
]
with open(log_file, "w") as f:
process = subprocess.Popen(
cmd,
stdout=f,
stderr=subprocess.STDOUT,
start_new_session=True
)
print(f"\n✓ 后台任务已启动 (PID: {process.pid})")
print(f"日志文件: {log_file}")
print(f"输出目录: {output_dir}")
print("\n使用以下命令查看进度:")
print(f" tail -f {log_file}")
def main():
parser = argparse.ArgumentParser(
description="KB FunASR - 本地语音识别工具",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
python funasr.py --input video.mp4
python funasr.py --input podcast.mp3 --model paraformer-zh
python funasr.py --input video.mp4 --output ./transcripts --background
"""
)
parser.add_argument("--input", "-i", required=True, help="输入音视频文件路径")
parser.add_argument("--model", "-m", default="paraformer-zh",
choices=["paraformer-zh", "paraformer", "sensevoice"],
help="模型名称 (默认: paraformer-zh)")
parser.add_argument("--output", "-o", default=None, help="输出目录 (默认: 文件所在目录)")
parser.add_argument("--background", "-b", action="store_true", help="后台异步执行")
args = parser.parse_args()
# 确定输出目录
if args.output:
output_dir = args.output
else:
output_dir = os.path.dirname(args.input) or "."
# 后台模式
if args.background:
run_as_background(args.input, args.model, output_dir)
return
# 前台模式
transcriber = FunASRTranscriber(model=args.model)
result = transcriber.transcribe(args.input)
if result.get("success"):
transcriber.save_outputs(result, output_dir)
print(f"\n✓ 处理完成!")
print(f" 时长: {result['duration_seconds']} 秒")
print(f" 分段数: {len(result['segments'])}")
else:
print(f"\n✗ 处理失败: {result.get('error')}")
sys.exit(1)
if __name__ == "__main__":
main()
skills/kb-link-parser/SKILL.md
-43
View File
@@ -1,43 +0,0 @@
# KB Link Parser Skill
**kb-link-parser** - Extract and resolve IDs from various platform URLs.
## Features
- Multi-platform support: Bilibili, YouTube, Zhihu
- Short link detection: b23.tv and other short links
- Multiple content types: video, article, up主, playlist, collection
- Batch processing from file
- JSON output format
## Commands
```bash
# Parse single URL
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
# Batch processing
python link-parser.py --batch urls.txt --output results.json
# Output format
python link-parser.py --url "..." --format json
```
## Output
```json
{
"success": true,
"data": {
"platform": "bilibili",
"type": "video",
"id": "BV1xx411c7mD"
}
}
```
## Installation
```bash
cp -r kb-link-parser/ ~/skills/
```
skills/kb-link-parser/__pycache__/link_parser_shortlink.cpython-312.pyc
BIN
View File
Binary file not shown.
skills/kb-link-parser/link-parser.py
-637
View File
@@ -1,637 +0,0 @@
#!/usr/bin/env python3
"""
Link Parser - Extract and resolve IDs from various platform URLs.
Supports:
- Bilibili (video, article, up主, collection, series)
- YouTube (video, channel, playlist)
- Zhihu (question, answer, user, collection)
- Xiaohongshu (note, user)
- Short links (b23.tv, xhslink.com, etc.)
Usage:
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
python link-parser.py --batch urls.txt --output results.json
"""
import argparse
import json
import re
import sys
from typing import Optional, Dict, Any
from urllib.parse import urlparse
from link_parser_shortlink import ShortLinkResolver
class LinkParser:
"""Main link parser class"""
# Bilibili patterns
BILIBILI_VIDEO_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV[a-zA-Z0-9]+)')
BILIBILI_VIDEO_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
BILIBILI_ARTICLE_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/(?:article|read)/([a-zA-Z0-9]+)')
BILIBILI_SPACE_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/(\d+)')
BILIBILI_MOBILE_SPACE_PATTERN = re.compile(r'https?://m\.bilibili\.com/space/(\d+)')
BILIBILI_MOBILE_OPUS_PATTERN = re.compile(r'https?://m\.bilibili\.com/opus/(\d+)')
BILIBILI_FAVLIST_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/favlist\?fid=(\d+)')
BILIBILI_OPUS_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/opus/(\d+)')
BILIBILI_SERIES_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/lists/(\d+)\?type=series')
BILIBILI_READLIST_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/read/readlist/(rl[a-zA-Z0-9]+)')
# YouTube patterns
YOUTUBE_VIDEO_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/)([a-zA-Z0-9_-]{11})')
YOUTUBE_CHANNEL_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/channel/([a-zA-Z0-9_-]+)')
YOUTUBE_USER_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/user/([a-zA-Z0-9_-]+)')
YOUTUBE_PLAYLIST_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/playlist\?list=([a-zA-Z0-9_-]+)')
# Xiaohongshu (小红书) patterns
XIAOHONGSHU_SHORT_PATTERN = re.compile(r'https?://xhslink\.com/o/([a-zA-Z0-9]+)')
XIAOHONGSHU_NOTE_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/explore/([a-zA-Z0-9]+)')
XIAOHONGSHU_NOTE_PATTERN2 = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/notes/([a-zA-Z0-9]+)')
XIAOHONGSHU_DISCOVERY_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/discovery/item/([a-zA-Z0-9]+)')
XIAOHONGSHU_USER_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/user/profile/(\d+)')
# Zhihu patterns
ZHIHAU_QUESTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/(\d+)')
ZHIHAU_ANSWER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)')
ZHIHAU_USER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/people/([a-zA-Z0-9_-]+)')
ZHIHAU_COLUMN_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/column/([a-zA-Z0-9_-]+)')
ZHIHAU_COLLECTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/collection/(\d+)')
# Additional Zhihu patterns
ZHIHAU_ZHUANLAN_PATTERN = re.compile(r'https?://zhuanlan\.zhihu\.com/p/(\d+)')
ZHIHAU_ORG_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/([a-zA-Z0-9_-]+)')
ZHIHAU_ORG_ANSWERS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/answers')
ZHIHAU_ORG_VIDEOS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/zvideos')
ZHIHAU_ORG_ASKS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/asks')
ZHIHAU_ORG_POSTS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/posts')
ZHIHAU_ORG_PINS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/pins')
ZHIHAU_ZVIDEO_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/zvideo/(\d+)')
def __init__(self):
self.shortlink_resolver = ShortLinkResolver()
def resolve_shortlink(self, url: str) -> Optional[str]:
"""
Resolve short link to get final URL.
Args:
url: Short URL to resolve
Returns:
Final URL after resolution, or None if not resolvable
"""
# Check if it's a known short link pattern
if self.shortlink_resolver.is_bilibili_short_link(url):
# Try to resolve using requests (follows redirects)
try:
import requests
response = requests.head(url, allow_redirects=True, timeout=10)
return response.url
except Exception:
# Fallback: return original URL
return url
# Check if it's Xiaohongshu short link
if self.XIAOHONGSHU_SHORT_PATTERN.match(url):
try:
import requests
response = requests.get(url, allow_redirects=True, timeout=10)
return response.url
except Exception:
return url
return url
def extract_bilibili_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Bilibili ID from URL.
Args:
url: Bilibili URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try series/playlist (must check before space pattern)
match = self.BILIBILI_SERIES_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "playlist",
"id": match.group(1)
}
# Try readlist (专栏合集)
match = self.BILIBILI_READLIST_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "readlist",
"id": match.group(1)
}
# Try favlist (收藏夹)
match = self.BILIBILI_FAVLIST_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "favlist",
"id": match.group(1)
}
# Try opus (动态)
match = self.BILIBILI_OPUS_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "opus",
"id": match.group(1)
}
# Try mobile opus (移动端动态)
match = self.BILIBILI_MOBILE_OPUS_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "opus",
"id": match.group(1)
}
# Try video
match = self.BILIBILI_VIDEO_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "video",
"id": match.group(1)
}
# Try article/read
match = self.BILIBILI_ARTICLE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "article",
"id": match.group(1)
}
# Try space (UP主) - must check after specific patterns
match = self.BILIBILI_SPACE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "up主",
"id": match.group(1)
}
# Try mobile space (移动端UP主空间)
match = self.BILIBILI_MOBILE_SPACE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "up主",
"id": match.group(1)
}
return None
def extract_youtube_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract YouTube ID from URL.
Args:
url: YouTube URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try video
match = self.YOUTUBE_VIDEO_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "video",
"id": match.group(1)
}
# Try channel
match = self.YOUTUBE_CHANNEL_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "channel",
"id": match.group(1)
}
# Try user
match = self.YOUTUBE_USER_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "user",
"id": match.group(1)
}
# Try playlist
match = self.YOUTUBE_PLAYLIST_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "playlist",
"id": match.group(1)
}
return None
def extract_zhihu_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Zhihu ID from URL.
Args:
url: Zhihu URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try question
match = self.ZHIHAU_QUESTION_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "question",
"id": match.group(1)
}
# Try answer
match = self.ZHIHAU_ANSWER_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "answer",
"id": match.group(1)
}
# Try user
match = self.ZHIHAU_USER_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "user",
"id": match.group(1)
}
# Try column
match = self.ZHIHAU_COLUMN_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "column",
"id": match.group(1)
}
# Try collection
match = self.ZHIHAU_COLLECTION_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "collection",
"id": match.group(1)
}
# Try zhuanlan article (专栏文章)
match = self.ZHIHAU_ZHUANLAN_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "zhuanlan",
"id": match.group(1)
}
# Try org page (机构页面)
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org",
"id": match.group(1)
}
# Try org answers (机构回答)
if self.ZHIHAU_ORG_ANSWERS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_answers",
"id": match.group(1)
}
# Try org zvideos (机构视频)
if self.ZHIHAU_ORG_VIDEOS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_zvideos",
"id": match.group(1)
}
# Try org asks (机构提问)
if self.ZHIHAU_ORG_ASKS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_asks",
"id": match.group(1)
}
# Try org posts (机构文章)
if self.ZHIHAU_ORG_POSTS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_posts",
"id": match.group(1)
}
# Try org pins (机构想法)
if self.ZHIHAU_ORG_PINS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_pins",
"id": match.group(1)
}
# Try zvideo (知乎视频)
match = self.ZHIHAU_ZVIDEO_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "zvideo",
"id": match.group(1)
}
return None
def extract_xiaohongshu_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Xiaohongshu ID from URL.
Args:
url: Xiaohongshu URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try short link
match = self.XIAOHONGSHU_SHORT_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try explore link
match = self.XIAOHONGSHU_NOTE_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try notes link
match = self.XIAOHONGSHU_NOTE_PATTERN2.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try discovery item link (resolved from short link)
match = self.XIAOHONGSHU_DISCOVERY_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try user profile
match = self.XIAOHONGSHU_USER_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "user",
"id": match.group(1)
}
return None
def parse(self, url: str) -> Dict[str, Any]:
"""
Parse URL and extract platform-specific ID.
Args:
url: URL to parse
Returns:
Dict with platform, type, id, and original_url fields
"""
result = {
"platform": None,
"type": None,
"id": None,
"original_url": url,
"resolved_url": None,
"success": False,
"error": None
}
# First, resolve short links
resolved = self.resolve_shortlink(url)
if resolved:
result["resolved_url"] = resolved
url = resolved
# Try each platform
bilibili_result = self.extract_bilibili_id(url)
if bilibili_result:
result.update(bilibili_result)
result["success"] = True
return result
youtube_result = self.extract_youtube_id(url)
if youtube_result:
result.update(youtube_result)
result["success"] = True
return result
zhihu_result = self.extract_zhihu_id(url)
if zhihu_result:
result.update(zhihu_result)
result["success"] = True
return result
# Try Xiaohongshu
xiaohongshu_result = self.extract_xiaohongshu_id(url)
if xiaohongshu_result:
result.update(xiaohongshu_result)
result["success"] = True
return result
# Check for short links that need resolution
short_links = self.shortlink_resolver.detect_short_links(url)
if short_links:
result["error"] = "Short link detected. Please provide the full URL."
return result
result["error"] = "Unsupported URL format"
return result
def format_output(data: Dict[str, Any], output_format: str = "json") -> str:
"""Format output based on specified format."""
if output_format == "json":
return json.dumps(data, ensure_ascii=False, indent=2)
elif output_format == "compact":
return json.dumps(data, ensure_ascii=False, separators=(',', ':'))
elif output_format == "text":
lines = []
for key, value in data.items():
if value is not None:
lines.append(f"{key}: {value}")
return "\n".join(lines)
else:
return json.dumps(data, ensure_ascii=False, indent=2)
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(
description="Parse and extract IDs from various platform URLs",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s --url "https://www.bilibili.com/video/BV1xx411c7mD"
%(prog)s --url "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
%(prog)s --url "https://www.zhihu.com/question/266309957"
%(prog)s --batch urls.txt --output results.json
"""
)
parser.add_argument(
"--url", "-u",
help="Single URL to parse"
)
parser.add_argument(
"--batch", "-b",
help="File containing URLs to parse (one per line)"
)
parser.add_argument(
"--output", "-o",
help="Output file path (for batch mode)"
)
parser.add_argument(
"--format", "-f",
choices=["json", "compact", "text"],
default="json",
help="Output format (default: json)"
)
args = parser.parse_args()
# Check if we're in shortlink help mode
if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']:
parser.print_help()
return 0
if not args.url and not args.batch:
parser.print_help()
print("\nError: Please provide --url or --batch")
return 1
link_parser = LinkParser()
if args.url:
# Single URL mode
parsed = link_parser.parse(args.url)
# 统一输出格式
if parsed["success"]:
output = {
"success": True,
"platform": parsed.get("platform"),
"type": parsed.get("type"),
"id": parsed.get("id"),
"filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None,
"message": "URL解析成功"
}
else:
output = {
"success": False,
"error": parsed.get("error", "解析失败"),
"detail": f"URL: {args.url}"
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return 0 if output["success"] else 1
elif args.batch:
try:
with open(args.batch, 'r', encoding='utf-8') as f:
urls = [line.strip() for line in f if line.strip()]
except Exception as e:
print(json.dumps({
"success": False,
"error": "读取文件失败",
"detail": str(e)
}, ensure_ascii=False, indent=2))
return 1
results = []
for url in urls:
parsed = link_parser.parse(url)
if parsed["success"]:
results.append({
"success": True,
"platform": parsed.get("platform"),
"type": parsed.get("type"),
"id": parsed.get("id"),
"filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None
})
else:
results.append({
"success": False,
"error": parsed.get("error", "解析失败"),
"detail": f"URL: {url}"
})
if args.output:
with open(args.output, 'w', encoding='utf-8') as f:
json.dump(results, f, ensure_ascii=False, indent=2)
print(json.dumps({
"success": True,
"message": f"结果已保存到 {args.output}",
"total": len(results)
}, ensure_ascii=False, indent=2))
else:
for result in results:
print(json.dumps(result, ensure_ascii=False, indent=2))
print("-" * 40)
return 0
if __name__ == '__main__':
sys.exit(main())
skills/kb-link-parser/link_parser_shortlink.py
-296
View File
@@ -1,296 +0,0 @@
#!/usr/bin/env python3
"""
Short link resolver for link-parser skill.
Provides utilities for detecting and resolving short links, particularly
Bilibili short links (b23.tv) which may be blocked by anti-scraping measures.
Supported short link services:
- Bilibili: b23.tv
- Generic URL shorteners (detection only)
"""
import re
from typing import Optional, Dict, List, Tuple
from urllib.parse import urlparse
class ShortLinkResolver:
"""Short link detection and resolution handler"""
# Pattern for Bilibili short links
BILIBILI_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
# Pattern for other short link services
OTHER_SHORT_PATTERNS = {
'bit.ly': re.compile(r'https?://bit\.ly/\w+'),
't.cn': re.compile(r'https?://t\.cn/\w+'),
'dwz.cn': re.compile(r'https?://dwz\.cn/\w+'),
'goo.gl': re.compile(r'https?://goo\.gl/\w+'),
'tinyurl.com': re.compile(r'https?://tinyurl\.com/\w+'),
}
@staticmethod
def detect_short_links(text: str) -> List[Dict[str, str]]:
"""
Detect short links in text.
Args:
text: Input text to search
Returns:
List of detected short links with metadata:
[
{
'url': 'https://b23.tv/xxxxx',
'platform': 'bilibili',
'is_short': True
},
...
]
"""
detected = []
# Check for Bilibili short links
for match in ShortLinkResolver.BILIBILI_SHORT_PATTERN.finditer(text):
detected.append({
'url': match.group(0),
'code': match.group(1),
'platform': 'bilibili',
'is_short': True
})
# Check for other short link services
for platform, pattern in ShortLinkResolver.OTHER_SHORT_PATTERNS.items():
for match in pattern.finditer(text):
detected.append({
'url': match.group(0),
'platform': platform,
'is_short': True
})
return detected
@staticmethod
def is_bilibili_short_link(url: str) -> bool:
"""
Check if URL is a Bilibili short link.
Args:
url: URL to check
Returns:
True if URL is a b23.tv short link
"""
return bool(ShortLinkResolver.BILIBILI_SHORT_PATTERN.match(url))
@staticmethod
def format_short_link_help(short_link: Optional[str] = None) -> str:
"""
Format help message for short links.
Args:
short_link: Optional short link URL to mention
Returns:
Help message
"""
lines = [
"Detection: Short link",
""
]
if short_link:
lines.append(f"Short link: {short_link}")
lines.append("")
lines.extend([
"Due to anti-scraping protection, short links cannot be automatically resolved.",
"Please provide the full URL or the content ID (e.g., BV号 for Bilibili).",
"",
"Full URL format: https://www.bilibili.com/video/BVxxxxx"
])
return "\n".join(lines)
@staticmethod
def format_short_link_warning(short_links: List[Dict[str, str]]) -> str:
"""
Format warning message for multiple short links.
Args:
short_links: List of short link metadata
Returns:
Warning message
"""
if not short_links:
return ""
lines = [
"Short links detected:",
""
]
for i, link in enumerate(short_links, 1):
platform = link.get('platform', 'unknown')
url = link['url']
if platform == 'bilibili':
lines.append(f" [{i}] Bilibili short link: {url}")
lines.append(" Full URL or BV ID required")
else:
lines.append(f" [{i}] {platform}: {url}")
lines.append("")
return "\n".join(lines)
class PlaywrightResolver:
"""
Browser-based short link resolver using Playwright.
This is an optional implementation for environments where Playwright
is available. It can bypass anti-scraping measures by simulating
a real browser.
Requirements:
pip install playwright
playwright install chromium
"""
def __init__(self, headless: bool = True, timeout: int = 10000):
"""
Initialize Playwright resolver.
Args:
headless: Whether to run browser in headless mode
timeout: Page navigation timeout in milliseconds
"""
self.headless = headless
self.timeout = timeout
self._playwright = None
def _check_playwright_available(self) -> bool:
"""Check if Playwright is installed and available"""
try:
from playwright.sync_api import sync_playwright
return True
except ImportError:
return False
def resolve(self, short_url: str) -> Optional[str]:
"""
Resolve short link using Playwright browser automation.
Args:
short_url: Short URL to resolve
Returns:
Final URL after redirects, or None if resolution fails
"""
if not self._check_playwright_available():
raise ImportError(
"Playwright not installed. Install with: pip install playwright && playwright install chromium"
)
from playwright.sync_api import sync_playwright
try:
with sync_playwright() as p:
# Launch browser
browser = p.chromium.launch(headless=self.headless)
context = browser.new_context(
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
viewport={'width': 1920, 'height': 1080}
)
page = context.new_page()
# Navigate to short URL
page.goto(short_url, timeout=self.timeout, wait_until='networkidle')
# Get final URL
final_url = page.url
# Close browser
browser.close()
return final_url
except Exception as e:
print(f"Playwright resolution failed: {e}")
return None
def resolve_bilibili_short(self, short_url: str) -> Optional[str]:
"""
Resolve Bilibili short link and extract BV ID.
Args:
short_url: Bilibili short URL (e.g., https://b23.tv/xxxxx)
Returns:
Full Bilibili URL with BV ID, or None if resolution fails
"""
final_url = self.resolve(short_url)
if not final_url:
return None
# Check if final URL contains BV ID
if 'bilibili.com/video/' in final_url:
return final_url
# Extract BV ID if present in URL
bv_pattern = re.compile(r'(BV[a-zA-Z0-9]{10})')
match = bv_pattern.search(final_url)
if match:
return f"https://www.bilibili.com/video/{match.group(1)}"
return None
def get_resolver() -> ShortLinkResolver:
"""
Get the default resolver instance.
Returns:
ShortLinkResolver instance
"""
return ShortLinkResolver()
def detect_short_links(text: str) -> List[Dict[str, str]]:
"""
Detect short links in text.
Args:
text: Input text to search
Returns:
List of detected short links with metadata
"""
resolver = ShortLinkResolver()
return resolver.detect_short_links(text)
# CLI interface for testing
def main():
"""CLI for testing the short link resolver."""
import sys
if len(sys.argv) < 2:
print("Usage: python3 link_parser_shortlink.py 'text to test'")
print("\nExamples:")
print(' python3 link_parser_shortlink.py "Check this: https://b23.tv/oLBZZfK"')
print(' python3 link_parser_shortlink.py "https://bit.ly/xxxxx"')
sys.exit(1)
text = ' '.join(sys.argv[1:])
short_links = detect_short_links(text)
import json
print(json.dumps(short_links, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()
skills/kb-ocr-processor/SKILL.md
-66
View File
@@ -1,66 +0,0 @@
# KB OCR Processor Skill
**kb-ocr-processor** - Extract text from video frames using OCR.
## Features
- **视频截帧**: 直接从视频文件截取帧图片
- **批量处理**: 支持处理帧目录
- **双引擎**: EasyOCR / Tesseract
- **置信度评分**: 每帧识别结果带置信度
- **文本合并**: 自动合并所有帧的识别结果
## Commands
```bash
# 方式1: 直接从视频截帧并OCR（推荐）
python ocr_processor.py --video ./video.flv --output ocr.json
# 方式2: 处理已截取的帧目录
python ocr_processor.py --input ./frames/ --output ocr.json
# 指定截帧帧率（默认1帧/秒）
python ocr_processor.py --video ./video.flv --fps 2 --output ocr.json
# 同时生成合并的文本文件
python ocr_processor.py --video ./video.flv --output ocr.json --merge
# 使用 Tesseract 引擎
python ocr_processor.py --input ./frames/ --engine tesseract --output ocr.json
```
## Workflow 中的使用
```bash
# 步骤5: OCR处理（视频无字幕时）
python ocr_processor.py --video ${CONTENT_VIDEO_PATH} --output ${CONTENT_DIR}/ocr.json --merge
```
## Output
```json
{
"success": true,
"engine": "easyocr",
"total_frames": 619,
"results": [
{
"source": "0001.jpg",
"text": "字幕文字",
"confidence": 0.95
}
]
}
```
## Installation
```bash
cp -r kb-ocr-processor/ ~/skills/
pip install -r requirements.txt
# 依赖说明
# - easyocr: 主要OCR引擎（自动下载模型）
# - pytesseract: Tesseract引擎
# - ffmpeg: 视频截帧（apt install ffmpeg）
```
skills/kb-ocr-processor/ocr_processor.py
-360
View File
@@ -1,360 +0,0 @@
#!/usr/bin/env python3
"""
OCR Processor - 视频帧文字识别工具
支持 EasyOCR 和 Tesseract 引擎
"""
import argparse
import json
import os
import glob
from pathlib import Path
from typing import List, Dict, Optional
import easyocr
import pytesseract
from PIL import Image
class OCRProcessor:
"""OCR 文字识别处理器"""
def __init__(self, engine: str = "easyocr", languages: List[str] = ["ch_sim", "en"]):
"""
初始化 OCR 处理器
Args:
engine: OCR 引擎 ("easyocr" 或 "tesseract")
languages: 语言列表
"""
self.engine = engine
self.languages = languages
self.reader = None
if engine == "easyocr":
self.reader = easyocr.Reader(languages, gpu=False)
def process_image(self, image_path: str) -> Dict:
"""
处理单张图片进行 OCR 识别
Args:
image_path: 图片路径
Returns:
包含 text 和 confidence 的字典
"""
if not os.path.exists(image_path):
return {"text": "", "confidence": 0.0, "error": "File not found"}
try:
if self.engine == "easyocr":
return self._process_with_easyocr(image_path)
elif self.engine == "tesseract":
return self._process_with_tesseract(image_path)
else:
return {"text": "", "confidence": 0.0, "error": f"Unknown engine: {self.engine}"}
except Exception as e:
return {"text": "", "confidence": 0.0, "error": str(e)}
def _process_with_easyocr(self, image_path: str) -> Dict:
"""使用 EasyOCR 处理图片"""
results = self.reader.readtext(image_path)
if not results:
return {"text": "", "confidence": 0.0}
texts = []
confidences = []
for bbox, text, conf in results:
texts.append(text)
confidences.append(conf)
merged_text = "\n".join(texts)
avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0
return {
"text": merged_text,
"confidence": round(avg_confidence, 4),
"details": [
{"text": text, "confidence": round(conf, 4)}
for text, conf in zip(texts, confidences)
]
}
def _process_with_tesseract(self, image_path: str) -> Dict:
"""使用 Tesseract 处理图片"""
image = Image.open(image_path)
text = pytesseract.image_to_string(image, lang="+".join(self.languages))
# Tesseract 不直接返回置信度，返回 0.0
return {
"text": text.strip(),
"confidence": 0.0
}
def process_frames(self, input_dir: str, fps: float = 0.1, progress_file: Optional[str] = None) -> List[Dict]:
"""
处理目录下所有帧图片
Args:
input_dir: 帧图片目录
fps: 采样帧率
progress_file: 可选，进度文件路径（会实时更新）
Returns:
OCR 结果列表
"""
if not os.path.isdir(input_dir):
print(f"Error: Directory not found: {input_dir}")
return []
# 支持的图片格式
image_extensions = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.webp"]
image_files = []
for ext in image_extensions:
image_files.extend(glob.glob(os.path.join(input_dir, ext)))
image_files.extend(glob.glob(os.path.join(input_dir, "**", ext), recursive=True))
# 按文件名排序
image_files = sorted(image_files)
if not image_files:
print(f"No images found in {input_dir}")
return []
total = len(image_files)
print(f"Found {total} images in {input_dir}")
print(f"Processing with {self.engine} engine...")
def update_progress(current, status="running", last_result=None):
"""更新进度文件"""
if progress_file:
import json
import time
prog = {
"current": current,
"total": total,
"percent": round(current / total * 100, 1) if total > 0 else 0,
"status": status,
"last_result": last_result,
"updated_at": time.strftime("%Y-%m-%d %H:%M:%S")
}
with open(progress_file, 'w', encoding='utf-8') as f:
json.dump(prog, f, ensure_ascii=False, indent=2)
# 初始化进度
update_progress(0, "started")
results = []
for i, image_path in enumerate(image_files, 1):
basename = os.path.basename(image_path)
print(f"Processing [{i}/{total}]: {basename}")
ocr_result = self.process_image(image_path)
ocr_result["source"] = basename
# 记录结果摘要
text_count = len(ocr_result.get("text", "").split()) if ocr_result.get("text") else 0
last_result = f"✓ {text_count} 处文字" if ocr_result.get("text") else "✗ 无文字"
results.append(ocr_result)
# 实时更新进度
update_progress(i, "running", last_result)
# 完成
update_progress(total, "completed", f"完成 {total} 张")
print(f"\n✅ Processed {total} frames")
return results
def merge_texts(self, results: List[Dict]) -> str:
"""
合并多个 OCR 结果的文本
Args:
results: OCR 结果列表
Returns:
合并后的文本
"""
texts = []
for result in results:
if result.get("text"):
texts.append(result["text"])
return "\n\n---\n\n".join(texts)
def save_results(self, results: List[Dict], output_path: str, merge: bool = False):
"""
保存 OCR 结果到 JSON 文件
Args:
results: OCR 结果列表
output_path: 输出文件路径
merge: 是否同时保存合并后的文本文件
"""
output_data = {
"engine": self.engine,
"total_frames": len(results),
"results": results
}
# 保存 JSON
with open(output_path, "w", encoding="utf-8") as f:
json.dump(output_data, f, ensure_ascii=False, indent=2)
print(f"Results saved to {output_path}")
# 如果需要，同时保存合并的文本
if merge:
text_path = output_path.replace(".json", ".txt")
merged_text = self.merge_texts(results)
with open(text_path, "w", encoding="utf-8") as f:
f.write(merged_text)
print(f"Merged text saved to {text_path}")
def extract_frames(video_path: str, output_dir: str, fps: float = 1.0) -> str:
"""
使用 ffmpeg 从视频中截取帧图片
Args:
video_path: 视频文件路径
output_dir: 输出目录
fps: 每秒截取帧数 (默认: 1)
Returns:
帧图片目录路径
"""
import subprocess
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# ffmpeg 命令: -i 输入 -vf fps=fps 输出格式
cmd = [
"ffmpeg", "-y",
"-i", video_path,
"-vf", f"fps={fps}",
"-q:v", "2", # 质量等级 (2=高质量)
str(output_dir / "%04d.jpg")
]
print(f"🔄 截取视频帧: {video_path}")
print(f" 输出目录: {output_dir}")
try:
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0:
# 统计截取的帧数
frame_count = len(list(output_dir.glob("*.jpg")))
print(f"✅ 截取完成: {frame_count} 帧")
return str(output_dir)
else:
print(f"❌ ffmpeg 错误: {result.stderr}")
return ""
except FileNotFoundError:
print("❌ 错误: 未找到 ffmpeg，请先安装: apt install ffmpeg")
return ""
def main():
"""命令行入口"""
parser = argparse.ArgumentParser(
description="OCR 视频帧文字识别工具",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
# 使用 EasyOCR 处理帧目录
python ocr_processor.py --input ./frames/ --output ./ocr.json
# 直接从视频截帧并 OCR
python ocr_processor.py --video ./video.flv --output ./ocr.json
# 使用 Tesseract 处理
python ocr_processor.py --input ./frames/ --output ./ocr.json --engine tesseract
# 指定截帧帧率
python ocr_processor.py --video ./video.flv --output ./ocr.json --fps 2
# 同时生成合并的文本文件
python ocr_processor.py --input ./frames/ --output ./ocr.json --merge
"""
)
# 输入源：帧目录 或 视频文件（二选一）
input_group = parser.add_mutually_exclusive_group(required=True)
input_group.add_argument("--input", "-i", help="输入帧图片目录")
input_group.add_argument("--video", "-v", help="输入视频文件（会自动截帧）")
parser.add_argument("--output", "-o", required=True, help="输出 JSON 文件路径")
parser.add_argument("--fps", "-f", type=float, default=1.0, help="截帧帧率 (默认: 1帧/秒)")
parser.add_argument("--engine", "-e", choices=["easyocr", "tesseract"],
default="easyocr", help="OCR 引擎 (默认: easyocr)")
parser.add_argument("--merge", action="store_true", help="同时保存合并的文本文件")
parser.add_argument("--languages", "-l", nargs="+",
default=["ch_sim", "en"], help="语言列表")
parser.add_argument("--progress", "-p", help="可选：进度文件路径（会实时更新）")
args = parser.parse_args()
# 如果指定了视频文件，先截帧
input_dir = args.input
if args.video:
if not os.path.exists(args.video):
print(json.dumps({
"success": False,
"error": f"视频文件不存在: {args.video}"
}, ensure_ascii=False, indent=2))
return 1
# 创建临时帧目录
frames_dir = Path(args.video).stem + "_frames"
input_dir = extract_frames(args.video, frames_dir, fps=args.fps)
if not input_dir:
print(json.dumps({
"success": False,
"error": "截帧失败"
}, ensure_ascii=False, indent=2))
return 1
# 初始化处理器
processor = OCRProcessor(engine=args.engine, languages=args.languages)
# 处理帧
results = processor.process_frames(input_dir, fps=args.fps, progress_file=args.progress)
if results:
# 保存结果
processor.save_results(results, args.output, merge=args.merge)
# 统一输出格式
total_texts = sum(1 for r in results if r.get("text"))
avg_conf = sum(r.get("confidence", 0) for r in results) / len(results) if results else 0
output = {
"success": True,
"filename": Path(args.output).name,
"filepath": str(Path(args.output).resolve()),
"message": f"OCR处理完成",
"stats": {
"total_frames": len(results),
"frames_with_text": total_texts,
"avg_confidence": round(avg_conf, 4)
}
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return 0
else:
print(json.dumps({
"success": False,
"error": "未处理到任何结果"
}, ensure_ascii=False, indent=2))
return 1
if __name__ == "__main__":
exit(main())
skills/kb-ocr-processor/requirements.txt
-3
View File
@@ -1,3 +0,0 @@
easyocr>=1.7.0
pytesseract>=0.3.10
Pillow>=9.0.0
skills/kb-whisper-diarization/.env.example
-9
View File
@@ -1,9 +0,0 @@
# KB Whisper Diarization 环境变量
# Whisper.cpp 安装路径
WHISPER_CPP_PATH=~/whisper.cpp
# HuggingFace Token（必需，用于下载 pyannote 模型）
# 获取方式: https://huggingface.co/settings/tokens
# 注意: 需要先申请 pyannote/speaker-diarization 模型访问权限
HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
skills/kb-whisper-diarization/SKILL.md
-151
View File
@@ -1,151 +0,0 @@
# KB Whisper Diarization Skill
**kb-whisper-diarization** - 使用 Whisper + pyannote.audio 实现语音识别和说话者分离。
## 什么时候使用
当用户需要：
- 区分多个说话者（如访谈、对话）
- 获取带说话者标签的字幕
- 分析对话中谁说了什么
## 环境变量
在使用之前，必须检查环境变量是否已经设置。
| 变量名 | 用途 | 示例 |
|--------|------|------|
| `WHISPER_CPP_PATH` | whisper.cpp 安装路径 | `~/whisper.cpp` |
| `HF_TOKEN` | HuggingFace Token | `hf_xxx...` |
### 获取 HuggingFace Token（必需）
pyannote.audio 需要从 HuggingFace 下载模型，按以下步骤操作：
**步骤 1: 注册/登录 HuggingFace**
- 访问 https://huggingface.co
- 注册或登录账号
**步骤 2: 申请模型访问权限**
需要申请以下两个模型的访问权限（点击页面上的 "Access repository" 按钮）：
- https://huggingface.co/pyannote/speaker-diarization
- https://huggingface.co/pyannote/segmentation
> ⚠️ 申请后需要等待几分钟到几小时才能获得访问权限
**步骤 3: 创建 Access Token**
1. 访问 https://huggingface.co/settings/tokens
2. 点击 "New token"
3. 选择 "Read" 权限
4. 复制生成的 token（以 `hf_` 开头）
**步骤 4: 设置环境变量**
```bash
# 添加到 ~/.kb_env 文件
echo 'export HF_TOKEN="hf_xxxxxxxxxxxxxxxxxxxxxx"' >> ~/.kb_env
source ~/.kb_env
# 验证
echo $HF_TOKEN
```
## 使用方法
```bash
# 基本用法（带说话者分离）
python whisper_diarization.py --input video.mp4
# 说话者分离
python whisper_diarization.py --input interview.mp4 --diarize
# 指定说话者数量
python whisper_diarization.py --input meeting.mp4 --diarize --num-speakers 2
# 指定语言
python whisper_diarization.py --input video.mp4 --language zh
```
## 命令参数
| 参数 | 说明 | 默认值 |
|------|------|--------|
| `--input` | 输入音视频文件路径 | 必需 |
| `--output` | 输出目录 | 文件所在目录 |
| `--diarize` | 启用说话者分离 | False |
| `--num-speakers` | 估计说话者数量 | None（自动检测） |
| `--language` | 语言 | 自动检测 |
| `--model` | Whisper 模型 | base |
| `--background` | 后台异步执行 | False |
## 模型选项
- **Whisper 模型**: tiny, base, small, medium, large
- **说话者分离**: pyannote/speaker-diarization (需申请权限)
## 输出文件
对于输入 `interview.mp4`，将生成：
- `interview.mp4.json` - 完整结果（包含说话者标签）
- `interview.mp4.txt` - 纯文本
- `interview.mp4.srt` - 字幕格式（带说话者标签）
## 输出格式示例
### JSON 格式
```json
{
"success": true,
"input_file": "interview.mp4",
"diarization": true,
"speakers": ["SPEAKER_00", "SPEAKER_01"],
"segments": [
{
"id": 0,
"speaker": "SPEAKER_00",
"start": 0.0,
"end": 5.2,
"text": "大家好，欢迎来到访谈节目"
},
{
"id": 1,
"speaker": "SPEAKER_01",
"start": 5.2,
"end": 10.8,
"text": "谢谢邀请，我很荣幸"
}
]
}
```
### SRT 字幕格式
```
1
00:00:00,000 --> 00:00:05,200
[SPEAKER_00] 大家好，欢迎来到访谈节目
2
00:00:05,200 --> 00:00:10,800
[SPEAKER_01] 谢谢邀请，我很荣幸
```
## 安装
```bash
# 1. 安装依赖
pip install pyannote.audio torch
# 2. 设置 HuggingFace Token
export HF_TOKEN="hf_xxx..."
# 3. 首次运行会自动下载模型
python whisper_diarization.py --input test.mp4
```
## 注意事项
- **说话者分离需要申请权限**: https://huggingface.co/pyannote/speaker-diarization
- **处理时间较长**: 说话者分离会增加处理时间
- **对立体声效果更好**: 多声道音频的分离效果更佳
skills/kb-whisper-diarization/requirements.txt
-8
View File
@@ -1,8 +0,0 @@
# KB Whisper Diarization 依赖
# 核心依赖
torch>=2.0.0
pyannote.audio>=1.0.0
# 可选: GPU 加速
# torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
skills/kb-whisper-diarization/whisper_diarization.py
-339
View File
@@ -1,339 +0,0 @@
#!/usr/bin/env python3
"""
KB Whisper Diarization - 语音识别 + 说话者分离
使用 Whisper 进行语音识别，pyannote.audio 进行说话者分离
"""
import argparse
import json
import os
import subprocess
import sys
import tempfile
from datetime import datetime
from pathlib import Path
import re
def parse_arguments():
"""解析命令行参数"""
parser = argparse.ArgumentParser(
description='KB Whisper Diarization - 语音识别 + 说话者分离',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
python whisper_diarization.py --input video.mp4
python whisper_diarization.py --input interview.mp4 --diarize
python whisper_diarization.py --input meeting.mp4 --diarize --num-speakers 2
"""
)
parser.add_argument('--input', required=True, help='输入音视频文件路径')
parser.add_argument('--output', help='输出目录 (默认: 文件所在目录)')
parser.add_argument('--diarize', action='store_true', help='启用说话者分离')
parser.add_argument('--num-speakers', type=int, help='估计说话者数量')
parser.add_argument('--language', help='指定语言 (默认: 自动检测)')
parser.add_argument('--model', default='base', choices=['tiny', 'base', 'small', 'medium', 'large'],
help='Whisper 模型 (默认: base)')
parser.add_argument('--background', action='store_true', help='后台异步执行')
return parser.parse_args()
def get_whisper_path():
"""获取 whisper.cpp 路径"""
path = os.environ.get('WHISPER_CPP_PATH')
if path:
path = os.path.expanduser(path)
if os.path.exists(path):
return path
return os.path.expanduser('~/whisper.cpp')
def check_whisper_model(whisper_path, model):
"""检查 Whisper 模型"""
model_path = os.path.join(whisper_path, 'models', f'ggml-{model}.bin')
if not os.path.exists(model_path):
raise FileNotFoundError(f"Whisper 模型不存在: {model_path}")
return model_path
def convert_to_wav(input_path, output_dir):
"""转换为 16kHz WAV"""
if input_path.lower().endswith('.wav'):
try:
result = subprocess.run(
['ffprobe', '-v', 'error', '-show_entries', 'stream=sample_rate',
'-of', 'default=noprint_wrappers=1:nokey=1', input_path],
capture_output=True, text=True
)
if result.returncode == 0 and int(result.stdout.strip()) == 16000:
return input_path
except Exception:
pass
output_path = os.path.join(output_dir, f"{Path(input_path).stem}_audio.wav")
print(f"转换音频: {input_path} -> {output_path}")
subprocess.run([
'ffmpeg', '-y', '-i', input_path,
'-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',
'-loglevel', 'error', output_path
], check=True, capture_output=True)
print("✓ 音频转换完成")
return output_path
def run_whisper(whisper_path, audio_path, model, output_dir, language=None):
"""运行 Whisper 识别"""
basename = Path(audio_path).stem.replace('_audio', '')
output_file = os.path.join(output_dir, basename)
cmd = [
os.path.join(whisper_path, 'whisper-cli'),
'-m', os.path.join(whisper_path, 'models', f'ggml-{model}.bin'),
'-f', audio_path,
'-of', output_file,
'-l', language or 'auto',
'--output-txt', '--output-srt', '--output-json',
]
print(f"执行 Whisper: {' '.join(cmd[:6])} ...")
result = subprocess.run(
cmd, capture_output=True, text=True, cwd=whisper_path, timeout=3600
)
if result.returncode != 0 and not os.path.exists(f"{output_file}.txt"):
raise RuntimeError(f"Whisper 失败: {result.stderr}")
return basename
def run_diarization(audio_path, num_speakers=None):
"""运行 pyannote.audio 说话者分离"""
print("\n🎙️ 步骤2: 说话者分离...")
try:
import torch
from pyannote.audio import Pipeline
except ImportError as e:
print(f"✗ 缺少依赖: {e}")
print("请安装: pip install pyannote.audio torch")
raise
# 检查 HuggingFace Token
hf_token = os.environ.get('HF_TOKEN')
if not hf_token:
raise RuntimeError("需要设置 HF_TOKEN 环境变量\n"
"获取方式: https://huggingface.co/settings/tokens")
# 新版 pyannote 需要先登录
from huggingface_hub import login
print("登录 HuggingFace...")
login(token=hf_token)
print("加载 pyannote 流水线...")
pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization")
# 设置设备
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"使用设备: {device}")
pipeline.to(device)
# 运行分离
print("执行说话者分离...")
diarization = pipeline(
audio_path,
num_speakers=num_speakers
)
# 解析结果
speakers = {}
for turn, _, speaker in diarization.itertracks(yield_label=True):
if speaker not in speakers:
speakers[speaker] = []
speakers[speaker].append({
"start": round(turn.start, 2),
"end": round(turn.end, 2)
})
print(f"✓ 检测到 {len(speakers)} 个说话者: {list(speakers.keys())}")
return speakers
def merge_results(whisper_txt, whisper_srt, diarization, output_dir, basename):
"""合并 Whisper 和说话者分离结果"""
print("\n📝 步骤3: 合并结果...")
# 读取 Whisper 文本
text_segments = []
if os.path.exists(whisper_txt):
with open(whisper_txt, 'r', encoding='utf-8') as f:
content = f.read()
# 简单按行分段
for i, line in enumerate(content.strip().split('\n')):
if line.strip():
text_segments.append({
"id": i,
"text": line.strip(),
"start": i * 3.0, # 估算
"end": (i + 1) * 3.0
})
# 读取 SRT 获取精确时间
srt_segments = []
if os.path.exists(whisper_srt):
with open(whisper_srt, 'r', encoding='utf-8') as f:
content = f.read()
# 解析 SRT
pattern = r'(\d+)\n(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\n(.+?)(?=\n\n|\Z)'
matches = re.findall(pattern, content, re.DOTALL)
def parse_time(t):
h, m, s_ms = t.split(':')
s, ms = s_ms.split(',')
return float(h) * 3600 + float(m) * 60 + float(s) + float(ms) / 1000
for idx, num, start, end, text in matches:
srt_segments.append({
"id": int(idx),
"start": parse_time(start),
"end": parse_time(end),
"text": text.strip()
})
# 为每个 Whisper 段落分配说话者
merged_segments = []
for seg in srt_segments:
speaker = assign_speaker(seg['start'], seg['end'], diarization)
merged_segments.append({
"id": len(merged_segments),
"speaker": speaker,
"start": seg['start'],
"end": seg['end'],
"text": seg['text']
})
# 生成带说话者的 SRT
output_srt = os.path.join(output_dir, f"{basename}_diarized.srt")
with open(output_srt, 'w', encoding='utf-8') as f:
for seg in merged_segments:
start = format_time(seg['start'])
end = format_time(seg['end'])
f.write(f"{seg['id'] + 1}\n")
f.write(f"{start} --> {end}\n")
f.write(f"[{seg['speaker']}] {seg['text']}\n\n")
# 生成合并的 TXT
output_txt = os.path.join(output_dir, f"{basename}_diarized.txt")
with open(output_txt, 'w', encoding='utf-8') as f:
for seg in merged_segments:
f.write(f"[{seg['speaker']}] {seg['text']}\n")
# 生成 JSON
output_json = os.path.join(output_dir, f"{basename}_diarized.json")
json_data = {
"success": True,
"tool": "whisper-diarization",
"diarization": True,
"speakers": list(diarization.keys()),
"input_file": basename,
"output_files": {
"txt": f"{basename}_diarized.txt",
"srt": f"{basename}_diarized.srt",
"json": f"{basename}_diarized.json"
},
"segments": merged_segments
}
with open(output_json, 'w', encoding='utf-8') as f:
json.dump(json_data, f, ensure_ascii=False, indent=2)
print(f"✓ 生成文件:")
print(f" - {output_srt}")
print(f" - {output_txt}")
print(f" - {output_json}")
return json_data
def assign_speaker(start, end, diarization):
"""将时间段分配给说话者"""
best_speaker = "UNKNOWN"
max_overlap = 0
for speaker, segments in diarization.items():
for seg in segments:
overlap = min(end, seg['end']) - max(start, seg['start'])
if overlap > max_overlap:
max_overlap = overlap
best_speaker = speaker
return best_speaker
def format_time(seconds):
"""秒数转换为 SRT 时间格式"""
hours = int(seconds // 3600)
minutes = int((seconds % 3600) // 60)
secs = int(seconds % 60)
ms = int((seconds % 1) * 1000)
return f"{hours:02d}:{minutes:02d}:{secs:02d},{ms:03d}"
def main():
args = parse_arguments()
input_file = os.path.abspath(args.input)
if not os.path.exists(input_file):
print(f"✗ 文件不存在: {input_file}")
sys.exit(1)
output_dir = os.path.abspath(args.output) if args.output else os.path.dirname(input_file)
os.makedirs(output_dir, exist_ok=True)
basename = Path(input_file).stem
print(f"\n📁 输入: {input_file}")
print(f"📁 输出: {output_dir}")
print(f"🎙️ 说话者分离: {'是' if args.diarize else '否'}")
try:
# 步骤1: Whisper 识别
print(f"\n🎙️ 步骤1: 语音识别...")
whisper_path = get_whisper_path()
check_whisper_model(whisper_path, args.model)
audio_path = convert_to_wav(input_file, output_dir)
run_whisper(whisper_path, audio_path, args.model, output_dir, args.language)
# 步骤2: 说话者分离
if args.diarize:
diarization = run_diarization(audio_path, args.num_speakers)
else:
diarization = {}
# 步骤3: 合并结果
whisper_txt = os.path.join(output_dir, f"{basename}.txt")
whisper_srt = os.path.join(output_dir, f"{basename}.srt")
result = merge_results(whisper_txt, whisper_srt, diarization, output_dir, basename)
# 清理临时音频
if audio_path != input_file and audio_path.endswith('_audio.wav'):
try:
os.remove(audio_path)
except Exception:
pass
print(f"\n✅ 处理完成!")
print(f" 说话者数量: {len(diarization) if args.diarize else 1}")
print(f" 段落数量: {len(result['segments'])}")
except Exception as e:
print(f"\n✗ 错误: {e}")
sys.exit(1)
if __name__ == '__main__':
main()
skills/kb-whisper/.env.example
-5
View File
@@ -1,5 +0,0 @@
# KB Whisper 环境变量配置示例
# 复制此文件为 .env 并填写实际值
# Whisper.cpp 安装路径
WHISPER_CPP_PATH=~/whisper.cpp
skills/kb-whisper/README.md
-211
View File
@@ -1,211 +0,0 @@
# KB Whisper - 本地语音识别
基于 Whisper.cpp 的本地语音识别工具，支持将音视频文件转换为文本、字幕和 JSON 格式。
## 功能特性
- 🎯 **本地执行** - 完全离线运行，保护隐私
- 🚀 **多模型支持** - 从 tiny 到 large，根据需求选择速度/精度
- 📝 **多格式输出** - JSON、SRT、TXT 三种格式
- ⚡ **异步执行** - 支持后台运行，不阻塞主代理
- 🎬 **广泛兼容** - 支持常见音视频格式（MP3、MP4、WAV 等）
## 安装前提
### 1. 安装 whisper.cpp
```bash
# 克隆仓库
git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp
# 编译
cd ~/whisper.cpp
make
# 下载模型（至少需要一个模型）
./models/download-ggml-model.sh tiny
./models/download-ggml-model.sh base # 可选
```
### 2. 配置环境变量
在 `~/.clawdbot/clawdbot.json` 中添加：
```json5
{
skills: {
entries: {
"kb-whisper": {
env: {
WHISPER_CPP_PATH: "~/whisper.cpp"
}
}
}
}
}
```
或通过命令行配置：
```bash
clawdbot skills config kb-whisper WHISPER_CPP_PATH "~/whisper.cpp"
```
## 使用方法
### 基本用法
```bash
# 使用默认 tiny 模型处理
python whisper.py --input video.mp4
# 使用 base 模型（推荐，平衡速度和精度）
python whisper.py --input video.mp4 --model base
# 后台异步执行
python whisper.py --input video.mp4 --background
```
### 指定输出目录
```bash
python whisper.py --input video.mp4 --output ./transcripts
```
### 处理音频文件
```bash
python whisper.py --input podcast.mp3 --model base
```
## 输出文件
对于输入 `video.mp4`，将生成以下文件：
```
output/
├── video.json # 详细结果（含时间戳）
├── video.srt # 字幕格式
└── video.txt # 纯文本
```
### JSON 格式
```json
{
"success": true,
"input_file": "video.mp4",
"model": "base",
"output_files": {
"json": "video.json",
"srt": "video.srt",
"txt": "video.txt"
},
"duration_seconds": 120.5,
"segments": [
{
"id": 0,
"start": 0.0,
"end": 3.2,
"text": "这是第一段字幕"
},
{
"id": 1,
"start": 3.2,
"end": 6.8,
"text": "这是第二段字幕"
}
]
}
```
## 模型对比
| 模型 | 速度 | 精度 | 适用场景 |
|------|------|------|----------|
| tiny | ⚡⚡⚡ | ⭐ | 快速预览、实时转录 |
| base | ⚡⚡ | ⭐⭐⭐ | 日常使用（推荐） |
| small | ⚡ | ⭐⭐⭐⭐ | 高质量需求 |
| medium | ⭐ | ⭐⭐⭐⭐⭐ | 专业用途 |
| large | 🐢 | ⭐⭐⭐⭐⭐ | 最高精度要求 |
## 完整示例
### 示例 1: 转录播客
```bash
python whisper.py --input podcast.mp3 --model base --background
```
### 示例 2: 生成视频字幕
```bash
python whisper.py --input tutorial.mp4 --model small
```
### 示例 3: 批量处理（配合脚本）
```bash
#!/bin/bash
for video in videos/*.mp4; do
python whisper.py --input "$video" --model base
done
```
## 在 Workflow 中使用
在知识库处理流程中，如果视频没有字幕：
```bash
# 步骤4: 下载视频后，如果没有字幕
# 步骤5: 使用 Whisper 生成字幕
python whisper.py --input ${video_path} --model base --background
# 等待 Whisper 完成，然后继续后续步骤
```
## 技术细节
- **引擎**: Whisper.cpp（C++ 实现，性能优异）
- **音频采样率**: 自动转换为 16kHz
- **语言**: 自动检测（支持 99 种语言）
- **GPU 支持**: whisper.cpp 支持 CUDA 加速（需编译时启用）
## 故障排查
### 问题: 找不到 whisper.cpp
```bash
# 检查环境变量
echo $WHISPER_CPP_PATH
# 手动测试
~/whisper.cpp/main --help
```
### 问题: 模型未下载
```bash
cd ~/whisper.cpp
./models/download-ggml-model.sh tiny
```
### 问题: 音频格式不支持
```bash
# 转换音频格式
ffmpeg -i input.avi -acodec libmp3lame output.mp3
python whisper.py --input output.mp3
```
## 性能参考
在 CPU 环境下（Intel i7）：
- **tiny 模型**: ~1x 实时速度（1分钟音频 ~1分钟处理）
- **base 模型**: ~0.5x 实时速度（1分钟音频 ~2分钟处理）
- **small 模型**: ~0.25x 实时速度（1分钟音频 ~4分钟处理）
## 许可证
本技能使用 MIT 许可证，Whisper.cpp 使用 MIT 许可证。
skills/kb-whisper/SKILL.md
-124
View File
@@ -1,124 +0,0 @@
---
name: kb-whisper
description: 本地语音识别，支持音视频转文字（基于 Whisper.cpp）
---
# KB Whisper Skill
**kb-whisper** - 本地语音识别，使用 Whisper.cpp 将音视频文件转换为文本、字幕和 JSON 格式输出。
## 什么时候使用
当用户需要：
- 将视频/音频文件转换为文字
- 生成视频字幕（SRT 格式）
- 获取带时间戳的详细识别结果
## 环境变量
在使用之前，必须检查环境变量是否已经设置。
| 变量名 | 用途 | 示例 |
|--------|------|------|
| `WHISPER_CPP_PATH` | whisper.cpp 安装路径 | `~/whisper.cpp` |
## 使用方法
```bash
# 基本用法
python whisper.py --input video.mp4
# 指定模型（默认 tiny）
python whisper.py --input video.mp4 --model base
# 指定输出目录
python whisper.py --input video.mp4 --output ./output
# 异步执行（推荐）
python whisper.py --input video.mp4 --background
```
## 命令参数
| 参数 | 说明 | 默认值 |
|------|------|--------|
| `--input` | 输入音视频文件路径 | 必需 |
| `--model` | Whisper 模型类型 | `tiny` |
| `--output` | 输出目录 | 文件所在目录 |
| `--background` | 后台异步执行 | False |
## 模型选项
- `tiny` - 最小模型，速度快，精度较低（推荐快速识别）
- `base` - 基础模型，平衡速度和精度（推荐）
- `small` - 小型模型，精度更高，速度较慢
- `medium` - 中型模型，高精度
- `large` - 大型模型，最高精度，速度慢
## 输出文件
对于输入 `video.mp4`，将生成：
- `video.json` - 详细结果（包含时间戳、分段信息）
- `video.srt` - 字幕格式文件
- `video.txt` - 纯文本内容
## 输出格式示例
### JSON 格式
```json
{
"success": true,
"input_file": "video.mp4",
"model": "tiny",
"output_files": {
"json": "video.json",
"srt": "video.srt",
"txt": "video.txt"
},
"duration_seconds": 120.5,
"segments": [
{
"id": 0,
"start": 0.0,
"end": 3.2,
"text": "这是第一段字幕"
}
]
}
```
## Workflow 中的使用
```bash
# 步骤4之后：如果已有音频但无字幕，可以使用 whisper
python whisper.py --input ${video_path} --model base --background
```
## 安装
```bash
# 1. 克隆 whisper.cpp
git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp
# 2. 编译
cd ~/whisper.cpp
make
# 3. 下载模型（tiny）
./models/download-ggml-model.sh tiny
# 4. 下载更多模型（可选）
# ./models/download-ggml-model.sh base
# ./models/download-ggml-model.sh small
# 5. 复制技能到 skills 目录
cp -r kb-whisper/ ~/skills/
```
## 注意事项
- **异步执行**: 使用 `--background` 参数时，命令会在后台运行，不阻塞主代理
- **模型选择**: tiny 模型适合快速识别，base 模型适合一般用途，large 模型适合高精度需求
- **文件格式**: 支持 MP3、MP4、WAV、FLAC、OGG、M4A 等常见格式
skills/kb-whisper/__pycache__/whisper.cpython-312.pyc
BIN
View File
Binary file not shown.
skills/kb-whisper/examples.md
-273
View File
@@ -1,273 +0,0 @@
# KB Whisper 使用示例
## 快速开始
### 1. 基础使用
```bash
# 使用默认 tiny 模型转录视频
python whisper.py --input video.mp4
# 使用 base 模型（推荐）
python whisper.py --input video.mp4 --model base
```
### 2. 异步执行
```bash
# 后台处理，不阻塞主代理
python whisper.py --input video.mp4 --model base --background
```
### 3. 指定输出目录
```bash
python whisper.py --input video.mp4 --output ./transcripts
```
### 4. 指定语言
```bash
# 指定中文
python whisper.py --input video.mp4 --language zh
# 指定英文
python whisper.py --input video.mp4 --language en
# 指定日语
python whisper.py --input video.mp4 --language ja
```
## 实际应用场景
### 场景 1: 转录播客
```bash
python whisper.py --input podcast.mp3 --model base --background
```
**输出**:
- `podcast.json` - 带时间戳的详细结果
- `podcast.srt` - 字幕格式
- `podcast.txt` - 纯文本
### 场景 2: 生成视频字幕
```bash
python whisper.py --input tutorial.mp4 --model small
```
**用途**:
- 为教程视频生成字幕
- 导出 SRT 文件到视频编辑软件
- 生成多语言字幕版本
### 场景 3: 会议录音转文字
```bash
python whisper.py --input meeting.mp3 --model base --output ./meetings
```
**输出示例**:
```
meetings/
├── meeting.json # 完整 JSON（含时间戳）
├── meeting.srt # 字幕格式
└── meeting.txt # 纯文本
```
### 场景 4: 批量处理
创建脚本 `batch_transcribe.sh`:
```bash
#!/bin/bash
OUTPUT_DIR="./transcripts"
mkdir -p "$OUTPUT_DIR"
for video in videos/*.mp4; do
echo "处理: $video"
python whisper.py --input "$video" --output "$OUTPUT_DIR" --model base
done
echo "批量处理完成!"
```
```bash
chmod +x batch_transcribe.sh
./batch_transcribe.sh
```
## 在 Workflow 中使用
### 知识库处理流程
在 `kb-workflow` 技能中，如果视频没有字幕：
```bash
# 步骤 4: 下载视频后
# 检查是否有字幕文件
if [ ! -f "${subtitle_file}" ]; then
echo "无字幕文件，使用 Whisper 生成..."
# 使用 Whisper 生成字幕（后台执行）
python /path/to/whisper.py \
--input "${video_path}" \
--model base \
--output "${content_dir}" \
--background
# 等待 Whisper 完成
sleep 5 # 等待后台任务启动
while ! [ -f "${content_dir}/$(basename ${video_path} .mp4).srt" ]; do
sleep 2
done
echo "Whisper 处理完成"
fi
```
## 输出文件使用
### JSON 格式
```python
import json
# 读取 JSON 结果
with open('video.json', 'r', encoding='utf-8') as f:
data = json.load(f)
# 访问分段
for segment in data['segments']:
print(f"[{segment['start']:.2f}-{segment['end']:.2f}] {segment['text']}")
```
### SRT 格式
```bash
# 直接在视频播放器中使用
mpv video.mp4 --sub-file=video.srt
# 或转换字幕格式
ffmpeg -i video.srt video.ass
```
### TXT 格式
```bash
# 搜索关键词
grep "重要信息" video.txt
# 统计字数
wc -m video.txt
```
## 性能优化建议
### 1. 根据内容选择模型
```bash
# 快速预览（短视频、测试）
python whisper.py --input video.mp4 --model tiny
# 日常使用（播客、会议）
python whisper.py --input video.mp4 --model base
# 高质量需求（正式内容、多语言）
python whisper.py --input video.mp4 --model small
```
### 2. 使用后台执行
```bash
# 处理长文件时使用后台模式
python whisper.py --input long_video.mp4 --model base --background
```
### 3. 指定语言提升准确度
```bash
# 如果知道内容语言，指定后准确度更高
python whisper.py --input video.mp4 --language zh
```
## 故障排查
### 问题 1: whisper.cpp 未找到
```bash
# 检查环境变量
echo $WHISPER_CPP_PATH
# 或配置
clawdbot skills config kb-whisper WHISPER_CPP_PATH "~/whisper.cpp"
```
### 问题 2: 模型未下载
```bash
cd ~/whisper.cpp
./models/download-ggml-model.sh base
```
### 问题 3: 音频格式不支持
```bash
# 转换格式
ffmpeg -i input.avi -acodec libmp3lame output.mp3
python whisper.py --input output.mp3
```
### 问题 4: 后台任务未完成
```bash
# 查看日志
cat video_whisper.log
# 检查进程
ps aux | grep whisper
```
## 集成到其他工具
### 集成到 Python 脚本
```python
import subprocess
import json
def transcribe_audio(audio_path, model='base'):
"""调用 whisper.py 转录音频"""
cmd = [
'python3', '/path/to/whisper.py',
'--input', audio_path,
'--model', model
]
result = subprocess.run(cmd, capture_output=True, text=True)
return json.loads(result.stdout)
# 使用
result = transcribe_audio('podcast.mp3', 'base')
print(f"识别到 {result['total_segments']} 个片段")
```
### 集成到 Shell 脚本
```bash
#!/bin/bash
transcribe() {
local file=$1
local model=${2:-base}
echo "转录: $file"
python3 whisper.py --input "$file" --model "$model" --background
}
# 批量处理
transcribe "video1.mp4"
transcribe "video2.mp4" "tiny"
```
skills/kb-whisper/whisper.py
-403
View File
@@ -1,403 +0,0 @@
#!/usr/bin/env python3
"""
KB Whisper - 本地语音识别工具
基于 Whisper.cpp 的音视频转文字
功能：
- 自动将视频/音频转换为 16kHz WAV 格式
- 支持指定语言（默认中文）
- 输出标准 JSON 格式供 KB Workflow 使用
"""
import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
import re
def parse_arguments():
"""解析命令行参数"""
parser = argparse.ArgumentParser(
description='KB Whisper - 本地语音识别工具',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
示例:
python whisper.py --input video.mp4
python whisper.py --input video.mp4 --model base
python whisper.py --input video.mp4 --output ./output --background
"""
)
parser.add_argument(
'--input',
required=True,
help='输入音视频文件路径'
)
parser.add_argument(
'--model',
default='base',
choices=['tiny', 'base', 'small', 'medium', 'large'],
help='Whisper 模型类型 (默认: base)'
)
parser.add_argument(
'--output',
help='输出目录 (默认: 输入文件所在目录)'
)
parser.add_argument(
'--background',
action='store_true',
help='后台异步执行'
)
parser.add_argument(
'--language',
default='zh',
help='指定语言 (默认: zh)，例如: en, zh, ja'
)
return parser.parse_args()
def get_whisper_cpp_path():
"""获取 whisper.cpp 路径"""
# 优先从环境变量获取
path = os.environ.get('WHISPER_CPP_PATH')
if path:
path = os.path.expanduser(path)
if os.path.exists(path):
return path
# 尝试常见位置
common_paths = [
os.path.expanduser('~/whisper.cpp'),
'/usr/local/bin/whisper',
'/opt/whisper.cpp',
]
for cp in common_paths:
if os.path.exists(cp):
return cp
raise FileNotFoundError(
"未找到 whisper.cpp。请设置环境变量 WHISPER_CPP_PATH "
"或安装到 ~/whisper.cpp\n"
"安装方法: git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp && cd ~/whisper.cpp && make"
)
def check_model(whisper_path, model):
"""检查模型是否存在"""
model_path = os.path.join(whisper_path, 'models', f'ggml-{model}.bin')
if not os.path.exists(model_path):
raise FileNotFoundError(
f"模型文件不存在: {model_path}\n"
f"请运行以下命令下载模型:\n"
f"cd {whisper_path} && ./models/download-ggml-model.sh {model}"
)
return model_path
def convert_to_wav(input_path, output_dir):
"""将音视频转换为 16kHz WAV 格式
Whisper 需要 WAV 格式的输入文件。
"""
input_path = os.path.abspath(input_path)
# 如果已经是 WAV 且采样率正确，直接返回
if input_path.lower().endswith('.wav'):
try:
result = subprocess.run(
['ffprobe', '-v', 'error', '-show_entries',
'stream=sample_rate', '-of', 'default=noprint_wrappers=1:nokey=1',
input_path],
capture_output=True, text=True
)
if result.returncode == 0 and int(result.stdout.strip()) == 16000:
print(f"输入文件已是 16kHz WAV 格式，直接使用")
return input_path
except Exception:
pass
# 转换
output_path = os.path.join(output_dir, f"{Path(input_path).stem}_audio.wav")
print(f"转换音频格式: {input_path} -> {output_path}")
try:
subprocess.run([
'ffmpeg', '-y', '-i', input_path,
'-acodec', 'pcm_s16le',
'-ar', '16000',
'-ac', '1',
'-loglevel', 'error',
output_path
], check=True, capture_output=True)
print("音频转换完成")
return output_path
except subprocess.CalledProcessError as e:
print(f"音频转换失败: {e}")
raise RuntimeError(f"无法转换音频格式: {e}")
except FileNotFoundError:
raise RuntimeError("未找到 ffmpeg，请安装: sudo apt-get install ffmpeg")
def run_whisper(whisper_path, audio_path, model, output_dir, language='zh'):
"""运行 Whisper 语音识别
Args:
whisper_path: whisper.cpp 目录
audio_path: WAV 音频文件路径
model: 模型名称
output_dir: 输出目录
language: 语言代码
"""
# 输出文件名（使用原始输入文件名，不是 audio.wav）
# 从 output_dir 和 audio_path 推断原始文件名
original_stem = Path(audio_path).stem.replace('_audio', '')
output_basename = original_stem
# 构建命令
cmd = [
os.path.join(whisper_path, 'whisper-cli'),
'-m', os.path.join(whisper_path, 'models', f'ggml-{model}.bin'),
'-f', audio_path,
'-of', os.path.join(output_dir, output_basename),
'-l', language,
'--output-txt',
'--output-srt',
'--output-json',
]
print(f"执行命令: {' '.join(cmd)}")
# 运行
result = subprocess.run(
cmd,
capture_output=True,
text=True,
cwd=whisper_path,
timeout=3600 # 1小时超时
)
if result.returncode != 0:
# 检查是否有警告但仍成功
if "error" in result.stderr.lower() and not os.path.exists(
os.path.join(output_dir, f"{output_basename}.txt")
):
raise RuntimeError(
f"Whisper 执行失败:\n"
f"标准输出: {result.stdout}\n"
f"错误输出: {result.stderr}"
)
return result
def parse_srt_for_duration(srt_file):
"""从 SRT 文件解析时长"""
if not os.path.exists(srt_file):
return 0
with open(srt_file, 'r', encoding='utf-8') as f:
content = f.read()
# 找到最后一个时间戳
pattern = r'(\d{2}:\d{2}:\d{2}),\d{3} --> \d{2}:\d{2}:\d{2},\d{3}'
matches = re.findall(pattern, content)
if not matches:
return 0
# 解析最后一个时间戳
last_time = matches[-1]
h, m, s = last_time.split(':')
return int(h) * 3600 + int(m) * 60 + int(s)
def create_workflow_json(input_file, output_dir, model, language, duration):
"""创建标准格式的 JSON 输出，供 KB Workflow 使用"""
input_path = Path(input_file)
basename = input_path.stem
txt_file = os.path.join(output_dir, f"{basename}.txt")
srt_file = os.path.join(output_dir, f"{basename}.srt")
json_file = os.path.join(output_dir, f"{basename}.json")
# 读取文本内容
text_content = ""
if os.path.exists(txt_file):
with open(txt_file, 'r', encoding='utf-8') as f:
text_content = f.read().strip()
# 解析 SRT 获取分段
segments = []
if os.path.exists(srt_file):
with open(srt_file, 'r', encoding='utf-8') as f:
content = f.read()
# SRT 格式: 序号 -> 时间 -> 文本
pattern = r'(\d+)\n(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\n(.+?)(?=\n\n|\Z)'
matches = re.findall(pattern, content, re.DOTALL)
def parse_time(t):
h, m, s_ms = t.split(':')
s, ms = s_ms.split(',')
return float(h) * 3600 + float(m) * 60 + float(s) + float(ms) / 1000
for idx, (num, start, end, text) in enumerate(matches):
segments.append({
"id": idx,
"start": round(parse_time(start), 2),
"end": round(parse_time(end), 2),
"text": text.strip()
})
# 构建 JSON
workflow_json = {
"success": True,
"tool": "whisper",
"model": model,
"language": language,
"input_file": str(input_path),
"output_files": {
"txt": f"{basename}.txt",
"srt": f"{basename}.srt",
"json": f"{basename}.json"
},
"duration_seconds": duration,
"segments": segments,
"text": text_content
}
# 写入 JSON
with open(json_file, 'w', encoding='utf-8') as f:
json.dump(workflow_json, f, ensure_ascii=False, indent=2)
return workflow_json
def main():
"""主函数"""
args = parse_arguments()
# 检查输入文件
input_file = os.path.abspath(args.input)
if not os.path.exists(input_file):
print(f"错误: 输入文件不存在: {input_file}", file=sys.stderr)
sys.exit(1)
# 获取输出目录
if args.output:
output_dir = os.path.abspath(args.output)
else:
output_dir = os.path.dirname(input_file)
os.makedirs(output_dir, exist_ok=True)
# 获取 whisper.cpp 路径
try:
whisper_path = get_whisper_cpp_path()
print(f"✓ Whisper.cpp 路径: {whisper_path}")
except FileNotFoundError as e:
print(f"✗ 错误: {e}", file=sys.stderr)
sys.exit(1)
# 检查模型
try:
check_model(whisper_path, args.model)
print(f"✓ 使用模型: {args.model}")
except FileNotFoundError as e:
print(f"✗ 错误: {e}", file=sys.stderr)
sys.exit(1)
# 后台执行
if args.background:
print("启动后台处理...")
log_file = os.path.join(output_dir, f"{Path(input_file).stem}_whisper.log")
with open(log_file, 'w') as log:
subprocess.Popen(
[sys.executable, __file__,
'--input', input_file,
'--model', args.model,
'--output', output_dir,
'--language', args.language],
stdout=log,
stderr=log,
start_new_session=True
)
print(f"✓ 后台任务已启动")
print(f" 日志: {log_file}")
print(f" 输出: {output_dir}")
result = {
"success": True,
"background": True,
"status": "running",
"log_file": log_file,
"output_dir": output_dir
}
print(json.dumps(result, ensure_ascii=False, indent=2))
return
# 前台执行
print(f"\n📁 处理文件: {input_file}")
print(f"📁 输出目录: {output_dir}")
try:
# 1. 转换为 WAV
print(f"\n🔄 步骤1: 转换音频格式...")
audio_path = convert_to_wav(input_file, output_dir)
# 2. 运行 Whisper
print(f"\n🎙️ 步骤2: 语音识别 (语言: {args.language})...")
run_whisper(whisper_path, audio_path, args.model, output_dir, args.language)
# 3. 清理临时 WAV 文件
if audio_path != input_file and audio_path.endswith('_audio.wav'):
try:
os.remove(audio_path)
print("✓ 清理临时文件")
except Exception:
pass
# 4. 获取时长并创建 JSON
print(f"\n📝 步骤3: 生成输出文件...")
basename = Path(input_file).stem
srt_file = os.path.join(output_dir, f"{basename}.srt")
duration = parse_srt_for_duration(srt_file)
workflow_json = create_workflow_json(
input_file, output_dir, args.model, args.language, duration
)
# 输出结果
print(f"\n✅ 处理完成!")
print(f" 时长: {duration} 秒")
print(f" 分段数: {len(workflow_json['segments'])}")
print(f"\n📄 输出文件:")
for fmt in ['txt', 'srt', 'json']:
fpath = os.path.join(output_dir, f"{basename}.{fmt}")
if os.path.exists(fpath):
size = os.path.getsize(fpath)
print(f" • {fmt.upper()}: {fpath} ({size:,} bytes)")
print(f"\n📋 Workflow JSON:")
print(json.dumps({
"success": workflow_json["success"],
"tool": "whisper",
"text_length": len(workflow_json["text"]),
"segments_count": len(workflow_json["segments"])
}, ensure_ascii=False, indent=2))
except Exception as e:
print(f"\n✗ 错误: {e}", file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()
skills/kb-workflow/README.md
-247
View File
@@ -1,247 +0,0 @@
# KB Workflow 环境安装脚本
一键检查并安装 kb-workflow 知识库工作流所需的全部环境依赖。
## 功能特性
- ✅ 自动检测系统类型 (Ubuntu/Debian/Fedora/Arch)
- ✅ 检查系统依赖 (Python3, pip3, ffmpeg, tesseract, git)
- ✅ 检查 Python 依赖包 (httpx, aiohttp, playwright, easyocr 等)
- ✅ 自动创建知识库目录结构
- ✅ 生成环境变量配置示例
- ✅ 汇总显示所有需要执行的安装命令
## 使用方法
### 运行检查脚本
```bash
cd /home/starmen1265/.openclaw/workspace/skills
./install-kb-workflow.sh
```
脚本会检查所有依赖，并显示需要执行的安装命令。
### 示例输出
```
========================================
第一步: 检查系统依赖
========================================
[SUCCESS] Python3 已安装: Python 3.12.3
[WARN] pip3 未安装
[WARN] ffmpeg 未安装
...
========================================
需要安装的系统依赖
========================================
检测到以下依赖未安装:
- python3-pip
- ffmpeg
- tesseract-ocr
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
请运行以下命令安装:
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
sudo apt-get update
sudo apt-get install -y python3-pip ffmpeg tesseract-ocr tesseract-ocr-chi-sim tesseract-ocr-eng
```
## 依赖清单
### 系统依赖
| 依赖 | 用途 | 安装命令 |
|------|------|----------|
| **python3** | Python 运行时 | `sudo apt-get install python3` |
| **python3-pip** | Python 包管理器 | `sudo apt-get install python3-pip` |
| **ffmpeg** | 视频处理（OCR 截帧） | `sudo apt-get install ffmpeg` |
| **tesseract-ocr** | OCR 引擎（备用方案） | `sudo apt-get install tesseract-ocr` |
| **git** | 版本控制（归档提交） | `sudo apt-get install git` |
### Tesseract 语言包
```bash
# Ubuntu/Debian
sudo apt-get install tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-eng
# Fedora/RHEL
sudo dnf install tesseract-langpack-chi_sim tesseract-langpack-chi_tra tesseract-langpack-eng
```
### Python 依赖
```bash
# 基础 HTTP 库
pip3 install --user httpx aiohttp requests
# 图像和 OCR
pip3 install --user Pillow easyocr pytesseract
# 浏览器自动化
pip3 install --user playwright
python3 -m playwright install chromium
```
## 目录结构
安装完成后会创建以下目录结构：
```
~/work/knowledge-base/ # 知识库根目录（可配置）
├── archive/ # 归档目录
│ ├── pending/ # 待处理
│ └── subscriptions/ # 订阅信息
├── metadata/ # 元数据文件
├── content/ # 内容文件
│ ├── downloads/ # 视频下载
│ └── subtitles/ # 字幕文件
└── analyzed/ # 深度分析报告
```
## 环境变量
### 必需
- `KB_BASE_PATH` - 知识库根目录（默认: `~/work/knowledge-base`）
### 可选
- `BILIBILI_COOKIE` - B站 Cookie，用于下载会员视频
- `AI_API_KEY` - AI API 密钥，用于深度内容分析
- `AI_PROVIDER` - AI 提供商（openai/zhipu/minimax）
- `AI_MODEL` - AI 模型名称（如 gpt-4）
- `AI_BASE_URL` - 自定义 API 端点
- `KB_GIT_USER` - Git 用户名（用于归档提交）
- `KB_GIT_EMAIL` - Git 邮箱（用于归档提交）
- `KB_GIT_REMOTE` - Git 远程仓库地址
### 配置方法
将以下内容添加到 `~/.bashrc` 或 `~/.zshrc`：
```bash
# KB Workflow 配置
export KB_BASE_PATH="~/work/knowledge-base"
# 可选：B站 Cookie
export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx"
# 可选：AI 配置
export AI_API_KEY="your-api-key"
export AI_PROVIDER="openai"
export AI_MODEL="gpt-4"
```
然后重新加载配置：
```bash
source ~/.bashrc # 或 source ~/.zshrc
```
## 故障排除
### 1. pip3 命令不存在
```bash
# 方法 1: 安装 pip3
sudo apt-get install python3-pip
# 方法 2: 使用 python3 -m pip
python3 -m pip install --user <package>
```
### 2. Playwright 浏览器安装失败
```bash
# 手动安装 Chromium
python3 -m playwright install chromium
# 如果仍然失败，尝试安装依赖
sudo apt-get install libnss3 libatk-bridge2.0-0 libxss1 libgtk-3-0
```
### 3. EasyOCR 安装失败
EasyOCR 依赖 PyTorch，可能需要较长时间安装：
```bash
# 先安装 PyTorch
pip3 install --user torch torchvision --index-url https://download.pytorch.org/whl/cpu
# 再安装 easyocr
pip3 install --user easyocr
```
### 4. 权限问题
如果使用 `--user` 安装的包无法找到，检查 PATH：
```bash
# 查看用户安装路径
python3 -m site --user-base
# 添加到 PATH（添加到 ~/.bashrc）
export PATH="$PATH:$(python3 -m site --user-base)/bin"
```
### 5. OCR 中文识别问题
确保已安装中文语言包：
```bash
# 检查已安装的语言包
tesseract --list-langs
# 应该能看到 chi_sim 和 chi_tra
```
## 更新
如果技能更新了依赖，可以重新运行检查脚本：
```bash
./install-kb-workflow.sh
```
## 子技能清单
| 技能 | 路径 | 功能 |
|------|------|------|
| kb-workflow | `kb-workflow/kb_workflow.py` | 主流程协调 |
| kb-link-parser | `kb-link-parser/link-parser.py` | 链接解析 |
| kb-bilibili-metadata | `kb-bilibili-metadata/bilibili_metadata.py` | B站元数据提取 |
| kb-bilibili-content | `kb-bilibili-content/bilibili_content.py` | B站内容下载 |
| kb-ocr-processor | `kb-ocr-processor/ocr_processor.py` | 视频 OCR 识别 |
| kb-deep-analyzer | `kb-deep-analyzer/deep_analyzer.py` | AI 深度分析 |
## 使用 kb-workflow
安装完成后，可以通过以下方式使用：
### 方式 1：通过主代理
直接向主代理发送消息：
```
添加到知识库 https://www.bilibili.com/video/BVxxxxx
```
### 方式 2：手动运行子技能
```bash
# 1. 链接解析
python3 kb-link-parser/link-parser.py --url "https://www.bilibili.com/video/BVxxxxx"
# 2. 元数据提取
python3 kb-bilibili-metadata/bilibili_metadata.py --id "BVxxxxx" --type video
# 3. 内容下载
python3 kb-bilibili-content/bilibili_content.py --id "BVxxxxx" --type subtitle-download
# 4. 深度分析
python3 kb-deep-analyzer/deep_analyzer.py --metadata meta.json --content content.json --output analysis.md
```
## 许可证
MIT License
skills/kb-workflow/SKILL.md
+1 -32
View File
@@ -60,7 +60,7 @@ echo "✅ 环境变量检查通过"
## Workflow 执行方式
### 方式一：一键处理（推荐）
### 一键处理（推荐）
使用管道 CLI 一键完成所有步骤：
@@ -85,21 +85,6 @@ python3 kb-workflow/kb_workflow.py process "https://..." -o /path/to/output
python3 kb-workflow/kb_workflow.py process "https://..." --no-skip
```
### 方式二：手动分步执行（高级）
如需更细粒度控制，可手动调用各子技能：
```bash
# 1. 查重
python3 kb-workflow/kb_workflow.py --check --url "https://..."
# 2. 一键处理
python3 kb-workflow/kb_workflow.py process "https://..."
# 3. 归档
python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md"
```
## 支持的平台和内容类型
| 平台 | 内容类型 | URL 示例 |
@@ -152,22 +137,6 @@ python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md"
3. **禁止降级处理**：不能使用预设模板或简单摘要来"假装成功"
4. **每次执行都要使用 AI**：必须有真实的 AI API 调用记录
## 辅助命令
```bash
# 查重
python3 kb-workflow/kb_workflow.py --check --url "https://..."
# 获取目录路径
python3 kb-workflow/kb_workflow.py --base-path
python3 kb-workflow/kb_workflow.py --metadata
python3 kb-workflow/kb_workflow.py --content
python3 kb-workflow/kb_workflow.py --analyzed
# 归档分析报告
python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md"
```
## 示例
```
skills/kb-workflow/SkillBug.md
-101
View File
@@ -1,101 +0,0 @@
# SkillBug.md
## 问题记录
### 2026-02-10: 子代理权限问题
**问题：** `sessions_spawn` 调用子代理时返回 `forbidden` 错误。
**原因：** OpenClaw 配置中 allowlist 为空，Python 脚本没有权限调用子代理。
**解决方案：** 改用 Prompt-based 模式（方案B）。
---
## 架构重构
### 之前（Script-based）
```
kb_workflow.py (Python脚本)
↓
sessions_spawn() # ❌ 失败（脚本没有权限）
```
### 之后（Prompt-based）
```
主代理读取 SKILL.md
↓
按照指导调用 sessions_spawn # ✅ 成功（主代理有权限）
```
### 文件变更
| 文件 | 变更 | 说明 |
|------|------|------|
| `SKILL.md` | 重写 | 详细定义 workflow 步骤，主代理直接读取执行 |
| `kb_workflow.py` | 简化 | 保留为辅助工具函数库（查重、归档、Git） |
| `skill_main.py` | 简化 | 只做触发检测，返回指导信息 |
| `SkillBug.md` | 更新 | 记录架构变更 |
### SKILL.md 结构
```
1. 触发条件
2. 步骤 1: 查重
3. 步骤 2: kb-link-parser
4. 步骤 3: 元数据技能
5. 步骤 4: 内容下载
6. 步骤 5: OCR（30分钟超时）
7. 步骤 6: 深度分析
8. 步骤 7: 归档和 Git
9. 步骤 8: 汇报完成
10. 目录结构
11. 支持平台
12. 注意事项
```
### kb_workflow.py 保留的函数
```python
extract_platform(filename) # 从文件名提取平台
extract_base_name(filename) # 从文件名提取基础名称
get_knowledge_base_path() # 获取知识库根目录
ensure_directories(base_path) # 确保目录存在
check_duplicate(url) # 查重
archive_and_git(analyzed_file) # 归档和 Git
generate_summary(...) # 生成汇报
```
---
### 2026-02-12: B站视频下载失败 - 403 Forbidden
**问题：** 使用 kb-bilibili-content 下载视频时返回 403 Forbidden 错误。
**原因：**
1. 视频可能需要B站会员权限
2. 或者需要登录cookie才能下载
3. 当前未配置 `BILIBILI_COOKIE` 环境变量
**复现场景：**
- 链接：https://www.bilibili.com/video/BV1Z9nXzUEhv
- 下载命令：`python3 bilibili_content.py --id "BV1Z9nXzUEhv" --type video`
- 结果：返回403错误页面，文件只有548字节
**影响：** 无法进行后续的OCR处理和深度分析
**建议解决方案：**
1. 配置 `BILIBILI_COOKIE` 环境变量
2. 或提供cookies文件路径
3. 或跳过OCR步骤，使用元数据+简介进行深度分析
**当前进度：**
- ✅ 步骤1：环境变量检查完成
- ✅ 步骤2：链接解析成功（bilibili_BV1Z9nXzUEhv）
- ✅ 步骤3：元数据提取成功（标题：新加坡分区漫游指南...）
- ❌ 步骤4：视频下载失败（403）
- ⏸️ 步骤5：OCR处理（阻塞）
- ⏸️ 步骤6：深度分析（阻塞）
- ⏸️ 步骤7：归档（阻塞）
skills/kb-workflow/kb_workflow.py
+23 -389
View File
@@ -3,330 +3,24 @@
"""
KB Workflow - Knowledge Base Workflow Orchestrator
知识库工作流辅助工具函数。
知识库工作流管道入口。
此模块提供查重、文件操作、Git 管理等工具函数，
供主代理在执行 SKILL.md 中定义的 workflow 时调用。
此模块提供 process_command() 函数作为管道入口，
内部使用 kb_pipeline 包实现完整的内容处理流程。
使用方式:
python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
python kb_workflow.py process "https://..." -o /path/to/output
"""
import os
import sys
import json
import argparse
import subprocess
import shutil
import re
from datetime import datetime
import asyncio
import json
import sys
from pathlib import Path
from typing import Dict, List, Optional, Any
from typing import Any, Dict, Optional
# ==================== 子技能映射 ====================
METADATA_AGENTS = {
"bilibili": "kb-bilibili-metadata",
"zhihu": "kb-zhihu-metadata",
}
CONTENT_AGENTS = {
"bilibili": "kb-bilibili-content",
"zhihu": "kb-zhihu-content",
}
# ==================== 内容类型判断 ====================
# 需要下载视频/字幕的内容类型
VIDEO_CONTENT_TYPES = {"video", "zvideo"}
# 需要提取文字的内容类型
TEXT_CONTENT_TYPES = {"article", "zhuanlan", "question", "answer"}
# 需要下载图片的内容类型
IMAGE_CONTENT_TYPES = {"note"}
def is_video_content(content_type: str) -> bool:
"""判断是否为视频内容类型"""
return content_type in VIDEO_CONTENT_TYPES
def is_text_content(content_type: str) -> bool:
"""判断是否为文字内容类型（专栏、文章、问答等）"""
return content_type in TEXT_CONTENT_TYPES
def is_image_content(content_type: str) -> bool:
"""判断是否为图片内容类型（小红书等）"""
return content_type in IMAGE_CONTENT_TYPES
def get_content_handler(content_type: str) -> str:
"""
获取内容处理方式
Returns:
"video" - 需要下载视频/字幕
"text" - 需要提取文字
"image" - 需要下载图片
"unknown" - 未知类型
"""
if is_video_content(content_type):
return "video"
elif is_text_content(content_type):
return "text"
elif is_image_content(content_type):
return "image"
else:
return "unknown"
# ==================== 工具函数 ====================
def extract_platform(filename: str) -> str:
"""从文件名提取平台（第一个下划线之前）"""
match = re.match(r'^([a-zA-Z0-9]+)_(.+)$', filename)
if match:
return match.group(1)
return "unknown"
def extract_base_name(filename: str) -> str:
"""从文件名提取基础名称（去掉扩展名）"""
return Path(filename).stem
def get_knowledge_base_path() -> Path:
"""获取知识库根目录"""
base_path_str = os.environ.get('KB_BASE_PATH', '')
if base_path_str:
return Path(base_path_str).expanduser()
return Path.home() / "work" / "knowledge-base"
def get_archive_path() -> Path:
"""获取归档目录（AI分析文档存放位置）"""
archive_path_str = os.environ.get('KB_ARCHIVE_PATH', '')
if archive_path_str:
return Path(archive_path_str).expanduser()
# 默认：~/work/zq/30.资源/AI分析
return Path.home() / "work" / "zq" / "30.资源" / "AI分析"
def get_git_root() -> Path:
"""获取 Git 仓库根目录（用于归档提交）"""
git_root_str = os.environ.get('KB_GIT_ROOT', '')
if git_root_str:
return Path(git_root_str).expanduser()
# 默认：~/work/zq
return Path.home() / "work" / "zq"
def ensure_directories(base_path: Path = None) -> Dict[str, Path]:
"""确保目录存在"""
if base_path is None:
base_path = get_knowledge_base_path()
dirs = {
"base": base_path,
"pending": base_path / "pending",
"metadata": base_path / "metadata",
"content": base_path / "content",
"content_downloads": base_path / "content" / "downloads",
"analyzed": base_path / "analyzed",
"archive": base_path / "archive",
"archive_subscriptions": base_path / "archive" / "subscriptions",
}
for name, dir_path in dirs.items():
if isinstance(dir_path, Path):
dir_path.mkdir(parents=True, exist_ok=True)
return dirs
def check_duplicate(base_path: Path = None, url: str = None, platform: str = None, content_id: str = None) -> Dict[str, Any]:
"""查重检查"""
if base_path is None:
base_path = get_knowledge_base_path()
metadata_dir = base_path / "metadata"
archive_dir = base_path / "archive"
# 从URL提取ID
if url and not content_id:
bv_match = re.search(r'BV[a-zA-Z0-9]+', url)
content_id = bv_match.group() if bv_match else None
result = {"duplicate": False, "status": None, "files": []}
if metadata_dir.exists():
for f in metadata_dir.iterdir():
if f.is_file() and content_id and content_id in f.name:
result["duplicate"] = True
result["status"] = "metadata"
result["files"].append(str(f))
if archive_dir.exists():
for date_dir in archive_dir.iterdir():
if date_dir.is_dir():
for f in date_dir.iterdir():
if f.is_file() and content_id and content_id in f.name:
result["duplicate"] = True
result["status"] = "archived"
result["files"].append(str(f))
return result
def archive_and_git(
analyzed_file: str,
base_path: Path = None,
git_root: Path = None,
git_user: str = None,
git_email: str = None,
git_remote: str = None
) -> Dict[str, Any]:
"""归档并提交Git"""
if base_path is None:
base_path = get_knowledge_base_path()
if git_root is None:
git_root = get_git_root()
if git_user is None:
git_user = os.environ.get('KB_GIT_USER', '')
if git_email is None:
git_email = os.environ.get('KB_GIT_EMAIL', '')
if git_remote is None:
git_remote = os.environ.get('KB_GIT_REMOTE', '')
result = {
"success": False,
"archive_path": "",
"git_commit": False,
"git_push": None,
"error": None
}
try:
# 1. 检查并初始化 Git 仓库
git_dir = git_root / ".git"
if not git_dir.exists():
# 初始化仓库
subprocess.run(['git', 'init'], cwd=str(git_root), capture_output=True)
subprocess.run(['git', 'config', 'user.name', git_user], cwd=str(git_root))
subprocess.run(['git', 'config', 'user.email', git_email], cwd=str(git_root))
# 关联远端仓库
if git_remote:
subprocess.run(['git', 'remote', 'add', 'origin', git_remote], cwd=str(git_root), capture_output=True)
# 拉取远端分支
subprocess.run(['git', 'fetch', 'origin'], cwd=str(git_root), capture_output=True)
# 设置 main 分支并拉取
subprocess.run(['git', 'branch', '--set-upstream-to=origin/main', 'main'], cwd=str(git_root), capture_output=True)
subprocess.run(['git', 'pull', 'origin', 'main', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
else:
# 已有仓库，检查远端是否配置
if git_remote:
remote_check = subprocess.run(['git', 'remote', '-v'], cwd=str(git_root), capture_output=True, text=True)
if 'origin' not in remote_check.stdout:
subprocess.run(['git', 'remote', 'add', 'origin', git_remote], cwd=str(git_root), capture_output=True)
# 拉取远端最新代码
subprocess.run(['git', 'fetch', 'origin'], cwd=str(git_root), capture_output=True)
# 尝试拉取 main 分支，如果失败则尝试 master
pull_result = subprocess.run(['git', 'pull', 'origin', 'main', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
if pull_result.returncode != 0:
subprocess.run(['git', 'pull', 'origin', 'master', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
# 2. 创建归档目录
date_str = datetime.now().strftime('%Y-%m-%d')
archive_base = get_archive_path()
date_dir = archive_base / date_str
date_dir.mkdir(parents=True, exist_ok=True)
# 3. 复制分析报告
src_file = Path(analyzed_file)
archive_file = date_dir / src_file.name
shutil.copy(src_file, archive_file)
result["archive_path"] = str(archive_file)
# 4. 复制元数据
meta_file = base_path / "metadata" / f"{src_file.stem}.meta.json"
if meta_file.exists():
shutil.copy(meta_file, date_dir / meta_file.name)
# 5. Git 提交并推送
if (git_root / ".git").exists():
# 添加所有变更
subprocess.run(['git', 'add', '.'], cwd=str(git_root), capture_output=True)
# 检查是否有变更需要提交
status_result = subprocess.run(['git', 'status', '--porcelain'], cwd=str(git_root), capture_output=True, text=True)
if status_result.stdout.strip():
commit_msg = f"Archive: {src_file.name} - {datetime.now().isoformat()}"
commit_result = subprocess.run(
['git', 'commit', '-m', commit_msg],
cwd=str(git_root), capture_output=True, text=True
)
result["git_commit"] = commit_result.returncode == 0
# 推送到远端
if git_remote and commit_result.returncode == 0:
push_result = subprocess.run(
['git', 'push', 'origin', 'main'],
cwd=str(git_root), capture_output=True, text=True, timeout=60
)
if push_result.returncode != 0:
# 如果 main 分支推送失败，尝试 master
push_result = subprocess.run(
['git', 'push', 'origin', 'master'],
cwd=str(git_root), capture_output=True, text=True, timeout=60
)
result["git_push"] = {"success": push_result.returncode == 0, "output": push_result.stdout}
result["success"] = True
except Exception as e:
result["error"] = str(e)
return result
def generate_summary(url: str, stages: List[Dict], final_file: str, archive_result: Dict = None) -> str:
"""生成处理完成汇报"""
# 提取视频标题（如果有）
title_match = re.search(r'com/video/([^?]+)', url)
title = title_match.group(1) if title_match else "未知"
summary = f"""
✅ **处理完成！**
**视频:** {title}
**处理阶段:**
"""
for stage in stages:
stage_name = stage.get('stage', 'unknown')
emoji = "✓" if stage.get('success') else "○"
summary += f"{emoji} {stage_name}\n"
if final_file:
summary += f"\n**最终文件:** `{final_file}`"
if archive_result:
if archive_result.get('git_commit'):
summary += f"\n**Git:** 已提交"
if archive_result.get('git_push', {}).get('success'):
summary += " ✓"
elif archive_result.get('git_push'):
summary += " ✗"
return summary
# ==================== CLI Process Command ====================
def process_command(
url: str,
output_dir: Optional[str] = None,
@@ -342,8 +36,6 @@ def process_command(
Returns:
Dictionary with success status, content_dir, and results
"""
import asyncio
# Add project root to path for kb_pipeline imports
project_root = Path(__file__).parent.parent
if str(project_root) not in sys.path:
@@ -351,7 +43,7 @@ def process_command(
try:
from kb_pipeline.pipeline import Pipeline
from kb_pipeline.processors import ParseProcessor, DownloadProcessor, ASRProcessor, AnalyzeProcessor
from kb_pipeline.processors import ASRProcessor, AnalyzeProcessor, DownloadProcessor, ParseProcessor
from kb_pipeline.schemas import PipelineConfig
async def run_pipeline():
@@ -402,24 +94,22 @@ def process_command(
}
# ==================== CLI ====================
def main():
"""CLI入口 - 用于命令行测试"""
"""CLI入口 - 用于命令行调用"""
parser = argparse.ArgumentParser(
description='KB Workflow - Knowledge Base Workflow Orchestrator',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
epilog='''
使用示例:
# 查重
python kb_workflow.py --check --url "https://www.bilibili.com/video/BVxxx"
# 归档
python kb_workflow.py --archive "/path/to/analyzed.md"
# 处理URL (通过完整管道)
python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
"""
# 指定输出目录
python kb_workflow.py process "https://..." -o /path/to/output
# 强制重新执行（忽略已完成状态）
python kb_workflow.py process "https://..." --no-skip
'''
)
# Create subparsers for commands
@@ -430,29 +120,12 @@ def main():
process_parser.add_argument('url', type=str, help='URL to process')
process_parser.add_argument('--output-dir', '-o', type=str, help='Output directory for content')
process_parser.add_argument('--skip-completed', action='store_true', default=True,
help='Skip completed steps (default: True)')
help='Skip completed steps (default: True)')
process_parser.add_argument('--no-skip', action='store_false', dest='skip_completed',
help='Re-run all steps')
help='Re-run all steps')
# Legacy arguments (for backward compatibility)
parser.add_argument('--check', action='store_true', help='查重')
parser.add_argument('--url', type=str, help='要检查的URL')
parser.add_argument('--archive-file', type=str, help='归档的文件路径')
parser.add_argument('--base-path', type=str, help='获取知识库根目录')
parser.add_argument('--pending', action='store_true', help='获取url解析目录')
parser.add_argument('--metadata', action='store_true', help='获取元数据目录')
parser.add_argument('--content', action='store_true', help='获取内容目录')
parser.add_argument('--content_downloads', action='store_true', help='获取主内容下载目录')
parser.add_argument('--analyzed', action='store_true', help='获取深度分析目录')
parser.add_argument('--archive_dir', action='store_true', help='获取归档目录')
parser.add_argument('--archive_subscriptions', action='store_true', help='获取订阅信息目录')
parser.add_argument('--extract_platform', type=str, help='要提取平台名称的文件名')
args = parser.parse_args()
base_path = Path(args.base_path) if args.base_path else None
dirs = ensure_directories(base_path)
# Handle process subcommand
if args.command == 'process':
result = process_command(
@@ -461,45 +134,6 @@ def main():
skip_completed=args.skip_completed
)
print(json.dumps(result, ensure_ascii=False, indent=2))
return
# Legacy argument handling
if args.check and args.url:
result = check_duplicate(base_path, args.url)
print(json.dumps(result, ensure_ascii=False, indent=2))
elif args.archive_file:
result = archive_and_git(args.archive_file, base_path)
print(json.dumps(result, ensure_ascii=False, indent=2))
elif args.pending:
print(json.dumps({"path": str(dirs["pending"])}, ensure_ascii=False, indent=2))
elif args.metadata:
print(json.dumps({"path": str(dirs["metadata"])}, ensure_ascii=False, indent=2))
elif args.content:
print(json.dumps({"path": str(dirs["content"])}, ensure_ascii=False, indent=2))
elif args.content_downloads:
print(json.dumps({"path": str(dirs["content_downloads"])}, ensure_ascii=False, indent=2))
elif args.analyzed:
print(json.dumps({"path": str(dirs["analyzed"])}, ensure_ascii=False, indent=2))
elif args.archive_dir:
print(json.dumps({"path": str(get_archive_path())}, ensure_ascii=False, indent=2))
elif args.archive_subscriptions:
print(json.dumps({"path": str(dirs["archive_subscriptions"])}, ensure_ascii=False, indent=2))
elif args.extract_platform:
platform = extract_platform(args.extract_platform)
print(json.dumps({"platform": platform}, ensure_ascii=False, indent=2))
elif args.base_path:
print(json.dumps({"path": str(dirs["base"])}, ensure_ascii=False, indent=2))
else:
parser.print_help()
skills/kb-zhihu-content/SKILL.md
-86
View File
@@ -1,86 +0,0 @@
# KB Zhihu Content Skill
**kb-zhihu-content** - Download Zhihu videos and extract content.
## Features
- Video download using **yt-dlp**
- Subtitle extraction (returns no subtitle, triggers Whisper)
- **Read article content from MediaCrawler data** (NEW!)
- Supports `ZHIHU_COOKIE` environment variable
## Environment Variables
| Variable | Required | Description |
|----------|----------|-------------|
| `ZHIHU_COOKIE` | No | 知乎登录 Cookie (for member-only videos) |
## Commands
```bash
# Download video
python zhihu_content.py --id "12345678" --type video --output "/path/to/downloads/"
# Extract subtitle (returns no subtitle, triggers Whisper)
python zhihu_content.py --id "12345678" --type subtitle --output "/path/to/subtitles/"
# 从 MediaCrawler 读取文章内容 (⚡ NEW!)
python zhihu_content.py --type article-from-media --input "~/work/MediaCrawler/data/zhihu/json/detail_contents_2026-02-27.json" --output "~/work/knowledge-base/data/zhihu/"
```
## MediaCrawler 数据读取
**新功能**：从 MediaCrawler 爬取的数据中提取知乎文章内容
### 使用方法
```bash
# 从 MediaCrawler JSON 文件读取文章内容
python zhihu_content.py --type article-from-media \
--input "~/work/MediaCrawler/data/zhihu/json/detail_contents_2026-02-27.json" \
--output "~/work/knowledge-base/data/zhihu/"
# 查找最新的爬取数据
python zhihu_content.py --type article-from-media \
--output "~/work/knowledge-base/data/zhihu/"
```
### 参数说明
| 参数 | 说明 |
|------|------|
| `--type article-from-media` | 从 MediaCrawler 数据读取 |
| `--input` | MediaCrawler 输出文件路径（可选，不指定则自动查找最新） |
| `--output` | 输出目录 |
### 输出格式
```json
{
"success": true,
"type": "article",
"filename": "zhihu_article_如何高效的爬取微信公众号.md",
"filepath": "/home/xxx/work/knowledge-base/data/zhihu/zhihu_article_如何高效的爬取微信公众号.md",
"message": "文章内容已导出"
}
```
## 原有功能
### Video Download
Video download uses **yt-dlp**:
1. Parses `ZHIHU_COOKIE` environment variable (optional)
2. Creates temporary Netscape-format cookie file
3. Runs `yt-dlp -f best -o <output> <url>`
4. Cleans up temporary file
**Note**: 知乎视频通常没有官方字幕，将自动触发 Whisper 语音识别
## Installation
```bash
# Already included in kb-skills repo
# Ensure yt-dlp is installed
pip install yt-dlp aiohttp
```
skills/kb-zhihu-content/zhihu_content.py
-365
View File
@@ -1,365 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
知乎内容获取工具
支持：
- zvideo: 知乎视频下载
- article-from-media: 从 MediaCrawler 数据读取知乎文章
- 从页面提取视频播放地址
"""
import argparse
import asyncio
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Optional, Dict, Any, List
# MediaCrawler 目录
MEDIACRAWLER_DIR = Path.home() / "work" / "MediaCrawler"
class ZhihuContent:
"""知乎内容获取器"""
def __init__(self, output_dir: str = "./output", cookie: str = None):
self.output_dir = Path(output_dir)
self.output_dir.mkdir(parents=True, exist_ok=True)
self.cookie = cookie
def _create_cookie_file(self) -> str:
"""创建 Netscape 格式的 Cookie 文件"""
if not self.cookie:
return None
cookies = {}
for item in self.cookie.split(';'):
item = item.strip()
if '=' in item:
key, value = item.split('=', 1)
cookies[key.strip()] = value.strip()
cookie_content = """# Netscape HTTP Cookie File
# Generated by zhihu_content.py
"""
for name, value in cookies.items():
cookie_content += f".zhihu.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n"
fd, path = tempfile.mkstemp(suffix='.txt')
with os.fdopen(fd, 'w') as f:
f.write(cookie_content)
return path
async def download_video(self, video_id: str, quality: int = 720) -> Dict[str, Any]:
"""下载知乎视频（使用 yt-dlp）"""
result = {"success": False, "video_id": video_id, "message": "", "filepath": None, "metadata": {}}
try:
print(f"获取知乎视频信息: {video_id}")
cookie_file = self._create_cookie_file()
url = f"https://www.zhihu.com/zvideo/{video_id}"
# 先获取视频标题
try:
if cookie_file:
cmd = ["yt-dlp", "--cookies", cookie_file, "--get-title", url]
else:
cmd = ["yt-dlp", "--get-title", url]
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
title = stdout.decode('utf-8').strip()
if not title or 'error' in title.lower():
title = f"zhihu_video_{video_id}"
except Exception as e:
title = f"zhihu_video_{video_id}"
# 构建 yt-dlp 命令
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
print(f"下载视频: {title}")
output_template = str(self.output_dir / f"{safe_title}.%(ext)s")
# 构建下载命令
cmd = [
"yt-dlp",
"-f", "bestvideo+bestaudio/best",
"-o", output_template,
"--no-download-archive",
url
]
if cookie_file:
cmd.extend(["--cookies", cookie_file])
print(f"执行命令: {' '.join(cmd[:4])}...")
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
# 查找下载的文件
downloaded_files = list(self.output_dir.glob(f"{safe_title}.*"))
if downloaded_files:
video_file = downloaded_files[0]
result["success"] = True
result["filepath"] = str(video_file)
result["message"] = f"下载完成: {video_file.name}"
result["metadata"] = {"title": title, "video_id": video_id}
print(f"✅ {result['message']}")
else:
result["message"] = "未找到下载的文件"
print(f"❌ {result['message']}")
# 清理临时 Cookie 文件
if cookie_file and os.path.exists(cookie_file):
os.remove(cookie_file)
except Exception as e:
result["message"] = f"错误: {str(e)}"
print(f"❌ {result['message']}")
return result
async def extract_subtitle(self, video_id: str) -> Dict[str, Any]:
"""
提取字幕
知乎视频通常没有官方字幕，需要使用 Whisper 进行语音识别。
此函数返回 has_subtitle=False，让工作流继续执行 Whisper 步骤。
"""
result = {"success": True, "video_id": video_id, "has_subtitle": False, "message": "知乎视频无官方字幕，将使用 Whisper 进行语音识别"}
return result
def read_article_from_mediacrawler(self, input_file: str = None, output_dir: str = None) -> Dict[str, Any]:
"""
从 MediaCrawler 爬取的数据中读取知乎文章内容
Args:
input_file: MediaCrawler 输出文件路径，不指定则自动查找最新
output_dir: 输出目录
Returns:
处理结果字典
"""
# 确定输出目录
if output_dir:
self.output_dir = Path(output_dir)
self.output_dir.mkdir(parents=True, exist_ok=True)
# 查找数据文件
data_dir = MEDIACRAWLER_DIR / "data" / "zhihu" / "json"
if input_file:
# 使用指定的文件
json_file = Path(input_file)
if not json_file.exists():
return {"success": False, "message": f"文件不存在: {input_file}"}
else:
# 查找最新的数据文件
json_files = sorted(data_dir.glob("detail_contents_*.json"), reverse=True)
if not json_files:
return {"success": False, "message": "未找到 MediaCrawler 爬取的数据，请先运行爬虫"}
json_file = json_files[0]
try:
with open(json_file, encoding="utf-8") as f:
data = json.load(f)
if not data:
return {"success": False, "message": "数据文件为空"}
# 提取第一篇文章
item = data[0]
# 准备数据
title = item.get("title", "无标题")
content = item.get("content_text", "")
url = item.get("content_url", "")
author = item.get("user_nickname", "未知")
created_ts = item.get("created_time", 0)
created = datetime.fromtimestamp(created_ts).strftime("%Y-%m-%d %H:%M:%S") if created_ts else ""
voteup = item.get("voteup_count", 0)
comments = item.get("comment_count", 0)
# 生成文件名
safe_title = "".join(c for c in title if c.isalnum() or c in " -_")[:50]
filename = f"zhihu_article_{safe_title}.md"
output_path = self.output_dir / filename
# 写入 Markdown 文件
markdown_content = f"""---
title: "{title}"
author: "{author}"
url: "{url}"
created: "{created}"
voteup: {voteup}
comments: {comments}
platform: zhihu
type: article
---
# {title}
**作者**: {author}
**来源**: [知乎]({url})
**发布时间**: {created}
**点赞**: {voteup}
**评论**: {comments}
---
{content}
"""
with open(output_path, "w", encoding="utf-8") as f:
f.write(markdown_content)
return {
"success": True,
"type": "article",
"title": title,
"filename": filename,
"filepath": str(output_path),
"content_length": len(content),
"message": f"文章内容已导出: {title}"
}
except Exception as e:
return {"success": False, "message": f"读取失败: {str(e)}"}
async def main():
parser = argparse.ArgumentParser(description='知乎内容获取工具')
parser.add_argument('--id', '-i', help='知乎视频ID (用于 video/subtitle 类型)')
parser.add_argument('--type', '-t', required=True,
choices=['video', 'subtitle', 'article-from-media', 'zhuanlan', 'article'],
help='内容类型: video=视频下载, subtitle=字幕提取, article-from-media=从MediaCrawler读取文章, zhuanlan/article=爬取知乎专栏文章')
parser.add_argument('--output', '-o', default='./output', help='输出目录')
parser.add_argument('--quality', '-q', type=int, default=720, help='视频质量')
parser.add_argument('--input', '-in', help='MediaCrawler 数据文件路径 (用于 article-from-media 类型)')
args = parser.parse_args()
cookie = os.environ.get('ZHIHU_COOKIE')
downloader = ZhihuContent(output_dir=args.output, cookie=cookie)
if args.type == 'video':
result = await downloader.download_video(args.id, quality=args.quality)
elif args.type == 'subtitle':
result = await downloader.extract_subtitle(args.id)
elif args.type == 'article-from-media':
# 同步调用，不需要 asyncio
result = downloader.read_article_from_mediacrawler(
input_file=args.input,
output_dir=args.output
)
elif args.type in ('zhuanlan', 'article'):
# 知乎专栏文章：先爬取再读取
# 需要从 --id 或构造完整 URL
if args.id:
# 如果传入的是完整 URL
if args.id.startswith('http'):
article_url = args.id
else:
# 如果只传入 ID，构造 URL
article_url = f"https://zhuanlan.zhihu.com/p/{args.id}"
else:
result = {"success": False, "message": "zhuanlan/article 类型需要 --id 参数（文章ID或完整URL）"}
print(json.dumps(result, ensure_ascii=False, indent=2))
return result
# 调用 zhihu_article_crawler.py 爬取
crawler_script = Path(__file__).parent.parent / "kb-zhihu-metadata" / "zhihu_article_crawler.py"
if not crawler_script.exists():
# 尝试另一种路径
crawler_script = Path.home() / ".openclaw" / "agents" / "knowledge-base" / "workspace" / "skills" / "kb-zhihu-metadata" / "zhihu_article_crawler.py"
if crawler_script.exists():
import subprocess
cmd = ["python3", str(crawler_script), "--url", article_url, "--output", args.output]
print(f"调用爬虫: {' '.join(cmd)}")
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
result = {"success": False, "message": f"爬取失败: {proc.stderr[:200]}"}
else:
# 解析爬虫输出获取文件路径
try:
crawler_output = json.loads(proc.stdout.split("--- JSON 输出 ---")[-1].strip())
if crawler_output.get("success"):
# 直接使用爬虫输出的文件
result = {
"success": True,
"type": "article",
"title": crawler_output.get("title"),
"filename": Path(crawler_output["filepath"]).name,
"filepath": crawler_output["filepath"],
"content_length": crawler_output.get("content_length"),
"message": f"知乎专栏文章已爬取: {crawler_output.get('title')}"
}
else:
result = {"success": False, "message": crawler_output.get("error", "爬取失败")}
except Exception as e:
result = {"success": False, "message": f"解析爬虫输出失败: {str(e)}"}
else:
result = {"success": False, "message": f"未找到 zhihu_article_crawler.py: {crawler_script}"}
else:
result = {"success": False, "message": f"不支持的类型: {args.type}"}
# 统一输出格式
if result.get("success"):
output = {
"success": True,
"type": args.type,
"has_subtitle": result.get("has_subtitle", False),
"filename": result.get('filename') or (Path(result.get('filepath', '')).name if result.get('filepath') else None),
"filepath": str(Path(result.get('filepath', '')).resolve()) if result.get('filepath') else None,
"message": result.get('message', '处理成功')
}
if result.get('title'):
output['title'] = result['title']
if result.get('content_length'):
output['content_length'] = result['content_length']
else:
output = {
"success": False,
"error": result.get('message', '处理失败'),
"detail": f"ID: {args.id}, 类型: {args.type}"
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return result
if __name__ == '__main__':
try:
result = asyncio.run(main())
sys.exit(0 if result.get("success") else 1)
except KeyboardInterrupt:
print(json.dumps({"success": False, "error": "用户取消"}, ensure_ascii=False, indent=2))
sys.exit(1)
except Exception as e:
print(json.dumps({"success": False, "error": str(e)}, ensure_ascii=False, indent=2))
sys.exit(1)
skills/kb-zhihu-metadata/markdown_converter.py
+7
View File
@@ -38,6 +38,9 @@ class MarkdownConverter:
将 data-actualsrc 或 data-original 属性的值替换到 src 属性中，
以便 markdownify 能够正确提取真实图片 URL。
同时移除 <noscript> 标签，因为知乎等平台在 noscript 中包含备用图片，
会导致修复后图片重复出现。
Args:
html: 输入的 HTML 字符串
@@ -46,6 +49,10 @@ class MarkdownConverter:
"""
soup = BeautifulSoup(html, 'html.parser')
# 先移除 noscript 标签，避免图片重复
for noscript in soup.find_all('noscript'):
noscript.decompose()
for img in soup.find_all('img'):
src = img.get('src', '')