diff --git a/agents/dev-assist/workspace/AGENTS.md b/agents/dev-assist/workspace/AGENTS.md index c472b63..d045f0f 100644 --- a/agents/dev-assist/workspace/AGENTS.md +++ b/agents/dev-assist/workspace/AGENTS.md @@ -69,9 +69,14 @@ This folder is home. Treat it that way. ### 用户的工作区 - 路径：~/work/zq -- 这个工作区目录通过rclone与Nas同步 -- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"` -- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"` +- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`） +  - 所有操作实时同步到NAS，无需手动sync +  - 管理命令：`systemctl --user status|restart|stop rclone-zq` +  - 日志：`~/.cache/rclone/zq-mount.log` +- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录** +  - 该目录已通过 rclone mount 挂载，直接读写挂载点即可 +  - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新 +  - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步 - 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md - 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒 diff --git a/agents/haonan/sessions/sessions.json b/agents/haonan/sessions/sessions.json index 67faa07..296fa05 100644 --- a/agents/haonan/sessions/sessions.json +++ b/agents/haonan/sessions/sessions.json @@ -1,7 +1,7 @@ { "agent:haonan:main": { - "sessionId": "40e80fbb-07c8-440d-a1d4-1e876b86b82d", - "updatedAt": 1773508722585, + "sessionId": "e4845dbe-7d4d-41b6-adb5-b8a739e6520d", + "updatedAt": 1773620936802, "systemSent": true, "skillsSnapshot": { "prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n\n \n capability-evolver\n A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.\n ~/.openclaw/skills/capability-evolver/SKILL.md\n \n \n clawlist\n MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.\n ~/.openclaw/skills/clawlist/SKILL.md\n \n", @@ -59,11 +59,11 @@ "to": "user:ou_be83febac9ee2afaa343c642bc697c48", "accountId": "feishu-pm" }, - "sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/40e80fbb-07c8-440d-a1d4-1e876b86b82d.jsonl", + "sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/e4845dbe-7d4d-41b6-adb5-b8a739e6520d.jsonl", "systemPromptReport": { "source": "run", - "generatedAt": 1773508545805, - "sessionId": "40e80fbb-07c8-440d-a1d4-1e876b86b82d", + "generatedAt": 1773620921082, + "sessionId": "e4845dbe-7d4d-41b6-adb5-b8a739e6520d", "sessionKey": "agent:haonan:main", "provider": "zhipu", "model": "glm-5", @@ -75,8 +75,8 @@ "sandboxed": false }, "systemPrompt": { - "chars": 29493, - "projectContextChars": 16491, + "chars": 28195, + "projectContextChars": 15193, "nonProjectContextChars": 13002 }, "injectedWorkspaceFiles": [ @@ -144,18 +144,6 @@ "injectedChars": 1168, "truncated": false }, - { - "path": "SELF_IMPROVEMENT_REMINDER.md", - "rawChars": 616, - "injectedChars": 616, - "truncated": false - }, - { - "path": "SELF_IMPROVEMENT_REMINDER.md", - "rawChars": 616, - "injectedChars": 616, - "truncated": false - }, { "path": "SELF_IMPROVEMENT_REMINDER.md", "rawChars": 616, @@ -391,12 +379,12 @@ "totalTokensFresh": true, "lastHeartbeatText": "已在飞书向蒋先生发送日历同步提醒！✅", "lastHeartbeatSentAt": 1772924419184, - "cacheRead": 29760, + "cacheRead": 5376, "cacheWrite": 0, "contextTokens": 200000, - "inputTokens": 13411, - "outputTokens": 1288, - "totalTokens": 30249 + "inputTokens": 12014, + "outputTokens": 510, + "totalTokens": 17390 }, "agent:haonan:wecom:dm:zhangqing": { "origin": { @@ -727,8 +715,8 @@ "memoryFlushCompactionCount": 0 }, "agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc": { - "sessionId": "a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda", - "updatedAt": 1773597600006, + "sessionId": "3941961e-62e9-42a2-9ec7-0847ef6734fb", + "updatedAt": 1773684000005, "systemSent": true, "model": "glm-5", "contextTokens": 200000, @@ -769,11 +757,11 @@ "version": 0 }, "modelProvider": "zhipu", - "inputTokens": 11356, - "outputTokens": 531, - "totalTokens": 17361, + "inputTokens": 10481, + "outputTokens": 227, + "totalTokens": 16606, "totalTokensFresh": true, - "cacheRead": 90112, + "cacheRead": 22592, "cacheWrite": 0 }, "agent:haonan:cron:f03b184f-5f1d-4093-ae1b-fc9fe0b59eef": { @@ -819,9 +807,9 @@ "version": 0 }, "modelProvider": "zhipu", - "inputTokens": 153698, - "outputTokens": 6902, - "totalTokens": 33385, + "inputTokens": 41440, + "outputTokens": 3044, + "totalTokens": 25129, "deliveryContext": { "channel": "webchat" }, @@ -830,7 +818,7 @@ "sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl", "abortedLastRun": false, "totalTokensFresh": true, - "cacheRead": 384894, + "cacheRead": 90423, "cacheWrite": 0 }, "agent:haonan:cron:3245fca7-045b-4977-8fab-020f6df2e453": { @@ -873,8 +861,8 @@ "cacheWrite": 0 }, "agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc": { - "sessionId": "69fcc73b-3185-4ad9-aacd-e44989b770d9", - "updatedAt": 1773598200007, + "sessionId": "5432f300-44b1-496e-b169-19a4593e5352", + "updatedAt": 1773684600005, "systemSent": true, "label": "Cron: openclaw-git-sync", "skillsSnapshot": { @@ -922,113 +910,6 @@ "cacheRead": 32768, "cacheWrite": 0 }, - "agent:haonan:cron:f03b184f-5f1d-4093-ae1b-fc9fe0b59eef:run:ed66991b-9963-45f0-b7c1-c2b2655e0463": { - "sessionId": "ed66991b-9963-45f0-b7c1-c2b2655e0463", - "updatedAt": 1773512400007, - "systemSent": true, - "model": "glm-4.7-FlashX", - "contextTokens": 200000, - "label": "Cron: capability-evolver-daily", - "skillsSnapshot": { - "prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n\n \n capability-evolver\n A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.\n ~/.openclaw/skills/capability-evolver/SKILL.md\n \n \n clawlist\n MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.\n ~/.openclaw/skills/clawlist/SKILL.md\n \n", - "skills": [ - { - "name": "capability-evolver" - }, - { - "name": "clawlist" - } - ], - "skillFilter": [ - "clawlist", - "capability-evolver", - "self-improving-agent" - ], - "resolvedSkills": [ - { - "name": "capability-evolver", - "description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.", - "filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md", - "baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver", - "source": "openclaw-managed", - "disableModelInvocation": false - }, - { - "name": "clawlist", - "description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.", - "filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md", - "baseDir": "/home/starmen1265/.openclaw/skills/clawlist", - "source": "openclaw-managed", - "disableModelInvocation": false - } - ], - "version": 0 - }, - "modelProvider": "zhipu", - "inputTokens": 153698, - "outputTokens": 6902, - "totalTokens": 33385, - "deliveryContext": { - "channel": "webchat" - }, - "lastChannel": "webchat", - "channel": "webchat", - "sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl", - "abortedLastRun": false, - "totalTokensFresh": true, - "cacheRead": 384894, - "cacheWrite": 0 - }, - "agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc:run:a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda": { - "sessionId": "a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda", - "updatedAt": 1773597600006, - "systemSent": true, - "model": "glm-5", - "contextTokens": 200000, - "label": "Cron: backup-openclaw-json", - "skillsSnapshot": { - "prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n\n \n capability-evolver\n A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.\n ~/.openclaw/skills/capability-evolver/SKILL.md\n \n \n clawlist\n MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.\n ~/.openclaw/skills/clawlist/SKILL.md\n \n", - "skills": [ - { - "name": "capability-evolver" - }, - { - "name": "clawlist" - } - ], - "skillFilter": [ - "clawlist", - "capability-evolver", - "self-improving-agent" - ], - "resolvedSkills": [ - { - "name": "capability-evolver", - "description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.", - "filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md", - "baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver", - "source": "openclaw-managed", - "disableModelInvocation": false - }, - { - "name": "clawlist", - "description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.", - "filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md", - "baseDir": "/home/starmen1265/.openclaw/skills/clawlist", - "source": "openclaw-managed", - "disableModelInvocation": false - } - ], - "version": 0 - }, - "modelProvider": "zhipu", - "inputTokens": 11356, - "outputTokens": 531, - "totalTokens": 17361, - "totalTokensFresh": true, - "cacheRead": 90112, - "cacheWrite": 0 - }, "agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc:run:69fcc73b-3185-4ad9-aacd-e44989b770d9": { "sessionId": "69fcc73b-3185-4ad9-aacd-e44989b770d9", "updatedAt": 1773598200007, @@ -1122,9 +1003,9 @@ "version": 0 }, "modelProvider": "zhipu", - "inputTokens": 153698, - "outputTokens": 6902, - "totalTokens": 33385, + "inputTokens": 41440, + "outputTokens": 3044, + "totalTokens": 25129, "deliveryContext": { "channel": "webchat" }, @@ -1133,7 +1014,107 @@ "sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl", "abortedLastRun": false, "totalTokensFresh": true, - "cacheRead": 384894, + "cacheRead": 90423, + "cacheWrite": 0 + }, + "agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc:run:3941961e-62e9-42a2-9ec7-0847ef6734fb": { + "sessionId": "3941961e-62e9-42a2-9ec7-0847ef6734fb", + "updatedAt": 1773684000005, + "systemSent": true, + "model": "glm-5", + "contextTokens": 200000, + "label": "Cron: backup-openclaw-json", + "skillsSnapshot": { + "prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n\n \n capability-evolver\n A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.\n ~/.openclaw/skills/capability-evolver/SKILL.md\n \n \n clawlist\n MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.\n ~/.openclaw/skills/clawlist/SKILL.md\n \n", + "skills": [ + { + "name": "capability-evolver" + }, + { + "name": "clawlist" + } + ], + "skillFilter": [ + "clawlist", + "capability-evolver", + "self-improving-agent" + ], + "resolvedSkills": [ + { + "name": "capability-evolver", + "description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.", + "filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md", + "baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver", + "source": "openclaw-managed", + "disableModelInvocation": false + }, + { + "name": "clawlist", + "description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.", + "filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md", + "baseDir": "/home/starmen1265/.openclaw/skills/clawlist", + "source": "openclaw-managed", + "disableModelInvocation": false + } + ], + "version": 0 + }, + "modelProvider": "zhipu", + "inputTokens": 10481, + "outputTokens": 227, + "totalTokens": 16606, + "totalTokensFresh": true, + "cacheRead": 22592, + "cacheWrite": 0 + }, + "agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc:run:5432f300-44b1-496e-b169-19a4593e5352": { + "sessionId": "5432f300-44b1-496e-b169-19a4593e5352", + "updatedAt": 1773684600005, + "systemSent": true, + "label": "Cron: openclaw-git-sync", + "skillsSnapshot": { + "prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n\n \n capability-evolver\n A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.\n ~/.openclaw/skills/capability-evolver/SKILL.md\n \n \n clawlist\n MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.\n ~/.openclaw/skills/clawlist/SKILL.md\n \n", + "skills": [ + { + "name": "capability-evolver" + }, + { + "name": "clawlist" + } + ], + "skillFilter": [ + "clawlist", + "capability-evolver", + "self-improving-agent" + ], + "resolvedSkills": [ + { + "name": "capability-evolver", + "description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.", + "filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md", + "baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver", + "source": "openclaw-managed", + "disableModelInvocation": false + }, + { + "name": "clawlist", + "description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.", + "filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md", + "baseDir": "/home/starmen1265/.openclaw/skills/clawlist", + "source": "openclaw-managed", + "disableModelInvocation": false + } + ], + "version": 0 + }, + "modelProvider": "zhipu", + "model": "glm-5", + "contextTokens": 200000, + "inputTokens": 579, + "outputTokens": 363, + "totalTokens": 16886, + "totalTokensFresh": true, + "cacheRead": 32768, "cacheWrite": 0 } } \ No newline at end of file diff --git a/agents/haonan/workspace/AGENTS.md b/agents/haonan/workspace/AGENTS.md index 806ce61..db925cb 100644 --- a/agents/haonan/workspace/AGENTS.md +++ b/agents/haonan/workspace/AGENTS.md @@ -69,9 +69,14 @@ This folder is home. Treat it that way. ### 用户的工作区 - 路径：~/work/zq -- 这个工作区目录通过rclone与Nas同步 -- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"` -- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"` +- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`） +  - 所有操作实时同步到NAS，无需手动sync +  - 管理命令：`systemctl --user status|restart|stop rclone-zq` +  - 日志：`~/.cache/rclone/zq-mount.log` +- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录** +  - 该目录已通过 rclone mount 挂载，直接读写挂载点即可 +  - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新 +  - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步 - 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md - 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒 diff --git a/agents/knowledge-base/workspace/.learnings/ERRORS.md b/agents/knowledge-base/workspace/.learnings/ERRORS.md index 3cae7b2..ebe6764 100644 --- a/agents/knowledge-base/workspace/.learnings/ERRORS.md +++ b/agents/knowledge-base/workspace/.learnings/ERRORS.md @@ -1,39 +1,40 @@ # ERRORS.md - 错误记录 -## 2026-03-10: kb-workflow 流程执行不完整 +## 2026-03-16: rclone sync 覆盖了用户更新 -### 发生了什么 -用户请求"添加到知识库 + 链接"，我启动了 kb-workflow 技能，但两次漏掉步骤： -1. 第一次：只做了链接解析、元数据提取、内容提取，漏掉了深度分析 -2. 第二次：深度分析完成后，漏掉了归档步骤 +**问题描述**： +- `~/work/zq` 已通过 `rclone mount` 挂载，实时同步到 NAS +- 我在 18:25 执行了 `rclone sync ~/work/zq/ "nas-webdav:..."`（直接操作远程） +- 这绕过了挂载点缓存，覆盖了用户 18:30 在 NAS 上的更新 -### 根本原因 -1. **没有建立步骤检查清单** - SKILL.md 明确列出了8个步骤，但我没有逐项核对 -2. **并行子代理导致认知偏差** - 启动3个子代理处理后以为工作结束了 -3. **过早汇报完成** - 在步骤4完成后就向用户汇报，没有继续后续步骤 -4. **忘记归档是必须步骤** - 即使分析完成了，归档才是最终目标 +**根本原因**： +- AGENTS.md 中没有明确禁止对已挂载目录使用 rclone sync +- 误以为需要手动同步 -### 正确做法 -``` -1. 执行技能前，先列出完整步骤清单 -2. 每完成一步，标记状态 [x] -3. 全部完成后，再次核对清单 -4. 全部 ✅ 才能向用户汇报完成 -``` +**教训**： +- **禁止对已挂载目录（~/work/zq）使用 rclone sync** +- 直接读写挂载点即可，挂载点会自动同步 -### kb-workflow 完整步骤清单（必须全部执行） -| 步骤 | 内容 | 状态 | -|------|------|------| -| 1 | 环境变量检查 | [ ] | -| 2 | 链接解析 (kb-link-parser) | [ ] | -| 3 | 元数据提取 (kb-xxx-metadata) | [ ] | -| 4 | 内容下载/提取 (kb-xxx-content) | [ ] | -| 5 | 语音识别（视频才需要）| [ ] 或 [跳过] | -| 6 | 深度分析 (kb-deep-analyzer) | [ ] | -| 7 | 归档 (kb_workflow.py --archive-file) | [ ] | -| 8 | NAS 同步 + 汇报完成 | [ ] | +**已修正**： +- 更新了 AGENTS.md，添加了明确的禁止规则 -### 如何避免再犯 -- 每次执行 kb-workflow，必须使用上面的清单逐项核对 -- 最后一步"归档"没完成，不算完成 -- 用户问"为什么没XX"时，立即检查是否漏步骤 +--- + +## 2026-03-16: kb-workflow 技能缺少归档步骤 + +**问题描述**： +- SKILL.md 文档描述了 5 步流程（parse → download → asr → analyze → archive） +- 实际 `kb_workflow.py` 只实现了 4 步，缺少 archive 步骤 +- `KB_ARCHIVE_PATH` 环境变量被定义但未被使用 + +**影响**： +- 处理后的文件留在 workspace 而非归档到工作区 +- 需要手动移动文件 + +**临时解决方案**： +- 手动执行 `cp` 到归档目录（直接操作挂载点 ~/work/zq，不用 rclone sync） + +**根本解决方案**（需修改技能代码）： +1. 在 pipeline 中添加 ArchiveProcessor +2. 读取 `KB_ARCHIVE_PATH` 环境变量 +3. 按日期创建子目录并移动文件 diff --git a/agents/knowledge-base/workspace/AGENTS.md b/agents/knowledge-base/workspace/AGENTS.md index c472b63..2a16492 100644 --- a/agents/knowledge-base/workspace/AGENTS.md +++ b/agents/knowledge-base/workspace/AGENTS.md @@ -69,9 +69,14 @@ This folder is home. Treat it that way. ### 用户的工作区 - 路径：~/work/zq -- 这个工作区目录通过rclone与Nas同步 -- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"` -- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"` +- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`） + - 所有操作实时同步到NAS，无需手动sync + - 管理命令：`systemctl --user status|restart|stop rclone-zq` + - 日志：`~/.cache/rclone/zq-mount.log` +- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录** + - 该目录已通过 rclone mount 挂载，直接读写挂载点即可 + - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新 + - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步 - 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md - 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒 diff --git a/agents/knowledge-base/workspace/memory/2026-03-16-0549.md b/agents/knowledge-base/workspace/memory/2026-03-16-0549.md new file mode 100644 index 0000000..3a34deb --- /dev/null +++ b/agents/knowledge-base/workspace/memory/2026-03-16-0549.md @@ -0,0 +1,70 @@ +# Session: 2026-03-16 05:49:59 UTC + +- **Session Key**: agent:knowledge-base:main +- **Session ID**: ad89af23-d276-4fd4-bd7c-e1a06ed88000 +- **Source**: feishu + +## Conversation Summary + +user: Conversation info (untrusted metadata): +```json +{ + "message_id": "om_x100b544d24d77ca4c4da0769c820e25", + "sender_id": "ou_9351c287f42069fa30366ac641d24df7", + "sender": "张青", + "timestamp": "Mon 2026-03-16 13:35 GMT+8" +} +``` + +Sender (untrusted metadata): +```json +{ + "label": "张青 (ou_9351c287f42069fa30366ac641d24df7)", + "id": "ou_9351c287f42069fa30366ac641d24df7", + "name": "张青" +} +``` + +[message_id: om_x100b544d24d77ca4c4da0769c820e25] +张青: 现在哪里能找到南京90年代到2000年出头的城市样貌照片或者视频 +assistant: 我来帮你搜索一下南京90年代到2000年代初的城市照片和视频资料。 +assistant: 蒋先生，根据搜索结果，我为您找到了几个可以找到南京90年代到2000年代初城市样貌照片和视频的渠道： + +## 📱 视频平台 + +### 抖音账号 +- **"南京老照片"** - 这是一个专注南京历史文化的账号，发布了大量珍贵的历史影像 + - 近期爆款：《2000年的南京大厂区》 + - 《九十年代末的三牌楼大街和娄子巷》 + - 拥有10万粉丝，持续输出高质量内容 + +### B站（哔哩哔哩） +- 搜索关键词："南京 90年代"、"南京 老照片" +- 有相关纪录片和用户上传的老视频 + +## 📸 照片资源 + +### 网站文章 +1. **我苏网** - 《40年南京变化有多大？看这些"老照片"就懂了》 + - 包含1990年代的玄武湖公园等场景 + +2. **搜狐网** - 《寻找消失的南京！40张老照片》 + - 有90年代河西地区、7路公交车等珍贵影像 + +3. **游侠客旅行** - 《2025南京古今大对比，超百张珍贵老照片》 + +4. **南京市地方志办公室官网** + - 多篇关于南京高校、城市变迁的老照片文章 + +## 🎬 纪录片 + +- **《百年南京》** - 央视纪录片，包含南京历史变迁内容 + +## 💡 建议搜索方式 + +1. **抖音**：搜索"南京老照片"账号 +2. **B站**：搜索"南京 90年代 老视频" +3. **百度图片**：搜索"南京 90年代 街景" +4. **微博**：搜索相关话题 + +需要我帮您进一步查看某个具体资源的详细内容吗？ diff --git a/agents/knowledge-base/workspace/memory/2026-03-16-0856.md b/agents/knowledge-base/workspace/memory/2026-03-16-0856.md new file mode 100644 index 0000000..6b018a4 --- /dev/null +++ b/agents/knowledge-base/workspace/memory/2026-03-16-0856.md @@ -0,0 +1,226 @@ +# Session: 2026-03-16 08:56:28 UTC + +- **Session Key**: agent:knowledge-base:main +- **Session ID**: 9db16f34-09e8-4f14-810a-c210bf13a275 +- **Source**: feishu + +## Conversation Summary + +assistant: 短链接格式不被识别。让我先获取真实URL： +assistant: 正在处理中，让我检查进度： +assistant: 处理进展： +- ✅ 解析成功 +- ✅ 下载成功（已获取音频） +- ✅ 语音识别完成 +- ❌ 深度分析失败：`AI_API_KEY 未设置` + +让我检查环境变量配置： +assistant: 环境变量文件存在。问题是脚本执行时没有加载。让我手动加载后重新运行分析步骤： +assistant: ✅ 处理成功！现在执行归档步骤： +assistant: --- + +✅ **知识库添加完成！** + +**标题**：2-3月宏观经济运行状态复盘与推演 + 2026年政府工作报告解读 + +**作者**：阿周的Eureka | **时长**：07:16 | **日期**：2026-03-08 + +**处理结果**： +- ✅ 解析链接 +- ✅ 下载音频 +- ✅ 语音识别 +- ✅ AI深度分析 +- ✅ 归档到知识库 +- ✅ 同步到NAS + +**归档位置**：`~/work/zq/30.资源/AI分析/2026-03-16/bilibili_BV16cNMzpExh.md` + +**核心内容摘要**： +视频复盘了2026年2-3月宏观经济数据，解读了政府工作报告。主要观点： +1. **通缩压力缓解**：猪肉价格触底回升，PPI环比改善 +2. **M1增速回升**：源于定期存款集中到期，非内生扩张 +3. **社融创新高**：主要靠政府债券融资，企业/居民意愿仍低 +4. **房地产筑底**：成交量回暖是刚需入场，政策转向"稳预期" +5. **外部风险**：伊朗局势可能带来输入性通胀 +user: Conversation info (untrusted metadata): +```json +{ + "message_id": "om_x100b544e420910a4c14cf91d01d5da0", + "sender_id": "ou_9351c287f42069fa30366ac641d24df7", + "sender": "张青", + "timestamp": "Mon 2026-03-16 14:18 GMT+8" +} +``` + +Sender (untrusted metadata): +```json +{ + "label": "张青 (ou_9351c287f42069fa30366ac641d24df7)", + "id": "ou_9351c287f42069fa30366ac641d24df7", + "name": "张青" +} +``` + +[message_id: om_x100b544e420910a4c14cf91d01d5da0] +张青: 工作区现在同步方式有问题，只要修改之前没有进行rclone操作，就会导致本地旧版的文件同步到Nas上。 +user: Note: The previous agent run was aborted by the user. Resume carefully or ask for clarification. + +Conversation info (untrusted metadata): +```json +{ + "message_id": "om_x100b544e41da6c94c37ec270f11b68e", + "sender_id": "ou_9351c287f42069fa30366ac641d24df7", + "sender": "张青", + "timestamp": "Mon 2026-03-16 14:19 GMT+8" +} +``` + +Sender (untrusted metadata): +```json +{ + "label": "张青 (ou_9351c287f42069fa30366ac641d24df7)", + "id": "ou_9351c287f42069fa30366ac641d24df7", + "name": "张青" +} +``` + +[message_id: om_x100b544e41da6c94c37ec270f11b68e] +张青: 工作区现在同步方式有问题，只要修改之前没有进行rclone操作，就会导致本地旧版的文件同步到Nas上。看看有没有更实时的同步方案。比如SynologyNas官方的linux部署方案 +assistant: 明白问题所在。当前的双向 `rclone sync` 模式存在风险：如果本地有旧文件，会反向覆盖NAS上的新版本。 + +让我搜索 Synology NAS 官方的 Linux 同步方案： +assistant: 找到了几种方案。让我整理对比： + +--- + +## 工作区实时同步方案对比 + +### 当前问题 +`rclone sync` 是单向覆盖式同步，如果本地有旧文件，会把NAS新版本覆盖掉。 + +### 可选方案 + +| 方案 | 类型 | 实时性 | 优点 | 缺点 | +|------|------|--------|------|------| +| **Synology Drive Client** | 官方客户端 | ✅ 实时双向 | 官方支持、冲突处理、版本控制 | 需要GUI或有headless限制 | +| **rclone mount** | 文件系统挂载 | ✅ 实时读写 | 轻量、直接操作NAS文件 | 依赖网络稳定性 | +| **Syncthing** | P2P同步 | ✅ 实时双向 | 开源、无中心服务器 | 需要在NAS上安装 | + +--- + +### 方案1：Synology Drive Client（官方推荐） + +您的系统是 Ubuntu，支持安装： + +```bash +# 下载 deb 包（从 Synology 官网或 NAS 套件中心获取） +wget https://global.download.synology.com/download/Package/spk/SynologyDriveClient/3.5.0-16084/Synology%20Drive%20Client-3.5.0-16084-x86_64.deb + +# 安装 +sudo dpkg -i "Synology Drive Client-3.5.0-16084-x86_64.deb" +sudo apt -f install # 修复依赖 +``` + +**优点**： +- 真正的双向实时同步 +- 自动处理冲突 +- 支持版本历史 +- 离线编辑后自动同步 + +**注意**：需要确认NAS上已安装 **Synology Drive Server** 套件。 + +--- + +### 方案2：rclone mount（轻量替代） + +直接把NAS挂载为本地目录，所有操作实时写入： + +```bash +# 挂载 +rclone mount nas-webdav:/home/Drive/工作区/张青/工作区/ ~/work/zq/ --vfs-cache-mode full --daemon + +# 卸载 +fusermount -u ~/work/zq/ +``` + +**优点**： +- 无需额外安装 +- 操作即同步，不会有"旧版本覆盖"问题 +- 配合 `--vfs-cache-mode full` 提升性能 + +**缺点**： +- 网络断开时无法操作 +- 大文件操作可能较慢 + +--- + +### 建议 + +蒋先生，您倾向哪种方案？ + +1. **Synology Drive Client** - 功能最完整，需要先确认NAS上是否已安装 Drive Server 套件 +2. **rclone mount** - 最快上手，我可以帮您配置自动挂载 +3. **Syncthing** - 需要在NAS上额外安装 + +我可以帮您检查NAS上的配置，或者直接配置 rclone mount 方案？ +user: Conversation info (untrusted metadata): +```json +{ + "message_id": "om_x100b544e744c5080c2c7031b6254cec", + "sender_id": "ou_9351c287f42069fa30366ac641d24df7", + "sender": "张青", + "timestamp": "Mon 2026-03-16 14:31 GMT+8" +} +``` + +Sender (untrusted metadata): +```json +{ + "label": "张青 (ou_9351c287f42069fa30366ac641d24df7)", + "id": "ou_9351c287f42069fa30366ac641d24df7", + "name": "张青" +} +``` + +[message_id: om_x100b544e744c5080c2c7031b6254cec] +张青: 2。rclone mount +assistant: 好的，配置 rclone mount 方案。 +assistant: rclone 已配置好。现在创建 systemd 自动挂载服务： +assistant: ✅ **rclone mount 配置完成！** + +**挂载状态**： +- 路径：`~/work/zq/` → NAS WebDAV +- 类型：实时读写（操作即同步） +- 缓存：启用（10G 本地缓存，提升性能） +- 自动启动：✅ 开机自动挂载 + +**常用命令**： +```bash +# 查看状态 +systemctl --user status rclone-zq + +# 手动重启 +systemctl --user restart rclone-zq + +# 查看日志 +tail -f ~/.cache/rclone/zq-mount.log +``` + +现在更新 AGENTS.md，移除旧的 sync 步骤： +assistant: ✅ **配置完成！** + +**变更摘要**： + +| 项目 | 旧方式 | 新方式 | +|------|--------|--------| +| 同步模式 | 手动 `rclone sync` | 实时挂载 `rclone mount` | +| 操作前 | 需先从NAS拉取 | 直接操作，实时写入 | +| 操作后 | 需推送到NAS | 自动同步 | +| 覆盖风险 | ❌ 本地旧文件可能覆盖NAS | ✅ 直接读写NAS，无覆盖风险 | + +**服务状态**： +- 服务名：`rclone-zq.service` +- 状态：运行中（开机自启） +- 挂载点：`~/work/zq/` ↔ NAS WebDAV + +现在对 `~/work/zq/` 的任何操作都会实时同步到 NAS，不再需要手动 sync 了。 diff --git a/agents/knowledge-base/workspace/memory/2026-03-16-0928.md b/agents/knowledge-base/workspace/memory/2026-03-16-0928.md new file mode 100644 index 0000000..7502e24 --- /dev/null +++ b/agents/knowledge-base/workspace/memory/2026-03-16-0928.md @@ -0,0 +1,53 @@ +# Session: 2026-03-16 09:28:44 UTC + +- **Session Key**: agent:knowledge-base:main +- **Session ID**: fa3cb5a4-5a5b-4d65-96e9-0b17095e5445 +- **Source**: feishu + +## Conversation Summary + +assistant: ✅ New session started · model: zhipu/glm-5 +user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning. +assistant: 📚 蒋先生您好，我是达康书记。今天是周一，下午4点56分。有什么工作或生活事务需要我协助处理吗？ +user: Conversation info (untrusted metadata): +```json +{ + "message_id": "om_x100b54482e7f9480c39f23560aab683", + "sender_id": "ou_9351c287f42069fa30366ac641d24df7", + "sender": "张青", + "timestamp": "Mon 2026-03-16 16:57 GMT+8" +} +``` + +Sender (untrusted metadata): +```json +{ + "label": "张青 (ou_9351c287f42069fa30366ac641d24df7)", + "id": "ou_9351c287f42069fa30366ac641d24df7", + "name": "张青" +} +``` + +[message_id: om_x100b54482e7f9480c39f23560aab683] +张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库 +assistant: ❌ **处理失败 - 环境依赖缺失** + +蒋先生，在执行"添加到知识库"任务时遇到了意料之外的问题： + +**问题详情：** +- kb-workflow 技能缺少必要的 Python 包：`kb_zhihu_metadata` +- 该包未安装在虚拟环境中，也未在 kb-skills 仓库中找到源代码 +- 这是一个知乎内容下载所需的依赖包 + +**已尝试的解决方法：** +1. ✅ 环境变量检查通过 +2. ✅ 找到正确的 kb-workflow 执行位置 +3. ❌ 缺少 kb_zhihu_metadata 包导致下载步骤失败 + +**建议解决方案：** +需要安装或获取 kb-zhihu-metadata 包。这可能需要： +- 从 PyPI 安装（如果已发布） +- 从其他仓库获取源代码并安装 +- 联系技能维护者获取安装说明 + +根据工作规范，我无法通过修改代码或配置来解决这个问题。需要您协助处理这个环境依赖问题。 diff --git a/agents/knowledge-base/workspace/memory/2026-03-16-0933.md b/agents/knowledge-base/workspace/memory/2026-03-16-0933.md new file mode 100644 index 0000000..c5b97a4 --- /dev/null +++ b/agents/knowledge-base/workspace/memory/2026-03-16-0933.md @@ -0,0 +1,34 @@ +# Session: 2026-03-16 09:33:25 UTC + +- **Session Key**: agent:knowledge-base:main +- **Session ID**: bf3272a6-1140-4b72-815a-2c8c4e769629 +- **Source**: feishu + +## Conversation Summary + +assistant: ✅ New session started · model: zhipu/glm-5 +user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning. +assistant: 📚 **达康书记为您服务，蒋先生。** + +新会话已就绪，有什么需要我协助的？无论是工作区管理、知识库整理，还是日历提醒、邮件处理，随时吩咐。 +user: Conversation info (untrusted metadata): +```json +{ + "message_id": "om_x100b544898f834b0c2998c7ae1dd83b", + "sender_id": "ou_9351c287f42069fa30366ac641d24df7", + "sender": "张青", + "timestamp": "Mon 2026-03-16 17:29 GMT+8" +} +``` + +Sender (untrusted metadata): +```json +{ + "label": "张青 (ou_9351c287f42069fa30366ac641d24df7)", + "id": "ou_9351c287f42069fa30366ac641d24df7", + "name": "张青" +} +``` + +[message_id: om_x100b544898f834b0c2998c7ae1dd83b] +张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库 diff --git a/agents/knowledge-base/workspace/memory/2026-03-16-1021.md b/agents/knowledge-base/workspace/memory/2026-03-16-1021.md new file mode 100644 index 0000000..57ea28c --- /dev/null +++ b/agents/knowledge-base/workspace/memory/2026-03-16-1021.md @@ -0,0 +1,48 @@ +# Session: 2026-03-16 10:21:10 UTC + +- **Session Key**: agent:knowledge-base:main +- **Session ID**: e5fb9409-8568-482c-9e4d-0dc39679edf3 +- **Source**: feishu + +## Conversation Summary + +assistant: ✅ New session started · model: zhipu/glm-5 +user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning. +assistant: 📚 蒋先生您好，我是达康书记，您的私人秘书。请问今天有什么需要我协助您处理的？ +user: Conversation info (untrusted metadata): +```json +{ + "message_id": "om_x100b5448be14cd10c314363dea64b82", + "sender_id": "ou_9351c287f42069fa30366ac641d24df7", + "sender": "张青", + "timestamp": "Mon 2026-03-16 17:36 GMT+8" +} +``` + +Sender (untrusted metadata): +```json +{ + "label": "张青 (ou_9351c287f42069fa30366ac641d24df7)", + "id": "ou_9351c287f42069fa30366ac641d24df7", + "name": "张青" +} +``` + +[message_id: om_x100b5448be14cd10c314363dea64b82] +张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库 +assistant: ✅ **已添加到知识库** + +**标题**：怎么成为一个 ai agent 工程师？ +**来源**：知乎回答 +**作者**：啦啦啦啦 + +**内容摘要**： +这份内容针对希望转型成为 AI Agent 工程师的人群，提出了一套以"商业项目落地"为核心的敏捷转型方案，核心包含四个步骤： +1. 掌握大模型的基础调用与提示词控制 +2. 赋予模型使用工具的能力 +3. 解决工程化难题（上下文管理、稳定性、日志监控） +4. 将项目经验转化为高含金量的简历话术 + +**归档位置**：`~/work/zq/30.资源/AI分析/2026-03-16/zhihu_2003232153480226496.md` + +已同步至 NAS。 diff --git a/agents/knowledge-base/workspace/~/work/knowledge-base/data/zhihu/zhihu_article_VS Code 1111 炸了自动驾驶Autopilot模式上线这是要干掉 Cursor.md b/agents/knowledge-base/workspace/~/work/knowledge-base/data/zhihu/zhihu_article_VS Code 1111 炸了自动驾驶Autopilot模式上线这是要干掉 Cursor.md deleted file mode 100644 index c19ef57..0000000 --- a/agents/knowledge-base/workspace/~/work/knowledge-base/data/zhihu/zhihu_article_VS Code 1111 炸了自动驾驶Autopilot模式上线这是要干掉 Cursor.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: "VS Code 1.111 炸了：自动驾驶Autopilot模式上线，这是要干掉 Cursor？" -author: "王几行XING" -url: "https://zhuanlan.zhihu.com/p/2014737517937731380" -created: "2026-03-10 16:24:06" -voteup: 0 -comments: 0 -platform: zhihu -type: article ---- - -# VS Code 1.111 炸了：自动驾驶Autopilot模式上线，这是要干掉 Cursor？ - -**作者**: 王几行XING -**来源**: [知乎专栏](https://zhuanlan.zhihu.com/p/2014737517937731380) -**发布时间**: 2026-03-10 16:24:06 -**点赞**: 0 -**评论**: 0 - ---- - -2026年3月9日，微软悄悄干了一件大事。VS Code 发布了 1.111 版本——这不是一个普通的版本号。它是 VS Code 历史上第一个每周稳定发布版本，标志着这款全球市占率超过 70% 的代码编辑器，正式从”月更选手”变成了”周更狂魔”。更炸裂的是，这个版本带来了一个让人又兴奋又后怕的新功能：Autopilot 自动驾驶模式。AI 代理可以自动批准所有操作、自动重试错误、自动回答后续问题，一路狂飙到任务完成——中间不需要你点任何确认按钮。这到底是开发者的福音，还是潘多拉的魔盒？为什么突然改成每周更新？背后藏着 AI 军备竞赛VS Code 从 2015 年发布以来，一直保持着每月一次的稳定版发布节奏。十年如一日，雷打不动。但现在，一切变了。VS Code 团队在官方公告中表示：”通过转向每周稳定版本发布，我们继续改进工程流程，以更快的速度推送高质量功能。”听起来很官方，但真正的原因，明眼人都看得出来——AI 模型的迭代速度太快了。Anthropic 和 OpenAI 几乎每个月都在发布新模型。Claude Opus 4.6、GPT-5.3-Codex、Gemini 3 Pro……AI 模型的更新周期已经压缩到了周级别。如果 VS Code 还是一个月更新一次，等新功能上线的时候，竞争对手可能已经跑了三轮了。更重要的是，Cursor 正在疯狂蚕食 VS Code 的市场。这款基于 VS Code 内核打造的 AI 原生编辑器，仅用 16 个月就突破了 100 万用户和 36 万付费客户。Salesforce 超过 90% 的开发者已经转向 Cursor，带来了 PR 速度和代码质量的双位数提升。面对这样的竞争压力，VS Code 的选择很简单：要么跟上 AI 的节奏，要么被淘汰。每周更新意味着什么？意味着一个功能一旦准备就绪，开发者不用再苦等 2-3 周。虽然单次更新的内容量会比月更少，但累积效果一样——只是颗粒度更细了。Autopilot 自动驾驶模式：AI 全权接管你的编辑器1.111 版本最重磅的更新，毫无疑问是 Autopilot 自动驾驶模式。在此之前，VS Code 的 AI 代理每执行一个操作都需要你点”确认”。想删个文件？确认。想跑个命令？确认。想改个配置？还是确认。对于简单任务来说，这种交互简直是折磨。现在，VS Code 引入了一个全新的权限选择器，取代了旧的硬编码审批流程。它提供三个档位：默认审批（Default Approvals）：按你配置的审批规则来，该确认的确认跳过审批（Bypass Approvals）：自动批准所有工具调用，不再弹确认框自动驾驶（Autopilot）：这才是真正的”YOLO 模式”——自动批准、自动重试、自动回答后续问题，直到 task_complete 信号发出简单来说，开了 Autopilot，AI 代理就像一个拿到了管理员权限的实习生——它会不停地干活，遇到错误自己重试，被问问题自己回答，一路干到任务完成为止。这个功能目前还是预览版，需要手动开启 chat.autopilot.enabled 设置。而且微软特意设计了一个很难忽略的警告对话框——因为一旦开启，AI 代理就会绕过你配置的所有安全检查。有开发者反馈说，不小心开启 Autopilot 后，AI 代理开始自顾自地执行终端命令，完全不等人批准。这提醒我们：Autopilot 虽然爽，但用之前一定要想清楚你在干什么。我的建议是：对于你非常熟悉的、风险可控的任务（比如写单元测试、格式化代码），大胆用 Autopilot。但涉及到数据库操作、文件删除、生产环境部署这类事情，还是老老实实用默认审批吧。Agent 作用域钩子：给每个 AI 代理定制专属工作流1.111 版本的另一个重要更新是 Agent 作用域钩子（Agent-Scoped Hooks），目前也是预览版。什么意思呢？简单来说，你现在可以为不同的自定义 AI 代理设定不同的自动化规则。VS Code 的钩子系统支持 8 种生命周期事件：事件触发时机典型用途SessionStart新会话开始初始化资源、验证项目状态UserPromptSubmit用户提交提示审计请求、注入上下文PreToolUse工具调用前拦截危险操作PostToolUse工具调用后运行格式化、记录结果PreCompact上下文压缩前导出重要数据SubagentStart子代理启动追踪嵌套代理SubagentStop子代理结束聚合结果、清理Stop会话结束生成报告、发送通知以前，这些钩子是全局生效的——不管你用哪个代理，钩子都会执行。但现在，你可以在自定义代理的 YAML 前置元数据中定义专属钩子，让它只在选择该代理时才运行。举个实际的例子：你有一个专门写 Python 代码的代理，可以给它配一个 PostToolUse 钩子，每次代码生成后自动跑 black 格式化和 mypy 类型检查。而你的前端代理则配另一套钩子，跑 prettier 和 eslint。两套规则互不干扰。这看起来是个小功能，但它代表了一个重要趋势：VS Code 正在从”代码编辑器”进化为”AI 代理编排平台”。调试事件快照：给 AI 代理做”体检”你有没有遇到过这种情况：AI 代理给出了一个莫名其妙的回答，你完全不知道它是怎么推理出来的？1.111 版本新增了 调试事件快照（Debug Events Snapshot） 功能，专门解决这个痛点。在代理调试面板中，你可以点击星形图标，一键将所有调试事件的快照附加到当前聊天。系统会自动按时间戳筛选日志，让你清楚地看到：加载了哪些提示词文件调用了哪些工具消耗了多少 token哪些自定义配置生效了配合新的 /troubleshoot 斜杠命令，你甚至可以直接问 AI：”你刚才为什么会这样做？用了多少 token？加载了哪些自定义配置？”这个功能对于调试复杂的多代理工作流特别有价值。当你有多个代理协同工作时，出了问题不再是一头雾水——你可以精确定位到某个代理在某个时间点的行为，就像给 AI 做了一次全面体检。编辑模式被判”死刑”：Agent 模式全面上位在 VS Code 的 AI 辅助编码历史上，有三种模式并存：Ask 模式：聊天问答，不修改文件Edit 模式：告诉 AI 你想改什么，它提出编辑建议Agent 模式：AI 自主规划、跨文件修改、运行命令、迭代调试从 1.111 版本开始，Edit 模式正式被标记为弃用，将在 1.125 版本彻底移除。这意味着 VS Code 正式宣告：Agent 模式才是未来。为什么？因为 Agent 模式的能力远超 Edit 模式。Edit 模式只能在你指定的范围内做修改，而 Agent 模式可以：自主扫描整个代码库，找到相关文件跨多个文件进行协调修改运行终端命令并根据结果调整方案自动测试、自动调试、自动修复简单来说，Edit 模式是”你指哪它打哪”，Agent 模式是”你说目标它自己想办法”。如果你还在用 Edit 模式，可以临时设置 chat.editMode.hidden: false 找回它，但建议尽早适应 Agent 模式——因为这就是 AI 编程的未来方向。过渡期间，微软会创建一个内置的自定义 Agent 模式来模拟 Edit 模式的行为，降低迁移门槛。VS Code 的 AI 生态：从编辑器到代理编排平台如果把视野拉长，你会发现 VS Code 在 2026 年的进化速度堪称恐怖。1.109 版本（2026年1月）：引入多代理编排，Claude 作为一等公民代理与 Copilot 并行运行，技能系统正式发布。1.110 版本（2026年2月）：推出代理插件（预打包的技能+命令+MCP 服务器+钩子），浏览器自动化工具让代理能直接操作网页，会话派生和上下文压缩优化长对话体验。1.111 版本（2026年3月）：Autopilot 模式、代理钩子、调试快照、每周发布节奏——一切都在加速。与此同时，GitHub Copilot CLI 也在2月25日正式 GA（全面可用）。这个终端原生的编程代理已经从一个简单的命令行助手，进化成了一个完整的代理开发环境——支持计划模式、自动驾驶模式、Fleet 并行模式，支持 Claude Opus 4.6、GPT-5.3-Codex、Gemini 3 Pro 等多种模型。一个有意思的数据：根据 JetBrains 2026年1月的调查，93% 的开发者已经在日常工作中使用 AI 工具。GitHub 报告称，使用 Copilot 的开发者任务完成速度平均提升 55%。我们正在见证一个转折点：VS Code 不再只是一个代码编辑器，它正在成为一个AI 代理的控制中心。VS Code vs Cursor：传统编辑器的反击说到 AI 代码编辑器，就不得不提 Cursor。两者的核心区别用一句话概括：在 VS Code 中，AI 是乘客；在 Cursor 中，AI 是司机。维度VS Code + CopilotCursor定价10美元/月（Pro）20美元/月（Pro）AI 集成方式插件/扩展原生内置多文件重构逐步增强中业界标杆扩展生态最大最全兼容 VS Code企业安全完善的审计日志相对薄弱模型选择多模型支持多模型支持Cursor 的杀手锏是 Composer 多文件编辑和整个代码库的本地索引，在复杂重构场景下效率提升约 40%。但 VS Code 的优势在于生态和价格。每月 10美元 vs 20美元，一年就差 120美元。而且 VS Code 的扩展生态无人能敌，你可以自由组合 Copilot、Claude Code、Continue 等多种 AI 工具。随着 1.111 版本的发布，VS Code 正在快速缩小与 Cursor 的差距。多代理编排、自动驾驶模式、代理插件系统——这些曾经是 Cursor 独有的优势，VS Code 都在逐步追上。我的判断是：对于大多数开发者来说，VS Code + Copilot Pro 仍然是性价比最高的选择。但如果你的工作大量涉及多文件重构和复杂架构变更，Cursor 仍然值得那额外的 10美元。写在最后VS Code 1.111 版本虽然看起来只是一个”小更新”，但它传递出的信号非常清晰：速度就是一切——从月更到周更，微软在用行动表明 AI 时代的竞争节奏自主性是方向——从手动确认到 Autopilot，AI 代理正在获得越来越多的自主权平台化是终局——从编辑器到代理编排平台，VS Code 的野心远不止写代码2026年的 AI 编程工具市场，已经不再是”谁的代码补全更准”的竞争了。它变成了一个更深层次的问题：谁能更好地让 AI 代理为开发者工作？在这场竞赛中，VS Code 凭借 70% 的市占率和微软的资源，依然是最有力的竞争者。但 Cursor、Windsurf、Claude Code 等挑战者也不可小觑——它们正在用更激进的 AI 原生设计，重新定义”代码编辑器”这四个字。你现在用的是哪款 AI 编程工具？升级到 1.111 后体验如何？欢迎在评论区聊聊。参考来源中文来源VS Code 官方更新页面 v1.111 - Visual Studio Code 官方vscode发布最新版本，全面拥抱AI Agent，功能概览 - 知乎Cursor压力山大：VSCode 3月更新，原地化身免费AI代码编辑器 - CSDNVisual Studio Code 发布王炸更新：Agent 模式上线 - 掘金微软要将 VS Code 打造成开源 AI 编辑器 - OSCHINA2026年2月 (版本 1.110) - VSCode 中文网Visual Studio 2026 现已正式发布，更快、更智能！ - 博客园Visual Studio 2026 和 VS2022：功能全面对比 - Home1024Visual Studio 2026 正式发布：全球首款深度智能IDE - 腾讯云编程神器诞生！在 VSCode 中使用 GitHub Copilot 进行 AI 辅助编程 - 知乎杀疯了！GitHub 发布 Copilot CLI！！！ - 知乎「从夯到拉」2026年AI编程工具全景测评 - 知乎2026 年 12 款最佳 AI 编程工具：实测排名 - SimilarLabsKilo Code：开源 VS Code AI 代理 - OSCHINAGitHub Copilot 编码代理 - VSCode 中文网在 Visual Studio Code 中使用代理 - VSCode 中文网英文来源Visual Studio Code 1.111 Release: Autopilot, Agent Hooks, and Debug Snapshots Explained - NTCompatibleVisual Studio Code 1.111 Release - Warp2SearchVisual Studio Code is switching to faster releases, starting with v1.111 - How-To GeekVS Code March 2026 Latest Updates - ReleasebotVS Code 1.110 Ships with Agent Plugins, Browser Tools and Session Memory - Visual Studio MagazineGitHub Copilot CLI is now generally available - GitHub BlogGitHub Copilot CLI Reaches General Availability - Visual Studio MagazineGitHub Copilot CLI Goes Generally Available - Autopilot, Multi-Model, and Fleet Mode - Awesome AgentsCopilot CLI Weekly: General Availability Has Landed - DEV CommunityVS Code’s January 2026 Update Turns Your Editor Into an AI Agent Platform - DEV CommunityVisual Studio Code previews agent plugins - InfoWorldVS Code, Cursor, Windsurf, JetBrains or Web IDEs: Which Wins in 2026 - Amplifi LabsVS Code vs Cursor: Traditional vs AI Code Editor in 2026 - DEV CommunityCursor vs GitHub Copilot vs Continue: AI Code Editor Showdown 2026 - DEV CommunityTop 5 AI Coding Assistants of 2026 Compared - Deepak GuptaGitHub Copilot Agent mode vs. Ask and Edit mode in VS Code - 4sysopsAgent hooks in Visual Studio Code (Preview) - VS Code DocsUse agent mode in VS Code - VS Code DocsClaude and Codex now available for Copilot Business & Pro users - GitHub BlogGitHub Copilot vs Claude Code: 2026 Accuracy & Speed Analysis - SitePoint(文章结束) diff --git a/agents/thinker/workspace/AGENTS.md b/agents/thinker/workspace/AGENTS.md index c472b63..d045f0f 100644 --- a/agents/thinker/workspace/AGENTS.md +++ b/agents/thinker/workspace/AGENTS.md @@ -69,9 +69,14 @@ This folder is home. Treat it that way. ### 用户的工作区 - 路径：~/work/zq -- 这个工作区目录通过rclone与Nas同步 -- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"` -- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"` +- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`） +  - 所有操作实时同步到NAS，无需手动sync +  - 管理命令：`systemctl --user status|restart|stop rclone-zq` +  - 日志：`~/.cache/rclone/zq-mount.log` +- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录** +  - 该目录已通过 rclone mount 挂载，直接读写挂载点即可 +  - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新 +  - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步 - 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md - 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒 diff --git a/agents/ui-designer/sessions/sessions.json b/agents/ui-designer/sessions/sessions.json index a63ba6f..dd098c9 100644 --- a/agents/ui-designer/sessions/sessions.json +++ b/agents/ui-designer/sessions/sessions.json @@ -1,7 +1,7 @@ { "agent:ui-designer:main": { - "sessionId": "fe685776-cc8c-489c-b409-e461e3d3cbfa", - "updatedAt": 1773564045652, + "sessionId": "2790af05-ac44-4b18-82fa-23471948d4d1", + "updatedAt": 1773677918741, "systemSent": true, "abortedLastRun": false, "chatType": "direct", @@ -22,7 +22,7 @@ "to": "user:ou_9847ce84b960736c14554730b41e4957", "accountId": "feishu-art" }, - "sessionFile": "/home/starmen1265/.openclaw/agents/ui-designer/sessions/fe685776-cc8c-489c-b409-e461e3d3cbfa.jsonl", + "sessionFile": "/home/starmen1265/.openclaw/agents/ui-designer/sessions/2790af05-ac44-4b18-82fa-23471948d4d1.jsonl", "compactionCount": 0, "skillsSnapshot": { "prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n\n \n capability-evolver\n A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.\n ~/.openclaw/skills/capability-evolver/SKILL.md\n \n", @@ -51,8 +51,8 @@ "model": "glm-5", "systemPromptReport": { "source": "run", - "generatedAt": 1773564010472, - "sessionId": "fe685776-cc8c-489c-b409-e461e3d3cbfa", + "generatedAt": 1773677890931, + "sessionId": "2790af05-ac44-4b18-82fa-23471948d4d1", "sessionKey": "agent:ui-designer:main", "provider": "zhipu", "model": "glm-5", @@ -64,8 +64,8 @@ "sandboxed": false }, "systemPrompt": { - "chars": 22552, - "projectContextChars": 10067, + "chars": 23850, + "projectContextChars": 11365, "nonProjectContextChars": 12485 }, "injectedWorkspaceFiles": [ @@ -169,6 +169,18 @@ "injectedChars": 616, "truncated": false }, + { + "path": "SELF_IMPROVEMENT_REMINDER.md", + "rawChars": 616, + "injectedChars": 616, + "truncated": false + }, + { + "path": "SELF_IMPROVEMENT_REMINDER.md", + "rawChars": 616, + "injectedChars": 616, + "truncated": false + }, { "path": "SELF_IMPROVEMENT_REMINDER.md", "rawChars": 616, @@ -396,13 +408,13 @@ ] } }, - "cacheRead": 15232, + "cacheRead": 15552, "cacheWrite": 0, "totalTokensFresh": true, "contextTokens": 200000, - "inputTokens": 13122, - "outputTokens": 1320, - "totalTokens": 28354 + "inputTokens": 2711, + "outputTokens": 882, + "totalTokens": 18263 }, "agent:ui-designer:cron:78b64c2f-7f91-47ff-9250-d13f546eb21f": { "sessionId": "03e44b46-c707-4a9f-bf3d-1818aafdcde2", diff --git a/agents/ui-designer/workspace/AGENTS.md b/agents/ui-designer/workspace/AGENTS.md index cb09a86..23721a1 100644 --- a/agents/ui-designer/workspace/AGENTS.md +++ b/agents/ui-designer/workspace/AGENTS.md @@ -69,9 +69,14 @@ This folder is home. Treat it that way. ### 用户的工作区 - 路径：~/work/zq -- 这个工作区目录通过rclone与Nas同步 -- **任何对~/work/zq目录的操作（新建/修改/删除）之前必须先从NAS同步到本地：** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"` -- **操作完成后必须同步到NAS：** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"` +- **同步方式：rclone mount 实时挂载**（systemd 服务：`rclone-zq.service`） +  - 所有操作实时同步到NAS，无需手动sync +  - 管理命令：`systemctl --user status|restart|stop rclone-zq` +  - 日志：`~/.cache/rclone/zq-mount.log` +- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录** +  - 该目录已通过 rclone mount 挂载，直接读写挂载点即可 +  - 使用 rclone sync 会绕过挂载点缓存，导致数据不一致或覆盖用户更新 +  - 正确做法：直接操作 `~/work/zq/` 下的文件，挂载点会自动同步 - 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md - 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒 diff --git a/agents/ui-designer/workspace/memory/2026-03-17.md b/agents/ui-designer/workspace/memory/2026-03-17.md new file mode 100644 index 0000000..ade2c1f --- /dev/null +++ b/agents/ui-designer/workspace/memory/2026-03-17.md @@ -0,0 +1,36 @@ +# 2026-03-17 财务咨询记录 + +## 用户信息 +- 用户：张青 (ou_9847ce84b960736c14554730b41e4957) + +--- + +## 资产配置优化咨询 + +### 用户背景 +- 总资产：190万 +- 当前配置：大部分固定收益理财，年化 2% +- 目标：整体年化收益提升到 8% + +### 给出的方案 + +| 方案 | 股票占比 | 股票金额 | 股票目标收益率 | 风险等级 | +|------|---------|---------|--------------|---------| +| 保守型 | 30% | 57万 | 22% | 中高 | +| 平衡型 ⭐ | 50% | 95万 | 14% | 中 | +| 进取型 | 70% | 133万 | 10.6% | 中低（但敞口大）| + +### 推荐方案 +**平衡型（50% 股票配置）** +- 固定收益：95万（2%） +- 股票投资：95万（目标 14%） +- 理由：风险可控、目标现实、心理压力适中 + +### 股票投资建议 +1. 指数基金打底（60%）：沪深300、中证500 +2. 优质个股精选（30%）：高股息蓝筹、行业龙头 +3. 灵活仓位（10%）：机会加仓 + +--- + +*记录时间：2026-03-17 00:13* diff --git a/cron/jobs.json b/cron/jobs.json index 559077b..ed8a169 100644 --- a/cron/jobs.json +++ b/cron/jobs.json @@ -7,7 +7,7 @@ "name": "backup-openclaw-json", "enabled": true, "createdAtMs": 1772261581796, - "updatedAtMs": 1773597621815, + "updatedAtMs": 1773684008663, "schedule": { "kind": "cron", "expr": "0 2 * * *", @@ -23,10 +23,10 @@ "mode": "announce" }, "state": { - "nextRunAtMs": 1773684000000, - "lastRunAtMs": 1773597600004, + "nextRunAtMs": 1773770400000, + "lastRunAtMs": 1773684000004, "lastStatus": "ok", - "lastDurationMs": 21811, + "lastDurationMs": 8659, "consecutiveErrors": 0, "lastRunStatus": "ok", "lastDelivered": true, @@ -59,7 +59,8 @@ "consecutiveErrors": 0, "lastRunStatus": "ok", "lastDeliveryStatus": "delivered", - "lastDelivered": true + "lastDelivered": true, + "runningAtMs": 1773684600003 }, "delivery": { "mode": "announce" @@ -71,7 +72,7 @@ "name": "capability-evolver-daily", "enabled": true, "createdAtMs": 1772550743528, - "updatedAtMs": 1773513000030, + "updatedAtMs": 1773599129019, "schedule": { "kind": "cron", "expr": "20 2 * * *", @@ -92,14 +93,16 @@ "to": "feishu-pm" }, "state": { - "nextRunAtMs": 1773598800000, - "lastRunAtMs": 1773512400004, + "nextRunAtMs": 1773685200000, + "lastRunAtMs": 1773598800006, "lastStatus": "error", - "lastDurationMs": 600026, - "consecutiveErrors": 2, + "lastDurationMs": 329013, + "consecutiveErrors": 3, "lastRunStatus": "error", - "lastDeliveryStatus": "unknown", - "runningAtMs": 1773598800004 + "lastDeliveryStatus": "not-delivered", + "lastError": "⚠️ ✉️ Message failed", + "lastDelivered": false, + "lastDeliveryError": "⚠️ ✉️ Message failed" } }, { @@ -109,7 +112,7 @@ "description": "每天凌晨2:30整理昨天的已完成任务到每日回顾", "enabled": true, "createdAtMs": 1741484880000, - "updatedAtMs": 1773513010858, + "updatedAtMs": 1773599416835, "schedule": { "kind": "cron", "expr": "30 2 * * *", @@ -127,11 +130,11 @@ "channel": "last" }, "state": { - "nextRunAtMs": 1773599400000, - "lastRunAtMs": 1773513002054, + "nextRunAtMs": 1773685800000, + "lastRunAtMs": 1773599400005, "lastRunStatus": "ok", "lastStatus": "ok", - "lastDurationMs": 8804, + "lastDurationMs": 16830, "lastDelivered": true, "lastDeliveryStatus": "delivered", "consecutiveErrors": 0 @@ -143,7 +146,7 @@ "name": "zq_workspace_sync", "enabled": true, "createdAtMs": 1772551807455, - "updatedAtMs": 1773513600005, + "updatedAtMs": 1773600000005, "schedule": { "expr": "40 2 * * *", "kind": "cron", @@ -156,10 +159,10 @@ "text": "执行 zq 工作区同步：bash ~/.openclaw/cron/scripts/zq_workspace_sync.sh" }, "state": { - "nextRunAtMs": 1773600000000, - "lastRunAtMs": 1773513600005, + "nextRunAtMs": 1773686400000, + "lastRunAtMs": 1773600000004, "lastStatus": "skipped", - "lastDurationMs": 0, + "lastDurationMs": 1, "consecutiveErrors": 0, "lastRunStatus": "skipped", "lastDeliveryStatus": "not-requested", @@ -173,7 +176,7 @@ "description": "每天早上8点整理与各个 AI 代理的对话记录", "enabled": true, "createdAtMs": 1772776369645, - "updatedAtMs": 1773532816520, + "updatedAtMs": 1773619224324, "schedule": { "kind": "cron", "expr": "0 8 * * *", @@ -191,11 +194,11 @@ "channel": "last" }, "state": { - "nextRunAtMs": 1773619200000, - "lastRunAtMs": 1773532800006, + "nextRunAtMs": 1773705600000, + "lastRunAtMs": 1773619200004, "lastRunStatus": "ok", "lastStatus": "ok", - "lastDurationMs": 16514, + "lastDurationMs": 24320, "lastDelivered": true, "lastDeliveryStatus": "delivered", "consecutiveErrors": 0 @@ -208,7 +211,7 @@ "description": "每天7点同步提醒到CalDAV日历", "enabled": true, "createdAtMs": 1772791528528, - "updatedAtMs": 1773529221353, + "updatedAtMs": 1773615616422, "schedule": { "kind": "cron", "expr": "0 7 * * *", @@ -226,11 +229,11 @@ "channel": "last" }, "state": { - "nextRunAtMs": 1773615600000, - "lastRunAtMs": 1773529200004, + "nextRunAtMs": 1773702000000, + "lastRunAtMs": 1773615600034, "lastRunStatus": "ok", "lastStatus": "ok", - "lastDurationMs": 21349, + "lastDurationMs": 16388, "lastDeliveryStatus": "delivered", "consecutiveErrors": 0, "lastDelivered": true diff --git a/cron/jobs.json.bak b/cron/jobs.json.bak index eab9073..84319d3 100644 --- a/cron/jobs.json.bak +++ b/cron/jobs.json.bak @@ -7,7 +7,7 @@ "name": "backup-openclaw-json", "enabled": true, "createdAtMs": 1772261581796, - "updatedAtMs": 1773597621815, + "updatedAtMs": 1773684008663, "schedule": { "kind": "cron", "expr": "0 2 * * *", @@ -23,10 +23,10 @@ "mode": "announce" }, "state": { - "nextRunAtMs": 1773684000000, - "lastRunAtMs": 1773597600004, + "nextRunAtMs": 1773770400000, + "lastRunAtMs": 1773684000004, "lastStatus": "ok", - "lastDurationMs": 21811, + "lastDurationMs": 8659, "consecutiveErrors": 0, "lastRunStatus": "ok", "lastDelivered": true, @@ -71,7 +71,7 @@ "name": "capability-evolver-daily", "enabled": true, "createdAtMs": 1772550743528, - "updatedAtMs": 1773513000030, + "updatedAtMs": 1773599129019, "schedule": { "kind": "cron", "expr": "20 2 * * *", @@ -92,14 +92,16 @@ "to": "feishu-pm" }, "state": { - "nextRunAtMs": 1773598800000, - "lastRunAtMs": 1773512400004, + "nextRunAtMs": 1773685200000, + "lastRunAtMs": 1773598800006, "lastStatus": "error", - "lastDurationMs": 600026, - "consecutiveErrors": 2, + "lastDurationMs": 329013, + "consecutiveErrors": 3, "lastRunStatus": "error", - "lastDeliveryStatus": "unknown", - "lastError": "cron: job execution timed out" + "lastDeliveryStatus": "not-delivered", + "lastError": "⚠️ ✉️ Message failed", + "lastDelivered": false, + "lastDeliveryError": "⚠️ ✉️ Message failed" } }, { @@ -109,7 +111,7 @@ "description": "每天凌晨2:30整理昨天的已完成任务到每日回顾", "enabled": true, "createdAtMs": 1741484880000, - "updatedAtMs": 1773513010858, + "updatedAtMs": 1773599416835, "schedule": { "kind": "cron", "expr": "30 2 * * *", @@ -127,11 +129,11 @@ "channel": "last" }, "state": { - "nextRunAtMs": 1773599400000, - "lastRunAtMs": 1773513002054, + "nextRunAtMs": 1773685800000, + "lastRunAtMs": 1773599400005, "lastRunStatus": "ok", "lastStatus": "ok", - "lastDurationMs": 8804, + "lastDurationMs": 16830, "lastDelivered": true, "lastDeliveryStatus": "delivered", "consecutiveErrors": 0 @@ -143,7 +145,7 @@ "name": "zq_workspace_sync", "enabled": true, "createdAtMs": 1772551807455, - "updatedAtMs": 1773513600005, + "updatedAtMs": 1773600000005, "schedule": { "expr": "40 2 * * *", "kind": "cron", @@ -156,10 +158,10 @@ "text": "执行 zq 工作区同步：bash ~/.openclaw/cron/scripts/zq_workspace_sync.sh" }, "state": { - "nextRunAtMs": 1773600000000, - "lastRunAtMs": 1773513600005, + "nextRunAtMs": 1773686400000, + "lastRunAtMs": 1773600000004, "lastStatus": "skipped", - "lastDurationMs": 0, + "lastDurationMs": 1, "consecutiveErrors": 0, "lastRunStatus": "skipped", "lastDeliveryStatus": "not-requested", @@ -173,7 +175,7 @@ "description": "每天早上8点整理与各个 AI 代理的对话记录", "enabled": true, "createdAtMs": 1772776369645, - "updatedAtMs": 1773532816520, + "updatedAtMs": 1773619224324, "schedule": { "kind": "cron", "expr": "0 8 * * *", @@ -191,11 +193,11 @@ "channel": "last" }, "state": { - "nextRunAtMs": 1773619200000, - "lastRunAtMs": 1773532800006, + "nextRunAtMs": 1773705600000, + "lastRunAtMs": 1773619200004, "lastRunStatus": "ok", "lastStatus": "ok", - "lastDurationMs": 16514, + "lastDurationMs": 24320, "lastDelivered": true, "lastDeliveryStatus": "delivered", "consecutiveErrors": 0 @@ -208,7 +210,7 @@ "description": "每天7点同步提醒到CalDAV日历", "enabled": true, "createdAtMs": 1772791528528, - "updatedAtMs": 1773529221353, + "updatedAtMs": 1773615616422, "schedule": { "kind": "cron", "expr": "0 7 * * *", @@ -226,11 +228,11 @@ "channel": "last" }, "state": { - "nextRunAtMs": 1773615600000, - "lastRunAtMs": 1773529200004, + "nextRunAtMs": 1773702000000, + "lastRunAtMs": 1773615600034, "lastRunStatus": "ok", "lastStatus": "ok", - "lastDurationMs": 21349, + "lastDurationMs": 16388, "lastDeliveryStatus": "delivered", "consecutiveErrors": 0, "lastDelivered": true diff --git a/delivery-queue/4b56a737-d726-4c48-8b10-493f0e9df335.json b/delivery-queue/4b56a737-d726-4c48-8b10-493f0e9df335.json new file mode 100644 index 0000000..19905cd --- /dev/null +++ b/delivery-queue/4b56a737-d726-4c48-8b10-493f0e9df335.json @@ -0,0 +1,21 @@ +{ + "id": "4b56a737-d726-4c48-8b10-493f0e9df335", + "enqueuedAt": 1773599128798, + "channel": "feishu", + "to": "feishu-pm", + "accountId": "feishu-pm", + "payloads": [ + { + "text": "## 🧬 Capability Evolver 每日报告\n\n**时间**: 2026年 03月 16日 星期一 02:20:22 CST\n**周期**: #0011\n\n---\n\n### 📊 变更概览\n- **意图**: REPAIR (修复)\n- **状态**: SUCCESS\n- **变更范围**: 4 文件修改，2 文件新增\n\n---\n\n### 📝 Git 提交记录\n\n1. **Auto backup before evolver run**\n - 4 files changed, 70 insertions(+), 64 deletions(-)\n\n2. **Evolver auto evolution** \n - 6 files changed, 550 insertions(+), 13 deletions(-)\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 📂 变更文件\n\n**修改文件**:\n- `memory/evolution/evolution_solidify_state.json`\n- `memory/evolution/evolution_state.json`\n- `memory/evolution/memory_graph_state.json`\n- `memory/evolution/personality_state.json`\n\n**新增文件**:\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 🎯 变更原因\n命中信号:\n- memory_missing\n- session_logs_missing\n\n系统检测到记忆缺失和会话日志缺失，触发修复模式，更新了进化状态和记忆图谱。" + } + ], + "threadId": null, + "mirror": { + "sessionKey": "agent:haonan:main", + "agentId": "haonan", + "text": "## 🧬 Capability Evolver 每日报告\n\n**时间**: 2026年 03月 16日 星期一 02:20:22 CST\n**周期**: #0011\n\n---\n\n### 📊 变更概览\n- **意图**: REPAIR (修复)\n- **状态**: SUCCESS\n- **变更范围**: 4 文件修改，2 文件新增\n\n---\n\n### 📝 Git 提交记录\n\n1. **Auto backup before evolver run**\n - 4 files changed, 70 insertions(+), 64 deletions(-)\n\n2. **Evolver auto evolution** \n - 6 files changed, 550 insertions(+), 13 deletions(-)\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 📂 变更文件\n\n**修改文件**:\n- `memory/evolution/evolution_solidify_state.json`\n- `memory/evolution/evolution_state.json`\n- `memory/evolution/memory_graph_state.json`\n- `memory/evolution/personality_state.json`\n\n**新增文件**:\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 🎯 变更原因\n命中信号:\n- memory_missing\n- session_logs_missing\n\n系统检测到记忆缺失和会话日志缺失，触发修复模式，更新了进化状态和记忆图谱。" + }, + "retryCount": 1, + "lastAttemptAt": 1773599129011, + "lastError": "Request failed with status code 400" +} \ No newline at end of file diff --git a/feishu/dedup/feishu-art.json b/feishu/dedup/feishu-art.json index a520fc5..fc9eef3 100644 --- a/feishu/dedup/feishu-art.json +++ b/feishu/dedup/feishu-art.json @@ -1,9 +1,4 @@ { - "om_x100b546dc09400acb4a5d0af66ca3c9": 1773508838286, - "om_x100b546dde1c34a4b49a888912755ac": 1773508877713, - "om_x100b545236d06ca0b100846cfa37e9d": 1773561730503, - "om_x100b5452d99e18a4b32ab9bdc02463e": 1773562229918, - "om_x100b5452972138a8b213d14996ffdbd": 1773563295479, - "om_x100b5452a60eaca0b10319a26ed640f": 1773563532727, - "om_x100b5453485b4ca0b37c3413c41bba3": 1773564010102 + "om_x100b54b68a9658a8c3597587eaea165": 1773677638200, + "om_x100b54b69ae988acc4e308675a1599b": 1773677890863 } diff --git a/feishu/dedup/feishu-pm.json b/feishu/dedup/feishu-pm.json index 2449029..daab491 100644 --- a/feishu/dedup/feishu-pm.json +++ b/feishu/dedup/feishu-pm.json @@ -1,6 +1,3 @@ { - "om_x100b546d172f3c68c2cea708421b99a": 1773507998614, - "om_x100b546d2b32cca0c329a33c339cfb5": 1773508192506, - "om_x100b546d32de24a8b316c197e60d6e4": 1773508545727, - "om_x100b546dc9d61ca0b33d802bc7615e0": 1773508722267 + "om_x100b5440a54d84acb3898a37411f37b": 1773620920592 } diff --git a/feishu/dedup/feishu-worker.json b/feishu/dedup/feishu-worker.json index b4ac2e5..6367afa 100644 --- a/feishu/dedup/feishu-worker.json +++ b/feishu/dedup/feishu-worker.json @@ -1,32 +1,34 @@ { - "om_x100b546ddd8b14a0b3ddda14ce768c5": 1773508916933, - "om_x100b546dd78c8ca0b31b0314f910115": 1773509012676, - "om_x100b546def8ac8a4b38d921a2bfc8b8": 1773509140955, - "om_x100b546dedb654a4b3ce916a129facf": 1773509176311, - "om_x100b546de975dca0b27e2f24aad0d8c": 1773509244102, - "om_x100b546de53a94a0b38693fbd7e385c": 1773509311993, - "om_x100b546dffaa3cb0b35aef2747ed228": 1773509398933, - "om_x100b546df49c10a0b106d597fd348a3": 1773509541678, - "om_x100b546d16e6e4a0c42bef93bca2224": 1773511894148, - "om_x100b546d144668a0c3583fa56a2597d": 1773511924836, - "om_x100b546d112090acc3d096e37b38dc9": 1773512010988, - "om_x100b545c390898a4c379f319d607993": 1773569917108, - "om_x100b545c365b94a0c44e709df0f5422": 1773569929804, - "om_x100b545cce0a04a0c4a26c0203c2b13": 1773570060994, - "om_x100b545ccf6e6ca0c4571d12c405d8f": 1773570074717, - "om_x100b545cccde84acc4d8fa9b0f0c852": 1773570081700, - "om_x100b545ccab08ca0c12b5e6c715a1aa": 1773570120321, - "om_x100b545cca14e8a4c4dcc592a9c425d": 1773570126138, - "om_x100b545c9fd988a4b4bcc03ec7b3368": 1773571345806, - "om_x100b5458d25a5ca0c2cf04550bf07cc": 1773586890090, - "om_x100b5458d055dca4c4dd20469577549": 1773586922136, - "om_x100b54588d4de4a0c14df6ea4b56be0": 1773587512581, - "om_x100b54599445b4a0c4def897c8a2700": 1773591976996, - "om_x100b545992d568a0c3511684eae15a5": 1773592002087, - "om_x100b5459a51f60a4c11dbfc6068d690": 1773592253777, - "om_x100b5459be53f0a0c42f8fe338d2510": 1773592330439, - "om_x100b5459be2d7ca4c32ee845e40c605": 1773592334472, - "om_x100b5459bd4168a0c36ca5cf1588b13": 1773592377222, - "om_x100b5459bae828acc347e53e7e5de8d": 1773592386779, - "om_x100b5459bb6404a0c4ade83756a546a": 1773592411047 + "om_x100b544d24d77ca4c4da0769c820e25": 1773639330290, + "om_x100b544deea46100c29a100e8840656": 1773640199255, + "om_x100b544dfc78d090c1266487d499834": 1773640492047, + "om_x100b544df47a2c80c36bffb914f76a1": 1773640620036, + "om_x100b544e420910a4c14cf91d01d5da0": 1773641933039, + "om_x100b544e43f1d4acc2bcd0631ad4df1": 1773641940383, + "om_x100b544e41da6c94c37ec270f11b68e": 1773641970288, + "om_x100b544e744c5080c2c7031b6254cec": 1773642664923, + "om_x100b54481574e8bcc311e8d58fb4c75": 1773651388362, + "om_x100b54482e7f9480c39f23560aab683": 1773651467804, + "om_x100b54489a79ecbcc142ca87d3f064b": 1773653324009, + "om_x100b544898f834b0c2998c7ae1dd83b": 1773653348152, + "om_x100b5448aba858acc4c8be1f8439827": 1773653591178, + "om_x100b5448ab152c80c2a6860a990a363": 1773653598227, + "om_x100b5448a8d4f0a0c32f5cc81fda1b3": 1773653602205, + "om_x100b5448a887a4a0c3d2cacc2350d0f": 1773653605264, + "om_x100b5448be14cd10c314363dea64b82": 1773653774367, + "om_x100b5449d79ebca8c3e54daed500100": 1773656469908, + "om_x100b5449d129e0acc3831f0b28c0dca": 1773656574965, + "om_x100b5449e4e594a0c4504c3f849787d": 1773656739113, + "om_x100b54499819dc84c31ee62a28e622a": 1773657453822, + "om_x100b544994dbc8a0c37f7406c922a18": 1773657506038, + "om_x100b5449953c8ca0c10d2892ab60482": 1773657535728, + "om_x100b5449a63174b4c2c98d2bc1f7fcc": 1773657744382, + "om_x100b544a62d8fc80c4e14309469ce27": 1773658818051, + "om_x100b544a7b1c14a0c49f7000df992cc": 1773658973700, + "om_x100b544a775f1080c3d624954a8460e": 1773659033826, + "om_x100b54b47a79e8a8b37ac4b2ddad147": 1773667147854, + "om_x100b54b760e4b0a0c445e4e92b151e5": 1773679331046, + "om_x100b54b709932ca8b32f9afaf823ea9": 1773679734328, + "om_x100b54b7042de0acb108eaf062898fd": 1773679790410, + "om_x100b54b7174820b0b2678b67a63e045": 1773680024819 } diff --git a/identity/device-auth.json b/identity/device-auth.json index 04e9908..35a7bfd 100644 --- a/identity/device-auth.json +++ b/identity/device-auth.json @@ -10,7 +10,7 @@ "operator.approvals", "operator.pairing" ], - "updatedAtMs": 1773598217705 + "updatedAtMs": 1773684008206 } } } diff --git a/logs/commands.log b/logs/commands.log index 7395a47..1d4ef80 100644 --- a/logs/commands.log +++ b/logs/commands.log @@ -45,3 +45,8 @@ {"timestamp":"2026-03-15T16:26:17.361Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"} {"timestamp":"2026-03-15T16:32:14.537Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"} {"timestamp":"2026-03-15T16:33:06.821Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"} +{"timestamp":"2026-03-16T05:49:59.593Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"} +{"timestamp":"2026-03-16T08:56:28.934Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"} +{"timestamp":"2026-03-16T09:28:44.393Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"} +{"timestamp":"2026-03-16T09:33:25.309Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"} +{"timestamp":"2026-03-16T10:21:10.561Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"} diff --git a/skills/kb-bilibili-content/SKILL.md b/skills/kb-bilibili-content/SKILL.md deleted file mode 100644 index 4701426..0000000 --- a/skills/kb-bilibili-content/SKILL.md +++ /dev/null @@ -1,91 +0,0 @@ -# KB Bilibili Content Skill - -**kb-bilibili-content** - Download Bilibili videos and extract content. - -## Features - -- Video download using **yt-dlp** (recommended) -- Subtitle extraction -- Article content extraction -- Multiple quality options (360P/480P/720P/1080P) -- Supports `BILIBILI_COOKIE` environment variable - -## Environment Variables - -| Variable | Required | Description | -|----------|----------|-------------| -| `BILIBILI_COOKIE` | No | B站登录 Cookie (for member-only videos) | - -## Commands - -```bash -# Download video (720P default) -python bilibili_content.py --id "BV1xx411c7mD" --type video --output "/path/to/downloads/" - -# Download video (1080P) -python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 80 --output "/path/to/downloads/" - -# Download video (480P) -python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 32 --output "/path/to/downloads/" - -# Extract subtitle -python bilibili_content.py --id "BV1xx411c7mD" --type subtitle --output "/path/to/subtitles/" - -# Get article content -python bilibili_content.py --id "cv43780186" --type article --output "/path/to/article.md" -``` - -## Output - -```json -{ - "success": true, - "type": "video", - "filename": "video_title.mp4", - "filepath": "/path/to/downloads/video_title.mp4", - "message": "下载完成: video_title.mp4" -} -``` - -## Cookie Setup - -### Get Cookie from Browser -1. Login to bilibili.com -2. Press F12 → Network → Refresh page -3. Copy Cookie from request headers - -### Set Environment Variable -```bash -# Add to ~/.bashrc for persistence -export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx; DedeUserID=xxx; DedeUserID__ckMd5=xxx" - -# Or set temporarily -export BILIBILI_COOKIE="your_cookie_here" -``` - -## Installation - -```bash -cp -r kb-bilibili-content/ ~/skills/ -pip install aiohttp playwright -playwright install chromium -``` - -## How It Works - -Video download uses **yt-dlp** with Netscape-format cookie file: -1. Parses `BILIBILI_COOKIE` environment variable -2. Creates temporary Netscape-format cookie file -3. Runs `yt-dlp --cookies -f -o ` -4. Cleans up temporary file - -## Quality Options - -| Quality | ID | Resolution | Notes | -|---------|-----|------------|-------| -| 16 | 360P | 640x360 | Fastest | -| 32 | 480P | 852x480 | Default for low bandwidth | -| 64 | 720P | 1280x720 | **Recommended** | -| 80 | 1080P | 1920x1080 | Best quality | - -Default quality: **64 (720P)** diff --git a/skills/kb-bilibili-content/__pycache__/bilibili_content.cpython-312.pyc b/skills/kb-bilibili-content/__pycache__/bilibili_content.cpython-312.pyc deleted file mode 100644 index d8b5ed3..0000000 Binary files a/skills/kb-bilibili-content/__pycache__/bilibili_content.cpython-312.pyc and /dev/null differ diff --git a/skills/kb-bilibili-content/bilibili_content.py b/skills/kb-bilibili-content/bilibili_content.py deleted file mode 100644 index 08bfbef..0000000 --- a/skills/kb-bilibili-content/bilibili_content.py +++ /dev/null @@ -1,419 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Bilibili Content Downloader - B站内容获取工具 - -功能: -- download_video(bvid, quality=480) - 下载视频（使用 yt-dlp） -- extract_subtitle(bvid) - 提取字幕 -- extract_article(cvid) - 获取专栏全文（使用Playwright） -- 命令行参数支持 -- JSON 输出 -""" - -import argparse -import asyncio -import json -import os -import re -import subprocess -import sys -import tempfile -from datetime import datetime -from pathlib import Path -from typing import Optional, Dict, Any, List - - -class BilibiliContent: - """B站内容获取器""" - - def __init__(self, output_dir: str = "./output", cookie: str = None): - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - self.cookie = cookie - - def _create_cookie_file(self) -> str: - """创建 Netscape 格式的 Cookie 文件""" - if not self.cookie: - return None - - # 解析 Cookie 字符串 - cookies = {} - for item in self.cookie.split(';'): - item = item.strip() - if '=' in item: - key, value = item.split('=', 1) - cookies[key.strip()] = value.strip() - - # 创建 Netscape 格式的 Cookie 文件 - cookie_content = """# Netscape HTTP Cookie File -# Generated by bilibili_content.py - -""" - - for name, value in cookies.items(): - cookie_content += f".bilibili.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n" - - # 写入临时文件 - fd, path = tempfile.mkstemp(suffix='.txt') - with os.fdopen(fd, 'w') as f: - f.write(cookie_content) - - return path - - async def download_video(self, bvid: str, quality: int = 64) -> Dict[str, Any]: - """下载B站视频（使用 yt-dlp）""" - result = {"success": False, "bvid": bvid, "message": "", "filepath": None, "metadata": {}} - - try: - # 先获取视频信息 - print(f"获取视频信息: {bvid}") - - cookie_file = self._create_cookie_file() - url = f"https://www.bilibili.com/video/{bvid}" - - try: - # 获取视频标题（用于文件名） - if cookie_file: - cmd = ["yt-dlp", "--cookies", cookie_file, "--get-title", url] - else: - cmd = ["yt-dlp", "--get-title", url] - - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - - title = stdout.decode('utf-8').strip() - if not title or 'error' in title.lower(): - # 尝试备用方法 - title = f"video_{bvid}" - except Exception as e: - title = f"video_{bvid}" - - # 构建 yt-dlp 命令 - safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_', '丨')).strip()[:50] - - # 选择视频格式 - if quality == 80: - # 1080P - format_str = "30112+30280" - elif quality == 64: - # 720P - format_str = "30064+30280" - elif quality == 32: - # 480P - format_str = "30032+30280" - else: - # 360P - format_str = "30016+30280" - - print(f"下载视频: {title}") - print(f"格式: {format_str}") - - output_template = str(self.output_dir / f"{safe_title}.%(ext)s") - - cmd = [ - "yt-dlp", - "-f", format_str, - "-o", output_template, - "--no-download-archive", - "-q", - url - ] - - if cookie_file: - cmd.extend(["--cookies", cookie_file]) - - print(f"执行命令: {' '.join(cmd[:4])}...") - - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - - # 查找下载的文件 - downloaded_files = list(self.output_dir.glob(f"{safe_title}.*")) - if downloaded_files: - video_file = downloaded_files[0] - result["success"] = True - result["filepath"] = str(video_file) - result["message"] = f"下载完成: {video_file.name}" - result["metadata"] = {"title": title, "bvid": bvid} - print(f"✅ {result['message']}") - else: - result["message"] = "未找到下载的文件" - print(f"❌ {result['message']}") - - # 清理临时 Cookie 文件 - if cookie_file and os.path.exists(cookie_file): - os.remove(cookie_file) - - except Exception as e: - result["message"] = f"错误: {str(e)}" - print(f"❌ {result['message']}") - - return result - - async def extract_subtitle(self, bvid: str) -> Dict[str, Any]: - """ - 提取字幕 - - 注意：没有找到官方字幕是正常的！很多视频没有自动生成的字幕。 - 返回 success=True 即使没有字幕，让工作流继续执行后续步骤（如 Whisper）。 - """ - result = {"success": False, "bvid": bvid, "subtitles": [], "has_subtitle": False, "message": ""} - - try: - import aiohttp - - api_url = f"https://api.bilibili.com/x/player/pagelist?bvid={bvid}" - - async with aiohttp.ClientSession() as session: - async with session.get(api_url) as resp: - data = await resp.json() - - pages = data.get('data', []) - if not pages: - result["message"] = "未找到分P（这是正常的）" - # 仍然返回 success=True，让工作流继续 - result["success"] = True - return result - - cid = pages[0].get('cid', '') - - async with aiohttp.ClientSession() as session: - async with session.get(f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}") as resp: - sub_data = await resp.json() - - subtitles = sub_data.get('data', {}).get('subtitle', {}).get('list', []) - - if not subtitles: - result["message"] = "没有官方字幕（这是正常的），将使用 Whisper 进行语音识别" - # 仍然返回 success=True，让工作流继续 - result["success"] = True - return result - - result["success"] = True - result["has_subtitle"] = True - result["subtitles"] = [{"lan": s.get('lan'), "title": s.get('lan_doc')} for s in subtitles] - result["message"] = f"✅ 找到 {len(subtitles)} 个字幕，将使用官方字幕进行深度分析" - print(result["message"]) - - except Exception as e: - result["message"] = f"获取字幕出错（这是正常的），将使用 Whisper: {str(e)}" - # 出错时也返回 success=True，让工作流继续 - result["success"] = True - - return result - - async def download_subtitle(self, bvid: str) -> Dict[str, Any]: - """ - 下载字幕并转换为 SRT 格式 - - 如果没有官方字幕，返回 success=True 但 has_subtitle=False， - 让工作流继续使用 Whisper 进行语音识别。 - """ - result = {"success": False, "bvid": bvid, "filepath": None, "has_subtitle": False, "message": ""} - - try: - import aiohttp - - # 获取 CID - api_url = f"https://api.bilibili.com/x/player/pagelist?bvid={bvid}" - - async with aiohttp.ClientSession() as session: - async with session.get(api_url) as resp: - data = await resp.json() - - pages = data.get('data', []) - if not pages: - result["message"] = "未找到分P，将使用 Whisper 进行语音识别" - result["success"] = True - return result - - cid = pages[0].get('cid', '') - - # 获取字幕列表 - async with aiohttp.ClientSession() as session: - async with session.get(f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}") as resp: - sub_data = await resp.json() - - subtitles = sub_data.get('data', {}).get('subtitle', {}).get('list', []) - - if not subtitles: - result["message"] = "没有官方字幕，将使用 Whisper 进行语音识别" - result["success"] = True - return result - - # 选择中文字幕，如果没有则选第一个 - zh_sub = None - for s in subtitles: - if s.get('lan') in ('ai-zh', 'zh-CN', 'zh'): - zh_sub = s - break - if not zh_sub: - zh_sub = subtitles[0] - - # 下载字幕文件 - sub_url = zh_sub.get('subtitle_url', '') - if not sub_url: - result["message"] = "字幕URL为空，将使用 Whisper" - result["success"] = True - return result - - if sub_url.startswith('//'): - sub_url = 'https:' + sub_url - - async with aiohttp.ClientSession() as session: - async with session.get(sub_url) as resp: - sub_content = await resp.json() - - # 转换为 SRT 格式 - body = sub_content.get('body', []) - if not body: - result["message"] = "字幕内容为空，将使用 Whisper" - result["success"] = True - return result - - def format_srt_time(t: float) -> str: - """将秒数转换为 SRT 时间格式""" - h = int(t // 3600) - m = int((t % 3600) // 60) - s = int(t % 60) - ms = int((t - int(t)) * 1000) - return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}" - - srt_lines = [] - for i, line in enumerate(body, 1): - start = line.get('from', 0) - end = line.get('to', 0) - content = line.get('content', '') - srt_lines.extend([str(i), f"{format_srt_time(start)} --> {format_srt_time(end)}", content, ""]) - - srt_content = "\n".join(srt_lines) - - # 保存 SRT 文件 - output_path = self.output_dir / f"{bvid}.srt" - with open(output_path, 'w', encoding='utf-8') as f: - f.write(srt_content) - - result["success"] = True - result["has_subtitle"] = True - result["filepath"] = str(output_path) - result["message"] = f"✅ 官方字幕已下载: {output_path.name} ({len(body)}条)" - print(result["message"]) - - except Exception as e: - result["message"] = f"下载字幕出错: {str(e)}，将使用 Whisper" - result["success"] = True # 仍然返回成功，让工作流继续 - - return result - - async def extract_article(self, cvid: str) -> Dict[str, Any]: - """获取专栏全文（使用Playwright异步API）""" - result = {"success": False, "cvid": cvid, "title": "", "content": "", "message": ""} - - try: - from playwright.async_api import async_playwright - - cvid_clean = cvid.replace('cv', '') - url = f"https://www.bilibili.com/read/cv{cvid_clean}" - print(f"获取专栏: {url}") - - async with async_playwright() as p: - browser = await p.chromium.launch(headless=True) - page = await browser.new_page() - - await page.goto(url, wait_until="networkidle") - await page.wait_for_timeout(3000) - - # 提取标题 - title_elem = await page.query_selector('h1') - if title_elem: - result["title"] = (await title_elem.inner_text()).strip() - print(f"标题: {result['title'][:50]}...") - - # 提取正文内容 - content_elem = await page.query_selector('.read-article-holder') - if content_elem: - html_content = await content_elem.inner_html() - text_content = await content_elem.inner_text() - result["content"] = text_content.strip() - result["html_content"] = html_content.strip() - result["success"] = bool(result["title"] and result["content"]) - print(f"内容长度: {len(result['content'])} 字符") - else: - result["message"] = "未找到专栏内容" - - await browser.close() - - if result["success"]: - result["message"] = f"✅ 获取成功: {result['title'][:30]}..." - else: - result["message"] = f"获取失败: {result.get('message', '未知错误')}" - - except Exception as e: - result["message"] = f"错误: {str(e)}" - print(f"错误: {e}") - - return result - - -async def main(): - parser = argparse.ArgumentParser(description='Bilibili Content Downloader') - parser.add_argument('--id', '-i', required=True, help='B站内容ID') - parser.add_argument('--type', '-t', required=True, choices=['video', 'subtitle', 'subtitle-download', 'article'], help='内容类型') - parser.add_argument('--output', '-o', default='./output', help='输出目录') - parser.add_argument('--quality', '-q', type=int, default=64, help='视频质量: 16=360P, 32=480P, 64=720P, 80=1080P') - - args = parser.parse_args() - - cookie = os.environ.get('BILIBILI_COOKIE') - downloader = BilibiliContent(output_dir=args.output, cookie=cookie) - - if args.type == 'video': - result = await downloader.download_video(args.id, quality=args.quality) - elif args.type == 'subtitle': - result = await downloader.extract_subtitle(args.id) - elif args.type == 'subtitle-download': - # 下载并转换字幕为 SRT 格式 - result = await downloader.download_subtitle(args.id) - else: - result = await downloader.extract_article(args.id.replace('cv', '')) - - # 统一输出格式 - if result.get("success"): - output = { - "success": True, - "type": args.type, - "has_subtitle": result.get("has_subtitle", False), - "filename": Path(result.get('filepath', '')).name if result.get('filepath') else None, - "filepath": str(Path(result.get('filepath', '')).resolve()) if result.get('filepath') else None, - "message": result.get('message', '处理成功') - } - else: - output = { - "success": False, - "error": result.get('message', '处理失败'), - "detail": f"ID: {args.id}, 类型: {args.type}" - } - - print(json.dumps(output, ensure_ascii=False, indent=2)) - return result - - -if __name__ == '__main__': - try: - result = asyncio.run(main()) - sys.exit(0 if result.get("success") else 1) - except KeyboardInterrupt: - print(json.dumps({"success": False, "error": "用户取消"}, ensure_ascii=False, indent=2)) - sys.exit(1) - except Exception as e: - print(json.dumps({"success": False, "error": str(e)}, ensure_ascii=False, indent=2)) - sys.exit(1) diff --git a/skills/kb-bilibili-content/bilibili_extractor.py b/skills/kb-bilibili-content/bilibili_extractor.py deleted file mode 100644 index 971d239..0000000 --- a/skills/kb-bilibili-content/bilibili_extractor.py +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -B站视频提取脚本 -提取视频信息、统计数据、字幕并生成Markdown笔记 -""" - -import sys -import os -import json -import re -from datetime import datetime - -# 添加skills路径 -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb')) - -from src.extractors.bilibili_extractor import BilibiliExtractor -from src.config import get_config - - -def format_number(num): - """格式化数字""" - if num >= 100000000: - return f"{num / 100000000:.1f}亿" - elif num >= 10000: - return f"{num / 10000:.1f}万" - else: - return f"{num:,}" - - -def generate_markdown(result, bvid): - """生成Markdown笔记""" - metadata = result['metadata'] - stats = result['stats'] - description = result['description'] - subtitle = result['subtitle'] - - # 获取当前时间 - now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - - md = f"""# {metadata['title']} - -> **来源**: [B站](https://www.bilibili.com/video/{bvid}) -> **作者**: {metadata['author']} -> **发布时间**: {metadata['pubtime']} -> **提取时间**: {now} -> **时长**: {metadata['duration']} - -## 📊 统计数据 - -| 指标 | 数量 | -|------|------| -| 👀 播放 | {format_number(stats['view'])} | -| 💬 弹幕 | {format_number(stats['danmaku'])} | -| 📝 评论 | {format_number(stats['reply'])} | -| ⭐ 收藏 | {format_number(stats['favorite'])} | -| 🪙 投币 | {format_number(stats['coin'])} | -| 🔗 分享 | {format_number(stats['share'])} | -| 👍 点赞 | {format_number(stats['like'])} | - -## 📝 视频简介 - -{description if description else '(暂无简介)'} - -## 📌 标签 - -{metadata['tags'] if metadata['tags'] else '(无标签)'} - -## 🎬 字幕 - -{subtitle if subtitle else '(无字幕)'} - ---- - -**元数据**: -- BV号: `{metadata['bvid']}` -- AV号: `{metadata['aid']}` -- UP主ID: `{metadata['mid']}` -""" - - return md - - -def main(): - """主函数""" - # BV号 - bvid = "BV1N5foBjE5K" # 从 https://b23.tv/btMGoOi 解析得到 - - print(f"开始提取B站视频: {bvid}") - print("=" * 60) - - # 获取配置 - try: - config = get_config() - except Exception as e: - print(f"⚠️ 获取配置失败，使用默认配置: {e}") - config = None - - # 创建提取器 - extractor = BilibiliExtractor(config=config) - - # 提取视频信息 - try: - result = extractor.extract(bvid) - - print("\n✅ 提取成功！\n") - print("=" * 60) - print("基本信息:") - print("=" * 60) - print(f"标题: {result['metadata']['title']}") - print(f"作者: {result['metadata']['author']}") - print(f"时长: {result['metadata']['duration']}") - print(f"发布: {result['metadata']['pubtime']}") - - print("\n" + "=" * 60) - print("统计数据:") - print("=" * 60) - print(f"播放: {format_number(result['stats']['view'])}") - print(f"弹幕: {format_number(result['stats']['danmaku'])}") - print(f"评论: {format_number(result['stats']['reply'])}") - print(f"收藏: {format_number(result['stats']['favorite'])}") - print(f"投币: {format_number(result['stats']['coin'])}") - print(f"点赞: {format_number(result['stats']['like'])}") - - # 生成 Markdown 笔记 - md_content = generate_markdown(result, bvid) - - # 确保 notes 目录存在 - notes_dir = "/home/admin/.openclaw/workspace/notes" - os.makedirs(notes_dir, exist_ok=True) - - # 生成文件名（使用BV号作为文件名） - filename = f"{bvid}.md" - filepath = os.path.join(notes_dir, filename) - - # 写入文件 - with open(filepath, 'w', encoding='utf-8') as f: - f.write(md_content) - - print("\n" + "=" * 60) - print(f"✅ Markdown笔记已生成: {filepath}") - print("=" * 60) - - # 返回笔记内容 - return md_content - - except Exception as e: - print(f"\n❌ 提取失败: {e}") - import traceback - traceback.print_exc() - return None - - -if __name__ == '__main__': - main() diff --git a/skills/kb-bilibili-content/video_downloader.py b/skills/kb-bilibili-content/video_downloader.py deleted file mode 100644 index f5d25c4..0000000 --- a/skills/kb-bilibili-content/video_downloader.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -使用旧版 API 获取并下载视频流 -""" -import asyncio -import json -import os -from playwright.async_api import async_playwright - -async def download_video_stream(bvid, output_dir="/tmp/bilibili_downloads"): - """下载视频流""" - url = f"https://www.bilibili.com/video/{bvid}" - - # 创建输出目录 - os.makedirs(output_dir, exist_ok=True) - - async with async_playwright() as playwright: - browser = await playwright.chromium.launch(headless=True) - context = await browser.new_context() - page = await context.new_page() - - print(f"正在下载视频: {url}") - print("=" * 80) - - # 首先获取 aid 和 cid - print("\n1. 获取视频信息...") - try: - api_url = "https://api.bilibili.com/x/web-interface/view" - response = await page.request.get(api_url, params={'bvid': bvid}) - data = await response.json() - - if data.get('code') != 0: - print(f" ❌ 失败: {data.get('message')}") - await browser.close() - return None - - video_data = data.get('data', {}) - title = video_data.get('title') - aid = video_data.get('aid') - cid = video_data.get('cid') - duration = video_data.get('duration') - print(f" 标题: {title}") - print(f" 时长: {duration}秒") - except Exception as e: - print(f" ❌ 异常: {e}") - await browser.close() - return None - - # 获取视频流 URL - print("\n2. 获取视频流 URL...") - try: - playurl_api = "https://api.bilibili.com/x/player/playurl" - params = { - 'bvid': bvid, - 'cid': cid, - 'qn': 80, # 1080P - 'otype': 'json', - 'fnver': 0, - 'fnval': 0, # FLV 格式 - } - - playurl_response = await page.request.get(playurl_api, params=params) - - if not playurl_response.ok: - print(f" ❌ HTTP 错误: {playurl_response.status}") - await browser.close() - return None - - playurl_data = await playurl_response.json() - - if playurl_data.get('code') != 0: - print(f" ❌ API 错误: {playurl_data.get('message')}") - await browser.close() - return None - - play_data = playurl_data.get('data', {}) - - # 检查支持的清晰度 - print(f"\n 支持的清晰度:") - for i, (q, desc) in enumerate(zip(play_data.get('accept_quality', []), play_data.get('accept_description', []))): - current = " (当前)" if q == play_data.get('quality') else "" - print(f" {desc} ({q}){current}") - - # 获取视频 URL - durl = play_data.get('durl', []) - if not durl: - print(f" ❌ 没有找到视频流") - await browser.close() - return None - - print(f"\n 找到 {len(durl)} 个视频分段") - video_urls = [] - for i, segment in enumerate(durl): - segment_url = segment.get('url') - segment_size = segment.get('size', 0) - video_urls.append(segment_url) - print(f" 分段 {i+1}: {segment_size / 1024 / 1024:.2f}MB") - print(f" {segment_url[:100]}...") - - except Exception as e: - print(f" ❌ 异常: {e}") - import traceback - traceback.print_exc() - await browser.close() - return None - - # 下载视频 - print(f"\n3. 开始下载视频...") - all_segments = [] - total_size = 0 - - for i, video_url in enumerate(video_urls, 1): - print(f"\n 下载分段 {i}/{len(video_urls)}...") - - try: - # 添加必要的请求头 - headers = { - 'Referer': 'https://www.bilibili.com/', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', - } - - # 下载分段 - segment_response = await page.request.get( - video_url, - headers=headers, - timeout=60000 - ) - - if not segment_response.ok: - print(f" ❌ 下载失败: {segment_response.status}") - continue - - segment_data = await segment_response.body() - segment_size = len(segment_data) - total_size += segment_size - all_segments.append(segment_data) - - print(f" ✅ 下载成功: {segment_size / 1024 / 1024:.2f}MB") - - except Exception as e: - print(f" ❌ 下载异常: {e}") - continue - - if not all_segments: - print(f"\n ❌ 没有成功下载任何分段") - await browser.close() - return None - - # 合并分段 - print(f"\n4. 合并视频分段...") - video_data = b''.join(all_segments) - - # 保存文件 - safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50] - output_path = os.path.join(output_dir, f"{safe_title}.flv") - - with open(output_path, 'wb') as f: - f.write(video_data) - - print(f" ✅ 视频已保存: {output_path}") - print(f" 文件大小: {total_size / 1024 / 1024:.2f}MB") - - await browser.close() - - return output_path - -if __name__ == '__main__': - bvid = "BV1KtFzzqEUG" - result = asyncio.run(download_video_stream(bvid)) - if result: - print(f"\n✅ 下载完成！") - print(f"文件路径: {result}") - else: - print(f"\n❌ 下载失败") diff --git a/skills/kb-deep-analyzer/deep_analyzer.py b/skills/kb-deep-analyzer/deep_analyzer.py index 93d4298..2f947ce 100644 --- a/skills/kb-deep-analyzer/deep_analyzer.py +++ b/skills/kb-deep-analyzer/deep_analyzer.py @@ -141,14 +141,26 @@ class DeepAnalyzer: # 标准化元数据 metadata = self._normalize_metadata(metadata) - + title = metadata.get('title', '未知') duration = metadata.get('duration', 0) author = metadata.get('author', '') pubtime = metadata.get('pubtime', '') url = metadata.get('url', '') bvid = metadata.get('bvid', '') - platform = metadata.get('platform', 'bilibili') + + # 推断 platform：优先使用元数据中的值，否则根据 URL 或 content_type 推断 + platform = metadata.get('platform', '') + if not platform: + if 'zhihu.com' in url or content_type in ('answer', 'zhuanlan', 'question'): + platform = 'zhihu' + elif 'bilibili.com' in url or bvid: + platform = 'bilibili' + else: + platform = 'bilibili' # 默认值 + + # 清理 author 字段：移除零宽度空格和前后空白 + author = self._clean_author(author) # 获取内容 - 支持多种字段名 text = content.get('text', '') or content.get('subtitle', '') or content.get('subtitles', '') or content.get('description', '') @@ -186,11 +198,11 @@ class DeepAnalyzer: return result - def _build_prompt(self, title: str, duration: int, duration_str: str, author: str, - pubtime: str, url: str, bvid: str, platform: str, text: str, + def _build_prompt(self, title: str, duration: int, duration_str: str, author: str, + pubtime: str, url: str, bvid: str, platform: str, text: str, content_type: str = "video") -> str: """构建分析 prompt，根据内容类型选择不同模板""" - if content_type == "article": + if content_type in ("article", "answer", "zhuanlan"): return self._build_article_prompt(title, author, pubtime, url, platform, text) else: return self._build_video_prompt(title, duration, duration_str, author, pubtime, url, bvid, platform, text) @@ -422,6 +434,22 @@ URL：{url} return f"{h}:{m:02d}:{s:02d}" return f"{m}:{s:02d}" + def _clean_author(self, author: str) -> str: + """清理 author 字段：移除零宽度空格和前后空白""" + if not author: + return '' + # 移除零宽度空格 (U+200B) 和其他不可见字符 + invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u00ad'] + cleaned = author + for char in invisible_chars: + cleaned = cleaned.replace(char, '') + # 移除前后空白和换行 + cleaned = cleaned.strip() + # 如果包含换行符，只取第一行 + if '\n' in cleaned: + cleaned = cleaned.split('\n')[0].strip() + return cleaned + # ============== 工具函数 ============== @@ -437,19 +465,34 @@ def generate_frontmatter(metadata: Dict, result: AnalysisResult, content_type: s """生成 Markdown frontmatter""" # 提取标签 tags = result.recommended_tags or [] - + # 格式化日期 pubtime = metadata.get('pubtime', '') if pubtime: date_str = str(pubtime)[:10] else: date_str = datetime.now().strftime('%Y-%m-%d') - - fm = ["---", + + # 清理 author 字段 + author = metadata.get('author', '未知') + author = _clean_string(author) + + # 推断 platform + url = metadata.get('url', '') + platform = metadata.get('platform', '') + if not platform: + if 'zhihu.com' in url or content_type in ('answer', 'zhuanlan', 'question'): + platform = 'zhihu' + elif 'bilibili.com' in url or metadata.get('bvid'): + platform = 'bilibili' + else: + platform = 'bilibili' + + fm = ["---", f"title: \"{metadata.get('title', '未命名')}\"", f"date: {date_str}", - f"author: {metadata.get('author', '未知')}", - f"platform: {metadata.get('platform', 'bilibili')}"] + f"author: {author}", + f"platform: {platform}"] # 视频类型添加 duration 和 bvid if content_type == "video": @@ -470,6 +513,23 @@ def generate_frontmatter(metadata: Dict, result: AnalysisResult, content_type: s return '\n'.join(fm) +def _clean_string(text: str) -> str: + """清理字符串：移除零宽度空格和前后空白""" + if not text: + return '' + # 移除零宽度空格 (U+200B) 和其他不可见字符 + invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u00ad'] + cleaned = text + for char in invisible_chars: + cleaned = cleaned.replace(char, '') + # 移除前后空白和换行 + cleaned = cleaned.strip() + # 如果包含换行符，只取第一行 + if '\n' in cleaned: + cleaned = cleaned.split('\n')[0].strip() + return cleaned + + # ============== 主程序 ============== def main(): diff --git a/skills/kb-funasr/.env.example b/skills/kb-funasr/.env.example deleted file mode 100644 index 1c21fe6..0000000 --- a/skills/kb-funasr/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -# FunASR 模型缓存目录 -FUNASR_CACHE_DIR=~/.cache/modelscope/hub diff --git a/skills/kb-funasr/README.md b/skills/kb-funasr/README.md deleted file mode 100644 index baef2d8..0000000 --- a/skills/kb-funasr/README.md +++ /dev/null @@ -1,212 +0,0 @@ -# KB FunASR - 本地语音识别 - -基于阿里 FunASR 的本地语音识别工具，支持将音视频文件转换为文本和 JSON 格式。 - -## 功能特性 - -- 🎯 **本地执行** - 完全离线运行，保护隐私 -- 🚀 **多模型支持** - Paraformer、SenseVoice 等模型可选 -- 📝 **多格式输出** - JSON、TXT 两种格式 -- ⚡ **异步执行** - 支持后台运行，不阻塞主代理 -- 🎬 **广泛兼容** - 支持常见音视频格式（MP3、MP4、WAV 等） -- 🇨🇳 **中文优化** - Paraformer 模型针对中文语音优化 - -## 安装前提 - -### 1. 安装依赖 - -```bash -pip install funasr modelscope -``` - -### 2. 配置环境变量（可选） - -在 `~/.clawdbot/clawdbot.json` 中添加： - -```json5 -{ - skills: { - entries: { - "kb-funasr": { - env: { - FUNASR_CACHE_DIR: "~/.cache/modelscope/hub" - } - } - } - } -} -``` - -或通过命令行配置： - -```bash -clawdbot skills config kb-funasr FUNASR_CACHE_DIR "~/.cache/modelscope/hub" -``` - -## 使用方法 - -### 基本用法 - -```bash -# 使用默认 Paraformer 中文模型处理 -python funasr.py --input video.mp4 - -# 后台异步执行 -python funasr.py --input video.mp4 --background -``` - -### 指定模型 - -```bash -# Paraformer 中文模型（推荐） -python funasr.py --input video.mp4 --model paraformer-zh - -# Paraformer 多语言模型 -python funasr.py --input video.mp4 --model paraformer - -# SenseVoice 多语言模型 -python funasr.py --input video.mp4 --model sensevoice -``` - -### 指定输出目录 - -```bash -python funasr.py --input video.mp4 --output ./transcripts -``` - -## 输出文件 - -对于输入 `video.mp4`，将生成以下文件： - -``` -output/ -├── video.json # 详细结果（含时间戳） -└── video.txt # 纯文本 -``` - -### JSON 格式 - -```json -{ - "success": true, - "input_file": "video.mp4", - "model": "paraformer-zh", - "output_files": { - "json": "video.json", - "txt": "video.txt" - }, - "duration_seconds": 120.5, - "segments": [ - { - "id": 0, - "start": 0.0, - "end": 3.2, - "text": "这是第一段识别结果" - }, - { - "id": 1, - "start": 3.2, - "end": 6.8, - "text": "这是第二段识别结果" - } - ] -} -``` - -## 模型对比 - -| 模型 | 速度 | 精度 | 适用场景 | -|------|------|------|----------| -| paraformer-zh | ⚡⚡⚡ | ⭐⭐⭐⭐ | 中文语音（推荐） | -| paraformer | ⚡⚡ | ⭐⭐⭐⭐ | 多语言识别 | -| sensevoice | ⚡ | ⭐⭐⭐⭐⭐ | 高精度多语言 | - -## 完整示例 - -### 示例 1: 转录播客 - -```bash -python funasr.py --input podcast.mp3 --model paraformer-zh --background -``` - -### 示例 2: 生成视频识别结果 - -```bash -python funasr.py --input tutorial.mp4 --model paraformer -``` - -### 示例 3: 批量处理（配合脚本） - -```bash -#!/bin/bash -for video in videos/*.mp4; do - python funasr.py --input "$video" --model paraformer-zh -done -``` - -## 在 Workflow 中使用 - -在知识库处理流程中，如果视频没有识别结果： - -```bash -# 步骤4: 下载视频后 -# 步骤5: 使用 FunASR 生成识别结果 -python funasr.py --input ${video_path} --model paraformer-zh --background - -# 等待 FunASR 完成，然后继续后续步骤 -``` - -## 技术细节 - -- **引擎**: 阿里 FunASR -- **模型框架**: ModelScope -- **音频采样率**: 自动转换为 16kHz -- **语言**: 根据模型支持不同语言 - -## 故障排查 - -### 问题: 模型下载失败 - -```bash -# 首次运行会自动下载，确保网络稳定 -python funasr.py --input test.mp4 - -# 手动检查缓存目录 -ls ~/.cache/modelscope/hub/ -``` - -### 问题: 音频格式不支持 - -```bash -# 转换音频格式 -ffmpeg -i input.avi -acodec libmp3lame output.mp3 -python funasr.py --input output.mp3 -``` - -### 问题: 依赖安装失败 - -```bash -# 使用清华镜像源 -pip install funasr modelscope -i https://pypi.tuna.tsinghua.edu.cn/simple -``` - -## 性能参考 - -在 CPU 环境下（Intel i7）： - -- **paraformer-zh 模型**: ~2x 实时速度（1分钟音频 ~30秒处理） -- **paraformer 模型**: ~1.5x 实时速度（1分钟音频 ~40秒处理） -- **sensevoice 模型**: ~1x 实时速度（1分钟音频 ~1分钟处理） - -## 与 Whisper 对比 - -| 特性 | FunASR | Whisper | -|------|--------|---------| -| 中文识别 | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | -| 速度 | ⚡⚡⚡ | ⚡⚡ | -| 模型大小 | 较小 | 较大 | -| 多语言支持 | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ | - -## 许可证 - -本技能使用 MIT 许可证，FunASR 使用 Apache 2.0 许可证。 diff --git a/skills/kb-funasr/SKILL.md b/skills/kb-funasr/SKILL.md deleted file mode 100644 index 886012c..0000000 --- a/skills/kb-funasr/SKILL.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -name: kb-funasr -description: 本地语音识别，支持音视频转文字（基于阿里 FunASR） ---- - -# KB FunASR Skill - -**kb-funasr** - 本地语音识别，使用阿里 FunASR 将音视频文件转换为文本和 JSON 格式输出。 - -## 什么时候使用 - -当用户需要： -- 将视频/音频文件转换为文字 -- 获取带时间戳的详细识别结果 -- 需要中文优化的语音识别（Paraformer 模型） - -## 环境变量 - -在使用之前，必须检查环境变量是否已经设置。 - -| 变量名 | 用途 | 示例 | -|--------|------|------| -| `FUNASR_CACHE_DIR` | FunASR 模型缓存目录 | `~/.cache/modelscope/hub` | - -## 使用方法 - -```bash -# 基本用法（使用 Paraformer 模型） -python funasr.py --input video.mp4 - -# 指定模型 -python funasr.py --input video.mp4 --model paraformer-zh - -# 指定输出目录 -python funasr.py --input video.mp4 --output ./output - -# 异步执行（推荐） -python funasr.py --input video.mp4 --background -``` - -## 命令参数 - -| 参数 | 说明 | 默认值 | -|------|------|--------| -| `--input` | 输入音视频文件路径 | 必需 | -| `--model` | FunASR 模型类型 | `paraformer-zh` | -| `--output` | 输出目录 | 文件所在目录 | -| `--background` | 后台异步执行 | False | - -## 模型选项 - -- `paraformer-zh` - Paraformer 中文模型（轻量，中文优化，推荐） -- `paraformer` - Paraformer 多语言模型 -- `sensevoice` - SenseVoice 多语言模型 - -## 输出文件 - -对于输入 `video.mp4`，将生成： - -- `video.json` - 详细结果（包含时间戳、分段信息） -- `video.txt` - 纯文本内容 - -## 输出格式示例 - -### JSON 格式 - -```json -{ - "success": true, - "input_file": "video.mp4", - "model": "paraformer-zh", - "output_files": { - "json": "video.json", - "txt": "video.txt" - }, - "duration_seconds": 120.5, - "segments": [ - { - "id": 0, - "start": 0.0, - "end": 3.2, - "text": "这是第一段识别结果" - } - ] -} -``` - -## Workflow 中的使用 - -```bash -# 步骤4之后：如果已有音频但无字幕，可以使用 FunASR -python funasr.py --input ${video_path} --model paraformer-zh --background -``` - -## 安装 - -```bash -# 1. 安装依赖 -pip install funasr modelscope - -# 2. 首次运行时会自动下载模型到缓存目录 -python funasr.py --input test.mp4 - -# 3. 复制技能到 skills 目录 -cp -r kb-funasr/ ~/skills/ -``` - -## 注意事项 - -- **异步执行**: 使用 `--background` 参数时，命令会在后台运行，不阻塞主代理 -- **模型选择**: paraformer-zh 模型适合中文语音识别，速度较快，精度较高 -- **文件格式**: 支持 MP3、MP4、WAV、FLAC、OGG、M4A 等常见格式 -- **首次运行**: 首次运行时会自动下载模型文件，需要稳定的网络连接 diff --git a/skills/kb-funasr/__pycache__/funasr.cpython-312.pyc b/skills/kb-funasr/__pycache__/funasr.cpython-312.pyc deleted file mode 100644 index 17e5daf..0000000 Binary files a/skills/kb-funasr/__pycache__/funasr.cpython-312.pyc and /dev/null differ diff --git a/skills/kb-funasr/funasr_tool.py b/skills/kb-funasr/funasr_tool.py deleted file mode 100644 index b082155..0000000 --- a/skills/kb-funasr/funasr_tool.py +++ /dev/null @@ -1,313 +0,0 @@ -#!/usr/bin/env python3 -""" -KB FunASR - 本地语音识别工具 -基于阿里 FunASR 实现音视频转文字 -""" - -import argparse -import json -import os -import subprocess -import sys -import tempfile -from pathlib import Path -from datetime import datetime -from typing import Dict, List, Optional - - -class FunASRTranscriber: - """FunASR 语音识别器""" - - # 模型映射 - MODEL_MAP = { - "paraformer-zh": "damo/speech_paraformer-large-vad_punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch", - "paraformer": "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", - "sensevoice": "FunASR/SenseVoiceSmall" - } - - def __init__(self, model: str = "paraformer-zh", cache_dir: Optional[str] = None): - """初始化 FunASR 识别器 - - Args: - model: 模型名称 - cache_dir: 模型缓存目录 - """ - self.model_name = model - self.model_id = self.MODEL_MAP.get(model, self.MODEL_MAP["paraformer-zh"]) - self.cache_dir = cache_dir or os.path.expanduser("~/.cache/modelscope/hub") - self._model = None - - def _load_model(self): - """延迟加载模型""" - if self._model is None: - try: - from funasr import AutoModel - print(f"正在加载模型: {self.model_name} ({self.model_id})") - print(f"缓存目录: {self.cache_dir}") - self._model = AutoModel( - model=self.model_id, - disable_pbar=False, - disable_log=False, - ) - print("模型加载完成") - except ImportError: - print("错误: 未安装 funasr 库") - print("请运行: pip install funasr modelscope") - sys.exit(1) - - def _convert_audio(self, input_path: str) -> str: - """转换音频格式为 16kHz WAV - - Args: - input_path: 输入文件路径 - - Returns: - 转换后的文件路径（如果不需要转换则返回原路径） - """ - # 检查是否已经是 WAV 格式 - if input_path.lower().endswith('.wav'): - # 检查采样率 - try: - result = subprocess.run( - ['ffprobe', '-v', 'error', '-show_entries', - 'stream=sample_rate', '-of', 'default=noprint_wrappers=1:nokey=1', - input_path], - capture_output=True, text=True - ) - if result.returncode == 0: - sample_rate = int(result.stdout.strip()) - if sample_rate == 16000: - return input_path - except Exception: - pass - - # 需要转换 - output_path = input_path.rsplit('.', 1)[0] + '_converted.wav' - print(f"正在转换音频格式: {input_path} -> {output_path}") - - try: - subprocess.run([ - 'ffmpeg', '-y', '-i', input_path, - '-acodec', 'pcm_s16le', - '-ar', '16000', - '-ac', '1', - output_path - ], check=True, capture_output=True) - print("音频转换完成") - return output_path - except subprocess.CalledProcessError as e: - print(f"音频转换失败: {e}") - # 如果转换失败，尝试使用原文件 - print("尝试使用原文件继续处理") - return input_path - except FileNotFoundError: - print("警告: 未找到 ffmpeg，跳过音频转换") - return input_path - - def transcribe(self, input_path: str) -> Dict: - """转录音频文件 - - Args: - input_path: 输入文件路径 - - Returns: - 识别结果字典 - """ - input_path = os.path.abspath(input_path) - - if not os.path.exists(input_path): - return { - "success": False, - "error": f"文件不存在: {input_path}" - } - - print(f"\n开始处理: {input_path}") - print(f"使用模型: {self.model_name}") - - # 转换音频格式 - audio_path = self._convert_audio(input_path) - - # 加载模型 - self._load_model() - - # 执行识别 - print("正在进行语音识别...") - try: - result = self._model.generate( - input=audio_path, - batch_size_s=300, - ) - except Exception as e: - return { - "success": False, - "error": f"识别失败: {str(e)}" - } - - # 解析结果 - if not result or not isinstance(result, list): - return { - "success": False, - "error": "识别结果格式错误" - } - - # FunASR 返回格式处理 - segments = [] - full_text = "" - - for seg in result: - if isinstance(seg, dict): - text = seg.get("text", "").strip() - if "timestamp" in seg and seg["timestamp"]: - # 包含时间戳 - timestamp = seg["timestamp"] - if len(timestamp) >= 2: - start_time = timestamp[0][0] if timestamp[0] else 0 - end_time = timestamp[-1][1] if timestamp[-1] else start_time - else: - start_time = 0 - end_time = 0 - else: - start_time = 0 - end_time = 0 - - if text: - segments.append({ - "id": len(segments), - "start": round(start_time, 2), - "end": round(end_time, 2), - "text": text - }) - full_text += text + "\n" - - # 清理临时文件 - if audio_path != input_path and "_converted.wav" in audio_path: - try: - os.remove(audio_path) - except Exception: - pass - - # 计算时长 - duration = segments[-1]["end"] if segments else 0 - - return { - "success": True, - "input_file": os.path.basename(input_path), - "model": self.model_name, - "output_files": {}, - "duration_seconds": round(duration, 2), - "segments": segments, - "text": full_text.strip() - } - - def save_outputs(self, result: Dict, output_dir: str): - """保存输出文件 - - Args: - result: 识别结果 - output_dir: 输出目录 - """ - if not result.get("success"): - return - - output_dir = Path(output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - basename = Path(result["input_file"]).stem - - # 保存 JSON - json_path = output_dir / f"{basename}.json" - with open(json_path, "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) - print(f"已保存: {json_path}") - result["output_files"]["json"] = str(json_path) - - # 保存 TXT - txt_path = output_dir / f"{basename}.txt" - with open(txt_path, "w", encoding="utf-8") as f: - f.write(result.get("text", "")) - print(f"已保存: {txt_path}") - result["output_files"]["txt"] = str(txt_path) - - -def run_as_background(input_path: str, model: str, output_dir: str): - """后台运行识别 - - Args: - input_path: 输入文件路径 - model: 模型名称 - output_dir: 输出目录 - """ - # 创建日志文件 - log_file = os.path.join(output_dir, f"funasr_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log") - - # 重新执行命令（后台模式） - cmd = [ - sys.executable, __file__, - "--input", input_path, - "--model", model, - "--output", output_dir - ] - - with open(log_file, "w") as f: - process = subprocess.Popen( - cmd, - stdout=f, - stderr=subprocess.STDOUT, - start_new_session=True - ) - - print(f"\n✓ 后台任务已启动 (PID: {process.pid})") - print(f"日志文件: {log_file}") - print(f"输出目录: {output_dir}") - print("\n使用以下命令查看进度:") - print(f" tail -f {log_file}") - - -def main(): - parser = argparse.ArgumentParser( - description="KB FunASR - 本地语音识别工具", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -示例: - python funasr.py --input video.mp4 - python funasr.py --input podcast.mp3 --model paraformer-zh - python funasr.py --input video.mp4 --output ./transcripts --background - """ - ) - - parser.add_argument("--input", "-i", required=True, help="输入音视频文件路径") - parser.add_argument("--model", "-m", default="paraformer-zh", - choices=["paraformer-zh", "paraformer", "sensevoice"], - help="模型名称 (默认: paraformer-zh)") - parser.add_argument("--output", "-o", default=None, help="输出目录 (默认: 文件所在目录)") - parser.add_argument("--background", "-b", action="store_true", help="后台异步执行") - - args = parser.parse_args() - - # 确定输出目录 - if args.output: - output_dir = args.output - else: - output_dir = os.path.dirname(args.input) or "." - - # 后台模式 - if args.background: - run_as_background(args.input, args.model, output_dir) - return - - # 前台模式 - transcriber = FunASRTranscriber(model=args.model) - result = transcriber.transcribe(args.input) - - if result.get("success"): - transcriber.save_outputs(result, output_dir) - print(f"\n✓ 处理完成!") - print(f" 时长: {result['duration_seconds']} 秒") - print(f" 分段数: {len(result['segments'])}") - else: - print(f"\n✗ 处理失败: {result.get('error')}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/skills/kb-link-parser/SKILL.md b/skills/kb-link-parser/SKILL.md deleted file mode 100644 index b8c97cc..0000000 --- a/skills/kb-link-parser/SKILL.md +++ /dev/null @@ -1,43 +0,0 @@ -# KB Link Parser Skill - -**kb-link-parser** - Extract and resolve IDs from various platform URLs. - -## Features - -- Multi-platform support: Bilibili, YouTube, Zhihu -- Short link detection: b23.tv and other short links -- Multiple content types: video, article, up主, playlist, collection -- Batch processing from file -- JSON output format - -## Commands - -```bash -# Parse single URL -python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD" - -# Batch processing -python link-parser.py --batch urls.txt --output results.json - -# Output format -python link-parser.py --url "..." --format json -``` - -## Output - -```json -{ - "success": true, - "data": { - "platform": "bilibili", - "type": "video", - "id": "BV1xx411c7mD" - } -} -``` - -## Installation - -```bash -cp -r kb-link-parser/ ~/skills/ -``` diff --git a/skills/kb-link-parser/__pycache__/link_parser_shortlink.cpython-312.pyc b/skills/kb-link-parser/__pycache__/link_parser_shortlink.cpython-312.pyc deleted file mode 100644 index eeed198..0000000 Binary files a/skills/kb-link-parser/__pycache__/link_parser_shortlink.cpython-312.pyc and /dev/null differ diff --git a/skills/kb-link-parser/link-parser.py b/skills/kb-link-parser/link-parser.py deleted file mode 100644 index b8cb635..0000000 --- a/skills/kb-link-parser/link-parser.py +++ /dev/null @@ -1,637 +0,0 @@ -#!/usr/bin/env python3 -""" -Link Parser - Extract and resolve IDs from various platform URLs. - -Supports: -- Bilibili (video, article, up主, collection, series) -- YouTube (video, channel, playlist) -- Zhihu (question, answer, user, collection) -- Xiaohongshu (note, user) -- Short links (b23.tv, xhslink.com, etc.) - -Usage: - python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD" - python link-parser.py --batch urls.txt --output results.json -""" - -import argparse -import json -import re -import sys -from typing import Optional, Dict, Any -from urllib.parse import urlparse - -from link_parser_shortlink import ShortLinkResolver - - -class LinkParser: - """Main link parser class""" - - # Bilibili patterns - BILIBILI_VIDEO_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV[a-zA-Z0-9]+)') - BILIBILI_VIDEO_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)') - BILIBILI_ARTICLE_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/(?:article|read)/([a-zA-Z0-9]+)') - BILIBILI_SPACE_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/(\d+)') - BILIBILI_MOBILE_SPACE_PATTERN = re.compile(r'https?://m\.bilibili\.com/space/(\d+)') - BILIBILI_MOBILE_OPUS_PATTERN = re.compile(r'https?://m\.bilibili\.com/opus/(\d+)') - BILIBILI_FAVLIST_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/favlist\?fid=(\d+)') - BILIBILI_OPUS_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/opus/(\d+)') - BILIBILI_SERIES_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/lists/(\d+)\?type=series') - BILIBILI_READLIST_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/read/readlist/(rl[a-zA-Z0-9]+)') - - # YouTube patterns - YOUTUBE_VIDEO_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/)([a-zA-Z0-9_-]{11})') - YOUTUBE_CHANNEL_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/channel/([a-zA-Z0-9_-]+)') - YOUTUBE_USER_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/user/([a-zA-Z0-9_-]+)') - YOUTUBE_PLAYLIST_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/playlist\?list=([a-zA-Z0-9_-]+)') - - # Xiaohongshu (小红书) patterns - XIAOHONGSHU_SHORT_PATTERN = re.compile(r'https?://xhslink\.com/o/([a-zA-Z0-9]+)') - XIAOHONGSHU_NOTE_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/explore/([a-zA-Z0-9]+)') - XIAOHONGSHU_NOTE_PATTERN2 = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/notes/([a-zA-Z0-9]+)') - XIAOHONGSHU_DISCOVERY_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/discovery/item/([a-zA-Z0-9]+)') - XIAOHONGSHU_USER_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/user/profile/(\d+)') - - # Zhihu patterns - ZHIHAU_QUESTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/(\d+)') - ZHIHAU_ANSWER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)') - ZHIHAU_USER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/people/([a-zA-Z0-9_-]+)') - ZHIHAU_COLUMN_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/column/([a-zA-Z0-9_-]+)') - ZHIHAU_COLLECTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/collection/(\d+)') - # Additional Zhihu patterns - ZHIHAU_ZHUANLAN_PATTERN = re.compile(r'https?://zhuanlan\.zhihu\.com/p/(\d+)') - ZHIHAU_ORG_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/([a-zA-Z0-9_-]+)') - ZHIHAU_ORG_ANSWERS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/answers') - ZHIHAU_ORG_VIDEOS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/zvideos') - ZHIHAU_ORG_ASKS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/asks') - ZHIHAU_ORG_POSTS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/posts') - ZHIHAU_ORG_PINS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/pins') - ZHIHAU_ZVIDEO_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/zvideo/(\d+)') - - def __init__(self): - self.shortlink_resolver = ShortLinkResolver() - - def resolve_shortlink(self, url: str) -> Optional[str]: - """ - Resolve short link to get final URL. - - Args: - url: Short URL to resolve - - Returns: - Final URL after resolution, or None if not resolvable - """ - # Check if it's a known short link pattern - if self.shortlink_resolver.is_bilibili_short_link(url): - # Try to resolve using requests (follows redirects) - try: - import requests - response = requests.head(url, allow_redirects=True, timeout=10) - return response.url - except Exception: - # Fallback: return original URL - return url - - # Check if it's Xiaohongshu short link - if self.XIAOHONGSHU_SHORT_PATTERN.match(url): - try: - import requests - response = requests.get(url, allow_redirects=True, timeout=10) - return response.url - except Exception: - return url - - return url - - def extract_bilibili_id(self, url: str) -> Optional[Dict[str, str]]: - """ - Extract Bilibili ID from URL. - - Args: - url: Bilibili URL - - Returns: - Dict with platform, type, and id fields, or None if not matched - """ - # Try series/playlist (must check before space pattern) - match = self.BILIBILI_SERIES_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "playlist", - "id": match.group(1) - } - - # Try readlist (专栏合集) - match = self.BILIBILI_READLIST_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "readlist", - "id": match.group(1) - } - - # Try favlist (收藏夹) - match = self.BILIBILI_FAVLIST_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "favlist", - "id": match.group(1) - } - - # Try opus (动态) - match = self.BILIBILI_OPUS_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "opus", - "id": match.group(1) - } - - # Try mobile opus (移动端动态) - match = self.BILIBILI_MOBILE_OPUS_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "opus", - "id": match.group(1) - } - - # Try video - match = self.BILIBILI_VIDEO_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "video", - "id": match.group(1) - } - - # Try article/read - match = self.BILIBILI_ARTICLE_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "article", - "id": match.group(1) - } - - # Try space (UP主) - must check after specific patterns - match = self.BILIBILI_SPACE_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "up主", - "id": match.group(1) - } - - # Try mobile space (移动端UP主空间) - match = self.BILIBILI_MOBILE_SPACE_PATTERN.match(url) - if match: - return { - "platform": "bilibili", - "type": "up主", - "id": match.group(1) - } - - return None - - def extract_youtube_id(self, url: str) -> Optional[Dict[str, str]]: - """ - Extract YouTube ID from URL. - - Args: - url: YouTube URL - - Returns: - Dict with platform, type, and id fields, or None if not matched - """ - # Try video - match = self.YOUTUBE_VIDEO_PATTERN.match(url) - if match: - return { - "platform": "youtube", - "type": "video", - "id": match.group(1) - } - - # Try channel - match = self.YOUTUBE_CHANNEL_PATTERN.match(url) - if match: - return { - "platform": "youtube", - "type": "channel", - "id": match.group(1) - } - - # Try user - match = self.YOUTUBE_USER_PATTERN.match(url) - if match: - return { - "platform": "youtube", - "type": "user", - "id": match.group(1) - } - - # Try playlist - match = self.YOUTUBE_PLAYLIST_PATTERN.match(url) - if match: - return { - "platform": "youtube", - "type": "playlist", - "id": match.group(1) - } - - return None - - def extract_zhihu_id(self, url: str) -> Optional[Dict[str, str]]: - """ - Extract Zhihu ID from URL. - - Args: - url: Zhihu URL - - Returns: - Dict with platform, type, and id fields, or None if not matched - """ - # Try question - match = self.ZHIHAU_QUESTION_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "question", - "id": match.group(1) - } - - # Try answer - match = self.ZHIHAU_ANSWER_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "answer", - "id": match.group(1) - } - - # Try user - match = self.ZHIHAU_USER_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "user", - "id": match.group(1) - } - - # Try column - match = self.ZHIHAU_COLUMN_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "column", - "id": match.group(1) - } - - # Try collection - match = self.ZHIHAU_COLLECTION_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "collection", - "id": match.group(1) - } - - # Try zhuanlan article (专栏文章) - match = self.ZHIHAU_ZHUANLAN_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "zhuanlan", - "id": match.group(1) - } - - # Try org page (机构页面) - match = self.ZHIHAU_ORG_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "org", - "id": match.group(1) - } - - # Try org answers (机构回答) - if self.ZHIHAU_ORG_ANSWERS_PATTERN.match(url): - match = self.ZHIHAU_ORG_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "org_answers", - "id": match.group(1) - } - - # Try org zvideos (机构视频) - if self.ZHIHAU_ORG_VIDEOS_PATTERN.match(url): - match = self.ZHIHAU_ORG_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "org_zvideos", - "id": match.group(1) - } - - # Try org asks (机构提问) - if self.ZHIHAU_ORG_ASKS_PATTERN.match(url): - match = self.ZHIHAU_ORG_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "org_asks", - "id": match.group(1) - } - - # Try org posts (机构文章) - if self.ZHIHAU_ORG_POSTS_PATTERN.match(url): - match = self.ZHIHAU_ORG_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "org_posts", - "id": match.group(1) - } - - # Try org pins (机构想法) - if self.ZHIHAU_ORG_PINS_PATTERN.match(url): - match = self.ZHIHAU_ORG_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "org_pins", - "id": match.group(1) - } - - # Try zvideo (知乎视频) - match = self.ZHIHAU_ZVIDEO_PATTERN.match(url) - if match: - return { - "platform": "zhihu", - "type": "zvideo", - "id": match.group(1) - } - - return None - - def extract_xiaohongshu_id(self, url: str) -> Optional[Dict[str, str]]: - """ - Extract Xiaohongshu ID from URL. - - Args: - url: Xiaohongshu URL - - Returns: - Dict with platform, type, and id fields, or None if not matched - """ - # Try short link - match = self.XIAOHONGSHU_SHORT_PATTERN.match(url) - if match: - return { - "platform": "xiaohongshu", - "type": "note", - "id": match.group(1) - } - - # Try explore link - match = self.XIAOHONGSHU_NOTE_PATTERN.match(url) - if match: - return { - "platform": "xiaohongshu", - "type": "note", - "id": match.group(1) - } - - # Try notes link - match = self.XIAOHONGSHU_NOTE_PATTERN2.match(url) - if match: - return { - "platform": "xiaohongshu", - "type": "note", - "id": match.group(1) - } - - # Try discovery item link (resolved from short link) - match = self.XIAOHONGSHU_DISCOVERY_PATTERN.match(url) - if match: - return { - "platform": "xiaohongshu", - "type": "note", - "id": match.group(1) - } - - # Try user profile - match = self.XIAOHONGSHU_USER_PATTERN.match(url) - if match: - return { - "platform": "xiaohongshu", - "type": "user", - "id": match.group(1) - } - - return None - - def parse(self, url: str) -> Dict[str, Any]: - """ - Parse URL and extract platform-specific ID. - - Args: - url: URL to parse - - Returns: - Dict with platform, type, id, and original_url fields - """ - result = { - "platform": None, - "type": None, - "id": None, - "original_url": url, - "resolved_url": None, - "success": False, - "error": None - } - - # First, resolve short links - resolved = self.resolve_shortlink(url) - if resolved: - result["resolved_url"] = resolved - url = resolved - - # Try each platform - bilibili_result = self.extract_bilibili_id(url) - if bilibili_result: - result.update(bilibili_result) - result["success"] = True - return result - - youtube_result = self.extract_youtube_id(url) - if youtube_result: - result.update(youtube_result) - result["success"] = True - return result - - zhihu_result = self.extract_zhihu_id(url) - if zhihu_result: - result.update(zhihu_result) - result["success"] = True - return result - - # Try Xiaohongshu - xiaohongshu_result = self.extract_xiaohongshu_id(url) - if xiaohongshu_result: - result.update(xiaohongshu_result) - result["success"] = True - return result - - # Check for short links that need resolution - short_links = self.shortlink_resolver.detect_short_links(url) - if short_links: - result["error"] = "Short link detected. Please provide the full URL." - return result - - result["error"] = "Unsupported URL format" - return result - - -def format_output(data: Dict[str, Any], output_format: str = "json") -> str: - """Format output based on specified format.""" - if output_format == "json": - return json.dumps(data, ensure_ascii=False, indent=2) - elif output_format == "compact": - return json.dumps(data, ensure_ascii=False, separators=(',', ':')) - elif output_format == "text": - lines = [] - for key, value in data.items(): - if value is not None: - lines.append(f"{key}: {value}") - return "\n".join(lines) - else: - return json.dumps(data, ensure_ascii=False, indent=2) - - -def main(): - """Main entry point.""" - parser = argparse.ArgumentParser( - description="Parse and extract IDs from various platform URLs", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - %(prog)s --url "https://www.bilibili.com/video/BV1xx411c7mD" - %(prog)s --url "https://www.youtube.com/watch?v=dQw4w9WgXcQ" - %(prog)s --url "https://www.zhihu.com/question/266309957" - %(prog)s --batch urls.txt --output results.json - """ - ) - - parser.add_argument( - "--url", "-u", - help="Single URL to parse" - ) - - parser.add_argument( - "--batch", "-b", - help="File containing URLs to parse (one per line)" - ) - - parser.add_argument( - "--output", "-o", - help="Output file path (for batch mode)" - ) - - parser.add_argument( - "--format", "-f", - choices=["json", "compact", "text"], - default="json", - help="Output format (default: json)" - ) - - args = parser.parse_args() - - # Check if we're in shortlink help mode - if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']: - parser.print_help() - return 0 - - if not args.url and not args.batch: - parser.print_help() - print("\nError: Please provide --url or --batch") - return 1 - - link_parser = LinkParser() - - if args.url: - # Single URL mode - parsed = link_parser.parse(args.url) - - # 统一输出格式 - if parsed["success"]: - output = { - "success": True, - "platform": parsed.get("platform"), - "type": parsed.get("type"), - "id": parsed.get("id"), - "filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None, - "message": "URL解析成功" - } - else: - output = { - "success": False, - "error": parsed.get("error", "解析失败"), - "detail": f"URL: {args.url}" - } - - print(json.dumps(output, ensure_ascii=False, indent=2)) - return 0 if output["success"] else 1 - - elif args.batch: - try: - with open(args.batch, 'r', encoding='utf-8') as f: - urls = [line.strip() for line in f if line.strip()] - except Exception as e: - print(json.dumps({ - "success": False, - "error": "读取文件失败", - "detail": str(e) - }, ensure_ascii=False, indent=2)) - return 1 - - results = [] - for url in urls: - parsed = link_parser.parse(url) - if parsed["success"]: - results.append({ - "success": True, - "platform": parsed.get("platform"), - "type": parsed.get("type"), - "id": parsed.get("id"), - "filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None - }) - else: - results.append({ - "success": False, - "error": parsed.get("error", "解析失败"), - "detail": f"URL: {url}" - }) - - if args.output: - with open(args.output, 'w', encoding='utf-8') as f: - json.dump(results, f, ensure_ascii=False, indent=2) - print(json.dumps({ - "success": True, - "message": f"结果已保存到 {args.output}", - "total": len(results) - }, ensure_ascii=False, indent=2)) - else: - for result in results: - print(json.dumps(result, ensure_ascii=False, indent=2)) - print("-" * 40) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/skills/kb-link-parser/link_parser_shortlink.py b/skills/kb-link-parser/link_parser_shortlink.py deleted file mode 100644 index 20739e7..0000000 --- a/skills/kb-link-parser/link_parser_shortlink.py +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env python3 -""" -Short link resolver for link-parser skill. - -Provides utilities for detecting and resolving short links, particularly -Bilibili short links (b23.tv) which may be blocked by anti-scraping measures. - -Supported short link services: -- Bilibili: b23.tv -- Generic URL shorteners (detection only) -""" - -import re -from typing import Optional, Dict, List, Tuple -from urllib.parse import urlparse - - -class ShortLinkResolver: - """Short link detection and resolution handler""" - - # Pattern for Bilibili short links - BILIBILI_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)') - - # Pattern for other short link services - OTHER_SHORT_PATTERNS = { - 'bit.ly': re.compile(r'https?://bit\.ly/\w+'), - 't.cn': re.compile(r'https?://t\.cn/\w+'), - 'dwz.cn': re.compile(r'https?://dwz\.cn/\w+'), - 'goo.gl': re.compile(r'https?://goo\.gl/\w+'), - 'tinyurl.com': re.compile(r'https?://tinyurl\.com/\w+'), - } - - @staticmethod - def detect_short_links(text: str) -> List[Dict[str, str]]: - """ - Detect short links in text. - - Args: - text: Input text to search - - Returns: - List of detected short links with metadata: - [ - { - 'url': 'https://b23.tv/xxxxx', - 'platform': 'bilibili', - 'is_short': True - }, - ... - ] - """ - detected = [] - - # Check for Bilibili short links - for match in ShortLinkResolver.BILIBILI_SHORT_PATTERN.finditer(text): - detected.append({ - 'url': match.group(0), - 'code': match.group(1), - 'platform': 'bilibili', - 'is_short': True - }) - - # Check for other short link services - for platform, pattern in ShortLinkResolver.OTHER_SHORT_PATTERNS.items(): - for match in pattern.finditer(text): - detected.append({ - 'url': match.group(0), - 'platform': platform, - 'is_short': True - }) - - return detected - - @staticmethod - def is_bilibili_short_link(url: str) -> bool: - """ - Check if URL is a Bilibili short link. - - Args: - url: URL to check - - Returns: - True if URL is a b23.tv short link - """ - return bool(ShortLinkResolver.BILIBILI_SHORT_PATTERN.match(url)) - - @staticmethod - def format_short_link_help(short_link: Optional[str] = None) -> str: - """ - Format help message for short links. - - Args: - short_link: Optional short link URL to mention - - Returns: - Help message - """ - lines = [ - "Detection: Short link", - "" - ] - - if short_link: - lines.append(f"Short link: {short_link}") - lines.append("") - - lines.extend([ - "Due to anti-scraping protection, short links cannot be automatically resolved.", - "Please provide the full URL or the content ID (e.g., BV号 for Bilibili).", - "", - "Full URL format: https://www.bilibili.com/video/BVxxxxx" - ]) - - return "\n".join(lines) - - @staticmethod - def format_short_link_warning(short_links: List[Dict[str, str]]) -> str: - """ - Format warning message for multiple short links. - - Args: - short_links: List of short link metadata - - Returns: - Warning message - """ - if not short_links: - return "" - - lines = [ - "Short links detected:", - "" - ] - - for i, link in enumerate(short_links, 1): - platform = link.get('platform', 'unknown') - url = link['url'] - - if platform == 'bilibili': - lines.append(f" [{i}] Bilibili short link: {url}") - lines.append(" Full URL or BV ID required") - else: - lines.append(f" [{i}] {platform}: {url}") - - lines.append("") - - return "\n".join(lines) - - -class PlaywrightResolver: - """ - Browser-based short link resolver using Playwright. - - This is an optional implementation for environments where Playwright - is available. It can bypass anti-scraping measures by simulating - a real browser. - - Requirements: - pip install playwright - playwright install chromium - """ - - def __init__(self, headless: bool = True, timeout: int = 10000): - """ - Initialize Playwright resolver. - - Args: - headless: Whether to run browser in headless mode - timeout: Page navigation timeout in milliseconds - """ - self.headless = headless - self.timeout = timeout - self._playwright = None - - def _check_playwright_available(self) -> bool: - """Check if Playwright is installed and available""" - try: - from playwright.sync_api import sync_playwright - return True - except ImportError: - return False - - def resolve(self, short_url: str) -> Optional[str]: - """ - Resolve short link using Playwright browser automation. - - Args: - short_url: Short URL to resolve - - Returns: - Final URL after redirects, or None if resolution fails - """ - if not self._check_playwright_available(): - raise ImportError( - "Playwright not installed. Install with: pip install playwright && playwright install chromium" - ) - - from playwright.sync_api import sync_playwright - - try: - with sync_playwright() as p: - # Launch browser - browser = p.chromium.launch(headless=self.headless) - context = browser.new_context( - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - viewport={'width': 1920, 'height': 1080} - ) - page = context.new_page() - - # Navigate to short URL - page.goto(short_url, timeout=self.timeout, wait_until='networkidle') - - # Get final URL - final_url = page.url - - # Close browser - browser.close() - - return final_url - - except Exception as e: - print(f"Playwright resolution failed: {e}") - return None - - def resolve_bilibili_short(self, short_url: str) -> Optional[str]: - """ - Resolve Bilibili short link and extract BV ID. - - Args: - short_url: Bilibili short URL (e.g., https://b23.tv/xxxxx) - - Returns: - Full Bilibili URL with BV ID, or None if resolution fails - """ - final_url = self.resolve(short_url) - if not final_url: - return None - - # Check if final URL contains BV ID - if 'bilibili.com/video/' in final_url: - return final_url - - # Extract BV ID if present in URL - bv_pattern = re.compile(r'(BV[a-zA-Z0-9]{10})') - match = bv_pattern.search(final_url) - if match: - return f"https://www.bilibili.com/video/{match.group(1)}" - - return None - - -def get_resolver() -> ShortLinkResolver: - """ - Get the default resolver instance. - - Returns: - ShortLinkResolver instance - """ - return ShortLinkResolver() - - -def detect_short_links(text: str) -> List[Dict[str, str]]: - """ - Detect short links in text. - - Args: - text: Input text to search - - Returns: - List of detected short links with metadata - """ - resolver = ShortLinkResolver() - return resolver.detect_short_links(text) - - -# CLI interface for testing -def main(): - """CLI for testing the short link resolver.""" - import sys - - if len(sys.argv) < 2: - print("Usage: python3 link_parser_shortlink.py 'text to test'") - print("\nExamples:") - print(' python3 link_parser_shortlink.py "Check this: https://b23.tv/oLBZZfK"') - print(' python3 link_parser_shortlink.py "https://bit.ly/xxxxx"') - sys.exit(1) - - text = ' '.join(sys.argv[1:]) - short_links = detect_short_links(text) - - import json - print(json.dumps(short_links, ensure_ascii=False, indent=2)) - - -if __name__ == '__main__': - main() diff --git a/skills/kb-ocr-processor/SKILL.md b/skills/kb-ocr-processor/SKILL.md deleted file mode 100644 index cfd9ef5..0000000 --- a/skills/kb-ocr-processor/SKILL.md +++ /dev/null @@ -1,66 +0,0 @@ -# KB OCR Processor Skill - -**kb-ocr-processor** - Extract text from video frames using OCR. - -## Features - -- **视频截帧**: 直接从视频文件截取帧图片 -- **批量处理**: 支持处理帧目录 -- **双引擎**: EasyOCR / Tesseract -- **置信度评分**: 每帧识别结果带置信度 -- **文本合并**: 自动合并所有帧的识别结果 - -## Commands - -```bash -# 方式1: 直接从视频截帧并OCR（推荐） -python ocr_processor.py --video ./video.flv --output ocr.json - -# 方式2: 处理已截取的帧目录 -python ocr_processor.py --input ./frames/ --output ocr.json - -# 指定截帧帧率（默认1帧/秒） -python ocr_processor.py --video ./video.flv --fps 2 --output ocr.json - -# 同时生成合并的文本文件 -python ocr_processor.py --video ./video.flv --output ocr.json --merge - -# 使用 Tesseract 引擎 -python ocr_processor.py --input ./frames/ --engine tesseract --output ocr.json -``` - -## Workflow 中的使用 - -```bash -# 步骤5: OCR处理（视频无字幕时） -python ocr_processor.py --video ${CONTENT_VIDEO_PATH} --output ${CONTENT_DIR}/ocr.json --merge -``` - -## Output - -```json -{ - "success": true, - "engine": "easyocr", - "total_frames": 619, - "results": [ - { - "source": "0001.jpg", - "text": "字幕文字", - "confidence": 0.95 - } - ] -} -``` - -## Installation - -```bash -cp -r kb-ocr-processor/ ~/skills/ -pip install -r requirements.txt - -# 依赖说明 -# - easyocr: 主要OCR引擎（自动下载模型） -# - pytesseract: Tesseract引擎 -# - ffmpeg: 视频截帧（apt install ffmpeg） -``` diff --git a/skills/kb-ocr-processor/ocr_processor.py b/skills/kb-ocr-processor/ocr_processor.py deleted file mode 100644 index b58d618..0000000 --- a/skills/kb-ocr-processor/ocr_processor.py +++ /dev/null @@ -1,360 +0,0 @@ -#!/usr/bin/env python3 -""" -OCR Processor - 视频帧文字识别工具 -支持 EasyOCR 和 Tesseract 引擎 -""" - -import argparse -import json -import os -import glob -from pathlib import Path -from typing import List, Dict, Optional -import easyocr -import pytesseract -from PIL import Image - - -class OCRProcessor: - """OCR 文字识别处理器""" - - def __init__(self, engine: str = "easyocr", languages: List[str] = ["ch_sim", "en"]): - """ - 初始化 OCR 处理器 - - Args: - engine: OCR 引擎 ("easyocr" 或 "tesseract") - languages: 语言列表 - """ - self.engine = engine - self.languages = languages - self.reader = None - - if engine == "easyocr": - self.reader = easyocr.Reader(languages, gpu=False) - - def process_image(self, image_path: str) -> Dict: - """ - 处理单张图片进行 OCR 识别 - - Args: - image_path: 图片路径 - - Returns: - 包含 text 和 confidence 的字典 - """ - if not os.path.exists(image_path): - return {"text": "", "confidence": 0.0, "error": "File not found"} - - try: - if self.engine == "easyocr": - return self._process_with_easyocr(image_path) - elif self.engine == "tesseract": - return self._process_with_tesseract(image_path) - else: - return {"text": "", "confidence": 0.0, "error": f"Unknown engine: {self.engine}"} - except Exception as e: - return {"text": "", "confidence": 0.0, "error": str(e)} - - def _process_with_easyocr(self, image_path: str) -> Dict: - """使用 EasyOCR 处理图片""" - results = self.reader.readtext(image_path) - - if not results: - return {"text": "", "confidence": 0.0} - - texts = [] - confidences = [] - - for bbox, text, conf in results: - texts.append(text) - confidences.append(conf) - - merged_text = "\n".join(texts) - avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0 - - return { - "text": merged_text, - "confidence": round(avg_confidence, 4), - "details": [ - {"text": text, "confidence": round(conf, 4)} - for text, conf in zip(texts, confidences) - ] - } - - def _process_with_tesseract(self, image_path: str) -> Dict: - """使用 Tesseract 处理图片""" - image = Image.open(image_path) - text = pytesseract.image_to_string(image, lang="+".join(self.languages)) - - # Tesseract 不直接返回置信度，返回 0.0 - return { - "text": text.strip(), - "confidence": 0.0 - } - - def process_frames(self, input_dir: str, fps: float = 0.1, progress_file: Optional[str] = None) -> List[Dict]: - """ - 处理目录下所有帧图片 - - Args: - input_dir: 帧图片目录 - fps: 采样帧率 - progress_file: 可选，进度文件路径（会实时更新） - - Returns: - OCR 结果列表 - """ - if not os.path.isdir(input_dir): - print(f"Error: Directory not found: {input_dir}") - return [] - - # 支持的图片格式 - image_extensions = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.webp"] - image_files = [] - - for ext in image_extensions: - image_files.extend(glob.glob(os.path.join(input_dir, ext))) - image_files.extend(glob.glob(os.path.join(input_dir, "**", ext), recursive=True)) - - # 按文件名排序 - image_files = sorted(image_files) - - if not image_files: - print(f"No images found in {input_dir}") - return [] - - total = len(image_files) - print(f"Found {total} images in {input_dir}") - print(f"Processing with {self.engine} engine...") - - def update_progress(current, status="running", last_result=None): - """更新进度文件""" - if progress_file: - import json - import time - prog = { - "current": current, - "total": total, - "percent": round(current / total * 100, 1) if total > 0 else 0, - "status": status, - "last_result": last_result, - "updated_at": time.strftime("%Y-%m-%d %H:%M:%S") - } - with open(progress_file, 'w', encoding='utf-8') as f: - json.dump(prog, f, ensure_ascii=False, indent=2) - - # 初始化进度 - update_progress(0, "started") - - results = [] - for i, image_path in enumerate(image_files, 1): - basename = os.path.basename(image_path) - print(f"Processing [{i}/{total}]: {basename}") - - ocr_result = self.process_image(image_path) - ocr_result["source"] = basename - - # 记录结果摘要 - text_count = len(ocr_result.get("text", "").split()) if ocr_result.get("text") else 0 - last_result = f"✓ {text_count} 处文字" if ocr_result.get("text") else "✗ 无文字" - - results.append(ocr_result) - - # 实时更新进度 - update_progress(i, "running", last_result) - - # 完成 - update_progress(total, "completed", f"完成 {total} 张") - - print(f"\n✅ Processed {total} frames") - return results - - def merge_texts(self, results: List[Dict]) -> str: - """ - 合并多个 OCR 结果的文本 - - Args: - results: OCR 结果列表 - - Returns: - 合并后的文本 - """ - texts = [] - for result in results: - if result.get("text"): - texts.append(result["text"]) - return "\n\n---\n\n".join(texts) - - def save_results(self, results: List[Dict], output_path: str, merge: bool = False): - """ - 保存 OCR 结果到 JSON 文件 - - Args: - results: OCR 结果列表 - output_path: 输出文件路径 - merge: 是否同时保存合并后的文本文件 - """ - output_data = { - "engine": self.engine, - "total_frames": len(results), - "results": results - } - - # 保存 JSON - with open(output_path, "w", encoding="utf-8") as f: - json.dump(output_data, f, ensure_ascii=False, indent=2) - - print(f"Results saved to {output_path}") - - # 如果需要，同时保存合并的文本 - if merge: - text_path = output_path.replace(".json", ".txt") - merged_text = self.merge_texts(results) - with open(text_path, "w", encoding="utf-8") as f: - f.write(merged_text) - print(f"Merged text saved to {text_path}") - - -def extract_frames(video_path: str, output_dir: str, fps: float = 1.0) -> str: - """ - 使用 ffmpeg 从视频中截取帧图片 - - Args: - video_path: 视频文件路径 - output_dir: 输出目录 - fps: 每秒截取帧数 (默认: 1) - - Returns: - 帧图片目录路径 - """ - import subprocess - - output_dir = Path(output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - # ffmpeg 命令: -i 输入 -vf fps=fps 输出格式 - cmd = [ - "ffmpeg", "-y", - "-i", video_path, - "-vf", f"fps={fps}", - "-q:v", "2", # 质量等级 (2=高质量) - str(output_dir / "%04d.jpg") - ] - - print(f"🔄 截取视频帧: {video_path}") - print(f" 输出目录: {output_dir}") - - try: - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode == 0: - # 统计截取的帧数 - frame_count = len(list(output_dir.glob("*.jpg"))) - print(f"✅ 截取完成: {frame_count} 帧") - return str(output_dir) - else: - print(f"❌ ffmpeg 错误: {result.stderr}") - return "" - except FileNotFoundError: - print("❌ 错误: 未找到 ffmpeg，请先安装: apt install ffmpeg") - return "" - - -def main(): - """命令行入口""" - parser = argparse.ArgumentParser( - description="OCR 视频帧文字识别工具", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -示例: - # 使用 EasyOCR 处理帧目录 - python ocr_processor.py --input ./frames/ --output ./ocr.json - - # 直接从视频截帧并 OCR - python ocr_processor.py --video ./video.flv --output ./ocr.json - - # 使用 Tesseract 处理 - python ocr_processor.py --input ./frames/ --output ./ocr.json --engine tesseract - - # 指定截帧帧率 - python ocr_processor.py --video ./video.flv --output ./ocr.json --fps 2 - - # 同时生成合并的文本文件 - python ocr_processor.py --input ./frames/ --output ./ocr.json --merge - """ - ) - - # 输入源：帧目录 或 视频文件（二选一） - input_group = parser.add_mutually_exclusive_group(required=True) - input_group.add_argument("--input", "-i", help="输入帧图片目录") - input_group.add_argument("--video", "-v", help="输入视频文件（会自动截帧）") - - parser.add_argument("--output", "-o", required=True, help="输出 JSON 文件路径") - parser.add_argument("--fps", "-f", type=float, default=1.0, help="截帧帧率 (默认: 1帧/秒)") - parser.add_argument("--engine", "-e", choices=["easyocr", "tesseract"], - default="easyocr", help="OCR 引擎 (默认: easyocr)") - parser.add_argument("--merge", action="store_true", help="同时保存合并的文本文件") - parser.add_argument("--languages", "-l", nargs="+", - default=["ch_sim", "en"], help="语言列表") - parser.add_argument("--progress", "-p", help="可选：进度文件路径（会实时更新）") - - args = parser.parse_args() - - # 如果指定了视频文件，先截帧 - input_dir = args.input - if args.video: - if not os.path.exists(args.video): - print(json.dumps({ - "success": False, - "error": f"视频文件不存在: {args.video}" - }, ensure_ascii=False, indent=2)) - return 1 - - # 创建临时帧目录 - frames_dir = Path(args.video).stem + "_frames" - input_dir = extract_frames(args.video, frames_dir, fps=args.fps) - - if not input_dir: - print(json.dumps({ - "success": False, - "error": "截帧失败" - }, ensure_ascii=False, indent=2)) - return 1 - - # 初始化处理器 - processor = OCRProcessor(engine=args.engine, languages=args.languages) - - # 处理帧 - results = processor.process_frames(input_dir, fps=args.fps, progress_file=args.progress) - - if results: - # 保存结果 - processor.save_results(results, args.output, merge=args.merge) - - # 统一输出格式 - total_texts = sum(1 for r in results if r.get("text")) - avg_conf = sum(r.get("confidence", 0) for r in results) / len(results) if results else 0 - - output = { - "success": True, - "filename": Path(args.output).name, - "filepath": str(Path(args.output).resolve()), - "message": f"OCR处理完成", - "stats": { - "total_frames": len(results), - "frames_with_text": total_texts, - "avg_confidence": round(avg_conf, 4) - } - } - print(json.dumps(output, ensure_ascii=False, indent=2)) - return 0 - else: - print(json.dumps({ - "success": False, - "error": "未处理到任何结果" - }, ensure_ascii=False, indent=2)) - return 1 - - -if __name__ == "__main__": - exit(main()) diff --git a/skills/kb-ocr-processor/requirements.txt b/skills/kb-ocr-processor/requirements.txt deleted file mode 100644 index f3f72c2..0000000 --- a/skills/kb-ocr-processor/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -easyocr>=1.7.0 -pytesseract>=0.3.10 -Pillow>=9.0.0 diff --git a/skills/kb-whisper-diarization/.env.example b/skills/kb-whisper-diarization/.env.example deleted file mode 100644 index 108647b..0000000 --- a/skills/kb-whisper-diarization/.env.example +++ /dev/null @@ -1,9 +0,0 @@ -# KB Whisper Diarization 环境变量 - -# Whisper.cpp 安装路径 -WHISPER_CPP_PATH=~/whisper.cpp - -# HuggingFace Token（必需，用于下载 pyannote 模型） -# 获取方式: https://huggingface.co/settings/tokens -# 注意: 需要先申请 pyannote/speaker-diarization 模型访问权限 -HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx diff --git a/skills/kb-whisper-diarization/SKILL.md b/skills/kb-whisper-diarization/SKILL.md deleted file mode 100644 index c9fde27..0000000 --- a/skills/kb-whisper-diarization/SKILL.md +++ /dev/null @@ -1,151 +0,0 @@ -# KB Whisper Diarization Skill - -**kb-whisper-diarization** - 使用 Whisper + pyannote.audio 实现语音识别和说话者分离。 - -## 什么时候使用 - -当用户需要： -- 区分多个说话者（如访谈、对话） -- 获取带说话者标签的字幕 -- 分析对话中谁说了什么 - -## 环境变量 - -在使用之前，必须检查环境变量是否已经设置。 - -| 变量名 | 用途 | 示例 | -|--------|------|------| -| `WHISPER_CPP_PATH` | whisper.cpp 安装路径 | `~/whisper.cpp` | -| `HF_TOKEN` | HuggingFace Token | `hf_xxx...` | - -### 获取 HuggingFace Token（必需） - -pyannote.audio 需要从 HuggingFace 下载模型，按以下步骤操作： - -**步骤 1: 注册/登录 HuggingFace** -- 访问 https://huggingface.co -- 注册或登录账号 - -**步骤 2: 申请模型访问权限** -需要申请以下两个模型的访问权限（点击页面上的 "Access repository" 按钮）： -- https://huggingface.co/pyannote/speaker-diarization -- https://huggingface.co/pyannote/segmentation - -> ⚠️ 申请后需要等待几分钟到几小时才能获得访问权限 - -**步骤 3: 创建 Access Token** -1. 访问 https://huggingface.co/settings/tokens -2. 点击 "New token" -3. 选择 "Read" 权限 -4. 复制生成的 token（以 `hf_` 开头） - -**步骤 4: 设置环境变量** -```bash -# 添加到 ~/.kb_env 文件 -echo 'export HF_TOKEN="hf_xxxxxxxxxxxxxxxxxxxxxx"' >> ~/.kb_env -source ~/.kb_env - -# 验证 -echo $HF_TOKEN -``` - -## 使用方法 - -```bash -# 基本用法（带说话者分离） -python whisper_diarization.py --input video.mp4 - -# 说话者分离 -python whisper_diarization.py --input interview.mp4 --diarize - -# 指定说话者数量 -python whisper_diarization.py --input meeting.mp4 --diarize --num-speakers 2 - -# 指定语言 -python whisper_diarization.py --input video.mp4 --language zh -``` - -## 命令参数 - -| 参数 | 说明 | 默认值 | -|------|------|--------| -| `--input` | 输入音视频文件路径 | 必需 | -| `--output` | 输出目录 | 文件所在目录 | -| `--diarize` | 启用说话者分离 | False | -| `--num-speakers` | 估计说话者数量 | None（自动检测） | -| `--language` | 语言 | 自动检测 | -| `--model` | Whisper 模型 | base | -| `--background` | 后台异步执行 | False | - -## 模型选项 - -- **Whisper 模型**: tiny, base, small, medium, large -- **说话者分离**: pyannote/speaker-diarization (需申请权限) - -## 输出文件 - -对于输入 `interview.mp4`，将生成： - -- `interview.mp4.json` - 完整结果（包含说话者标签） -- `interview.mp4.txt` - 纯文本 -- `interview.mp4.srt` - 字幕格式（带说话者标签） - -## 输出格式示例 - -### JSON 格式 - -```json -{ - "success": true, - "input_file": "interview.mp4", - "diarization": true, - "speakers": ["SPEAKER_00", "SPEAKER_01"], - "segments": [ - { - "id": 0, - "speaker": "SPEAKER_00", - "start": 0.0, - "end": 5.2, - "text": "大家好，欢迎来到访谈节目" - }, - { - "id": 1, - "speaker": "SPEAKER_01", - "start": 5.2, - "end": 10.8, - "text": "谢谢邀请，我很荣幸" - } - ] -} -``` - -### SRT 字幕格式 - -``` -1 -00:00:00,000 --> 00:00:05,200 -[SPEAKER_00] 大家好，欢迎来到访谈节目 - -2 -00:00:05,200 --> 00:00:10,800 -[SPEAKER_01] 谢谢邀请，我很荣幸 -``` - -## 安装 - -```bash -# 1. 安装依赖 -pip install pyannote.audio torch - -# 2. 设置 HuggingFace Token -export HF_TOKEN="hf_xxx..." - -# 3. 首次运行会自动下载模型 -python whisper_diarization.py --input test.mp4 -``` - -## 注意事项 - -- **说话者分离需要申请权限**: https://huggingface.co/pyannote/speaker-diarization -- **处理时间较长**: 说话者分离会增加处理时间 -- **对立体声效果更好**: 多声道音频的分离效果更佳 diff --git a/skills/kb-whisper-diarization/requirements.txt b/skills/kb-whisper-diarization/requirements.txt deleted file mode 100644 index d465929..0000000 --- a/skills/kb-whisper-diarization/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# KB Whisper Diarization 依赖 - -# 核心依赖 -torch>=2.0.0 -pyannote.audio>=1.0.0 - -# 可选: GPU 加速 -# torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 diff --git a/skills/kb-whisper-diarization/whisper_diarization.py b/skills/kb-whisper-diarization/whisper_diarization.py deleted file mode 100644 index aa0cb58..0000000 --- a/skills/kb-whisper-diarization/whisper_diarization.py +++ /dev/null @@ -1,339 +0,0 @@ -#!/usr/bin/env python3 -""" -KB Whisper Diarization - 语音识别 + 说话者分离 -使用 Whisper 进行语音识别，pyannote.audio 进行说话者分离 -""" - -import argparse -import json -import os -import subprocess -import sys -import tempfile -from datetime import datetime -from pathlib import Path -import re - - -def parse_arguments(): - """解析命令行参数""" - parser = argparse.ArgumentParser( - description='KB Whisper Diarization - 语音识别 + 说话者分离', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -示例: - python whisper_diarization.py --input video.mp4 - python whisper_diarization.py --input interview.mp4 --diarize - python whisper_diarization.py --input meeting.mp4 --diarize --num-speakers 2 - """ - ) - parser.add_argument('--input', required=True, help='输入音视频文件路径') - parser.add_argument('--output', help='输出目录 (默认: 文件所在目录)') - parser.add_argument('--diarize', action='store_true', help='启用说话者分离') - parser.add_argument('--num-speakers', type=int, help='估计说话者数量') - parser.add_argument('--language', help='指定语言 (默认: 自动检测)') - parser.add_argument('--model', default='base', choices=['tiny', 'base', 'small', 'medium', 'large'], - help='Whisper 模型 (默认: base)') - parser.add_argument('--background', action='store_true', help='后台异步执行') - return parser.parse_args() - - -def get_whisper_path(): - """获取 whisper.cpp 路径""" - path = os.environ.get('WHISPER_CPP_PATH') - if path: - path = os.path.expanduser(path) - if os.path.exists(path): - return path - return os.path.expanduser('~/whisper.cpp') - - -def check_whisper_model(whisper_path, model): - """检查 Whisper 模型""" - model_path = os.path.join(whisper_path, 'models', f'ggml-{model}.bin') - if not os.path.exists(model_path): - raise FileNotFoundError(f"Whisper 模型不存在: {model_path}") - return model_path - - -def convert_to_wav(input_path, output_dir): - """转换为 16kHz WAV""" - if input_path.lower().endswith('.wav'): - try: - result = subprocess.run( - ['ffprobe', '-v', 'error', '-show_entries', 'stream=sample_rate', - '-of', 'default=noprint_wrappers=1:nokey=1', input_path], - capture_output=True, text=True - ) - if result.returncode == 0 and int(result.stdout.strip()) == 16000: - return input_path - except Exception: - pass - - output_path = os.path.join(output_dir, f"{Path(input_path).stem}_audio.wav") - print(f"转换音频: {input_path} -> {output_path}") - - subprocess.run([ - 'ffmpeg', '-y', '-i', input_path, - '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1', - '-loglevel', 'error', output_path - ], check=True, capture_output=True) - - print("✓ 音频转换完成") - return output_path - - -def run_whisper(whisper_path, audio_path, model, output_dir, language=None): - """运行 Whisper 识别""" - basename = Path(audio_path).stem.replace('_audio', '') - output_file = os.path.join(output_dir, basename) - - cmd = [ - os.path.join(whisper_path, 'whisper-cli'), - '-m', os.path.join(whisper_path, 'models', f'ggml-{model}.bin'), - '-f', audio_path, - '-of', output_file, - '-l', language or 'auto', - '--output-txt', '--output-srt', '--output-json', - ] - - print(f"执行 Whisper: {' '.join(cmd[:6])} ...") - - result = subprocess.run( - cmd, capture_output=True, text=True, cwd=whisper_path, timeout=3600 - ) - - if result.returncode != 0 and not os.path.exists(f"{output_file}.txt"): - raise RuntimeError(f"Whisper 失败: {result.stderr}") - - return basename - - -def run_diarization(audio_path, num_speakers=None): - """运行 pyannote.audio 说话者分离""" - print("\n🎙️ 步骤2: 说话者分离...") - - try: - import torch - from pyannote.audio import Pipeline - except ImportError as e: - print(f"✗ 缺少依赖: {e}") - print("请安装: pip install pyannote.audio torch") - raise - - # 检查 HuggingFace Token - hf_token = os.environ.get('HF_TOKEN') - if not hf_token: - raise RuntimeError("需要设置 HF_TOKEN 环境变量\n" - "获取方式: https://huggingface.co/settings/tokens") - - # 新版 pyannote 需要先登录 - from huggingface_hub import login - print("登录 HuggingFace...") - login(token=hf_token) - - print("加载 pyannote 流水线...") - pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization") - - # 设置设备 - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - print(f"使用设备: {device}") - pipeline.to(device) - - # 运行分离 - print("执行说话者分离...") - diarization = pipeline( - audio_path, - num_speakers=num_speakers - ) - - # 解析结果 - speakers = {} - for turn, _, speaker in diarization.itertracks(yield_label=True): - if speaker not in speakers: - speakers[speaker] = [] - speakers[speaker].append({ - "start": round(turn.start, 2), - "end": round(turn.end, 2) - }) - - print(f"✓ 检测到 {len(speakers)} 个说话者: {list(speakers.keys())}") - return speakers - - -def merge_results(whisper_txt, whisper_srt, diarization, output_dir, basename): - """合并 Whisper 和说话者分离结果""" - print("\n📝 步骤3: 合并结果...") - - # 读取 Whisper 文本 - text_segments = [] - if os.path.exists(whisper_txt): - with open(whisper_txt, 'r', encoding='utf-8') as f: - content = f.read() - - # 简单按行分段 - for i, line in enumerate(content.strip().split('\n')): - if line.strip(): - text_segments.append({ - "id": i, - "text": line.strip(), - "start": i * 3.0, # 估算 - "end": (i + 1) * 3.0 - }) - - # 读取 SRT 获取精确时间 - srt_segments = [] - if os.path.exists(whisper_srt): - with open(whisper_srt, 'r', encoding='utf-8') as f: - content = f.read() - - # 解析 SRT - pattern = r'(\d+)\n(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\n(.+?)(?=\n\n|\Z)' - matches = re.findall(pattern, content, re.DOTALL) - - def parse_time(t): - h, m, s_ms = t.split(':') - s, ms = s_ms.split(',') - return float(h) * 3600 + float(m) * 60 + float(s) + float(ms) / 1000 - - for idx, num, start, end, text in matches: - srt_segments.append({ - "id": int(idx), - "start": parse_time(start), - "end": parse_time(end), - "text": text.strip() - }) - - # 为每个 Whisper 段落分配说话者 - merged_segments = [] - for seg in srt_segments: - speaker = assign_speaker(seg['start'], seg['end'], diarization) - merged_segments.append({ - "id": len(merged_segments), - "speaker": speaker, - "start": seg['start'], - "end": seg['end'], - "text": seg['text'] - }) - - # 生成带说话者的 SRT - output_srt = os.path.join(output_dir, f"{basename}_diarized.srt") - with open(output_srt, 'w', encoding='utf-8') as f: - for seg in merged_segments: - start = format_time(seg['start']) - end = format_time(seg['end']) - f.write(f"{seg['id'] + 1}\n") - f.write(f"{start} --> {end}\n") - f.write(f"[{seg['speaker']}] {seg['text']}\n\n") - - # 生成合并的 TXT - output_txt = os.path.join(output_dir, f"{basename}_diarized.txt") - with open(output_txt, 'w', encoding='utf-8') as f: - for seg in merged_segments: - f.write(f"[{seg['speaker']}] {seg['text']}\n") - - # 生成 JSON - output_json = os.path.join(output_dir, f"{basename}_diarized.json") - json_data = { - "success": True, - "tool": "whisper-diarization", - "diarization": True, - "speakers": list(diarization.keys()), - "input_file": basename, - "output_files": { - "txt": f"{basename}_diarized.txt", - "srt": f"{basename}_diarized.srt", - "json": f"{basename}_diarized.json" - }, - "segments": merged_segments - } - with open(output_json, 'w', encoding='utf-8') as f: - json.dump(json_data, f, ensure_ascii=False, indent=2) - - print(f"✓ 生成文件:") - print(f" - {output_srt}") - print(f" - {output_txt}") - print(f" - {output_json}") - - return json_data - - -def assign_speaker(start, end, diarization): - """将时间段分配给说话者""" - best_speaker = "UNKNOWN" - max_overlap = 0 - - for speaker, segments in diarization.items(): - for seg in segments: - overlap = min(end, seg['end']) - max(start, seg['start']) - if overlap > max_overlap: - max_overlap = overlap - best_speaker = speaker - - return best_speaker - - -def format_time(seconds): - """秒数转换为 SRT 时间格式""" - hours = int(seconds // 3600) - minutes = int((seconds % 3600) // 60) - secs = int(seconds % 60) - ms = int((seconds % 1) * 1000) - return f"{hours:02d}:{minutes:02d}:{secs:02d},{ms:03d}" - - -def main(): - args = parse_arguments() - - input_file = os.path.abspath(args.input) - if not os.path.exists(input_file): - print(f"✗ 文件不存在: {input_file}") - sys.exit(1) - - output_dir = os.path.abspath(args.output) if args.output else os.path.dirname(input_file) - os.makedirs(output_dir, exist_ok=True) - - basename = Path(input_file).stem - - print(f"\n📁 输入: {input_file}") - print(f"📁 输出: {output_dir}") - print(f"🎙️ 说话者分离: {'是' if args.diarize else '否'}") - - try: - # 步骤1: Whisper 识别 - print(f"\n🎙️ 步骤1: 语音识别...") - whisper_path = get_whisper_path() - check_whisper_model(whisper_path, args.model) - - audio_path = convert_to_wav(input_file, output_dir) - run_whisper(whisper_path, audio_path, args.model, output_dir, args.language) - - # 步骤2: 说话者分离 - if args.diarize: - diarization = run_diarization(audio_path, args.num_speakers) - else: - diarization = {} - - # 步骤3: 合并结果 - whisper_txt = os.path.join(output_dir, f"{basename}.txt") - whisper_srt = os.path.join(output_dir, f"{basename}.srt") - result = merge_results(whisper_txt, whisper_srt, diarization, output_dir, basename) - - # 清理临时音频 - if audio_path != input_file and audio_path.endswith('_audio.wav'): - try: - os.remove(audio_path) - except Exception: - pass - - print(f"\n✅ 处理完成!") - print(f" 说话者数量: {len(diarization) if args.diarize else 1}") - print(f" 段落数量: {len(result['segments'])}") - - except Exception as e: - print(f"\n✗ 错误: {e}") - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/skills/kb-whisper/.env.example b/skills/kb-whisper/.env.example deleted file mode 100644 index 8ffc440..0000000 --- a/skills/kb-whisper/.env.example +++ /dev/null @@ -1,5 +0,0 @@ -# KB Whisper 环境变量配置示例 -# 复制此文件为 .env 并填写实际值 - -# Whisper.cpp 安装路径 -WHISPER_CPP_PATH=~/whisper.cpp diff --git a/skills/kb-whisper/README.md b/skills/kb-whisper/README.md deleted file mode 100644 index 7b84ea8..0000000 --- a/skills/kb-whisper/README.md +++ /dev/null @@ -1,211 +0,0 @@ -# KB Whisper - 本地语音识别 - -基于 Whisper.cpp 的本地语音识别工具，支持将音视频文件转换为文本、字幕和 JSON 格式。 - -## 功能特性 - -- 🎯 **本地执行** - 完全离线运行，保护隐私 -- 🚀 **多模型支持** - 从 tiny 到 large，根据需求选择速度/精度 -- 📝 **多格式输出** - JSON、SRT、TXT 三种格式 -- ⚡ **异步执行** - 支持后台运行，不阻塞主代理 -- 🎬 **广泛兼容** - 支持常见音视频格式（MP3、MP4、WAV 等） - -## 安装前提 - -### 1. 安装 whisper.cpp - -```bash -# 克隆仓库 -git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp - -# 编译 -cd ~/whisper.cpp -make - -# 下载模型（至少需要一个模型） -./models/download-ggml-model.sh tiny -./models/download-ggml-model.sh base # 可选 -``` - -### 2. 配置环境变量 - -在 `~/.clawdbot/clawdbot.json` 中添加： - -```json5 -{ - skills: { - entries: { - "kb-whisper": { - env: { - WHISPER_CPP_PATH: "~/whisper.cpp" - } - } - } - } -} -``` - -或通过命令行配置： - -```bash -clawdbot skills config kb-whisper WHISPER_CPP_PATH "~/whisper.cpp" -``` - -## 使用方法 - -### 基本用法 - -```bash -# 使用默认 tiny 模型处理 -python whisper.py --input video.mp4 - -# 使用 base 模型（推荐，平衡速度和精度） -python whisper.py --input video.mp4 --model base - -# 后台异步执行 -python whisper.py --input video.mp4 --background -``` - -### 指定输出目录 - -```bash -python whisper.py --input video.mp4 --output ./transcripts -``` - -### 处理音频文件 - -```bash -python whisper.py --input podcast.mp3 --model base -``` - -## 输出文件 - -对于输入 `video.mp4`，将生成以下文件： - -``` -output/ -├── video.json # 详细结果（含时间戳） -├── video.srt # 字幕格式 -└── video.txt # 纯文本 -``` - -### JSON 格式 - -```json -{ - "success": true, - "input_file": "video.mp4", - "model": "base", - "output_files": { - "json": "video.json", - "srt": "video.srt", - "txt": "video.txt" - }, - "duration_seconds": 120.5, - "segments": [ - { - "id": 0, - "start": 0.0, - "end": 3.2, - "text": "这是第一段字幕" - }, - { - "id": 1, - "start": 3.2, - "end": 6.8, - "text": "这是第二段字幕" - } - ] -} -``` - -## 模型对比 - -| 模型 | 速度 | 精度 | 适用场景 | -|------|------|------|----------| -| tiny | ⚡⚡⚡ | ⭐ | 快速预览、实时转录 | -| base | ⚡⚡ | ⭐⭐⭐ | 日常使用（推荐） | -| small | ⚡ | ⭐⭐⭐⭐ | 高质量需求 | -| medium | ⭐ | ⭐⭐⭐⭐⭐ | 专业用途 | -| large | 🐢 | ⭐⭐⭐⭐⭐ | 最高精度要求 | - -## 完整示例 - -### 示例 1: 转录播客 - -```bash -python whisper.py --input podcast.mp3 --model base --background -``` - -### 示例 2: 生成视频字幕 - -```bash -python whisper.py --input tutorial.mp4 --model small -``` - -### 示例 3: 批量处理（配合脚本） - -```bash -#!/bin/bash -for video in videos/*.mp4; do - python whisper.py --input "$video" --model base -done -``` - -## 在 Workflow 中使用 - -在知识库处理流程中，如果视频没有字幕： - -```bash -# 步骤4: 下载视频后，如果没有字幕 -# 步骤5: 使用 Whisper 生成字幕 -python whisper.py --input ${video_path} --model base --background - -# 等待 Whisper 完成，然后继续后续步骤 -``` - -## 技术细节 - -- **引擎**: Whisper.cpp（C++ 实现，性能优异） -- **音频采样率**: 自动转换为 16kHz -- **语言**: 自动检测（支持 99 种语言） -- **GPU 支持**: whisper.cpp 支持 CUDA 加速（需编译时启用） - -## 故障排查 - -### 问题: 找不到 whisper.cpp - -```bash -# 检查环境变量 -echo $WHISPER_CPP_PATH - -# 手动测试 -~/whisper.cpp/main --help -``` - -### 问题: 模型未下载 - -```bash -cd ~/whisper.cpp -./models/download-ggml-model.sh tiny -``` - -### 问题: 音频格式不支持 - -```bash -# 转换音频格式 -ffmpeg -i input.avi -acodec libmp3lame output.mp3 -python whisper.py --input output.mp3 -``` - -## 性能参考 - -在 CPU 环境下（Intel i7）： - -- **tiny 模型**: ~1x 实时速度（1分钟音频 ~1分钟处理） -- **base 模型**: ~0.5x 实时速度（1分钟音频 ~2分钟处理） -- **small 模型**: ~0.25x 实时速度（1分钟音频 ~4分钟处理） - -## 许可证 - -本技能使用 MIT 许可证，Whisper.cpp 使用 MIT 许可证。 diff --git a/skills/kb-whisper/SKILL.md b/skills/kb-whisper/SKILL.md deleted file mode 100644 index 9ebb999..0000000 --- a/skills/kb-whisper/SKILL.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -name: kb-whisper -description: 本地语音识别，支持音视频转文字（基于 Whisper.cpp） ---- - -# KB Whisper Skill - -**kb-whisper** - 本地语音识别，使用 Whisper.cpp 将音视频文件转换为文本、字幕和 JSON 格式输出。 - -## 什么时候使用 - -当用户需要： -- 将视频/音频文件转换为文字 -- 生成视频字幕（SRT 格式） -- 获取带时间戳的详细识别结果 - -## 环境变量 - -在使用之前，必须检查环境变量是否已经设置。 - -| 变量名 | 用途 | 示例 | -|--------|------|------| -| `WHISPER_CPP_PATH` | whisper.cpp 安装路径 | `~/whisper.cpp` | - -## 使用方法 - -```bash -# 基本用法 -python whisper.py --input video.mp4 - -# 指定模型（默认 tiny） -python whisper.py --input video.mp4 --model base - -# 指定输出目录 -python whisper.py --input video.mp4 --output ./output - -# 异步执行（推荐） -python whisper.py --input video.mp4 --background -``` - -## 命令参数 - -| 参数 | 说明 | 默认值 | -|------|------|--------| -| `--input` | 输入音视频文件路径 | 必需 | -| `--model` | Whisper 模型类型 | `tiny` | -| `--output` | 输出目录 | 文件所在目录 | -| `--background` | 后台异步执行 | False | - -## 模型选项 - -- `tiny` - 最小模型，速度快，精度较低（推荐快速识别） -- `base` - 基础模型，平衡速度和精度（推荐） -- `small` - 小型模型，精度更高，速度较慢 -- `medium` - 中型模型，高精度 -- `large` - 大型模型，最高精度，速度慢 - -## 输出文件 - -对于输入 `video.mp4`，将生成： - -- `video.json` - 详细结果（包含时间戳、分段信息） -- `video.srt` - 字幕格式文件 -- `video.txt` - 纯文本内容 - -## 输出格式示例 - -### JSON 格式 - -```json -{ - "success": true, - "input_file": "video.mp4", - "model": "tiny", - "output_files": { - "json": "video.json", - "srt": "video.srt", - "txt": "video.txt" - }, - "duration_seconds": 120.5, - "segments": [ - { - "id": 0, - "start": 0.0, - "end": 3.2, - "text": "这是第一段字幕" - } - ] -} -``` - -## Workflow 中的使用 - -```bash -# 步骤4之后：如果已有音频但无字幕，可以使用 whisper -python whisper.py --input ${video_path} --model base --background -``` - -## 安装 - -```bash -# 1. 克隆 whisper.cpp -git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp - -# 2. 编译 -cd ~/whisper.cpp -make - -# 3. 下载模型（tiny） -./models/download-ggml-model.sh tiny - -# 4. 下载更多模型（可选） -# ./models/download-ggml-model.sh base -# ./models/download-ggml-model.sh small - -# 5. 复制技能到 skills 目录 -cp -r kb-whisper/ ~/skills/ -``` - -## 注意事项 - -- **异步执行**: 使用 `--background` 参数时，命令会在后台运行，不阻塞主代理 -- **模型选择**: tiny 模型适合快速识别，base 模型适合一般用途，large 模型适合高精度需求 -- **文件格式**: 支持 MP3、MP4、WAV、FLAC、OGG、M4A 等常见格式 diff --git a/skills/kb-whisper/__pycache__/whisper.cpython-312.pyc b/skills/kb-whisper/__pycache__/whisper.cpython-312.pyc deleted file mode 100644 index c605169..0000000 Binary files a/skills/kb-whisper/__pycache__/whisper.cpython-312.pyc and /dev/null differ diff --git a/skills/kb-whisper/examples.md b/skills/kb-whisper/examples.md deleted file mode 100644 index 02905d0..0000000 --- a/skills/kb-whisper/examples.md +++ /dev/null @@ -1,273 +0,0 @@ -# KB Whisper 使用示例 - -## 快速开始 - -### 1. 基础使用 - -```bash -# 使用默认 tiny 模型转录视频 -python whisper.py --input video.mp4 - -# 使用 base 模型（推荐） -python whisper.py --input video.mp4 --model base -``` - -### 2. 异步执行 - -```bash -# 后台处理，不阻塞主代理 -python whisper.py --input video.mp4 --model base --background -``` - -### 3. 指定输出目录 - -```bash -python whisper.py --input video.mp4 --output ./transcripts -``` - -### 4. 指定语言 - -```bash -# 指定中文 -python whisper.py --input video.mp4 --language zh - -# 指定英文 -python whisper.py --input video.mp4 --language en - -# 指定日语 -python whisper.py --input video.mp4 --language ja -``` - -## 实际应用场景 - -### 场景 1: 转录播客 - -```bash -python whisper.py --input podcast.mp3 --model base --background -``` - -**输出**: -- `podcast.json` - 带时间戳的详细结果 -- `podcast.srt` - 字幕格式 -- `podcast.txt` - 纯文本 - -### 场景 2: 生成视频字幕 - -```bash -python whisper.py --input tutorial.mp4 --model small -``` - -**用途**: -- 为教程视频生成字幕 -- 导出 SRT 文件到视频编辑软件 -- 生成多语言字幕版本 - -### 场景 3: 会议录音转文字 - -```bash -python whisper.py --input meeting.mp3 --model base --output ./meetings -``` - -**输出示例**: -``` -meetings/ -├── meeting.json # 完整 JSON（含时间戳） -├── meeting.srt # 字幕格式 -└── meeting.txt # 纯文本 -``` - -### 场景 4: 批量处理 - -创建脚本 `batch_transcribe.sh`: - -```bash -#!/bin/bash - -OUTPUT_DIR="./transcripts" -mkdir -p "$OUTPUT_DIR" - -for video in videos/*.mp4; do - echo "处理: $video" - python whisper.py --input "$video" --output "$OUTPUT_DIR" --model base -done - -echo "批量处理完成!" -``` - -```bash -chmod +x batch_transcribe.sh -./batch_transcribe.sh -``` - -## 在 Workflow 中使用 - -### 知识库处理流程 - -在 `kb-workflow` 技能中，如果视频没有字幕： - -```bash -# 步骤 4: 下载视频后 -# 检查是否有字幕文件 -if [ ! -f "${subtitle_file}" ]; then - echo "无字幕文件，使用 Whisper 生成..." - - # 使用 Whisper 生成字幕（后台执行） - python /path/to/whisper.py \ - --input "${video_path}" \ - --model base \ - --output "${content_dir}" \ - --background - - # 等待 Whisper 完成 - sleep 5 # 等待后台任务启动 - while ! [ -f "${content_dir}/$(basename ${video_path} .mp4).srt" ]; do - sleep 2 - done - - echo "Whisper 处理完成" -fi -``` - -## 输出文件使用 - -### JSON 格式 - -```python -import json - -# 读取 JSON 结果 -with open('video.json', 'r', encoding='utf-8') as f: - data = json.load(f) - -# 访问分段 -for segment in data['segments']: - print(f"[{segment['start']:.2f}-{segment['end']:.2f}] {segment['text']}") -``` - -### SRT 格式 - -```bash -# 直接在视频播放器中使用 -mpv video.mp4 --sub-file=video.srt - -# 或转换字幕格式 -ffmpeg -i video.srt video.ass -``` - -### TXT 格式 - -```bash -# 搜索关键词 -grep "重要信息" video.txt - -# 统计字数 -wc -m video.txt -``` - -## 性能优化建议 - -### 1. 根据内容选择模型 - -```bash -# 快速预览（短视频、测试） -python whisper.py --input video.mp4 --model tiny - -# 日常使用（播客、会议） -python whisper.py --input video.mp4 --model base - -# 高质量需求（正式内容、多语言） -python whisper.py --input video.mp4 --model small -``` - -### 2. 使用后台执行 - -```bash -# 处理长文件时使用后台模式 -python whisper.py --input long_video.mp4 --model base --background -``` - -### 3. 指定语言提升准确度 - -```bash -# 如果知道内容语言，指定后准确度更高 -python whisper.py --input video.mp4 --language zh -``` - -## 故障排查 - -### 问题 1: whisper.cpp 未找到 - -```bash -# 检查环境变量 -echo $WHISPER_CPP_PATH - -# 或配置 -clawdbot skills config kb-whisper WHISPER_CPP_PATH "~/whisper.cpp" -``` - -### 问题 2: 模型未下载 - -```bash -cd ~/whisper.cpp -./models/download-ggml-model.sh base -``` - -### 问题 3: 音频格式不支持 - -```bash -# 转换格式 -ffmpeg -i input.avi -acodec libmp3lame output.mp3 -python whisper.py --input output.mp3 -``` - -### 问题 4: 后台任务未完成 - -```bash -# 查看日志 -cat video_whisper.log - -# 检查进程 -ps aux | grep whisper -``` - -## 集成到其他工具 - -### 集成到 Python 脚本 - -```python -import subprocess -import json - -def transcribe_audio(audio_path, model='base'): - """调用 whisper.py 转录音频""" - cmd = [ - 'python3', '/path/to/whisper.py', - '--input', audio_path, - '--model', model - ] - - result = subprocess.run(cmd, capture_output=True, text=True) - return json.loads(result.stdout) - -# 使用 -result = transcribe_audio('podcast.mp3', 'base') -print(f"识别到 {result['total_segments']} 个片段") -``` - -### 集成到 Shell 脚本 - -```bash -#!/bin/bash - -transcribe() { - local file=$1 - local model=${2:-base} - - echo "转录: $file" - python3 whisper.py --input "$file" --model "$model" --background -} - -# 批量处理 -transcribe "video1.mp4" -transcribe "video2.mp4" "tiny" -``` diff --git a/skills/kb-whisper/whisper.py b/skills/kb-whisper/whisper.py deleted file mode 100644 index 4d95f3a..0000000 --- a/skills/kb-whisper/whisper.py +++ /dev/null @@ -1,403 +0,0 @@ -#!/usr/bin/env python3 -""" -KB Whisper - 本地语音识别工具 -基于 Whisper.cpp 的音视频转文字 - -功能： -- 自动将视频/音频转换为 16kHz WAV 格式 -- 支持指定语言（默认中文） -- 输出标准 JSON 格式供 KB Workflow 使用 -""" - -import argparse -import json -import os -import subprocess -import sys -import tempfile -from pathlib import Path -import re - - -def parse_arguments(): - """解析命令行参数""" - parser = argparse.ArgumentParser( - description='KB Whisper - 本地语音识别工具', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -示例: - python whisper.py --input video.mp4 - python whisper.py --input video.mp4 --model base - python whisper.py --input video.mp4 --output ./output --background - """ - ) - parser.add_argument( - '--input', - required=True, - help='输入音视频文件路径' - ) - parser.add_argument( - '--model', - default='base', - choices=['tiny', 'base', 'small', 'medium', 'large'], - help='Whisper 模型类型 (默认: base)' - ) - parser.add_argument( - '--output', - help='输出目录 (默认: 输入文件所在目录)' - ) - parser.add_argument( - '--background', - action='store_true', - help='后台异步执行' - ) - parser.add_argument( - '--language', - default='zh', - help='指定语言 (默认: zh)，例如: en, zh, ja' - ) - - return parser.parse_args() - - -def get_whisper_cpp_path(): - """获取 whisper.cpp 路径""" - # 优先从环境变量获取 - path = os.environ.get('WHISPER_CPP_PATH') - if path: - path = os.path.expanduser(path) - if os.path.exists(path): - return path - - # 尝试常见位置 - common_paths = [ - os.path.expanduser('~/whisper.cpp'), - '/usr/local/bin/whisper', - '/opt/whisper.cpp', - ] - - for cp in common_paths: - if os.path.exists(cp): - return cp - - raise FileNotFoundError( - "未找到 whisper.cpp。请设置环境变量 WHISPER_CPP_PATH " - "或安装到 ~/whisper.cpp\n" - "安装方法: git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp && cd ~/whisper.cpp && make" - ) - - -def check_model(whisper_path, model): - """检查模型是否存在""" - model_path = os.path.join(whisper_path, 'models', f'ggml-{model}.bin') - if not os.path.exists(model_path): - raise FileNotFoundError( - f"模型文件不存在: {model_path}\n" - f"请运行以下命令下载模型:\n" - f"cd {whisper_path} && ./models/download-ggml-model.sh {model}" - ) - return model_path - - -def convert_to_wav(input_path, output_dir): - """将音视频转换为 16kHz WAV 格式 - - Whisper 需要 WAV 格式的输入文件。 - """ - input_path = os.path.abspath(input_path) - - # 如果已经是 WAV 且采样率正确，直接返回 - if input_path.lower().endswith('.wav'): - try: - result = subprocess.run( - ['ffprobe', '-v', 'error', '-show_entries', - 'stream=sample_rate', '-of', 'default=noprint_wrappers=1:nokey=1', - input_path], - capture_output=True, text=True - ) - if result.returncode == 0 and int(result.stdout.strip()) == 16000: - print(f"输入文件已是 16kHz WAV 格式，直接使用") - return input_path - except Exception: - pass - - # 转换 - output_path = os.path.join(output_dir, f"{Path(input_path).stem}_audio.wav") - - print(f"转换音频格式: {input_path} -> {output_path}") - - try: - subprocess.run([ - 'ffmpeg', '-y', '-i', input_path, - '-acodec', 'pcm_s16le', - '-ar', '16000', - '-ac', '1', - '-loglevel', 'error', - output_path - ], check=True, capture_output=True) - print("音频转换完成") - return output_path - except subprocess.CalledProcessError as e: - print(f"音频转换失败: {e}") - raise RuntimeError(f"无法转换音频格式: {e}") - except FileNotFoundError: - raise RuntimeError("未找到 ffmpeg，请安装: sudo apt-get install ffmpeg") - - -def run_whisper(whisper_path, audio_path, model, output_dir, language='zh'): - """运行 Whisper 语音识别 - - Args: - whisper_path: whisper.cpp 目录 - audio_path: WAV 音频文件路径 - model: 模型名称 - output_dir: 输出目录 - language: 语言代码 - """ - # 输出文件名（使用原始输入文件名，不是 audio.wav） - # 从 output_dir 和 audio_path 推断原始文件名 - original_stem = Path(audio_path).stem.replace('_audio', '') - output_basename = original_stem - - # 构建命令 - cmd = [ - os.path.join(whisper_path, 'whisper-cli'), - '-m', os.path.join(whisper_path, 'models', f'ggml-{model}.bin'), - '-f', audio_path, - '-of', os.path.join(output_dir, output_basename), - '-l', language, - '--output-txt', - '--output-srt', - '--output-json', - ] - - print(f"执行命令: {' '.join(cmd)}") - - # 运行 - result = subprocess.run( - cmd, - capture_output=True, - text=True, - cwd=whisper_path, - timeout=3600 # 1小时超时 - ) - - if result.returncode != 0: - # 检查是否有警告但仍成功 - if "error" in result.stderr.lower() and not os.path.exists( - os.path.join(output_dir, f"{output_basename}.txt") - ): - raise RuntimeError( - f"Whisper 执行失败:\n" - f"标准输出: {result.stdout}\n" - f"错误输出: {result.stderr}" - ) - - return result - - -def parse_srt_for_duration(srt_file): - """从 SRT 文件解析时长""" - if not os.path.exists(srt_file): - return 0 - - with open(srt_file, 'r', encoding='utf-8') as f: - content = f.read() - - # 找到最后一个时间戳 - pattern = r'(\d{2}:\d{2}:\d{2}),\d{3} --> \d{2}:\d{2}:\d{2},\d{3}' - matches = re.findall(pattern, content) - - if not matches: - return 0 - - # 解析最后一个时间戳 - last_time = matches[-1] - h, m, s = last_time.split(':') - return int(h) * 3600 + int(m) * 60 + int(s) - - -def create_workflow_json(input_file, output_dir, model, language, duration): - """创建标准格式的 JSON 输出，供 KB Workflow 使用""" - input_path = Path(input_file) - basename = input_path.stem - - txt_file = os.path.join(output_dir, f"{basename}.txt") - srt_file = os.path.join(output_dir, f"{basename}.srt") - json_file = os.path.join(output_dir, f"{basename}.json") - - # 读取文本内容 - text_content = "" - if os.path.exists(txt_file): - with open(txt_file, 'r', encoding='utf-8') as f: - text_content = f.read().strip() - - # 解析 SRT 获取分段 - segments = [] - if os.path.exists(srt_file): - with open(srt_file, 'r', encoding='utf-8') as f: - content = f.read() - - # SRT 格式: 序号 -> 时间 -> 文本 - pattern = r'(\d+)\n(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\n(.+?)(?=\n\n|\Z)' - matches = re.findall(pattern, content, re.DOTALL) - - def parse_time(t): - h, m, s_ms = t.split(':') - s, ms = s_ms.split(',') - return float(h) * 3600 + float(m) * 60 + float(s) + float(ms) / 1000 - - for idx, (num, start, end, text) in enumerate(matches): - segments.append({ - "id": idx, - "start": round(parse_time(start), 2), - "end": round(parse_time(end), 2), - "text": text.strip() - }) - - # 构建 JSON - workflow_json = { - "success": True, - "tool": "whisper", - "model": model, - "language": language, - "input_file": str(input_path), - "output_files": { - "txt": f"{basename}.txt", - "srt": f"{basename}.srt", - "json": f"{basename}.json" - }, - "duration_seconds": duration, - "segments": segments, - "text": text_content - } - - # 写入 JSON - with open(json_file, 'w', encoding='utf-8') as f: - json.dump(workflow_json, f, ensure_ascii=False, indent=2) - - return workflow_json - - -def main(): - """主函数""" - args = parse_arguments() - - # 检查输入文件 - input_file = os.path.abspath(args.input) - if not os.path.exists(input_file): - print(f"错误: 输入文件不存在: {input_file}", file=sys.stderr) - sys.exit(1) - - # 获取输出目录 - if args.output: - output_dir = os.path.abspath(args.output) - else: - output_dir = os.path.dirname(input_file) - - os.makedirs(output_dir, exist_ok=True) - - # 获取 whisper.cpp 路径 - try: - whisper_path = get_whisper_cpp_path() - print(f"✓ Whisper.cpp 路径: {whisper_path}") - except FileNotFoundError as e: - print(f"✗ 错误: {e}", file=sys.stderr) - sys.exit(1) - - # 检查模型 - try: - check_model(whisper_path, args.model) - print(f"✓ 使用模型: {args.model}") - except FileNotFoundError as e: - print(f"✗ 错误: {e}", file=sys.stderr) - sys.exit(1) - - # 后台执行 - if args.background: - print("启动后台处理...") - log_file = os.path.join(output_dir, f"{Path(input_file).stem}_whisper.log") - - with open(log_file, 'w') as log: - subprocess.Popen( - [sys.executable, __file__, - '--input', input_file, - '--model', args.model, - '--output', output_dir, - '--language', args.language], - stdout=log, - stderr=log, - start_new_session=True - ) - - print(f"✓ 后台任务已启动") - print(f" 日志: {log_file}") - print(f" 输出: {output_dir}") - - result = { - "success": True, - "background": True, - "status": "running", - "log_file": log_file, - "output_dir": output_dir - } - print(json.dumps(result, ensure_ascii=False, indent=2)) - return - - # 前台执行 - print(f"\n📁 处理文件: {input_file}") - print(f"📁 输出目录: {output_dir}") - - try: - # 1. 转换为 WAV - print(f"\n🔄 步骤1: 转换音频格式...") - audio_path = convert_to_wav(input_file, output_dir) - - # 2. 运行 Whisper - print(f"\n🎙️ 步骤2: 语音识别 (语言: {args.language})...") - run_whisper(whisper_path, audio_path, args.model, output_dir, args.language) - - # 3. 清理临时 WAV 文件 - if audio_path != input_file and audio_path.endswith('_audio.wav'): - try: - os.remove(audio_path) - print("✓ 清理临时文件") - except Exception: - pass - - # 4. 获取时长并创建 JSON - print(f"\n📝 步骤3: 生成输出文件...") - basename = Path(input_file).stem - srt_file = os.path.join(output_dir, f"{basename}.srt") - duration = parse_srt_for_duration(srt_file) - - workflow_json = create_workflow_json( - input_file, output_dir, args.model, args.language, duration - ) - - # 输出结果 - print(f"\n✅ 处理完成!") - print(f" 时长: {duration} 秒") - print(f" 分段数: {len(workflow_json['segments'])}") - print(f"\n📄 输出文件:") - for fmt in ['txt', 'srt', 'json']: - fpath = os.path.join(output_dir, f"{basename}.{fmt}") - if os.path.exists(fpath): - size = os.path.getsize(fpath) - print(f" • {fmt.upper()}: {fpath} ({size:,} bytes)") - - print(f"\n📋 Workflow JSON:") - print(json.dumps({ - "success": workflow_json["success"], - "tool": "whisper", - "text_length": len(workflow_json["text"]), - "segments_count": len(workflow_json["segments"]) - }, ensure_ascii=False, indent=2)) - - except Exception as e: - print(f"\n✗ 错误: {e}", file=sys.stderr) - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/skills/kb-workflow/README.md b/skills/kb-workflow/README.md deleted file mode 100644 index 9fda761..0000000 --- a/skills/kb-workflow/README.md +++ /dev/null @@ -1,247 +0,0 @@ -# KB Workflow 环境安装脚本 - -一键检查并安装 kb-workflow 知识库工作流所需的全部环境依赖。 - -## 功能特性 - -- ✅ 自动检测系统类型 (Ubuntu/Debian/Fedora/Arch) -- ✅ 检查系统依赖 (Python3, pip3, ffmpeg, tesseract, git) -- ✅ 检查 Python 依赖包 (httpx, aiohttp, playwright, easyocr 等) -- ✅ 自动创建知识库目录结构 -- ✅ 生成环境变量配置示例 -- ✅ 汇总显示所有需要执行的安装命令 - -## 使用方法 - -### 运行检查脚本 - -```bash -cd /home/starmen1265/.openclaw/workspace/skills -./install-kb-workflow.sh -``` - -脚本会检查所有依赖，并显示需要执行的安装命令。 - -### 示例输出 - -``` -======================================== -第一步: 检查系统依赖 -======================================== -[SUCCESS] Python3 已安装: Python 3.12.3 -[WARN] pip3 未安装 -[WARN] ffmpeg 未安装 -... - -======================================== -需要安装的系统依赖 -======================================== - -检测到以下依赖未安装: - - python3-pip - - ffmpeg - - tesseract-ocr - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -请运行以下命令安装: -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -sudo apt-get update -sudo apt-get install -y python3-pip ffmpeg tesseract-ocr tesseract-ocr-chi-sim tesseract-ocr-eng -``` - -## 依赖清单 - -### 系统依赖 - -| 依赖 | 用途 | 安装命令 | -|------|------|----------| -| **python3** | Python 运行时 | `sudo apt-get install python3` | -| **python3-pip** | Python 包管理器 | `sudo apt-get install python3-pip` | -| **ffmpeg** | 视频处理（OCR 截帧） | `sudo apt-get install ffmpeg` | -| **tesseract-ocr** | OCR 引擎（备用方案） | `sudo apt-get install tesseract-ocr` | -| **git** | 版本控制（归档提交） | `sudo apt-get install git` | - -### Tesseract 语言包 - -```bash -# Ubuntu/Debian -sudo apt-get install tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-eng - -# Fedora/RHEL -sudo dnf install tesseract-langpack-chi_sim tesseract-langpack-chi_tra tesseract-langpack-eng -``` - -### Python 依赖 - -```bash -# 基础 HTTP 库 -pip3 install --user httpx aiohttp requests - -# 图像和 OCR -pip3 install --user Pillow easyocr pytesseract - -# 浏览器自动化 -pip3 install --user playwright -python3 -m playwright install chromium -``` - -## 目录结构 - -安装完成后会创建以下目录结构： - -``` -~/work/knowledge-base/ # 知识库根目录（可配置） -├── archive/ # 归档目录 -│ ├── pending/ # 待处理 -│ └── subscriptions/ # 订阅信息 -├── metadata/ # 元数据文件 -├── content/ # 内容文件 -│ ├── downloads/ # 视频下载 -│ └── subtitles/ # 字幕文件 -└── analyzed/ # 深度分析报告 -``` - -## 环境变量 - -### 必需 -- `KB_BASE_PATH` - 知识库根目录（默认: `~/work/knowledge-base`） - -### 可选 -- `BILIBILI_COOKIE` - B站 Cookie，用于下载会员视频 -- `AI_API_KEY` - AI API 密钥，用于深度内容分析 -- `AI_PROVIDER` - AI 提供商（openai/zhipu/minimax） -- `AI_MODEL` - AI 模型名称（如 gpt-4） -- `AI_BASE_URL` - 自定义 API 端点 -- `KB_GIT_USER` - Git 用户名（用于归档提交） -- `KB_GIT_EMAIL` - Git 邮箱（用于归档提交） -- `KB_GIT_REMOTE` - Git 远程仓库地址 - -### 配置方法 - -将以下内容添加到 `~/.bashrc` 或 `~/.zshrc`： - -```bash -# KB Workflow 配置 -export KB_BASE_PATH="~/work/knowledge-base" - -# 可选：B站 Cookie -export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx" - -# 可选：AI 配置 -export AI_API_KEY="your-api-key" -export AI_PROVIDER="openai" -export AI_MODEL="gpt-4" -``` - -然后重新加载配置： -```bash -source ~/.bashrc # 或 source ~/.zshrc -``` - -## 故障排除 - -### 1. pip3 命令不存在 - -```bash -# 方法 1: 安装 pip3 -sudo apt-get install python3-pip - -# 方法 2: 使用 python3 -m pip -python3 -m pip install --user -``` - -### 2. Playwright 浏览器安装失败 - -```bash -# 手动安装 Chromium -python3 -m playwright install chromium - -# 如果仍然失败，尝试安装依赖 -sudo apt-get install libnss3 libatk-bridge2.0-0 libxss1 libgtk-3-0 -``` - -### 3. EasyOCR 安装失败 - -EasyOCR 依赖 PyTorch，可能需要较长时间安装： - -```bash -# 先安装 PyTorch -pip3 install --user torch torchvision --index-url https://download.pytorch.org/whl/cpu - -# 再安装 easyocr -pip3 install --user easyocr -``` - -### 4. 权限问题 - -如果使用 `--user` 安装的包无法找到，检查 PATH： - -```bash -# 查看用户安装路径 -python3 -m site --user-base - -# 添加到 PATH（添加到 ~/.bashrc） -export PATH="$PATH:$(python3 -m site --user-base)/bin" -``` - -### 5. OCR 中文识别问题 - -确保已安装中文语言包： - -```bash -# 检查已安装的语言包 -tesseract --list-langs - -# 应该能看到 chi_sim 和 chi_tra -``` - -## 更新 - -如果技能更新了依赖，可以重新运行检查脚本： - -```bash -./install-kb-workflow.sh -``` - -## 子技能清单 - -| 技能 | 路径 | 功能 | -|------|------|------| -| kb-workflow | `kb-workflow/kb_workflow.py` | 主流程协调 | -| kb-link-parser | `kb-link-parser/link-parser.py` | 链接解析 | -| kb-bilibili-metadata | `kb-bilibili-metadata/bilibili_metadata.py` | B站元数据提取 | -| kb-bilibili-content | `kb-bilibili-content/bilibili_content.py` | B站内容下载 | -| kb-ocr-processor | `kb-ocr-processor/ocr_processor.py` | 视频 OCR 识别 | -| kb-deep-analyzer | `kb-deep-analyzer/deep_analyzer.py` | AI 深度分析 | - -## 使用 kb-workflow - -安装完成后，可以通过以下方式使用： - -### 方式 1：通过主代理 - -直接向主代理发送消息： -``` -添加到知识库 https://www.bilibili.com/video/BVxxxxx -``` - -### 方式 2：手动运行子技能 - -```bash -# 1. 链接解析 -python3 kb-link-parser/link-parser.py --url "https://www.bilibili.com/video/BVxxxxx" - -# 2. 元数据提取 -python3 kb-bilibili-metadata/bilibili_metadata.py --id "BVxxxxx" --type video - -# 3. 内容下载 -python3 kb-bilibili-content/bilibili_content.py --id "BVxxxxx" --type subtitle-download - -# 4. 深度分析 -python3 kb-deep-analyzer/deep_analyzer.py --metadata meta.json --content content.json --output analysis.md -``` - -## 许可证 - -MIT License diff --git a/skills/kb-workflow/SKILL.md b/skills/kb-workflow/SKILL.md index a4c8573..8b536d9 100644 --- a/skills/kb-workflow/SKILL.md +++ b/skills/kb-workflow/SKILL.md @@ -60,7 +60,7 @@ echo "✅ 环境变量检查通过" ## Workflow 执行方式 -### 方式一：一键处理（推荐） +### 一键处理（推荐） 使用管道 CLI 一键完成所有步骤： @@ -85,21 +85,6 @@ python3 kb-workflow/kb_workflow.py process "https://..." -o /path/to/output python3 kb-workflow/kb_workflow.py process "https://..." --no-skip ``` -### 方式二：手动分步执行（高级） - -如需更细粒度控制，可手动调用各子技能： - -```bash -# 1. 查重 -python3 kb-workflow/kb_workflow.py --check --url "https://..." - -# 2. 一键处理 -python3 kb-workflow/kb_workflow.py process "https://..." - -# 3. 归档 -python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md" -``` - ## 支持的平台和内容类型 | 平台 | 内容类型 | URL 示例 | @@ -152,22 +137,6 @@ python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md" 3. **禁止降级处理**：不能使用预设模板或简单摘要来"假装成功" 4. **每次执行都要使用 AI**：必须有真实的 AI API 调用记录 -## 辅助命令 - -```bash -# 查重 -python3 kb-workflow/kb_workflow.py --check --url "https://..." - -# 获取目录路径 -python3 kb-workflow/kb_workflow.py --base-path -python3 kb-workflow/kb_workflow.py --metadata -python3 kb-workflow/kb_workflow.py --content -python3 kb-workflow/kb_workflow.py --analyzed - -# 归档分析报告 -python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md" -``` - ## 示例 ``` diff --git a/skills/kb-workflow/SkillBug.md b/skills/kb-workflow/SkillBug.md deleted file mode 100644 index ae5df03..0000000 --- a/skills/kb-workflow/SkillBug.md +++ /dev/null @@ -1,101 +0,0 @@ -# SkillBug.md - -## 问题记录 - -### 2026-02-10: 子代理权限问题 - -**问题：** `sessions_spawn` 调用子代理时返回 `forbidden` 错误。 - -**原因：** OpenClaw 配置中 allowlist 为空，Python 脚本没有权限调用子代理。 - -**解决方案：** 改用 Prompt-based 模式（方案B）。 - ---- - -## 架构重构 - -### 之前（Script-based） - -``` -kb_workflow.py (Python脚本) - ↓ -sessions_spawn() # ❌ 失败（脚本没有权限） -``` - -### 之后（Prompt-based） - -``` -主代理读取 SKILL.md - ↓ -按照指导调用 sessions_spawn # ✅ 成功（主代理有权限） -``` - -### 文件变更 - -| 文件 | 变更 | 说明 | -|------|------|------| -| `SKILL.md` | 重写 | 详细定义 workflow 步骤，主代理直接读取执行 | -| `kb_workflow.py` | 简化 | 保留为辅助工具函数库（查重、归档、Git） | -| `skill_main.py` | 简化 | 只做触发检测，返回指导信息 | -| `SkillBug.md` | 更新 | 记录架构变更 | - -### SKILL.md 结构 - -``` -1. 触发条件 -2. 步骤 1: 查重 -3. 步骤 2: kb-link-parser -4. 步骤 3: 元数据技能 -5. 步骤 4: 内容下载 -6. 步骤 5: OCR（30分钟超时） -7. 步骤 6: 深度分析 -8. 步骤 7: 归档和 Git -9. 步骤 8: 汇报完成 -10. 目录结构 -11. 支持平台 -12. 注意事项 -``` - -### kb_workflow.py 保留的函数 - -```python -extract_platform(filename) # 从文件名提取平台 -extract_base_name(filename) # 从文件名提取基础名称 -get_knowledge_base_path() # 获取知识库根目录 -ensure_directories(base_path) # 确保目录存在 -check_duplicate(url) # 查重 -archive_and_git(analyzed_file) # 归档和 Git -generate_summary(...) # 生成汇报 -``` - ---- - -### 2026-02-12: B站视频下载失败 - 403 Forbidden - -**问题：** 使用 kb-bilibili-content 下载视频时返回 403 Forbidden 错误。 - -**原因：** -1. 视频可能需要B站会员权限 -2. 或者需要登录cookie才能下载 -3. 当前未配置 `BILIBILI_COOKIE` 环境变量 - -**复现场景：** -- 链接：https://www.bilibili.com/video/BV1Z9nXzUEhv -- 下载命令：`python3 bilibili_content.py --id "BV1Z9nXzUEhv" --type video` -- 结果：返回403错误页面，文件只有548字节 - -**影响：** 无法进行后续的OCR处理和深度分析 - -**建议解决方案：** -1. 配置 `BILIBILI_COOKIE` 环境变量 -2. 或提供cookies文件路径 -3. 或跳过OCR步骤，使用元数据+简介进行深度分析 - -**当前进度：** -- ✅ 步骤1：环境变量检查完成 -- ✅ 步骤2：链接解析成功（bilibili_BV1Z9nXzUEhv） -- ✅ 步骤3：元数据提取成功（标题：新加坡分区漫游指南...） -- ❌ 步骤4：视频下载失败（403） -- ⏸️ 步骤5：OCR处理（阻塞） -- ⏸️ 步骤6：深度分析（阻塞） -- ⏸️ 步骤7：归档（阻塞） diff --git a/skills/kb-workflow/kb_workflow.py b/skills/kb-workflow/kb_workflow.py index 540b0ae..1851637 100644 --- a/skills/kb-workflow/kb_workflow.py +++ b/skills/kb-workflow/kb_workflow.py @@ -3,330 +3,24 @@ """ KB Workflow - Knowledge Base Workflow Orchestrator -知识库工作流辅助工具函数。 +知识库工作流管道入口。 -此模块提供查重、文件操作、Git 管理等工具函数， -供主代理在执行 SKILL.md 中定义的 workflow 时调用。 +此模块提供 process_command() 函数作为管道入口， +内部使用 kb_pipeline 包实现完整的内容处理流程。 + +使用方式: + python kb_workflow.py process "https://www.bilibili.com/video/BVxxx" + python kb_workflow.py process "https://..." -o /path/to/output """ -import os -import sys -import json import argparse -import subprocess -import shutil -import re -from datetime import datetime +import asyncio +import json +import sys from pathlib import Path -from typing import Dict, List, Optional, Any +from typing import Any, Dict, Optional -# ==================== 子技能映射 ==================== - -METADATA_AGENTS = { - "bilibili": "kb-bilibili-metadata", - "zhihu": "kb-zhihu-metadata", -} - -CONTENT_AGENTS = { - "bilibili": "kb-bilibili-content", - "zhihu": "kb-zhihu-content", -} - - -# ==================== 内容类型判断 ==================== - -# 需要下载视频/字幕的内容类型 -VIDEO_CONTENT_TYPES = {"video", "zvideo"} - -# 需要提取文字的内容类型 -TEXT_CONTENT_TYPES = {"article", "zhuanlan", "question", "answer"} - -# 需要下载图片的内容类型 -IMAGE_CONTENT_TYPES = {"note"} - - -def is_video_content(content_type: str) -> bool: - """判断是否为视频内容类型""" - return content_type in VIDEO_CONTENT_TYPES - - -def is_text_content(content_type: str) -> bool: - """判断是否为文字内容类型（专栏、文章、问答等）""" - return content_type in TEXT_CONTENT_TYPES - - -def is_image_content(content_type: str) -> bool: - """判断是否为图片内容类型（小红书等）""" - return content_type in IMAGE_CONTENT_TYPES - - -def get_content_handler(content_type: str) -> str: - """ - 获取内容处理方式 - - Returns: - "video" - 需要下载视频/字幕 - "text" - 需要提取文字 - "image" - 需要下载图片 - "unknown" - 未知类型 - """ - if is_video_content(content_type): - return "video" - elif is_text_content(content_type): - return "text" - elif is_image_content(content_type): - return "image" - else: - return "unknown" - - -# ==================== 工具函数 ==================== - -def extract_platform(filename: str) -> str: - """从文件名提取平台（第一个下划线之前）""" - match = re.match(r'^([a-zA-Z0-9]+)_(.+)$', filename) - if match: - return match.group(1) - return "unknown" - - -def extract_base_name(filename: str) -> str: - """从文件名提取基础名称（去掉扩展名）""" - return Path(filename).stem - - -def get_knowledge_base_path() -> Path: - """获取知识库根目录""" - base_path_str = os.environ.get('KB_BASE_PATH', '') - if base_path_str: - return Path(base_path_str).expanduser() - return Path.home() / "work" / "knowledge-base" - - -def get_archive_path() -> Path: - """获取归档目录（AI分析文档存放位置）""" - archive_path_str = os.environ.get('KB_ARCHIVE_PATH', '') - if archive_path_str: - return Path(archive_path_str).expanduser() - # 默认：~/work/zq/30.资源/AI分析 - return Path.home() / "work" / "zq" / "30.资源" / "AI分析" - - -def get_git_root() -> Path: - """获取 Git 仓库根目录（用于归档提交）""" - git_root_str = os.environ.get('KB_GIT_ROOT', '') - if git_root_str: - return Path(git_root_str).expanduser() - # 默认：~/work/zq - return Path.home() / "work" / "zq" - - -def ensure_directories(base_path: Path = None) -> Dict[str, Path]: - """确保目录存在""" - if base_path is None: - base_path = get_knowledge_base_path() - - dirs = { - "base": base_path, - "pending": base_path / "pending", - "metadata": base_path / "metadata", - "content": base_path / "content", - "content_downloads": base_path / "content" / "downloads", - "analyzed": base_path / "analyzed", - "archive": base_path / "archive", - "archive_subscriptions": base_path / "archive" / "subscriptions", - } - - for name, dir_path in dirs.items(): - if isinstance(dir_path, Path): - dir_path.mkdir(parents=True, exist_ok=True) - - return dirs - - -def check_duplicate(base_path: Path = None, url: str = None, platform: str = None, content_id: str = None) -> Dict[str, Any]: - """查重检查""" - if base_path is None: - base_path = get_knowledge_base_path() - - metadata_dir = base_path / "metadata" - archive_dir = base_path / "archive" - - # 从URL提取ID - if url and not content_id: - bv_match = re.search(r'BV[a-zA-Z0-9]+', url) - content_id = bv_match.group() if bv_match else None - - result = {"duplicate": False, "status": None, "files": []} - - if metadata_dir.exists(): - for f in metadata_dir.iterdir(): - if f.is_file() and content_id and content_id in f.name: - result["duplicate"] = True - result["status"] = "metadata" - result["files"].append(str(f)) - - if archive_dir.exists(): - for date_dir in archive_dir.iterdir(): - if date_dir.is_dir(): - for f in date_dir.iterdir(): - if f.is_file() and content_id and content_id in f.name: - result["duplicate"] = True - result["status"] = "archived" - result["files"].append(str(f)) - - return result - - -def archive_and_git( - analyzed_file: str, - base_path: Path = None, - git_root: Path = None, - git_user: str = None, - git_email: str = None, - git_remote: str = None -) -> Dict[str, Any]: - """归档并提交Git""" - if base_path is None: - base_path = get_knowledge_base_path() - if git_root is None: - git_root = get_git_root() - if git_user is None: - git_user = os.environ.get('KB_GIT_USER', '') - if git_email is None: - git_email = os.environ.get('KB_GIT_EMAIL', '') - if git_remote is None: - git_remote = os.environ.get('KB_GIT_REMOTE', '') - - result = { - "success": False, - "archive_path": "", - "git_commit": False, - "git_push": None, - "error": None - } - - try: - # 1. 检查并初始化 Git 仓库 - git_dir = git_root / ".git" - if not git_dir.exists(): - # 初始化仓库 - subprocess.run(['git', 'init'], cwd=str(git_root), capture_output=True) - subprocess.run(['git', 'config', 'user.name', git_user], cwd=str(git_root)) - subprocess.run(['git', 'config', 'user.email', git_email], cwd=str(git_root)) - - # 关联远端仓库 - if git_remote: - subprocess.run(['git', 'remote', 'add', 'origin', git_remote], cwd=str(git_root), capture_output=True) - # 拉取远端分支 - subprocess.run(['git', 'fetch', 'origin'], cwd=str(git_root), capture_output=True) - # 设置 main 分支并拉取 - subprocess.run(['git', 'branch', '--set-upstream-to=origin/main', 'main'], cwd=str(git_root), capture_output=True) - subprocess.run(['git', 'pull', 'origin', 'main', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120) - else: - # 已有仓库，检查远端是否配置 - if git_remote: - remote_check = subprocess.run(['git', 'remote', '-v'], cwd=str(git_root), capture_output=True, text=True) - if 'origin' not in remote_check.stdout: - subprocess.run(['git', 'remote', 'add', 'origin', git_remote], cwd=str(git_root), capture_output=True) - - # 拉取远端最新代码 - subprocess.run(['git', 'fetch', 'origin'], cwd=str(git_root), capture_output=True) - # 尝试拉取 main 分支，如果失败则尝试 master - pull_result = subprocess.run(['git', 'pull', 'origin', 'main', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120) - if pull_result.returncode != 0: - subprocess.run(['git', 'pull', 'origin', 'master', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120) - - # 2. 创建归档目录 - date_str = datetime.now().strftime('%Y-%m-%d') - archive_base = get_archive_path() - date_dir = archive_base / date_str - date_dir.mkdir(parents=True, exist_ok=True) - - # 3. 复制分析报告 - src_file = Path(analyzed_file) - archive_file = date_dir / src_file.name - shutil.copy(src_file, archive_file) - result["archive_path"] = str(archive_file) - - # 4. 复制元数据 - meta_file = base_path / "metadata" / f"{src_file.stem}.meta.json" - if meta_file.exists(): - shutil.copy(meta_file, date_dir / meta_file.name) - - # 5. Git 提交并推送 - if (git_root / ".git").exists(): - # 添加所有变更 - subprocess.run(['git', 'add', '.'], cwd=str(git_root), capture_output=True) - - # 检查是否有变更需要提交 - status_result = subprocess.run(['git', 'status', '--porcelain'], cwd=str(git_root), capture_output=True, text=True) - - if status_result.stdout.strip(): - commit_msg = f"Archive: {src_file.name} - {datetime.now().isoformat()}" - commit_result = subprocess.run( - ['git', 'commit', '-m', commit_msg], - cwd=str(git_root), capture_output=True, text=True - ) - result["git_commit"] = commit_result.returncode == 0 - - # 推送到远端 - if git_remote and commit_result.returncode == 0: - push_result = subprocess.run( - ['git', 'push', 'origin', 'main'], - cwd=str(git_root), capture_output=True, text=True, timeout=60 - ) - if push_result.returncode != 0: - # 如果 main 分支推送失败，尝试 master - push_result = subprocess.run( - ['git', 'push', 'origin', 'master'], - cwd=str(git_root), capture_output=True, text=True, timeout=60 - ) - result["git_push"] = {"success": push_result.returncode == 0, "output": push_result.stdout} - - result["success"] = True - - except Exception as e: - result["error"] = str(e) - - return result - - -def generate_summary(url: str, stages: List[Dict], final_file: str, archive_result: Dict = None) -> str: - """生成处理完成汇报""" - # 提取视频标题（如果有） - title_match = re.search(r'com/video/([^?]+)', url) - title = title_match.group(1) if title_match else "未知" - - summary = f""" -✅ **处理完成！** - -**视频:** {title} - -**处理阶段:** -""" - for stage in stages: - stage_name = stage.get('stage', 'unknown') - emoji = "✓" if stage.get('success') else "○" - summary += f"{emoji} {stage_name}\n" - - if final_file: - summary += f"\n**最终文件:** `{final_file}`" - - if archive_result: - if archive_result.get('git_commit'): - summary += f"\n**Git:** 已提交" - if archive_result.get('git_push', {}).get('success'): - summary += " ✓" - elif archive_result.get('git_push'): - summary += " ✗" - - return summary - - -# ==================== CLI Process Command ==================== - def process_command( url: str, output_dir: Optional[str] = None, @@ -342,8 +36,6 @@ def process_command( Returns: Dictionary with success status, content_dir, and results """ - import asyncio - # Add project root to path for kb_pipeline imports project_root = Path(__file__).parent.parent if str(project_root) not in sys.path: @@ -351,7 +43,7 @@ def process_command( try: from kb_pipeline.pipeline import Pipeline - from kb_pipeline.processors import ParseProcessor, DownloadProcessor, ASRProcessor, AnalyzeProcessor + from kb_pipeline.processors import ASRProcessor, AnalyzeProcessor, DownloadProcessor, ParseProcessor from kb_pipeline.schemas import PipelineConfig async def run_pipeline(): @@ -402,24 +94,22 @@ def process_command( } -# ==================== CLI ==================== - def main(): - """CLI入口 - 用于命令行测试""" + """CLI入口 - 用于命令行调用""" parser = argparse.ArgumentParser( description='KB Workflow - Knowledge Base Workflow Orchestrator', formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" + epilog=''' 使用示例: - # 查重 - python kb_workflow.py --check --url "https://www.bilibili.com/video/BVxxx" - - # 归档 - python kb_workflow.py --archive "/path/to/analyzed.md" - # 处理URL (通过完整管道) python kb_workflow.py process "https://www.bilibili.com/video/BVxxx" - """ + + # 指定输出目录 + python kb_workflow.py process "https://..." -o /path/to/output + + # 强制重新执行（忽略已完成状态） + python kb_workflow.py process "https://..." --no-skip + ''' ) # Create subparsers for commands @@ -430,29 +120,12 @@ def main(): process_parser.add_argument('url', type=str, help='URL to process') process_parser.add_argument('--output-dir', '-o', type=str, help='Output directory for content') process_parser.add_argument('--skip-completed', action='store_true', default=True, - help='Skip completed steps (default: True)') + help='Skip completed steps (default: True)') process_parser.add_argument('--no-skip', action='store_false', dest='skip_completed', - help='Re-run all steps') + help='Re-run all steps') - # Legacy arguments (for backward compatibility) - parser.add_argument('--check', action='store_true', help='查重') - parser.add_argument('--url', type=str, help='要检查的URL') - parser.add_argument('--archive-file', type=str, help='归档的文件路径') - parser.add_argument('--base-path', type=str, help='获取知识库根目录') - - parser.add_argument('--pending', action='store_true', help='获取url解析目录') - parser.add_argument('--metadata', action='store_true', help='获取元数据目录') - parser.add_argument('--content', action='store_true', help='获取内容目录') - parser.add_argument('--content_downloads', action='store_true', help='获取主内容下载目录') - parser.add_argument('--analyzed', action='store_true', help='获取深度分析目录') - parser.add_argument('--archive_dir', action='store_true', help='获取归档目录') - parser.add_argument('--archive_subscriptions', action='store_true', help='获取订阅信息目录') - parser.add_argument('--extract_platform', type=str, help='要提取平台名称的文件名') args = parser.parse_args() - base_path = Path(args.base_path) if args.base_path else None - dirs = ensure_directories(base_path) - # Handle process subcommand if args.command == 'process': result = process_command( @@ -461,45 +134,6 @@ def main(): skip_completed=args.skip_completed ) print(json.dumps(result, ensure_ascii=False, indent=2)) - return - - # Legacy argument handling - if args.check and args.url: - result = check_duplicate(base_path, args.url) - print(json.dumps(result, ensure_ascii=False, indent=2)) - - elif args.archive_file: - result = archive_and_git(args.archive_file, base_path) - print(json.dumps(result, ensure_ascii=False, indent=2)) - - elif args.pending: - print(json.dumps({"path": str(dirs["pending"])}, ensure_ascii=False, indent=2)) - - elif args.metadata: - print(json.dumps({"path": str(dirs["metadata"])}, ensure_ascii=False, indent=2)) - - elif args.content: - print(json.dumps({"path": str(dirs["content"])}, ensure_ascii=False, indent=2)) - - elif args.content_downloads: - print(json.dumps({"path": str(dirs["content_downloads"])}, ensure_ascii=False, indent=2)) - - elif args.analyzed: - print(json.dumps({"path": str(dirs["analyzed"])}, ensure_ascii=False, indent=2)) - - elif args.archive_dir: - print(json.dumps({"path": str(get_archive_path())}, ensure_ascii=False, indent=2)) - - elif args.archive_subscriptions: - print(json.dumps({"path": str(dirs["archive_subscriptions"])}, ensure_ascii=False, indent=2)) - - elif args.extract_platform: - platform = extract_platform(args.extract_platform) - print(json.dumps({"platform": platform}, ensure_ascii=False, indent=2)) - - elif args.base_path: - print(json.dumps({"path": str(dirs["base"])}, ensure_ascii=False, indent=2)) - else: parser.print_help() diff --git a/skills/kb-zhihu-content/SKILL.md b/skills/kb-zhihu-content/SKILL.md deleted file mode 100644 index 9009c3e..0000000 --- a/skills/kb-zhihu-content/SKILL.md +++ /dev/null @@ -1,86 +0,0 @@ -# KB Zhihu Content Skill - -**kb-zhihu-content** - Download Zhihu videos and extract content. - -## Features - -- Video download using **yt-dlp** -- Subtitle extraction (returns no subtitle, triggers Whisper) -- **Read article content from MediaCrawler data** (NEW!) -- Supports `ZHIHU_COOKIE` environment variable - -## Environment Variables - -| Variable | Required | Description | -|----------|----------|-------------| -| `ZHIHU_COOKIE` | No | 知乎登录 Cookie (for member-only videos) | - -## Commands - -```bash -# Download video -python zhihu_content.py --id "12345678" --type video --output "/path/to/downloads/" - -# Extract subtitle (returns no subtitle, triggers Whisper) -python zhihu_content.py --id "12345678" --type subtitle --output "/path/to/subtitles/" - -# 从 MediaCrawler 读取文章内容 (⚡ NEW!) -python zhihu_content.py --type article-from-media --input "~/work/MediaCrawler/data/zhihu/json/detail_contents_2026-02-27.json" --output "~/work/knowledge-base/data/zhihu/" -``` - -## MediaCrawler 数据读取 - -**新功能**：从 MediaCrawler 爬取的数据中提取知乎文章内容 - -### 使用方法 - -```bash -# 从 MediaCrawler JSON 文件读取文章内容 -python zhihu_content.py --type article-from-media \ - --input "~/work/MediaCrawler/data/zhihu/json/detail_contents_2026-02-27.json" \ - --output "~/work/knowledge-base/data/zhihu/" - -# 查找最新的爬取数据 -python zhihu_content.py --type article-from-media \ - --output "~/work/knowledge-base/data/zhihu/" -``` - -### 参数说明 - -| 参数 | 说明 | -|------|------| -| `--type article-from-media` | 从 MediaCrawler 数据读取 | -| `--input` | MediaCrawler 输出文件路径（可选，不指定则自动查找最新） | -| `--output` | 输出目录 | - -### 输出格式 - -```json -{ - "success": true, - "type": "article", - "filename": "zhihu_article_如何高效的爬取微信公众号.md", - "filepath": "/home/xxx/work/knowledge-base/data/zhihu/zhihu_article_如何高效的爬取微信公众号.md", - "message": "文章内容已导出" -} -``` - -## 原有功能 - -### Video Download - -Video download uses **yt-dlp**: -1. Parses `ZHIHU_COOKIE` environment variable (optional) -2. Creates temporary Netscape-format cookie file -3. Runs `yt-dlp -f best -o ` -4. Cleans up temporary file - -**Note**: 知乎视频通常没有官方字幕，将自动触发 Whisper 语音识别 - -## Installation - -```bash -# Already included in kb-skills repo -# Ensure yt-dlp is installed -pip install yt-dlp aiohttp -``` diff --git a/skills/kb-zhihu-content/zhihu_content.py b/skills/kb-zhihu-content/zhihu_content.py deleted file mode 100644 index ff9c4b4..0000000 --- a/skills/kb-zhihu-content/zhihu_content.py +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -知乎内容获取工具 - -支持： -- zvideo: 知乎视频下载 -- article-from-media: 从 MediaCrawler 数据读取知乎文章 -- 从页面提取视频播放地址 -""" - -import argparse -import asyncio -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -from datetime import datetime -from pathlib import Path -from typing import Optional, Dict, Any, List - - -# MediaCrawler 目录 -MEDIACRAWLER_DIR = Path.home() / "work" / "MediaCrawler" - - -class ZhihuContent: - """知乎内容获取器""" - - def __init__(self, output_dir: str = "./output", cookie: str = None): - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - self.cookie = cookie - - def _create_cookie_file(self) -> str: - """创建 Netscape 格式的 Cookie 文件""" - if not self.cookie: - return None - - cookies = {} - for item in self.cookie.split(';'): - item = item.strip() - if '=' in item: - key, value = item.split('=', 1) - cookies[key.strip()] = value.strip() - - cookie_content = """# Netscape HTTP Cookie File -# Generated by zhihu_content.py - -""" - - for name, value in cookies.items(): - cookie_content += f".zhihu.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n" - - fd, path = tempfile.mkstemp(suffix='.txt') - with os.fdopen(fd, 'w') as f: - f.write(cookie_content) - - return path - - async def download_video(self, video_id: str, quality: int = 720) -> Dict[str, Any]: - """下载知乎视频（使用 yt-dlp）""" - result = {"success": False, "video_id": video_id, "message": "", "filepath": None, "metadata": {}} - - try: - print(f"获取知乎视频信息: {video_id}") - - cookie_file = self._create_cookie_file() - url = f"https://www.zhihu.com/zvideo/{video_id}" - - # 先获取视频标题 - try: - if cookie_file: - cmd = ["yt-dlp", "--cookies", cookie_file, "--get-title", url] - else: - cmd = ["yt-dlp", "--get-title", url] - - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - - title = stdout.decode('utf-8').strip() - if not title or 'error' in title.lower(): - title = f"zhihu_video_{video_id}" - except Exception as e: - title = f"zhihu_video_{video_id}" - - # 构建 yt-dlp 命令 - safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50] - - print(f"下载视频: {title}") - output_template = str(self.output_dir / f"{safe_title}.%(ext)s") - - # 构建下载命令 - cmd = [ - "yt-dlp", - "-f", "bestvideo+bestaudio/best", - "-o", output_template, - "--no-download-archive", - url - ] - - if cookie_file: - cmd.extend(["--cookies", cookie_file]) - - print(f"执行命令: {' '.join(cmd[:4])}...") - - proc = await asyncio.create_subprocess_exec( - *cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await proc.communicate() - - # 查找下载的文件 - downloaded_files = list(self.output_dir.glob(f"{safe_title}.*")) - if downloaded_files: - video_file = downloaded_files[0] - result["success"] = True - result["filepath"] = str(video_file) - result["message"] = f"下载完成: {video_file.name}" - result["metadata"] = {"title": title, "video_id": video_id} - print(f"✅ {result['message']}") - else: - result["message"] = "未找到下载的文件" - print(f"❌ {result['message']}") - - # 清理临时 Cookie 文件 - if cookie_file and os.path.exists(cookie_file): - os.remove(cookie_file) - - except Exception as e: - result["message"] = f"错误: {str(e)}" - print(f"❌ {result['message']}") - - return result - - async def extract_subtitle(self, video_id: str) -> Dict[str, Any]: - """ - 提取字幕 - - 知乎视频通常没有官方字幕，需要使用 Whisper 进行语音识别。 - 此函数返回 has_subtitle=False，让工作流继续执行 Whisper 步骤。 - """ - result = {"success": True, "video_id": video_id, "has_subtitle": False, "message": "知乎视频无官方字幕，将使用 Whisper 进行语音识别"} - - return result - - def read_article_from_mediacrawler(self, input_file: str = None, output_dir: str = None) -> Dict[str, Any]: - """ - 从 MediaCrawler 爬取的数据中读取知乎文章内容 - - Args: - input_file: MediaCrawler 输出文件路径，不指定则自动查找最新 - output_dir: 输出目录 - - Returns: - 处理结果字典 - """ - # 确定输出目录 - if output_dir: - self.output_dir = Path(output_dir) - self.output_dir.mkdir(parents=True, exist_ok=True) - - # 查找数据文件 - data_dir = MEDIACRAWLER_DIR / "data" / "zhihu" / "json" - - if input_file: - # 使用指定的文件 - json_file = Path(input_file) - if not json_file.exists(): - return {"success": False, "message": f"文件不存在: {input_file}"} - else: - # 查找最新的数据文件 - json_files = sorted(data_dir.glob("detail_contents_*.json"), reverse=True) - if not json_files: - return {"success": False, "message": "未找到 MediaCrawler 爬取的数据，请先运行爬虫"} - json_file = json_files[0] - - try: - with open(json_file, encoding="utf-8") as f: - data = json.load(f) - - if not data: - return {"success": False, "message": "数据文件为空"} - - # 提取第一篇文章 - item = data[0] - - # 准备数据 - title = item.get("title", "无标题") - content = item.get("content_text", "") - url = item.get("content_url", "") - author = item.get("user_nickname", "未知") - created_ts = item.get("created_time", 0) - created = datetime.fromtimestamp(created_ts).strftime("%Y-%m-%d %H:%M:%S") if created_ts else "" - voteup = item.get("voteup_count", 0) - comments = item.get("comment_count", 0) - - # 生成文件名 - safe_title = "".join(c for c in title if c.isalnum() or c in " -_")[:50] - filename = f"zhihu_article_{safe_title}.md" - output_path = self.output_dir / filename - - # 写入 Markdown 文件 - markdown_content = f"""--- -title: "{title}" -author: "{author}" -url: "{url}" -created: "{created}" -voteup: {voteup} -comments: {comments} -platform: zhihu -type: article ---- - -# {title} - -**作者**: {author} -**来源**: [知乎]({url}) -**发布时间**: {created} -**点赞**: {voteup} -**评论**: {comments} - ---- - -{content} -""" - - with open(output_path, "w", encoding="utf-8") as f: - f.write(markdown_content) - - return { - "success": True, - "type": "article", - "title": title, - "filename": filename, - "filepath": str(output_path), - "content_length": len(content), - "message": f"文章内容已导出: {title}" - } - - except Exception as e: - return {"success": False, "message": f"读取失败: {str(e)}"} - - -async def main(): - parser = argparse.ArgumentParser(description='知乎内容获取工具') - parser.add_argument('--id', '-i', help='知乎视频ID (用于 video/subtitle 类型)') - parser.add_argument('--type', '-t', required=True, - choices=['video', 'subtitle', 'article-from-media', 'zhuanlan', 'article'], - help='内容类型: video=视频下载, subtitle=字幕提取, article-from-media=从MediaCrawler读取文章, zhuanlan/article=爬取知乎专栏文章') - parser.add_argument('--output', '-o', default='./output', help='输出目录') - parser.add_argument('--quality', '-q', type=int, default=720, help='视频质量') - parser.add_argument('--input', '-in', help='MediaCrawler 数据文件路径 (用于 article-from-media 类型)') - - args = parser.parse_args() - - cookie = os.environ.get('ZHIHU_COOKIE') - downloader = ZhihuContent(output_dir=args.output, cookie=cookie) - - if args.type == 'video': - result = await downloader.download_video(args.id, quality=args.quality) - elif args.type == 'subtitle': - result = await downloader.extract_subtitle(args.id) - elif args.type == 'article-from-media': - # 同步调用，不需要 asyncio - result = downloader.read_article_from_mediacrawler( - input_file=args.input, - output_dir=args.output - ) - elif args.type in ('zhuanlan', 'article'): - # 知乎专栏文章：先爬取再读取 - # 需要从 --id 或构造完整 URL - if args.id: - # 如果传入的是完整 URL - if args.id.startswith('http'): - article_url = args.id - else: - # 如果只传入 ID，构造 URL - article_url = f"https://zhuanlan.zhihu.com/p/{args.id}" - else: - result = {"success": False, "message": "zhuanlan/article 类型需要 --id 参数（文章ID或完整URL）"} - print(json.dumps(result, ensure_ascii=False, indent=2)) - return result - - # 调用 zhihu_article_crawler.py 爬取 - crawler_script = Path(__file__).parent.parent / "kb-zhihu-metadata" / "zhihu_article_crawler.py" - if not crawler_script.exists(): - # 尝试另一种路径 - crawler_script = Path.home() / ".openclaw" / "agents" / "knowledge-base" / "workspace" / "skills" / "kb-zhihu-metadata" / "zhihu_article_crawler.py" - - if crawler_script.exists(): - import subprocess - cmd = ["python3", str(crawler_script), "--url", article_url, "--output", args.output] - print(f"调用爬虫: {' '.join(cmd)}") - proc = subprocess.run(cmd, capture_output=True, text=True) - - if proc.returncode != 0: - result = {"success": False, "message": f"爬取失败: {proc.stderr[:200]}"} - else: - # 解析爬虫输出获取文件路径 - try: - crawler_output = json.loads(proc.stdout.split("--- JSON 输出 ---")[-1].strip()) - if crawler_output.get("success"): - # 直接使用爬虫输出的文件 - result = { - "success": True, - "type": "article", - "title": crawler_output.get("title"), - "filename": Path(crawler_output["filepath"]).name, - "filepath": crawler_output["filepath"], - "content_length": crawler_output.get("content_length"), - "message": f"知乎专栏文章已爬取: {crawler_output.get('title')}" - } - else: - result = {"success": False, "message": crawler_output.get("error", "爬取失败")} - except Exception as e: - result = {"success": False, "message": f"解析爬虫输出失败: {str(e)}"} - else: - result = {"success": False, "message": f"未找到 zhihu_article_crawler.py: {crawler_script}"} - else: - result = {"success": False, "message": f"不支持的类型: {args.type}"} - - # 统一输出格式 - if result.get("success"): - output = { - "success": True, - "type": args.type, - "has_subtitle": result.get("has_subtitle", False), - "filename": result.get('filename') or (Path(result.get('filepath', '')).name if result.get('filepath') else None), - "filepath": str(Path(result.get('filepath', '')).resolve()) if result.get('filepath') else None, - "message": result.get('message', '处理成功') - } - if result.get('title'): - output['title'] = result['title'] - if result.get('content_length'): - output['content_length'] = result['content_length'] - else: - output = { - "success": False, - "error": result.get('message', '处理失败'), - "detail": f"ID: {args.id}, 类型: {args.type}" - } - - print(json.dumps(output, ensure_ascii=False, indent=2)) - return result - - -if __name__ == '__main__': - try: - result = asyncio.run(main()) - sys.exit(0 if result.get("success") else 1) - except KeyboardInterrupt: - print(json.dumps({"success": False, "error": "用户取消"}, ensure_ascii=False, indent=2)) - sys.exit(1) - except Exception as e: - print(json.dumps({"success": False, "error": str(e)}, ensure_ascii=False, indent=2)) - sys.exit(1) diff --git a/skills/kb-zhihu-metadata/markdown_converter.py b/skills/kb-zhihu-metadata/markdown_converter.py index 81faa54..eaae319 100644 --- a/skills/kb-zhihu-metadata/markdown_converter.py +++ b/skills/kb-zhihu-metadata/markdown_converter.py @@ -38,6 +38,9 @@ class MarkdownConverter: 将 data-actualsrc 或 data-original 属性的值替换到 src 属性中， 以便 markdownify 能够正确提取真实图片 URL。 + 同时移除