自动同步: 2026-03-17 02:10
This commit is contained in:
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
|
|||||||
|
|
||||||
### 用户的工作区
|
### 用户的工作区
|
||||||
- 路径:~/work/zq
|
- 路径:~/work/zq
|
||||||
- 这个工作区目录通过rclone与Nas同步
|
- **同步方式:rclone mount 实时挂载**(systemd 服务:`rclone-zq.service`)
|
||||||
- **任何对~/work/zq目录的操作(新建/修改/删除)之前必须先从NAS同步到本地:** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
|
- 所有操作实时同步到NAS,无需手动sync
|
||||||
- **操作完成后必须同步到NAS:** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
|
- 管理命令:`systemctl --user status|restart|stop rclone-zq`
|
||||||
|
- 日志:`~/.cache/rclone/zq-mount.log`
|
||||||
|
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
|
||||||
|
- 该目录已通过 rclone mount 挂载,直接读写挂载点即可
|
||||||
|
- 使用 rclone sync 会绕过挂载点缓存,导致数据不一致或覆盖用户更新
|
||||||
|
- 正确做法:直接操作 `~/work/zq/` 下的文件,挂载点会自动同步
|
||||||
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
||||||
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"agent:haonan:main": {
|
"agent:haonan:main": {
|
||||||
"sessionId": "40e80fbb-07c8-440d-a1d4-1e876b86b82d",
|
"sessionId": "e4845dbe-7d4d-41b6-adb5-b8a739e6520d",
|
||||||
"updatedAt": 1773508722585,
|
"updatedAt": 1773620936802,
|
||||||
"systemSent": true,
|
"systemSent": true,
|
||||||
"skillsSnapshot": {
|
"skillsSnapshot": {
|
||||||
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
|
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
|
||||||
@@ -59,11 +59,11 @@
|
|||||||
"to": "user:ou_be83febac9ee2afaa343c642bc697c48",
|
"to": "user:ou_be83febac9ee2afaa343c642bc697c48",
|
||||||
"accountId": "feishu-pm"
|
"accountId": "feishu-pm"
|
||||||
},
|
},
|
||||||
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/40e80fbb-07c8-440d-a1d4-1e876b86b82d.jsonl",
|
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/e4845dbe-7d4d-41b6-adb5-b8a739e6520d.jsonl",
|
||||||
"systemPromptReport": {
|
"systemPromptReport": {
|
||||||
"source": "run",
|
"source": "run",
|
||||||
"generatedAt": 1773508545805,
|
"generatedAt": 1773620921082,
|
||||||
"sessionId": "40e80fbb-07c8-440d-a1d4-1e876b86b82d",
|
"sessionId": "e4845dbe-7d4d-41b6-adb5-b8a739e6520d",
|
||||||
"sessionKey": "agent:haonan:main",
|
"sessionKey": "agent:haonan:main",
|
||||||
"provider": "zhipu",
|
"provider": "zhipu",
|
||||||
"model": "glm-5",
|
"model": "glm-5",
|
||||||
@@ -75,8 +75,8 @@
|
|||||||
"sandboxed": false
|
"sandboxed": false
|
||||||
},
|
},
|
||||||
"systemPrompt": {
|
"systemPrompt": {
|
||||||
"chars": 29493,
|
"chars": 28195,
|
||||||
"projectContextChars": 16491,
|
"projectContextChars": 15193,
|
||||||
"nonProjectContextChars": 13002
|
"nonProjectContextChars": 13002
|
||||||
},
|
},
|
||||||
"injectedWorkspaceFiles": [
|
"injectedWorkspaceFiles": [
|
||||||
@@ -144,18 +144,6 @@
|
|||||||
"injectedChars": 1168,
|
"injectedChars": 1168,
|
||||||
"truncated": false
|
"truncated": false
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"path": "SELF_IMPROVEMENT_REMINDER.md",
|
|
||||||
"rawChars": 616,
|
|
||||||
"injectedChars": 616,
|
|
||||||
"truncated": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"path": "SELF_IMPROVEMENT_REMINDER.md",
|
|
||||||
"rawChars": 616,
|
|
||||||
"injectedChars": 616,
|
|
||||||
"truncated": false
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"path": "SELF_IMPROVEMENT_REMINDER.md",
|
"path": "SELF_IMPROVEMENT_REMINDER.md",
|
||||||
"rawChars": 616,
|
"rawChars": 616,
|
||||||
@@ -391,12 +379,12 @@
|
|||||||
"totalTokensFresh": true,
|
"totalTokensFresh": true,
|
||||||
"lastHeartbeatText": "已在飞书向蒋先生发送日历同步提醒!✅",
|
"lastHeartbeatText": "已在飞书向蒋先生发送日历同步提醒!✅",
|
||||||
"lastHeartbeatSentAt": 1772924419184,
|
"lastHeartbeatSentAt": 1772924419184,
|
||||||
"cacheRead": 29760,
|
"cacheRead": 5376,
|
||||||
"cacheWrite": 0,
|
"cacheWrite": 0,
|
||||||
"contextTokens": 200000,
|
"contextTokens": 200000,
|
||||||
"inputTokens": 13411,
|
"inputTokens": 12014,
|
||||||
"outputTokens": 1288,
|
"outputTokens": 510,
|
||||||
"totalTokens": 30249
|
"totalTokens": 17390
|
||||||
},
|
},
|
||||||
"agent:haonan:wecom:dm:zhangqing": {
|
"agent:haonan:wecom:dm:zhangqing": {
|
||||||
"origin": {
|
"origin": {
|
||||||
@@ -727,8 +715,8 @@
|
|||||||
"memoryFlushCompactionCount": 0
|
"memoryFlushCompactionCount": 0
|
||||||
},
|
},
|
||||||
"agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc": {
|
"agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc": {
|
||||||
"sessionId": "a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda",
|
"sessionId": "3941961e-62e9-42a2-9ec7-0847ef6734fb",
|
||||||
"updatedAt": 1773597600006,
|
"updatedAt": 1773684000005,
|
||||||
"systemSent": true,
|
"systemSent": true,
|
||||||
"model": "glm-5",
|
"model": "glm-5",
|
||||||
"contextTokens": 200000,
|
"contextTokens": 200000,
|
||||||
@@ -769,11 +757,11 @@
|
|||||||
"version": 0
|
"version": 0
|
||||||
},
|
},
|
||||||
"modelProvider": "zhipu",
|
"modelProvider": "zhipu",
|
||||||
"inputTokens": 11356,
|
"inputTokens": 10481,
|
||||||
"outputTokens": 531,
|
"outputTokens": 227,
|
||||||
"totalTokens": 17361,
|
"totalTokens": 16606,
|
||||||
"totalTokensFresh": true,
|
"totalTokensFresh": true,
|
||||||
"cacheRead": 90112,
|
"cacheRead": 22592,
|
||||||
"cacheWrite": 0
|
"cacheWrite": 0
|
||||||
},
|
},
|
||||||
"agent:haonan:cron:f03b184f-5f1d-4093-ae1b-fc9fe0b59eef": {
|
"agent:haonan:cron:f03b184f-5f1d-4093-ae1b-fc9fe0b59eef": {
|
||||||
@@ -819,9 +807,9 @@
|
|||||||
"version": 0
|
"version": 0
|
||||||
},
|
},
|
||||||
"modelProvider": "zhipu",
|
"modelProvider": "zhipu",
|
||||||
"inputTokens": 153698,
|
"inputTokens": 41440,
|
||||||
"outputTokens": 6902,
|
"outputTokens": 3044,
|
||||||
"totalTokens": 33385,
|
"totalTokens": 25129,
|
||||||
"deliveryContext": {
|
"deliveryContext": {
|
||||||
"channel": "webchat"
|
"channel": "webchat"
|
||||||
},
|
},
|
||||||
@@ -830,7 +818,7 @@
|
|||||||
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl",
|
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl",
|
||||||
"abortedLastRun": false,
|
"abortedLastRun": false,
|
||||||
"totalTokensFresh": true,
|
"totalTokensFresh": true,
|
||||||
"cacheRead": 384894,
|
"cacheRead": 90423,
|
||||||
"cacheWrite": 0
|
"cacheWrite": 0
|
||||||
},
|
},
|
||||||
"agent:haonan:cron:3245fca7-045b-4977-8fab-020f6df2e453": {
|
"agent:haonan:cron:3245fca7-045b-4977-8fab-020f6df2e453": {
|
||||||
@@ -873,8 +861,8 @@
|
|||||||
"cacheWrite": 0
|
"cacheWrite": 0
|
||||||
},
|
},
|
||||||
"agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc": {
|
"agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc": {
|
||||||
"sessionId": "69fcc73b-3185-4ad9-aacd-e44989b770d9",
|
"sessionId": "5432f300-44b1-496e-b169-19a4593e5352",
|
||||||
"updatedAt": 1773598200007,
|
"updatedAt": 1773684600005,
|
||||||
"systemSent": true,
|
"systemSent": true,
|
||||||
"label": "Cron: openclaw-git-sync",
|
"label": "Cron: openclaw-git-sync",
|
||||||
"skillsSnapshot": {
|
"skillsSnapshot": {
|
||||||
@@ -922,113 +910,6 @@
|
|||||||
"cacheRead": 32768,
|
"cacheRead": 32768,
|
||||||
"cacheWrite": 0
|
"cacheWrite": 0
|
||||||
},
|
},
|
||||||
"agent:haonan:cron:f03b184f-5f1d-4093-ae1b-fc9fe0b59eef:run:ed66991b-9963-45f0-b7c1-c2b2655e0463": {
|
|
||||||
"sessionId": "ed66991b-9963-45f0-b7c1-c2b2655e0463",
|
|
||||||
"updatedAt": 1773512400007,
|
|
||||||
"systemSent": true,
|
|
||||||
"model": "glm-4.7-FlashX",
|
|
||||||
"contextTokens": 200000,
|
|
||||||
"label": "Cron: capability-evolver-daily",
|
|
||||||
"skillsSnapshot": {
|
|
||||||
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
|
|
||||||
"skills": [
|
|
||||||
{
|
|
||||||
"name": "capability-evolver"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "clawlist"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"skillFilter": [
|
|
||||||
"clawlist",
|
|
||||||
"capability-evolver",
|
|
||||||
"self-improving-agent"
|
|
||||||
],
|
|
||||||
"resolvedSkills": [
|
|
||||||
{
|
|
||||||
"name": "capability-evolver",
|
|
||||||
"description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.",
|
|
||||||
"filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md",
|
|
||||||
"baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver",
|
|
||||||
"source": "openclaw-managed",
|
|
||||||
"disableModelInvocation": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "clawlist",
|
|
||||||
"description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.",
|
|
||||||
"filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md",
|
|
||||||
"baseDir": "/home/starmen1265/.openclaw/skills/clawlist",
|
|
||||||
"source": "openclaw-managed",
|
|
||||||
"disableModelInvocation": false
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"version": 0
|
|
||||||
},
|
|
||||||
"modelProvider": "zhipu",
|
|
||||||
"inputTokens": 153698,
|
|
||||||
"outputTokens": 6902,
|
|
||||||
"totalTokens": 33385,
|
|
||||||
"deliveryContext": {
|
|
||||||
"channel": "webchat"
|
|
||||||
},
|
|
||||||
"lastChannel": "webchat",
|
|
||||||
"channel": "webchat",
|
|
||||||
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl",
|
|
||||||
"abortedLastRun": false,
|
|
||||||
"totalTokensFresh": true,
|
|
||||||
"cacheRead": 384894,
|
|
||||||
"cacheWrite": 0
|
|
||||||
},
|
|
||||||
"agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc:run:a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda": {
|
|
||||||
"sessionId": "a42c9fa0-04d2-41a8-ae29-fedfa8c1bdda",
|
|
||||||
"updatedAt": 1773597600006,
|
|
||||||
"systemSent": true,
|
|
||||||
"model": "glm-5",
|
|
||||||
"contextTokens": 200000,
|
|
||||||
"label": "Cron: backup-openclaw-json",
|
|
||||||
"skillsSnapshot": {
|
|
||||||
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
|
|
||||||
"skills": [
|
|
||||||
{
|
|
||||||
"name": "capability-evolver"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "clawlist"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"skillFilter": [
|
|
||||||
"clawlist",
|
|
||||||
"capability-evolver",
|
|
||||||
"self-improving-agent"
|
|
||||||
],
|
|
||||||
"resolvedSkills": [
|
|
||||||
{
|
|
||||||
"name": "capability-evolver",
|
|
||||||
"description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.",
|
|
||||||
"filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md",
|
|
||||||
"baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver",
|
|
||||||
"source": "openclaw-managed",
|
|
||||||
"disableModelInvocation": false
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "clawlist",
|
|
||||||
"description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.",
|
|
||||||
"filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md",
|
|
||||||
"baseDir": "/home/starmen1265/.openclaw/skills/clawlist",
|
|
||||||
"source": "openclaw-managed",
|
|
||||||
"disableModelInvocation": false
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"version": 0
|
|
||||||
},
|
|
||||||
"modelProvider": "zhipu",
|
|
||||||
"inputTokens": 11356,
|
|
||||||
"outputTokens": 531,
|
|
||||||
"totalTokens": 17361,
|
|
||||||
"totalTokensFresh": true,
|
|
||||||
"cacheRead": 90112,
|
|
||||||
"cacheWrite": 0
|
|
||||||
},
|
|
||||||
"agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc:run:69fcc73b-3185-4ad9-aacd-e44989b770d9": {
|
"agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc:run:69fcc73b-3185-4ad9-aacd-e44989b770d9": {
|
||||||
"sessionId": "69fcc73b-3185-4ad9-aacd-e44989b770d9",
|
"sessionId": "69fcc73b-3185-4ad9-aacd-e44989b770d9",
|
||||||
"updatedAt": 1773598200007,
|
"updatedAt": 1773598200007,
|
||||||
@@ -1122,9 +1003,9 @@
|
|||||||
"version": 0
|
"version": 0
|
||||||
},
|
},
|
||||||
"modelProvider": "zhipu",
|
"modelProvider": "zhipu",
|
||||||
"inputTokens": 153698,
|
"inputTokens": 41440,
|
||||||
"outputTokens": 6902,
|
"outputTokens": 3044,
|
||||||
"totalTokens": 33385,
|
"totalTokens": 25129,
|
||||||
"deliveryContext": {
|
"deliveryContext": {
|
||||||
"channel": "webchat"
|
"channel": "webchat"
|
||||||
},
|
},
|
||||||
@@ -1133,7 +1014,107 @@
|
|||||||
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl",
|
"sessionFile": "/home/starmen1265/.openclaw/agents/haonan/sessions/ccc40ca7-7876-4d82-ab84-7b790feddc19.jsonl",
|
||||||
"abortedLastRun": false,
|
"abortedLastRun": false,
|
||||||
"totalTokensFresh": true,
|
"totalTokensFresh": true,
|
||||||
"cacheRead": 384894,
|
"cacheRead": 90423,
|
||||||
|
"cacheWrite": 0
|
||||||
|
},
|
||||||
|
"agent:haonan:cron:41396aeb-6e3a-47b2-9bd3-9156a348c1dc:run:3941961e-62e9-42a2-9ec7-0847ef6734fb": {
|
||||||
|
"sessionId": "3941961e-62e9-42a2-9ec7-0847ef6734fb",
|
||||||
|
"updatedAt": 1773684000005,
|
||||||
|
"systemSent": true,
|
||||||
|
"model": "glm-5",
|
||||||
|
"contextTokens": 200000,
|
||||||
|
"label": "Cron: backup-openclaw-json",
|
||||||
|
"skillsSnapshot": {
|
||||||
|
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
|
||||||
|
"skills": [
|
||||||
|
{
|
||||||
|
"name": "capability-evolver"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "clawlist"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"skillFilter": [
|
||||||
|
"clawlist",
|
||||||
|
"capability-evolver",
|
||||||
|
"self-improving-agent"
|
||||||
|
],
|
||||||
|
"resolvedSkills": [
|
||||||
|
{
|
||||||
|
"name": "capability-evolver",
|
||||||
|
"description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.",
|
||||||
|
"filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md",
|
||||||
|
"baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver",
|
||||||
|
"source": "openclaw-managed",
|
||||||
|
"disableModelInvocation": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "clawlist",
|
||||||
|
"description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.",
|
||||||
|
"filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md",
|
||||||
|
"baseDir": "/home/starmen1265/.openclaw/skills/clawlist",
|
||||||
|
"source": "openclaw-managed",
|
||||||
|
"disableModelInvocation": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"version": 0
|
||||||
|
},
|
||||||
|
"modelProvider": "zhipu",
|
||||||
|
"inputTokens": 10481,
|
||||||
|
"outputTokens": 227,
|
||||||
|
"totalTokens": 16606,
|
||||||
|
"totalTokensFresh": true,
|
||||||
|
"cacheRead": 22592,
|
||||||
|
"cacheWrite": 0
|
||||||
|
},
|
||||||
|
"agent:haonan:cron:aa4d2de9-79be-4262-83ee-af8824793bfc:run:5432f300-44b1-496e-b169-19a4593e5352": {
|
||||||
|
"sessionId": "5432f300-44b1-496e-b169-19a4593e5352",
|
||||||
|
"updatedAt": 1773684600005,
|
||||||
|
"systemSent": true,
|
||||||
|
"label": "Cron: openclaw-git-sync",
|
||||||
|
"skillsSnapshot": {
|
||||||
|
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n <skill>\n <name>clawlist</name>\n <description>MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.</description>\n <location>~/.openclaw/skills/clawlist/SKILL.md</location>\n </skill>\n</available_skills>",
|
||||||
|
"skills": [
|
||||||
|
{
|
||||||
|
"name": "capability-evolver"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "clawlist"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"skillFilter": [
|
||||||
|
"clawlist",
|
||||||
|
"capability-evolver",
|
||||||
|
"self-improving-agent"
|
||||||
|
],
|
||||||
|
"resolvedSkills": [
|
||||||
|
{
|
||||||
|
"name": "capability-evolver",
|
||||||
|
"description": "A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.",
|
||||||
|
"filePath": "/home/starmen1265/.openclaw/skills/capability-evolver/SKILL.md",
|
||||||
|
"baseDir": "/home/starmen1265/.openclaw/skills/capability-evolver",
|
||||||
|
"source": "openclaw-managed",
|
||||||
|
"disableModelInvocation": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "clawlist",
|
||||||
|
"description": "MUST use for any multi-step project, long-running task, or infinite monitoring workflow. Plan, execute, track, and verify tasks with checkpoint validation. For projects, automation, and ongoing operations.",
|
||||||
|
"filePath": "/home/starmen1265/.openclaw/skills/clawlist/SKILL.md",
|
||||||
|
"baseDir": "/home/starmen1265/.openclaw/skills/clawlist",
|
||||||
|
"source": "openclaw-managed",
|
||||||
|
"disableModelInvocation": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"version": 0
|
||||||
|
},
|
||||||
|
"modelProvider": "zhipu",
|
||||||
|
"model": "glm-5",
|
||||||
|
"contextTokens": 200000,
|
||||||
|
"inputTokens": 579,
|
||||||
|
"outputTokens": 363,
|
||||||
|
"totalTokens": 16886,
|
||||||
|
"totalTokensFresh": true,
|
||||||
|
"cacheRead": 32768,
|
||||||
"cacheWrite": 0
|
"cacheWrite": 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
|
|||||||
|
|
||||||
### 用户的工作区
|
### 用户的工作区
|
||||||
- 路径:~/work/zq
|
- 路径:~/work/zq
|
||||||
- 这个工作区目录通过rclone与Nas同步
|
- **同步方式:rclone mount 实时挂载**(systemd 服务:`rclone-zq.service`)
|
||||||
- **任何对~/work/zq目录的操作(新建/修改/删除)之前必须先从NAS同步到本地:** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
|
- 所有操作实时同步到NAS,无需手动sync
|
||||||
- **操作完成后必须同步到NAS:** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
|
- 管理命令:`systemctl --user status|restart|stop rclone-zq`
|
||||||
|
- 日志:`~/.cache/rclone/zq-mount.log`
|
||||||
|
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
|
||||||
|
- 该目录已通过 rclone mount 挂载,直接读写挂载点即可
|
||||||
|
- 使用 rclone sync 会绕过挂载点缓存,导致数据不一致或覆盖用户更新
|
||||||
|
- 正确做法:直接操作 `~/work/zq/` 下的文件,挂载点会自动同步
|
||||||
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
||||||
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
||||||
|
|
||||||
|
|||||||
@@ -1,39 +1,40 @@
|
|||||||
# ERRORS.md - 错误记录
|
# ERRORS.md - 错误记录
|
||||||
|
|
||||||
## 2026-03-10: kb-workflow 流程执行不完整
|
## 2026-03-16: rclone sync 覆盖了用户更新
|
||||||
|
|
||||||
### 发生了什么
|
**问题描述**:
|
||||||
用户请求"添加到知识库 + 链接",我启动了 kb-workflow 技能,但两次漏掉步骤:
|
- `~/work/zq` 已通过 `rclone mount` 挂载,实时同步到 NAS
|
||||||
1. 第一次:只做了链接解析、元数据提取、内容提取,漏掉了深度分析
|
- 我在 18:25 执行了 `rclone sync ~/work/zq/ "nas-webdav:..."`(直接操作远程)
|
||||||
2. 第二次:深度分析完成后,漏掉了归档步骤
|
- 这绕过了挂载点缓存,覆盖了用户 18:30 在 NAS 上的更新
|
||||||
|
|
||||||
### 根本原因
|
**根本原因**:
|
||||||
1. **没有建立步骤检查清单** - SKILL.md 明确列出了8个步骤,但我没有逐项核对
|
- AGENTS.md 中没有明确禁止对已挂载目录使用 rclone sync
|
||||||
2. **并行子代理导致认知偏差** - 启动3个子代理处理后以为工作结束了
|
- 误以为需要手动同步
|
||||||
3. **过早汇报完成** - 在步骤4完成后就向用户汇报,没有继续后续步骤
|
|
||||||
4. **忘记归档是必须步骤** - 即使分析完成了,归档才是最终目标
|
|
||||||
|
|
||||||
### 正确做法
|
**教训**:
|
||||||
```
|
- **禁止对已挂载目录(~/work/zq)使用 rclone sync**
|
||||||
1. 执行技能前,先列出完整步骤清单
|
- 直接读写挂载点即可,挂载点会自动同步
|
||||||
2. 每完成一步,标记状态 [x]
|
|
||||||
3. 全部完成后,再次核对清单
|
|
||||||
4. 全部 ✅ 才能向用户汇报完成
|
|
||||||
```
|
|
||||||
|
|
||||||
### kb-workflow 完整步骤清单(必须全部执行)
|
**已修正**:
|
||||||
| 步骤 | 内容 | 状态 |
|
- 更新了 AGENTS.md,添加了明确的禁止规则
|
||||||
|------|------|------|
|
|
||||||
| 1 | 环境变量检查 | [ ] |
|
|
||||||
| 2 | 链接解析 (kb-link-parser) | [ ] |
|
|
||||||
| 3 | 元数据提取 (kb-xxx-metadata) | [ ] |
|
|
||||||
| 4 | 内容下载/提取 (kb-xxx-content) | [ ] |
|
|
||||||
| 5 | 语音识别(视频才需要)| [ ] 或 [跳过] |
|
|
||||||
| 6 | 深度分析 (kb-deep-analyzer) | [ ] |
|
|
||||||
| 7 | 归档 (kb_workflow.py --archive-file) | [ ] |
|
|
||||||
| 8 | NAS 同步 + 汇报完成 | [ ] |
|
|
||||||
|
|
||||||
### 如何避免再犯
|
---
|
||||||
- 每次执行 kb-workflow,必须使用上面的清单逐项核对
|
|
||||||
- 最后一步"归档"没完成,不算完成
|
## 2026-03-16: kb-workflow 技能缺少归档步骤
|
||||||
- 用户问"为什么没XX"时,立即检查是否漏步骤
|
|
||||||
|
**问题描述**:
|
||||||
|
- SKILL.md 文档描述了 5 步流程(parse → download → asr → analyze → archive)
|
||||||
|
- 实际 `kb_workflow.py` 只实现了 4 步,缺少 archive 步骤
|
||||||
|
- `KB_ARCHIVE_PATH` 环境变量被定义但未被使用
|
||||||
|
|
||||||
|
**影响**:
|
||||||
|
- 处理后的文件留在 workspace 而非归档到工作区
|
||||||
|
- 需要手动移动文件
|
||||||
|
|
||||||
|
**临时解决方案**:
|
||||||
|
- 手动执行 `cp` 到归档目录(直接操作挂载点 ~/work/zq,不用 rclone sync)
|
||||||
|
|
||||||
|
**根本解决方案**(需修改技能代码):
|
||||||
|
1. 在 pipeline 中添加 ArchiveProcessor
|
||||||
|
2. 读取 `KB_ARCHIVE_PATH` 环境变量
|
||||||
|
3. 按日期创建子目录并移动文件
|
||||||
|
|||||||
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
|
|||||||
|
|
||||||
### 用户的工作区
|
### 用户的工作区
|
||||||
- 路径:~/work/zq
|
- 路径:~/work/zq
|
||||||
- 这个工作区目录通过rclone与Nas同步
|
- **同步方式:rclone mount 实时挂载**(systemd 服务:`rclone-zq.service`)
|
||||||
- **任何对~/work/zq目录的操作(新建/修改/删除)之前必须先从NAS同步到本地:** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
|
- 所有操作实时同步到NAS,无需手动sync
|
||||||
- **操作完成后必须同步到NAS:** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
|
- 管理命令:`systemctl --user status|restart|stop rclone-zq`
|
||||||
|
- 日志:`~/.cache/rclone/zq-mount.log`
|
||||||
|
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
|
||||||
|
- 该目录已通过 rclone mount 挂载,直接读写挂载点即可
|
||||||
|
- 使用 rclone sync 会绕过挂载点缓存,导致数据不一致或覆盖用户更新
|
||||||
|
- 正确做法:直接操作 `~/work/zq/` 下的文件,挂载点会自动同步
|
||||||
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
||||||
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# Session: 2026-03-16 05:49:59 UTC
|
||||||
|
|
||||||
|
- **Session Key**: agent:knowledge-base:main
|
||||||
|
- **Session ID**: ad89af23-d276-4fd4-bd7c-e1a06ed88000
|
||||||
|
- **Source**: feishu
|
||||||
|
|
||||||
|
## Conversation Summary
|
||||||
|
|
||||||
|
user: Conversation info (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message_id": "om_x100b544d24d77ca4c4da0769c820e25",
|
||||||
|
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"sender": "张青",
|
||||||
|
"timestamp": "Mon 2026-03-16 13:35 GMT+8"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Sender (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
|
||||||
|
"id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"name": "张青"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
[message_id: om_x100b544d24d77ca4c4da0769c820e25]
|
||||||
|
张青: 现在哪里能找到南京90年代到2000年出头的城市样貌照片或者视频
|
||||||
|
assistant: 我来帮你搜索一下南京90年代到2000年代初的城市照片和视频资料。
|
||||||
|
assistant: 蒋先生,根据搜索结果,我为您找到了几个可以找到南京90年代到2000年代初城市样貌照片和视频的渠道:
|
||||||
|
|
||||||
|
## 📱 视频平台
|
||||||
|
|
||||||
|
### 抖音账号
|
||||||
|
- **"南京老照片"** - 这是一个专注南京历史文化的账号,发布了大量珍贵的历史影像
|
||||||
|
- 近期爆款:《2000年的南京大厂区》
|
||||||
|
- 《九十年代末的三牌楼大街和娄子巷》
|
||||||
|
- 拥有10万粉丝,持续输出高质量内容
|
||||||
|
|
||||||
|
### B站(哔哩哔哩)
|
||||||
|
- 搜索关键词:"南京 90年代"、"南京 老照片"
|
||||||
|
- 有相关纪录片和用户上传的老视频
|
||||||
|
|
||||||
|
## 📸 照片资源
|
||||||
|
|
||||||
|
### 网站文章
|
||||||
|
1. **我苏网** - 《40年南京变化有多大?看这些"老照片"就懂了》
|
||||||
|
- 包含1990年代的玄武湖公园等场景
|
||||||
|
|
||||||
|
2. **搜狐网** - 《寻找消失的南京!40张老照片》
|
||||||
|
- 有90年代河西地区、7路公交车等珍贵影像
|
||||||
|
|
||||||
|
3. **游侠客旅行** - 《2025南京古今大对比,超百张珍贵老照片》
|
||||||
|
|
||||||
|
4. **南京市地方志办公室官网**
|
||||||
|
- 多篇关于南京高校、城市变迁的老照片文章
|
||||||
|
|
||||||
|
## 🎬 纪录片
|
||||||
|
|
||||||
|
- **《百年南京》** - 央视纪录片,包含南京历史变迁内容
|
||||||
|
|
||||||
|
## 💡 建议搜索方式
|
||||||
|
|
||||||
|
1. **抖音**:搜索"南京老照片"账号
|
||||||
|
2. **B站**:搜索"南京 90年代 老视频"
|
||||||
|
3. **百度图片**:搜索"南京 90年代 街景"
|
||||||
|
4. **微博**:搜索相关话题
|
||||||
|
|
||||||
|
需要我帮您进一步查看某个具体资源的详细内容吗?
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
# Session: 2026-03-16 08:56:28 UTC
|
||||||
|
|
||||||
|
- **Session Key**: agent:knowledge-base:main
|
||||||
|
- **Session ID**: 9db16f34-09e8-4f14-810a-c210bf13a275
|
||||||
|
- **Source**: feishu
|
||||||
|
|
||||||
|
## Conversation Summary
|
||||||
|
|
||||||
|
assistant: 短链接格式不被识别。让我先获取真实URL:
|
||||||
|
assistant: 正在处理中,让我检查进度:
|
||||||
|
assistant: 处理进展:
|
||||||
|
- ✅ 解析成功
|
||||||
|
- ✅ 下载成功(已获取音频)
|
||||||
|
- ✅ 语音识别完成
|
||||||
|
- ❌ 深度分析失败:`AI_API_KEY 未设置`
|
||||||
|
|
||||||
|
让我检查环境变量配置:
|
||||||
|
assistant: 环境变量文件存在。问题是脚本执行时没有加载。让我手动加载后重新运行分析步骤:
|
||||||
|
assistant: ✅ 处理成功!现在执行归档步骤:
|
||||||
|
assistant: ---
|
||||||
|
|
||||||
|
✅ **知识库添加完成!**
|
||||||
|
|
||||||
|
**标题**:2-3月宏观经济运行状态复盘与推演 + 2026年政府工作报告解读
|
||||||
|
|
||||||
|
**作者**:阿周的Eureka | **时长**:07:16 | **日期**:2026-03-08
|
||||||
|
|
||||||
|
**处理结果**:
|
||||||
|
- ✅ 解析链接
|
||||||
|
- ✅ 下载音频
|
||||||
|
- ✅ 语音识别
|
||||||
|
- ✅ AI深度分析
|
||||||
|
- ✅ 归档到知识库
|
||||||
|
- ✅ 同步到NAS
|
||||||
|
|
||||||
|
**归档位置**:`~/work/zq/30.资源/AI分析/2026-03-16/bilibili_BV16cNMzpExh.md`
|
||||||
|
|
||||||
|
**核心内容摘要**:
|
||||||
|
视频复盘了2026年2-3月宏观经济数据,解读了政府工作报告。主要观点:
|
||||||
|
1. **通缩压力缓解**:猪肉价格触底回升,PPI环比改善
|
||||||
|
2. **M1增速回升**:源于定期存款集中到期,非内生扩张
|
||||||
|
3. **社融创新高**:主要靠政府债券融资,企业/居民意愿仍低
|
||||||
|
4. **房地产筑底**:成交量回暖是刚需入场,政策转向"稳预期"
|
||||||
|
5. **外部风险**:伊朗局势可能带来输入性通胀
|
||||||
|
user: Conversation info (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message_id": "om_x100b544e420910a4c14cf91d01d5da0",
|
||||||
|
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"sender": "张青",
|
||||||
|
"timestamp": "Mon 2026-03-16 14:18 GMT+8"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Sender (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
|
||||||
|
"id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"name": "张青"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
[message_id: om_x100b544e420910a4c14cf91d01d5da0]
|
||||||
|
张青: 工作区现在同步方式有问题,只要修改之前没有进行rclone操作,就会导致本地旧版的文件同步到Nas上。
|
||||||
|
user: Note: The previous agent run was aborted by the user. Resume carefully or ask for clarification.
|
||||||
|
|
||||||
|
Conversation info (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message_id": "om_x100b544e41da6c94c37ec270f11b68e",
|
||||||
|
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"sender": "张青",
|
||||||
|
"timestamp": "Mon 2026-03-16 14:19 GMT+8"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Sender (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
|
||||||
|
"id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"name": "张青"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
[message_id: om_x100b544e41da6c94c37ec270f11b68e]
|
||||||
|
张青: 工作区现在同步方式有问题,只要修改之前没有进行rclone操作,就会导致本地旧版的文件同步到Nas上。看看有没有更实时的同步方案。比如SynologyNas官方的linux部署方案
|
||||||
|
assistant: 明白问题所在。当前的双向 `rclone sync` 模式存在风险:如果本地有旧文件,会反向覆盖NAS上的新版本。
|
||||||
|
|
||||||
|
让我搜索 Synology NAS 官方的 Linux 同步方案:
|
||||||
|
assistant: 找到了几种方案。让我整理对比:
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 工作区实时同步方案对比
|
||||||
|
|
||||||
|
### 当前问题
|
||||||
|
`rclone sync` 是单向覆盖式同步,如果本地有旧文件,会把NAS新版本覆盖掉。
|
||||||
|
|
||||||
|
### 可选方案
|
||||||
|
|
||||||
|
| 方案 | 类型 | 实时性 | 优点 | 缺点 |
|
||||||
|
|------|------|--------|------|------|
|
||||||
|
| **Synology Drive Client** | 官方客户端 | ✅ 实时双向 | 官方支持、冲突处理、版本控制 | 需要GUI或有headless限制 |
|
||||||
|
| **rclone mount** | 文件系统挂载 | ✅ 实时读写 | 轻量、直接操作NAS文件 | 依赖网络稳定性 |
|
||||||
|
| **Syncthing** | P2P同步 | ✅ 实时双向 | 开源、无中心服务器 | 需要在NAS上安装 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 方案1:Synology Drive Client(官方推荐)
|
||||||
|
|
||||||
|
您的系统是 Ubuntu,支持安装:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 下载 deb 包(从 Synology 官网或 NAS 套件中心获取)
|
||||||
|
wget https://global.download.synology.com/download/Package/spk/SynologyDriveClient/3.5.0-16084/Synology%20Drive%20Client-3.5.0-16084-x86_64.deb
|
||||||
|
|
||||||
|
# 安装
|
||||||
|
sudo dpkg -i "Synology Drive Client-3.5.0-16084-x86_64.deb"
|
||||||
|
sudo apt -f install # 修复依赖
|
||||||
|
```
|
||||||
|
|
||||||
|
**优点**:
|
||||||
|
- 真正的双向实时同步
|
||||||
|
- 自动处理冲突
|
||||||
|
- 支持版本历史
|
||||||
|
- 离线编辑后自动同步
|
||||||
|
|
||||||
|
**注意**:需要确认NAS上已安装 **Synology Drive Server** 套件。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 方案2:rclone mount(轻量替代)
|
||||||
|
|
||||||
|
直接把NAS挂载为本地目录,所有操作实时写入:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 挂载
|
||||||
|
rclone mount nas-webdav:/home/Drive/工作区/张青/工作区/ ~/work/zq/ --vfs-cache-mode full --daemon
|
||||||
|
|
||||||
|
# 卸载
|
||||||
|
fusermount -u ~/work/zq/
|
||||||
|
```
|
||||||
|
|
||||||
|
**优点**:
|
||||||
|
- 无需额外安装
|
||||||
|
- 操作即同步,不会有"旧版本覆盖"问题
|
||||||
|
- 配合 `--vfs-cache-mode full` 提升性能
|
||||||
|
|
||||||
|
**缺点**:
|
||||||
|
- 网络断开时无法操作
|
||||||
|
- 大文件操作可能较慢
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 建议
|
||||||
|
|
||||||
|
蒋先生,您倾向哪种方案?
|
||||||
|
|
||||||
|
1. **Synology Drive Client** - 功能最完整,需要先确认NAS上是否已安装 Drive Server 套件
|
||||||
|
2. **rclone mount** - 最快上手,我可以帮您配置自动挂载
|
||||||
|
3. **Syncthing** - 需要在NAS上额外安装
|
||||||
|
|
||||||
|
我可以帮您检查NAS上的配置,或者直接配置 rclone mount 方案?
|
||||||
|
user: Conversation info (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message_id": "om_x100b544e744c5080c2c7031b6254cec",
|
||||||
|
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"sender": "张青",
|
||||||
|
"timestamp": "Mon 2026-03-16 14:31 GMT+8"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Sender (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
|
||||||
|
"id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"name": "张青"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
[message_id: om_x100b544e744c5080c2c7031b6254cec]
|
||||||
|
张青: 2。rclone mount
|
||||||
|
assistant: 好的,配置 rclone mount 方案。
|
||||||
|
assistant: rclone 已配置好。现在创建 systemd 自动挂载服务:
|
||||||
|
assistant: ✅ **rclone mount 配置完成!**
|
||||||
|
|
||||||
|
**挂载状态**:
|
||||||
|
- 路径:`~/work/zq/` → NAS WebDAV
|
||||||
|
- 类型:实时读写(操作即同步)
|
||||||
|
- 缓存:启用(10G 本地缓存,提升性能)
|
||||||
|
- 自动启动:✅ 开机自动挂载
|
||||||
|
|
||||||
|
**常用命令**:
|
||||||
|
```bash
|
||||||
|
# 查看状态
|
||||||
|
systemctl --user status rclone-zq
|
||||||
|
|
||||||
|
# 手动重启
|
||||||
|
systemctl --user restart rclone-zq
|
||||||
|
|
||||||
|
# 查看日志
|
||||||
|
tail -f ~/.cache/rclone/zq-mount.log
|
||||||
|
```
|
||||||
|
|
||||||
|
现在更新 AGENTS.md,移除旧的 sync 步骤:
|
||||||
|
assistant: ✅ **配置完成!**
|
||||||
|
|
||||||
|
**变更摘要**:
|
||||||
|
|
||||||
|
| 项目 | 旧方式 | 新方式 |
|
||||||
|
|------|--------|--------|
|
||||||
|
| 同步模式 | 手动 `rclone sync` | 实时挂载 `rclone mount` |
|
||||||
|
| 操作前 | 需先从NAS拉取 | 直接操作,实时写入 |
|
||||||
|
| 操作后 | 需推送到NAS | 自动同步 |
|
||||||
|
| 覆盖风险 | ❌ 本地旧文件可能覆盖NAS | ✅ 直接读写NAS,无覆盖风险 |
|
||||||
|
|
||||||
|
**服务状态**:
|
||||||
|
- 服务名:`rclone-zq.service`
|
||||||
|
- 状态:运行中(开机自启)
|
||||||
|
- 挂载点:`~/work/zq/` ↔ NAS WebDAV
|
||||||
|
|
||||||
|
现在对 `~/work/zq/` 的任何操作都会实时同步到 NAS,不再需要手动 sync 了。
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# Session: 2026-03-16 09:28:44 UTC
|
||||||
|
|
||||||
|
- **Session Key**: agent:knowledge-base:main
|
||||||
|
- **Session ID**: fa3cb5a4-5a5b-4d65-96e9-0b17095e5445
|
||||||
|
- **Source**: feishu
|
||||||
|
|
||||||
|
## Conversation Summary
|
||||||
|
|
||||||
|
assistant: ✅ New session started · model: zhipu/glm-5
|
||||||
|
user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning.
|
||||||
|
assistant: 📚 蒋先生您好,我是达康书记。今天是周一,下午4点56分。有什么工作或生活事务需要我协助处理吗?
|
||||||
|
user: Conversation info (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message_id": "om_x100b54482e7f9480c39f23560aab683",
|
||||||
|
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"sender": "张青",
|
||||||
|
"timestamp": "Mon 2026-03-16 16:57 GMT+8"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Sender (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
|
||||||
|
"id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"name": "张青"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
[message_id: om_x100b54482e7f9480c39f23560aab683]
|
||||||
|
张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库
|
||||||
|
assistant: ❌ **处理失败 - 环境依赖缺失**
|
||||||
|
|
||||||
|
蒋先生,在执行"添加到知识库"任务时遇到了意料之外的问题:
|
||||||
|
|
||||||
|
**问题详情:**
|
||||||
|
- kb-workflow 技能缺少必要的 Python 包:`kb_zhihu_metadata`
|
||||||
|
- 该包未安装在虚拟环境中,也未在 kb-skills 仓库中找到源代码
|
||||||
|
- 这是一个知乎内容下载所需的依赖包
|
||||||
|
|
||||||
|
**已尝试的解决方法:**
|
||||||
|
1. ✅ 环境变量检查通过
|
||||||
|
2. ✅ 找到正确的 kb-workflow 执行位置
|
||||||
|
3. ❌ 缺少 kb_zhihu_metadata 包导致下载步骤失败
|
||||||
|
|
||||||
|
**建议解决方案:**
|
||||||
|
需要安装或获取 kb-zhihu-metadata 包。这可能需要:
|
||||||
|
- 从 PyPI 安装(如果已发布)
|
||||||
|
- 从其他仓库获取源代码并安装
|
||||||
|
- 联系技能维护者获取安装说明
|
||||||
|
|
||||||
|
根据工作规范,我无法通过修改代码或配置来解决这个问题。需要您协助处理这个环境依赖问题。
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# Session: 2026-03-16 09:33:25 UTC
|
||||||
|
|
||||||
|
- **Session Key**: agent:knowledge-base:main
|
||||||
|
- **Session ID**: bf3272a6-1140-4b72-815a-2c8c4e769629
|
||||||
|
- **Source**: feishu
|
||||||
|
|
||||||
|
## Conversation Summary
|
||||||
|
|
||||||
|
assistant: ✅ New session started · model: zhipu/glm-5
|
||||||
|
user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning.
|
||||||
|
assistant: 📚 **达康书记为您服务,蒋先生。**
|
||||||
|
|
||||||
|
新会话已就绪,有什么需要我协助的?无论是工作区管理、知识库整理,还是日历提醒、邮件处理,随时吩咐。
|
||||||
|
user: Conversation info (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message_id": "om_x100b544898f834b0c2998c7ae1dd83b",
|
||||||
|
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"sender": "张青",
|
||||||
|
"timestamp": "Mon 2026-03-16 17:29 GMT+8"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Sender (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
|
||||||
|
"id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"name": "张青"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
[message_id: om_x100b544898f834b0c2998c7ae1dd83b]
|
||||||
|
张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# Session: 2026-03-16 10:21:10 UTC
|
||||||
|
|
||||||
|
- **Session Key**: agent:knowledge-base:main
|
||||||
|
- **Session ID**: e5fb9409-8568-482c-9e4d-0dc39679edf3
|
||||||
|
- **Source**: feishu
|
||||||
|
|
||||||
|
## Conversation Summary
|
||||||
|
|
||||||
|
assistant: ✅ New session started · model: zhipu/glm-5
|
||||||
|
user: A new session was started via /new or /reset. Execute your Session Startup sequence now - read the required files before responding to the user. Then greet the user in your configured persona, if one is provided. Be yourself - use your defined voice, mannerisms, and mood. Keep it to 1-3 sentences and ask what they want to do. If the runtime model differs from default_model in the system prompt, mention the default model. Do not mention internal steps, files, tools, or reasoning.
|
||||||
|
assistant: 📚 蒋先生您好,我是达康书记,您的私人秘书。请问今天有什么需要我协助您处理的?
|
||||||
|
user: Conversation info (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"message_id": "om_x100b5448be14cd10c314363dea64b82",
|
||||||
|
"sender_id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"sender": "张青",
|
||||||
|
"timestamp": "Mon 2026-03-16 17:36 GMT+8"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Sender (untrusted metadata):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"label": "张青 (ou_9351c287f42069fa30366ac641d24df7)",
|
||||||
|
"id": "ou_9351c287f42069fa30366ac641d24df7",
|
||||||
|
"name": "张青"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
[message_id: om_x100b5448be14cd10c314363dea64b82]
|
||||||
|
张青: https://www.zhihu.com/question/1936375725931361485/answer/2003232153480226496?share_code=P3Sg6A8B8rNK&utm_psn=2016585020039517457 添加到知识库
|
||||||
|
assistant: ✅ **已添加到知识库**
|
||||||
|
|
||||||
|
**标题**:怎么成为一个 ai agent 工程师?
|
||||||
|
**来源**:知乎回答
|
||||||
|
**作者**:啦啦啦啦
|
||||||
|
|
||||||
|
**内容摘要**:
|
||||||
|
这份内容针对希望转型成为 AI Agent 工程师的人群,提出了一套以"商业项目落地"为核心的敏捷转型方案,核心包含四个步骤:
|
||||||
|
1. 掌握大模型的基础调用与提示词控制
|
||||||
|
2. 赋予模型使用工具的能力
|
||||||
|
3. 解决工程化难题(上下文管理、稳定性、日志监控)
|
||||||
|
4. 将项目经验转化为高含金量的简历话术
|
||||||
|
|
||||||
|
**归档位置**:`~/work/zq/30.资源/AI分析/2026-03-16/zhihu_2003232153480226496.md`
|
||||||
|
|
||||||
|
已同步至 NAS。
|
||||||
-22
File diff suppressed because one or more lines are too long
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
|
|||||||
|
|
||||||
### 用户的工作区
|
### 用户的工作区
|
||||||
- 路径:~/work/zq
|
- 路径:~/work/zq
|
||||||
- 这个工作区目录通过rclone与Nas同步
|
- **同步方式:rclone mount 实时挂载**(systemd 服务:`rclone-zq.service`)
|
||||||
- **任何对~/work/zq目录的操作(新建/修改/删除)之前必须先从NAS同步到本地:** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
|
- 所有操作实时同步到NAS,无需手动sync
|
||||||
- **操作完成后必须同步到NAS:** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
|
- 管理命令:`systemctl --user status|restart|stop rclone-zq`
|
||||||
|
- 日志:`~/.cache/rclone/zq-mount.log`
|
||||||
|
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
|
||||||
|
- 该目录已通过 rclone mount 挂载,直接读写挂载点即可
|
||||||
|
- 使用 rclone sync 会绕过挂载点缓存,导致数据不一致或覆盖用户更新
|
||||||
|
- 正确做法:直接操作 `~/work/zq/` 下的文件,挂载点会自动同步
|
||||||
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
||||||
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"agent:ui-designer:main": {
|
"agent:ui-designer:main": {
|
||||||
"sessionId": "fe685776-cc8c-489c-b409-e461e3d3cbfa",
|
"sessionId": "2790af05-ac44-4b18-82fa-23471948d4d1",
|
||||||
"updatedAt": 1773564045652,
|
"updatedAt": 1773677918741,
|
||||||
"systemSent": true,
|
"systemSent": true,
|
||||||
"abortedLastRun": false,
|
"abortedLastRun": false,
|
||||||
"chatType": "direct",
|
"chatType": "direct",
|
||||||
@@ -22,7 +22,7 @@
|
|||||||
"to": "user:ou_9847ce84b960736c14554730b41e4957",
|
"to": "user:ou_9847ce84b960736c14554730b41e4957",
|
||||||
"accountId": "feishu-art"
|
"accountId": "feishu-art"
|
||||||
},
|
},
|
||||||
"sessionFile": "/home/starmen1265/.openclaw/agents/ui-designer/sessions/fe685776-cc8c-489c-b409-e461e3d3cbfa.jsonl",
|
"sessionFile": "/home/starmen1265/.openclaw/agents/ui-designer/sessions/2790af05-ac44-4b18-82fa-23471948d4d1.jsonl",
|
||||||
"compactionCount": 0,
|
"compactionCount": 0,
|
||||||
"skillsSnapshot": {
|
"skillsSnapshot": {
|
||||||
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n</available_skills>",
|
"prompt": "\n\nThe following skills provide specialized instructions for specific tasks.\nUse the read tool to load a skill's file when the task matches its description.\nWhen a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\n\n<available_skills>\n <skill>\n <name>capability-evolver</name>\n <description>A self-evolution engine for AI agents. Analyzes runtime history to identify improvements and applies protocol-constrained evolution.</description>\n <location>~/.openclaw/skills/capability-evolver/SKILL.md</location>\n </skill>\n</available_skills>",
|
||||||
@@ -51,8 +51,8 @@
|
|||||||
"model": "glm-5",
|
"model": "glm-5",
|
||||||
"systemPromptReport": {
|
"systemPromptReport": {
|
||||||
"source": "run",
|
"source": "run",
|
||||||
"generatedAt": 1773564010472,
|
"generatedAt": 1773677890931,
|
||||||
"sessionId": "fe685776-cc8c-489c-b409-e461e3d3cbfa",
|
"sessionId": "2790af05-ac44-4b18-82fa-23471948d4d1",
|
||||||
"sessionKey": "agent:ui-designer:main",
|
"sessionKey": "agent:ui-designer:main",
|
||||||
"provider": "zhipu",
|
"provider": "zhipu",
|
||||||
"model": "glm-5",
|
"model": "glm-5",
|
||||||
@@ -64,8 +64,8 @@
|
|||||||
"sandboxed": false
|
"sandboxed": false
|
||||||
},
|
},
|
||||||
"systemPrompt": {
|
"systemPrompt": {
|
||||||
"chars": 22552,
|
"chars": 23850,
|
||||||
"projectContextChars": 10067,
|
"projectContextChars": 11365,
|
||||||
"nonProjectContextChars": 12485
|
"nonProjectContextChars": 12485
|
||||||
},
|
},
|
||||||
"injectedWorkspaceFiles": [
|
"injectedWorkspaceFiles": [
|
||||||
@@ -169,6 +169,18 @@
|
|||||||
"injectedChars": 616,
|
"injectedChars": 616,
|
||||||
"truncated": false
|
"truncated": false
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "SELF_IMPROVEMENT_REMINDER.md",
|
||||||
|
"rawChars": 616,
|
||||||
|
"injectedChars": 616,
|
||||||
|
"truncated": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "SELF_IMPROVEMENT_REMINDER.md",
|
||||||
|
"rawChars": 616,
|
||||||
|
"injectedChars": 616,
|
||||||
|
"truncated": false
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "SELF_IMPROVEMENT_REMINDER.md",
|
"path": "SELF_IMPROVEMENT_REMINDER.md",
|
||||||
"rawChars": 616,
|
"rawChars": 616,
|
||||||
@@ -396,13 +408,13 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"cacheRead": 15232,
|
"cacheRead": 15552,
|
||||||
"cacheWrite": 0,
|
"cacheWrite": 0,
|
||||||
"totalTokensFresh": true,
|
"totalTokensFresh": true,
|
||||||
"contextTokens": 200000,
|
"contextTokens": 200000,
|
||||||
"inputTokens": 13122,
|
"inputTokens": 2711,
|
||||||
"outputTokens": 1320,
|
"outputTokens": 882,
|
||||||
"totalTokens": 28354
|
"totalTokens": 18263
|
||||||
},
|
},
|
||||||
"agent:ui-designer:cron:78b64c2f-7f91-47ff-9250-d13f546eb21f": {
|
"agent:ui-designer:cron:78b64c2f-7f91-47ff-9250-d13f546eb21f": {
|
||||||
"sessionId": "03e44b46-c707-4a9f-bf3d-1818aafdcde2",
|
"sessionId": "03e44b46-c707-4a9f-bf3d-1818aafdcde2",
|
||||||
|
|||||||
@@ -69,9 +69,14 @@ This folder is home. Treat it that way.
|
|||||||
|
|
||||||
### 用户的工作区
|
### 用户的工作区
|
||||||
- 路径:~/work/zq
|
- 路径:~/work/zq
|
||||||
- 这个工作区目录通过rclone与Nas同步
|
- **同步方式:rclone mount 实时挂载**(systemd 服务:`rclone-zq.service`)
|
||||||
- **任何对~/work/zq目录的操作(新建/修改/删除)之前必须先从NAS同步到本地:** `rclone sync "nas-webdav:/home/Drive/工作区/张青/工作区/" ~/work/zq/ --exclude ".git/" --exclude ".gitignore"`
|
- 所有操作实时同步到NAS,无需手动sync
|
||||||
- **操作完成后必须同步到NAS:** `rclone sync ~/work/zq/ "nas-webdav:/home/Drive/工作区/张青/工作区/" --exclude ".git/" --exclude ".gitignore"`
|
- 管理命令:`systemctl --user status|restart|stop rclone-zq`
|
||||||
|
- 日志:`~/.cache/rclone/zq-mount.log`
|
||||||
|
- ⚠️ **禁止使用 rclone sync 操作 ~/work/zq 目录**
|
||||||
|
- 该目录已通过 rclone mount 挂载,直接读写挂载点即可
|
||||||
|
- 使用 rclone sync 会绕过挂载点缓存,导致数据不一致或覆盖用户更新
|
||||||
|
- 正确做法:直接操作 `~/work/zq/` 下的文件,挂载点会自动同步
|
||||||
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
- 用户的待办记录到~/work/zq/00.收件箱/01-速记本.md
|
||||||
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
- 用户的提醒记录到~/work/zq/20.领域/23-生活管理/提醒
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# 2026-03-17 财务咨询记录
|
||||||
|
|
||||||
|
## 用户信息
|
||||||
|
- 用户:张青 (ou_9847ce84b960736c14554730b41e4957)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 资产配置优化咨询
|
||||||
|
|
||||||
|
### 用户背景
|
||||||
|
- 总资产:190万
|
||||||
|
- 当前配置:大部分固定收益理财,年化 2%
|
||||||
|
- 目标:整体年化收益提升到 8%
|
||||||
|
|
||||||
|
### 给出的方案
|
||||||
|
|
||||||
|
| 方案 | 股票占比 | 股票金额 | 股票目标收益率 | 风险等级 |
|
||||||
|
|------|---------|---------|--------------|---------|
|
||||||
|
| 保守型 | 30% | 57万 | 22% | 中高 |
|
||||||
|
| 平衡型 ⭐ | 50% | 95万 | 14% | 中 |
|
||||||
|
| 进取型 | 70% | 133万 | 10.6% | 中低(但敞口大)|
|
||||||
|
|
||||||
|
### 推荐方案
|
||||||
|
**平衡型(50% 股票配置)**
|
||||||
|
- 固定收益:95万(2%)
|
||||||
|
- 股票投资:95万(目标 14%)
|
||||||
|
- 理由:风险可控、目标现实、心理压力适中
|
||||||
|
|
||||||
|
### 股票投资建议
|
||||||
|
1. 指数基金打底(60%):沪深300、中证500
|
||||||
|
2. 优质个股精选(30%):高股息蓝筹、行业龙头
|
||||||
|
3. 灵活仓位(10%):机会加仓
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*记录时间:2026-03-17 00:13*
|
||||||
+31
-28
@@ -7,7 +7,7 @@
|
|||||||
"name": "backup-openclaw-json",
|
"name": "backup-openclaw-json",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772261581796,
|
"createdAtMs": 1772261581796,
|
||||||
"updatedAtMs": 1773597621815,
|
"updatedAtMs": 1773684008663,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "0 2 * * *",
|
"expr": "0 2 * * *",
|
||||||
@@ -23,10 +23,10 @@
|
|||||||
"mode": "announce"
|
"mode": "announce"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773684000000,
|
"nextRunAtMs": 1773770400000,
|
||||||
"lastRunAtMs": 1773597600004,
|
"lastRunAtMs": 1773684000004,
|
||||||
"lastStatus": "ok",
|
"lastStatus": "ok",
|
||||||
"lastDurationMs": 21811,
|
"lastDurationMs": 8659,
|
||||||
"consecutiveErrors": 0,
|
"consecutiveErrors": 0,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastDelivered": true,
|
"lastDelivered": true,
|
||||||
@@ -59,7 +59,8 @@
|
|||||||
"consecutiveErrors": 0,
|
"consecutiveErrors": 0,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastDeliveryStatus": "delivered",
|
"lastDeliveryStatus": "delivered",
|
||||||
"lastDelivered": true
|
"lastDelivered": true,
|
||||||
|
"runningAtMs": 1773684600003
|
||||||
},
|
},
|
||||||
"delivery": {
|
"delivery": {
|
||||||
"mode": "announce"
|
"mode": "announce"
|
||||||
@@ -71,7 +72,7 @@
|
|||||||
"name": "capability-evolver-daily",
|
"name": "capability-evolver-daily",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772550743528,
|
"createdAtMs": 1772550743528,
|
||||||
"updatedAtMs": 1773513000030,
|
"updatedAtMs": 1773599129019,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "20 2 * * *",
|
"expr": "20 2 * * *",
|
||||||
@@ -92,14 +93,16 @@
|
|||||||
"to": "feishu-pm"
|
"to": "feishu-pm"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773598800000,
|
"nextRunAtMs": 1773685200000,
|
||||||
"lastRunAtMs": 1773512400004,
|
"lastRunAtMs": 1773598800006,
|
||||||
"lastStatus": "error",
|
"lastStatus": "error",
|
||||||
"lastDurationMs": 600026,
|
"lastDurationMs": 329013,
|
||||||
"consecutiveErrors": 2,
|
"consecutiveErrors": 3,
|
||||||
"lastRunStatus": "error",
|
"lastRunStatus": "error",
|
||||||
"lastDeliveryStatus": "unknown",
|
"lastDeliveryStatus": "not-delivered",
|
||||||
"runningAtMs": 1773598800004
|
"lastError": "⚠️ ✉️ Message failed",
|
||||||
|
"lastDelivered": false,
|
||||||
|
"lastDeliveryError": "⚠️ ✉️ Message failed"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -109,7 +112,7 @@
|
|||||||
"description": "每天凌晨2:30整理昨天的已完成任务到每日回顾",
|
"description": "每天凌晨2:30整理昨天的已完成任务到每日回顾",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1741484880000,
|
"createdAtMs": 1741484880000,
|
||||||
"updatedAtMs": 1773513010858,
|
"updatedAtMs": 1773599416835,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "30 2 * * *",
|
"expr": "30 2 * * *",
|
||||||
@@ -127,11 +130,11 @@
|
|||||||
"channel": "last"
|
"channel": "last"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773599400000,
|
"nextRunAtMs": 1773685800000,
|
||||||
"lastRunAtMs": 1773513002054,
|
"lastRunAtMs": 1773599400005,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastStatus": "ok",
|
"lastStatus": "ok",
|
||||||
"lastDurationMs": 8804,
|
"lastDurationMs": 16830,
|
||||||
"lastDelivered": true,
|
"lastDelivered": true,
|
||||||
"lastDeliveryStatus": "delivered",
|
"lastDeliveryStatus": "delivered",
|
||||||
"consecutiveErrors": 0
|
"consecutiveErrors": 0
|
||||||
@@ -143,7 +146,7 @@
|
|||||||
"name": "zq_workspace_sync",
|
"name": "zq_workspace_sync",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772551807455,
|
"createdAtMs": 1772551807455,
|
||||||
"updatedAtMs": 1773513600005,
|
"updatedAtMs": 1773600000005,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"expr": "40 2 * * *",
|
"expr": "40 2 * * *",
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
@@ -156,10 +159,10 @@
|
|||||||
"text": "执行 zq 工作区同步:bash ~/.openclaw/cron/scripts/zq_workspace_sync.sh"
|
"text": "执行 zq 工作区同步:bash ~/.openclaw/cron/scripts/zq_workspace_sync.sh"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773600000000,
|
"nextRunAtMs": 1773686400000,
|
||||||
"lastRunAtMs": 1773513600005,
|
"lastRunAtMs": 1773600000004,
|
||||||
"lastStatus": "skipped",
|
"lastStatus": "skipped",
|
||||||
"lastDurationMs": 0,
|
"lastDurationMs": 1,
|
||||||
"consecutiveErrors": 0,
|
"consecutiveErrors": 0,
|
||||||
"lastRunStatus": "skipped",
|
"lastRunStatus": "skipped",
|
||||||
"lastDeliveryStatus": "not-requested",
|
"lastDeliveryStatus": "not-requested",
|
||||||
@@ -173,7 +176,7 @@
|
|||||||
"description": "每天早上8点整理与各个 AI 代理的对话记录",
|
"description": "每天早上8点整理与各个 AI 代理的对话记录",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772776369645,
|
"createdAtMs": 1772776369645,
|
||||||
"updatedAtMs": 1773532816520,
|
"updatedAtMs": 1773619224324,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "0 8 * * *",
|
"expr": "0 8 * * *",
|
||||||
@@ -191,11 +194,11 @@
|
|||||||
"channel": "last"
|
"channel": "last"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773619200000,
|
"nextRunAtMs": 1773705600000,
|
||||||
"lastRunAtMs": 1773532800006,
|
"lastRunAtMs": 1773619200004,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastStatus": "ok",
|
"lastStatus": "ok",
|
||||||
"lastDurationMs": 16514,
|
"lastDurationMs": 24320,
|
||||||
"lastDelivered": true,
|
"lastDelivered": true,
|
||||||
"lastDeliveryStatus": "delivered",
|
"lastDeliveryStatus": "delivered",
|
||||||
"consecutiveErrors": 0
|
"consecutiveErrors": 0
|
||||||
@@ -208,7 +211,7 @@
|
|||||||
"description": "每天7点同步提醒到CalDAV日历",
|
"description": "每天7点同步提醒到CalDAV日历",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772791528528,
|
"createdAtMs": 1772791528528,
|
||||||
"updatedAtMs": 1773529221353,
|
"updatedAtMs": 1773615616422,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "0 7 * * *",
|
"expr": "0 7 * * *",
|
||||||
@@ -226,11 +229,11 @@
|
|||||||
"channel": "last"
|
"channel": "last"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773615600000,
|
"nextRunAtMs": 1773702000000,
|
||||||
"lastRunAtMs": 1773529200004,
|
"lastRunAtMs": 1773615600034,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastStatus": "ok",
|
"lastStatus": "ok",
|
||||||
"lastDurationMs": 21349,
|
"lastDurationMs": 16388,
|
||||||
"lastDeliveryStatus": "delivered",
|
"lastDeliveryStatus": "delivered",
|
||||||
"consecutiveErrors": 0,
|
"consecutiveErrors": 0,
|
||||||
"lastDelivered": true
|
"lastDelivered": true
|
||||||
|
|||||||
+29
-27
@@ -7,7 +7,7 @@
|
|||||||
"name": "backup-openclaw-json",
|
"name": "backup-openclaw-json",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772261581796,
|
"createdAtMs": 1772261581796,
|
||||||
"updatedAtMs": 1773597621815,
|
"updatedAtMs": 1773684008663,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "0 2 * * *",
|
"expr": "0 2 * * *",
|
||||||
@@ -23,10 +23,10 @@
|
|||||||
"mode": "announce"
|
"mode": "announce"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773684000000,
|
"nextRunAtMs": 1773770400000,
|
||||||
"lastRunAtMs": 1773597600004,
|
"lastRunAtMs": 1773684000004,
|
||||||
"lastStatus": "ok",
|
"lastStatus": "ok",
|
||||||
"lastDurationMs": 21811,
|
"lastDurationMs": 8659,
|
||||||
"consecutiveErrors": 0,
|
"consecutiveErrors": 0,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastDelivered": true,
|
"lastDelivered": true,
|
||||||
@@ -71,7 +71,7 @@
|
|||||||
"name": "capability-evolver-daily",
|
"name": "capability-evolver-daily",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772550743528,
|
"createdAtMs": 1772550743528,
|
||||||
"updatedAtMs": 1773513000030,
|
"updatedAtMs": 1773599129019,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "20 2 * * *",
|
"expr": "20 2 * * *",
|
||||||
@@ -92,14 +92,16 @@
|
|||||||
"to": "feishu-pm"
|
"to": "feishu-pm"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773598800000,
|
"nextRunAtMs": 1773685200000,
|
||||||
"lastRunAtMs": 1773512400004,
|
"lastRunAtMs": 1773598800006,
|
||||||
"lastStatus": "error",
|
"lastStatus": "error",
|
||||||
"lastDurationMs": 600026,
|
"lastDurationMs": 329013,
|
||||||
"consecutiveErrors": 2,
|
"consecutiveErrors": 3,
|
||||||
"lastRunStatus": "error",
|
"lastRunStatus": "error",
|
||||||
"lastDeliveryStatus": "unknown",
|
"lastDeliveryStatus": "not-delivered",
|
||||||
"lastError": "cron: job execution timed out"
|
"lastError": "⚠️ ✉️ Message failed",
|
||||||
|
"lastDelivered": false,
|
||||||
|
"lastDeliveryError": "⚠️ ✉️ Message failed"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -109,7 +111,7 @@
|
|||||||
"description": "每天凌晨2:30整理昨天的已完成任务到每日回顾",
|
"description": "每天凌晨2:30整理昨天的已完成任务到每日回顾",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1741484880000,
|
"createdAtMs": 1741484880000,
|
||||||
"updatedAtMs": 1773513010858,
|
"updatedAtMs": 1773599416835,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "30 2 * * *",
|
"expr": "30 2 * * *",
|
||||||
@@ -127,11 +129,11 @@
|
|||||||
"channel": "last"
|
"channel": "last"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773599400000,
|
"nextRunAtMs": 1773685800000,
|
||||||
"lastRunAtMs": 1773513002054,
|
"lastRunAtMs": 1773599400005,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastStatus": "ok",
|
"lastStatus": "ok",
|
||||||
"lastDurationMs": 8804,
|
"lastDurationMs": 16830,
|
||||||
"lastDelivered": true,
|
"lastDelivered": true,
|
||||||
"lastDeliveryStatus": "delivered",
|
"lastDeliveryStatus": "delivered",
|
||||||
"consecutiveErrors": 0
|
"consecutiveErrors": 0
|
||||||
@@ -143,7 +145,7 @@
|
|||||||
"name": "zq_workspace_sync",
|
"name": "zq_workspace_sync",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772551807455,
|
"createdAtMs": 1772551807455,
|
||||||
"updatedAtMs": 1773513600005,
|
"updatedAtMs": 1773600000005,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"expr": "40 2 * * *",
|
"expr": "40 2 * * *",
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
@@ -156,10 +158,10 @@
|
|||||||
"text": "执行 zq 工作区同步:bash ~/.openclaw/cron/scripts/zq_workspace_sync.sh"
|
"text": "执行 zq 工作区同步:bash ~/.openclaw/cron/scripts/zq_workspace_sync.sh"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773600000000,
|
"nextRunAtMs": 1773686400000,
|
||||||
"lastRunAtMs": 1773513600005,
|
"lastRunAtMs": 1773600000004,
|
||||||
"lastStatus": "skipped",
|
"lastStatus": "skipped",
|
||||||
"lastDurationMs": 0,
|
"lastDurationMs": 1,
|
||||||
"consecutiveErrors": 0,
|
"consecutiveErrors": 0,
|
||||||
"lastRunStatus": "skipped",
|
"lastRunStatus": "skipped",
|
||||||
"lastDeliveryStatus": "not-requested",
|
"lastDeliveryStatus": "not-requested",
|
||||||
@@ -173,7 +175,7 @@
|
|||||||
"description": "每天早上8点整理与各个 AI 代理的对话记录",
|
"description": "每天早上8点整理与各个 AI 代理的对话记录",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772776369645,
|
"createdAtMs": 1772776369645,
|
||||||
"updatedAtMs": 1773532816520,
|
"updatedAtMs": 1773619224324,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "0 8 * * *",
|
"expr": "0 8 * * *",
|
||||||
@@ -191,11 +193,11 @@
|
|||||||
"channel": "last"
|
"channel": "last"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773619200000,
|
"nextRunAtMs": 1773705600000,
|
||||||
"lastRunAtMs": 1773532800006,
|
"lastRunAtMs": 1773619200004,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastStatus": "ok",
|
"lastStatus": "ok",
|
||||||
"lastDurationMs": 16514,
|
"lastDurationMs": 24320,
|
||||||
"lastDelivered": true,
|
"lastDelivered": true,
|
||||||
"lastDeliveryStatus": "delivered",
|
"lastDeliveryStatus": "delivered",
|
||||||
"consecutiveErrors": 0
|
"consecutiveErrors": 0
|
||||||
@@ -208,7 +210,7 @@
|
|||||||
"description": "每天7点同步提醒到CalDAV日历",
|
"description": "每天7点同步提醒到CalDAV日历",
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"createdAtMs": 1772791528528,
|
"createdAtMs": 1772791528528,
|
||||||
"updatedAtMs": 1773529221353,
|
"updatedAtMs": 1773615616422,
|
||||||
"schedule": {
|
"schedule": {
|
||||||
"kind": "cron",
|
"kind": "cron",
|
||||||
"expr": "0 7 * * *",
|
"expr": "0 7 * * *",
|
||||||
@@ -226,11 +228,11 @@
|
|||||||
"channel": "last"
|
"channel": "last"
|
||||||
},
|
},
|
||||||
"state": {
|
"state": {
|
||||||
"nextRunAtMs": 1773615600000,
|
"nextRunAtMs": 1773702000000,
|
||||||
"lastRunAtMs": 1773529200004,
|
"lastRunAtMs": 1773615600034,
|
||||||
"lastRunStatus": "ok",
|
"lastRunStatus": "ok",
|
||||||
"lastStatus": "ok",
|
"lastStatus": "ok",
|
||||||
"lastDurationMs": 21349,
|
"lastDurationMs": 16388,
|
||||||
"lastDeliveryStatus": "delivered",
|
"lastDeliveryStatus": "delivered",
|
||||||
"consecutiveErrors": 0,
|
"consecutiveErrors": 0,
|
||||||
"lastDelivered": true
|
"lastDelivered": true
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"id": "4b56a737-d726-4c48-8b10-493f0e9df335",
|
||||||
|
"enqueuedAt": 1773599128798,
|
||||||
|
"channel": "feishu",
|
||||||
|
"to": "feishu-pm",
|
||||||
|
"accountId": "feishu-pm",
|
||||||
|
"payloads": [
|
||||||
|
{
|
||||||
|
"text": "## 🧬 Capability Evolver 每日报告\n\n**时间**: 2026年 03月 16日 星期一 02:20:22 CST\n**周期**: #0011\n\n---\n\n### 📊 变更概览\n- **意图**: REPAIR (修复)\n- **状态**: SUCCESS\n- **变更范围**: 4 文件修改,2 文件新增\n\n---\n\n### 📝 Git 提交记录\n\n1. **Auto backup before evolver run**\n - 4 files changed, 70 insertions(+), 64 deletions(-)\n\n2. **Evolver auto evolution** \n - 6 files changed, 550 insertions(+), 13 deletions(-)\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 📂 变更文件\n\n**修改文件**:\n- `memory/evolution/evolution_solidify_state.json`\n- `memory/evolution/evolution_state.json`\n- `memory/evolution/memory_graph_state.json`\n- `memory/evolution/personality_state.json`\n\n**新增文件**:\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 🎯 变更原因\n命中信号:\n- memory_missing\n- session_logs_missing\n\n系统检测到记忆缺失和会话日志缺失,触发修复模式,更新了进化状态和记忆图谱。"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"threadId": null,
|
||||||
|
"mirror": {
|
||||||
|
"sessionKey": "agent:haonan:main",
|
||||||
|
"agentId": "haonan",
|
||||||
|
"text": "## 🧬 Capability Evolver 每日报告\n\n**时间**: 2026年 03月 16日 星期一 02:20:22 CST\n**周期**: #0011\n\n---\n\n### 📊 变更概览\n- **意图**: REPAIR (修复)\n- **状态**: SUCCESS\n- **变更范围**: 4 文件修改,2 文件新增\n\n---\n\n### 📝 Git 提交记录\n\n1. **Auto backup before evolver run**\n - 4 files changed, 70 insertions(+), 64 deletions(-)\n\n2. **Evolver auto evolution** \n - 6 files changed, 550 insertions(+), 13 deletions(-)\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n - 新增: `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 📂 变更文件\n\n**修改文件**:\n- `memory/evolution/evolution_solidify_state.json`\n- `memory/evolution/evolution_state.json`\n- `memory/evolution/memory_graph_state.json`\n- `memory/evolution/personality_state.json`\n\n**新增文件**:\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.json`\n- `memory/evolution/gep_prompt_Cycle_#0011_run_1773598824487.txt`\n\n---\n\n### 🎯 变更原因\n命中信号:\n- memory_missing\n- session_logs_missing\n\n系统检测到记忆缺失和会话日志缺失,触发修复模式,更新了进化状态和记忆图谱。"
|
||||||
|
},
|
||||||
|
"retryCount": 1,
|
||||||
|
"lastAttemptAt": 1773599129011,
|
||||||
|
"lastError": "Request failed with status code 400"
|
||||||
|
}
|
||||||
@@ -1,9 +1,4 @@
|
|||||||
{
|
{
|
||||||
"om_x100b546dc09400acb4a5d0af66ca3c9": 1773508838286,
|
"om_x100b54b68a9658a8c3597587eaea165": 1773677638200,
|
||||||
"om_x100b546dde1c34a4b49a888912755ac": 1773508877713,
|
"om_x100b54b69ae988acc4e308675a1599b": 1773677890863
|
||||||
"om_x100b545236d06ca0b100846cfa37e9d": 1773561730503,
|
|
||||||
"om_x100b5452d99e18a4b32ab9bdc02463e": 1773562229918,
|
|
||||||
"om_x100b5452972138a8b213d14996ffdbd": 1773563295479,
|
|
||||||
"om_x100b5452a60eaca0b10319a26ed640f": 1773563532727,
|
|
||||||
"om_x100b5453485b4ca0b37c3413c41bba3": 1773564010102
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,3 @@
|
|||||||
{
|
{
|
||||||
"om_x100b546d172f3c68c2cea708421b99a": 1773507998614,
|
"om_x100b5440a54d84acb3898a37411f37b": 1773620920592
|
||||||
"om_x100b546d2b32cca0c329a33c339cfb5": 1773508192506,
|
|
||||||
"om_x100b546d32de24a8b316c197e60d6e4": 1773508545727,
|
|
||||||
"om_x100b546dc9d61ca0b33d802bc7615e0": 1773508722267
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,32 +1,34 @@
|
|||||||
{
|
{
|
||||||
"om_x100b546ddd8b14a0b3ddda14ce768c5": 1773508916933,
|
"om_x100b544d24d77ca4c4da0769c820e25": 1773639330290,
|
||||||
"om_x100b546dd78c8ca0b31b0314f910115": 1773509012676,
|
"om_x100b544deea46100c29a100e8840656": 1773640199255,
|
||||||
"om_x100b546def8ac8a4b38d921a2bfc8b8": 1773509140955,
|
"om_x100b544dfc78d090c1266487d499834": 1773640492047,
|
||||||
"om_x100b546dedb654a4b3ce916a129facf": 1773509176311,
|
"om_x100b544df47a2c80c36bffb914f76a1": 1773640620036,
|
||||||
"om_x100b546de975dca0b27e2f24aad0d8c": 1773509244102,
|
"om_x100b544e420910a4c14cf91d01d5da0": 1773641933039,
|
||||||
"om_x100b546de53a94a0b38693fbd7e385c": 1773509311993,
|
"om_x100b544e43f1d4acc2bcd0631ad4df1": 1773641940383,
|
||||||
"om_x100b546dffaa3cb0b35aef2747ed228": 1773509398933,
|
"om_x100b544e41da6c94c37ec270f11b68e": 1773641970288,
|
||||||
"om_x100b546df49c10a0b106d597fd348a3": 1773509541678,
|
"om_x100b544e744c5080c2c7031b6254cec": 1773642664923,
|
||||||
"om_x100b546d16e6e4a0c42bef93bca2224": 1773511894148,
|
"om_x100b54481574e8bcc311e8d58fb4c75": 1773651388362,
|
||||||
"om_x100b546d144668a0c3583fa56a2597d": 1773511924836,
|
"om_x100b54482e7f9480c39f23560aab683": 1773651467804,
|
||||||
"om_x100b546d112090acc3d096e37b38dc9": 1773512010988,
|
"om_x100b54489a79ecbcc142ca87d3f064b": 1773653324009,
|
||||||
"om_x100b545c390898a4c379f319d607993": 1773569917108,
|
"om_x100b544898f834b0c2998c7ae1dd83b": 1773653348152,
|
||||||
"om_x100b545c365b94a0c44e709df0f5422": 1773569929804,
|
"om_x100b5448aba858acc4c8be1f8439827": 1773653591178,
|
||||||
"om_x100b545cce0a04a0c4a26c0203c2b13": 1773570060994,
|
"om_x100b5448ab152c80c2a6860a990a363": 1773653598227,
|
||||||
"om_x100b545ccf6e6ca0c4571d12c405d8f": 1773570074717,
|
"om_x100b5448a8d4f0a0c32f5cc81fda1b3": 1773653602205,
|
||||||
"om_x100b545cccde84acc4d8fa9b0f0c852": 1773570081700,
|
"om_x100b5448a887a4a0c3d2cacc2350d0f": 1773653605264,
|
||||||
"om_x100b545ccab08ca0c12b5e6c715a1aa": 1773570120321,
|
"om_x100b5448be14cd10c314363dea64b82": 1773653774367,
|
||||||
"om_x100b545cca14e8a4c4dcc592a9c425d": 1773570126138,
|
"om_x100b5449d79ebca8c3e54daed500100": 1773656469908,
|
||||||
"om_x100b545c9fd988a4b4bcc03ec7b3368": 1773571345806,
|
"om_x100b5449d129e0acc3831f0b28c0dca": 1773656574965,
|
||||||
"om_x100b5458d25a5ca0c2cf04550bf07cc": 1773586890090,
|
"om_x100b5449e4e594a0c4504c3f849787d": 1773656739113,
|
||||||
"om_x100b5458d055dca4c4dd20469577549": 1773586922136,
|
"om_x100b54499819dc84c31ee62a28e622a": 1773657453822,
|
||||||
"om_x100b54588d4de4a0c14df6ea4b56be0": 1773587512581,
|
"om_x100b544994dbc8a0c37f7406c922a18": 1773657506038,
|
||||||
"om_x100b54599445b4a0c4def897c8a2700": 1773591976996,
|
"om_x100b5449953c8ca0c10d2892ab60482": 1773657535728,
|
||||||
"om_x100b545992d568a0c3511684eae15a5": 1773592002087,
|
"om_x100b5449a63174b4c2c98d2bc1f7fcc": 1773657744382,
|
||||||
"om_x100b5459a51f60a4c11dbfc6068d690": 1773592253777,
|
"om_x100b544a62d8fc80c4e14309469ce27": 1773658818051,
|
||||||
"om_x100b5459be53f0a0c42f8fe338d2510": 1773592330439,
|
"om_x100b544a7b1c14a0c49f7000df992cc": 1773658973700,
|
||||||
"om_x100b5459be2d7ca4c32ee845e40c605": 1773592334472,
|
"om_x100b544a775f1080c3d624954a8460e": 1773659033826,
|
||||||
"om_x100b5459bd4168a0c36ca5cf1588b13": 1773592377222,
|
"om_x100b54b47a79e8a8b37ac4b2ddad147": 1773667147854,
|
||||||
"om_x100b5459bae828acc347e53e7e5de8d": 1773592386779,
|
"om_x100b54b760e4b0a0c445e4e92b151e5": 1773679331046,
|
||||||
"om_x100b5459bb6404a0c4ade83756a546a": 1773592411047
|
"om_x100b54b709932ca8b32f9afaf823ea9": 1773679734328,
|
||||||
|
"om_x100b54b7042de0acb108eaf062898fd": 1773679790410,
|
||||||
|
"om_x100b54b7174820b0b2678b67a63e045": 1773680024819
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
"operator.approvals",
|
"operator.approvals",
|
||||||
"operator.pairing"
|
"operator.pairing"
|
||||||
],
|
],
|
||||||
"updatedAtMs": 1773598217705
|
"updatedAtMs": 1773684008206
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,3 +45,8 @@
|
|||||||
{"timestamp":"2026-03-15T16:26:17.361Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
{"timestamp":"2026-03-15T16:26:17.361Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
||||||
{"timestamp":"2026-03-15T16:32:14.537Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
{"timestamp":"2026-03-15T16:32:14.537Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
||||||
{"timestamp":"2026-03-15T16:33:06.821Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
{"timestamp":"2026-03-15T16:33:06.821Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
||||||
|
{"timestamp":"2026-03-16T05:49:59.593Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
||||||
|
{"timestamp":"2026-03-16T08:56:28.934Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
||||||
|
{"timestamp":"2026-03-16T09:28:44.393Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
||||||
|
{"timestamp":"2026-03-16T09:33:25.309Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
||||||
|
{"timestamp":"2026-03-16T10:21:10.561Z","action":"new","sessionKey":"agent:knowledge-base:main","senderId":"ou_9351c287f42069fa30366ac641d24df7","source":"feishu"}
|
||||||
|
|||||||
@@ -1,91 +0,0 @@
|
|||||||
# KB Bilibili Content Skill
|
|
||||||
|
|
||||||
**kb-bilibili-content** - Download Bilibili videos and extract content.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- Video download using **yt-dlp** (recommended)
|
|
||||||
- Subtitle extraction
|
|
||||||
- Article content extraction
|
|
||||||
- Multiple quality options (360P/480P/720P/1080P)
|
|
||||||
- Supports `BILIBILI_COOKIE` environment variable
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
| Variable | Required | Description |
|
|
||||||
|----------|----------|-------------|
|
|
||||||
| `BILIBILI_COOKIE` | No | B站登录 Cookie (for member-only videos) |
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Download video (720P default)
|
|
||||||
python bilibili_content.py --id "BV1xx411c7mD" --type video --output "/path/to/downloads/"
|
|
||||||
|
|
||||||
# Download video (1080P)
|
|
||||||
python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 80 --output "/path/to/downloads/"
|
|
||||||
|
|
||||||
# Download video (480P)
|
|
||||||
python bilibili_content.py --id "BV1xx411c7mD" --type video --quality 32 --output "/path/to/downloads/"
|
|
||||||
|
|
||||||
# Extract subtitle
|
|
||||||
python bilibili_content.py --id "BV1xx411c7mD" --type subtitle --output "/path/to/subtitles/"
|
|
||||||
|
|
||||||
# Get article content
|
|
||||||
python bilibili_content.py --id "cv43780186" --type article --output "/path/to/article.md"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Output
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"type": "video",
|
|
||||||
"filename": "video_title.mp4",
|
|
||||||
"filepath": "/path/to/downloads/video_title.mp4",
|
|
||||||
"message": "下载完成: video_title.mp4"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Cookie Setup
|
|
||||||
|
|
||||||
### Get Cookie from Browser
|
|
||||||
1. Login to bilibili.com
|
|
||||||
2. Press F12 → Network → Refresh page
|
|
||||||
3. Copy Cookie from request headers
|
|
||||||
|
|
||||||
### Set Environment Variable
|
|
||||||
```bash
|
|
||||||
# Add to ~/.bashrc for persistence
|
|
||||||
export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx; DedeUserID=xxx; DedeUserID__ckMd5=xxx"
|
|
||||||
|
|
||||||
# Or set temporarily
|
|
||||||
export BILIBILI_COOKIE="your_cookie_here"
|
|
||||||
```
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp -r kb-bilibili-content/ ~/skills/
|
|
||||||
pip install aiohttp playwright
|
|
||||||
playwright install chromium
|
|
||||||
```
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
Video download uses **yt-dlp** with Netscape-format cookie file:
|
|
||||||
1. Parses `BILIBILI_COOKIE` environment variable
|
|
||||||
2. Creates temporary Netscape-format cookie file
|
|
||||||
3. Runs `yt-dlp --cookies <file> -f <format> -o <output>`
|
|
||||||
4. Cleans up temporary file
|
|
||||||
|
|
||||||
## Quality Options
|
|
||||||
|
|
||||||
| Quality | ID | Resolution | Notes |
|
|
||||||
|---------|-----|------------|-------|
|
|
||||||
| 16 | 360P | 640x360 | Fastest |
|
|
||||||
| 32 | 480P | 852x480 | Default for low bandwidth |
|
|
||||||
| 64 | 720P | 1280x720 | **Recommended** |
|
|
||||||
| 80 | 1080P | 1920x1080 | Best quality |
|
|
||||||
|
|
||||||
Default quality: **64 (720P)**
|
|
||||||
Binary file not shown.
@@ -1,419 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
Bilibili Content Downloader - B站内容获取工具
|
|
||||||
|
|
||||||
功能:
|
|
||||||
- download_video(bvid, quality=480) - 下载视频(使用 yt-dlp)
|
|
||||||
- extract_subtitle(bvid) - 提取字幕
|
|
||||||
- extract_article(cvid) - 获取专栏全文(使用Playwright)
|
|
||||||
- 命令行参数支持
|
|
||||||
- JSON 输出
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from datetime import datetime
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Optional, Dict, Any, List
|
|
||||||
|
|
||||||
|
|
||||||
class BilibiliContent:
|
|
||||||
"""B站内容获取器"""
|
|
||||||
|
|
||||||
def __init__(self, output_dir: str = "./output", cookie: str = None):
|
|
||||||
self.output_dir = Path(output_dir)
|
|
||||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.cookie = cookie
|
|
||||||
|
|
||||||
def _create_cookie_file(self) -> str:
|
|
||||||
"""创建 Netscape 格式的 Cookie 文件"""
|
|
||||||
if not self.cookie:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# 解析 Cookie 字符串
|
|
||||||
cookies = {}
|
|
||||||
for item in self.cookie.split(';'):
|
|
||||||
item = item.strip()
|
|
||||||
if '=' in item:
|
|
||||||
key, value = item.split('=', 1)
|
|
||||||
cookies[key.strip()] = value.strip()
|
|
||||||
|
|
||||||
# 创建 Netscape 格式的 Cookie 文件
|
|
||||||
cookie_content = """# Netscape HTTP Cookie File
|
|
||||||
# Generated by bilibili_content.py
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
for name, value in cookies.items():
|
|
||||||
cookie_content += f".bilibili.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n"
|
|
||||||
|
|
||||||
# 写入临时文件
|
|
||||||
fd, path = tempfile.mkstemp(suffix='.txt')
|
|
||||||
with os.fdopen(fd, 'w') as f:
|
|
||||||
f.write(cookie_content)
|
|
||||||
|
|
||||||
return path
|
|
||||||
|
|
||||||
async def download_video(self, bvid: str, quality: int = 64) -> Dict[str, Any]:
|
|
||||||
"""下载B站视频(使用 yt-dlp)"""
|
|
||||||
result = {"success": False, "bvid": bvid, "message": "", "filepath": None, "metadata": {}}
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 先获取视频信息
|
|
||||||
print(f"获取视频信息: {bvid}")
|
|
||||||
|
|
||||||
cookie_file = self._create_cookie_file()
|
|
||||||
url = f"https://www.bilibili.com/video/{bvid}"
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 获取视频标题(用于文件名)
|
|
||||||
if cookie_file:
|
|
||||||
cmd = ["yt-dlp", "--cookies", cookie_file, "--get-title", url]
|
|
||||||
else:
|
|
||||||
cmd = ["yt-dlp", "--get-title", url]
|
|
||||||
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
|
||||||
*cmd,
|
|
||||||
stdout=asyncio.subprocess.PIPE,
|
|
||||||
stderr=asyncio.subprocess.PIPE
|
|
||||||
)
|
|
||||||
stdout, stderr = await proc.communicate()
|
|
||||||
|
|
||||||
title = stdout.decode('utf-8').strip()
|
|
||||||
if not title or 'error' in title.lower():
|
|
||||||
# 尝试备用方法
|
|
||||||
title = f"video_{bvid}"
|
|
||||||
except Exception as e:
|
|
||||||
title = f"video_{bvid}"
|
|
||||||
|
|
||||||
# 构建 yt-dlp 命令
|
|
||||||
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_', '丨')).strip()[:50]
|
|
||||||
|
|
||||||
# 选择视频格式
|
|
||||||
if quality == 80:
|
|
||||||
# 1080P
|
|
||||||
format_str = "30112+30280"
|
|
||||||
elif quality == 64:
|
|
||||||
# 720P
|
|
||||||
format_str = "30064+30280"
|
|
||||||
elif quality == 32:
|
|
||||||
# 480P
|
|
||||||
format_str = "30032+30280"
|
|
||||||
else:
|
|
||||||
# 360P
|
|
||||||
format_str = "30016+30280"
|
|
||||||
|
|
||||||
print(f"下载视频: {title}")
|
|
||||||
print(f"格式: {format_str}")
|
|
||||||
|
|
||||||
output_template = str(self.output_dir / f"{safe_title}.%(ext)s")
|
|
||||||
|
|
||||||
cmd = [
|
|
||||||
"yt-dlp",
|
|
||||||
"-f", format_str,
|
|
||||||
"-o", output_template,
|
|
||||||
"--no-download-archive",
|
|
||||||
"-q",
|
|
||||||
url
|
|
||||||
]
|
|
||||||
|
|
||||||
if cookie_file:
|
|
||||||
cmd.extend(["--cookies", cookie_file])
|
|
||||||
|
|
||||||
print(f"执行命令: {' '.join(cmd[:4])}...")
|
|
||||||
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
|
||||||
*cmd,
|
|
||||||
stdout=asyncio.subprocess.PIPE,
|
|
||||||
stderr=asyncio.subprocess.PIPE
|
|
||||||
)
|
|
||||||
stdout, stderr = await proc.communicate()
|
|
||||||
|
|
||||||
# 查找下载的文件
|
|
||||||
downloaded_files = list(self.output_dir.glob(f"{safe_title}.*"))
|
|
||||||
if downloaded_files:
|
|
||||||
video_file = downloaded_files[0]
|
|
||||||
result["success"] = True
|
|
||||||
result["filepath"] = str(video_file)
|
|
||||||
result["message"] = f"下载完成: {video_file.name}"
|
|
||||||
result["metadata"] = {"title": title, "bvid": bvid}
|
|
||||||
print(f"✅ {result['message']}")
|
|
||||||
else:
|
|
||||||
result["message"] = "未找到下载的文件"
|
|
||||||
print(f"❌ {result['message']}")
|
|
||||||
|
|
||||||
# 清理临时 Cookie 文件
|
|
||||||
if cookie_file and os.path.exists(cookie_file):
|
|
||||||
os.remove(cookie_file)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result["message"] = f"错误: {str(e)}"
|
|
||||||
print(f"❌ {result['message']}")
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def extract_subtitle(self, bvid: str) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
提取字幕
|
|
||||||
|
|
||||||
注意:没有找到官方字幕是正常的!很多视频没有自动生成的字幕。
|
|
||||||
返回 success=True 即使没有字幕,让工作流继续执行后续步骤(如 Whisper)。
|
|
||||||
"""
|
|
||||||
result = {"success": False, "bvid": bvid, "subtitles": [], "has_subtitle": False, "message": ""}
|
|
||||||
|
|
||||||
try:
|
|
||||||
import aiohttp
|
|
||||||
|
|
||||||
api_url = f"https://api.bilibili.com/x/player/pagelist?bvid={bvid}"
|
|
||||||
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
async with session.get(api_url) as resp:
|
|
||||||
data = await resp.json()
|
|
||||||
|
|
||||||
pages = data.get('data', [])
|
|
||||||
if not pages:
|
|
||||||
result["message"] = "未找到分P(这是正常的)"
|
|
||||||
# 仍然返回 success=True,让工作流继续
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
cid = pages[0].get('cid', '')
|
|
||||||
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
async with session.get(f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}") as resp:
|
|
||||||
sub_data = await resp.json()
|
|
||||||
|
|
||||||
subtitles = sub_data.get('data', {}).get('subtitle', {}).get('list', [])
|
|
||||||
|
|
||||||
if not subtitles:
|
|
||||||
result["message"] = "没有官方字幕(这是正常的),将使用 Whisper 进行语音识别"
|
|
||||||
# 仍然返回 success=True,让工作流继续
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
result["success"] = True
|
|
||||||
result["has_subtitle"] = True
|
|
||||||
result["subtitles"] = [{"lan": s.get('lan'), "title": s.get('lan_doc')} for s in subtitles]
|
|
||||||
result["message"] = f"✅ 找到 {len(subtitles)} 个字幕,将使用官方字幕进行深度分析"
|
|
||||||
print(result["message"])
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result["message"] = f"获取字幕出错(这是正常的),将使用 Whisper: {str(e)}"
|
|
||||||
# 出错时也返回 success=True,让工作流继续
|
|
||||||
result["success"] = True
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def download_subtitle(self, bvid: str) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
下载字幕并转换为 SRT 格式
|
|
||||||
|
|
||||||
如果没有官方字幕,返回 success=True 但 has_subtitle=False,
|
|
||||||
让工作流继续使用 Whisper 进行语音识别。
|
|
||||||
"""
|
|
||||||
result = {"success": False, "bvid": bvid, "filepath": None, "has_subtitle": False, "message": ""}
|
|
||||||
|
|
||||||
try:
|
|
||||||
import aiohttp
|
|
||||||
|
|
||||||
# 获取 CID
|
|
||||||
api_url = f"https://api.bilibili.com/x/player/pagelist?bvid={bvid}"
|
|
||||||
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
async with session.get(api_url) as resp:
|
|
||||||
data = await resp.json()
|
|
||||||
|
|
||||||
pages = data.get('data', [])
|
|
||||||
if not pages:
|
|
||||||
result["message"] = "未找到分P,将使用 Whisper 进行语音识别"
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
cid = pages[0].get('cid', '')
|
|
||||||
|
|
||||||
# 获取字幕列表
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
async with session.get(f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}") as resp:
|
|
||||||
sub_data = await resp.json()
|
|
||||||
|
|
||||||
subtitles = sub_data.get('data', {}).get('subtitle', {}).get('list', [])
|
|
||||||
|
|
||||||
if not subtitles:
|
|
||||||
result["message"] = "没有官方字幕,将使用 Whisper 进行语音识别"
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
# 选择中文字幕,如果没有则选第一个
|
|
||||||
zh_sub = None
|
|
||||||
for s in subtitles:
|
|
||||||
if s.get('lan') in ('ai-zh', 'zh-CN', 'zh'):
|
|
||||||
zh_sub = s
|
|
||||||
break
|
|
||||||
if not zh_sub:
|
|
||||||
zh_sub = subtitles[0]
|
|
||||||
|
|
||||||
# 下载字幕文件
|
|
||||||
sub_url = zh_sub.get('subtitle_url', '')
|
|
||||||
if not sub_url:
|
|
||||||
result["message"] = "字幕URL为空,将使用 Whisper"
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
if sub_url.startswith('//'):
|
|
||||||
sub_url = 'https:' + sub_url
|
|
||||||
|
|
||||||
async with aiohttp.ClientSession() as session:
|
|
||||||
async with session.get(sub_url) as resp:
|
|
||||||
sub_content = await resp.json()
|
|
||||||
|
|
||||||
# 转换为 SRT 格式
|
|
||||||
body = sub_content.get('body', [])
|
|
||||||
if not body:
|
|
||||||
result["message"] = "字幕内容为空,将使用 Whisper"
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
def format_srt_time(t: float) -> str:
|
|
||||||
"""将秒数转换为 SRT 时间格式"""
|
|
||||||
h = int(t // 3600)
|
|
||||||
m = int((t % 3600) // 60)
|
|
||||||
s = int(t % 60)
|
|
||||||
ms = int((t - int(t)) * 1000)
|
|
||||||
return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
|
|
||||||
|
|
||||||
srt_lines = []
|
|
||||||
for i, line in enumerate(body, 1):
|
|
||||||
start = line.get('from', 0)
|
|
||||||
end = line.get('to', 0)
|
|
||||||
content = line.get('content', '')
|
|
||||||
srt_lines.extend([str(i), f"{format_srt_time(start)} --> {format_srt_time(end)}", content, ""])
|
|
||||||
|
|
||||||
srt_content = "\n".join(srt_lines)
|
|
||||||
|
|
||||||
# 保存 SRT 文件
|
|
||||||
output_path = self.output_dir / f"{bvid}.srt"
|
|
||||||
with open(output_path, 'w', encoding='utf-8') as f:
|
|
||||||
f.write(srt_content)
|
|
||||||
|
|
||||||
result["success"] = True
|
|
||||||
result["has_subtitle"] = True
|
|
||||||
result["filepath"] = str(output_path)
|
|
||||||
result["message"] = f"✅ 官方字幕已下载: {output_path.name} ({len(body)}条)"
|
|
||||||
print(result["message"])
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result["message"] = f"下载字幕出错: {str(e)},将使用 Whisper"
|
|
||||||
result["success"] = True # 仍然返回成功,让工作流继续
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def extract_article(self, cvid: str) -> Dict[str, Any]:
|
|
||||||
"""获取专栏全文(使用Playwright异步API)"""
|
|
||||||
result = {"success": False, "cvid": cvid, "title": "", "content": "", "message": ""}
|
|
||||||
|
|
||||||
try:
|
|
||||||
from playwright.async_api import async_playwright
|
|
||||||
|
|
||||||
cvid_clean = cvid.replace('cv', '')
|
|
||||||
url = f"https://www.bilibili.com/read/cv{cvid_clean}"
|
|
||||||
print(f"获取专栏: {url}")
|
|
||||||
|
|
||||||
async with async_playwright() as p:
|
|
||||||
browser = await p.chromium.launch(headless=True)
|
|
||||||
page = await browser.new_page()
|
|
||||||
|
|
||||||
await page.goto(url, wait_until="networkidle")
|
|
||||||
await page.wait_for_timeout(3000)
|
|
||||||
|
|
||||||
# 提取标题
|
|
||||||
title_elem = await page.query_selector('h1')
|
|
||||||
if title_elem:
|
|
||||||
result["title"] = (await title_elem.inner_text()).strip()
|
|
||||||
print(f"标题: {result['title'][:50]}...")
|
|
||||||
|
|
||||||
# 提取正文内容
|
|
||||||
content_elem = await page.query_selector('.read-article-holder')
|
|
||||||
if content_elem:
|
|
||||||
html_content = await content_elem.inner_html()
|
|
||||||
text_content = await content_elem.inner_text()
|
|
||||||
result["content"] = text_content.strip()
|
|
||||||
result["html_content"] = html_content.strip()
|
|
||||||
result["success"] = bool(result["title"] and result["content"])
|
|
||||||
print(f"内容长度: {len(result['content'])} 字符")
|
|
||||||
else:
|
|
||||||
result["message"] = "未找到专栏内容"
|
|
||||||
|
|
||||||
await browser.close()
|
|
||||||
|
|
||||||
if result["success"]:
|
|
||||||
result["message"] = f"✅ 获取成功: {result['title'][:30]}..."
|
|
||||||
else:
|
|
||||||
result["message"] = f"获取失败: {result.get('message', '未知错误')}"
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result["message"] = f"错误: {str(e)}"
|
|
||||||
print(f"错误: {e}")
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
|
||||||
parser = argparse.ArgumentParser(description='Bilibili Content Downloader')
|
|
||||||
parser.add_argument('--id', '-i', required=True, help='B站内容ID')
|
|
||||||
parser.add_argument('--type', '-t', required=True, choices=['video', 'subtitle', 'subtitle-download', 'article'], help='内容类型')
|
|
||||||
parser.add_argument('--output', '-o', default='./output', help='输出目录')
|
|
||||||
parser.add_argument('--quality', '-q', type=int, default=64, help='视频质量: 16=360P, 32=480P, 64=720P, 80=1080P')
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
cookie = os.environ.get('BILIBILI_COOKIE')
|
|
||||||
downloader = BilibiliContent(output_dir=args.output, cookie=cookie)
|
|
||||||
|
|
||||||
if args.type == 'video':
|
|
||||||
result = await downloader.download_video(args.id, quality=args.quality)
|
|
||||||
elif args.type == 'subtitle':
|
|
||||||
result = await downloader.extract_subtitle(args.id)
|
|
||||||
elif args.type == 'subtitle-download':
|
|
||||||
# 下载并转换字幕为 SRT 格式
|
|
||||||
result = await downloader.download_subtitle(args.id)
|
|
||||||
else:
|
|
||||||
result = await downloader.extract_article(args.id.replace('cv', ''))
|
|
||||||
|
|
||||||
# 统一输出格式
|
|
||||||
if result.get("success"):
|
|
||||||
output = {
|
|
||||||
"success": True,
|
|
||||||
"type": args.type,
|
|
||||||
"has_subtitle": result.get("has_subtitle", False),
|
|
||||||
"filename": Path(result.get('filepath', '')).name if result.get('filepath') else None,
|
|
||||||
"filepath": str(Path(result.get('filepath', '')).resolve()) if result.get('filepath') else None,
|
|
||||||
"message": result.get('message', '处理成功')
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
output = {
|
|
||||||
"success": False,
|
|
||||||
"error": result.get('message', '处理失败'),
|
|
||||||
"detail": f"ID: {args.id}, 类型: {args.type}"
|
|
||||||
}
|
|
||||||
|
|
||||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
try:
|
|
||||||
result = asyncio.run(main())
|
|
||||||
sys.exit(0 if result.get("success") else 1)
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
print(json.dumps({"success": False, "error": "用户取消"}, ensure_ascii=False, indent=2))
|
|
||||||
sys.exit(1)
|
|
||||||
except Exception as e:
|
|
||||||
print(json.dumps({"success": False, "error": str(e)}, ensure_ascii=False, indent=2))
|
|
||||||
sys.exit(1)
|
|
||||||
@@ -1,155 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
B站视频提取脚本
|
|
||||||
提取视频信息、统计数据、字幕并生成Markdown笔记
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
# 添加skills路径
|
|
||||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb'))
|
|
||||||
|
|
||||||
from src.extractors.bilibili_extractor import BilibiliExtractor
|
|
||||||
from src.config import get_config
|
|
||||||
|
|
||||||
|
|
||||||
def format_number(num):
|
|
||||||
"""格式化数字"""
|
|
||||||
if num >= 100000000:
|
|
||||||
return f"{num / 100000000:.1f}亿"
|
|
||||||
elif num >= 10000:
|
|
||||||
return f"{num / 10000:.1f}万"
|
|
||||||
else:
|
|
||||||
return f"{num:,}"
|
|
||||||
|
|
||||||
|
|
||||||
def generate_markdown(result, bvid):
|
|
||||||
"""生成Markdown笔记"""
|
|
||||||
metadata = result['metadata']
|
|
||||||
stats = result['stats']
|
|
||||||
description = result['description']
|
|
||||||
subtitle = result['subtitle']
|
|
||||||
|
|
||||||
# 获取当前时间
|
|
||||||
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|
||||||
|
|
||||||
md = f"""# {metadata['title']}
|
|
||||||
|
|
||||||
> **来源**: [B站](https://www.bilibili.com/video/{bvid})
|
|
||||||
> **作者**: {metadata['author']}
|
|
||||||
> **发布时间**: {metadata['pubtime']}
|
|
||||||
> **提取时间**: {now}
|
|
||||||
> **时长**: {metadata['duration']}
|
|
||||||
|
|
||||||
## 📊 统计数据
|
|
||||||
|
|
||||||
| 指标 | 数量 |
|
|
||||||
|------|------|
|
|
||||||
| 👀 播放 | {format_number(stats['view'])} |
|
|
||||||
| 💬 弹幕 | {format_number(stats['danmaku'])} |
|
|
||||||
| 📝 评论 | {format_number(stats['reply'])} |
|
|
||||||
| ⭐ 收藏 | {format_number(stats['favorite'])} |
|
|
||||||
| 🪙 投币 | {format_number(stats['coin'])} |
|
|
||||||
| 🔗 分享 | {format_number(stats['share'])} |
|
|
||||||
| 👍 点赞 | {format_number(stats['like'])} |
|
|
||||||
|
|
||||||
## 📝 视频简介
|
|
||||||
|
|
||||||
{description if description else '(暂无简介)'}
|
|
||||||
|
|
||||||
## 📌 标签
|
|
||||||
|
|
||||||
{metadata['tags'] if metadata['tags'] else '(无标签)'}
|
|
||||||
|
|
||||||
## 🎬 字幕
|
|
||||||
|
|
||||||
{subtitle if subtitle else '(无字幕)'}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**元数据**:
|
|
||||||
- BV号: `{metadata['bvid']}`
|
|
||||||
- AV号: `{metadata['aid']}`
|
|
||||||
- UP主ID: `{metadata['mid']}`
|
|
||||||
"""
|
|
||||||
|
|
||||||
return md
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""主函数"""
|
|
||||||
# BV号
|
|
||||||
bvid = "BV1N5foBjE5K" # 从 https://b23.tv/btMGoOi 解析得到
|
|
||||||
|
|
||||||
print(f"开始提取B站视频: {bvid}")
|
|
||||||
print("=" * 60)
|
|
||||||
|
|
||||||
# 获取配置
|
|
||||||
try:
|
|
||||||
config = get_config()
|
|
||||||
except Exception as e:
|
|
||||||
print(f"⚠️ 获取配置失败,使用默认配置: {e}")
|
|
||||||
config = None
|
|
||||||
|
|
||||||
# 创建提取器
|
|
||||||
extractor = BilibiliExtractor(config=config)
|
|
||||||
|
|
||||||
# 提取视频信息
|
|
||||||
try:
|
|
||||||
result = extractor.extract(bvid)
|
|
||||||
|
|
||||||
print("\n✅ 提取成功!\n")
|
|
||||||
print("=" * 60)
|
|
||||||
print("基本信息:")
|
|
||||||
print("=" * 60)
|
|
||||||
print(f"标题: {result['metadata']['title']}")
|
|
||||||
print(f"作者: {result['metadata']['author']}")
|
|
||||||
print(f"时长: {result['metadata']['duration']}")
|
|
||||||
print(f"发布: {result['metadata']['pubtime']}")
|
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
|
||||||
print("统计数据:")
|
|
||||||
print("=" * 60)
|
|
||||||
print(f"播放: {format_number(result['stats']['view'])}")
|
|
||||||
print(f"弹幕: {format_number(result['stats']['danmaku'])}")
|
|
||||||
print(f"评论: {format_number(result['stats']['reply'])}")
|
|
||||||
print(f"收藏: {format_number(result['stats']['favorite'])}")
|
|
||||||
print(f"投币: {format_number(result['stats']['coin'])}")
|
|
||||||
print(f"点赞: {format_number(result['stats']['like'])}")
|
|
||||||
|
|
||||||
# 生成 Markdown 笔记
|
|
||||||
md_content = generate_markdown(result, bvid)
|
|
||||||
|
|
||||||
# 确保 notes 目录存在
|
|
||||||
notes_dir = "/home/admin/.openclaw/workspace/notes"
|
|
||||||
os.makedirs(notes_dir, exist_ok=True)
|
|
||||||
|
|
||||||
# 生成文件名(使用BV号作为文件名)
|
|
||||||
filename = f"{bvid}.md"
|
|
||||||
filepath = os.path.join(notes_dir, filename)
|
|
||||||
|
|
||||||
# 写入文件
|
|
||||||
with open(filepath, 'w', encoding='utf-8') as f:
|
|
||||||
f.write(md_content)
|
|
||||||
|
|
||||||
print("\n" + "=" * 60)
|
|
||||||
print(f"✅ Markdown笔记已生成: {filepath}")
|
|
||||||
print("=" * 60)
|
|
||||||
|
|
||||||
# 返回笔记内容
|
|
||||||
return md_content
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"\n❌ 提取失败: {e}")
|
|
||||||
import traceback
|
|
||||||
traceback.print_exc()
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,175 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
使用旧版 API 获取并下载视频流
|
|
||||||
"""
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from playwright.async_api import async_playwright
|
|
||||||
|
|
||||||
async def download_video_stream(bvid, output_dir="/tmp/bilibili_downloads"):
|
|
||||||
"""下载视频流"""
|
|
||||||
url = f"https://www.bilibili.com/video/{bvid}"
|
|
||||||
|
|
||||||
# 创建输出目录
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
|
|
||||||
async with async_playwright() as playwright:
|
|
||||||
browser = await playwright.chromium.launch(headless=True)
|
|
||||||
context = await browser.new_context()
|
|
||||||
page = await context.new_page()
|
|
||||||
|
|
||||||
print(f"正在下载视频: {url}")
|
|
||||||
print("=" * 80)
|
|
||||||
|
|
||||||
# 首先获取 aid 和 cid
|
|
||||||
print("\n1. 获取视频信息...")
|
|
||||||
try:
|
|
||||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
|
||||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
|
||||||
data = await response.json()
|
|
||||||
|
|
||||||
if data.get('code') != 0:
|
|
||||||
print(f" ❌ 失败: {data.get('message')}")
|
|
||||||
await browser.close()
|
|
||||||
return None
|
|
||||||
|
|
||||||
video_data = data.get('data', {})
|
|
||||||
title = video_data.get('title')
|
|
||||||
aid = video_data.get('aid')
|
|
||||||
cid = video_data.get('cid')
|
|
||||||
duration = video_data.get('duration')
|
|
||||||
print(f" 标题: {title}")
|
|
||||||
print(f" 时长: {duration}秒")
|
|
||||||
except Exception as e:
|
|
||||||
print(f" ❌ 异常: {e}")
|
|
||||||
await browser.close()
|
|
||||||
return None
|
|
||||||
|
|
||||||
# 获取视频流 URL
|
|
||||||
print("\n2. 获取视频流 URL...")
|
|
||||||
try:
|
|
||||||
playurl_api = "https://api.bilibili.com/x/player/playurl"
|
|
||||||
params = {
|
|
||||||
'bvid': bvid,
|
|
||||||
'cid': cid,
|
|
||||||
'qn': 80, # 1080P
|
|
||||||
'otype': 'json',
|
|
||||||
'fnver': 0,
|
|
||||||
'fnval': 0, # FLV 格式
|
|
||||||
}
|
|
||||||
|
|
||||||
playurl_response = await page.request.get(playurl_api, params=params)
|
|
||||||
|
|
||||||
if not playurl_response.ok:
|
|
||||||
print(f" ❌ HTTP 错误: {playurl_response.status}")
|
|
||||||
await browser.close()
|
|
||||||
return None
|
|
||||||
|
|
||||||
playurl_data = await playurl_response.json()
|
|
||||||
|
|
||||||
if playurl_data.get('code') != 0:
|
|
||||||
print(f" ❌ API 错误: {playurl_data.get('message')}")
|
|
||||||
await browser.close()
|
|
||||||
return None
|
|
||||||
|
|
||||||
play_data = playurl_data.get('data', {})
|
|
||||||
|
|
||||||
# 检查支持的清晰度
|
|
||||||
print(f"\n 支持的清晰度:")
|
|
||||||
for i, (q, desc) in enumerate(zip(play_data.get('accept_quality', []), play_data.get('accept_description', []))):
|
|
||||||
current = " (当前)" if q == play_data.get('quality') else ""
|
|
||||||
print(f" {desc} ({q}){current}")
|
|
||||||
|
|
||||||
# 获取视频 URL
|
|
||||||
durl = play_data.get('durl', [])
|
|
||||||
if not durl:
|
|
||||||
print(f" ❌ 没有找到视频流")
|
|
||||||
await browser.close()
|
|
||||||
return None
|
|
||||||
|
|
||||||
print(f"\n 找到 {len(durl)} 个视频分段")
|
|
||||||
video_urls = []
|
|
||||||
for i, segment in enumerate(durl):
|
|
||||||
segment_url = segment.get('url')
|
|
||||||
segment_size = segment.get('size', 0)
|
|
||||||
video_urls.append(segment_url)
|
|
||||||
print(f" 分段 {i+1}: {segment_size / 1024 / 1024:.2f}MB")
|
|
||||||
print(f" {segment_url[:100]}...")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f" ❌ 异常: {e}")
|
|
||||||
import traceback
|
|
||||||
traceback.print_exc()
|
|
||||||
await browser.close()
|
|
||||||
return None
|
|
||||||
|
|
||||||
# 下载视频
|
|
||||||
print(f"\n3. 开始下载视频...")
|
|
||||||
all_segments = []
|
|
||||||
total_size = 0
|
|
||||||
|
|
||||||
for i, video_url in enumerate(video_urls, 1):
|
|
||||||
print(f"\n 下载分段 {i}/{len(video_urls)}...")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 添加必要的请求头
|
|
||||||
headers = {
|
|
||||||
'Referer': 'https://www.bilibili.com/',
|
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
|
||||||
}
|
|
||||||
|
|
||||||
# 下载分段
|
|
||||||
segment_response = await page.request.get(
|
|
||||||
video_url,
|
|
||||||
headers=headers,
|
|
||||||
timeout=60000
|
|
||||||
)
|
|
||||||
|
|
||||||
if not segment_response.ok:
|
|
||||||
print(f" ❌ 下载失败: {segment_response.status}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
segment_data = await segment_response.body()
|
|
||||||
segment_size = len(segment_data)
|
|
||||||
total_size += segment_size
|
|
||||||
all_segments.append(segment_data)
|
|
||||||
|
|
||||||
print(f" ✅ 下载成功: {segment_size / 1024 / 1024:.2f}MB")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f" ❌ 下载异常: {e}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not all_segments:
|
|
||||||
print(f"\n ❌ 没有成功下载任何分段")
|
|
||||||
await browser.close()
|
|
||||||
return None
|
|
||||||
|
|
||||||
# 合并分段
|
|
||||||
print(f"\n4. 合并视频分段...")
|
|
||||||
video_data = b''.join(all_segments)
|
|
||||||
|
|
||||||
# 保存文件
|
|
||||||
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
|
|
||||||
output_path = os.path.join(output_dir, f"{safe_title}.flv")
|
|
||||||
|
|
||||||
with open(output_path, 'wb') as f:
|
|
||||||
f.write(video_data)
|
|
||||||
|
|
||||||
print(f" ✅ 视频已保存: {output_path}")
|
|
||||||
print(f" 文件大小: {total_size / 1024 / 1024:.2f}MB")
|
|
||||||
|
|
||||||
await browser.close()
|
|
||||||
|
|
||||||
return output_path
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
bvid = "BV1KtFzzqEUG"
|
|
||||||
result = asyncio.run(download_video_stream(bvid))
|
|
||||||
if result:
|
|
||||||
print(f"\n✅ 下载完成!")
|
|
||||||
print(f"文件路径: {result}")
|
|
||||||
else:
|
|
||||||
print(f"\n❌ 下载失败")
|
|
||||||
@@ -141,14 +141,26 @@ class DeepAnalyzer:
|
|||||||
|
|
||||||
# 标准化元数据
|
# 标准化元数据
|
||||||
metadata = self._normalize_metadata(metadata)
|
metadata = self._normalize_metadata(metadata)
|
||||||
|
|
||||||
title = metadata.get('title', '未知')
|
title = metadata.get('title', '未知')
|
||||||
duration = metadata.get('duration', 0)
|
duration = metadata.get('duration', 0)
|
||||||
author = metadata.get('author', '')
|
author = metadata.get('author', '')
|
||||||
pubtime = metadata.get('pubtime', '')
|
pubtime = metadata.get('pubtime', '')
|
||||||
url = metadata.get('url', '')
|
url = metadata.get('url', '')
|
||||||
bvid = metadata.get('bvid', '')
|
bvid = metadata.get('bvid', '')
|
||||||
platform = metadata.get('platform', 'bilibili')
|
|
||||||
|
# 推断 platform:优先使用元数据中的值,否则根据 URL 或 content_type 推断
|
||||||
|
platform = metadata.get('platform', '')
|
||||||
|
if not platform:
|
||||||
|
if 'zhihu.com' in url or content_type in ('answer', 'zhuanlan', 'question'):
|
||||||
|
platform = 'zhihu'
|
||||||
|
elif 'bilibili.com' in url or bvid:
|
||||||
|
platform = 'bilibili'
|
||||||
|
else:
|
||||||
|
platform = 'bilibili' # 默认值
|
||||||
|
|
||||||
|
# 清理 author 字段:移除零宽度空格和前后空白
|
||||||
|
author = self._clean_author(author)
|
||||||
|
|
||||||
# 获取内容 - 支持多种字段名
|
# 获取内容 - 支持多种字段名
|
||||||
text = content.get('text', '') or content.get('subtitle', '') or content.get('subtitles', '') or content.get('description', '')
|
text = content.get('text', '') or content.get('subtitle', '') or content.get('subtitles', '') or content.get('description', '')
|
||||||
@@ -186,11 +198,11 @@ class DeepAnalyzer:
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def _build_prompt(self, title: str, duration: int, duration_str: str, author: str,
|
def _build_prompt(self, title: str, duration: int, duration_str: str, author: str,
|
||||||
pubtime: str, url: str, bvid: str, platform: str, text: str,
|
pubtime: str, url: str, bvid: str, platform: str, text: str,
|
||||||
content_type: str = "video") -> str:
|
content_type: str = "video") -> str:
|
||||||
"""构建分析 prompt,根据内容类型选择不同模板"""
|
"""构建分析 prompt,根据内容类型选择不同模板"""
|
||||||
if content_type == "article":
|
if content_type in ("article", "answer", "zhuanlan"):
|
||||||
return self._build_article_prompt(title, author, pubtime, url, platform, text)
|
return self._build_article_prompt(title, author, pubtime, url, platform, text)
|
||||||
else:
|
else:
|
||||||
return self._build_video_prompt(title, duration, duration_str, author, pubtime, url, bvid, platform, text)
|
return self._build_video_prompt(title, duration, duration_str, author, pubtime, url, bvid, platform, text)
|
||||||
@@ -422,6 +434,22 @@ URL:{url}
|
|||||||
return f"{h}:{m:02d}:{s:02d}"
|
return f"{h}:{m:02d}:{s:02d}"
|
||||||
return f"{m}:{s:02d}"
|
return f"{m}:{s:02d}"
|
||||||
|
|
||||||
|
def _clean_author(self, author: str) -> str:
|
||||||
|
"""清理 author 字段:移除零宽度空格和前后空白"""
|
||||||
|
if not author:
|
||||||
|
return ''
|
||||||
|
# 移除零宽度空格 (U+200B) 和其他不可见字符
|
||||||
|
invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u00ad']
|
||||||
|
cleaned = author
|
||||||
|
for char in invisible_chars:
|
||||||
|
cleaned = cleaned.replace(char, '')
|
||||||
|
# 移除前后空白和换行
|
||||||
|
cleaned = cleaned.strip()
|
||||||
|
# 如果包含换行符,只取第一行
|
||||||
|
if '\n' in cleaned:
|
||||||
|
cleaned = cleaned.split('\n')[0].strip()
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
# ============== 工具函数 ==============
|
# ============== 工具函数 ==============
|
||||||
|
|
||||||
@@ -437,19 +465,34 @@ def generate_frontmatter(metadata: Dict, result: AnalysisResult, content_type: s
|
|||||||
"""生成 Markdown frontmatter"""
|
"""生成 Markdown frontmatter"""
|
||||||
# 提取标签
|
# 提取标签
|
||||||
tags = result.recommended_tags or []
|
tags = result.recommended_tags or []
|
||||||
|
|
||||||
# 格式化日期
|
# 格式化日期
|
||||||
pubtime = metadata.get('pubtime', '')
|
pubtime = metadata.get('pubtime', '')
|
||||||
if pubtime:
|
if pubtime:
|
||||||
date_str = str(pubtime)[:10]
|
date_str = str(pubtime)[:10]
|
||||||
else:
|
else:
|
||||||
date_str = datetime.now().strftime('%Y-%m-%d')
|
date_str = datetime.now().strftime('%Y-%m-%d')
|
||||||
|
|
||||||
fm = ["---",
|
# 清理 author 字段
|
||||||
|
author = metadata.get('author', '未知')
|
||||||
|
author = _clean_string(author)
|
||||||
|
|
||||||
|
# 推断 platform
|
||||||
|
url = metadata.get('url', '')
|
||||||
|
platform = metadata.get('platform', '')
|
||||||
|
if not platform:
|
||||||
|
if 'zhihu.com' in url or content_type in ('answer', 'zhuanlan', 'question'):
|
||||||
|
platform = 'zhihu'
|
||||||
|
elif 'bilibili.com' in url or metadata.get('bvid'):
|
||||||
|
platform = 'bilibili'
|
||||||
|
else:
|
||||||
|
platform = 'bilibili'
|
||||||
|
|
||||||
|
fm = ["---",
|
||||||
f"title: \"{metadata.get('title', '未命名')}\"",
|
f"title: \"{metadata.get('title', '未命名')}\"",
|
||||||
f"date: {date_str}",
|
f"date: {date_str}",
|
||||||
f"author: {metadata.get('author', '未知')}",
|
f"author: {author}",
|
||||||
f"platform: {metadata.get('platform', 'bilibili')}"]
|
f"platform: {platform}"]
|
||||||
|
|
||||||
# 视频类型添加 duration 和 bvid
|
# 视频类型添加 duration 和 bvid
|
||||||
if content_type == "video":
|
if content_type == "video":
|
||||||
@@ -470,6 +513,23 @@ def generate_frontmatter(metadata: Dict, result: AnalysisResult, content_type: s
|
|||||||
return '\n'.join(fm)
|
return '\n'.join(fm)
|
||||||
|
|
||||||
|
|
||||||
|
def _clean_string(text: str) -> str:
|
||||||
|
"""清理字符串:移除零宽度空格和前后空白"""
|
||||||
|
if not text:
|
||||||
|
return ''
|
||||||
|
# 移除零宽度空格 (U+200B) 和其他不可见字符
|
||||||
|
invisible_chars = ['\u200b', '\u200c', '\u200d', '\ufeff', '\u00ad']
|
||||||
|
cleaned = text
|
||||||
|
for char in invisible_chars:
|
||||||
|
cleaned = cleaned.replace(char, '')
|
||||||
|
# 移除前后空白和换行
|
||||||
|
cleaned = cleaned.strip()
|
||||||
|
# 如果包含换行符,只取第一行
|
||||||
|
if '\n' in cleaned:
|
||||||
|
cleaned = cleaned.split('\n')[0].strip()
|
||||||
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
# ============== 主程序 ==============
|
# ============== 主程序 ==============
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
# FunASR 模型缓存目录
|
|
||||||
FUNASR_CACHE_DIR=~/.cache/modelscope/hub
|
|
||||||
@@ -1,212 +0,0 @@
|
|||||||
# KB FunASR - 本地语音识别
|
|
||||||
|
|
||||||
基于阿里 FunASR 的本地语音识别工具,支持将音视频文件转换为文本和 JSON 格式。
|
|
||||||
|
|
||||||
## 功能特性
|
|
||||||
|
|
||||||
- 🎯 **本地执行** - 完全离线运行,保护隐私
|
|
||||||
- 🚀 **多模型支持** - Paraformer、SenseVoice 等模型可选
|
|
||||||
- 📝 **多格式输出** - JSON、TXT 两种格式
|
|
||||||
- ⚡ **异步执行** - 支持后台运行,不阻塞主代理
|
|
||||||
- 🎬 **广泛兼容** - 支持常见音视频格式(MP3、MP4、WAV 等)
|
|
||||||
- 🇨🇳 **中文优化** - Paraformer 模型针对中文语音优化
|
|
||||||
|
|
||||||
## 安装前提
|
|
||||||
|
|
||||||
### 1. 安装依赖
|
|
||||||
|
|
||||||
```bash
|
|
||||||
pip install funasr modelscope
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. 配置环境变量(可选)
|
|
||||||
|
|
||||||
在 `~/.clawdbot/clawdbot.json` 中添加:
|
|
||||||
|
|
||||||
```json5
|
|
||||||
{
|
|
||||||
skills: {
|
|
||||||
entries: {
|
|
||||||
"kb-funasr": {
|
|
||||||
env: {
|
|
||||||
FUNASR_CACHE_DIR: "~/.cache/modelscope/hub"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
或通过命令行配置:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
clawdbot skills config kb-funasr FUNASR_CACHE_DIR "~/.cache/modelscope/hub"
|
|
||||||
```
|
|
||||||
|
|
||||||
## 使用方法
|
|
||||||
|
|
||||||
### 基本用法
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 使用默认 Paraformer 中文模型处理
|
|
||||||
python funasr.py --input video.mp4
|
|
||||||
|
|
||||||
# 后台异步执行
|
|
||||||
python funasr.py --input video.mp4 --background
|
|
||||||
```
|
|
||||||
|
|
||||||
### 指定模型
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Paraformer 中文模型(推荐)
|
|
||||||
python funasr.py --input video.mp4 --model paraformer-zh
|
|
||||||
|
|
||||||
# Paraformer 多语言模型
|
|
||||||
python funasr.py --input video.mp4 --model paraformer
|
|
||||||
|
|
||||||
# SenseVoice 多语言模型
|
|
||||||
python funasr.py --input video.mp4 --model sensevoice
|
|
||||||
```
|
|
||||||
|
|
||||||
### 指定输出目录
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python funasr.py --input video.mp4 --output ./transcripts
|
|
||||||
```
|
|
||||||
|
|
||||||
## 输出文件
|
|
||||||
|
|
||||||
对于输入 `video.mp4`,将生成以下文件:
|
|
||||||
|
|
||||||
```
|
|
||||||
output/
|
|
||||||
├── video.json # 详细结果(含时间戳)
|
|
||||||
└── video.txt # 纯文本
|
|
||||||
```
|
|
||||||
|
|
||||||
### JSON 格式
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"input_file": "video.mp4",
|
|
||||||
"model": "paraformer-zh",
|
|
||||||
"output_files": {
|
|
||||||
"json": "video.json",
|
|
||||||
"txt": "video.txt"
|
|
||||||
},
|
|
||||||
"duration_seconds": 120.5,
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 0,
|
|
||||||
"start": 0.0,
|
|
||||||
"end": 3.2,
|
|
||||||
"text": "这是第一段识别结果"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"start": 3.2,
|
|
||||||
"end": 6.8,
|
|
||||||
"text": "这是第二段识别结果"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 模型对比
|
|
||||||
|
|
||||||
| 模型 | 速度 | 精度 | 适用场景 |
|
|
||||||
|------|------|------|----------|
|
|
||||||
| paraformer-zh | ⚡⚡⚡ | ⭐⭐⭐⭐ | 中文语音(推荐) |
|
|
||||||
| paraformer | ⚡⚡ | ⭐⭐⭐⭐ | 多语言识别 |
|
|
||||||
| sensevoice | ⚡ | ⭐⭐⭐⭐⭐ | 高精度多语言 |
|
|
||||||
|
|
||||||
## 完整示例
|
|
||||||
|
|
||||||
### 示例 1: 转录播客
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python funasr.py --input podcast.mp3 --model paraformer-zh --background
|
|
||||||
```
|
|
||||||
|
|
||||||
### 示例 2: 生成视频识别结果
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python funasr.py --input tutorial.mp4 --model paraformer
|
|
||||||
```
|
|
||||||
|
|
||||||
### 示例 3: 批量处理(配合脚本)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
#!/bin/bash
|
|
||||||
for video in videos/*.mp4; do
|
|
||||||
python funasr.py --input "$video" --model paraformer-zh
|
|
||||||
done
|
|
||||||
```
|
|
||||||
|
|
||||||
## 在 Workflow 中使用
|
|
||||||
|
|
||||||
在知识库处理流程中,如果视频没有识别结果:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 步骤4: 下载视频后
|
|
||||||
# 步骤5: 使用 FunASR 生成识别结果
|
|
||||||
python funasr.py --input ${video_path} --model paraformer-zh --background
|
|
||||||
|
|
||||||
# 等待 FunASR 完成,然后继续后续步骤
|
|
||||||
```
|
|
||||||
|
|
||||||
## 技术细节
|
|
||||||
|
|
||||||
- **引擎**: 阿里 FunASR
|
|
||||||
- **模型框架**: ModelScope
|
|
||||||
- **音频采样率**: 自动转换为 16kHz
|
|
||||||
- **语言**: 根据模型支持不同语言
|
|
||||||
|
|
||||||
## 故障排查
|
|
||||||
|
|
||||||
### 问题: 模型下载失败
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 首次运行会自动下载,确保网络稳定
|
|
||||||
python funasr.py --input test.mp4
|
|
||||||
|
|
||||||
# 手动检查缓存目录
|
|
||||||
ls ~/.cache/modelscope/hub/
|
|
||||||
```
|
|
||||||
|
|
||||||
### 问题: 音频格式不支持
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 转换音频格式
|
|
||||||
ffmpeg -i input.avi -acodec libmp3lame output.mp3
|
|
||||||
python funasr.py --input output.mp3
|
|
||||||
```
|
|
||||||
|
|
||||||
### 问题: 依赖安装失败
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 使用清华镜像源
|
|
||||||
pip install funasr modelscope -i https://pypi.tuna.tsinghua.edu.cn/simple
|
|
||||||
```
|
|
||||||
|
|
||||||
## 性能参考
|
|
||||||
|
|
||||||
在 CPU 环境下(Intel i7):
|
|
||||||
|
|
||||||
- **paraformer-zh 模型**: ~2x 实时速度(1分钟音频 ~30秒处理)
|
|
||||||
- **paraformer 模型**: ~1.5x 实时速度(1分钟音频 ~40秒处理)
|
|
||||||
- **sensevoice 模型**: ~1x 实时速度(1分钟音频 ~1分钟处理)
|
|
||||||
|
|
||||||
## 与 Whisper 对比
|
|
||||||
|
|
||||||
| 特性 | FunASR | Whisper |
|
|
||||||
|------|--------|---------|
|
|
||||||
| 中文识别 | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ |
|
|
||||||
| 速度 | ⚡⚡⚡ | ⚡⚡ |
|
|
||||||
| 模型大小 | 较小 | 较大 |
|
|
||||||
| 多语言支持 | ⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
|
|
||||||
|
|
||||||
## 许可证
|
|
||||||
|
|
||||||
本技能使用 MIT 许可证,FunASR 使用 Apache 2.0 许可证。
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
---
|
|
||||||
name: kb-funasr
|
|
||||||
description: 本地语音识别,支持音视频转文字(基于阿里 FunASR)
|
|
||||||
---
|
|
||||||
|
|
||||||
# KB FunASR Skill
|
|
||||||
|
|
||||||
**kb-funasr** - 本地语音识别,使用阿里 FunASR 将音视频文件转换为文本和 JSON 格式输出。
|
|
||||||
|
|
||||||
## 什么时候使用
|
|
||||||
|
|
||||||
当用户需要:
|
|
||||||
- 将视频/音频文件转换为文字
|
|
||||||
- 获取带时间戳的详细识别结果
|
|
||||||
- 需要中文优化的语音识别(Paraformer 模型)
|
|
||||||
|
|
||||||
## 环境变量
|
|
||||||
|
|
||||||
在使用之前,必须检查环境变量是否已经设置。
|
|
||||||
|
|
||||||
| 变量名 | 用途 | 示例 |
|
|
||||||
|--------|------|------|
|
|
||||||
| `FUNASR_CACHE_DIR` | FunASR 模型缓存目录 | `~/.cache/modelscope/hub` |
|
|
||||||
|
|
||||||
## 使用方法
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 基本用法(使用 Paraformer 模型)
|
|
||||||
python funasr.py --input video.mp4
|
|
||||||
|
|
||||||
# 指定模型
|
|
||||||
python funasr.py --input video.mp4 --model paraformer-zh
|
|
||||||
|
|
||||||
# 指定输出目录
|
|
||||||
python funasr.py --input video.mp4 --output ./output
|
|
||||||
|
|
||||||
# 异步执行(推荐)
|
|
||||||
python funasr.py --input video.mp4 --background
|
|
||||||
```
|
|
||||||
|
|
||||||
## 命令参数
|
|
||||||
|
|
||||||
| 参数 | 说明 | 默认值 |
|
|
||||||
|------|------|--------|
|
|
||||||
| `--input` | 输入音视频文件路径 | 必需 |
|
|
||||||
| `--model` | FunASR 模型类型 | `paraformer-zh` |
|
|
||||||
| `--output` | 输出目录 | 文件所在目录 |
|
|
||||||
| `--background` | 后台异步执行 | False |
|
|
||||||
|
|
||||||
## 模型选项
|
|
||||||
|
|
||||||
- `paraformer-zh` - Paraformer 中文模型(轻量,中文优化,推荐)
|
|
||||||
- `paraformer` - Paraformer 多语言模型
|
|
||||||
- `sensevoice` - SenseVoice 多语言模型
|
|
||||||
|
|
||||||
## 输出文件
|
|
||||||
|
|
||||||
对于输入 `video.mp4`,将生成:
|
|
||||||
|
|
||||||
- `video.json` - 详细结果(包含时间戳、分段信息)
|
|
||||||
- `video.txt` - 纯文本内容
|
|
||||||
|
|
||||||
## 输出格式示例
|
|
||||||
|
|
||||||
### JSON 格式
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"input_file": "video.mp4",
|
|
||||||
"model": "paraformer-zh",
|
|
||||||
"output_files": {
|
|
||||||
"json": "video.json",
|
|
||||||
"txt": "video.txt"
|
|
||||||
},
|
|
||||||
"duration_seconds": 120.5,
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 0,
|
|
||||||
"start": 0.0,
|
|
||||||
"end": 3.2,
|
|
||||||
"text": "这是第一段识别结果"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Workflow 中的使用
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 步骤4之后:如果已有音频但无字幕,可以使用 FunASR
|
|
||||||
python funasr.py --input ${video_path} --model paraformer-zh --background
|
|
||||||
```
|
|
||||||
|
|
||||||
## 安装
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. 安装依赖
|
|
||||||
pip install funasr modelscope
|
|
||||||
|
|
||||||
# 2. 首次运行时会自动下载模型到缓存目录
|
|
||||||
python funasr.py --input test.mp4
|
|
||||||
|
|
||||||
# 3. 复制技能到 skills 目录
|
|
||||||
cp -r kb-funasr/ ~/skills/
|
|
||||||
```
|
|
||||||
|
|
||||||
## 注意事项
|
|
||||||
|
|
||||||
- **异步执行**: 使用 `--background` 参数时,命令会在后台运行,不阻塞主代理
|
|
||||||
- **模型选择**: paraformer-zh 模型适合中文语音识别,速度较快,精度较高
|
|
||||||
- **文件格式**: 支持 MP3、MP4、WAV、FLAC、OGG、M4A 等常见格式
|
|
||||||
- **首次运行**: 首次运行时会自动下载模型文件,需要稳定的网络连接
|
|
||||||
Binary file not shown.
@@ -1,313 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
KB FunASR - 本地语音识别工具
|
|
||||||
基于阿里 FunASR 实现音视频转文字
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from pathlib import Path
|
|
||||||
from datetime import datetime
|
|
||||||
from typing import Dict, List, Optional
|
|
||||||
|
|
||||||
|
|
||||||
class FunASRTranscriber:
|
|
||||||
"""FunASR 语音识别器"""
|
|
||||||
|
|
||||||
# 模型映射
|
|
||||||
MODEL_MAP = {
|
|
||||||
"paraformer-zh": "damo/speech_paraformer-large-vad_punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
|
||||||
"paraformer": "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
|
||||||
"sensevoice": "FunASR/SenseVoiceSmall"
|
|
||||||
}
|
|
||||||
|
|
||||||
def __init__(self, model: str = "paraformer-zh", cache_dir: Optional[str] = None):
|
|
||||||
"""初始化 FunASR 识别器
|
|
||||||
|
|
||||||
Args:
|
|
||||||
model: 模型名称
|
|
||||||
cache_dir: 模型缓存目录
|
|
||||||
"""
|
|
||||||
self.model_name = model
|
|
||||||
self.model_id = self.MODEL_MAP.get(model, self.MODEL_MAP["paraformer-zh"])
|
|
||||||
self.cache_dir = cache_dir or os.path.expanduser("~/.cache/modelscope/hub")
|
|
||||||
self._model = None
|
|
||||||
|
|
||||||
def _load_model(self):
|
|
||||||
"""延迟加载模型"""
|
|
||||||
if self._model is None:
|
|
||||||
try:
|
|
||||||
from funasr import AutoModel
|
|
||||||
print(f"正在加载模型: {self.model_name} ({self.model_id})")
|
|
||||||
print(f"缓存目录: {self.cache_dir}")
|
|
||||||
self._model = AutoModel(
|
|
||||||
model=self.model_id,
|
|
||||||
disable_pbar=False,
|
|
||||||
disable_log=False,
|
|
||||||
)
|
|
||||||
print("模型加载完成")
|
|
||||||
except ImportError:
|
|
||||||
print("错误: 未安装 funasr 库")
|
|
||||||
print("请运行: pip install funasr modelscope")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def _convert_audio(self, input_path: str) -> str:
|
|
||||||
"""转换音频格式为 16kHz WAV
|
|
||||||
|
|
||||||
Args:
|
|
||||||
input_path: 输入文件路径
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
转换后的文件路径(如果不需要转换则返回原路径)
|
|
||||||
"""
|
|
||||||
# 检查是否已经是 WAV 格式
|
|
||||||
if input_path.lower().endswith('.wav'):
|
|
||||||
# 检查采样率
|
|
||||||
try:
|
|
||||||
result = subprocess.run(
|
|
||||||
['ffprobe', '-v', 'error', '-show_entries',
|
|
||||||
'stream=sample_rate', '-of', 'default=noprint_wrappers=1:nokey=1',
|
|
||||||
input_path],
|
|
||||||
capture_output=True, text=True
|
|
||||||
)
|
|
||||||
if result.returncode == 0:
|
|
||||||
sample_rate = int(result.stdout.strip())
|
|
||||||
if sample_rate == 16000:
|
|
||||||
return input_path
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# 需要转换
|
|
||||||
output_path = input_path.rsplit('.', 1)[0] + '_converted.wav'
|
|
||||||
print(f"正在转换音频格式: {input_path} -> {output_path}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
subprocess.run([
|
|
||||||
'ffmpeg', '-y', '-i', input_path,
|
|
||||||
'-acodec', 'pcm_s16le',
|
|
||||||
'-ar', '16000',
|
|
||||||
'-ac', '1',
|
|
||||||
output_path
|
|
||||||
], check=True, capture_output=True)
|
|
||||||
print("音频转换完成")
|
|
||||||
return output_path
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print(f"音频转换失败: {e}")
|
|
||||||
# 如果转换失败,尝试使用原文件
|
|
||||||
print("尝试使用原文件继续处理")
|
|
||||||
return input_path
|
|
||||||
except FileNotFoundError:
|
|
||||||
print("警告: 未找到 ffmpeg,跳过音频转换")
|
|
||||||
return input_path
|
|
||||||
|
|
||||||
def transcribe(self, input_path: str) -> Dict:
|
|
||||||
"""转录音频文件
|
|
||||||
|
|
||||||
Args:
|
|
||||||
input_path: 输入文件路径
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
识别结果字典
|
|
||||||
"""
|
|
||||||
input_path = os.path.abspath(input_path)
|
|
||||||
|
|
||||||
if not os.path.exists(input_path):
|
|
||||||
return {
|
|
||||||
"success": False,
|
|
||||||
"error": f"文件不存在: {input_path}"
|
|
||||||
}
|
|
||||||
|
|
||||||
print(f"\n开始处理: {input_path}")
|
|
||||||
print(f"使用模型: {self.model_name}")
|
|
||||||
|
|
||||||
# 转换音频格式
|
|
||||||
audio_path = self._convert_audio(input_path)
|
|
||||||
|
|
||||||
# 加载模型
|
|
||||||
self._load_model()
|
|
||||||
|
|
||||||
# 执行识别
|
|
||||||
print("正在进行语音识别...")
|
|
||||||
try:
|
|
||||||
result = self._model.generate(
|
|
||||||
input=audio_path,
|
|
||||||
batch_size_s=300,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
return {
|
|
||||||
"success": False,
|
|
||||||
"error": f"识别失败: {str(e)}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# 解析结果
|
|
||||||
if not result or not isinstance(result, list):
|
|
||||||
return {
|
|
||||||
"success": False,
|
|
||||||
"error": "识别结果格式错误"
|
|
||||||
}
|
|
||||||
|
|
||||||
# FunASR 返回格式处理
|
|
||||||
segments = []
|
|
||||||
full_text = ""
|
|
||||||
|
|
||||||
for seg in result:
|
|
||||||
if isinstance(seg, dict):
|
|
||||||
text = seg.get("text", "").strip()
|
|
||||||
if "timestamp" in seg and seg["timestamp"]:
|
|
||||||
# 包含时间戳
|
|
||||||
timestamp = seg["timestamp"]
|
|
||||||
if len(timestamp) >= 2:
|
|
||||||
start_time = timestamp[0][0] if timestamp[0] else 0
|
|
||||||
end_time = timestamp[-1][1] if timestamp[-1] else start_time
|
|
||||||
else:
|
|
||||||
start_time = 0
|
|
||||||
end_time = 0
|
|
||||||
else:
|
|
||||||
start_time = 0
|
|
||||||
end_time = 0
|
|
||||||
|
|
||||||
if text:
|
|
||||||
segments.append({
|
|
||||||
"id": len(segments),
|
|
||||||
"start": round(start_time, 2),
|
|
||||||
"end": round(end_time, 2),
|
|
||||||
"text": text
|
|
||||||
})
|
|
||||||
full_text += text + "\n"
|
|
||||||
|
|
||||||
# 清理临时文件
|
|
||||||
if audio_path != input_path and "_converted.wav" in audio_path:
|
|
||||||
try:
|
|
||||||
os.remove(audio_path)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# 计算时长
|
|
||||||
duration = segments[-1]["end"] if segments else 0
|
|
||||||
|
|
||||||
return {
|
|
||||||
"success": True,
|
|
||||||
"input_file": os.path.basename(input_path),
|
|
||||||
"model": self.model_name,
|
|
||||||
"output_files": {},
|
|
||||||
"duration_seconds": round(duration, 2),
|
|
||||||
"segments": segments,
|
|
||||||
"text": full_text.strip()
|
|
||||||
}
|
|
||||||
|
|
||||||
def save_outputs(self, result: Dict, output_dir: str):
|
|
||||||
"""保存输出文件
|
|
||||||
|
|
||||||
Args:
|
|
||||||
result: 识别结果
|
|
||||||
output_dir: 输出目录
|
|
||||||
"""
|
|
||||||
if not result.get("success"):
|
|
||||||
return
|
|
||||||
|
|
||||||
output_dir = Path(output_dir)
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
basename = Path(result["input_file"]).stem
|
|
||||||
|
|
||||||
# 保存 JSON
|
|
||||||
json_path = output_dir / f"{basename}.json"
|
|
||||||
with open(json_path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
||||||
print(f"已保存: {json_path}")
|
|
||||||
result["output_files"]["json"] = str(json_path)
|
|
||||||
|
|
||||||
# 保存 TXT
|
|
||||||
txt_path = output_dir / f"{basename}.txt"
|
|
||||||
with open(txt_path, "w", encoding="utf-8") as f:
|
|
||||||
f.write(result.get("text", ""))
|
|
||||||
print(f"已保存: {txt_path}")
|
|
||||||
result["output_files"]["txt"] = str(txt_path)
|
|
||||||
|
|
||||||
|
|
||||||
def run_as_background(input_path: str, model: str, output_dir: str):
|
|
||||||
"""后台运行识别
|
|
||||||
|
|
||||||
Args:
|
|
||||||
input_path: 输入文件路径
|
|
||||||
model: 模型名称
|
|
||||||
output_dir: 输出目录
|
|
||||||
"""
|
|
||||||
# 创建日志文件
|
|
||||||
log_file = os.path.join(output_dir, f"funasr_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log")
|
|
||||||
|
|
||||||
# 重新执行命令(后台模式)
|
|
||||||
cmd = [
|
|
||||||
sys.executable, __file__,
|
|
||||||
"--input", input_path,
|
|
||||||
"--model", model,
|
|
||||||
"--output", output_dir
|
|
||||||
]
|
|
||||||
|
|
||||||
with open(log_file, "w") as f:
|
|
||||||
process = subprocess.Popen(
|
|
||||||
cmd,
|
|
||||||
stdout=f,
|
|
||||||
stderr=subprocess.STDOUT,
|
|
||||||
start_new_session=True
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"\n✓ 后台任务已启动 (PID: {process.pid})")
|
|
||||||
print(f"日志文件: {log_file}")
|
|
||||||
print(f"输出目录: {output_dir}")
|
|
||||||
print("\n使用以下命令查看进度:")
|
|
||||||
print(f" tail -f {log_file}")
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="KB FunASR - 本地语音识别工具",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog="""
|
|
||||||
示例:
|
|
||||||
python funasr.py --input video.mp4
|
|
||||||
python funasr.py --input podcast.mp3 --model paraformer-zh
|
|
||||||
python funasr.py --input video.mp4 --output ./transcripts --background
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument("--input", "-i", required=True, help="输入音视频文件路径")
|
|
||||||
parser.add_argument("--model", "-m", default="paraformer-zh",
|
|
||||||
choices=["paraformer-zh", "paraformer", "sensevoice"],
|
|
||||||
help="模型名称 (默认: paraformer-zh)")
|
|
||||||
parser.add_argument("--output", "-o", default=None, help="输出目录 (默认: 文件所在目录)")
|
|
||||||
parser.add_argument("--background", "-b", action="store_true", help="后台异步执行")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# 确定输出目录
|
|
||||||
if args.output:
|
|
||||||
output_dir = args.output
|
|
||||||
else:
|
|
||||||
output_dir = os.path.dirname(args.input) or "."
|
|
||||||
|
|
||||||
# 后台模式
|
|
||||||
if args.background:
|
|
||||||
run_as_background(args.input, args.model, output_dir)
|
|
||||||
return
|
|
||||||
|
|
||||||
# 前台模式
|
|
||||||
transcriber = FunASRTranscriber(model=args.model)
|
|
||||||
result = transcriber.transcribe(args.input)
|
|
||||||
|
|
||||||
if result.get("success"):
|
|
||||||
transcriber.save_outputs(result, output_dir)
|
|
||||||
print(f"\n✓ 处理完成!")
|
|
||||||
print(f" 时长: {result['duration_seconds']} 秒")
|
|
||||||
print(f" 分段数: {len(result['segments'])}")
|
|
||||||
else:
|
|
||||||
print(f"\n✗ 处理失败: {result.get('error')}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
# KB Link Parser Skill
|
|
||||||
|
|
||||||
**kb-link-parser** - Extract and resolve IDs from various platform URLs.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- Multi-platform support: Bilibili, YouTube, Zhihu
|
|
||||||
- Short link detection: b23.tv and other short links
|
|
||||||
- Multiple content types: video, article, up主, playlist, collection
|
|
||||||
- Batch processing from file
|
|
||||||
- JSON output format
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Parse single URL
|
|
||||||
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
|
|
||||||
|
|
||||||
# Batch processing
|
|
||||||
python link-parser.py --batch urls.txt --output results.json
|
|
||||||
|
|
||||||
# Output format
|
|
||||||
python link-parser.py --url "..." --format json
|
|
||||||
```
|
|
||||||
|
|
||||||
## Output
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"data": {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "video",
|
|
||||||
"id": "BV1xx411c7mD"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp -r kb-link-parser/ ~/skills/
|
|
||||||
```
|
|
||||||
Binary file not shown.
@@ -1,637 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
Link Parser - Extract and resolve IDs from various platform URLs.
|
|
||||||
|
|
||||||
Supports:
|
|
||||||
- Bilibili (video, article, up主, collection, series)
|
|
||||||
- YouTube (video, channel, playlist)
|
|
||||||
- Zhihu (question, answer, user, collection)
|
|
||||||
- Xiaohongshu (note, user)
|
|
||||||
- Short links (b23.tv, xhslink.com, etc.)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
|
|
||||||
python link-parser.py --batch urls.txt --output results.json
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
from typing import Optional, Dict, Any
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
from link_parser_shortlink import ShortLinkResolver
|
|
||||||
|
|
||||||
|
|
||||||
class LinkParser:
|
|
||||||
"""Main link parser class"""
|
|
||||||
|
|
||||||
# Bilibili patterns
|
|
||||||
BILIBILI_VIDEO_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV[a-zA-Z0-9]+)')
|
|
||||||
BILIBILI_VIDEO_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
|
|
||||||
BILIBILI_ARTICLE_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/(?:article|read)/([a-zA-Z0-9]+)')
|
|
||||||
BILIBILI_SPACE_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/(\d+)')
|
|
||||||
BILIBILI_MOBILE_SPACE_PATTERN = re.compile(r'https?://m\.bilibili\.com/space/(\d+)')
|
|
||||||
BILIBILI_MOBILE_OPUS_PATTERN = re.compile(r'https?://m\.bilibili\.com/opus/(\d+)')
|
|
||||||
BILIBILI_FAVLIST_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/favlist\?fid=(\d+)')
|
|
||||||
BILIBILI_OPUS_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/opus/(\d+)')
|
|
||||||
BILIBILI_SERIES_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/lists/(\d+)\?type=series')
|
|
||||||
BILIBILI_READLIST_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/read/readlist/(rl[a-zA-Z0-9]+)')
|
|
||||||
|
|
||||||
# YouTube patterns
|
|
||||||
YOUTUBE_VIDEO_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/)([a-zA-Z0-9_-]{11})')
|
|
||||||
YOUTUBE_CHANNEL_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/channel/([a-zA-Z0-9_-]+)')
|
|
||||||
YOUTUBE_USER_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/user/([a-zA-Z0-9_-]+)')
|
|
||||||
YOUTUBE_PLAYLIST_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/playlist\?list=([a-zA-Z0-9_-]+)')
|
|
||||||
|
|
||||||
# Xiaohongshu (小红书) patterns
|
|
||||||
XIAOHONGSHU_SHORT_PATTERN = re.compile(r'https?://xhslink\.com/o/([a-zA-Z0-9]+)')
|
|
||||||
XIAOHONGSHU_NOTE_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/explore/([a-zA-Z0-9]+)')
|
|
||||||
XIAOHONGSHU_NOTE_PATTERN2 = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/notes/([a-zA-Z0-9]+)')
|
|
||||||
XIAOHONGSHU_DISCOVERY_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/discovery/item/([a-zA-Z0-9]+)')
|
|
||||||
XIAOHONGSHU_USER_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/user/profile/(\d+)')
|
|
||||||
|
|
||||||
# Zhihu patterns
|
|
||||||
ZHIHAU_QUESTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/(\d+)')
|
|
||||||
ZHIHAU_ANSWER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)')
|
|
||||||
ZHIHAU_USER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/people/([a-zA-Z0-9_-]+)')
|
|
||||||
ZHIHAU_COLUMN_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/column/([a-zA-Z0-9_-]+)')
|
|
||||||
ZHIHAU_COLLECTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/collection/(\d+)')
|
|
||||||
# Additional Zhihu patterns
|
|
||||||
ZHIHAU_ZHUANLAN_PATTERN = re.compile(r'https?://zhuanlan\.zhihu\.com/p/(\d+)')
|
|
||||||
ZHIHAU_ORG_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/([a-zA-Z0-9_-]+)')
|
|
||||||
ZHIHAU_ORG_ANSWERS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/answers')
|
|
||||||
ZHIHAU_ORG_VIDEOS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/zvideos')
|
|
||||||
ZHIHAU_ORG_ASKS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/asks')
|
|
||||||
ZHIHAU_ORG_POSTS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/posts')
|
|
||||||
ZHIHAU_ORG_PINS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/pins')
|
|
||||||
ZHIHAU_ZVIDEO_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/zvideo/(\d+)')
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.shortlink_resolver = ShortLinkResolver()
|
|
||||||
|
|
||||||
def resolve_shortlink(self, url: str) -> Optional[str]:
|
|
||||||
"""
|
|
||||||
Resolve short link to get final URL.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: Short URL to resolve
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Final URL after resolution, or None if not resolvable
|
|
||||||
"""
|
|
||||||
# Check if it's a known short link pattern
|
|
||||||
if self.shortlink_resolver.is_bilibili_short_link(url):
|
|
||||||
# Try to resolve using requests (follows redirects)
|
|
||||||
try:
|
|
||||||
import requests
|
|
||||||
response = requests.head(url, allow_redirects=True, timeout=10)
|
|
||||||
return response.url
|
|
||||||
except Exception:
|
|
||||||
# Fallback: return original URL
|
|
||||||
return url
|
|
||||||
|
|
||||||
# Check if it's Xiaohongshu short link
|
|
||||||
if self.XIAOHONGSHU_SHORT_PATTERN.match(url):
|
|
||||||
try:
|
|
||||||
import requests
|
|
||||||
response = requests.get(url, allow_redirects=True, timeout=10)
|
|
||||||
return response.url
|
|
||||||
except Exception:
|
|
||||||
return url
|
|
||||||
|
|
||||||
return url
|
|
||||||
|
|
||||||
def extract_bilibili_id(self, url: str) -> Optional[Dict[str, str]]:
|
|
||||||
"""
|
|
||||||
Extract Bilibili ID from URL.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: Bilibili URL
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with platform, type, and id fields, or None if not matched
|
|
||||||
"""
|
|
||||||
# Try series/playlist (must check before space pattern)
|
|
||||||
match = self.BILIBILI_SERIES_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "playlist",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try readlist (专栏合集)
|
|
||||||
match = self.BILIBILI_READLIST_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "readlist",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try favlist (收藏夹)
|
|
||||||
match = self.BILIBILI_FAVLIST_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "favlist",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try opus (动态)
|
|
||||||
match = self.BILIBILI_OPUS_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "opus",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try mobile opus (移动端动态)
|
|
||||||
match = self.BILIBILI_MOBILE_OPUS_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "opus",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try video
|
|
||||||
match = self.BILIBILI_VIDEO_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "video",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try article/read
|
|
||||||
match = self.BILIBILI_ARTICLE_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "article",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try space (UP主) - must check after specific patterns
|
|
||||||
match = self.BILIBILI_SPACE_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "up主",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try mobile space (移动端UP主空间)
|
|
||||||
match = self.BILIBILI_MOBILE_SPACE_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "bilibili",
|
|
||||||
"type": "up主",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def extract_youtube_id(self, url: str) -> Optional[Dict[str, str]]:
|
|
||||||
"""
|
|
||||||
Extract YouTube ID from URL.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: YouTube URL
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with platform, type, and id fields, or None if not matched
|
|
||||||
"""
|
|
||||||
# Try video
|
|
||||||
match = self.YOUTUBE_VIDEO_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "youtube",
|
|
||||||
"type": "video",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try channel
|
|
||||||
match = self.YOUTUBE_CHANNEL_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "youtube",
|
|
||||||
"type": "channel",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try user
|
|
||||||
match = self.YOUTUBE_USER_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "youtube",
|
|
||||||
"type": "user",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try playlist
|
|
||||||
match = self.YOUTUBE_PLAYLIST_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "youtube",
|
|
||||||
"type": "playlist",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def extract_zhihu_id(self, url: str) -> Optional[Dict[str, str]]:
|
|
||||||
"""
|
|
||||||
Extract Zhihu ID from URL.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: Zhihu URL
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with platform, type, and id fields, or None if not matched
|
|
||||||
"""
|
|
||||||
# Try question
|
|
||||||
match = self.ZHIHAU_QUESTION_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "question",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try answer
|
|
||||||
match = self.ZHIHAU_ANSWER_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "answer",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try user
|
|
||||||
match = self.ZHIHAU_USER_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "user",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try column
|
|
||||||
match = self.ZHIHAU_COLUMN_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "column",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try collection
|
|
||||||
match = self.ZHIHAU_COLLECTION_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "collection",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try zhuanlan article (专栏文章)
|
|
||||||
match = self.ZHIHAU_ZHUANLAN_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "zhuanlan",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try org page (机构页面)
|
|
||||||
match = self.ZHIHAU_ORG_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "org",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try org answers (机构回答)
|
|
||||||
if self.ZHIHAU_ORG_ANSWERS_PATTERN.match(url):
|
|
||||||
match = self.ZHIHAU_ORG_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "org_answers",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try org zvideos (机构视频)
|
|
||||||
if self.ZHIHAU_ORG_VIDEOS_PATTERN.match(url):
|
|
||||||
match = self.ZHIHAU_ORG_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "org_zvideos",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try org asks (机构提问)
|
|
||||||
if self.ZHIHAU_ORG_ASKS_PATTERN.match(url):
|
|
||||||
match = self.ZHIHAU_ORG_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "org_asks",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try org posts (机构文章)
|
|
||||||
if self.ZHIHAU_ORG_POSTS_PATTERN.match(url):
|
|
||||||
match = self.ZHIHAU_ORG_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "org_posts",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try org pins (机构想法)
|
|
||||||
if self.ZHIHAU_ORG_PINS_PATTERN.match(url):
|
|
||||||
match = self.ZHIHAU_ORG_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "org_pins",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try zvideo (知乎视频)
|
|
||||||
match = self.ZHIHAU_ZVIDEO_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "zhihu",
|
|
||||||
"type": "zvideo",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def extract_xiaohongshu_id(self, url: str) -> Optional[Dict[str, str]]:
|
|
||||||
"""
|
|
||||||
Extract Xiaohongshu ID from URL.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: Xiaohongshu URL
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with platform, type, and id fields, or None if not matched
|
|
||||||
"""
|
|
||||||
# Try short link
|
|
||||||
match = self.XIAOHONGSHU_SHORT_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "xiaohongshu",
|
|
||||||
"type": "note",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try explore link
|
|
||||||
match = self.XIAOHONGSHU_NOTE_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "xiaohongshu",
|
|
||||||
"type": "note",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try notes link
|
|
||||||
match = self.XIAOHONGSHU_NOTE_PATTERN2.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "xiaohongshu",
|
|
||||||
"type": "note",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try discovery item link (resolved from short link)
|
|
||||||
match = self.XIAOHONGSHU_DISCOVERY_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "xiaohongshu",
|
|
||||||
"type": "note",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
# Try user profile
|
|
||||||
match = self.XIAOHONGSHU_USER_PATTERN.match(url)
|
|
||||||
if match:
|
|
||||||
return {
|
|
||||||
"platform": "xiaohongshu",
|
|
||||||
"type": "user",
|
|
||||||
"id": match.group(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def parse(self, url: str) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Parse URL and extract platform-specific ID.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: URL to parse
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with platform, type, id, and original_url fields
|
|
||||||
"""
|
|
||||||
result = {
|
|
||||||
"platform": None,
|
|
||||||
"type": None,
|
|
||||||
"id": None,
|
|
||||||
"original_url": url,
|
|
||||||
"resolved_url": None,
|
|
||||||
"success": False,
|
|
||||||
"error": None
|
|
||||||
}
|
|
||||||
|
|
||||||
# First, resolve short links
|
|
||||||
resolved = self.resolve_shortlink(url)
|
|
||||||
if resolved:
|
|
||||||
result["resolved_url"] = resolved
|
|
||||||
url = resolved
|
|
||||||
|
|
||||||
# Try each platform
|
|
||||||
bilibili_result = self.extract_bilibili_id(url)
|
|
||||||
if bilibili_result:
|
|
||||||
result.update(bilibili_result)
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
youtube_result = self.extract_youtube_id(url)
|
|
||||||
if youtube_result:
|
|
||||||
result.update(youtube_result)
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
zhihu_result = self.extract_zhihu_id(url)
|
|
||||||
if zhihu_result:
|
|
||||||
result.update(zhihu_result)
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
# Try Xiaohongshu
|
|
||||||
xiaohongshu_result = self.extract_xiaohongshu_id(url)
|
|
||||||
if xiaohongshu_result:
|
|
||||||
result.update(xiaohongshu_result)
|
|
||||||
result["success"] = True
|
|
||||||
return result
|
|
||||||
|
|
||||||
# Check for short links that need resolution
|
|
||||||
short_links = self.shortlink_resolver.detect_short_links(url)
|
|
||||||
if short_links:
|
|
||||||
result["error"] = "Short link detected. Please provide the full URL."
|
|
||||||
return result
|
|
||||||
|
|
||||||
result["error"] = "Unsupported URL format"
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def format_output(data: Dict[str, Any], output_format: str = "json") -> str:
|
|
||||||
"""Format output based on specified format."""
|
|
||||||
if output_format == "json":
|
|
||||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
|
||||||
elif output_format == "compact":
|
|
||||||
return json.dumps(data, ensure_ascii=False, separators=(',', ':'))
|
|
||||||
elif output_format == "text":
|
|
||||||
lines = []
|
|
||||||
for key, value in data.items():
|
|
||||||
if value is not None:
|
|
||||||
lines.append(f"{key}: {value}")
|
|
||||||
return "\n".join(lines)
|
|
||||||
else:
|
|
||||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""Main entry point."""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Parse and extract IDs from various platform URLs",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog="""
|
|
||||||
Examples:
|
|
||||||
%(prog)s --url "https://www.bilibili.com/video/BV1xx411c7mD"
|
|
||||||
%(prog)s --url "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
|
||||||
%(prog)s --url "https://www.zhihu.com/question/266309957"
|
|
||||||
%(prog)s --batch urls.txt --output results.json
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--url", "-u",
|
|
||||||
help="Single URL to parse"
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--batch", "-b",
|
|
||||||
help="File containing URLs to parse (one per line)"
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--output", "-o",
|
|
||||||
help="Output file path (for batch mode)"
|
|
||||||
)
|
|
||||||
|
|
||||||
parser.add_argument(
|
|
||||||
"--format", "-f",
|
|
||||||
choices=["json", "compact", "text"],
|
|
||||||
default="json",
|
|
||||||
help="Output format (default: json)"
|
|
||||||
)
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Check if we're in shortlink help mode
|
|
||||||
if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']:
|
|
||||||
parser.print_help()
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if not args.url and not args.batch:
|
|
||||||
parser.print_help()
|
|
||||||
print("\nError: Please provide --url or --batch")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
link_parser = LinkParser()
|
|
||||||
|
|
||||||
if args.url:
|
|
||||||
# Single URL mode
|
|
||||||
parsed = link_parser.parse(args.url)
|
|
||||||
|
|
||||||
# 统一输出格式
|
|
||||||
if parsed["success"]:
|
|
||||||
output = {
|
|
||||||
"success": True,
|
|
||||||
"platform": parsed.get("platform"),
|
|
||||||
"type": parsed.get("type"),
|
|
||||||
"id": parsed.get("id"),
|
|
||||||
"filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None,
|
|
||||||
"message": "URL解析成功"
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
output = {
|
|
||||||
"success": False,
|
|
||||||
"error": parsed.get("error", "解析失败"),
|
|
||||||
"detail": f"URL: {args.url}"
|
|
||||||
}
|
|
||||||
|
|
||||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
|
||||||
return 0 if output["success"] else 1
|
|
||||||
|
|
||||||
elif args.batch:
|
|
||||||
try:
|
|
||||||
with open(args.batch, 'r', encoding='utf-8') as f:
|
|
||||||
urls = [line.strip() for line in f if line.strip()]
|
|
||||||
except Exception as e:
|
|
||||||
print(json.dumps({
|
|
||||||
"success": False,
|
|
||||||
"error": "读取文件失败",
|
|
||||||
"detail": str(e)
|
|
||||||
}, ensure_ascii=False, indent=2))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
results = []
|
|
||||||
for url in urls:
|
|
||||||
parsed = link_parser.parse(url)
|
|
||||||
if parsed["success"]:
|
|
||||||
results.append({
|
|
||||||
"success": True,
|
|
||||||
"platform": parsed.get("platform"),
|
|
||||||
"type": parsed.get("type"),
|
|
||||||
"id": parsed.get("id"),
|
|
||||||
"filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None
|
|
||||||
})
|
|
||||||
else:
|
|
||||||
results.append({
|
|
||||||
"success": False,
|
|
||||||
"error": parsed.get("error", "解析失败"),
|
|
||||||
"detail": f"URL: {url}"
|
|
||||||
})
|
|
||||||
|
|
||||||
if args.output:
|
|
||||||
with open(args.output, 'w', encoding='utf-8') as f:
|
|
||||||
json.dump(results, f, ensure_ascii=False, indent=2)
|
|
||||||
print(json.dumps({
|
|
||||||
"success": True,
|
|
||||||
"message": f"结果已保存到 {args.output}",
|
|
||||||
"total": len(results)
|
|
||||||
}, ensure_ascii=False, indent=2))
|
|
||||||
else:
|
|
||||||
for result in results:
|
|
||||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
||||||
print("-" * 40)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,296 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
Short link resolver for link-parser skill.
|
|
||||||
|
|
||||||
Provides utilities for detecting and resolving short links, particularly
|
|
||||||
Bilibili short links (b23.tv) which may be blocked by anti-scraping measures.
|
|
||||||
|
|
||||||
Supported short link services:
|
|
||||||
- Bilibili: b23.tv
|
|
||||||
- Generic URL shorteners (detection only)
|
|
||||||
"""
|
|
||||||
|
|
||||||
import re
|
|
||||||
from typing import Optional, Dict, List, Tuple
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
|
|
||||||
class ShortLinkResolver:
|
|
||||||
"""Short link detection and resolution handler"""
|
|
||||||
|
|
||||||
# Pattern for Bilibili short links
|
|
||||||
BILIBILI_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
|
|
||||||
|
|
||||||
# Pattern for other short link services
|
|
||||||
OTHER_SHORT_PATTERNS = {
|
|
||||||
'bit.ly': re.compile(r'https?://bit\.ly/\w+'),
|
|
||||||
't.cn': re.compile(r'https?://t\.cn/\w+'),
|
|
||||||
'dwz.cn': re.compile(r'https?://dwz\.cn/\w+'),
|
|
||||||
'goo.gl': re.compile(r'https?://goo\.gl/\w+'),
|
|
||||||
'tinyurl.com': re.compile(r'https?://tinyurl\.com/\w+'),
|
|
||||||
}
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def detect_short_links(text: str) -> List[Dict[str, str]]:
|
|
||||||
"""
|
|
||||||
Detect short links in text.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: Input text to search
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of detected short links with metadata:
|
|
||||||
[
|
|
||||||
{
|
|
||||||
'url': 'https://b23.tv/xxxxx',
|
|
||||||
'platform': 'bilibili',
|
|
||||||
'is_short': True
|
|
||||||
},
|
|
||||||
...
|
|
||||||
]
|
|
||||||
"""
|
|
||||||
detected = []
|
|
||||||
|
|
||||||
# Check for Bilibili short links
|
|
||||||
for match in ShortLinkResolver.BILIBILI_SHORT_PATTERN.finditer(text):
|
|
||||||
detected.append({
|
|
||||||
'url': match.group(0),
|
|
||||||
'code': match.group(1),
|
|
||||||
'platform': 'bilibili',
|
|
||||||
'is_short': True
|
|
||||||
})
|
|
||||||
|
|
||||||
# Check for other short link services
|
|
||||||
for platform, pattern in ShortLinkResolver.OTHER_SHORT_PATTERNS.items():
|
|
||||||
for match in pattern.finditer(text):
|
|
||||||
detected.append({
|
|
||||||
'url': match.group(0),
|
|
||||||
'platform': platform,
|
|
||||||
'is_short': True
|
|
||||||
})
|
|
||||||
|
|
||||||
return detected
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def is_bilibili_short_link(url: str) -> bool:
|
|
||||||
"""
|
|
||||||
Check if URL is a Bilibili short link.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url: URL to check
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
True if URL is a b23.tv short link
|
|
||||||
"""
|
|
||||||
return bool(ShortLinkResolver.BILIBILI_SHORT_PATTERN.match(url))
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def format_short_link_help(short_link: Optional[str] = None) -> str:
|
|
||||||
"""
|
|
||||||
Format help message for short links.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
short_link: Optional short link URL to mention
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Help message
|
|
||||||
"""
|
|
||||||
lines = [
|
|
||||||
"Detection: Short link",
|
|
||||||
""
|
|
||||||
]
|
|
||||||
|
|
||||||
if short_link:
|
|
||||||
lines.append(f"Short link: {short_link}")
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
lines.extend([
|
|
||||||
"Due to anti-scraping protection, short links cannot be automatically resolved.",
|
|
||||||
"Please provide the full URL or the content ID (e.g., BV号 for Bilibili).",
|
|
||||||
"",
|
|
||||||
"Full URL format: https://www.bilibili.com/video/BVxxxxx"
|
|
||||||
])
|
|
||||||
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def format_short_link_warning(short_links: List[Dict[str, str]]) -> str:
|
|
||||||
"""
|
|
||||||
Format warning message for multiple short links.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
short_links: List of short link metadata
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Warning message
|
|
||||||
"""
|
|
||||||
if not short_links:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
lines = [
|
|
||||||
"Short links detected:",
|
|
||||||
""
|
|
||||||
]
|
|
||||||
|
|
||||||
for i, link in enumerate(short_links, 1):
|
|
||||||
platform = link.get('platform', 'unknown')
|
|
||||||
url = link['url']
|
|
||||||
|
|
||||||
if platform == 'bilibili':
|
|
||||||
lines.append(f" [{i}] Bilibili short link: {url}")
|
|
||||||
lines.append(" Full URL or BV ID required")
|
|
||||||
else:
|
|
||||||
lines.append(f" [{i}] {platform}: {url}")
|
|
||||||
|
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
class PlaywrightResolver:
|
|
||||||
"""
|
|
||||||
Browser-based short link resolver using Playwright.
|
|
||||||
|
|
||||||
This is an optional implementation for environments where Playwright
|
|
||||||
is available. It can bypass anti-scraping measures by simulating
|
|
||||||
a real browser.
|
|
||||||
|
|
||||||
Requirements:
|
|
||||||
pip install playwright
|
|
||||||
playwright install chromium
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, headless: bool = True, timeout: int = 10000):
|
|
||||||
"""
|
|
||||||
Initialize Playwright resolver.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
headless: Whether to run browser in headless mode
|
|
||||||
timeout: Page navigation timeout in milliseconds
|
|
||||||
"""
|
|
||||||
self.headless = headless
|
|
||||||
self.timeout = timeout
|
|
||||||
self._playwright = None
|
|
||||||
|
|
||||||
def _check_playwright_available(self) -> bool:
|
|
||||||
"""Check if Playwright is installed and available"""
|
|
||||||
try:
|
|
||||||
from playwright.sync_api import sync_playwright
|
|
||||||
return True
|
|
||||||
except ImportError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def resolve(self, short_url: str) -> Optional[str]:
|
|
||||||
"""
|
|
||||||
Resolve short link using Playwright browser automation.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
short_url: Short URL to resolve
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Final URL after redirects, or None if resolution fails
|
|
||||||
"""
|
|
||||||
if not self._check_playwright_available():
|
|
||||||
raise ImportError(
|
|
||||||
"Playwright not installed. Install with: pip install playwright && playwright install chromium"
|
|
||||||
)
|
|
||||||
|
|
||||||
from playwright.sync_api import sync_playwright
|
|
||||||
|
|
||||||
try:
|
|
||||||
with sync_playwright() as p:
|
|
||||||
# Launch browser
|
|
||||||
browser = p.chromium.launch(headless=self.headless)
|
|
||||||
context = browser.new_context(
|
|
||||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
||||||
viewport={'width': 1920, 'height': 1080}
|
|
||||||
)
|
|
||||||
page = context.new_page()
|
|
||||||
|
|
||||||
# Navigate to short URL
|
|
||||||
page.goto(short_url, timeout=self.timeout, wait_until='networkidle')
|
|
||||||
|
|
||||||
# Get final URL
|
|
||||||
final_url = page.url
|
|
||||||
|
|
||||||
# Close browser
|
|
||||||
browser.close()
|
|
||||||
|
|
||||||
return final_url
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"Playwright resolution failed: {e}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def resolve_bilibili_short(self, short_url: str) -> Optional[str]:
|
|
||||||
"""
|
|
||||||
Resolve Bilibili short link and extract BV ID.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
short_url: Bilibili short URL (e.g., https://b23.tv/xxxxx)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Full Bilibili URL with BV ID, or None if resolution fails
|
|
||||||
"""
|
|
||||||
final_url = self.resolve(short_url)
|
|
||||||
if not final_url:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# Check if final URL contains BV ID
|
|
||||||
if 'bilibili.com/video/' in final_url:
|
|
||||||
return final_url
|
|
||||||
|
|
||||||
# Extract BV ID if present in URL
|
|
||||||
bv_pattern = re.compile(r'(BV[a-zA-Z0-9]{10})')
|
|
||||||
match = bv_pattern.search(final_url)
|
|
||||||
if match:
|
|
||||||
return f"https://www.bilibili.com/video/{match.group(1)}"
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def get_resolver() -> ShortLinkResolver:
|
|
||||||
"""
|
|
||||||
Get the default resolver instance.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
ShortLinkResolver instance
|
|
||||||
"""
|
|
||||||
return ShortLinkResolver()
|
|
||||||
|
|
||||||
|
|
||||||
def detect_short_links(text: str) -> List[Dict[str, str]]:
|
|
||||||
"""
|
|
||||||
Detect short links in text.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: Input text to search
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of detected short links with metadata
|
|
||||||
"""
|
|
||||||
resolver = ShortLinkResolver()
|
|
||||||
return resolver.detect_short_links(text)
|
|
||||||
|
|
||||||
|
|
||||||
# CLI interface for testing
|
|
||||||
def main():
|
|
||||||
"""CLI for testing the short link resolver."""
|
|
||||||
import sys
|
|
||||||
|
|
||||||
if len(sys.argv) < 2:
|
|
||||||
print("Usage: python3 link_parser_shortlink.py 'text to test'")
|
|
||||||
print("\nExamples:")
|
|
||||||
print(' python3 link_parser_shortlink.py "Check this: https://b23.tv/oLBZZfK"')
|
|
||||||
print(' python3 link_parser_shortlink.py "https://bit.ly/xxxxx"')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
text = ' '.join(sys.argv[1:])
|
|
||||||
short_links = detect_short_links(text)
|
|
||||||
|
|
||||||
import json
|
|
||||||
print(json.dumps(short_links, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,66 +0,0 @@
|
|||||||
# KB OCR Processor Skill
|
|
||||||
|
|
||||||
**kb-ocr-processor** - Extract text from video frames using OCR.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- **视频截帧**: 直接从视频文件截取帧图片
|
|
||||||
- **批量处理**: 支持处理帧目录
|
|
||||||
- **双引擎**: EasyOCR / Tesseract
|
|
||||||
- **置信度评分**: 每帧识别结果带置信度
|
|
||||||
- **文本合并**: 自动合并所有帧的识别结果
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 方式1: 直接从视频截帧并OCR(推荐)
|
|
||||||
python ocr_processor.py --video ./video.flv --output ocr.json
|
|
||||||
|
|
||||||
# 方式2: 处理已截取的帧目录
|
|
||||||
python ocr_processor.py --input ./frames/ --output ocr.json
|
|
||||||
|
|
||||||
# 指定截帧帧率(默认1帧/秒)
|
|
||||||
python ocr_processor.py --video ./video.flv --fps 2 --output ocr.json
|
|
||||||
|
|
||||||
# 同时生成合并的文本文件
|
|
||||||
python ocr_processor.py --video ./video.flv --output ocr.json --merge
|
|
||||||
|
|
||||||
# 使用 Tesseract 引擎
|
|
||||||
python ocr_processor.py --input ./frames/ --engine tesseract --output ocr.json
|
|
||||||
```
|
|
||||||
|
|
||||||
## Workflow 中的使用
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 步骤5: OCR处理(视频无字幕时)
|
|
||||||
python ocr_processor.py --video ${CONTENT_VIDEO_PATH} --output ${CONTENT_DIR}/ocr.json --merge
|
|
||||||
```
|
|
||||||
|
|
||||||
## Output
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"engine": "easyocr",
|
|
||||||
"total_frames": 619,
|
|
||||||
"results": [
|
|
||||||
{
|
|
||||||
"source": "0001.jpg",
|
|
||||||
"text": "字幕文字",
|
|
||||||
"confidence": 0.95
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cp -r kb-ocr-processor/ ~/skills/
|
|
||||||
pip install -r requirements.txt
|
|
||||||
|
|
||||||
# 依赖说明
|
|
||||||
# - easyocr: 主要OCR引擎(自动下载模型)
|
|
||||||
# - pytesseract: Tesseract引擎
|
|
||||||
# - ffmpeg: 视频截帧(apt install ffmpeg)
|
|
||||||
```
|
|
||||||
@@ -1,360 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
OCR Processor - 视频帧文字识别工具
|
|
||||||
支持 EasyOCR 和 Tesseract 引擎
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import glob
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import List, Dict, Optional
|
|
||||||
import easyocr
|
|
||||||
import pytesseract
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
|
|
||||||
class OCRProcessor:
|
|
||||||
"""OCR 文字识别处理器"""
|
|
||||||
|
|
||||||
def __init__(self, engine: str = "easyocr", languages: List[str] = ["ch_sim", "en"]):
|
|
||||||
"""
|
|
||||||
初始化 OCR 处理器
|
|
||||||
|
|
||||||
Args:
|
|
||||||
engine: OCR 引擎 ("easyocr" 或 "tesseract")
|
|
||||||
languages: 语言列表
|
|
||||||
"""
|
|
||||||
self.engine = engine
|
|
||||||
self.languages = languages
|
|
||||||
self.reader = None
|
|
||||||
|
|
||||||
if engine == "easyocr":
|
|
||||||
self.reader = easyocr.Reader(languages, gpu=False)
|
|
||||||
|
|
||||||
def process_image(self, image_path: str) -> Dict:
|
|
||||||
"""
|
|
||||||
处理单张图片进行 OCR 识别
|
|
||||||
|
|
||||||
Args:
|
|
||||||
image_path: 图片路径
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
包含 text 和 confidence 的字典
|
|
||||||
"""
|
|
||||||
if not os.path.exists(image_path):
|
|
||||||
return {"text": "", "confidence": 0.0, "error": "File not found"}
|
|
||||||
|
|
||||||
try:
|
|
||||||
if self.engine == "easyocr":
|
|
||||||
return self._process_with_easyocr(image_path)
|
|
||||||
elif self.engine == "tesseract":
|
|
||||||
return self._process_with_tesseract(image_path)
|
|
||||||
else:
|
|
||||||
return {"text": "", "confidence": 0.0, "error": f"Unknown engine: {self.engine}"}
|
|
||||||
except Exception as e:
|
|
||||||
return {"text": "", "confidence": 0.0, "error": str(e)}
|
|
||||||
|
|
||||||
def _process_with_easyocr(self, image_path: str) -> Dict:
|
|
||||||
"""使用 EasyOCR 处理图片"""
|
|
||||||
results = self.reader.readtext(image_path)
|
|
||||||
|
|
||||||
if not results:
|
|
||||||
return {"text": "", "confidence": 0.0}
|
|
||||||
|
|
||||||
texts = []
|
|
||||||
confidences = []
|
|
||||||
|
|
||||||
for bbox, text, conf in results:
|
|
||||||
texts.append(text)
|
|
||||||
confidences.append(conf)
|
|
||||||
|
|
||||||
merged_text = "\n".join(texts)
|
|
||||||
avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0
|
|
||||||
|
|
||||||
return {
|
|
||||||
"text": merged_text,
|
|
||||||
"confidence": round(avg_confidence, 4),
|
|
||||||
"details": [
|
|
||||||
{"text": text, "confidence": round(conf, 4)}
|
|
||||||
for text, conf in zip(texts, confidences)
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
def _process_with_tesseract(self, image_path: str) -> Dict:
|
|
||||||
"""使用 Tesseract 处理图片"""
|
|
||||||
image = Image.open(image_path)
|
|
||||||
text = pytesseract.image_to_string(image, lang="+".join(self.languages))
|
|
||||||
|
|
||||||
# Tesseract 不直接返回置信度,返回 0.0
|
|
||||||
return {
|
|
||||||
"text": text.strip(),
|
|
||||||
"confidence": 0.0
|
|
||||||
}
|
|
||||||
|
|
||||||
def process_frames(self, input_dir: str, fps: float = 0.1, progress_file: Optional[str] = None) -> List[Dict]:
|
|
||||||
"""
|
|
||||||
处理目录下所有帧图片
|
|
||||||
|
|
||||||
Args:
|
|
||||||
input_dir: 帧图片目录
|
|
||||||
fps: 采样帧率
|
|
||||||
progress_file: 可选,进度文件路径(会实时更新)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
OCR 结果列表
|
|
||||||
"""
|
|
||||||
if not os.path.isdir(input_dir):
|
|
||||||
print(f"Error: Directory not found: {input_dir}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
# 支持的图片格式
|
|
||||||
image_extensions = ["*.jpg", "*.jpeg", "*.png", "*.bmp", "*.webp"]
|
|
||||||
image_files = []
|
|
||||||
|
|
||||||
for ext in image_extensions:
|
|
||||||
image_files.extend(glob.glob(os.path.join(input_dir, ext)))
|
|
||||||
image_files.extend(glob.glob(os.path.join(input_dir, "**", ext), recursive=True))
|
|
||||||
|
|
||||||
# 按文件名排序
|
|
||||||
image_files = sorted(image_files)
|
|
||||||
|
|
||||||
if not image_files:
|
|
||||||
print(f"No images found in {input_dir}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
total = len(image_files)
|
|
||||||
print(f"Found {total} images in {input_dir}")
|
|
||||||
print(f"Processing with {self.engine} engine...")
|
|
||||||
|
|
||||||
def update_progress(current, status="running", last_result=None):
|
|
||||||
"""更新进度文件"""
|
|
||||||
if progress_file:
|
|
||||||
import json
|
|
||||||
import time
|
|
||||||
prog = {
|
|
||||||
"current": current,
|
|
||||||
"total": total,
|
|
||||||
"percent": round(current / total * 100, 1) if total > 0 else 0,
|
|
||||||
"status": status,
|
|
||||||
"last_result": last_result,
|
|
||||||
"updated_at": time.strftime("%Y-%m-%d %H:%M:%S")
|
|
||||||
}
|
|
||||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
|
||||||
json.dump(prog, f, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
# 初始化进度
|
|
||||||
update_progress(0, "started")
|
|
||||||
|
|
||||||
results = []
|
|
||||||
for i, image_path in enumerate(image_files, 1):
|
|
||||||
basename = os.path.basename(image_path)
|
|
||||||
print(f"Processing [{i}/{total}]: {basename}")
|
|
||||||
|
|
||||||
ocr_result = self.process_image(image_path)
|
|
||||||
ocr_result["source"] = basename
|
|
||||||
|
|
||||||
# 记录结果摘要
|
|
||||||
text_count = len(ocr_result.get("text", "").split()) if ocr_result.get("text") else 0
|
|
||||||
last_result = f"✓ {text_count} 处文字" if ocr_result.get("text") else "✗ 无文字"
|
|
||||||
|
|
||||||
results.append(ocr_result)
|
|
||||||
|
|
||||||
# 实时更新进度
|
|
||||||
update_progress(i, "running", last_result)
|
|
||||||
|
|
||||||
# 完成
|
|
||||||
update_progress(total, "completed", f"完成 {total} 张")
|
|
||||||
|
|
||||||
print(f"\n✅ Processed {total} frames")
|
|
||||||
return results
|
|
||||||
|
|
||||||
def merge_texts(self, results: List[Dict]) -> str:
|
|
||||||
"""
|
|
||||||
合并多个 OCR 结果的文本
|
|
||||||
|
|
||||||
Args:
|
|
||||||
results: OCR 结果列表
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
合并后的文本
|
|
||||||
"""
|
|
||||||
texts = []
|
|
||||||
for result in results:
|
|
||||||
if result.get("text"):
|
|
||||||
texts.append(result["text"])
|
|
||||||
return "\n\n---\n\n".join(texts)
|
|
||||||
|
|
||||||
def save_results(self, results: List[Dict], output_path: str, merge: bool = False):
|
|
||||||
"""
|
|
||||||
保存 OCR 结果到 JSON 文件
|
|
||||||
|
|
||||||
Args:
|
|
||||||
results: OCR 结果列表
|
|
||||||
output_path: 输出文件路径
|
|
||||||
merge: 是否同时保存合并后的文本文件
|
|
||||||
"""
|
|
||||||
output_data = {
|
|
||||||
"engine": self.engine,
|
|
||||||
"total_frames": len(results),
|
|
||||||
"results": results
|
|
||||||
}
|
|
||||||
|
|
||||||
# 保存 JSON
|
|
||||||
with open(output_path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(output_data, f, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
print(f"Results saved to {output_path}")
|
|
||||||
|
|
||||||
# 如果需要,同时保存合并的文本
|
|
||||||
if merge:
|
|
||||||
text_path = output_path.replace(".json", ".txt")
|
|
||||||
merged_text = self.merge_texts(results)
|
|
||||||
with open(text_path, "w", encoding="utf-8") as f:
|
|
||||||
f.write(merged_text)
|
|
||||||
print(f"Merged text saved to {text_path}")
|
|
||||||
|
|
||||||
|
|
||||||
def extract_frames(video_path: str, output_dir: str, fps: float = 1.0) -> str:
|
|
||||||
"""
|
|
||||||
使用 ffmpeg 从视频中截取帧图片
|
|
||||||
|
|
||||||
Args:
|
|
||||||
video_path: 视频文件路径
|
|
||||||
output_dir: 输出目录
|
|
||||||
fps: 每秒截取帧数 (默认: 1)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
帧图片目录路径
|
|
||||||
"""
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
output_dir = Path(output_dir)
|
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
# ffmpeg 命令: -i 输入 -vf fps=fps 输出格式
|
|
||||||
cmd = [
|
|
||||||
"ffmpeg", "-y",
|
|
||||||
"-i", video_path,
|
|
||||||
"-vf", f"fps={fps}",
|
|
||||||
"-q:v", "2", # 质量等级 (2=高质量)
|
|
||||||
str(output_dir / "%04d.jpg")
|
|
||||||
]
|
|
||||||
|
|
||||||
print(f"🔄 截取视频帧: {video_path}")
|
|
||||||
print(f" 输出目录: {output_dir}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
||||||
if result.returncode == 0:
|
|
||||||
# 统计截取的帧数
|
|
||||||
frame_count = len(list(output_dir.glob("*.jpg")))
|
|
||||||
print(f"✅ 截取完成: {frame_count} 帧")
|
|
||||||
return str(output_dir)
|
|
||||||
else:
|
|
||||||
print(f"❌ ffmpeg 错误: {result.stderr}")
|
|
||||||
return ""
|
|
||||||
except FileNotFoundError:
|
|
||||||
print("❌ 错误: 未找到 ffmpeg,请先安装: apt install ffmpeg")
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""命令行入口"""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="OCR 视频帧文字识别工具",
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog="""
|
|
||||||
示例:
|
|
||||||
# 使用 EasyOCR 处理帧目录
|
|
||||||
python ocr_processor.py --input ./frames/ --output ./ocr.json
|
|
||||||
|
|
||||||
# 直接从视频截帧并 OCR
|
|
||||||
python ocr_processor.py --video ./video.flv --output ./ocr.json
|
|
||||||
|
|
||||||
# 使用 Tesseract 处理
|
|
||||||
python ocr_processor.py --input ./frames/ --output ./ocr.json --engine tesseract
|
|
||||||
|
|
||||||
# 指定截帧帧率
|
|
||||||
python ocr_processor.py --video ./video.flv --output ./ocr.json --fps 2
|
|
||||||
|
|
||||||
# 同时生成合并的文本文件
|
|
||||||
python ocr_processor.py --input ./frames/ --output ./ocr.json --merge
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
# 输入源:帧目录 或 视频文件(二选一)
|
|
||||||
input_group = parser.add_mutually_exclusive_group(required=True)
|
|
||||||
input_group.add_argument("--input", "-i", help="输入帧图片目录")
|
|
||||||
input_group.add_argument("--video", "-v", help="输入视频文件(会自动截帧)")
|
|
||||||
|
|
||||||
parser.add_argument("--output", "-o", required=True, help="输出 JSON 文件路径")
|
|
||||||
parser.add_argument("--fps", "-f", type=float, default=1.0, help="截帧帧率 (默认: 1帧/秒)")
|
|
||||||
parser.add_argument("--engine", "-e", choices=["easyocr", "tesseract"],
|
|
||||||
default="easyocr", help="OCR 引擎 (默认: easyocr)")
|
|
||||||
parser.add_argument("--merge", action="store_true", help="同时保存合并的文本文件")
|
|
||||||
parser.add_argument("--languages", "-l", nargs="+",
|
|
||||||
default=["ch_sim", "en"], help="语言列表")
|
|
||||||
parser.add_argument("--progress", "-p", help="可选:进度文件路径(会实时更新)")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# 如果指定了视频文件,先截帧
|
|
||||||
input_dir = args.input
|
|
||||||
if args.video:
|
|
||||||
if not os.path.exists(args.video):
|
|
||||||
print(json.dumps({
|
|
||||||
"success": False,
|
|
||||||
"error": f"视频文件不存在: {args.video}"
|
|
||||||
}, ensure_ascii=False, indent=2))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# 创建临时帧目录
|
|
||||||
frames_dir = Path(args.video).stem + "_frames"
|
|
||||||
input_dir = extract_frames(args.video, frames_dir, fps=args.fps)
|
|
||||||
|
|
||||||
if not input_dir:
|
|
||||||
print(json.dumps({
|
|
||||||
"success": False,
|
|
||||||
"error": "截帧失败"
|
|
||||||
}, ensure_ascii=False, indent=2))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# 初始化处理器
|
|
||||||
processor = OCRProcessor(engine=args.engine, languages=args.languages)
|
|
||||||
|
|
||||||
# 处理帧
|
|
||||||
results = processor.process_frames(input_dir, fps=args.fps, progress_file=args.progress)
|
|
||||||
|
|
||||||
if results:
|
|
||||||
# 保存结果
|
|
||||||
processor.save_results(results, args.output, merge=args.merge)
|
|
||||||
|
|
||||||
# 统一输出格式
|
|
||||||
total_texts = sum(1 for r in results if r.get("text"))
|
|
||||||
avg_conf = sum(r.get("confidence", 0) for r in results) / len(results) if results else 0
|
|
||||||
|
|
||||||
output = {
|
|
||||||
"success": True,
|
|
||||||
"filename": Path(args.output).name,
|
|
||||||
"filepath": str(Path(args.output).resolve()),
|
|
||||||
"message": f"OCR处理完成",
|
|
||||||
"stats": {
|
|
||||||
"total_frames": len(results),
|
|
||||||
"frames_with_text": total_texts,
|
|
||||||
"avg_confidence": round(avg_conf, 4)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
|
||||||
return 0
|
|
||||||
else:
|
|
||||||
print(json.dumps({
|
|
||||||
"success": False,
|
|
||||||
"error": "未处理到任何结果"
|
|
||||||
}, ensure_ascii=False, indent=2))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
exit(main())
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
easyocr>=1.7.0
|
|
||||||
pytesseract>=0.3.10
|
|
||||||
Pillow>=9.0.0
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
# KB Whisper Diarization 环境变量
|
|
||||||
|
|
||||||
# Whisper.cpp 安装路径
|
|
||||||
WHISPER_CPP_PATH=~/whisper.cpp
|
|
||||||
|
|
||||||
# HuggingFace Token(必需,用于下载 pyannote 模型)
|
|
||||||
# 获取方式: https://huggingface.co/settings/tokens
|
|
||||||
# 注意: 需要先申请 pyannote/speaker-diarization 模型访问权限
|
|
||||||
HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
||||||
@@ -1,151 +0,0 @@
|
|||||||
# KB Whisper Diarization Skill
|
|
||||||
|
|
||||||
**kb-whisper-diarization** - 使用 Whisper + pyannote.audio 实现语音识别和说话者分离。
|
|
||||||
|
|
||||||
## 什么时候使用
|
|
||||||
|
|
||||||
当用户需要:
|
|
||||||
- 区分多个说话者(如访谈、对话)
|
|
||||||
- 获取带说话者标签的字幕
|
|
||||||
- 分析对话中谁说了什么
|
|
||||||
|
|
||||||
## 环境变量
|
|
||||||
|
|
||||||
在使用之前,必须检查环境变量是否已经设置。
|
|
||||||
|
|
||||||
| 变量名 | 用途 | 示例 |
|
|
||||||
|--------|------|------|
|
|
||||||
| `WHISPER_CPP_PATH` | whisper.cpp 安装路径 | `~/whisper.cpp` |
|
|
||||||
| `HF_TOKEN` | HuggingFace Token | `hf_xxx...` |
|
|
||||||
|
|
||||||
### 获取 HuggingFace Token(必需)
|
|
||||||
|
|
||||||
pyannote.audio 需要从 HuggingFace 下载模型,按以下步骤操作:
|
|
||||||
|
|
||||||
**步骤 1: 注册/登录 HuggingFace**
|
|
||||||
- 访问 https://huggingface.co
|
|
||||||
- 注册或登录账号
|
|
||||||
|
|
||||||
**步骤 2: 申请模型访问权限**
|
|
||||||
需要申请以下两个模型的访问权限(点击页面上的 "Access repository" 按钮):
|
|
||||||
- https://huggingface.co/pyannote/speaker-diarization
|
|
||||||
- https://huggingface.co/pyannote/segmentation
|
|
||||||
|
|
||||||
> ⚠️ 申请后需要等待几分钟到几小时才能获得访问权限
|
|
||||||
|
|
||||||
**步骤 3: 创建 Access Token**
|
|
||||||
1. 访问 https://huggingface.co/settings/tokens
|
|
||||||
2. 点击 "New token"
|
|
||||||
3. 选择 "Read" 权限
|
|
||||||
4. 复制生成的 token(以 `hf_` 开头)
|
|
||||||
|
|
||||||
**步骤 4: 设置环境变量**
|
|
||||||
```bash
|
|
||||||
# 添加到 ~/.kb_env 文件
|
|
||||||
echo 'export HF_TOKEN="hf_xxxxxxxxxxxxxxxxxxxxxx"' >> ~/.kb_env
|
|
||||||
source ~/.kb_env
|
|
||||||
|
|
||||||
# 验证
|
|
||||||
echo $HF_TOKEN
|
|
||||||
```
|
|
||||||
|
|
||||||
## 使用方法
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 基本用法(带说话者分离)
|
|
||||||
python whisper_diarization.py --input video.mp4
|
|
||||||
|
|
||||||
# 说话者分离
|
|
||||||
python whisper_diarization.py --input interview.mp4 --diarize
|
|
||||||
|
|
||||||
# 指定说话者数量
|
|
||||||
python whisper_diarization.py --input meeting.mp4 --diarize --num-speakers 2
|
|
||||||
|
|
||||||
# 指定语言
|
|
||||||
python whisper_diarization.py --input video.mp4 --language zh
|
|
||||||
```
|
|
||||||
|
|
||||||
## 命令参数
|
|
||||||
|
|
||||||
| 参数 | 说明 | 默认值 |
|
|
||||||
|------|------|--------|
|
|
||||||
| `--input` | 输入音视频文件路径 | 必需 |
|
|
||||||
| `--output` | 输出目录 | 文件所在目录 |
|
|
||||||
| `--diarize` | 启用说话者分离 | False |
|
|
||||||
| `--num-speakers` | 估计说话者数量 | None(自动检测) |
|
|
||||||
| `--language` | 语言 | 自动检测 |
|
|
||||||
| `--model` | Whisper 模型 | base |
|
|
||||||
| `--background` | 后台异步执行 | False |
|
|
||||||
|
|
||||||
## 模型选项
|
|
||||||
|
|
||||||
- **Whisper 模型**: tiny, base, small, medium, large
|
|
||||||
- **说话者分离**: pyannote/speaker-diarization (需申请权限)
|
|
||||||
|
|
||||||
## 输出文件
|
|
||||||
|
|
||||||
对于输入 `interview.mp4`,将生成:
|
|
||||||
|
|
||||||
- `interview.mp4.json` - 完整结果(包含说话者标签)
|
|
||||||
- `interview.mp4.txt` - 纯文本
|
|
||||||
- `interview.mp4.srt` - 字幕格式(带说话者标签)
|
|
||||||
|
|
||||||
## 输出格式示例
|
|
||||||
|
|
||||||
### JSON 格式
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"input_file": "interview.mp4",
|
|
||||||
"diarization": true,
|
|
||||||
"speakers": ["SPEAKER_00", "SPEAKER_01"],
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 0,
|
|
||||||
"speaker": "SPEAKER_00",
|
|
||||||
"start": 0.0,
|
|
||||||
"end": 5.2,
|
|
||||||
"text": "大家好,欢迎来到访谈节目"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"speaker": "SPEAKER_01",
|
|
||||||
"start": 5.2,
|
|
||||||
"end": 10.8,
|
|
||||||
"text": "谢谢邀请,我很荣幸"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### SRT 字幕格式
|
|
||||||
|
|
||||||
```
|
|
||||||
1
|
|
||||||
00:00:00,000 --> 00:00:05,200
|
|
||||||
[SPEAKER_00] 大家好,欢迎来到访谈节目
|
|
||||||
|
|
||||||
2
|
|
||||||
00:00:05,200 --> 00:00:10,800
|
|
||||||
[SPEAKER_01] 谢谢邀请,我很荣幸
|
|
||||||
```
|
|
||||||
|
|
||||||
## 安装
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. 安装依赖
|
|
||||||
pip install pyannote.audio torch
|
|
||||||
|
|
||||||
# 2. 设置 HuggingFace Token
|
|
||||||
export HF_TOKEN="hf_xxx..."
|
|
||||||
|
|
||||||
# 3. 首次运行会自动下载模型
|
|
||||||
python whisper_diarization.py --input test.mp4
|
|
||||||
```
|
|
||||||
|
|
||||||
## 注意事项
|
|
||||||
|
|
||||||
- **说话者分离需要申请权限**: https://huggingface.co/pyannote/speaker-diarization
|
|
||||||
- **处理时间较长**: 说话者分离会增加处理时间
|
|
||||||
- **对立体声效果更好**: 多声道音频的分离效果更佳
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
# KB Whisper Diarization 依赖
|
|
||||||
|
|
||||||
# 核心依赖
|
|
||||||
torch>=2.0.0
|
|
||||||
pyannote.audio>=1.0.0
|
|
||||||
|
|
||||||
# 可选: GPU 加速
|
|
||||||
# torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
|
||||||
@@ -1,339 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
KB Whisper Diarization - 语音识别 + 说话者分离
|
|
||||||
使用 Whisper 进行语音识别,pyannote.audio 进行说话者分离
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from datetime import datetime
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
|
|
||||||
|
|
||||||
def parse_arguments():
|
|
||||||
"""解析命令行参数"""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description='KB Whisper Diarization - 语音识别 + 说话者分离',
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog="""
|
|
||||||
示例:
|
|
||||||
python whisper_diarization.py --input video.mp4
|
|
||||||
python whisper_diarization.py --input interview.mp4 --diarize
|
|
||||||
python whisper_diarization.py --input meeting.mp4 --diarize --num-speakers 2
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
parser.add_argument('--input', required=True, help='输入音视频文件路径')
|
|
||||||
parser.add_argument('--output', help='输出目录 (默认: 文件所在目录)')
|
|
||||||
parser.add_argument('--diarize', action='store_true', help='启用说话者分离')
|
|
||||||
parser.add_argument('--num-speakers', type=int, help='估计说话者数量')
|
|
||||||
parser.add_argument('--language', help='指定语言 (默认: 自动检测)')
|
|
||||||
parser.add_argument('--model', default='base', choices=['tiny', 'base', 'small', 'medium', 'large'],
|
|
||||||
help='Whisper 模型 (默认: base)')
|
|
||||||
parser.add_argument('--background', action='store_true', help='后台异步执行')
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def get_whisper_path():
|
|
||||||
"""获取 whisper.cpp 路径"""
|
|
||||||
path = os.environ.get('WHISPER_CPP_PATH')
|
|
||||||
if path:
|
|
||||||
path = os.path.expanduser(path)
|
|
||||||
if os.path.exists(path):
|
|
||||||
return path
|
|
||||||
return os.path.expanduser('~/whisper.cpp')
|
|
||||||
|
|
||||||
|
|
||||||
def check_whisper_model(whisper_path, model):
|
|
||||||
"""检查 Whisper 模型"""
|
|
||||||
model_path = os.path.join(whisper_path, 'models', f'ggml-{model}.bin')
|
|
||||||
if not os.path.exists(model_path):
|
|
||||||
raise FileNotFoundError(f"Whisper 模型不存在: {model_path}")
|
|
||||||
return model_path
|
|
||||||
|
|
||||||
|
|
||||||
def convert_to_wav(input_path, output_dir):
|
|
||||||
"""转换为 16kHz WAV"""
|
|
||||||
if input_path.lower().endswith('.wav'):
|
|
||||||
try:
|
|
||||||
result = subprocess.run(
|
|
||||||
['ffprobe', '-v', 'error', '-show_entries', 'stream=sample_rate',
|
|
||||||
'-of', 'default=noprint_wrappers=1:nokey=1', input_path],
|
|
||||||
capture_output=True, text=True
|
|
||||||
)
|
|
||||||
if result.returncode == 0 and int(result.stdout.strip()) == 16000:
|
|
||||||
return input_path
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
output_path = os.path.join(output_dir, f"{Path(input_path).stem}_audio.wav")
|
|
||||||
print(f"转换音频: {input_path} -> {output_path}")
|
|
||||||
|
|
||||||
subprocess.run([
|
|
||||||
'ffmpeg', '-y', '-i', input_path,
|
|
||||||
'-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',
|
|
||||||
'-loglevel', 'error', output_path
|
|
||||||
], check=True, capture_output=True)
|
|
||||||
|
|
||||||
print("✓ 音频转换完成")
|
|
||||||
return output_path
|
|
||||||
|
|
||||||
|
|
||||||
def run_whisper(whisper_path, audio_path, model, output_dir, language=None):
|
|
||||||
"""运行 Whisper 识别"""
|
|
||||||
basename = Path(audio_path).stem.replace('_audio', '')
|
|
||||||
output_file = os.path.join(output_dir, basename)
|
|
||||||
|
|
||||||
cmd = [
|
|
||||||
os.path.join(whisper_path, 'whisper-cli'),
|
|
||||||
'-m', os.path.join(whisper_path, 'models', f'ggml-{model}.bin'),
|
|
||||||
'-f', audio_path,
|
|
||||||
'-of', output_file,
|
|
||||||
'-l', language or 'auto',
|
|
||||||
'--output-txt', '--output-srt', '--output-json',
|
|
||||||
]
|
|
||||||
|
|
||||||
print(f"执行 Whisper: {' '.join(cmd[:6])} ...")
|
|
||||||
|
|
||||||
result = subprocess.run(
|
|
||||||
cmd, capture_output=True, text=True, cwd=whisper_path, timeout=3600
|
|
||||||
)
|
|
||||||
|
|
||||||
if result.returncode != 0 and not os.path.exists(f"{output_file}.txt"):
|
|
||||||
raise RuntimeError(f"Whisper 失败: {result.stderr}")
|
|
||||||
|
|
||||||
return basename
|
|
||||||
|
|
||||||
|
|
||||||
def run_diarization(audio_path, num_speakers=None):
|
|
||||||
"""运行 pyannote.audio 说话者分离"""
|
|
||||||
print("\n🎙️ 步骤2: 说话者分离...")
|
|
||||||
|
|
||||||
try:
|
|
||||||
import torch
|
|
||||||
from pyannote.audio import Pipeline
|
|
||||||
except ImportError as e:
|
|
||||||
print(f"✗ 缺少依赖: {e}")
|
|
||||||
print("请安装: pip install pyannote.audio torch")
|
|
||||||
raise
|
|
||||||
|
|
||||||
# 检查 HuggingFace Token
|
|
||||||
hf_token = os.environ.get('HF_TOKEN')
|
|
||||||
if not hf_token:
|
|
||||||
raise RuntimeError("需要设置 HF_TOKEN 环境变量\n"
|
|
||||||
"获取方式: https://huggingface.co/settings/tokens")
|
|
||||||
|
|
||||||
# 新版 pyannote 需要先登录
|
|
||||||
from huggingface_hub import login
|
|
||||||
print("登录 HuggingFace...")
|
|
||||||
login(token=hf_token)
|
|
||||||
|
|
||||||
print("加载 pyannote 流水线...")
|
|
||||||
pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization")
|
|
||||||
|
|
||||||
# 设置设备
|
|
||||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
||||||
print(f"使用设备: {device}")
|
|
||||||
pipeline.to(device)
|
|
||||||
|
|
||||||
# 运行分离
|
|
||||||
print("执行说话者分离...")
|
|
||||||
diarization = pipeline(
|
|
||||||
audio_path,
|
|
||||||
num_speakers=num_speakers
|
|
||||||
)
|
|
||||||
|
|
||||||
# 解析结果
|
|
||||||
speakers = {}
|
|
||||||
for turn, _, speaker in diarization.itertracks(yield_label=True):
|
|
||||||
if speaker not in speakers:
|
|
||||||
speakers[speaker] = []
|
|
||||||
speakers[speaker].append({
|
|
||||||
"start": round(turn.start, 2),
|
|
||||||
"end": round(turn.end, 2)
|
|
||||||
})
|
|
||||||
|
|
||||||
print(f"✓ 检测到 {len(speakers)} 个说话者: {list(speakers.keys())}")
|
|
||||||
return speakers
|
|
||||||
|
|
||||||
|
|
||||||
def merge_results(whisper_txt, whisper_srt, diarization, output_dir, basename):
|
|
||||||
"""合并 Whisper 和说话者分离结果"""
|
|
||||||
print("\n📝 步骤3: 合并结果...")
|
|
||||||
|
|
||||||
# 读取 Whisper 文本
|
|
||||||
text_segments = []
|
|
||||||
if os.path.exists(whisper_txt):
|
|
||||||
with open(whisper_txt, 'r', encoding='utf-8') as f:
|
|
||||||
content = f.read()
|
|
||||||
|
|
||||||
# 简单按行分段
|
|
||||||
for i, line in enumerate(content.strip().split('\n')):
|
|
||||||
if line.strip():
|
|
||||||
text_segments.append({
|
|
||||||
"id": i,
|
|
||||||
"text": line.strip(),
|
|
||||||
"start": i * 3.0, # 估算
|
|
||||||
"end": (i + 1) * 3.0
|
|
||||||
})
|
|
||||||
|
|
||||||
# 读取 SRT 获取精确时间
|
|
||||||
srt_segments = []
|
|
||||||
if os.path.exists(whisper_srt):
|
|
||||||
with open(whisper_srt, 'r', encoding='utf-8') as f:
|
|
||||||
content = f.read()
|
|
||||||
|
|
||||||
# 解析 SRT
|
|
||||||
pattern = r'(\d+)\n(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\n(.+?)(?=\n\n|\Z)'
|
|
||||||
matches = re.findall(pattern, content, re.DOTALL)
|
|
||||||
|
|
||||||
def parse_time(t):
|
|
||||||
h, m, s_ms = t.split(':')
|
|
||||||
s, ms = s_ms.split(',')
|
|
||||||
return float(h) * 3600 + float(m) * 60 + float(s) + float(ms) / 1000
|
|
||||||
|
|
||||||
for idx, num, start, end, text in matches:
|
|
||||||
srt_segments.append({
|
|
||||||
"id": int(idx),
|
|
||||||
"start": parse_time(start),
|
|
||||||
"end": parse_time(end),
|
|
||||||
"text": text.strip()
|
|
||||||
})
|
|
||||||
|
|
||||||
# 为每个 Whisper 段落分配说话者
|
|
||||||
merged_segments = []
|
|
||||||
for seg in srt_segments:
|
|
||||||
speaker = assign_speaker(seg['start'], seg['end'], diarization)
|
|
||||||
merged_segments.append({
|
|
||||||
"id": len(merged_segments),
|
|
||||||
"speaker": speaker,
|
|
||||||
"start": seg['start'],
|
|
||||||
"end": seg['end'],
|
|
||||||
"text": seg['text']
|
|
||||||
})
|
|
||||||
|
|
||||||
# 生成带说话者的 SRT
|
|
||||||
output_srt = os.path.join(output_dir, f"{basename}_diarized.srt")
|
|
||||||
with open(output_srt, 'w', encoding='utf-8') as f:
|
|
||||||
for seg in merged_segments:
|
|
||||||
start = format_time(seg['start'])
|
|
||||||
end = format_time(seg['end'])
|
|
||||||
f.write(f"{seg['id'] + 1}\n")
|
|
||||||
f.write(f"{start} --> {end}\n")
|
|
||||||
f.write(f"[{seg['speaker']}] {seg['text']}\n\n")
|
|
||||||
|
|
||||||
# 生成合并的 TXT
|
|
||||||
output_txt = os.path.join(output_dir, f"{basename}_diarized.txt")
|
|
||||||
with open(output_txt, 'w', encoding='utf-8') as f:
|
|
||||||
for seg in merged_segments:
|
|
||||||
f.write(f"[{seg['speaker']}] {seg['text']}\n")
|
|
||||||
|
|
||||||
# 生成 JSON
|
|
||||||
output_json = os.path.join(output_dir, f"{basename}_diarized.json")
|
|
||||||
json_data = {
|
|
||||||
"success": True,
|
|
||||||
"tool": "whisper-diarization",
|
|
||||||
"diarization": True,
|
|
||||||
"speakers": list(diarization.keys()),
|
|
||||||
"input_file": basename,
|
|
||||||
"output_files": {
|
|
||||||
"txt": f"{basename}_diarized.txt",
|
|
||||||
"srt": f"{basename}_diarized.srt",
|
|
||||||
"json": f"{basename}_diarized.json"
|
|
||||||
},
|
|
||||||
"segments": merged_segments
|
|
||||||
}
|
|
||||||
with open(output_json, 'w', encoding='utf-8') as f:
|
|
||||||
json.dump(json_data, f, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
print(f"✓ 生成文件:")
|
|
||||||
print(f" - {output_srt}")
|
|
||||||
print(f" - {output_txt}")
|
|
||||||
print(f" - {output_json}")
|
|
||||||
|
|
||||||
return json_data
|
|
||||||
|
|
||||||
|
|
||||||
def assign_speaker(start, end, diarization):
|
|
||||||
"""将时间段分配给说话者"""
|
|
||||||
best_speaker = "UNKNOWN"
|
|
||||||
max_overlap = 0
|
|
||||||
|
|
||||||
for speaker, segments in diarization.items():
|
|
||||||
for seg in segments:
|
|
||||||
overlap = min(end, seg['end']) - max(start, seg['start'])
|
|
||||||
if overlap > max_overlap:
|
|
||||||
max_overlap = overlap
|
|
||||||
best_speaker = speaker
|
|
||||||
|
|
||||||
return best_speaker
|
|
||||||
|
|
||||||
|
|
||||||
def format_time(seconds):
|
|
||||||
"""秒数转换为 SRT 时间格式"""
|
|
||||||
hours = int(seconds // 3600)
|
|
||||||
minutes = int((seconds % 3600) // 60)
|
|
||||||
secs = int(seconds % 60)
|
|
||||||
ms = int((seconds % 1) * 1000)
|
|
||||||
return f"{hours:02d}:{minutes:02d}:{secs:02d},{ms:03d}"
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
args = parse_arguments()
|
|
||||||
|
|
||||||
input_file = os.path.abspath(args.input)
|
|
||||||
if not os.path.exists(input_file):
|
|
||||||
print(f"✗ 文件不存在: {input_file}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
output_dir = os.path.abspath(args.output) if args.output else os.path.dirname(input_file)
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
|
|
||||||
basename = Path(input_file).stem
|
|
||||||
|
|
||||||
print(f"\n📁 输入: {input_file}")
|
|
||||||
print(f"📁 输出: {output_dir}")
|
|
||||||
print(f"🎙️ 说话者分离: {'是' if args.diarize else '否'}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 步骤1: Whisper 识别
|
|
||||||
print(f"\n🎙️ 步骤1: 语音识别...")
|
|
||||||
whisper_path = get_whisper_path()
|
|
||||||
check_whisper_model(whisper_path, args.model)
|
|
||||||
|
|
||||||
audio_path = convert_to_wav(input_file, output_dir)
|
|
||||||
run_whisper(whisper_path, audio_path, args.model, output_dir, args.language)
|
|
||||||
|
|
||||||
# 步骤2: 说话者分离
|
|
||||||
if args.diarize:
|
|
||||||
diarization = run_diarization(audio_path, args.num_speakers)
|
|
||||||
else:
|
|
||||||
diarization = {}
|
|
||||||
|
|
||||||
# 步骤3: 合并结果
|
|
||||||
whisper_txt = os.path.join(output_dir, f"{basename}.txt")
|
|
||||||
whisper_srt = os.path.join(output_dir, f"{basename}.srt")
|
|
||||||
result = merge_results(whisper_txt, whisper_srt, diarization, output_dir, basename)
|
|
||||||
|
|
||||||
# 清理临时音频
|
|
||||||
if audio_path != input_file and audio_path.endswith('_audio.wav'):
|
|
||||||
try:
|
|
||||||
os.remove(audio_path)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
print(f"\n✅ 处理完成!")
|
|
||||||
print(f" 说话者数量: {len(diarization) if args.diarize else 1}")
|
|
||||||
print(f" 段落数量: {len(result['segments'])}")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"\n✗ 错误: {e}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,5 +0,0 @@
|
|||||||
# KB Whisper 环境变量配置示例
|
|
||||||
# 复制此文件为 .env 并填写实际值
|
|
||||||
|
|
||||||
# Whisper.cpp 安装路径
|
|
||||||
WHISPER_CPP_PATH=~/whisper.cpp
|
|
||||||
@@ -1,211 +0,0 @@
|
|||||||
# KB Whisper - 本地语音识别
|
|
||||||
|
|
||||||
基于 Whisper.cpp 的本地语音识别工具,支持将音视频文件转换为文本、字幕和 JSON 格式。
|
|
||||||
|
|
||||||
## 功能特性
|
|
||||||
|
|
||||||
- 🎯 **本地执行** - 完全离线运行,保护隐私
|
|
||||||
- 🚀 **多模型支持** - 从 tiny 到 large,根据需求选择速度/精度
|
|
||||||
- 📝 **多格式输出** - JSON、SRT、TXT 三种格式
|
|
||||||
- ⚡ **异步执行** - 支持后台运行,不阻塞主代理
|
|
||||||
- 🎬 **广泛兼容** - 支持常见音视频格式(MP3、MP4、WAV 等)
|
|
||||||
|
|
||||||
## 安装前提
|
|
||||||
|
|
||||||
### 1. 安装 whisper.cpp
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 克隆仓库
|
|
||||||
git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp
|
|
||||||
|
|
||||||
# 编译
|
|
||||||
cd ~/whisper.cpp
|
|
||||||
make
|
|
||||||
|
|
||||||
# 下载模型(至少需要一个模型)
|
|
||||||
./models/download-ggml-model.sh tiny
|
|
||||||
./models/download-ggml-model.sh base # 可选
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. 配置环境变量
|
|
||||||
|
|
||||||
在 `~/.clawdbot/clawdbot.json` 中添加:
|
|
||||||
|
|
||||||
```json5
|
|
||||||
{
|
|
||||||
skills: {
|
|
||||||
entries: {
|
|
||||||
"kb-whisper": {
|
|
||||||
env: {
|
|
||||||
WHISPER_CPP_PATH: "~/whisper.cpp"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
或通过命令行配置:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
clawdbot skills config kb-whisper WHISPER_CPP_PATH "~/whisper.cpp"
|
|
||||||
```
|
|
||||||
|
|
||||||
## 使用方法
|
|
||||||
|
|
||||||
### 基本用法
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 使用默认 tiny 模型处理
|
|
||||||
python whisper.py --input video.mp4
|
|
||||||
|
|
||||||
# 使用 base 模型(推荐,平衡速度和精度)
|
|
||||||
python whisper.py --input video.mp4 --model base
|
|
||||||
|
|
||||||
# 后台异步执行
|
|
||||||
python whisper.py --input video.mp4 --background
|
|
||||||
```
|
|
||||||
|
|
||||||
### 指定输出目录
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python whisper.py --input video.mp4 --output ./transcripts
|
|
||||||
```
|
|
||||||
|
|
||||||
### 处理音频文件
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python whisper.py --input podcast.mp3 --model base
|
|
||||||
```
|
|
||||||
|
|
||||||
## 输出文件
|
|
||||||
|
|
||||||
对于输入 `video.mp4`,将生成以下文件:
|
|
||||||
|
|
||||||
```
|
|
||||||
output/
|
|
||||||
├── video.json # 详细结果(含时间戳)
|
|
||||||
├── video.srt # 字幕格式
|
|
||||||
└── video.txt # 纯文本
|
|
||||||
```
|
|
||||||
|
|
||||||
### JSON 格式
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"input_file": "video.mp4",
|
|
||||||
"model": "base",
|
|
||||||
"output_files": {
|
|
||||||
"json": "video.json",
|
|
||||||
"srt": "video.srt",
|
|
||||||
"txt": "video.txt"
|
|
||||||
},
|
|
||||||
"duration_seconds": 120.5,
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 0,
|
|
||||||
"start": 0.0,
|
|
||||||
"end": 3.2,
|
|
||||||
"text": "这是第一段字幕"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": 1,
|
|
||||||
"start": 3.2,
|
|
||||||
"end": 6.8,
|
|
||||||
"text": "这是第二段字幕"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 模型对比
|
|
||||||
|
|
||||||
| 模型 | 速度 | 精度 | 适用场景 |
|
|
||||||
|------|------|------|----------|
|
|
||||||
| tiny | ⚡⚡⚡ | ⭐ | 快速预览、实时转录 |
|
|
||||||
| base | ⚡⚡ | ⭐⭐⭐ | 日常使用(推荐) |
|
|
||||||
| small | ⚡ | ⭐⭐⭐⭐ | 高质量需求 |
|
|
||||||
| medium | ⭐ | ⭐⭐⭐⭐⭐ | 专业用途 |
|
|
||||||
| large | 🐢 | ⭐⭐⭐⭐⭐ | 最高精度要求 |
|
|
||||||
|
|
||||||
## 完整示例
|
|
||||||
|
|
||||||
### 示例 1: 转录播客
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python whisper.py --input podcast.mp3 --model base --background
|
|
||||||
```
|
|
||||||
|
|
||||||
### 示例 2: 生成视频字幕
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python whisper.py --input tutorial.mp4 --model small
|
|
||||||
```
|
|
||||||
|
|
||||||
### 示例 3: 批量处理(配合脚本)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
#!/bin/bash
|
|
||||||
for video in videos/*.mp4; do
|
|
||||||
python whisper.py --input "$video" --model base
|
|
||||||
done
|
|
||||||
```
|
|
||||||
|
|
||||||
## 在 Workflow 中使用
|
|
||||||
|
|
||||||
在知识库处理流程中,如果视频没有字幕:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 步骤4: 下载视频后,如果没有字幕
|
|
||||||
# 步骤5: 使用 Whisper 生成字幕
|
|
||||||
python whisper.py --input ${video_path} --model base --background
|
|
||||||
|
|
||||||
# 等待 Whisper 完成,然后继续后续步骤
|
|
||||||
```
|
|
||||||
|
|
||||||
## 技术细节
|
|
||||||
|
|
||||||
- **引擎**: Whisper.cpp(C++ 实现,性能优异)
|
|
||||||
- **音频采样率**: 自动转换为 16kHz
|
|
||||||
- **语言**: 自动检测(支持 99 种语言)
|
|
||||||
- **GPU 支持**: whisper.cpp 支持 CUDA 加速(需编译时启用)
|
|
||||||
|
|
||||||
## 故障排查
|
|
||||||
|
|
||||||
### 问题: 找不到 whisper.cpp
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 检查环境变量
|
|
||||||
echo $WHISPER_CPP_PATH
|
|
||||||
|
|
||||||
# 手动测试
|
|
||||||
~/whisper.cpp/main --help
|
|
||||||
```
|
|
||||||
|
|
||||||
### 问题: 模型未下载
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd ~/whisper.cpp
|
|
||||||
./models/download-ggml-model.sh tiny
|
|
||||||
```
|
|
||||||
|
|
||||||
### 问题: 音频格式不支持
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 转换音频格式
|
|
||||||
ffmpeg -i input.avi -acodec libmp3lame output.mp3
|
|
||||||
python whisper.py --input output.mp3
|
|
||||||
```
|
|
||||||
|
|
||||||
## 性能参考
|
|
||||||
|
|
||||||
在 CPU 环境下(Intel i7):
|
|
||||||
|
|
||||||
- **tiny 模型**: ~1x 实时速度(1分钟音频 ~1分钟处理)
|
|
||||||
- **base 模型**: ~0.5x 实时速度(1分钟音频 ~2分钟处理)
|
|
||||||
- **small 模型**: ~0.25x 实时速度(1分钟音频 ~4分钟处理)
|
|
||||||
|
|
||||||
## 许可证
|
|
||||||
|
|
||||||
本技能使用 MIT 许可证,Whisper.cpp 使用 MIT 许可证。
|
|
||||||
@@ -1,124 +0,0 @@
|
|||||||
---
|
|
||||||
name: kb-whisper
|
|
||||||
description: 本地语音识别,支持音视频转文字(基于 Whisper.cpp)
|
|
||||||
---
|
|
||||||
|
|
||||||
# KB Whisper Skill
|
|
||||||
|
|
||||||
**kb-whisper** - 本地语音识别,使用 Whisper.cpp 将音视频文件转换为文本、字幕和 JSON 格式输出。
|
|
||||||
|
|
||||||
## 什么时候使用
|
|
||||||
|
|
||||||
当用户需要:
|
|
||||||
- 将视频/音频文件转换为文字
|
|
||||||
- 生成视频字幕(SRT 格式)
|
|
||||||
- 获取带时间戳的详细识别结果
|
|
||||||
|
|
||||||
## 环境变量
|
|
||||||
|
|
||||||
在使用之前,必须检查环境变量是否已经设置。
|
|
||||||
|
|
||||||
| 变量名 | 用途 | 示例 |
|
|
||||||
|--------|------|------|
|
|
||||||
| `WHISPER_CPP_PATH` | whisper.cpp 安装路径 | `~/whisper.cpp` |
|
|
||||||
|
|
||||||
## 使用方法
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 基本用法
|
|
||||||
python whisper.py --input video.mp4
|
|
||||||
|
|
||||||
# 指定模型(默认 tiny)
|
|
||||||
python whisper.py --input video.mp4 --model base
|
|
||||||
|
|
||||||
# 指定输出目录
|
|
||||||
python whisper.py --input video.mp4 --output ./output
|
|
||||||
|
|
||||||
# 异步执行(推荐)
|
|
||||||
python whisper.py --input video.mp4 --background
|
|
||||||
```
|
|
||||||
|
|
||||||
## 命令参数
|
|
||||||
|
|
||||||
| 参数 | 说明 | 默认值 |
|
|
||||||
|------|------|--------|
|
|
||||||
| `--input` | 输入音视频文件路径 | 必需 |
|
|
||||||
| `--model` | Whisper 模型类型 | `tiny` |
|
|
||||||
| `--output` | 输出目录 | 文件所在目录 |
|
|
||||||
| `--background` | 后台异步执行 | False |
|
|
||||||
|
|
||||||
## 模型选项
|
|
||||||
|
|
||||||
- `tiny` - 最小模型,速度快,精度较低(推荐快速识别)
|
|
||||||
- `base` - 基础模型,平衡速度和精度(推荐)
|
|
||||||
- `small` - 小型模型,精度更高,速度较慢
|
|
||||||
- `medium` - 中型模型,高精度
|
|
||||||
- `large` - 大型模型,最高精度,速度慢
|
|
||||||
|
|
||||||
## 输出文件
|
|
||||||
|
|
||||||
对于输入 `video.mp4`,将生成:
|
|
||||||
|
|
||||||
- `video.json` - 详细结果(包含时间戳、分段信息)
|
|
||||||
- `video.srt` - 字幕格式文件
|
|
||||||
- `video.txt` - 纯文本内容
|
|
||||||
|
|
||||||
## 输出格式示例
|
|
||||||
|
|
||||||
### JSON 格式
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"input_file": "video.mp4",
|
|
||||||
"model": "tiny",
|
|
||||||
"output_files": {
|
|
||||||
"json": "video.json",
|
|
||||||
"srt": "video.srt",
|
|
||||||
"txt": "video.txt"
|
|
||||||
},
|
|
||||||
"duration_seconds": 120.5,
|
|
||||||
"segments": [
|
|
||||||
{
|
|
||||||
"id": 0,
|
|
||||||
"start": 0.0,
|
|
||||||
"end": 3.2,
|
|
||||||
"text": "这是第一段字幕"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Workflow 中的使用
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 步骤4之后:如果已有音频但无字幕,可以使用 whisper
|
|
||||||
python whisper.py --input ${video_path} --model base --background
|
|
||||||
```
|
|
||||||
|
|
||||||
## 安装
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. 克隆 whisper.cpp
|
|
||||||
git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp
|
|
||||||
|
|
||||||
# 2. 编译
|
|
||||||
cd ~/whisper.cpp
|
|
||||||
make
|
|
||||||
|
|
||||||
# 3. 下载模型(tiny)
|
|
||||||
./models/download-ggml-model.sh tiny
|
|
||||||
|
|
||||||
# 4. 下载更多模型(可选)
|
|
||||||
# ./models/download-ggml-model.sh base
|
|
||||||
# ./models/download-ggml-model.sh small
|
|
||||||
|
|
||||||
# 5. 复制技能到 skills 目录
|
|
||||||
cp -r kb-whisper/ ~/skills/
|
|
||||||
```
|
|
||||||
|
|
||||||
## 注意事项
|
|
||||||
|
|
||||||
- **异步执行**: 使用 `--background` 参数时,命令会在后台运行,不阻塞主代理
|
|
||||||
- **模型选择**: tiny 模型适合快速识别,base 模型适合一般用途,large 模型适合高精度需求
|
|
||||||
- **文件格式**: 支持 MP3、MP4、WAV、FLAC、OGG、M4A 等常见格式
|
|
||||||
Binary file not shown.
@@ -1,273 +0,0 @@
|
|||||||
# KB Whisper 使用示例
|
|
||||||
|
|
||||||
## 快速开始
|
|
||||||
|
|
||||||
### 1. 基础使用
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 使用默认 tiny 模型转录视频
|
|
||||||
python whisper.py --input video.mp4
|
|
||||||
|
|
||||||
# 使用 base 模型(推荐)
|
|
||||||
python whisper.py --input video.mp4 --model base
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. 异步执行
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 后台处理,不阻塞主代理
|
|
||||||
python whisper.py --input video.mp4 --model base --background
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. 指定输出目录
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python whisper.py --input video.mp4 --output ./transcripts
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. 指定语言
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 指定中文
|
|
||||||
python whisper.py --input video.mp4 --language zh
|
|
||||||
|
|
||||||
# 指定英文
|
|
||||||
python whisper.py --input video.mp4 --language en
|
|
||||||
|
|
||||||
# 指定日语
|
|
||||||
python whisper.py --input video.mp4 --language ja
|
|
||||||
```
|
|
||||||
|
|
||||||
## 实际应用场景
|
|
||||||
|
|
||||||
### 场景 1: 转录播客
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python whisper.py --input podcast.mp3 --model base --background
|
|
||||||
```
|
|
||||||
|
|
||||||
**输出**:
|
|
||||||
- `podcast.json` - 带时间戳的详细结果
|
|
||||||
- `podcast.srt` - 字幕格式
|
|
||||||
- `podcast.txt` - 纯文本
|
|
||||||
|
|
||||||
### 场景 2: 生成视频字幕
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python whisper.py --input tutorial.mp4 --model small
|
|
||||||
```
|
|
||||||
|
|
||||||
**用途**:
|
|
||||||
- 为教程视频生成字幕
|
|
||||||
- 导出 SRT 文件到视频编辑软件
|
|
||||||
- 生成多语言字幕版本
|
|
||||||
|
|
||||||
### 场景 3: 会议录音转文字
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python whisper.py --input meeting.mp3 --model base --output ./meetings
|
|
||||||
```
|
|
||||||
|
|
||||||
**输出示例**:
|
|
||||||
```
|
|
||||||
meetings/
|
|
||||||
├── meeting.json # 完整 JSON(含时间戳)
|
|
||||||
├── meeting.srt # 字幕格式
|
|
||||||
└── meeting.txt # 纯文本
|
|
||||||
```
|
|
||||||
|
|
||||||
### 场景 4: 批量处理
|
|
||||||
|
|
||||||
创建脚本 `batch_transcribe.sh`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
OUTPUT_DIR="./transcripts"
|
|
||||||
mkdir -p "$OUTPUT_DIR"
|
|
||||||
|
|
||||||
for video in videos/*.mp4; do
|
|
||||||
echo "处理: $video"
|
|
||||||
python whisper.py --input "$video" --output "$OUTPUT_DIR" --model base
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "批量处理完成!"
|
|
||||||
```
|
|
||||||
|
|
||||||
```bash
|
|
||||||
chmod +x batch_transcribe.sh
|
|
||||||
./batch_transcribe.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
## 在 Workflow 中使用
|
|
||||||
|
|
||||||
### 知识库处理流程
|
|
||||||
|
|
||||||
在 `kb-workflow` 技能中,如果视频没有字幕:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 步骤 4: 下载视频后
|
|
||||||
# 检查是否有字幕文件
|
|
||||||
if [ ! -f "${subtitle_file}" ]; then
|
|
||||||
echo "无字幕文件,使用 Whisper 生成..."
|
|
||||||
|
|
||||||
# 使用 Whisper 生成字幕(后台执行)
|
|
||||||
python /path/to/whisper.py \
|
|
||||||
--input "${video_path}" \
|
|
||||||
--model base \
|
|
||||||
--output "${content_dir}" \
|
|
||||||
--background
|
|
||||||
|
|
||||||
# 等待 Whisper 完成
|
|
||||||
sleep 5 # 等待后台任务启动
|
|
||||||
while ! [ -f "${content_dir}/$(basename ${video_path} .mp4).srt" ]; do
|
|
||||||
sleep 2
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "Whisper 处理完成"
|
|
||||||
fi
|
|
||||||
```
|
|
||||||
|
|
||||||
## 输出文件使用
|
|
||||||
|
|
||||||
### JSON 格式
|
|
||||||
|
|
||||||
```python
|
|
||||||
import json
|
|
||||||
|
|
||||||
# 读取 JSON 结果
|
|
||||||
with open('video.json', 'r', encoding='utf-8') as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
# 访问分段
|
|
||||||
for segment in data['segments']:
|
|
||||||
print(f"[{segment['start']:.2f}-{segment['end']:.2f}] {segment['text']}")
|
|
||||||
```
|
|
||||||
|
|
||||||
### SRT 格式
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 直接在视频播放器中使用
|
|
||||||
mpv video.mp4 --sub-file=video.srt
|
|
||||||
|
|
||||||
# 或转换字幕格式
|
|
||||||
ffmpeg -i video.srt video.ass
|
|
||||||
```
|
|
||||||
|
|
||||||
### TXT 格式
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 搜索关键词
|
|
||||||
grep "重要信息" video.txt
|
|
||||||
|
|
||||||
# 统计字数
|
|
||||||
wc -m video.txt
|
|
||||||
```
|
|
||||||
|
|
||||||
## 性能优化建议
|
|
||||||
|
|
||||||
### 1. 根据内容选择模型
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 快速预览(短视频、测试)
|
|
||||||
python whisper.py --input video.mp4 --model tiny
|
|
||||||
|
|
||||||
# 日常使用(播客、会议)
|
|
||||||
python whisper.py --input video.mp4 --model base
|
|
||||||
|
|
||||||
# 高质量需求(正式内容、多语言)
|
|
||||||
python whisper.py --input video.mp4 --model small
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. 使用后台执行
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 处理长文件时使用后台模式
|
|
||||||
python whisper.py --input long_video.mp4 --model base --background
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. 指定语言提升准确度
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 如果知道内容语言,指定后准确度更高
|
|
||||||
python whisper.py --input video.mp4 --language zh
|
|
||||||
```
|
|
||||||
|
|
||||||
## 故障排查
|
|
||||||
|
|
||||||
### 问题 1: whisper.cpp 未找到
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 检查环境变量
|
|
||||||
echo $WHISPER_CPP_PATH
|
|
||||||
|
|
||||||
# 或配置
|
|
||||||
clawdbot skills config kb-whisper WHISPER_CPP_PATH "~/whisper.cpp"
|
|
||||||
```
|
|
||||||
|
|
||||||
### 问题 2: 模型未下载
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd ~/whisper.cpp
|
|
||||||
./models/download-ggml-model.sh base
|
|
||||||
```
|
|
||||||
|
|
||||||
### 问题 3: 音频格式不支持
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 转换格式
|
|
||||||
ffmpeg -i input.avi -acodec libmp3lame output.mp3
|
|
||||||
python whisper.py --input output.mp3
|
|
||||||
```
|
|
||||||
|
|
||||||
### 问题 4: 后台任务未完成
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 查看日志
|
|
||||||
cat video_whisper.log
|
|
||||||
|
|
||||||
# 检查进程
|
|
||||||
ps aux | grep whisper
|
|
||||||
```
|
|
||||||
|
|
||||||
## 集成到其他工具
|
|
||||||
|
|
||||||
### 集成到 Python 脚本
|
|
||||||
|
|
||||||
```python
|
|
||||||
import subprocess
|
|
||||||
import json
|
|
||||||
|
|
||||||
def transcribe_audio(audio_path, model='base'):
|
|
||||||
"""调用 whisper.py 转录音频"""
|
|
||||||
cmd = [
|
|
||||||
'python3', '/path/to/whisper.py',
|
|
||||||
'--input', audio_path,
|
|
||||||
'--model', model
|
|
||||||
]
|
|
||||||
|
|
||||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
||||||
return json.loads(result.stdout)
|
|
||||||
|
|
||||||
# 使用
|
|
||||||
result = transcribe_audio('podcast.mp3', 'base')
|
|
||||||
print(f"识别到 {result['total_segments']} 个片段")
|
|
||||||
```
|
|
||||||
|
|
||||||
### 集成到 Shell 脚本
|
|
||||||
|
|
||||||
```bash
|
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
transcribe() {
|
|
||||||
local file=$1
|
|
||||||
local model=${2:-base}
|
|
||||||
|
|
||||||
echo "转录: $file"
|
|
||||||
python3 whisper.py --input "$file" --model "$model" --background
|
|
||||||
}
|
|
||||||
|
|
||||||
# 批量处理
|
|
||||||
transcribe "video1.mp4"
|
|
||||||
transcribe "video2.mp4" "tiny"
|
|
||||||
```
|
|
||||||
@@ -1,403 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
KB Whisper - 本地语音识别工具
|
|
||||||
基于 Whisper.cpp 的音视频转文字
|
|
||||||
|
|
||||||
功能:
|
|
||||||
- 自动将视频/音频转换为 16kHz WAV 格式
|
|
||||||
- 支持指定语言(默认中文)
|
|
||||||
- 输出标准 JSON 格式供 KB Workflow 使用
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
|
|
||||||
|
|
||||||
def parse_arguments():
|
|
||||||
"""解析命令行参数"""
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description='KB Whisper - 本地语音识别工具',
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog="""
|
|
||||||
示例:
|
|
||||||
python whisper.py --input video.mp4
|
|
||||||
python whisper.py --input video.mp4 --model base
|
|
||||||
python whisper.py --input video.mp4 --output ./output --background
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
'--input',
|
|
||||||
required=True,
|
|
||||||
help='输入音视频文件路径'
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
'--model',
|
|
||||||
default='base',
|
|
||||||
choices=['tiny', 'base', 'small', 'medium', 'large'],
|
|
||||||
help='Whisper 模型类型 (默认: base)'
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
'--output',
|
|
||||||
help='输出目录 (默认: 输入文件所在目录)'
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
'--background',
|
|
||||||
action='store_true',
|
|
||||||
help='后台异步执行'
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
'--language',
|
|
||||||
default='zh',
|
|
||||||
help='指定语言 (默认: zh),例如: en, zh, ja'
|
|
||||||
)
|
|
||||||
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def get_whisper_cpp_path():
|
|
||||||
"""获取 whisper.cpp 路径"""
|
|
||||||
# 优先从环境变量获取
|
|
||||||
path = os.environ.get('WHISPER_CPP_PATH')
|
|
||||||
if path:
|
|
||||||
path = os.path.expanduser(path)
|
|
||||||
if os.path.exists(path):
|
|
||||||
return path
|
|
||||||
|
|
||||||
# 尝试常见位置
|
|
||||||
common_paths = [
|
|
||||||
os.path.expanduser('~/whisper.cpp'),
|
|
||||||
'/usr/local/bin/whisper',
|
|
||||||
'/opt/whisper.cpp',
|
|
||||||
]
|
|
||||||
|
|
||||||
for cp in common_paths:
|
|
||||||
if os.path.exists(cp):
|
|
||||||
return cp
|
|
||||||
|
|
||||||
raise FileNotFoundError(
|
|
||||||
"未找到 whisper.cpp。请设置环境变量 WHISPER_CPP_PATH "
|
|
||||||
"或安装到 ~/whisper.cpp\n"
|
|
||||||
"安装方法: git clone https://github.com/ggerganov/whisper.cpp.git ~/whisper.cpp && cd ~/whisper.cpp && make"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def check_model(whisper_path, model):
|
|
||||||
"""检查模型是否存在"""
|
|
||||||
model_path = os.path.join(whisper_path, 'models', f'ggml-{model}.bin')
|
|
||||||
if not os.path.exists(model_path):
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"模型文件不存在: {model_path}\n"
|
|
||||||
f"请运行以下命令下载模型:\n"
|
|
||||||
f"cd {whisper_path} && ./models/download-ggml-model.sh {model}"
|
|
||||||
)
|
|
||||||
return model_path
|
|
||||||
|
|
||||||
|
|
||||||
def convert_to_wav(input_path, output_dir):
|
|
||||||
"""将音视频转换为 16kHz WAV 格式
|
|
||||||
|
|
||||||
Whisper 需要 WAV 格式的输入文件。
|
|
||||||
"""
|
|
||||||
input_path = os.path.abspath(input_path)
|
|
||||||
|
|
||||||
# 如果已经是 WAV 且采样率正确,直接返回
|
|
||||||
if input_path.lower().endswith('.wav'):
|
|
||||||
try:
|
|
||||||
result = subprocess.run(
|
|
||||||
['ffprobe', '-v', 'error', '-show_entries',
|
|
||||||
'stream=sample_rate', '-of', 'default=noprint_wrappers=1:nokey=1',
|
|
||||||
input_path],
|
|
||||||
capture_output=True, text=True
|
|
||||||
)
|
|
||||||
if result.returncode == 0 and int(result.stdout.strip()) == 16000:
|
|
||||||
print(f"输入文件已是 16kHz WAV 格式,直接使用")
|
|
||||||
return input_path
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# 转换
|
|
||||||
output_path = os.path.join(output_dir, f"{Path(input_path).stem}_audio.wav")
|
|
||||||
|
|
||||||
print(f"转换音频格式: {input_path} -> {output_path}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
subprocess.run([
|
|
||||||
'ffmpeg', '-y', '-i', input_path,
|
|
||||||
'-acodec', 'pcm_s16le',
|
|
||||||
'-ar', '16000',
|
|
||||||
'-ac', '1',
|
|
||||||
'-loglevel', 'error',
|
|
||||||
output_path
|
|
||||||
], check=True, capture_output=True)
|
|
||||||
print("音频转换完成")
|
|
||||||
return output_path
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print(f"音频转换失败: {e}")
|
|
||||||
raise RuntimeError(f"无法转换音频格式: {e}")
|
|
||||||
except FileNotFoundError:
|
|
||||||
raise RuntimeError("未找到 ffmpeg,请安装: sudo apt-get install ffmpeg")
|
|
||||||
|
|
||||||
|
|
||||||
def run_whisper(whisper_path, audio_path, model, output_dir, language='zh'):
|
|
||||||
"""运行 Whisper 语音识别
|
|
||||||
|
|
||||||
Args:
|
|
||||||
whisper_path: whisper.cpp 目录
|
|
||||||
audio_path: WAV 音频文件路径
|
|
||||||
model: 模型名称
|
|
||||||
output_dir: 输出目录
|
|
||||||
language: 语言代码
|
|
||||||
"""
|
|
||||||
# 输出文件名(使用原始输入文件名,不是 audio.wav)
|
|
||||||
# 从 output_dir 和 audio_path 推断原始文件名
|
|
||||||
original_stem = Path(audio_path).stem.replace('_audio', '')
|
|
||||||
output_basename = original_stem
|
|
||||||
|
|
||||||
# 构建命令
|
|
||||||
cmd = [
|
|
||||||
os.path.join(whisper_path, 'whisper-cli'),
|
|
||||||
'-m', os.path.join(whisper_path, 'models', f'ggml-{model}.bin'),
|
|
||||||
'-f', audio_path,
|
|
||||||
'-of', os.path.join(output_dir, output_basename),
|
|
||||||
'-l', language,
|
|
||||||
'--output-txt',
|
|
||||||
'--output-srt',
|
|
||||||
'--output-json',
|
|
||||||
]
|
|
||||||
|
|
||||||
print(f"执行命令: {' '.join(cmd)}")
|
|
||||||
|
|
||||||
# 运行
|
|
||||||
result = subprocess.run(
|
|
||||||
cmd,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
cwd=whisper_path,
|
|
||||||
timeout=3600 # 1小时超时
|
|
||||||
)
|
|
||||||
|
|
||||||
if result.returncode != 0:
|
|
||||||
# 检查是否有警告但仍成功
|
|
||||||
if "error" in result.stderr.lower() and not os.path.exists(
|
|
||||||
os.path.join(output_dir, f"{output_basename}.txt")
|
|
||||||
):
|
|
||||||
raise RuntimeError(
|
|
||||||
f"Whisper 执行失败:\n"
|
|
||||||
f"标准输出: {result.stdout}\n"
|
|
||||||
f"错误输出: {result.stderr}"
|
|
||||||
)
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def parse_srt_for_duration(srt_file):
|
|
||||||
"""从 SRT 文件解析时长"""
|
|
||||||
if not os.path.exists(srt_file):
|
|
||||||
return 0
|
|
||||||
|
|
||||||
with open(srt_file, 'r', encoding='utf-8') as f:
|
|
||||||
content = f.read()
|
|
||||||
|
|
||||||
# 找到最后一个时间戳
|
|
||||||
pattern = r'(\d{2}:\d{2}:\d{2}),\d{3} --> \d{2}:\d{2}:\d{2},\d{3}'
|
|
||||||
matches = re.findall(pattern, content)
|
|
||||||
|
|
||||||
if not matches:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
# 解析最后一个时间戳
|
|
||||||
last_time = matches[-1]
|
|
||||||
h, m, s = last_time.split(':')
|
|
||||||
return int(h) * 3600 + int(m) * 60 + int(s)
|
|
||||||
|
|
||||||
|
|
||||||
def create_workflow_json(input_file, output_dir, model, language, duration):
|
|
||||||
"""创建标准格式的 JSON 输出,供 KB Workflow 使用"""
|
|
||||||
input_path = Path(input_file)
|
|
||||||
basename = input_path.stem
|
|
||||||
|
|
||||||
txt_file = os.path.join(output_dir, f"{basename}.txt")
|
|
||||||
srt_file = os.path.join(output_dir, f"{basename}.srt")
|
|
||||||
json_file = os.path.join(output_dir, f"{basename}.json")
|
|
||||||
|
|
||||||
# 读取文本内容
|
|
||||||
text_content = ""
|
|
||||||
if os.path.exists(txt_file):
|
|
||||||
with open(txt_file, 'r', encoding='utf-8') as f:
|
|
||||||
text_content = f.read().strip()
|
|
||||||
|
|
||||||
# 解析 SRT 获取分段
|
|
||||||
segments = []
|
|
||||||
if os.path.exists(srt_file):
|
|
||||||
with open(srt_file, 'r', encoding='utf-8') as f:
|
|
||||||
content = f.read()
|
|
||||||
|
|
||||||
# SRT 格式: 序号 -> 时间 -> 文本
|
|
||||||
pattern = r'(\d+)\n(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})\n(.+?)(?=\n\n|\Z)'
|
|
||||||
matches = re.findall(pattern, content, re.DOTALL)
|
|
||||||
|
|
||||||
def parse_time(t):
|
|
||||||
h, m, s_ms = t.split(':')
|
|
||||||
s, ms = s_ms.split(',')
|
|
||||||
return float(h) * 3600 + float(m) * 60 + float(s) + float(ms) / 1000
|
|
||||||
|
|
||||||
for idx, (num, start, end, text) in enumerate(matches):
|
|
||||||
segments.append({
|
|
||||||
"id": idx,
|
|
||||||
"start": round(parse_time(start), 2),
|
|
||||||
"end": round(parse_time(end), 2),
|
|
||||||
"text": text.strip()
|
|
||||||
})
|
|
||||||
|
|
||||||
# 构建 JSON
|
|
||||||
workflow_json = {
|
|
||||||
"success": True,
|
|
||||||
"tool": "whisper",
|
|
||||||
"model": model,
|
|
||||||
"language": language,
|
|
||||||
"input_file": str(input_path),
|
|
||||||
"output_files": {
|
|
||||||
"txt": f"{basename}.txt",
|
|
||||||
"srt": f"{basename}.srt",
|
|
||||||
"json": f"{basename}.json"
|
|
||||||
},
|
|
||||||
"duration_seconds": duration,
|
|
||||||
"segments": segments,
|
|
||||||
"text": text_content
|
|
||||||
}
|
|
||||||
|
|
||||||
# 写入 JSON
|
|
||||||
with open(json_file, 'w', encoding='utf-8') as f:
|
|
||||||
json.dump(workflow_json, f, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
return workflow_json
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""主函数"""
|
|
||||||
args = parse_arguments()
|
|
||||||
|
|
||||||
# 检查输入文件
|
|
||||||
input_file = os.path.abspath(args.input)
|
|
||||||
if not os.path.exists(input_file):
|
|
||||||
print(f"错误: 输入文件不存在: {input_file}", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# 获取输出目录
|
|
||||||
if args.output:
|
|
||||||
output_dir = os.path.abspath(args.output)
|
|
||||||
else:
|
|
||||||
output_dir = os.path.dirname(input_file)
|
|
||||||
|
|
||||||
os.makedirs(output_dir, exist_ok=True)
|
|
||||||
|
|
||||||
# 获取 whisper.cpp 路径
|
|
||||||
try:
|
|
||||||
whisper_path = get_whisper_cpp_path()
|
|
||||||
print(f"✓ Whisper.cpp 路径: {whisper_path}")
|
|
||||||
except FileNotFoundError as e:
|
|
||||||
print(f"✗ 错误: {e}", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# 检查模型
|
|
||||||
try:
|
|
||||||
check_model(whisper_path, args.model)
|
|
||||||
print(f"✓ 使用模型: {args.model}")
|
|
||||||
except FileNotFoundError as e:
|
|
||||||
print(f"✗ 错误: {e}", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# 后台执行
|
|
||||||
if args.background:
|
|
||||||
print("启动后台处理...")
|
|
||||||
log_file = os.path.join(output_dir, f"{Path(input_file).stem}_whisper.log")
|
|
||||||
|
|
||||||
with open(log_file, 'w') as log:
|
|
||||||
subprocess.Popen(
|
|
||||||
[sys.executable, __file__,
|
|
||||||
'--input', input_file,
|
|
||||||
'--model', args.model,
|
|
||||||
'--output', output_dir,
|
|
||||||
'--language', args.language],
|
|
||||||
stdout=log,
|
|
||||||
stderr=log,
|
|
||||||
start_new_session=True
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"✓ 后台任务已启动")
|
|
||||||
print(f" 日志: {log_file}")
|
|
||||||
print(f" 输出: {output_dir}")
|
|
||||||
|
|
||||||
result = {
|
|
||||||
"success": True,
|
|
||||||
"background": True,
|
|
||||||
"status": "running",
|
|
||||||
"log_file": log_file,
|
|
||||||
"output_dir": output_dir
|
|
||||||
}
|
|
||||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
||||||
return
|
|
||||||
|
|
||||||
# 前台执行
|
|
||||||
print(f"\n📁 处理文件: {input_file}")
|
|
||||||
print(f"📁 输出目录: {output_dir}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 1. 转换为 WAV
|
|
||||||
print(f"\n🔄 步骤1: 转换音频格式...")
|
|
||||||
audio_path = convert_to_wav(input_file, output_dir)
|
|
||||||
|
|
||||||
# 2. 运行 Whisper
|
|
||||||
print(f"\n🎙️ 步骤2: 语音识别 (语言: {args.language})...")
|
|
||||||
run_whisper(whisper_path, audio_path, args.model, output_dir, args.language)
|
|
||||||
|
|
||||||
# 3. 清理临时 WAV 文件
|
|
||||||
if audio_path != input_file and audio_path.endswith('_audio.wav'):
|
|
||||||
try:
|
|
||||||
os.remove(audio_path)
|
|
||||||
print("✓ 清理临时文件")
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# 4. 获取时长并创建 JSON
|
|
||||||
print(f"\n📝 步骤3: 生成输出文件...")
|
|
||||||
basename = Path(input_file).stem
|
|
||||||
srt_file = os.path.join(output_dir, f"{basename}.srt")
|
|
||||||
duration = parse_srt_for_duration(srt_file)
|
|
||||||
|
|
||||||
workflow_json = create_workflow_json(
|
|
||||||
input_file, output_dir, args.model, args.language, duration
|
|
||||||
)
|
|
||||||
|
|
||||||
# 输出结果
|
|
||||||
print(f"\n✅ 处理完成!")
|
|
||||||
print(f" 时长: {duration} 秒")
|
|
||||||
print(f" 分段数: {len(workflow_json['segments'])}")
|
|
||||||
print(f"\n📄 输出文件:")
|
|
||||||
for fmt in ['txt', 'srt', 'json']:
|
|
||||||
fpath = os.path.join(output_dir, f"{basename}.{fmt}")
|
|
||||||
if os.path.exists(fpath):
|
|
||||||
size = os.path.getsize(fpath)
|
|
||||||
print(f" • {fmt.upper()}: {fpath} ({size:,} bytes)")
|
|
||||||
|
|
||||||
print(f"\n📋 Workflow JSON:")
|
|
||||||
print(json.dumps({
|
|
||||||
"success": workflow_json["success"],
|
|
||||||
"tool": "whisper",
|
|
||||||
"text_length": len(workflow_json["text"]),
|
|
||||||
"segments_count": len(workflow_json["segments"])
|
|
||||||
}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"\n✗ 错误: {e}", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,247 +0,0 @@
|
|||||||
# KB Workflow 环境安装脚本
|
|
||||||
|
|
||||||
一键检查并安装 kb-workflow 知识库工作流所需的全部环境依赖。
|
|
||||||
|
|
||||||
## 功能特性
|
|
||||||
|
|
||||||
- ✅ 自动检测系统类型 (Ubuntu/Debian/Fedora/Arch)
|
|
||||||
- ✅ 检查系统依赖 (Python3, pip3, ffmpeg, tesseract, git)
|
|
||||||
- ✅ 检查 Python 依赖包 (httpx, aiohttp, playwright, easyocr 等)
|
|
||||||
- ✅ 自动创建知识库目录结构
|
|
||||||
- ✅ 生成环境变量配置示例
|
|
||||||
- ✅ 汇总显示所有需要执行的安装命令
|
|
||||||
|
|
||||||
## 使用方法
|
|
||||||
|
|
||||||
### 运行检查脚本
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd /home/starmen1265/.openclaw/workspace/skills
|
|
||||||
./install-kb-workflow.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
脚本会检查所有依赖,并显示需要执行的安装命令。
|
|
||||||
|
|
||||||
### 示例输出
|
|
||||||
|
|
||||||
```
|
|
||||||
========================================
|
|
||||||
第一步: 检查系统依赖
|
|
||||||
========================================
|
|
||||||
[SUCCESS] Python3 已安装: Python 3.12.3
|
|
||||||
[WARN] pip3 未安装
|
|
||||||
[WARN] ffmpeg 未安装
|
|
||||||
...
|
|
||||||
|
|
||||||
========================================
|
|
||||||
需要安装的系统依赖
|
|
||||||
========================================
|
|
||||||
|
|
||||||
检测到以下依赖未安装:
|
|
||||||
- python3-pip
|
|
||||||
- ffmpeg
|
|
||||||
- tesseract-ocr
|
|
||||||
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
请运行以下命令安装:
|
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
||||||
|
|
||||||
sudo apt-get update
|
|
||||||
sudo apt-get install -y python3-pip ffmpeg tesseract-ocr tesseract-ocr-chi-sim tesseract-ocr-eng
|
|
||||||
```
|
|
||||||
|
|
||||||
## 依赖清单
|
|
||||||
|
|
||||||
### 系统依赖
|
|
||||||
|
|
||||||
| 依赖 | 用途 | 安装命令 |
|
|
||||||
|------|------|----------|
|
|
||||||
| **python3** | Python 运行时 | `sudo apt-get install python3` |
|
|
||||||
| **python3-pip** | Python 包管理器 | `sudo apt-get install python3-pip` |
|
|
||||||
| **ffmpeg** | 视频处理(OCR 截帧) | `sudo apt-get install ffmpeg` |
|
|
||||||
| **tesseract-ocr** | OCR 引擎(备用方案) | `sudo apt-get install tesseract-ocr` |
|
|
||||||
| **git** | 版本控制(归档提交) | `sudo apt-get install git` |
|
|
||||||
|
|
||||||
### Tesseract 语言包
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Ubuntu/Debian
|
|
||||||
sudo apt-get install tesseract-ocr-chi-sim tesseract-ocr-chi-tra tesseract-ocr-eng
|
|
||||||
|
|
||||||
# Fedora/RHEL
|
|
||||||
sudo dnf install tesseract-langpack-chi_sim tesseract-langpack-chi_tra tesseract-langpack-eng
|
|
||||||
```
|
|
||||||
|
|
||||||
### Python 依赖
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 基础 HTTP 库
|
|
||||||
pip3 install --user httpx aiohttp requests
|
|
||||||
|
|
||||||
# 图像和 OCR
|
|
||||||
pip3 install --user Pillow easyocr pytesseract
|
|
||||||
|
|
||||||
# 浏览器自动化
|
|
||||||
pip3 install --user playwright
|
|
||||||
python3 -m playwright install chromium
|
|
||||||
```
|
|
||||||
|
|
||||||
## 目录结构
|
|
||||||
|
|
||||||
安装完成后会创建以下目录结构:
|
|
||||||
|
|
||||||
```
|
|
||||||
~/work/knowledge-base/ # 知识库根目录(可配置)
|
|
||||||
├── archive/ # 归档目录
|
|
||||||
│ ├── pending/ # 待处理
|
|
||||||
│ └── subscriptions/ # 订阅信息
|
|
||||||
├── metadata/ # 元数据文件
|
|
||||||
├── content/ # 内容文件
|
|
||||||
│ ├── downloads/ # 视频下载
|
|
||||||
│ └── subtitles/ # 字幕文件
|
|
||||||
└── analyzed/ # 深度分析报告
|
|
||||||
```
|
|
||||||
|
|
||||||
## 环境变量
|
|
||||||
|
|
||||||
### 必需
|
|
||||||
- `KB_BASE_PATH` - 知识库根目录(默认: `~/work/knowledge-base`)
|
|
||||||
|
|
||||||
### 可选
|
|
||||||
- `BILIBILI_COOKIE` - B站 Cookie,用于下载会员视频
|
|
||||||
- `AI_API_KEY` - AI API 密钥,用于深度内容分析
|
|
||||||
- `AI_PROVIDER` - AI 提供商(openai/zhipu/minimax)
|
|
||||||
- `AI_MODEL` - AI 模型名称(如 gpt-4)
|
|
||||||
- `AI_BASE_URL` - 自定义 API 端点
|
|
||||||
- `KB_GIT_USER` - Git 用户名(用于归档提交)
|
|
||||||
- `KB_GIT_EMAIL` - Git 邮箱(用于归档提交)
|
|
||||||
- `KB_GIT_REMOTE` - Git 远程仓库地址
|
|
||||||
|
|
||||||
### 配置方法
|
|
||||||
|
|
||||||
将以下内容添加到 `~/.bashrc` 或 `~/.zshrc`:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# KB Workflow 配置
|
|
||||||
export KB_BASE_PATH="~/work/knowledge-base"
|
|
||||||
|
|
||||||
# 可选:B站 Cookie
|
|
||||||
export BILIBILI_COOKIE="SESSDATA=xxx; bili_jct=xxx"
|
|
||||||
|
|
||||||
# 可选:AI 配置
|
|
||||||
export AI_API_KEY="your-api-key"
|
|
||||||
export AI_PROVIDER="openai"
|
|
||||||
export AI_MODEL="gpt-4"
|
|
||||||
```
|
|
||||||
|
|
||||||
然后重新加载配置:
|
|
||||||
```bash
|
|
||||||
source ~/.bashrc # 或 source ~/.zshrc
|
|
||||||
```
|
|
||||||
|
|
||||||
## 故障排除
|
|
||||||
|
|
||||||
### 1. pip3 命令不存在
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 方法 1: 安装 pip3
|
|
||||||
sudo apt-get install python3-pip
|
|
||||||
|
|
||||||
# 方法 2: 使用 python3 -m pip
|
|
||||||
python3 -m pip install --user <package>
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Playwright 浏览器安装失败
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 手动安装 Chromium
|
|
||||||
python3 -m playwright install chromium
|
|
||||||
|
|
||||||
# 如果仍然失败,尝试安装依赖
|
|
||||||
sudo apt-get install libnss3 libatk-bridge2.0-0 libxss1 libgtk-3-0
|
|
||||||
```
|
|
||||||
|
|
||||||
### 3. EasyOCR 安装失败
|
|
||||||
|
|
||||||
EasyOCR 依赖 PyTorch,可能需要较长时间安装:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 先安装 PyTorch
|
|
||||||
pip3 install --user torch torchvision --index-url https://download.pytorch.org/whl/cpu
|
|
||||||
|
|
||||||
# 再安装 easyocr
|
|
||||||
pip3 install --user easyocr
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. 权限问题
|
|
||||||
|
|
||||||
如果使用 `--user` 安装的包无法找到,检查 PATH:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 查看用户安装路径
|
|
||||||
python3 -m site --user-base
|
|
||||||
|
|
||||||
# 添加到 PATH(添加到 ~/.bashrc)
|
|
||||||
export PATH="$PATH:$(python3 -m site --user-base)/bin"
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. OCR 中文识别问题
|
|
||||||
|
|
||||||
确保已安装中文语言包:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 检查已安装的语言包
|
|
||||||
tesseract --list-langs
|
|
||||||
|
|
||||||
# 应该能看到 chi_sim 和 chi_tra
|
|
||||||
```
|
|
||||||
|
|
||||||
## 更新
|
|
||||||
|
|
||||||
如果技能更新了依赖,可以重新运行检查脚本:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
./install-kb-workflow.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
## 子技能清单
|
|
||||||
|
|
||||||
| 技能 | 路径 | 功能 |
|
|
||||||
|------|------|------|
|
|
||||||
| kb-workflow | `kb-workflow/kb_workflow.py` | 主流程协调 |
|
|
||||||
| kb-link-parser | `kb-link-parser/link-parser.py` | 链接解析 |
|
|
||||||
| kb-bilibili-metadata | `kb-bilibili-metadata/bilibili_metadata.py` | B站元数据提取 |
|
|
||||||
| kb-bilibili-content | `kb-bilibili-content/bilibili_content.py` | B站内容下载 |
|
|
||||||
| kb-ocr-processor | `kb-ocr-processor/ocr_processor.py` | 视频 OCR 识别 |
|
|
||||||
| kb-deep-analyzer | `kb-deep-analyzer/deep_analyzer.py` | AI 深度分析 |
|
|
||||||
|
|
||||||
## 使用 kb-workflow
|
|
||||||
|
|
||||||
安装完成后,可以通过以下方式使用:
|
|
||||||
|
|
||||||
### 方式 1:通过主代理
|
|
||||||
|
|
||||||
直接向主代理发送消息:
|
|
||||||
```
|
|
||||||
添加到知识库 https://www.bilibili.com/video/BVxxxxx
|
|
||||||
```
|
|
||||||
|
|
||||||
### 方式 2:手动运行子技能
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. 链接解析
|
|
||||||
python3 kb-link-parser/link-parser.py --url "https://www.bilibili.com/video/BVxxxxx"
|
|
||||||
|
|
||||||
# 2. 元数据提取
|
|
||||||
python3 kb-bilibili-metadata/bilibili_metadata.py --id "BVxxxxx" --type video
|
|
||||||
|
|
||||||
# 3. 内容下载
|
|
||||||
python3 kb-bilibili-content/bilibili_content.py --id "BVxxxxx" --type subtitle-download
|
|
||||||
|
|
||||||
# 4. 深度分析
|
|
||||||
python3 kb-deep-analyzer/deep_analyzer.py --metadata meta.json --content content.json --output analysis.md
|
|
||||||
```
|
|
||||||
|
|
||||||
## 许可证
|
|
||||||
|
|
||||||
MIT License
|
|
||||||
@@ -60,7 +60,7 @@ echo "✅ 环境变量检查通过"
|
|||||||
|
|
||||||
## Workflow 执行方式
|
## Workflow 执行方式
|
||||||
|
|
||||||
### 方式一:一键处理(推荐)
|
### 一键处理(推荐)
|
||||||
|
|
||||||
使用管道 CLI 一键完成所有步骤:
|
使用管道 CLI 一键完成所有步骤:
|
||||||
|
|
||||||
@@ -85,21 +85,6 @@ python3 kb-workflow/kb_workflow.py process "https://..." -o /path/to/output
|
|||||||
python3 kb-workflow/kb_workflow.py process "https://..." --no-skip
|
python3 kb-workflow/kb_workflow.py process "https://..." --no-skip
|
||||||
```
|
```
|
||||||
|
|
||||||
### 方式二:手动分步执行(高级)
|
|
||||||
|
|
||||||
如需更细粒度控制,可手动调用各子技能:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. 查重
|
|
||||||
python3 kb-workflow/kb_workflow.py --check --url "https://..."
|
|
||||||
|
|
||||||
# 2. 一键处理
|
|
||||||
python3 kb-workflow/kb_workflow.py process "https://..."
|
|
||||||
|
|
||||||
# 3. 归档
|
|
||||||
python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md"
|
|
||||||
```
|
|
||||||
|
|
||||||
## 支持的平台和内容类型
|
## 支持的平台和内容类型
|
||||||
|
|
||||||
| 平台 | 内容类型 | URL 示例 |
|
| 平台 | 内容类型 | URL 示例 |
|
||||||
@@ -152,22 +137,6 @@ python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md"
|
|||||||
3. **禁止降级处理**:不能使用预设模板或简单摘要来"假装成功"
|
3. **禁止降级处理**:不能使用预设模板或简单摘要来"假装成功"
|
||||||
4. **每次执行都要使用 AI**:必须有真实的 AI API 调用记录
|
4. **每次执行都要使用 AI**:必须有真实的 AI API 调用记录
|
||||||
|
|
||||||
## 辅助命令
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 查重
|
|
||||||
python3 kb-workflow/kb_workflow.py --check --url "https://..."
|
|
||||||
|
|
||||||
# 获取目录路径
|
|
||||||
python3 kb-workflow/kb_workflow.py --base-path
|
|
||||||
python3 kb-workflow/kb_workflow.py --metadata
|
|
||||||
python3 kb-workflow/kb_workflow.py --content
|
|
||||||
python3 kb-workflow/kb_workflow.py --analyzed
|
|
||||||
|
|
||||||
# 归档分析报告
|
|
||||||
python3 kb-workflow/kb_workflow.py --archive-file "/path/to/report.md"
|
|
||||||
```
|
|
||||||
|
|
||||||
## 示例
|
## 示例
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -1,101 +0,0 @@
|
|||||||
# SkillBug.md
|
|
||||||
|
|
||||||
## 问题记录
|
|
||||||
|
|
||||||
### 2026-02-10: 子代理权限问题
|
|
||||||
|
|
||||||
**问题:** `sessions_spawn` 调用子代理时返回 `forbidden` 错误。
|
|
||||||
|
|
||||||
**原因:** OpenClaw 配置中 allowlist 为空,Python 脚本没有权限调用子代理。
|
|
||||||
|
|
||||||
**解决方案:** 改用 Prompt-based 模式(方案B)。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 架构重构
|
|
||||||
|
|
||||||
### 之前(Script-based)
|
|
||||||
|
|
||||||
```
|
|
||||||
kb_workflow.py (Python脚本)
|
|
||||||
↓
|
|
||||||
sessions_spawn() # ❌ 失败(脚本没有权限)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 之后(Prompt-based)
|
|
||||||
|
|
||||||
```
|
|
||||||
主代理读取 SKILL.md
|
|
||||||
↓
|
|
||||||
按照指导调用 sessions_spawn # ✅ 成功(主代理有权限)
|
|
||||||
```
|
|
||||||
|
|
||||||
### 文件变更
|
|
||||||
|
|
||||||
| 文件 | 变更 | 说明 |
|
|
||||||
|------|------|------|
|
|
||||||
| `SKILL.md` | 重写 | 详细定义 workflow 步骤,主代理直接读取执行 |
|
|
||||||
| `kb_workflow.py` | 简化 | 保留为辅助工具函数库(查重、归档、Git) |
|
|
||||||
| `skill_main.py` | 简化 | 只做触发检测,返回指导信息 |
|
|
||||||
| `SkillBug.md` | 更新 | 记录架构变更 |
|
|
||||||
|
|
||||||
### SKILL.md 结构
|
|
||||||
|
|
||||||
```
|
|
||||||
1. 触发条件
|
|
||||||
2. 步骤 1: 查重
|
|
||||||
3. 步骤 2: kb-link-parser
|
|
||||||
4. 步骤 3: 元数据技能
|
|
||||||
5. 步骤 4: 内容下载
|
|
||||||
6. 步骤 5: OCR(30分钟超时)
|
|
||||||
7. 步骤 6: 深度分析
|
|
||||||
8. 步骤 7: 归档和 Git
|
|
||||||
9. 步骤 8: 汇报完成
|
|
||||||
10. 目录结构
|
|
||||||
11. 支持平台
|
|
||||||
12. 注意事项
|
|
||||||
```
|
|
||||||
|
|
||||||
### kb_workflow.py 保留的函数
|
|
||||||
|
|
||||||
```python
|
|
||||||
extract_platform(filename) # 从文件名提取平台
|
|
||||||
extract_base_name(filename) # 从文件名提取基础名称
|
|
||||||
get_knowledge_base_path() # 获取知识库根目录
|
|
||||||
ensure_directories(base_path) # 确保目录存在
|
|
||||||
check_duplicate(url) # 查重
|
|
||||||
archive_and_git(analyzed_file) # 归档和 Git
|
|
||||||
generate_summary(...) # 生成汇报
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### 2026-02-12: B站视频下载失败 - 403 Forbidden
|
|
||||||
|
|
||||||
**问题:** 使用 kb-bilibili-content 下载视频时返回 403 Forbidden 错误。
|
|
||||||
|
|
||||||
**原因:**
|
|
||||||
1. 视频可能需要B站会员权限
|
|
||||||
2. 或者需要登录cookie才能下载
|
|
||||||
3. 当前未配置 `BILIBILI_COOKIE` 环境变量
|
|
||||||
|
|
||||||
**复现场景:**
|
|
||||||
- 链接:https://www.bilibili.com/video/BV1Z9nXzUEhv
|
|
||||||
- 下载命令:`python3 bilibili_content.py --id "BV1Z9nXzUEhv" --type video`
|
|
||||||
- 结果:返回403错误页面,文件只有548字节
|
|
||||||
|
|
||||||
**影响:** 无法进行后续的OCR处理和深度分析
|
|
||||||
|
|
||||||
**建议解决方案:**
|
|
||||||
1. 配置 `BILIBILI_COOKIE` 环境变量
|
|
||||||
2. 或提供cookies文件路径
|
|
||||||
3. 或跳过OCR步骤,使用元数据+简介进行深度分析
|
|
||||||
|
|
||||||
**当前进度:**
|
|
||||||
- ✅ 步骤1:环境变量检查完成
|
|
||||||
- ✅ 步骤2:链接解析成功(bilibili_BV1Z9nXzUEhv)
|
|
||||||
- ✅ 步骤3:元数据提取成功(标题:新加坡分区漫游指南...)
|
|
||||||
- ❌ 步骤4:视频下载失败(403)
|
|
||||||
- ⏸️ 步骤5:OCR处理(阻塞)
|
|
||||||
- ⏸️ 步骤6:深度分析(阻塞)
|
|
||||||
- ⏸️ 步骤7:归档(阻塞)
|
|
||||||
@@ -3,330 +3,24 @@
|
|||||||
"""
|
"""
|
||||||
KB Workflow - Knowledge Base Workflow Orchestrator
|
KB Workflow - Knowledge Base Workflow Orchestrator
|
||||||
|
|
||||||
知识库工作流辅助工具函数。
|
知识库工作流管道入口。
|
||||||
|
|
||||||
此模块提供查重、文件操作、Git 管理等工具函数,
|
此模块提供 process_command() 函数作为管道入口,
|
||||||
供主代理在执行 SKILL.md 中定义的 workflow 时调用。
|
内部使用 kb_pipeline 包实现完整的内容处理流程。
|
||||||
|
|
||||||
|
使用方式:
|
||||||
|
python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
|
||||||
|
python kb_workflow.py process "https://..." -o /path/to/output
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import json
|
|
||||||
import argparse
|
import argparse
|
||||||
import subprocess
|
import asyncio
|
||||||
import shutil
|
import json
|
||||||
import re
|
import sys
|
||||||
from datetime import datetime
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, List, Optional, Any
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
|
||||||
# ==================== 子技能映射 ====================
|
|
||||||
|
|
||||||
METADATA_AGENTS = {
|
|
||||||
"bilibili": "kb-bilibili-metadata",
|
|
||||||
"zhihu": "kb-zhihu-metadata",
|
|
||||||
}
|
|
||||||
|
|
||||||
CONTENT_AGENTS = {
|
|
||||||
"bilibili": "kb-bilibili-content",
|
|
||||||
"zhihu": "kb-zhihu-content",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ==================== 内容类型判断 ====================
|
|
||||||
|
|
||||||
# 需要下载视频/字幕的内容类型
|
|
||||||
VIDEO_CONTENT_TYPES = {"video", "zvideo"}
|
|
||||||
|
|
||||||
# 需要提取文字的内容类型
|
|
||||||
TEXT_CONTENT_TYPES = {"article", "zhuanlan", "question", "answer"}
|
|
||||||
|
|
||||||
# 需要下载图片的内容类型
|
|
||||||
IMAGE_CONTENT_TYPES = {"note"}
|
|
||||||
|
|
||||||
|
|
||||||
def is_video_content(content_type: str) -> bool:
|
|
||||||
"""判断是否为视频内容类型"""
|
|
||||||
return content_type in VIDEO_CONTENT_TYPES
|
|
||||||
|
|
||||||
|
|
||||||
def is_text_content(content_type: str) -> bool:
|
|
||||||
"""判断是否为文字内容类型(专栏、文章、问答等)"""
|
|
||||||
return content_type in TEXT_CONTENT_TYPES
|
|
||||||
|
|
||||||
|
|
||||||
def is_image_content(content_type: str) -> bool:
|
|
||||||
"""判断是否为图片内容类型(小红书等)"""
|
|
||||||
return content_type in IMAGE_CONTENT_TYPES
|
|
||||||
|
|
||||||
|
|
||||||
def get_content_handler(content_type: str) -> str:
|
|
||||||
"""
|
|
||||||
获取内容处理方式
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
"video" - 需要下载视频/字幕
|
|
||||||
"text" - 需要提取文字
|
|
||||||
"image" - 需要下载图片
|
|
||||||
"unknown" - 未知类型
|
|
||||||
"""
|
|
||||||
if is_video_content(content_type):
|
|
||||||
return "video"
|
|
||||||
elif is_text_content(content_type):
|
|
||||||
return "text"
|
|
||||||
elif is_image_content(content_type):
|
|
||||||
return "image"
|
|
||||||
else:
|
|
||||||
return "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
# ==================== 工具函数 ====================
|
|
||||||
|
|
||||||
def extract_platform(filename: str) -> str:
|
|
||||||
"""从文件名提取平台(第一个下划线之前)"""
|
|
||||||
match = re.match(r'^([a-zA-Z0-9]+)_(.+)$', filename)
|
|
||||||
if match:
|
|
||||||
return match.group(1)
|
|
||||||
return "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
def extract_base_name(filename: str) -> str:
|
|
||||||
"""从文件名提取基础名称(去掉扩展名)"""
|
|
||||||
return Path(filename).stem
|
|
||||||
|
|
||||||
|
|
||||||
def get_knowledge_base_path() -> Path:
|
|
||||||
"""获取知识库根目录"""
|
|
||||||
base_path_str = os.environ.get('KB_BASE_PATH', '')
|
|
||||||
if base_path_str:
|
|
||||||
return Path(base_path_str).expanduser()
|
|
||||||
return Path.home() / "work" / "knowledge-base"
|
|
||||||
|
|
||||||
|
|
||||||
def get_archive_path() -> Path:
|
|
||||||
"""获取归档目录(AI分析文档存放位置)"""
|
|
||||||
archive_path_str = os.environ.get('KB_ARCHIVE_PATH', '')
|
|
||||||
if archive_path_str:
|
|
||||||
return Path(archive_path_str).expanduser()
|
|
||||||
# 默认:~/work/zq/30.资源/AI分析
|
|
||||||
return Path.home() / "work" / "zq" / "30.资源" / "AI分析"
|
|
||||||
|
|
||||||
|
|
||||||
def get_git_root() -> Path:
|
|
||||||
"""获取 Git 仓库根目录(用于归档提交)"""
|
|
||||||
git_root_str = os.environ.get('KB_GIT_ROOT', '')
|
|
||||||
if git_root_str:
|
|
||||||
return Path(git_root_str).expanduser()
|
|
||||||
# 默认:~/work/zq
|
|
||||||
return Path.home() / "work" / "zq"
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_directories(base_path: Path = None) -> Dict[str, Path]:
|
|
||||||
"""确保目录存在"""
|
|
||||||
if base_path is None:
|
|
||||||
base_path = get_knowledge_base_path()
|
|
||||||
|
|
||||||
dirs = {
|
|
||||||
"base": base_path,
|
|
||||||
"pending": base_path / "pending",
|
|
||||||
"metadata": base_path / "metadata",
|
|
||||||
"content": base_path / "content",
|
|
||||||
"content_downloads": base_path / "content" / "downloads",
|
|
||||||
"analyzed": base_path / "analyzed",
|
|
||||||
"archive": base_path / "archive",
|
|
||||||
"archive_subscriptions": base_path / "archive" / "subscriptions",
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, dir_path in dirs.items():
|
|
||||||
if isinstance(dir_path, Path):
|
|
||||||
dir_path.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
return dirs
|
|
||||||
|
|
||||||
|
|
||||||
def check_duplicate(base_path: Path = None, url: str = None, platform: str = None, content_id: str = None) -> Dict[str, Any]:
|
|
||||||
"""查重检查"""
|
|
||||||
if base_path is None:
|
|
||||||
base_path = get_knowledge_base_path()
|
|
||||||
|
|
||||||
metadata_dir = base_path / "metadata"
|
|
||||||
archive_dir = base_path / "archive"
|
|
||||||
|
|
||||||
# 从URL提取ID
|
|
||||||
if url and not content_id:
|
|
||||||
bv_match = re.search(r'BV[a-zA-Z0-9]+', url)
|
|
||||||
content_id = bv_match.group() if bv_match else None
|
|
||||||
|
|
||||||
result = {"duplicate": False, "status": None, "files": []}
|
|
||||||
|
|
||||||
if metadata_dir.exists():
|
|
||||||
for f in metadata_dir.iterdir():
|
|
||||||
if f.is_file() and content_id and content_id in f.name:
|
|
||||||
result["duplicate"] = True
|
|
||||||
result["status"] = "metadata"
|
|
||||||
result["files"].append(str(f))
|
|
||||||
|
|
||||||
if archive_dir.exists():
|
|
||||||
for date_dir in archive_dir.iterdir():
|
|
||||||
if date_dir.is_dir():
|
|
||||||
for f in date_dir.iterdir():
|
|
||||||
if f.is_file() and content_id and content_id in f.name:
|
|
||||||
result["duplicate"] = True
|
|
||||||
result["status"] = "archived"
|
|
||||||
result["files"].append(str(f))
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def archive_and_git(
|
|
||||||
analyzed_file: str,
|
|
||||||
base_path: Path = None,
|
|
||||||
git_root: Path = None,
|
|
||||||
git_user: str = None,
|
|
||||||
git_email: str = None,
|
|
||||||
git_remote: str = None
|
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""归档并提交Git"""
|
|
||||||
if base_path is None:
|
|
||||||
base_path = get_knowledge_base_path()
|
|
||||||
if git_root is None:
|
|
||||||
git_root = get_git_root()
|
|
||||||
if git_user is None:
|
|
||||||
git_user = os.environ.get('KB_GIT_USER', '')
|
|
||||||
if git_email is None:
|
|
||||||
git_email = os.environ.get('KB_GIT_EMAIL', '')
|
|
||||||
if git_remote is None:
|
|
||||||
git_remote = os.environ.get('KB_GIT_REMOTE', '')
|
|
||||||
|
|
||||||
result = {
|
|
||||||
"success": False,
|
|
||||||
"archive_path": "",
|
|
||||||
"git_commit": False,
|
|
||||||
"git_push": None,
|
|
||||||
"error": None
|
|
||||||
}
|
|
||||||
|
|
||||||
try:
|
|
||||||
# 1. 检查并初始化 Git 仓库
|
|
||||||
git_dir = git_root / ".git"
|
|
||||||
if not git_dir.exists():
|
|
||||||
# 初始化仓库
|
|
||||||
subprocess.run(['git', 'init'], cwd=str(git_root), capture_output=True)
|
|
||||||
subprocess.run(['git', 'config', 'user.name', git_user], cwd=str(git_root))
|
|
||||||
subprocess.run(['git', 'config', 'user.email', git_email], cwd=str(git_root))
|
|
||||||
|
|
||||||
# 关联远端仓库
|
|
||||||
if git_remote:
|
|
||||||
subprocess.run(['git', 'remote', 'add', 'origin', git_remote], cwd=str(git_root), capture_output=True)
|
|
||||||
# 拉取远端分支
|
|
||||||
subprocess.run(['git', 'fetch', 'origin'], cwd=str(git_root), capture_output=True)
|
|
||||||
# 设置 main 分支并拉取
|
|
||||||
subprocess.run(['git', 'branch', '--set-upstream-to=origin/main', 'main'], cwd=str(git_root), capture_output=True)
|
|
||||||
subprocess.run(['git', 'pull', 'origin', 'main', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
|
|
||||||
else:
|
|
||||||
# 已有仓库,检查远端是否配置
|
|
||||||
if git_remote:
|
|
||||||
remote_check = subprocess.run(['git', 'remote', '-v'], cwd=str(git_root), capture_output=True, text=True)
|
|
||||||
if 'origin' not in remote_check.stdout:
|
|
||||||
subprocess.run(['git', 'remote', 'add', 'origin', git_remote], cwd=str(git_root), capture_output=True)
|
|
||||||
|
|
||||||
# 拉取远端最新代码
|
|
||||||
subprocess.run(['git', 'fetch', 'origin'], cwd=str(git_root), capture_output=True)
|
|
||||||
# 尝试拉取 main 分支,如果失败则尝试 master
|
|
||||||
pull_result = subprocess.run(['git', 'pull', 'origin', 'main', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
|
|
||||||
if pull_result.returncode != 0:
|
|
||||||
subprocess.run(['git', 'pull', 'origin', 'master', '--rebase'], cwd=str(git_root), capture_output=True, timeout=120)
|
|
||||||
|
|
||||||
# 2. 创建归档目录
|
|
||||||
date_str = datetime.now().strftime('%Y-%m-%d')
|
|
||||||
archive_base = get_archive_path()
|
|
||||||
date_dir = archive_base / date_str
|
|
||||||
date_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
# 3. 复制分析报告
|
|
||||||
src_file = Path(analyzed_file)
|
|
||||||
archive_file = date_dir / src_file.name
|
|
||||||
shutil.copy(src_file, archive_file)
|
|
||||||
result["archive_path"] = str(archive_file)
|
|
||||||
|
|
||||||
# 4. 复制元数据
|
|
||||||
meta_file = base_path / "metadata" / f"{src_file.stem}.meta.json"
|
|
||||||
if meta_file.exists():
|
|
||||||
shutil.copy(meta_file, date_dir / meta_file.name)
|
|
||||||
|
|
||||||
# 5. Git 提交并推送
|
|
||||||
if (git_root / ".git").exists():
|
|
||||||
# 添加所有变更
|
|
||||||
subprocess.run(['git', 'add', '.'], cwd=str(git_root), capture_output=True)
|
|
||||||
|
|
||||||
# 检查是否有变更需要提交
|
|
||||||
status_result = subprocess.run(['git', 'status', '--porcelain'], cwd=str(git_root), capture_output=True, text=True)
|
|
||||||
|
|
||||||
if status_result.stdout.strip():
|
|
||||||
commit_msg = f"Archive: {src_file.name} - {datetime.now().isoformat()}"
|
|
||||||
commit_result = subprocess.run(
|
|
||||||
['git', 'commit', '-m', commit_msg],
|
|
||||||
cwd=str(git_root), capture_output=True, text=True
|
|
||||||
)
|
|
||||||
result["git_commit"] = commit_result.returncode == 0
|
|
||||||
|
|
||||||
# 推送到远端
|
|
||||||
if git_remote and commit_result.returncode == 0:
|
|
||||||
push_result = subprocess.run(
|
|
||||||
['git', 'push', 'origin', 'main'],
|
|
||||||
cwd=str(git_root), capture_output=True, text=True, timeout=60
|
|
||||||
)
|
|
||||||
if push_result.returncode != 0:
|
|
||||||
# 如果 main 分支推送失败,尝试 master
|
|
||||||
push_result = subprocess.run(
|
|
||||||
['git', 'push', 'origin', 'master'],
|
|
||||||
cwd=str(git_root), capture_output=True, text=True, timeout=60
|
|
||||||
)
|
|
||||||
result["git_push"] = {"success": push_result.returncode == 0, "output": push_result.stdout}
|
|
||||||
|
|
||||||
result["success"] = True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result["error"] = str(e)
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def generate_summary(url: str, stages: List[Dict], final_file: str, archive_result: Dict = None) -> str:
|
|
||||||
"""生成处理完成汇报"""
|
|
||||||
# 提取视频标题(如果有)
|
|
||||||
title_match = re.search(r'com/video/([^?]+)', url)
|
|
||||||
title = title_match.group(1) if title_match else "未知"
|
|
||||||
|
|
||||||
summary = f"""
|
|
||||||
✅ **处理完成!**
|
|
||||||
|
|
||||||
**视频:** {title}
|
|
||||||
|
|
||||||
**处理阶段:**
|
|
||||||
"""
|
|
||||||
for stage in stages:
|
|
||||||
stage_name = stage.get('stage', 'unknown')
|
|
||||||
emoji = "✓" if stage.get('success') else "○"
|
|
||||||
summary += f"{emoji} {stage_name}\n"
|
|
||||||
|
|
||||||
if final_file:
|
|
||||||
summary += f"\n**最终文件:** `{final_file}`"
|
|
||||||
|
|
||||||
if archive_result:
|
|
||||||
if archive_result.get('git_commit'):
|
|
||||||
summary += f"\n**Git:** 已提交"
|
|
||||||
if archive_result.get('git_push', {}).get('success'):
|
|
||||||
summary += " ✓"
|
|
||||||
elif archive_result.get('git_push'):
|
|
||||||
summary += " ✗"
|
|
||||||
|
|
||||||
return summary
|
|
||||||
|
|
||||||
|
|
||||||
# ==================== CLI Process Command ====================
|
|
||||||
|
|
||||||
def process_command(
|
def process_command(
|
||||||
url: str,
|
url: str,
|
||||||
output_dir: Optional[str] = None,
|
output_dir: Optional[str] = None,
|
||||||
@@ -342,8 +36,6 @@ def process_command(
|
|||||||
Returns:
|
Returns:
|
||||||
Dictionary with success status, content_dir, and results
|
Dictionary with success status, content_dir, and results
|
||||||
"""
|
"""
|
||||||
import asyncio
|
|
||||||
|
|
||||||
# Add project root to path for kb_pipeline imports
|
# Add project root to path for kb_pipeline imports
|
||||||
project_root = Path(__file__).parent.parent
|
project_root = Path(__file__).parent.parent
|
||||||
if str(project_root) not in sys.path:
|
if str(project_root) not in sys.path:
|
||||||
@@ -351,7 +43,7 @@ def process_command(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
from kb_pipeline.pipeline import Pipeline
|
from kb_pipeline.pipeline import Pipeline
|
||||||
from kb_pipeline.processors import ParseProcessor, DownloadProcessor, ASRProcessor, AnalyzeProcessor
|
from kb_pipeline.processors import ASRProcessor, AnalyzeProcessor, DownloadProcessor, ParseProcessor
|
||||||
from kb_pipeline.schemas import PipelineConfig
|
from kb_pipeline.schemas import PipelineConfig
|
||||||
|
|
||||||
async def run_pipeline():
|
async def run_pipeline():
|
||||||
@@ -402,24 +94,22 @@ def process_command(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# ==================== CLI ====================
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
"""CLI入口 - 用于命令行测试"""
|
"""CLI入口 - 用于命令行调用"""
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description='KB Workflow - Knowledge Base Workflow Orchestrator',
|
description='KB Workflow - Knowledge Base Workflow Orchestrator',
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
epilog="""
|
epilog='''
|
||||||
使用示例:
|
使用示例:
|
||||||
# 查重
|
|
||||||
python kb_workflow.py --check --url "https://www.bilibili.com/video/BVxxx"
|
|
||||||
|
|
||||||
# 归档
|
|
||||||
python kb_workflow.py --archive "/path/to/analyzed.md"
|
|
||||||
|
|
||||||
# 处理URL (通过完整管道)
|
# 处理URL (通过完整管道)
|
||||||
python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
|
python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
|
||||||
"""
|
|
||||||
|
# 指定输出目录
|
||||||
|
python kb_workflow.py process "https://..." -o /path/to/output
|
||||||
|
|
||||||
|
# 强制重新执行(忽略已完成状态)
|
||||||
|
python kb_workflow.py process "https://..." --no-skip
|
||||||
|
'''
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create subparsers for commands
|
# Create subparsers for commands
|
||||||
@@ -430,29 +120,12 @@ def main():
|
|||||||
process_parser.add_argument('url', type=str, help='URL to process')
|
process_parser.add_argument('url', type=str, help='URL to process')
|
||||||
process_parser.add_argument('--output-dir', '-o', type=str, help='Output directory for content')
|
process_parser.add_argument('--output-dir', '-o', type=str, help='Output directory for content')
|
||||||
process_parser.add_argument('--skip-completed', action='store_true', default=True,
|
process_parser.add_argument('--skip-completed', action='store_true', default=True,
|
||||||
help='Skip completed steps (default: True)')
|
help='Skip completed steps (default: True)')
|
||||||
process_parser.add_argument('--no-skip', action='store_false', dest='skip_completed',
|
process_parser.add_argument('--no-skip', action='store_false', dest='skip_completed',
|
||||||
help='Re-run all steps')
|
help='Re-run all steps')
|
||||||
|
|
||||||
# Legacy arguments (for backward compatibility)
|
|
||||||
parser.add_argument('--check', action='store_true', help='查重')
|
|
||||||
parser.add_argument('--url', type=str, help='要检查的URL')
|
|
||||||
parser.add_argument('--archive-file', type=str, help='归档的文件路径')
|
|
||||||
parser.add_argument('--base-path', type=str, help='获取知识库根目录')
|
|
||||||
|
|
||||||
parser.add_argument('--pending', action='store_true', help='获取url解析目录')
|
|
||||||
parser.add_argument('--metadata', action='store_true', help='获取元数据目录')
|
|
||||||
parser.add_argument('--content', action='store_true', help='获取内容目录')
|
|
||||||
parser.add_argument('--content_downloads', action='store_true', help='获取主内容下载目录')
|
|
||||||
parser.add_argument('--analyzed', action='store_true', help='获取深度分析目录')
|
|
||||||
parser.add_argument('--archive_dir', action='store_true', help='获取归档目录')
|
|
||||||
parser.add_argument('--archive_subscriptions', action='store_true', help='获取订阅信息目录')
|
|
||||||
parser.add_argument('--extract_platform', type=str, help='要提取平台名称的文件名')
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
base_path = Path(args.base_path) if args.base_path else None
|
|
||||||
dirs = ensure_directories(base_path)
|
|
||||||
|
|
||||||
# Handle process subcommand
|
# Handle process subcommand
|
||||||
if args.command == 'process':
|
if args.command == 'process':
|
||||||
result = process_command(
|
result = process_command(
|
||||||
@@ -461,45 +134,6 @@ def main():
|
|||||||
skip_completed=args.skip_completed
|
skip_completed=args.skip_completed
|
||||||
)
|
)
|
||||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||||
return
|
|
||||||
|
|
||||||
# Legacy argument handling
|
|
||||||
if args.check and args.url:
|
|
||||||
result = check_duplicate(base_path, args.url)
|
|
||||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.archive_file:
|
|
||||||
result = archive_and_git(args.archive_file, base_path)
|
|
||||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.pending:
|
|
||||||
print(json.dumps({"path": str(dirs["pending"])}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.metadata:
|
|
||||||
print(json.dumps({"path": str(dirs["metadata"])}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.content:
|
|
||||||
print(json.dumps({"path": str(dirs["content"])}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.content_downloads:
|
|
||||||
print(json.dumps({"path": str(dirs["content_downloads"])}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.analyzed:
|
|
||||||
print(json.dumps({"path": str(dirs["analyzed"])}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.archive_dir:
|
|
||||||
print(json.dumps({"path": str(get_archive_path())}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.archive_subscriptions:
|
|
||||||
print(json.dumps({"path": str(dirs["archive_subscriptions"])}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.extract_platform:
|
|
||||||
platform = extract_platform(args.extract_platform)
|
|
||||||
print(json.dumps({"platform": platform}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
elif args.base_path:
|
|
||||||
print(json.dumps({"path": str(dirs["base"])}, ensure_ascii=False, indent=2))
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
parser.print_help()
|
parser.print_help()
|
||||||
|
|
||||||
|
|||||||
@@ -1,86 +0,0 @@
|
|||||||
# KB Zhihu Content Skill
|
|
||||||
|
|
||||||
**kb-zhihu-content** - Download Zhihu videos and extract content.
|
|
||||||
|
|
||||||
## Features
|
|
||||||
|
|
||||||
- Video download using **yt-dlp**
|
|
||||||
- Subtitle extraction (returns no subtitle, triggers Whisper)
|
|
||||||
- **Read article content from MediaCrawler data** (NEW!)
|
|
||||||
- Supports `ZHIHU_COOKIE` environment variable
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
| Variable | Required | Description |
|
|
||||||
|----------|----------|-------------|
|
|
||||||
| `ZHIHU_COOKIE` | No | 知乎登录 Cookie (for member-only videos) |
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Download video
|
|
||||||
python zhihu_content.py --id "12345678" --type video --output "/path/to/downloads/"
|
|
||||||
|
|
||||||
# Extract subtitle (returns no subtitle, triggers Whisper)
|
|
||||||
python zhihu_content.py --id "12345678" --type subtitle --output "/path/to/subtitles/"
|
|
||||||
|
|
||||||
# 从 MediaCrawler 读取文章内容 (⚡ NEW!)
|
|
||||||
python zhihu_content.py --type article-from-media --input "~/work/MediaCrawler/data/zhihu/json/detail_contents_2026-02-27.json" --output "~/work/knowledge-base/data/zhihu/"
|
|
||||||
```
|
|
||||||
|
|
||||||
## MediaCrawler 数据读取
|
|
||||||
|
|
||||||
**新功能**:从 MediaCrawler 爬取的数据中提取知乎文章内容
|
|
||||||
|
|
||||||
### 使用方法
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 从 MediaCrawler JSON 文件读取文章内容
|
|
||||||
python zhihu_content.py --type article-from-media \
|
|
||||||
--input "~/work/MediaCrawler/data/zhihu/json/detail_contents_2026-02-27.json" \
|
|
||||||
--output "~/work/knowledge-base/data/zhihu/"
|
|
||||||
|
|
||||||
# 查找最新的爬取数据
|
|
||||||
python zhihu_content.py --type article-from-media \
|
|
||||||
--output "~/work/knowledge-base/data/zhihu/"
|
|
||||||
```
|
|
||||||
|
|
||||||
### 参数说明
|
|
||||||
|
|
||||||
| 参数 | 说明 |
|
|
||||||
|------|------|
|
|
||||||
| `--type article-from-media` | 从 MediaCrawler 数据读取 |
|
|
||||||
| `--input` | MediaCrawler 输出文件路径(可选,不指定则自动查找最新) |
|
|
||||||
| `--output` | 输出目录 |
|
|
||||||
|
|
||||||
### 输出格式
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"type": "article",
|
|
||||||
"filename": "zhihu_article_如何高效的爬取微信公众号.md",
|
|
||||||
"filepath": "/home/xxx/work/knowledge-base/data/zhihu/zhihu_article_如何高效的爬取微信公众号.md",
|
|
||||||
"message": "文章内容已导出"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## 原有功能
|
|
||||||
|
|
||||||
### Video Download
|
|
||||||
|
|
||||||
Video download uses **yt-dlp**:
|
|
||||||
1. Parses `ZHIHU_COOKIE` environment variable (optional)
|
|
||||||
2. Creates temporary Netscape-format cookie file
|
|
||||||
3. Runs `yt-dlp -f best -o <output> <url>`
|
|
||||||
4. Cleans up temporary file
|
|
||||||
|
|
||||||
**Note**: 知乎视频通常没有官方字幕,将自动触发 Whisper 语音识别
|
|
||||||
|
|
||||||
## Installation
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Already included in kb-skills repo
|
|
||||||
# Ensure yt-dlp is installed
|
|
||||||
pip install yt-dlp aiohttp
|
|
||||||
```
|
|
||||||
@@ -1,365 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""
|
|
||||||
知乎内容获取工具
|
|
||||||
|
|
||||||
支持:
|
|
||||||
- zvideo: 知乎视频下载
|
|
||||||
- article-from-media: 从 MediaCrawler 数据读取知乎文章
|
|
||||||
- 从页面提取视频播放地址
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from datetime import datetime
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Optional, Dict, Any, List
|
|
||||||
|
|
||||||
|
|
||||||
# MediaCrawler 目录
|
|
||||||
MEDIACRAWLER_DIR = Path.home() / "work" / "MediaCrawler"
|
|
||||||
|
|
||||||
|
|
||||||
class ZhihuContent:
|
|
||||||
"""知乎内容获取器"""
|
|
||||||
|
|
||||||
def __init__(self, output_dir: str = "./output", cookie: str = None):
|
|
||||||
self.output_dir = Path(output_dir)
|
|
||||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
self.cookie = cookie
|
|
||||||
|
|
||||||
def _create_cookie_file(self) -> str:
|
|
||||||
"""创建 Netscape 格式的 Cookie 文件"""
|
|
||||||
if not self.cookie:
|
|
||||||
return None
|
|
||||||
|
|
||||||
cookies = {}
|
|
||||||
for item in self.cookie.split(';'):
|
|
||||||
item = item.strip()
|
|
||||||
if '=' in item:
|
|
||||||
key, value = item.split('=', 1)
|
|
||||||
cookies[key.strip()] = value.strip()
|
|
||||||
|
|
||||||
cookie_content = """# Netscape HTTP Cookie File
|
|
||||||
# Generated by zhihu_content.py
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
for name, value in cookies.items():
|
|
||||||
cookie_content += f".zhihu.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n"
|
|
||||||
|
|
||||||
fd, path = tempfile.mkstemp(suffix='.txt')
|
|
||||||
with os.fdopen(fd, 'w') as f:
|
|
||||||
f.write(cookie_content)
|
|
||||||
|
|
||||||
return path
|
|
||||||
|
|
||||||
async def download_video(self, video_id: str, quality: int = 720) -> Dict[str, Any]:
|
|
||||||
"""下载知乎视频(使用 yt-dlp)"""
|
|
||||||
result = {"success": False, "video_id": video_id, "message": "", "filepath": None, "metadata": {}}
|
|
||||||
|
|
||||||
try:
|
|
||||||
print(f"获取知乎视频信息: {video_id}")
|
|
||||||
|
|
||||||
cookie_file = self._create_cookie_file()
|
|
||||||
url = f"https://www.zhihu.com/zvideo/{video_id}"
|
|
||||||
|
|
||||||
# 先获取视频标题
|
|
||||||
try:
|
|
||||||
if cookie_file:
|
|
||||||
cmd = ["yt-dlp", "--cookies", cookie_file, "--get-title", url]
|
|
||||||
else:
|
|
||||||
cmd = ["yt-dlp", "--get-title", url]
|
|
||||||
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
|
||||||
*cmd,
|
|
||||||
stdout=asyncio.subprocess.PIPE,
|
|
||||||
stderr=asyncio.subprocess.PIPE
|
|
||||||
)
|
|
||||||
stdout, stderr = await proc.communicate()
|
|
||||||
|
|
||||||
title = stdout.decode('utf-8').strip()
|
|
||||||
if not title or 'error' in title.lower():
|
|
||||||
title = f"zhihu_video_{video_id}"
|
|
||||||
except Exception as e:
|
|
||||||
title = f"zhihu_video_{video_id}"
|
|
||||||
|
|
||||||
# 构建 yt-dlp 命令
|
|
||||||
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
|
|
||||||
|
|
||||||
print(f"下载视频: {title}")
|
|
||||||
output_template = str(self.output_dir / f"{safe_title}.%(ext)s")
|
|
||||||
|
|
||||||
# 构建下载命令
|
|
||||||
cmd = [
|
|
||||||
"yt-dlp",
|
|
||||||
"-f", "bestvideo+bestaudio/best",
|
|
||||||
"-o", output_template,
|
|
||||||
"--no-download-archive",
|
|
||||||
url
|
|
||||||
]
|
|
||||||
|
|
||||||
if cookie_file:
|
|
||||||
cmd.extend(["--cookies", cookie_file])
|
|
||||||
|
|
||||||
print(f"执行命令: {' '.join(cmd[:4])}...")
|
|
||||||
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
|
||||||
*cmd,
|
|
||||||
stdout=asyncio.subprocess.PIPE,
|
|
||||||
stderr=asyncio.subprocess.PIPE
|
|
||||||
)
|
|
||||||
stdout, stderr = await proc.communicate()
|
|
||||||
|
|
||||||
# 查找下载的文件
|
|
||||||
downloaded_files = list(self.output_dir.glob(f"{safe_title}.*"))
|
|
||||||
if downloaded_files:
|
|
||||||
video_file = downloaded_files[0]
|
|
||||||
result["success"] = True
|
|
||||||
result["filepath"] = str(video_file)
|
|
||||||
result["message"] = f"下载完成: {video_file.name}"
|
|
||||||
result["metadata"] = {"title": title, "video_id": video_id}
|
|
||||||
print(f"✅ {result['message']}")
|
|
||||||
else:
|
|
||||||
result["message"] = "未找到下载的文件"
|
|
||||||
print(f"❌ {result['message']}")
|
|
||||||
|
|
||||||
# 清理临时 Cookie 文件
|
|
||||||
if cookie_file and os.path.exists(cookie_file):
|
|
||||||
os.remove(cookie_file)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result["message"] = f"错误: {str(e)}"
|
|
||||||
print(f"❌ {result['message']}")
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def extract_subtitle(self, video_id: str) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
提取字幕
|
|
||||||
|
|
||||||
知乎视频通常没有官方字幕,需要使用 Whisper 进行语音识别。
|
|
||||||
此函数返回 has_subtitle=False,让工作流继续执行 Whisper 步骤。
|
|
||||||
"""
|
|
||||||
result = {"success": True, "video_id": video_id, "has_subtitle": False, "message": "知乎视频无官方字幕,将使用 Whisper 进行语音识别"}
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
def read_article_from_mediacrawler(self, input_file: str = None, output_dir: str = None) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
从 MediaCrawler 爬取的数据中读取知乎文章内容
|
|
||||||
|
|
||||||
Args:
|
|
||||||
input_file: MediaCrawler 输出文件路径,不指定则自动查找最新
|
|
||||||
output_dir: 输出目录
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
处理结果字典
|
|
||||||
"""
|
|
||||||
# 确定输出目录
|
|
||||||
if output_dir:
|
|
||||||
self.output_dir = Path(output_dir)
|
|
||||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
# 查找数据文件
|
|
||||||
data_dir = MEDIACRAWLER_DIR / "data" / "zhihu" / "json"
|
|
||||||
|
|
||||||
if input_file:
|
|
||||||
# 使用指定的文件
|
|
||||||
json_file = Path(input_file)
|
|
||||||
if not json_file.exists():
|
|
||||||
return {"success": False, "message": f"文件不存在: {input_file}"}
|
|
||||||
else:
|
|
||||||
# 查找最新的数据文件
|
|
||||||
json_files = sorted(data_dir.glob("detail_contents_*.json"), reverse=True)
|
|
||||||
if not json_files:
|
|
||||||
return {"success": False, "message": "未找到 MediaCrawler 爬取的数据,请先运行爬虫"}
|
|
||||||
json_file = json_files[0]
|
|
||||||
|
|
||||||
try:
|
|
||||||
with open(json_file, encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
if not data:
|
|
||||||
return {"success": False, "message": "数据文件为空"}
|
|
||||||
|
|
||||||
# 提取第一篇文章
|
|
||||||
item = data[0]
|
|
||||||
|
|
||||||
# 准备数据
|
|
||||||
title = item.get("title", "无标题")
|
|
||||||
content = item.get("content_text", "")
|
|
||||||
url = item.get("content_url", "")
|
|
||||||
author = item.get("user_nickname", "未知")
|
|
||||||
created_ts = item.get("created_time", 0)
|
|
||||||
created = datetime.fromtimestamp(created_ts).strftime("%Y-%m-%d %H:%M:%S") if created_ts else ""
|
|
||||||
voteup = item.get("voteup_count", 0)
|
|
||||||
comments = item.get("comment_count", 0)
|
|
||||||
|
|
||||||
# 生成文件名
|
|
||||||
safe_title = "".join(c for c in title if c.isalnum() or c in " -_")[:50]
|
|
||||||
filename = f"zhihu_article_{safe_title}.md"
|
|
||||||
output_path = self.output_dir / filename
|
|
||||||
|
|
||||||
# 写入 Markdown 文件
|
|
||||||
markdown_content = f"""---
|
|
||||||
title: "{title}"
|
|
||||||
author: "{author}"
|
|
||||||
url: "{url}"
|
|
||||||
created: "{created}"
|
|
||||||
voteup: {voteup}
|
|
||||||
comments: {comments}
|
|
||||||
platform: zhihu
|
|
||||||
type: article
|
|
||||||
---
|
|
||||||
|
|
||||||
# {title}
|
|
||||||
|
|
||||||
**作者**: {author}
|
|
||||||
**来源**: [知乎]({url})
|
|
||||||
**发布时间**: {created}
|
|
||||||
**点赞**: {voteup}
|
|
||||||
**评论**: {comments}
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
{content}
|
|
||||||
"""
|
|
||||||
|
|
||||||
with open(output_path, "w", encoding="utf-8") as f:
|
|
||||||
f.write(markdown_content)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"success": True,
|
|
||||||
"type": "article",
|
|
||||||
"title": title,
|
|
||||||
"filename": filename,
|
|
||||||
"filepath": str(output_path),
|
|
||||||
"content_length": len(content),
|
|
||||||
"message": f"文章内容已导出: {title}"
|
|
||||||
}
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
return {"success": False, "message": f"读取失败: {str(e)}"}
|
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
|
||||||
parser = argparse.ArgumentParser(description='知乎内容获取工具')
|
|
||||||
parser.add_argument('--id', '-i', help='知乎视频ID (用于 video/subtitle 类型)')
|
|
||||||
parser.add_argument('--type', '-t', required=True,
|
|
||||||
choices=['video', 'subtitle', 'article-from-media', 'zhuanlan', 'article'],
|
|
||||||
help='内容类型: video=视频下载, subtitle=字幕提取, article-from-media=从MediaCrawler读取文章, zhuanlan/article=爬取知乎专栏文章')
|
|
||||||
parser.add_argument('--output', '-o', default='./output', help='输出目录')
|
|
||||||
parser.add_argument('--quality', '-q', type=int, default=720, help='视频质量')
|
|
||||||
parser.add_argument('--input', '-in', help='MediaCrawler 数据文件路径 (用于 article-from-media 类型)')
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
cookie = os.environ.get('ZHIHU_COOKIE')
|
|
||||||
downloader = ZhihuContent(output_dir=args.output, cookie=cookie)
|
|
||||||
|
|
||||||
if args.type == 'video':
|
|
||||||
result = await downloader.download_video(args.id, quality=args.quality)
|
|
||||||
elif args.type == 'subtitle':
|
|
||||||
result = await downloader.extract_subtitle(args.id)
|
|
||||||
elif args.type == 'article-from-media':
|
|
||||||
# 同步调用,不需要 asyncio
|
|
||||||
result = downloader.read_article_from_mediacrawler(
|
|
||||||
input_file=args.input,
|
|
||||||
output_dir=args.output
|
|
||||||
)
|
|
||||||
elif args.type in ('zhuanlan', 'article'):
|
|
||||||
# 知乎专栏文章:先爬取再读取
|
|
||||||
# 需要从 --id 或构造完整 URL
|
|
||||||
if args.id:
|
|
||||||
# 如果传入的是完整 URL
|
|
||||||
if args.id.startswith('http'):
|
|
||||||
article_url = args.id
|
|
||||||
else:
|
|
||||||
# 如果只传入 ID,构造 URL
|
|
||||||
article_url = f"https://zhuanlan.zhihu.com/p/{args.id}"
|
|
||||||
else:
|
|
||||||
result = {"success": False, "message": "zhuanlan/article 类型需要 --id 参数(文章ID或完整URL)"}
|
|
||||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
||||||
return result
|
|
||||||
|
|
||||||
# 调用 zhihu_article_crawler.py 爬取
|
|
||||||
crawler_script = Path(__file__).parent.parent / "kb-zhihu-metadata" / "zhihu_article_crawler.py"
|
|
||||||
if not crawler_script.exists():
|
|
||||||
# 尝试另一种路径
|
|
||||||
crawler_script = Path.home() / ".openclaw" / "agents" / "knowledge-base" / "workspace" / "skills" / "kb-zhihu-metadata" / "zhihu_article_crawler.py"
|
|
||||||
|
|
||||||
if crawler_script.exists():
|
|
||||||
import subprocess
|
|
||||||
cmd = ["python3", str(crawler_script), "--url", article_url, "--output", args.output]
|
|
||||||
print(f"调用爬虫: {' '.join(cmd)}")
|
|
||||||
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
||||||
|
|
||||||
if proc.returncode != 0:
|
|
||||||
result = {"success": False, "message": f"爬取失败: {proc.stderr[:200]}"}
|
|
||||||
else:
|
|
||||||
# 解析爬虫输出获取文件路径
|
|
||||||
try:
|
|
||||||
crawler_output = json.loads(proc.stdout.split("--- JSON 输出 ---")[-1].strip())
|
|
||||||
if crawler_output.get("success"):
|
|
||||||
# 直接使用爬虫输出的文件
|
|
||||||
result = {
|
|
||||||
"success": True,
|
|
||||||
"type": "article",
|
|
||||||
"title": crawler_output.get("title"),
|
|
||||||
"filename": Path(crawler_output["filepath"]).name,
|
|
||||||
"filepath": crawler_output["filepath"],
|
|
||||||
"content_length": crawler_output.get("content_length"),
|
|
||||||
"message": f"知乎专栏文章已爬取: {crawler_output.get('title')}"
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
result = {"success": False, "message": crawler_output.get("error", "爬取失败")}
|
|
||||||
except Exception as e:
|
|
||||||
result = {"success": False, "message": f"解析爬虫输出失败: {str(e)}"}
|
|
||||||
else:
|
|
||||||
result = {"success": False, "message": f"未找到 zhihu_article_crawler.py: {crawler_script}"}
|
|
||||||
else:
|
|
||||||
result = {"success": False, "message": f"不支持的类型: {args.type}"}
|
|
||||||
|
|
||||||
# 统一输出格式
|
|
||||||
if result.get("success"):
|
|
||||||
output = {
|
|
||||||
"success": True,
|
|
||||||
"type": args.type,
|
|
||||||
"has_subtitle": result.get("has_subtitle", False),
|
|
||||||
"filename": result.get('filename') or (Path(result.get('filepath', '')).name if result.get('filepath') else None),
|
|
||||||
"filepath": str(Path(result.get('filepath', '')).resolve()) if result.get('filepath') else None,
|
|
||||||
"message": result.get('message', '处理成功')
|
|
||||||
}
|
|
||||||
if result.get('title'):
|
|
||||||
output['title'] = result['title']
|
|
||||||
if result.get('content_length'):
|
|
||||||
output['content_length'] = result['content_length']
|
|
||||||
else:
|
|
||||||
output = {
|
|
||||||
"success": False,
|
|
||||||
"error": result.get('message', '处理失败'),
|
|
||||||
"detail": f"ID: {args.id}, 类型: {args.type}"
|
|
||||||
}
|
|
||||||
|
|
||||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
try:
|
|
||||||
result = asyncio.run(main())
|
|
||||||
sys.exit(0 if result.get("success") else 1)
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
print(json.dumps({"success": False, "error": "用户取消"}, ensure_ascii=False, indent=2))
|
|
||||||
sys.exit(1)
|
|
||||||
except Exception as e:
|
|
||||||
print(json.dumps({"success": False, "error": str(e)}, ensure_ascii=False, indent=2))
|
|
||||||
sys.exit(1)
|
|
||||||
@@ -38,6 +38,9 @@ class MarkdownConverter:
|
|||||||
将 data-actualsrc 或 data-original 属性的值替换到 src 属性中,
|
将 data-actualsrc 或 data-original 属性的值替换到 src 属性中,
|
||||||
以便 markdownify 能够正确提取真实图片 URL。
|
以便 markdownify 能够正确提取真实图片 URL。
|
||||||
|
|
||||||
|
同时移除 <noscript> 标签,因为知乎等平台在 noscript 中包含备用图片,
|
||||||
|
会导致修复后图片重复出现。
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
html: 输入的 HTML 字符串
|
html: 输入的 HTML 字符串
|
||||||
|
|
||||||
@@ -46,6 +49,10 @@ class MarkdownConverter:
|
|||||||
"""
|
"""
|
||||||
soup = BeautifulSoup(html, 'html.parser')
|
soup = BeautifulSoup(html, 'html.parser')
|
||||||
|
|
||||||
|
# 先移除 noscript 标签,避免图片重复
|
||||||
|
for noscript in soup.find_all('noscript'):
|
||||||
|
noscript.decompose()
|
||||||
|
|
||||||
for img in soup.find_all('img'):
|
for img in soup.find_all('img'):
|
||||||
src = img.get('src', '')
|
src = img.get('src', '')
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user