Update obsidian-kb skill and add find-skills

- obsidian-kb: Add OCR processors (easyocr, tesseract)
- obsidian-kb: Fix subtitle detection with test files
- obsidian-kb: Add cookie verification script
- Add find-skills skill for discovering capabilities
- Add video processing scripts
- Update clawhub lock file
This commit is contained in:
OpenClaw Agent
2026-02-09 11:53:42 +08:00
parent d66eade77e
commit e0a2b323d9
33 changed files with 3555 additions and 36 deletions
+4
View File
@@ -60,6 +60,10 @@
"obsidian-kb": { "obsidian-kb": {
"version": "1.0.0", "version": "1.0.0",
"installedAt": 1738958400000 "installedAt": 1738958400000
},
"find-skills": {
"version": "0.1.0",
"installedAt": 1770607549813
} }
} }
} }
View File
File diff suppressed because one or more lines are too long
Binary file not shown.

After

Width:  |  Height:  |  Size: 31 KiB

+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
使用旧版 API 获取并下载视频流
"""
import asyncio
import json
import os
from playwright.async_api import async_playwright
async def download_video_stream(bvid, output_dir="/tmp/bilibili_downloads"):
"""下载视频流"""
url = f"https://www.bilibili.com/video/{bvid}"
# 创建输出目录
os.makedirs(output_dir, exist_ok=True)
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
print(f"正在下载视频: {url}")
print("=" * 80)
# 首先获取 aid 和 cid
print("\n1. 获取视频信息...")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') != 0:
print(f" ❌ 失败: {data.get('message')}")
await browser.close()
return None
video_data = data.get('data', {})
title = video_data.get('title')
aid = video_data.get('aid')
cid = video_data.get('cid')
duration = video_data.get('duration')
print(f" 标题: {title}")
print(f" 时长: {duration}秒")
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
return None
# 获取视频流 URL
print("\n2. 获取视频流 URL...")
try:
playurl_api = "https://api.bilibili.com/x/player/playurl"
params = {
'bvid': bvid,
'cid': cid,
'qn': 80, # 1080P
'otype': 'json',
'fnver': 0,
'fnval': 0, # FLV 格式
}
playurl_response = await page.request.get(playurl_api, params=params)
if not playurl_response.ok:
print(f" ❌ HTTP 错误: {playurl_response.status}")
await browser.close()
return None
playurl_data = await playurl_response.json()
if playurl_data.get('code') != 0:
print(f" ❌ API 错误: {playurl_data.get('message')}")
await browser.close()
return None
play_data = playurl_data.get('data', {})
# 检查支持的清晰度
print(f"\n 支持的清晰度:")
for i, (q, desc) in enumerate(zip(play_data.get('accept_quality', []), play_data.get('accept_description', []))):
current = " (当前)" if q == play_data.get('quality') else ""
print(f" {desc} ({q}){current}")
# 获取视频 URL
durl = play_data.get('durl', [])
if not durl:
print(f" ❌ 没有找到视频流")
await browser.close()
return None
print(f"\n 找到 {len(durl)} 个视频分段")
video_urls = []
for i, segment in enumerate(durl):
segment_url = segment.get('url')
segment_size = segment.get('size', 0)
video_urls.append(segment_url)
print(f" 分段 {i+1}: {segment_size / 1024 / 1024:.2f}MB")
print(f" {segment_url[:100]}...")
except Exception as e:
print(f" ❌ 异常: {e}")
import traceback
traceback.print_exc()
await browser.close()
return None
# 下载视频
print(f"\n3. 开始下载视频...")
all_segments = []
total_size = 0
for i, video_url in enumerate(video_urls, 1):
print(f"\n 下载分段 {i}/{len(video_urls)}...")
try:
# 添加必要的请求头
headers = {
'Referer': 'https://www.bilibili.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
}
# 下载分段
segment_response = await page.request.get(
video_url,
headers=headers,
timeout=60000
)
if not segment_response.ok:
print(f" ❌ 下载失败: {segment_response.status}")
continue
segment_data = await segment_response.body()
segment_size = len(segment_data)
total_size += segment_size
all_segments.append(segment_data)
print(f" ✅ 下载成功: {segment_size / 1024 / 1024:.2f}MB")
except Exception as e:
print(f" ❌ 下载异常: {e}")
continue
if not all_segments:
print(f"\n ❌ 没有成功下载任何分段")
await browser.close()
return None
# 合并分段
print(f"\n4. 合并视频分段...")
video_data = b''.join(all_segments)
# 保存文件
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
output_path = os.path.join(output_dir, f"{safe_title}.flv")
with open(output_path, 'wb') as f:
f.write(video_data)
print(f" ✅ 视频已保存: {output_path}")
print(f" 文件大小: {total_size / 1024 / 1024:.2f}MB")
await browser.close()
return output_path
if __name__ == '__main__':
bvid = "BV1KtFzzqEUG"
result = asyncio.run(download_video_stream(bvid))
if result:
print(f"\n✅ 下载完成!")
print(f"文件路径: {result}")
else:
print(f"\n❌ 下载失败")
+228
View File
@@ -0,0 +1,228 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
下载视频片段用于 OCR 提取硬字幕
策略:每隔 N 秒截取一帧,而不是下载整个视频
"""
import asyncio
import json
import os
import cv2
import numpy as np
from playwright.async_api import async_playwright
from urllib.request import Request, urlopen
async def extract_frames_for_ocr(bvid, output_dir="/tmp/bilibili_frames", interval_seconds=10, max_frames=200):
"""
提取视频帧用于 OCR
Args:
bvid: B站视频 BV 号
output_dir: 输出目录
interval_seconds: 截帧间隔(秒)
max_frames: 最大截取帧数
Returns:
str: 输出目录路径
"""
url = f"https://www.bilibili.com/video/{bvid}"
# 创建输出目录
os.makedirs(output_dir, exist_ok=True)
print(f"正在提取视频帧: {url}")
print(f"间隔: {interval_seconds}秒,最大帧数: {max_frames}")
print("=" * 80)
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
# 获取视频信息
print("\n1. 获取视频信息...")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') != 0:
print(f" ❌ 失败: {data.get('message')}")
await browser.close()
return None
video_data = data.get('data', {})
title = video_data.get('title')
aid = video_data.get('aid')
cid = video_data.get('cid')
duration = video_data.get('duration')
print(f" 标题: {title}")
print(f" 时长: {duration}秒")
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
return None
# 获取视频流 URL
print("\n2. 获取视频流 URL...")
try:
playurl_api = "https://api.bilibili.com/x/player/playurl"
params = {
'bvid': bvid,
'cid': cid,
'qn': 80, # 1080P
'otype': 'json',
'fnver': 0,
'fnval': 0, # FLV 格式
}
playurl_response = await page.request.get(playurl_api, params=params)
if not playurl_response.ok or playurl_response.status != 200:
print(f" ❌ HTTP 错误: {playurl_response.status}")
await browser.close()
return None
playurl_data = await playurl_response.json()
if playurl_data.get('code') != 0:
print(f" ❌ API 错误: {playurl_data.get('message')}")
await browser.close()
return None
play_data = playurl_data.get('data', {})
durl = play_data.get('durl', [])
if not durl:
print(f" ❌ 没有找到视频流")
await browser.close()
return None
# 获取第一个分段的 URL
first_segment_url = durl[0].get('url')
print(f" ✅ 获取到视频流 URL")
except Exception as e:
print(f" ❌ 异常: {e}")
import traceback
traceback.print_exc()
await browser.close()
return None
# 使用 OpenCV 直接从 URL 读取视频并提取帧
print(f"\n3. 提取视频帧...")
try:
# 添加请求头
headers = {
'Referer': 'https://www.bilibili.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
}
# 使用 urllib 下载视频到临时文件(只下载前 50MB)
print(f" 正在下载视频(前 50MB)...")
req = Request(first_segment_url, headers=headers)
# 下载视频数据(限制 50MB)
chunk_size = 8192
max_bytes = 50 * 1024 * 1024 # 50MB
video_data = bytearray()
with urlopen(req, timeout=60) as response:
while True:
chunk = response.read(chunk_size)
if not chunk or len(video_data) >= max_bytes:
break
video_data.extend(chunk)
print(f" 已下载 {len(video_data) / 1024 / 1024:.2f}MB")
# 保存为临时文件
temp_video_path = os.path.join(output_dir, "temp_video.flv")
with open(temp_video_path, 'wb') as f:
f.write(bytes(video_data))
# 使用 OpenCV 打开视频
cap = cv2.VideoCapture(temp_video_path)
if not cap.isOpened():
print(f" ❌ 无法打开视频文件")
await browser.close()
return None
# 获取视频帧率
fps = cap.get(cv2.CAP_PROP_FPS)
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
print(f" 帧率: {fps:.2f} fps")
print(f" 总帧数: {total_frames}")
# 计算需要提取的帧
interval_frames = int(fps * interval_seconds)
frame_count = 0
extracted_count = 0
print(f"\n 开始提取帧...")
while True:
# 跳到目标帧
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_count)
# 读取帧
ret, frame = cap.read()
if not ret:
break
# 保存帧
frame_path = os.path.join(output_dir, f"frame_{extracted_count:04d}_{frame_count}.jpg")
cv2.imwrite(frame_path, frame)
extracted_count += 1
print(f" 提取第 {extracted_count} 帧 (时间: {frame_count / fps:.1f}s)")
# 检查是否达到最大帧数
if extracted_count >= max_frames:
print(f" 已达到最大帧数限制")
break
# 跳到下一帧
frame_count += interval_frames
cap.release()
# 删除临时视频文件
os.remove(temp_video_path)
print(f"\n4. 完成!")
print(f" 成功提取 {extracted_count} 帧图像")
print(f" 输出目录: {output_dir}")
await browser.close()
return output_dir
except Exception as e:
print(f" ❌ 提取帧失败: {e}")
import traceback
traceback.print_exc()
await browser.close()
return None
if __name__ == '__main__':
import sys
bvid = "BV1KtFzzqEUG"
result = asyncio.run(extract_frames_for_ocr(
bvid,
interval_seconds=10, # 每 10 秒提取一帧
max_frames=100 # 最多提取 100 帧
))
if result:
print(f"\n✅ 提取完成!")
print(f"输出目录: {result}")
print(f"\n下一步:使用 OCR 工具识别字幕")
else:
print(f"\n❌ 提取失败")
sys.exit(1)
@@ -0,0 +1,30 @@
---
title: "为什么现在咖啡比矿泉水还便宜?【食录】"
source: "bilibili"
url: "https://www.bilibili.com/video/BV1KtFzzqEUG"
author: "赛博食录"
duration: "12:48"
date: "2026-02-06 11:44:59"
bvid: "BV1KtFzzqEUG"
created: "2026-02-08 23:32:02"
stats:
view: 977748
like: 32085
coin: 4368
favorite: 7642
tags:
---
## 视频简介
你买过最便宜的咖啡要多少钱?(ps:本期全程无广
## 数据统计
- 播放量: 977,748
- 点赞: 32,085
- 投币: 4,368
- 收藏: 7,642
- 分享: 1,653
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
处理B站视频并保存到知识库
"""
import asyncio
import sys
import os
from datetime import datetime
# 添加项目路径
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb'))
from src.extractors.bilibili_extractor import BilibiliExtractor
from src.outputs.obsidian_writer import ObsidianWriter
from src.ai_client import AIClient
async def process_video(video_url):
"""处理视频并生成 Obsidian 笔记"""
print(f"正在处理视频: {video_url}")
print("=" * 60)
# 1. 提取视频信息
print("\n步骤 1/3: 提取视频信息...")
extractor = BilibiliExtractor()
try:
result = await extractor._extract_async(video_url)
print("✅ 视频信息提取成功")
except Exception as e:
print(f"❌ 提取失败: {e}")
import traceback
traceback.print_exc()
return
# 显示提取的信息
print("\n视频信息:")
print("-" * 60)
print(f"标题: {result['metadata']['title']}")
print(f"UP主: {result['metadata']['author']}")
print(f"时长: {result['metadata']['duration']}")
print(f"发布时间: {result['metadata']['pubtime']}")
print(f"播放量: {result['stats']['view']:,}")
print(f"点赞: {result['stats']['like']:,}")
print(f"BV号: {result['metadata']['bvid']}")
# 显示字幕信息
subtitle = result['subtitle']
if subtitle:
lines = subtitle.split('\n')
print(f"\n字幕: 找到 {len(lines)} 条字幕")
else:
print("\n字幕: 该视频没有官方字幕")
# 2. 生成 Obsidian 笔记
print("\n步骤 2/3: 生成 Obsidian 笔记...")
# 创建笔记内容
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
# 构建标签
tags = result['metadata']['tags'].split(', ') if result['metadata']['tags'] else []
tags_str = ' '.join([f"#{tag}" for tag in tags if tag])
# 构建笔记
note_content = f"""---
title: "{result['metadata']['title']}"
source: "bilibili"
url: "{result['metadata']['url']}"
author: "{result['metadata']['author']}"
duration: "{result['metadata']['duration']}"
date: "{result['metadata']['pubtime']}"
bvid: "{result['metadata']['bvid']}"
created: "{now}"
stats:
view: {result['stats']['view']}
like: {result['stats']['like']}
coin: {result['stats']['coin']}
favorite: {result['stats']['favorite']}
tags:
{chr(10).join([f' - {tag}' for tag in tags if tag])}
---
{tags_str}
## 视频简介
{result['description'][:500]}{'...' if len(result['description']) > 500 else ''}
## 数据统计
- 播放量: {result['stats']['view']:,}
- 点赞: {result['stats']['like']:,}
- 投币: {result['stats']['coin']:,}
- 收藏: {result['stats']['favorite']:,}
- 分享: {result['stats']['share']:,}
"""
# 如果有字幕,添加到笔记
if subtitle:
note_content += "## 字幕内容\n\n"
note_content += f"共 {len(subtitle.split(chr(10)))} 条字幕\n\n"
# 只显示前50条字幕,避免过长
lines = subtitle.split('\n')
if len(lines) > 50:
note_content += '\n'.join(lines[:50])
note_content += f"\n\n...(还有 {len(lines) - 50} 条字幕,已省略)\n"
else:
note_content += subtitle + '\n'
print("✅ 笔记生成成功")
# 3. 保存笔记文件
print("\n步骤 3/3: 保存笔记文件...")
# 确定保存目录
kb_dir = os.path.expanduser("~/ObsidianVault")
if not os.path.exists(kb_dir):
# 如果用户的主目录没有 ObsidianVault,使用本地目录
kb_dir = os.path.join(os.path.dirname(__file__), 'kb_output')
if not os.path.exists(kb_dir):
os.makedirs(kb_dir)
print(f"创建输出目录: {kb_dir}")
# 生成文件名(使用日期和标题)
date_str = datetime.now().strftime('%Y%m%d')
safe_title = result['metadata']['title'][:50].replace('/', '-').replace('\\', '-').replace(':', '-')
filename = f"{date_str}-{safe_title}.md"
filepath = os.path.join(kb_dir, filename)
# 写入文件
with open(filepath, 'w', encoding='utf-8') as f:
f.write(note_content)
print(f"✅ 笔记已保存到: {filepath}")
print("\n" + "=" * 60)
print("🎉 处理完成!")
print("=" * 60)
if __name__ == '__main__':
# 从URL中提取BV号
import re
video_url = "https://www.bilibili.com/video/BV1KtFzzqEUG/?spm_id_from=333.1007.tianma.2-1-4.click&vd_source=d82ff6409be4a2d868295c58516354a2"
# 提取BV号
bv_match = re.search(r'BV[a-zA-Z0-9]{10}', video_url)
if bv_match:
bvid = bv_match.group(0)
print(f"识别到BV号: {bvid}\n")
asyncio.run(process_video(bvid))
else:
print("❌ 无法从URL中提取BV号")
+7
View File
@@ -0,0 +1,7 @@
{
"version": 1,
"registry": "https://clawhub.ai",
"slug": "find-skills",
"installedVersion": "0.1.0",
"installedAt": 1770607549810
}
+133
View File
@@ -0,0 +1,133 @@
---
name: find-skills
description: Helps users discover and install agent skills when they ask questions like "how do I do X", "find a skill for X", "is there a skill that can...", or express interest in extending capabilities. This skill should be used when the user is looking for functionality that might exist as an installable skill.
---
# Find Skills
This skill helps you discover and install skills from the open agent skills ecosystem.
## When to Use This Skill
Use this skill when the user:
- Asks "how do I do X" where X might be a common task with an existing skill
- Says "find a skill for X" or "is there a skill for X"
- Asks "can you do X" where X is a specialized capability
- Expresses interest in extending agent capabilities
- Wants to search for tools, templates, or workflows
- Mentions they wish they had help with a specific domain (design, testing, deployment, etc.)
## What is the Skills CLI?
The Skills CLI (`npx skills`) is the package manager for the open agent skills ecosystem. Skills are modular packages that extend agent capabilities with specialized knowledge, workflows, and tools.
**Key commands:**
- `npx skills find [query]` - Search for skills interactively or by keyword
- `npx skills add <package>` - Install a skill from GitHub or other sources
- `npx skills check` - Check for skill updates
- `npx skills update` - Update all installed skills
**Browse skills at:** https://skills.sh/
## How to Help Users Find Skills
### Step 1: Understand What They Need
When a user asks for help with something, identify:
1. The domain (e.g., React, testing, design, deployment)
2. The specific task (e.g., writing tests, creating animations, reviewing PRs)
3. Whether this is a common enough task that a skill likely exists
### Step 2: Search for Skills
Run the find command with a relevant query:
```bash
npx skills find [query]
```
For example:
- User asks "how do I make my React app faster?" → `npx skills find react performance`
- User asks "can you help me with PR reviews?" → `npx skills find pr review`
- User asks "I need to create a changelog" → `npx skills find changelog`
The command will return results like:
```
Install with npx skills add <owner/repo@skill>
vercel-labs/agent-skills@vercel-react-best-practices
└ https://skills.sh/vercel-labs/agent-skills/vercel-react-best-practices
```
### Step 3: Present Options to the User
When you find relevant skills, present them to the user with:
1. The skill name and what it does
2. The install command they can run
3. A link to learn more at skills.sh
Example response:
```
I found a skill that might help! The "vercel-react-best-practices" skill provides
React and Next.js performance optimization guidelines from Vercel Engineering.
To install it:
npx skills add vercel-labs/agent-skills@vercel-react-best-practices
Learn more: https://skills.sh/vercel-labs/agent-skills/vercel-react-best-practices
```
### Step 4: Offer to Install
If the user wants to proceed, you can install the skill for them:
```bash
npx skills add <owner/repo@skill> -g -y
```
The `-g` flag installs globally (user-level) and `-y` skips confirmation prompts.
## Common Skill Categories
When searching, consider these common categories:
| Category | Example Queries |
| --------------- | ---------------------------------------- |
| Web Development | react, nextjs, typescript, css, tailwind |
| Testing | testing, jest, playwright, e2e |
| DevOps | deploy, docker, kubernetes, ci-cd |
| Documentation | docs, readme, changelog, api-docs |
| Code Quality | review, lint, refactor, best-practices |
| Design | ui, ux, design-system, accessibility |
| Productivity | workflow, automation, git |
## Tips for Effective Searches
1. **Use specific keywords**: "react testing" is better than just "testing"
2. **Try alternative terms**: If "deploy" doesn't work, try "deployment" or "ci-cd"
3. **Check popular sources**: Many skills come from `vercel-labs/agent-skills` or `ComposioHQ/awesome-claude-skills`
## When No Skills Are Found
If no relevant skills exist:
1. Acknowledge that no existing skill was found
2. Offer to help with the task directly using your general capabilities
3. Suggest the user could create their own skill with `npx skills init`
Example:
```
I searched for skills related to "xyz" but didn't find any matches.
I can still help you with this task directly! Would you like me to proceed?
If this is something you do often, you could create your own skill:
npx skills init my-xyz-skill
```
+6
View File
@@ -0,0 +1,6 @@
{
"ownerId": "kn77ajmmqw3cgnc3ay1x3e0ccd805hsw",
"slug": "find-skills",
"version": "0.1.0",
"publishedAt": 1769698710765
}
+30
View File
@@ -0,0 +1,30 @@
#!/bin/bash
# 禁用 oneDNN 后端,解决 PaddleOCR 兼容性问题
echo "禁用 oneDNN..."
# 方法1:环境变量(临时)
export FLAGS_use_mkldnn=0
echo "✅ 环境变量已设置: FLAGS_use_mkldnn=0"
# 方法2:写入 ~/.bashrc(永久)
if ! grep -q "FLAGS_use_mkldnn=0" ~/.bashrc; then
echo "" >> ~/.bashrc
echo "# 禁用 PaddleOCR 的 oneDNN 后端(解决兼容性问题)" >> ~/.bashrc
echo "export FLAGS_use_mkldnn=0" >> ~/.bashrc
echo "✅ 已写入 ~/.bashrc"
else
echo "ℹ️ ~/.bashrc 中已存在此配置"
fi
echo ""
echo "验证配置..."
if [ "$FLAGS_use_mkldnn" = "0" ]; then
echo "✅ 当前会话: FLAGS_use_mkldnn=$FLAGS_use_mkldnn"
else
echo "⚠️ 当前会话: FLAGS_use_mkldnn=${FLAGS_use_mkldnn:-未设置}"
echo "💡 提示: 运行 'source ~/.bashrc' 使配置生效"
fi
echo ""
echo "完成!现在 PaddleOCR 将使用 CPU 原生计算,避免 oneDNN 兼容性问题。"
+42
View File
@@ -0,0 +1,42 @@
Collecting DrissionPage
Using cached DrissionPage-4.1.1.2-py3-none-any.whl.metadata (4.7 kB)
Requirement already satisfied: lxml in ./.venv/lib/python3.12/site-packages (from DrissionPage) (6.0.2)
Requirement already satisfied: requests in ./.venv/lib/python3.12/site-packages (from DrissionPage) (2.32.5)
Collecting cssselect (from DrissionPage)
Using cached cssselect-1.4.0-py3-none-any.whl.metadata (2.4 kB)
Collecting DownloadKit>=2.0.7 (from DrissionPage)
Using cached DownloadKit-2.0.7-py3-none-any.whl.metadata (1.7 kB)
Collecting websocket-client (from DrissionPage)
Using cached websocket_client-1.9.0-py3-none-any.whl.metadata (8.3 kB)
Requirement already satisfied: click in ./.venv/lib/python3.12/site-packages (from DrissionPage) (8.3.1)
Collecting tldextract>=3.4.4 (from DrissionPage)
Using cached tldextract-5.3.1-py3-none-any.whl.metadata (7.3 kB)
Collecting psutil (from DrissionPage)
Using cached psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl.metadata (22 kB)
Collecting DataRecorder>=3.4.11 (from DownloadKit>=2.0.7->DrissionPage)
Using cached DataRecorder-3.6.2-py3-none-any.whl.metadata (4.5 kB)
Requirement already satisfied: idna in ./.venv/lib/python3.12/site-packages (from tldextract>=3.4.4->DrissionPage) (3.11)
Collecting requests-file>=1.4 (from tldextract>=3.4.4->DrissionPage)
Using cached requests_file-3.0.1-py2.py3-none-any.whl.metadata (1.7 kB)
Collecting filelock>=3.0.8 (from tldextract>=3.4.4->DrissionPage)
Using cached filelock-3.20.3-py3-none-any.whl.metadata (2.1 kB)
Requirement already satisfied: charset_normalizer<4,>=2 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (3.4.4)
Requirement already satisfied: urllib3<3,>=1.21.1 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (2.6.3)
Requirement already satisfied: certifi>=2017.4.17 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (2026.1.4)
Collecting openpyxl (from DataRecorder>=3.4.11->DownloadKit>=2.0.7->DrissionPage)
Using cached openpyxl-3.1.5-py2.py3-none-any.whl.metadata (2.5 kB)
Collecting et-xmlfile (from openpyxl->DataRecorder>=3.4.11->DownloadKit>=2.0.7->DrissionPage)
Using cached et_xmlfile-2.0.0-py3-none-any.whl.metadata (2.7 kB)
Using cached DrissionPage-4.1.1.2-py3-none-any.whl (257 kB)
Using cached DownloadKit-2.0.7-py3-none-any.whl (21 kB)
Using cached tldextract-5.3.1-py3-none-any.whl (105 kB)
Using cached cssselect-1.4.0-py3-none-any.whl (18 kB)
Using cached psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl (155 kB)
Using cached websocket_client-1.9.0-py3-none-any.whl (82 kB)
Using cached DataRecorder-3.6.2-py3-none-any.whl (37 kB)
Using cached filelock-3.20.3-py3-none-any.whl (16 kB)
Using cached requests_file-3.0.1-py2.py3-none-any.whl (4.5 kB)
Using cached openpyxl-3.1.5-py2.py3-none-any.whl (250 kB)
Using cached et_xmlfile-2.0.0-py3-none-any.whl (18 kB)
Installing collected packages: websocket-client, psutil, filelock, et-xmlfile, cssselect, requests-file, openpyxl, tldextract, DataRecorder, DownloadKit, DrissionPage
Successfully installed DataRecorder-3.6.2 DownloadKit-2.0.7 DrissionPage-4.1.1.2 cssselect-1.4.0 et-xmlfile-2.0.0 filelock-3.20.3 openpyxl-3.1.5 psutil-7.2.2 requests-file-3.0.1 tldextract-5.3.1 websocket-client-1.9.0
+143
View File
@@ -208,6 +208,51 @@ bilibili_extraction:
enable_danmaku: false # 启用弹幕提取 enable_danmaku: false # 启用弹幕提取
timeout_seconds: 30 # 超时时间 timeout_seconds: 30 # 超时时间
max_retries: 3 # 最大重试次数 max_retries: 3 # 最大重试次数
# 硬字幕提取配置 (OCR 方式)
hard_subtitle:
# 是否启用硬字幕提取功能
enabled: true
# OCR 引擎选择
# 可选值: "paddleocr" (推荐), "tesseract"
engine: "paddleocr"
# 字幕区域检测配置
detection:
# 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测)
default_region: "bottom"
# 当默认区域未检测到字幕时,是否自动回退到全屏模式
fallback_auto: true
# 是否缓存字幕区域检测结果(提升处理速度)
cache_region: true
# OCR 识别配置
ocr:
# 置信度阈值: 低于此值的识别结果将被过滤
confidence_threshold: 0.7
# 批处理大小: 每次处理的帧数(影响内存使用和速度)
batch_size: 32
# 最大并发工作线程数
max_workers: 4
# 视频帧提取配置
video:
# 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别
interval_seconds: 5
# 最大抽取帧数: 防止处理过长视频导致资源耗尽
max_frames: 500
# 流式处理模式: 逐步读取视频帧,避免一次性加载整个视频
stream_mode: true
# 后处理配置
postprocess:
# 是否启用去重: 去除连续相同的字幕文本
deduplicate: true
# 最小文本长度: 少于此长度的识别结果将被视为噪声
min_length: 2
# 是否过滤噪声: 使用规则过滤明显错误的识别结果
filter_noise: true
``` ```
### 环境变量 ### 环境变量
@@ -319,6 +364,104 @@ else:
content["subtitle_text"] = subtitle_text content["subtitle_text"] = subtitle_text
``` ```
### 硬字幕提取功能
当视频没有内置字幕时,硬字幕提取功能可以通过 OCR(光学字符识别)技术从视频画面中识别字幕文本。
**功能介绍:**
- 自动检测和识别视频画面中的字幕文本
- 支持多种 OCR 引擎(PaddleOCR、Tesseract)
- 智能字幕区域检测(底部、全屏、自动)
- 自动去重和噪声过滤
- 支持流式处理,降低内存占用
**使用场景:**
- 视频没有提供 CC 字幕或 AI 字幕
- 字幕嵌入在视频画面中(硬编码字幕)
- 需要提取非标准格式的字幕
**配置说明:**
```yaml
hard_subtitle:
# 是否启用硬字幕提取功能
enabled: true
# OCR 引擎选择
# 可选值: "paddleocr" (推荐), "tesseract"
engine: "paddleocr"
# 字幕区域检测配置
detection:
# 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测)
default_region: "bottom"
# 当默认区域未检测到字幕时,是否自动回退到全屏模式
fallback_auto: true
# 是否缓存字幕区域检测结果(提升处理速度)
cache_region: true
# OCR 识别配置
ocr:
# 置信度阈值: 低于此值的识别结果将被过滤
confidence_threshold: 0.7
# 批处理大小: 每次处理的帧数(影响内存使用和速度)
batch_size: 32
# 最大并发工作线程数
max_workers: 4
# 视频帧提取配置
video:
# 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别
interval_seconds: 5
# 最大抽取帧数: 防止处理过长视频导致资源耗尽
max_frames: 500
# 流式处理模式: 逐步读取视频帧,避免一次性加载整个视频
stream_mode: true
# 后处理配置
postprocess:
# 是否启用去重: 去除连续相同的字幕文本
deduplicate: true
# 最小文本长度: 少于此长度的识别结果将被视为噪声
min_length: 2
# 是否过滤噪声: 使用规则过滤明显错误的识别结果
filter_noise: true
```
**使用示例:**
```python
from obsidian_kb.extractors.bilibili import BilibiliExtractor
# 创建提取器实例(自动启用硬字幕提取)
extractor = BilibiliExtractor(
use_playwright=True,
headless=True
)
# 当视频没有内置字幕时,会自动尝试硬字幕提取
subtitle = extractor.extract_subtitle("https://www.bilibili.com/video/BV1xx411c7mD")
print(f"识别到的字幕: {subtitle}")
```
**注意事项:**
1. **性能考虑**:硬字幕提取需要较高的计算资源,建议在配置合理的机器上运行
2. **识别准确度**:OCR 识别准确度受字幕字体、颜色、背景等因素影响,可能存在误差
3. **处理时间**:硬字幕提取比软字幕提取慢,对于长视频需要较长时间处理
4. **依赖安装**:使用前需要安装 OCR 引擎依赖:
- PaddleOCR: `pip install paddleocr paddlepaddle`
- Tesseract: 需要安装 Tesseract OCR 引擎和语言包
5. **网络视频**:流式处理需要稳定的网络连接
6. **字幕格式**:不同视频的字幕样式差异较大,可能需要调整配置参数以获得最佳效果
**最佳实践:**
- 对于短视频(<30分钟),可以减小 `interval_seconds` 以提高识别精度
- 对于长视频,增加 `interval_seconds` 和减少 `max_frames` 以平衡速度和质量
- 如果字幕位置固定在底部,使用 `default_region: "bottom"` 可以提升速度
- 首次使用建议先测试一段视频,根据结果调整配置参数
### ZhihuDrissionExtractor 类 ### ZhihuDrissionExtractor 类
知乎 DrissionPage 提取器提供知乎内容的完整提取功能: 知乎 DrissionPage 提取器提供知乎内容的完整提取功能:
+91
View File
@@ -0,0 +1,91 @@
# Obsidian KB - 待办事项
## 📌 当前任务:解决硬字幕 OCR 问题
### ✅ 已完成
1. [x] 分析硬字幕提取问题
- 发现 oneDNN 兼容性问题
- 发现 PaddleOCR 参数错误
2. [x] 禁用 oneDNN
- 设置环境变量 FLAGS_use_mkldnn=0
- 写入 ~/.bashrc
3. [x] 修复 PaddleOCR 参数
- 更新 `use_angle_cls` → `use_textline_orientation`
- 移除不支持的 `show_log` 参数
4. [x] 安装 EasyOCR
- 安装 EasyOCR 1.7.2
- 测试 EasyOCR 初始化成功
5. [x] 创建 EasyOCR 处理器
- 新文件:`src/extractors/easyocr_processor.py`
- 支持简体中文识别
- 避免所有 oneDNN 兼容性问题
### ⏳ 待完成
- [ ] 1. 修改硬字幕提取器,支持双 OCR 引擎
- 在 `hard_subtitle_extractor.py` 中添加引擎选择逻辑
- 根据 `config.hard_subtitle.engine` 参数选择使用 PaddleOCR 或 EasyOCR
- 保持向后兼容性
- [ ] 2. 更新配置文件
- 添加 `hard_subtitle.engine` 配置项(`paddleocr` | `easyocr`)
- 默认值设为 `easyocr`(更稳定)
- 更新配置文档
- [ ] 3. 实际测试 EasyOCR 字幕提取
- 使用真实视频测试硬字幕提取
- 对比 EasyOCR 和 PaddleOCR 的识别效果
- 测试处理速度和准确度
- [ ] 4. 性能优化(可选)
- 优化帧提取逻辑(当前只提取 4 帧有问题)
- 调整抽帧间隔和最大帧数
- 考虑多线程/多进程加速
- [ ] 5. 文档更新
- 更新 SKILL.md 中的硬字幕提取说明
- 添加 OCR 引擎选择指南
- 记录 EasyOCR 优势和限制
---
## 📋 备注
### 问题记录
- **PaddleOCR oneDNN 问题**:即使设置环境变量,仍出现兼容性错误
```
(Unimplemented) ConvertPirAttribute2RuntimeAttribute not support
[pir::ArrayAttribute<pir::DoubleAttribute>]
```
- **帧提取异常**:12:48的视频只提取了4帧(应该150+帧)
### EasyOCR 优势
- ✅ 不依赖 PaddlePaddle
- ✅ 安装简单
- ✅ 无底层框架兼容性问题
- ✅ 支持 70+ 种语言
- ✅ 活跃维护
### EasyOCR 限制
- ⚠️ 中文准确度略低于 PaddleOCR
- ⚠️ 首次初始化需要下载模型(约 40MB)
- ⚠️ 处理速度可能略慢
---
## 🔗 相关文件
- `src/extractors/ocr_processor.py` - PaddleOCR 处理器
- `src/extractors/easyocr_processor.py` - EasyOCR 处理器(新建)
- `src/extractors/hard_subtitle_extractor.py` - 硬字幕提取器
- `src/extractors/bilibili_extractor.py` - B站提取器(调用硬字幕提取)
- `.disable_onednn.sh` - 禁用 oneDNN 脚本
---
创建时间:2026-02-09 08:02
最后更新:2026-02-09 08:02
+99 -28
View File
@@ -1,38 +1,109 @@
obsidian: # Obsidian KB 配置文件
vault_path: /home/admin/.openclaw/workspace/knowledge-base # 优先级: ./config.yaml > ~/.obsidian-kb/config.yaml > ~/.config/obsidian-kb/config.yaml
git: # AI 配置
enabled: false ai:
vault_path: /home/admin/.openclaw/workspace/knowledge-base provider: openai # AI 提供商
remote: origin model: gpt-4 # 模型名称
branch: main api_key: sk-... # API Key (建议使用环境变量: OPENAI_API_KEY)
auto_push: false
commit_message_template: "Add note: {title}"
# 存储配置
storage:
inbox_dir: inbox # 收件箱目录
topics_dir: topics # 主题目录
data_dir: data # 数据目录
# 输出配置
output: output:
organize_by: author format: markdown # 输出格式
filename_format: "{date}_{title}.md" encoding: utf-8 # 文件编码
auto_tags: true include_timestamp: true # 包含时间戳
default_tags: include_source: true # 包含源链接
- video
- bilibili
# Git 同步配置
git:
enabled: true # 启用 Git 同步
vault_path: ~/ObsidianVault # Obsidian 知识库路径
remote: origin # 远程仓库名
branch: main # 分支名
auto_commit: true # 自动提交
commit_message_template: "Update from obsidian-kb: {count} items"
# 分段配置
chunking:
enabled: true # 启用长视频分段
chunk_duration_seconds: 900 # 分段时长 (15分钟)
max_chunk_duration_seconds: 1800 # 最大分段时长 (30分钟)
long_video_threshold: 1800 # 长视频阈值 (30分钟)
# 数据源配置
sources: sources:
bilibili: bilibili:
enabled: true enabled: true
extraction: cookie: '' # Bilibili Cookie (可选,环境变量: BILIBILI_COOKIE)
headless: true youtube:
timeout: 30 enabled: false
api_key: '' # YouTube API Key
zhihu:
enabled: true
use_drissionpage: true # 使用 DrissionPage 绕过检测(推荐)
cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径
cookie_str: "" # 可选:直接在配置中填入 Cookie JSON (环境变量: ZHIHU_COOKIE)
timeout: 30
headless: true
zhihu: # Bilibili 提取器配置
enabled: true bilibili_extraction:
use_drissionpage: true use_playwright: true # 使用 Playwright 绕过检测
cookie_file: ~/.obsidian-kb/zhihu_cookies.json headless: true # 无头模式
cookie_str: "" # 可选:直接在配置中填入 cookie_file: '' # Cookie 文件路径 (可选)
timeout: 30 enable_subtitle: true # 启用字幕提取
headless: true subtitle_language: zh-CN # 字幕语言
enable_danmaku: false # 启用弹幕提取
timeout_seconds: 30 # 超时时间
max_retries: 3 # 最大重试次数
chunking: # 硬字幕提取配置 (OCR 方式)
hard_subtitle:
# 是否启用硬字幕提取功能
enabled: true enabled: true
chunk_duration_seconds: 900
long_video_threshold: 1800 # OCR 引擎选择
# 可选值: "paddleocr" (推荐), "tesseract"
engine: "paddleocr"
# 字幕区域检测配置
detection:
# 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测)
default_region: "bottom"
# 当默认区域未检测到字幕时,是否自动回退到全屏模式
fallback_auto: true
# 是否缓存字幕区域检测结果(提升处理速度)
cache_region: true
# OCR 识别配置
ocr:
# 置信度阈值: 低于此值的识别结果将被过滤
confidence_threshold: 0.7
# 批处理大小: 每次处理的帧数(影响内存使用和速度)
batch_size: 32
# 最大并发工作线程数
max_workers: 4
# 视频帧提取配置
video:
# 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别
interval_seconds: 5
# 最大抽取帧数: 防止处理过长视频导致资源耗尽
max_frames: 500
# 流式处理模式: 逐步读取视频帧,避免一次性加载整个视频
stream_mode: true
# 后处理配置
postprocess:
# 是否启用去重: 去除连续相同的字幕文本
deduplicate: true
# 最小文本长度: 少于此长度的识别结果将被视为噪声
min_length: 2
# 是否过滤噪声: 使用规则过滤明显错误的识别结果
filter_noise: true
+3
View File
@@ -14,6 +14,9 @@ anthropic>=0.7.0
# undetected-playwright>=0.1.0 # undetected-playwright>=0.1.0
beautifulsoup4>=4.12.0 beautifulsoup4>=4.12.0
lxml>=4.9.0 lxml>=4.9.0
paddlepaddle>=2.5.0
paddleocr>=2.7.0
opencv-python>=4.8.0
# Zhihu support # Zhihu support
DrissionPage>=4.0.0 DrissionPage>=4.0.0
@@ -9,11 +9,14 @@ B站视频内容提取器
import asyncio import asyncio
import json import json
import logging
import re import re
from datetime import datetime from datetime import datetime
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
from urllib.parse import urlparse from urllib.parse import urlparse
logger = logging.getLogger(__name__)
from playwright.async_api import async_playwright from playwright.async_api import async_playwright
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
@@ -0,0 +1,355 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
OCR 文字识别处理器(EasyOCR 版本)
使用 EasyOCR 实现视频帧文字识别功能,作为 PaddleOCR 的替代方案
主要优势:不依赖 PaddlePaddle,避免 oneDNN 兼容性问题
"""
import re
import numpy as np
from typing import List, Dict, Any, Tuple
import logging
logger = logging.getLogger(__name__)
class EasyOCRProcessor:
"""
EasyOCR 文字识别处理器
使用 EasyOCR 从视频帧中识别文字,支持批量识别和结果过滤
相比 PaddleOCR 的优势:
- 不依赖 PaddlePaddle,避免 oneDNN 兼容性问题
- 安装更简单,依赖更少
- 支持多种语言(70+ 种)
"""
# 相似度阈值:用于判断两个文本是否相似(基于编辑距离)
SIMILARITY_THRESHOLD = 0.85
# 置信度阈值
CONFIDENCE_THRESHOLD = 0.7
# 最小字幕长度(字符数)
MIN_SUBTITLE_LENGTH = 2
def __init__(self, lang: str = 'ch_sim', use_gpu: bool = False, detail: int = 1):
"""
初始化 OCR 处理器
Args:
lang: 语言支持,'ch_sim' 为简体中文,'ch_tra' 为繁体中文
use_gpu: 是否使用 GPU 加速
detail: 识别结果详细程度(0=只返回文本,1=返回文本+坐标+置信度)
"""
self.lang = lang
self.use_gpu = use_gpu
self.detail = detail
# 初始化 EasyOCR 实例
self._ocr = None
self._initialize_ocr()
def _initialize_ocr(self):
"""初始化 EasyOCR 实例"""
try:
from easyocr import Reader
self._ocr = Reader(
[self.lang],
gpu=self.use_gpu,
verbose=False # 不显示详细日志
)
logger.info("EasyOCR 初始化成功")
except Exception as e:
logger.error(f"EasyOCR 初始化失败: {e}")
raise RuntimeError(f"无法初始化 EasyOCR: {e}")
def _initialize_ocr(self):
"""初始化 EasyOCR 实例"""
try:
from easyocr import Reader
self._ocr = Reader(
[self.lang],
gpu=self.use_gpu,
verbose=False # 不显示详细日志
)
logger.info("EasyOCR 初始化成功")
except Exception as e:
logger.error(f"EasyOCR 初始化失败: {e}")
raise RuntimeError(f"无法初始化 EasyOCR: {e}")
def recognize_frames(self, frames: List[Tuple[np.ndarray, float]]) -> List[Dict[str, Any]]:
"""
批量识别帧中的文字
Args:
frames: 帧列表,每个元素为 (图像数组, 时间戳) 元组
- 图像数组:numpy.ndarray,形状为 (H, W, 3) 的 RGB 图像
- 时间戳:float,单位为秒
Returns:
List[Dict[str, Any]]: 识别结果列表,每个结果包含:
- time: float, 时间戳(秒)
- text: str, 识别的文本
- confidence: float, 置信度 (0-1)
"""
if not frames:
logger.warning("输入帧列表为空")
return []
results = []
for image, timestamp in frames:
try:
# 调用 EasyOCR 进行识别
# EasyOCR 返回格式: [[[x1, y1], [x2, y2], [x3, y3], [x4, y4]], (text, confidence, ...)]
# 根据 detail 参数,返回格式不同:
# detail=0: 返回 [text1, text2, ...]
# detail=1: 返回 [[bbox, (text, confidence)], ...]
ocr_result = self._ocr.readtext(image, detail=self.detail)
if ocr_result:
if self.detail == 0:
# 简单模式:只返回文本列表
for i, text in enumerate(ocr_result):
if text:
results.append({
'time': timestamp,
'text': str(text),
'confidence': 1.0 # 简单模式下没有置信度
})
else:
# 详细模式:返回边界框+文本+置信度
for item in ocr_result:
if len(item) >= 2:
bbox = item[0]
text_info = item[1]
if len(text_info) >= 2:
text, confidence = text_info[0], text_info[1]
# 跳过空文本
if not text:
continue
# 处理置信度:可能是数字或特殊字符
try:
if isinstance(confidence, str):
# 如果是 '?' 或其他特殊字符,使用默认置信度
conf_float = 0.5
else:
conf_float = float(confidence)
except (ValueError, TypeError):
conf_float = 0.5
results.append({
'time': timestamp,
'text': text,
'confidence': conf_float
})
except Exception as e:
logger.warning(f"帧识别失败 (时间戳 {timestamp}s): {e}")
continue
logger.info(f"原始识别结果数量: {len(results)}")
# 过滤和清洗结果
filtered_results = self._filter_and_clean(results)
logger.info(f"过滤后结果数量: {len(filtered_results)}")
return filtered_results
def _filter_and_clean(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
过滤和清洗识别结果
Args:
results: 原始识别结果列表
Returns:
List[Dict[str, Any]]: 过滤后的结果列表
"""
if not results:
return []
# 步骤 1: 按置信度过滤
filtered = self._filter_by_confidence(results)
logger.debug(f"置信度过滤后: {len(filtered)}")
# 步骤 2: 过滤噪声(纯符号、单字符等)
filtered = self._filter_noise(filtered)
logger.debug(f"噪声过滤后: {len(filtered)}")
# 步骤 3: 去除重复或相似的文本
filtered = self._remove_duplicates(filtered)
logger.debug(f"去重后: {len(filtered)}")
# 步骤 4: 按时间戳排序
filtered.sort(key=lambda x: x['time'])
return filtered
def _filter_by_confidence(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
按置信度过滤结果
Args:
results: 识别结果列表
Returns:
List[Dict[str, Any]]: 高置信度结果
"""
return [r for r in results if r['confidence'] >= self.CONFIDENCE_THRESHOLD]
def _filter_noise(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
过滤噪声结果
过滤条件:
- 文本长度小于最小长度
- 纯符号或空白字符
Args:
results: 识别结果列表
Returns:
List[Dict[str, Any]]: 过滤后的结果
"""
filtered = []
for result in results:
text = result['text'].strip()
# 过滤过短的文本
if len(text) < self.MIN_SUBTITLE_LENGTH:
continue
# 过滤纯符号或空白字符
if self._is_noise(text):
continue
filtered.append(result)
return filtered
def _is_noise(self, text: str) -> bool:
"""
判断文本是否为噪声
Args:
text: 要判断的文本
Returns:
bool: 如果是噪声返回 True
"""
# 使用正则判断是否只包含符号和空白
try:
# 移除标点符号后检查是否为空
# 先替换常见的中文标点
cleaned = re.sub(r'[^\w\u4e00-\u9fff]', '', text)
return len(cleaned) == 0
except:
# 如果正则出错,保守处理,不算噪声
return False
def _remove_duplicates(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
去除重复或相似的文本
基于编辑距离(Levenshtein distance)判断文本相似度
Args:
results: 识别结果列表(假设已按时间戳排序)
Returns:
List[Dict[str, Any]]: 去重后的结果
"""
if not results:
return []
deduped = [results[0]]
for current in results[1:]:
last = deduped[-1]
# 计算相似度
similarity = self._calculate_similarity(current['text'], last['text'])
# 如果相似度低于阈值,则保留
if similarity < self.SIMILARITY_THRESHOLD:
deduped.append(current)
else:
# 相似文本:保留置信度更高的那个
if current['confidence'] > last['confidence']:
deduped[-1] = current
return deduped
def _calculate_similarity(self, text1: str, text2: str) -> float:
"""
计算两个文本的相似度(基于归一化编辑距离)
Args:
text1: 文本1
text2: 文本2
Returns:
float: 相似度 (0-1),1 表示完全相同
"""
if text1 == text2:
return 1.0
len1, len2 = len(text1), len(text2)
if len1 == 0 or len2 == 0:
return 0.0
# 动态规划计算编辑距离
dp = np.zeros((len1 + 1, len2 + 1))
for i in range(len1 + 1):
dp[i, 0] = i
for j in range(len2 + 1):
dp[0, j] = j
for i in range(1, len1 + 1):
for j in range(1, len2 + 1):
if text1[i-1] == text2[j-1]:
dp[i, j] = dp[i-1, j-1]
else:
dp[i, j] = min(
dp[i-1, j] + 1, # 删除
dp[i, j-1] + 1, # 插入
dp[i-1, j-1] + 1 # 替换
)
edit_distance = dp[len1, len2]
max_len = max(len1, len2)
# 归一化相似度:1 - (编辑距离 / 最大长度)
similarity = 1.0 - (edit_distance / max_len)
return similarity
def get_ocr_info(self) -> Dict[str, Any]:
"""
获取 OCR 处理器信息
Returns:
Dict[str, Any]: OCR 配置信息
"""
return {
'engine': 'easyocr',
'lang': self.lang,
'use_gpu': self.use_gpu,
'detail': self.detail,
'confidence_threshold': self.CONFIDENCE_THRESHOLD,
'similarity_threshold': self.SIMILARITY_THRESHOLD,
'min_subtitle_length': self.MIN_SUBTITLE_LENGTH
}
@@ -0,0 +1,309 @@
"""
OCR 文字识别处理器
使用 PaddleOCR 实现视频帧文字识别功能,主要用于提取硬字幕
"""
import os
import re
import numpy as np
from typing import List, Dict, Any, Tuple
from paddleocr import PaddleOCR
import logging
logger = logging.getLogger(__name__)
class OCRProcessor:
"""
OCR 文字识别处理器
使用 PaddleOCR 从视频帧中识别文字,支持批量识别和结果过滤
"""
# 相似度阈值:用于判断两个文本是否相似(基于编辑距离)
SIMILARITY_THRESHOLD = 0.85
# 置信度阈值
CONFIDENCE_THRESHOLD = 0.7
# 最小字幕长度(字符数)
MIN_SUBTITLE_LENGTH = 2
# 注意:Python re 模块不支持 \p{P} 等属性类,在 _is_noise 方法中使用自定义逻辑
def __init__(self, use_textline_orientation: bool = True, lang: str = 'ch',
use_gpu: bool = False):
"""
初始化 OCR 处理器
Args:
use_textline_orientation: 是否使用文本行方向分类器(PaddleOCR 3.x 参数名)
lang: 语言支持,'ch' 为中英文混合
use_gpu: 是否使用 GPU 加速(通过环境变量设置,如需启用请先设置 CUDA_VISIBLE_DEVICES)
"""
self.use_textline_orientation = use_textline_orientation
self.lang = lang
self.use_gpu = use_gpu
# 初始化 PaddleOCR 实例
self._ocr = None
self._initialize_ocr()
def _initialize_ocr(self):
"""初始化 PaddleOCR 实例"""
try:
# 强制禁用 oneDNN 和 MKL
os.environ['FLAGS_use_mkldnn'] = '0'
os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
os.environ['OMP_NUM_THREADS'] = '1'
os.environ['PADDLE_ON_INFER'] = '1'
# PaddleOCR 3.x 版本参数
self._ocr = PaddleOCR(
use_textline_orientation=self.use_textline_orientation,
lang=self.lang
)
logger.info("PaddleOCR 初始化成功(oneDNN 已禁用)")
except Exception as e:
logger.error(f"PaddleOCR 初始化失败: {e}")
raise RuntimeError(f"无法初始化 PaddleOCR: {e}")
def recognize_frames(self, frames: List[Tuple[np.ndarray, float]]) -> List[Dict[str, Any]]:
"""
批量识别帧中的文字
Args:
frames: 帧列表,每个元素为 (图像数组, 时间戳) 元组
- 图像数组:numpy.ndarray,形状为 (H, W, 3) 的 RGB 图像
- 时间戳:float,单位为秒
Returns:
List[Dict[str, Any]]: 识别结果列表,每个结果包含:
- time: float, 时间戳(秒)
- text: str, 识别的文本
- confidence: float, 置信度 (0-1)
"""
if not frames:
logger.warning("输入帧列表为空")
return []
results = []
for image, timestamp in frames:
try:
# 调用 PaddleOCR 进行识别
# 注意:PaddleOCR 3.x 版本的 ocr() 方法参数已简化
ocr_result = self._ocr.ocr(image)
# PaddleOCR 返回格式: [[[box], (text, confidence)], ...]
if ocr_result and ocr_result[0]:
for line in ocr_result[0]:
box, (text, confidence) = line
if text and confidence:
results.append({
'time': timestamp,
'text': text,
'confidence': float(confidence)
})
except Exception as e:
logger.warning(f"帧识别失败 (时间戳 {timestamp}s): {e}")
continue
logger.info(f"原始识别结果数量: {len(results)}")
# 过滤和清洗结果
filtered_results = self._filter_and_clean(results)
logger.info(f"过滤后结果数量: {len(filtered_results)}")
return filtered_results
def _filter_and_clean(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
过滤和清洗识别结果
Args:
results: 原始识别结果列表
Returns:
List[Dict[str, Any]]: 过滤后的结果列表
"""
if not results:
return []
# 步骤 1: 按置信度过滤
filtered = self._filter_by_confidence(results)
logger.debug(f"置信度过滤后: {len(filtered)}")
# 步骤 2: 过滤噪声(纯符号、单字符等)
filtered = self._filter_noise(filtered)
logger.debug(f"噪声过滤后: {len(filtered)}")
# 步骤 3: 去除重复或相似的文本
filtered = self._remove_duplicates(filtered)
logger.debug(f"去重后: {len(filtered)}")
# 步骤 4: 按时间戳排序
filtered.sort(key=lambda x: x['time'])
return filtered
def _filter_by_confidence(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
按置信度过滤结果
Args:
results: 识别结果列表
Returns:
List[Dict[str, Any]]: 高置信度结果
"""
return [r for r in results if r['confidence'] >= self.CONFIDENCE_THRESHOLD]
def _filter_noise(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
过滤噪声结果
过滤条件:
- 文本长度小于最小长度
- 纯符号或空白字符
Args:
results: 识别结果列表
Returns:
List[Dict[str, Any]]: 过滤后的结果
"""
filtered = []
for result in results:
text = result['text'].strip()
# 过滤过短的文本
if len(text) < self.MIN_SUBTITLE_LENGTH:
continue
# 过滤纯符号或空白字符
if self._is_noise(text):
continue
filtered.append(result)
return filtered
def _is_noise(self, text: str) -> bool:
"""
判断文本是否为噪声
Args:
text: 要判断的文本
Returns:
bool: 如果是噪声返回 True
"""
# 使用正则判断是否只包含符号和空白
# 注意:这里用简单的正则,更复杂的判断可以后续优化
try:
# 移除标点符号后检查是否为空
# 先替换常见的中文标点
cleaned = re.sub(r'[^\w\u4e00-\u9fff]', '', text)
return len(cleaned) == 0
except:
# 如果正则出错,保守处理,不算噪声
return False
def _remove_duplicates(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""
去除重复或相似的文本
基于编辑距离(Levenshtein distance)判断文本相似度
Args:
results: 识别结果列表(假设已按时间戳排序)
Returns:
List[Dict[str, Any]]: 去重后的结果
"""
if not results:
return []
deduped = [results[0]]
for current in results[1:]:
last = deduped[-1]
# 计算相似度
similarity = self._calculate_similarity(current['text'], last['text'])
# 如果相似度低于阈值,则保留
if similarity < self.SIMILARITY_THRESHOLD:
deduped.append(current)
else:
# 相似文本:保留置信度更高的那个
if current['confidence'] > last['confidence']:
deduped[-1] = current
return deduped
def _calculate_similarity(self, text1: str, text2: str) -> float:
"""
计算两个文本的相似度(基于归一化编辑距离)
Args:
text1: 文本1
text2: 文本2
Returns:
float: 相似度 (0-1),1 表示完全相同
"""
if text1 == text2:
return 1.0
len1, len2 = len(text1), len(text2)
if len1 == 0 or len2 == 0:
return 0.0
# 动态规划计算编辑距离
dp = np.zeros((len1 + 1, len2 + 1))
for i in range(len1 + 1):
dp[i, 0] = i
for j in range(len2 + 1):
dp[0, j] = j
for i in range(1, len1 + 1):
for j in range(1, len2 + 1):
if text1[i-1] == text2[j-1]:
dp[i, j] = dp[i-1, j-1]
else:
dp[i, j] = min(
dp[i-1, j] + 1, # 删除
dp[i, j-1] + 1, # 插入
dp[i-1, j-1] + 1 # 替换
)
edit_distance = dp[len1, len2]
max_len = max(len1, len2)
# 归一化相似度:1 - (编辑距离 / 最大长度)
similarity = 1.0 - (edit_distance / max_len)
return similarity
def get_ocr_info(self) -> Dict[str, Any]:
"""
获取 OCR 处理器信息
Returns:
Dict[str, Any]: OCR 配置信息
"""
return {
'use_textline_orientation': self.use_textline_orientation,
'lang': self.lang,
'use_gpu': self.use_gpu,
'confidence_threshold': self.CONFIDENCE_THRESHOLD,
'similarity_threshold': self.SIMILARITY_THRESHOLD,
'min_subtitle_length': self.MIN_SUBTITLE_LENGTH
}
@@ -35,6 +35,10 @@ class DrissionBrowser:
from DrissionPage import ChromiumOptions, ChromiumPage from DrissionPage import ChromiumOptions, ChromiumPage
options = ChromiumOptions() options = ChromiumOptions()
# 设置浏览器路径(服务器环境需要)
options.set_browser_path('/usr/bin/chromium-browser')
if self.headless: if self.headless:
options.headless() options.headless()
options.set_user_agent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36') options.set_user_agent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36')
@@ -116,10 +120,17 @@ class ZhihuDrissionExtractor:
self.browser = DrissionBrowser(headless=self.headless, timeout=self.timeout) self.browser = DrissionBrowser(headless=self.headless, timeout=self.timeout)
self.markdown_converter = MarkdownConverter() self.markdown_converter = MarkdownConverter()
self.cookie_file = os.path.join( # 从配置中读取 Cookie 文件路径,支持环境变量展开
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), # 直接读取 cookie_file(因为 self.config 已经是 zhihu 配置)
'data', 'cookies', 'zhihu.json' cookie_file = self.config.get('cookie_file')
) if cookie_file:
self.cookie_file = os.path.expanduser(cookie_file)
else:
# 默认路径
self.cookie_file = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'data', 'cookies', 'zhihu.json'
)
def identify_url_type(self, url: str) -> Optional[str]: def identify_url_type(self, url: str) -> Optional[str]:
""" """
@@ -180,9 +191,11 @@ class ZhihuDrissionExtractor:
cookies = [] cookies = []
# 1. 尝试从配置加载(优先级最高) # 1. 尝试从配置加载(优先级最高)
if self.config.get('zhihu.cookie_str'): # 支持扁平和嵌套配置结构
cookie_str = self.config.get('cookie_str')
if cookie_str:
try: try:
cookies = json.loads(self.config.get('zhihu.cookie_str')) cookies = json.loads(cookie_str)
print(f"从配置加载 Cookie,共 {len(cookies)} 个") print(f"从配置加载 Cookie,共 {len(cookies)} 个")
except json.JSONDecodeError as e: except json.JSONDecodeError as e:
print(f"配置中的 Cookie JSON 格式错误: {e}") print(f"配置中的 Cookie JSON 格式错误: {e}")
@@ -197,7 +210,16 @@ class ZhihuDrissionExtractor:
# 3. 尝试从文件加载(优先级最低) # 3. 尝试从文件加载(优先级最低)
else: else:
cookie_file = self.config.get('zhihu.cookie_file', self.cookie_file) # 直接从配置中读取 cookie_file(因为 self.config 已经是 zhihu 配置)
cookie_file = self.config.get('cookie_file')
if not cookie_file:
# 如果没有配置,使用默认路径
cookie_file = self.cookie_file
else:
# 展开环境变量(如 ~)
cookie_file = os.path.expanduser(cookie_file)
if cookie_file and Path(cookie_file).exists(): if cookie_file and Path(cookie_file).exists():
try: try:
with open(cookie_file, 'r', encoding='utf-8') as f: with open(cookie_file, 'r', encoding='utf-8') as f:
@@ -206,7 +228,8 @@ class ZhihuDrissionExtractor:
except (json.JSONDecodeError, FileNotFoundError, IOError) as e: except (json.JSONDecodeError, FileNotFoundError, IOError) as e:
print(f"从文件加载 Cookie 失败: {e}") print(f"从文件加载 Cookie 失败: {e}")
else: else:
print("未找到 Cookie 配置,将使用未登录状态访问知乎") print(f"未找到 Cookie 配置,将使用未登录状态访问知乎")
print(f" 期望路径: {cookie_file}")
return cookies return cookies
@@ -0,0 +1,296 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
字幕检测器测试脚本
验证:
1. 预设区域定义
2. 自动字幕区域检测
3. 缓存机制
"""
import sys
import os
import cv2
import numpy as np
# 添加项目路径
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, project_root)
from src.extractors.subtitle_detector import SubtitleDetector
def create_test_frame_with_text(width=1280, height=720):
"""
创建包含模拟字幕的测试帧
Args:
width: 帧宽度
height: 帧高度
Returns:
numpy.ndarray: 测试帧
"""
# 创建黑色背景
frame = np.zeros((height, width, 3), dtype=np.uint8)
# 添加一些噪声背景
noise = np.random.randint(0, 50, (height, width, 3), dtype=np.uint8)
frame = cv2.add(frame, noise)
# 在底部添加白色文字(模拟B站字幕)
font = cv2.FONT_HERSHEY_SIMPLEX
font_scale = 1.0
thickness = 2
# 添加多行字幕
texts = [
"这是测试字幕内容",
"B站硬字幕提取测试",
"00:01:23 第一行字幕",
"00:01:25 第二行字幕"
]
y_start = int(height * 0.85)
line_height = 35
for i, text in enumerate(texts):
y = y_start + i * line_height
x = int(width * 0.15)
# 绘制文字阴影(黑色)
cv2.putText(frame, text, (x + 2, y + 2), font, font_scale, (0, 0, 0), thickness + 2)
# 绘制白色文字
cv2.putText(frame, text, (x, y), font, font_scale, (255, 255, 255), thickness)
return frame
def test_preset_regions():
"""测试1: 预设区域定义"""
print("\n" + "="*60)
print("测试1: 预设区域定义")
print("="*60)
detector = SubtitleDetector()
frame_width, frame_height = 1280, 720
# 测试所有预设区域
for region_name in [detector.REGION_BOTTOM, detector.REGION_CENTER, detector.REGION_TOP]:
try:
x, y, width, height = detector.get_preset_region(region_name, frame_height, frame_width)
print(f"\n✓ 区域 '{region_name}':")
print(f" 坐标: x={x}, y={y}")
print(f" 尺寸: width={width}, height={height}")
print(f" 相对位置: top={y/frame_height:.2f}, bottom={(y+height)/frame_height:.2f}")
except Exception as e:
print(f"✗ 区域 '{region_name}' 失败: {e}")
# 测试无效区域名称
try:
detector.get_preset_region("invalid", frame_height, frame_width)
print("\n✗ 错误: 应该抛出异常但没有")
except ValueError as e:
print(f"\n✓ 无效区域名称正确抛出异常: {e}")
print("\n测试1完成!")
def test_auto_detection():
"""测试2: 自动字幕区域检测"""
print("\n" + "="*60)
print("测试2: 自动字幕区域检测")
print("="*60)
detector = SubtitleDetector()
# 创建测试帧(带字幕)
print("\n创建测试帧...")
test_frame = create_test_frame_with_text()
height, width = test_frame.shape[:2]
print(f"帧尺寸: {width}x{height}")
# 保存测试帧用于可视化
test_frame_path = "/tmp/test_subtitle_frame.png"
cv2.imwrite(test_frame_path, test_frame)
print(f"✓ 测试帧已保存到: {test_frame_path}")
# 执行自动检测
print("\n执行自动检测...")
try:
x, y, w, h = detector.auto_detect_subtitle_region(test_frame, video_id=None, use_cache=False)
print(f"✓ 检测到字幕区域:")
print(f" 坐标: x={x}, y={y}")
print(f" 尺寸: width={w}, height={h}")
print(f" 相对位置: top={y/height:.2f}, bottom={(y+h)/height:.2f}")
# 在测试帧上绘制检测框
frame_with_box = test_frame.copy()
cv2.rectangle(frame_with_box, (x, y), (x+w, y+h), (0, 255, 0), 2)
# 保存结果
result_path = "/tmp/test_subtitle_detected.png"
cv2.imwrite(result_path, frame_with_box)
print(f"✓ 检测结果已保存到: {result_path}")
# 验证检测区域是否合理(应该在底部)
region_bottom = (y + h) / height
if region_bottom > 0.7: # 底部30%
print(f"✓ 检测区域位置合理(底部)")
else:
print(f"! 警告: 检测区域可能不在底部 (bottom={region_bottom:.2f})")
except Exception as e:
print(f"✗ 自动检测失败: {e}")
import traceback
traceback.print_exc()
print("\n测试2完成!")
def test_cache_mechanism():
"""测试3: 缓存机制"""
print("\n" + "="*60)
print("测试3: 缓存机制")
print("="*60)
detector = SubtitleDetector(cache_size=3) # 小缓存便于测试
# 创建测试帧
test_frame = create_test_frame_with_text()
# 测试1: 首次检测并缓存
print("\n1. 首次检测并缓存...")
video_id1 = "BV1xx411c7mD"
region1 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id1, use_cache=True)
print(f"✓ 检测完成,区域: {region1}")
print(f"✓ 缓存大小: {detector.get_cache_size()}")
# 测试2: 再次检测同一视频(应使用缓存)
print("\n2. 再次检测同一视频(应使用缓存)...")
region2 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id1, use_cache=True)
print(f"✓ 缓存结果: {region2}")
if region1 == region2:
print(f"✓ 缓存命中,结果一致")
else:
print(f"✗ 警告: 缓存结果不一致")
# 测试3: 检测不同视频
print("\n3. 检测不同视频...")
video_id2 = "BV1yy411c7mE"
region3 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id2, use_cache=True)
print(f"✓ 新视频检测完成,区域: {region3}")
print(f"✓ 缓存大小: {detector.get_cache_size()}")
# 测试4: 查看所有缓存
print("\n4. 查看所有缓存...")
cached = detector.get_cached_regions()
print(f"✓ 缓存内容:")
for vid, reg in cached.items():
print(f" {vid}: {reg}")
# 测试5: 清空指定视频缓存
print("\n5. 清空指定视频缓存...")
detector.clear_cache(video_id=video_id1)
print(f"✓ 已删除 {video_id1}")
print(f"✓ 缓存大小: {detector.get_cache_size()}")
# 测试6: 清空所有缓存
print("\n6. 清空所有缓存...")
detector.clear_cache()
print(f"✓ 缓存已清空")
print(f"✓ 缓存大小: {detector.get_cache_size()}")
# 测试7: 缓存容量限制(LRU)
print("\n7. 测试缓存容量限制(LRU)...")
for i in range(5):
vid = f"BV{i:012d}"
detector.auto_detect_subtitle_region(test_frame, video_id=vid, use_cache=True)
print(f" 添加视频 {vid},缓存大小: {detector.get_cache_size()}")
cached = detector.get_cached_regions()
print(f"✓ 缓存中有 {len(cached)} 个视频(应为最大容量3)")
print(f"✓ 保留的视频: {list(cached.keys())}")
print("\n测试3完成!")
def test_integration():
"""测试4: 集成测试"""
print("\n" + "="*60)
print("测试4: 集成测试")
print("="*60)
detector = SubtitleDetector()
test_frame = create_test_frame_with_text()
height, width = test_frame.shape[:2]
print("\n模拟B站视频字幕检测流程...")
# 步骤1: 尝试自动检测
print("\n步骤1: 自动检测字幕区域")
video_id = "BV1xx411c7mD"
region = detector.auto_detect_subtitle_region(test_frame, video_id=video_id)
print(f"✓ 自动检测结果: {region}")
# 步骤2: 后续帧使用缓存
print("\n步骤2: 后续帧使用缓存")
for i in range(3):
cached_region = detector.auto_detect_subtitle_region(test_frame, video_id=video_id)
if cached_region == region:
print(f"✓ 帧 {i+1}: 使用缓存,结果一致")
else:
print(f"✗ 帧 {i+1}: 缓存结果异常")
# 步骤3: 对比预设区域
print("\n步骤3: 对比预设区域")
bottom_region = detector.get_preset_region(detector.REGION_BOTTOM, height, width)
print(f" 检测区域: y={region[1]}, height={region[3]}")
print(f" 底部预设: y={bottom_region[1]}, height={bottom_region[3]}")
# 步骤4: 提取字幕区域
print("\n步骤4: 提取字幕区域用于OCR")
x, y, w, h = region
subtitle_roi = test_frame[y:y+h, x:x+w]
subtitle_path = "/tmp/subtitle_roi.png"
cv2.imwrite(subtitle_path, subtitle_roi)
print(f"✓ 字幕ROI已保存到: {subtitle_path}")
print(f" ROI尺寸: {w}x{h}")
print("\n测试4完成!")
def main():
"""主测试函数"""
print("\n" + "="*60)
print("字幕检测器完整测试")
print("="*60)
try:
# 运行所有测试
test_preset_regions()
test_auto_detection()
test_cache_mechanism()
test_integration()
print("\n" + "="*60)
print("所有测试完成!")
print("="*60)
print("\n生成的测试文件:")
print(" - /tmp/test_subtitle_frame.png (原始测试帧)")
print(" - /tmp/test_subtitle_detected.png (带检测框的结果)")
print(" - /tmp/subtitle_roi.png (提取的字幕区域)")
except Exception as e:
print(f"\n✗ 测试失败: {e}")
import traceback
traceback.print_exc()
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""
验证知乎 Cookie 配置
"""
import json
import os
import sys
# 添加 src 目录到 Python 路径
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
from sources.zhihu_drission_extractor import ZhihuDrissionExtractor
def verify_cookie_config():
"""验证 Cookie 配置"""
print("=" * 60)
print("知乎 Cookie 配置验证")
print("=" * 60)
print()
# 1. 检查 Cookie 文件路径
cookie_file = os.path.expanduser('~/.obsidian-kb/zhihu_cookies.json')
print(f"1. Cookie 文件路径")
print(f" {cookie_file}")
if os.path.exists(cookie_file):
print(f" ✅ 文件存在")
else:
print(f" ❌ 文件不存在")
print(f" 请上传 Cookie 文件到此路径")
print()
return False
print()
# 2. 检查 Cookie 文件格式
print(f"2. Cookie 文件格式")
try:
with open(cookie_file, 'r', encoding='utf-8') as f:
cookies = json.load(f)
print(f" ✅ JSON 格式正确")
print(f" Cookie 数量: {len(cookies)}")
except json.JSONDecodeError as e:
print(f" ❌ JSON 格式错误: {e}")
return False
except Exception as e:
print(f" ❌ 读取失败: {e}")
return False
print()
# 3. 检查关键 Cookie
print(f"3. 关键 Cookie 检查")
cookie_names = [c['name'] for c in cookies]
required_cookies = ['z_c0', 'd_c0']
for cookie_name in required_cookies:
if cookie_name in cookie_names:
cookie = next(c for c in cookies if c['name'] == cookie_name)
value_preview = cookie['value'][:10] + '...' if len(cookie['value']) > 10 else cookie['value']
print(f" ✅ {cookie_name}: {value_preview}")
else:
print(f" ⚠️ {cookie_name}: 未找到(可能影响登录状态)")
print()
# 4. 检查配置文件
print(f"4. 配置文件检查")
config_file = os.path.join(os.path.dirname(__file__), 'config.yaml')
if os.path.exists(config_file):
print(f" ✅ config.yaml 存在")
with open(config_file, 'r') as f:
content = f.read()
if 'zhihu_cookies.json' in content:
print(f" ✅ Cookie 路径已配置")
else:
print(f" ⚠️ Cookie 路径未找到(可能是使用了其他配置)")
else:
print(f" ❌ config.yaml 不存在")
print()
# 5. 测试提取器初始化
print(f"5. 提取器初始化测试")
try:
# 加载配置文件并传递给提取器
import yaml
with open(config_file, 'r', encoding='utf-8') as f:
full_config = yaml.safe_load(f)
print(f" 配置文件: {config_file}")
print(f" zhihu 配置: {full_config.get('zhihu', {})}")
# 提取 zhihu 配置并传递
zhihu_config = full_config.get('zhihu', {})
extractor = ZhihuDrissionExtractor(config=zhihu_config)
print(f" ✅ ZhihuDrissionExtractor 初始化成功")
# 测试 Cookie 加载
loaded_cookies = extractor._load_cookies()
print(f" ✅ 成功加载 {len(loaded_cookies)} 个 Cookie")
if len(loaded_cookies) == 0:
print(f" ⚠️ 未加载到 Cookie,将使用未登录状态访问")
print(f" 这可能导致提取失败或内容不完整")
except Exception as e:
print(f" ❌ 初始化失败: {e}")
return False
print()
print("=" * 60)
print("✅ 配置验证完成!")
print("=" * 60)
print()
# 6. 下一步建议
print("下一步:")
print("1. 如果 Cookie 文件格式正确,可以直接开始提取")
print("2. 建议先测试一个简单的知乎回答链接")
print()
print("示例命令:")
print(" python3 -c \"from src.sources.zhihu_drission_extractor import ZhihuDrissionExtractor; print(ZhihuDrissionExtractor().extract('https://www.zhihu.com/question/266309957/answer/2003427891925509507')['title'])\"")
return True
if __name__ == '__main__':
success = verify_cookie_config()
sys.exit(0 if success else 1)
+219
View File
@@ -0,0 +1,219 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
深度测试B站字幕提取 - 尝试所有可能的方法
"""
import asyncio
import json
import re
from playwright.async_api import async_playwright
from bs4 import BeautifulSoup
async def test_all_methods(bvid):
"""尝试所有方法获取字幕"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
print(f"正在测试视频: {url}")
print("=" * 80)
# 等待页面完全加载
print("\n1. 访问页面...")
await page.goto(url, wait_until='networkidle', timeout=60000)
await asyncio.sleep(3)
# 获取基本信息
print("\n2. 获取视频基本信息...")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') == 0:
video_data = data.get('data', {})
print(f" 标题: {video_data.get('title')}")
print(f" aid: {video_data.get('aid')}")
print(f" cid: {video_data.get('cid')}")
cid = video_data.get('cid')
aid = video_data.get('aid')
else:
print(f" ❌ 获取基本信息失败: {data.get('message')}")
await browser.close()
return
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
return
# 方法1: 使用标准 API(官方 CC 字幕)
print("\n3. 方法1: 官方 CC 字幕 API...")
try:
player_url = "https://api.bilibili.com/x/player/v2"
player_response = await page.request.get(player_url, params={'bvid': bvid, 'cid': cid})
player_data = await player_response.json()
if player_data.get('code') == 0:
subtitle_info = player_data.get('data', {}).get('subtitle', {})
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
print(f" ✅ 找到 {len(subtitles_list)} 个官方字幕")
for i, sub in enumerate(subtitles_list):
print(f" [{i+1}] {sub.get('lan_doc')} (id={sub.get('id')})")
# 获取第一个字幕内容
subtitle_url = subtitles_list[0].get('subtitle_url', '')
if subtitle_url:
full_url = f"https:{subtitle_url}"
print(f" URL: {subtitle_url}")
sub_response = await page.request.get(full_url)
if sub_response.ok:
sub_data = await sub_response.json()
body = sub_data.get('body', [])
print(f" ✅ 获取成功,共 {len(body)} 条字幕")
print(f" 示例: {body[0] if body else '无'}")
else:
print(" ❌ 没有找到官方 CC 字幕")
print(f" Subtitle info: {json.dumps(subtitle_info, indent=8, ensure_ascii=False)}")
except Exception as e:
print(f" ❌ 异常: {e}")
# 方法2: 尝试 AI 字幕 API
print("\n4. 方法2: AI 自动生成字幕...")
try:
# B站 AI 字幕可能在另一个 API
ai_subtitle_url = "https://api.bilibili.com/x/player/wbi/v2/playurl"
# 这个通常用于视频流,但也可能包含字幕信息
ai_params = {
'bvid': bvid,
'cid': cid,
'qn': 80, # 清晰度
'fnval': 16, # 包含字幕的标志
}
ai_response = await page.request.get(ai_subtitle_url, params=ai_params)
ai_data = await ai_response.json()
if ai_data.get('code') == 0:
print(f" AI API 响应: {json.dumps(list(ai_data.get('data', {}).keys()), indent=8, ensure_ascii=False)}")
# 检查是否有字幕信息
if 'subtitle' in ai_data.get('data', {}):
subtitle = ai_data['data']['subtitle']
print(f" ✅ 找到 AI 字幕信息")
print(f" {json.dumps(subtitle, indent=8, ensure_ascii=False)}")
except Exception as e:
print(f" ❌ 异常: {e}")
# 方法3: 从页面脚本标签中提取(可能包含字幕配置)
print("\n5. 方法3: 从页面 HTML 中提取字幕配置...")
try:
content = await page.content()
soup = BeautifulSoup(content, 'lxml')
script_tags = soup.find_all('script')
found_subtitle = False
for script in script_tags:
if script.string and ('subtitle' in script.string or 'cc_subtitle' in script.string):
# 提取包含字幕配置的 JSON
matches = re.findall(r'window\.__(?:playinfo|INITIAL_STATE|subtitle__)\s*=\s*({.+?});', script.string)
for match in matches:
try:
data = json.loads(match)
# 查找字幕相关字段
if 'subtitle' in str(data) or 'subtitles' in str(data):
print(f" ✅ 找到字幕配置")
print(f" {json.dumps(data, indent=8, ensure_ascii=False)[:500]}")
found_subtitle = True
break
except:
pass
if found_subtitle:
break
if not found_subtitle:
print(" ❌ 页面中没有找到字幕配置")
except Exception as e:
print(f" ❌ 异常: {e}")
# 方法4: 尝试不同的播放器 API 端点
print("\n6. 方法4: 尝试其他 API 端点...")
endpoints = [
f"https://api.bilibili.com/x/player/wbi/v2/player?bvid={bvid}&cid={cid}",
f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}",
f"https://api.bilibili.com/x/player/playurl?bvid={bvid}&cid={cid}&qn=80&fnval=16",
]
for endpoint in endpoints:
try:
print(f" 测试: {endpoint}")
response = await page.request.get(endpoint)
if response.ok:
data = await response.json()
# 检查响应中是否有字幕信息
json_str = json.dumps(data, ensure_ascii=False)
if 'subtitle' in json_str.lower():
print(f" ✅ 可能包含字幕信息")
# 只显示相关部分
if 'data' in data and 'subtitle' in str(data['data']):
print(f" {json.dumps(data.get('data', {}).get('subtitle', {}), indent=12, ensure_ascii=False)}[:500]")
else:
print(f" 状态码: {response.status}")
except Exception as e:
print(f" 异常: {e}")
# 方法5: 尝试使用 aid 而不是 cid
print("\n7. 方法5: 尝试使用 aid 参数...")
try:
if aid:
player_url = "https://api.bilibili.com/x/player/v2"
response = await page.request.get(player_url, params={'aid': aid, 'bvid': bvid})
data = await response.json()
print(f" 响应码: {data.get('code')}")
if data.get('code') == 0:
subtitle_info = data.get('data', {}).get('subtitle', {})
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
print(f" ✅ 使用 aid 找到 {len(subtitles_list)} 个字幕")
else:
print(f" ❌ 使用 aid 也没有找到字幕")
except Exception as e:
print(f" ❌ 异常: {e}")
# 方法6: 检查视频是否有多 P(分集)
print("\n8. 方法6: 检查是否为分集视频...")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') == 0:
pages = data.get('data', {}).get('pages', [])
print(f" 共 {len(pages)} 个分集")
for i, p in enumerate(pages):
print(f" [{i+1}] cid={p.get('cid')}, page={p.get('page')}, title={p.get('part')}")
# 如果有多个分集,尝试每个分集的字幕
if len(pages) > 1:
print(f"\n 尝试获取第1个分集的字幕...")
first_cid = pages[0].get('cid')
player_url = "https://api.bilibili.com/x/player/v2"
response = await page.request.get(player_url, params={'bvid': bvid, 'cid': first_cid})
data = await response.json()
subtitle_info = data.get('data', {}).get('subtitle', {})
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
print(f" ✅ 第1个分集有 {len(subtitles_list)} 个字幕")
else:
print(f" ❌ 第1个分集也没有字幕")
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
if __name__ == '__main__':
bvid = "BV1KtFzzqEUG"
asyncio.run(test_all_methods(bvid))
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
测试修复后的 B 站字幕提取
"""
import asyncio
import json
import sys
import os
# 添加项目路径
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb'))
from src.extractors.bilibili_extractor import BilibiliExtractor
async def test_extract():
"""测试提取器"""
# 创建提取器实例
extractor = BilibiliExtractor()
# 测试视频
bvid = "BV1yx411s7Jq"
print(f"测试视频: {bvid}")
print("=" * 60)
try:
# 直接调用异步方法
result = await extractor._extract_async(bvid)
print("\n✅ 提取成功!\n")
print("=" * 60)
print("元数据:")
print("=" * 60)
for key, value in result['metadata'].items():
print(f" {key}: {value}")
print("\n" + "=" * 60)
print("字幕:")
print("=" * 60)
subtitle = result['subtitle']
if subtitle:
lines = subtitle.split('\n')
print(f"共 {len(lines)} 条字幕")
print("\n前 10 条字幕:")
print("\n".join(lines[:10]))
if len(lines) > 10:
print(f"\n...(还有 {len(lines) - 10} 条)")
else:
print(" 该视频没有字幕")
except Exception as e:
print(f"\n❌ 测试失败: {e}")
import traceback
traceback.print_exc()
if __name__ == '__main__':
asyncio.run(test_extract())
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
播放视频并截屏,查看字幕
"""
import asyncio
from playwright.async_api import async_playwright
async def play_and_screenshot(bvid):
"""播放视频并截屏"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=False) # 使用有头模式
context = await browser.new_context(viewport={'width': 1920, 'height': 1080})
page = await context.new_page()
print(f"正在访问视频: {url}")
print("=" * 80)
print("\n1. 访问页面...")
await page.goto(url, wait_until='networkidle', timeout=60000)
await asyncio.sleep(3)
print("2. 截取初始状态...")
await page.screenshot(path="/tmp/bilibili_1_initial.png")
print("3. 尝试点击播放按钮...")
try:
# 尝试多个可能的播放按钮选择器
selectors = [
'.bpx-player-ctrl-play',
'.bpx-player-shadow-play-area',
'video',
]
for selector in selectors:
element = await page.query_selector(selector)
if element:
await element.click()
print(f" ✅ 已点击 {selector}")
break
except Exception as e:
print(f" ⚠️ 点击播放失败: {e}")
print("4. 等待视频播放...")
await asyncio.sleep(10)
print("5. 截取播放中状态...")
await page.screenshot(path="/tmp/bilibili_2_playing.png")
print("6. 尝试点击字幕按钮...")
try:
# 查找字幕按钮
subtitle_btn = await page.query_selector('.bpx-player-ctrl-subtitle')
if subtitle_btn:
is_enabled = await subtitle_btn.is_enabled()
print(f" 字幕按钮状态: {'可用' if is_enabled else '不可用'}")
if is_enabled:
await subtitle_btn.click()
print(" ✅ 已点击字幕按钮")
await asyncio.sleep(2)
await page.screenshot(path="/tmp/bilibili_3_subtitle_on.png")
except Exception as e:
print(f" ⚠️ 点击字幕按钮失败: {e}")
print("7. 获取页面文本内容(包含字幕)...")
try:
# 获取所有文本
body_text = await page.evaluate('() => document.body.innerText')
# 查找可能包含字幕的元素
subtitle_elements = await page.query_selector_all('[class*="subtitle"], [class*="caption"]')
print(f" 找到 {len(subtitle_elements)} 个可能的字幕元素")
for i, elem in enumerate(subtitle_elements[:5]):
text = await elem.inner_text()
if text.strip():
print(f" [{i+1}] {text[:100]}")
except Exception as e:
print(f" ⚠️ 获取文本失败: {e}")
print("\n8. 检查页面中是否有动态字幕数据...")
try:
script_result = await page.evaluate('''
() => {
const results = {
hasSubtitlePlugin: false,
subtitleText: '',
captions: [],
};
// 检查播放器实例
const playerContainer = document.querySelector('.bpx-player-container');
if (playerContainer) {
results.playerExists = true;
}
// 查找字幕元素
const captionElements = document.querySelectorAll('.bpx-player-caption');
captionElements.forEach(el => {
results.captions.push(el.innerText);
});
return results;
}
''')
print(f" 检查结果: {script_result}")
except Exception as e:
print(f" ⚠️ 检查失败: {e}")
print("\n9. 等待30秒查看效果...")
await asyncio.sleep(30)
print("10. 截取最终状态...")
await page.screenshot(path="/tmp/bilibili_4_final.png")
print("\n" + "=" * 80)
print("测试完成!请检查以下截图:")
print(" /tmp/bilibili_1_initial.png - 初始状态")
print(" /tmp/bilibili_2_playing.png - 播放中")
print(" /tmp/bilibili_3_subtitle_on.png - 开启字幕后(如果有)")
print(" /tmp/bilibili_4_final.png - 最终状态")
print("=" * 80)
print("\n按回车键关闭浏览器...")
# 等待用户确认(但在自动化环境中会自动继续)
await asyncio.sleep(5)
await browser.close()
if __name__ == '__main__':
bvid = "BV1KtFzzqEUG"
asyncio.run(play_and_screenshot(bvid))
+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
测试动态字幕加载 - 播放视频并监听字幕请求
"""
import asyncio
import json
from playwright.async_api import async_playwright
async def test_dynamic_subtitle(bvid):
"""测试动态字幕加载"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
print(f"正在测试视频: {url}")
print("=" * 80)
# 监听所有网络请求,查找字幕相关的请求
subtitle_requests = []
def handle_request(request):
url = request.url
# 记录可能包含字幕的请求
if 'subtitle' in url.lower() or 'cc' in url.lower():
subtitle_requests.append({
'url': url,
'method': request.method,
'resource_type': request.resource_type
})
print(f"📡 发现字幕请求: {url[:100]}...")
page.on('request', handle_request)
print("\n1. 访问页面并等待加载...")
await page.goto(url, wait_until='networkidle', timeout=60000)
await asyncio.sleep(3)
# 模拟播放视频(点击播放按钮)
print("\n2. 尝试播放视频...")
try:
# 尝试点击播放按钮
play_button = await page.query_selector('.bpx-player-ctrl-play')
if play_button:
await play_button.click()
await asyncio.sleep(5) # 等待视频播放
print(" ✅ 已点击播放按钮")
else:
print(" ⚠️ 未找到播放按钮")
await asyncio.sleep(5) # 无论如何等待一下
except Exception as e:
print(f" ⚠️ 播放异常: {e}")
await asyncio.sleep(5)
# 检查是否有字幕按钮
print("\n3. 检查字幕按钮...")
try:
subtitle_button = await page.query_selector('.bpx-player-ctrl-subtitle')
if subtitle_button:
print(" ✅ 找到字幕按钮")
is_enabled = await subtitle_button.is_enabled()
print(f" 字幕按钮状态: {'可用' if is_enabled else '不可用'}")
# 尝试点击字幕按钮
if is_enabled:
await subtitle_button.click()
await asyncio.sleep(2)
print(" ✅ 已点击字幕按钮")
else:
print(" ❌ 未找到字幕按钮")
except Exception as e:
print(f" ⚠️ 检查字幕按钮异常: {e}")
# 等待更多网络请求
print("\n4. 等待字幕请求...")
await asyncio.sleep(3)
# 显示捕获的字幕请求
print(f"\n5. 捕获的字幕请求 (共 {len(subtitle_requests)} 个):")
if subtitle_requests:
for i, req in enumerate(subtitle_requests):
print(f"\n [{i+1}] {req['method']} {req['resource_type']}")
print(f" {req['url']}")
else:
print(" ❌ 未捕获到字幕请求")
# 尝试从页面中提取字幕文本
print("\n6. 尝试从页面提取字幕文本...")
try:
# B站字幕通常在 .bpx-player-subtitle-item 元素中
subtitle_items = await page.query_selector_all('.bpx-player-subtitle-item')
if subtitle_items:
print(f" ✅ 找到 {len(subtitle_items)} 个字幕元素")
# 获取前几个字幕
for i, item in enumerate(subtitle_items[:5]):
text = await item.inner_text()
print(f" [{i+1}] {text}")
else:
print(" ❌ 未找到字幕元素")
except Exception as e:
print(f" ⚠️ 提取字幕文本异常: {e}")
# 尝试获取所有文本内容(包含动态加载的字幕)
print("\n7. 检查页面中是否有字幕相关的数据...")
try:
# 尝试从 window 对象中获取字幕数据
script_result = await page.evaluate('''
() => {
const results = {};
// 检查不同的位置
if (window.__playinfo__?.data?.subtitle) {
results.playinfo_subtitle = window.__playinfo__.data.subtitle;
}
if (window.__subtitle__ || window.__subtitles__) {
results.subtitle_var = window.__subtitle__ || window.__subtitles__;
}
// 检查播放器实例
const player = document.querySelector('.bpx-player-container')?.__player__;
if (player?.subtitlePlugin) {
results.player_subtitle = Object.keys(player.subtitlePlugin);
}
return results;
}
''')
if script_result:
print(f" 找到的字幕数据: {json.dumps(script_result, indent=8, ensure_ascii=False)}")
else:
print(" 未找到字幕相关的数据")
except Exception as e:
print(f" ⚠️ 检查页面数据异常: {e}")
await browser.close()
print("\n" + "=" * 80)
print("测试完成")
print("=" * 80)
if __name__ == '__main__':
bvid = "BV1KtFzzqEUG"
asyncio.run(test_dynamic_subtitle(bvid))
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
测试B站字幕提取问题
"""
import asyncio
import json
from playwright.async_api import async_playwright
async def test_subtitle_api(bvid):
"""测试字幕API"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
await page.goto(url, wait_until='domcontentloaded')
await asyncio.sleep(2)
# 方法1: 获取 aid
print("=== 测试方法1: 使用 aid ===")
try:
aid = await page.evaluate('() => window.__INITIAL_STATE__?.videoData?.aid')
print(f"aid: {aid}")
if aid:
api_url = "https://api.bilibili.com/x/player/v2"
response = await page.request.get(api_url, params={'aid': aid, 'bvid': bvid})
data = await response.json()
print(f"Response code: {data.get('code')}")
print(f"Subtitle info: {json.dumps(data.get('data', {}).get('subtitle', {}), indent=2, ensure_ascii=False)}")
except Exception as e:
print(f"方法1失败: {e}")
# 方法2: 获取 cid
print("\n=== 测试方法2: 使用 cid ===")
try:
cid = await page.evaluate('() => window.__INITIAL_STATE__?.videoData?.cid')
print(f"cid: {cid}")
if cid:
api_url = "https://api.bilibili.com/x/player/v2"
response = await page.request.get(api_url, params={'bvid': bvid, 'cid': cid})
data = await response.json()
print(f"Response code: {data.get('code')}")
print(f"Subtitle info: {json.dumps(data.get('data', {}).get('subtitle', {}), indent=2, ensure_ascii=False)}")
# 如果有字幕,尝试获取字幕内容
subtitle_info = data.get('data', {}).get('subtitle', {})
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
subtitle_url = subtitles_list[0].get('subtitle_url', '')
print(f"\nSubtitle URL: {subtitle_url}")
full_url = f"https:{subtitle_url}"
response = await page.request.get(full_url)
if response.ok:
sub_data = await response.json()
print(f"First 3 subtitles:")
for i, item in enumerate(sub_data.get('body', [])[:3]):
print(f" [{i+1}] {item}")
except Exception as e:
print(f"方法2失败: {e}")
# 方法3: 从页面直接提取
print("\n=== 测试方法3: 从页面直接提取 ===")
try:
subtitle_data = await page.evaluate('() => window.__INITIAL_STATE__?.videoData?.subtitle')
print(f"Subtitle data from page: {json.dumps(subtitle_data, indent=2, ensure_ascii=False)}")
except Exception as e:
print(f"方法3失败: {e}")
await browser.close()
if __name__ == '__main__':
# 测试一个有字幕的视频
bvid = "BV1MnFNz5EDL" # 可以换成其他视频
asyncio.run(test_subtitle_api(bvid))
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
测试B站字幕提取问题 - 改进版
"""
import asyncio
import json
import re
from playwright.async_api import async_playwright
from bs4 import BeautifulSoup
async def test_subtitle_api(bvid):
"""测试字幕API"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
print("正在访问页面...")
await page.goto(url, wait_until='networkidle', timeout=60000)
await asyncio.sleep(3)
# 方法1: 检查 __INITIAL_STATE__ 是否存在
print("\n=== 测试方法1: 检查 __INITIAL_STATE__ 结构 ===")
try:
initial_state = await page.evaluate('() => window.__INITIAL_STATE__')
if initial_state:
print(f"__INITIAL_STATE__ keys: {list(initial_state.keys())}")
video_data = initial_state.get('videoData', {})
print(f"videoData keys: {list(video_data.keys())}")
print(f"aid: {video_data.get('aid')}")
print(f"cid: {video_data.get('cid')}")
else:
print("__INITIAL_STATE__ 不存在")
except Exception as e:
print(f"方法1失败: {e}")
# 方法2: 从 HTML 中提取
print("\n=== 测试方法2: 从 HTML script 标签提取 ===")
try:
content = await page.content()
soup = BeautifulSoup(content, 'lxml')
script_tags = soup.find_all('script')
for script in script_tags:
if script.string and '__INITIAL_STATE__' in script.string:
match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.+?});', script.string)
if match:
data = json.loads(match.group(1))
video_data = data.get('videoData', {})
print(f"aid: {video_data.get('aid')}")
print(f"cid: {video_data.get('cid')}")
print(f"subtitle: {video_data.get('subtitle')}")
# 如果找到了 cid,尝试获取字幕
cid = video_data.get('cid')
if cid:
await test_subtitle_api_with_cid(page, bvid, cid)
break
except Exception as e:
print(f"方法2失败: {e}")
# 方法3: 使用 API 获取视频信息
print("\n=== 测试方法3: 使用 API 获取视频信息 ===")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') == 0:
video_data = data.get('data', {})
print(f"aid: {video_data.get('aid')}")
print(f"cid: {video_data.get('cid')}")
print(f"title: {video_data.get('title')}")
cid = video_data.get('cid')
if cid:
await test_subtitle_api_with_cid(page, bvid, cid)
except Exception as e:
print(f"方法3失败: {e}")
await browser.close()
async def test_subtitle_api_with_cid(page, bvid, cid):
"""使用 cid 测试字幕 API"""
print(f"\n=== 使用 cid={cid} 获取字幕 ===")
try:
api_url = "https://api.bilibili.com/x/player/v2"
response = await page.request.get(api_url, params={'bvid': bvid, 'cid': cid})
data = await response.json()
print(f"Response code: {data.get('code')}")
if data.get('code') == 0:
player_data = data.get('data', {})
subtitle_info = player_data.get('subtitle', {})
print(f"Subtitle info:")
print(f" allow_submit: {subtitle_info.get('allow_submit')}")
print(f" subtitles count: {len(subtitle_info.get('subtitles', []))}")
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
print(f"\nFound {len(subtitles_list)} subtitles:")
for i, sub in enumerate(subtitles_list[:3]):
print(f" [{i+1}] {sub.get('id')} - {sub.get('lan_doc')}")
# 获取第一个字幕内容
subtitle_url = subtitles_list[0].get('subtitle_url', '')
print(f"\nFetching subtitle from: {subtitle_url}")
full_url = f"https:{subtitle_url}"
response = await page.request.get(full_url)
if response.ok:
sub_data = await response.json()
print(f"First 3 subtitle items:")
for i, item in enumerate(sub_data.get('body', [])[:3]):
print(f" [{i+1}] [{item.get('from')} - {item.get('to')}] {item.get('content')}")
else:
print(f"Failed to fetch subtitle: {response.status}")
else:
print("No subtitles found")
else:
print(f"API error: {data.get('message')}")
except Exception as e:
print(f"Failed: {e}")
if __name__ == '__main__':
# 测试一个有字幕的视频
bvid = "BV1MnFNz5EDL"
asyncio.run(test_subtitle_api(bvid))
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
测试B站字幕提取 - 测试多个视频
"""
import asyncio
import json
from playwright.async_api import async_playwright
async def test_video(bvid):
"""测试单个视频"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
# 使用 API 获取视频信息
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') == 0:
video_data = data.get('data', {})
title = video_data.get('title', '')
cid = video_data.get('cid', '')
print(f"\n视频: {title}")
print(f"BV号: {bvid}")
print(f"cid: {cid}")
# 获取字幕
if cid:
player_url = "https://api.bilibili.com/x/player/v2"
player_response = await page.request.get(player_url, params={'bvid': bvid, 'cid': cid})
player_data = await player_response.json()
if player_data.get('code') == 0:
subtitle_info = player_data.get('data', {}).get('subtitle', {})
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
print(f"✅ 找到 {len(subtitles_list)} 个字幕")
for i, sub in enumerate(subtitles_list[:3]):
print(f" [{i+1}] {sub.get('lan_doc')} - {sub.get('subtitle_url')}")
# 获取第一个字幕内容
subtitle_url = subtitles_list[0].get('subtitle_url', '')
if subtitle_url:
full_url = f"https:{subtitle_url}"
sub_response = await page.request.get(full_url)
if sub_response.ok:
sub_data = await sub_response.json()
body = sub_data.get('body', [])
print(f"\n字幕示例 (前3条):")
for i, item in enumerate(body[:3]):
from_time = item.get('from', 0)
to_time = item.get('to', 0)
content = item.get('content', '')
print(f" [{i+1}] [{from_time:.1f} - {to_time:.1f}] {content}")
else:
print(f"❌ 获取字幕内容失败: {sub_response.status}")
else:
print("❌ 该视频没有字幕")
else:
print(f"❌ 获取视频信息失败: {data.get('message')}")
await browser.close()
async def main():
"""测试多个视频"""
# 测试视频列表(包含不同类型的视频)
test_videos = [
"BV1XX4y1C7tF", # 教程类视频,可能有字幕
"BV1GJ411x7h7", # TED演讲,通常有字幕
"BV1uA411H7K4", # 科技类视频
"BV1MnFNz5EDL", # 之前的测试视频
]
for bvid in test_videos:
await test_video(bvid)
await asyncio.sleep(1)
if __name__ == '__main__':
asyncio.run(main())
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
测试B站字幕提取 - 使用确定有字幕的视频
"""
import asyncio
import json
from playwright.async_api import async_playwright
async def test_with_cookie(bvid, cookie=None):
"""测试带 Cookie 的字幕提取"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
# 如果有 cookie,添加到上下文
if cookie:
await context.add_cookies([cookie])
print("使用已登录 Cookie")
# 使用 API 获取视频信息
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') == 0:
video_data = data.get('data', {})
title = video_data.get('title', '')
cid = video_data.get('cid', '')
print(f"\n视频: {title}")
print(f"BV号: {bvid}")
print(f"cid: {cid}")
# 获取字幕
if cid:
player_url = "https://api.bilibili.com/x/player/v2"
player_response = await page.request.get(player_url, params={'bvid': bvid, 'cid': cid})
player_data = await player_response.json()
print(f"Player API 响应码: {player_data.get('code')}")
if player_data.get('code') == 0:
subtitle_info = player_data.get('data', {}).get('subtitle', {})
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
print(f"✅ 找到 {len(subtitles_list)} 个字幕")
for i, sub in enumerate(subtitles_list):
print(f" [{i+1}] 语言: {sub.get('lan_doc')}, ID: {sub.get('id')}")
print(f" URL: {sub.get('subtitle_url')}")
# 获取第一个字幕内容
subtitle_url = subtitles_list[0].get('subtitle_url', '')
if subtitle_url:
full_url = f"https:{subtitle_url}"
print(f"\n正在获取字幕内容: {full_url}")
sub_response = await page.request.get(full_url)
print(f"字幕请求状态: {sub_response.status}")
if sub_response.ok:
sub_data = await sub_response.json()
body = sub_data.get('body', [])
print(f"\n字幕内容示例 (前5条):")
for i, item in enumerate(body[:5]):
from_time = item.get('from', 0)
to_time = item.get('to', 0)
content = item.get('content', '')
print(f" [{i+1}] [{from_time:.1f}s - {to_time:.1f}s] {content}")
print(f"\n✅ 字幕获取成功!共 {len(body)} 条字幕")
else:
print(f"❌ 获取字幕内容失败: {sub_response.status}")
print(f"响应头: {sub_response.headers}")
else:
print("❌ 该视频没有字幕")
print(f"Subtitle info: {json.dumps(subtitle_info, indent=2, ensure_ascii=False)}")
else:
print(f"❌ Player API 失败: {player_data.get('message')}")
print(f"完整响应: {json.dumps(player_data, indent=2, ensure_ascii=False)}")
else:
print(f"❌ 获取视频信息失败: {data.get('message')}")
await browser.close()
async def main():
"""测试多个视频"""
# 罗翔的视频通常有官方字幕
test_videos = [
"BV1GJ411x7h7", # 罗翔 - 刑法
"BV1yx411s7Jq", # 可能的官方视频
"BV1zv4y1m7Qp", # B站官方视频
]
for bvid in test_videos:
print(f"\n{'='*60}")
print(f"测试视频: {bvid}")
print(f"{'='*60}")
try:
await test_with_cookie(bvid)
except Exception as e:
print(f"❌ 测试失败: {e}")
import traceback
traceback.print_exc()
await asyncio.sleep(2)
if __name__ == '__main__':
asyncio.run(main())
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
测试 B站 API 获取视频流
"""
import asyncio
import json
from playwright.async_api import async_playwright
async def test_video_stream(bvid):
"""测试获取视频流"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
print(f"正在测试视频: {url}")
print("=" * 80)
# 首先获取 aid 和 cid
print("\n1. 获取视频基本信息...")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') == 0:
video_data = data.get('data', {})
print(f" 标题: {video_data.get('title')}")
aid = video_data.get('aid')
cid = video_data.get('cid')
print(f" aid: {aid}")
print(f" cid: {cid}")
else:
print(f" ❌ 失败: {data.get('message')}")
await browser.close()
return
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
return
# 尝试获取视频流 URL
print("\n2. 尝试获取视频流 URL...")
try:
playurl_api = "https://api.bilibili.com/x/player/wbi/playurl"
params = {
'bvid': bvid,
'cid': cid,
'qn': 80, # 清晰度:80=1080P
'fnval': 16, # 包含 DASH 格式
'fnver': 0,
'fourk': 1,
}
playurl_response = await page.request.get(playurl_api, params=params)
print(f" 状态码: {playurl_response.status}")
if playurl_response.ok:
playurl_data = await playurl_response.json()
print(f" 响应码: {playurl_data.get('code')}")
if playurl_data.get('code') == 0:
play_data = playurl_data.get('data', {})
print(f"\n 数据类型:")
print(f" {json.dumps(list(play_data.keys()), indent=4, ensure_ascii=False)}")
# 检查 DASH 格式
if 'dash' in play_data:
dash = play_data['dash']
print(f"\n DASH 格式:")
print(f" duration: {dash.get('duration')}s")
print(f" video streams: {len(dash.get('video', []))}")
print(f" audio streams: {len(dash.get('audio', []))}")
# 显示视频流信息
print(f"\n 视频流列表:")
for i, video_stream in enumerate(dash.get('video', [])[:3]):
print(f" [{i+1}] id={video_stream.get('id')}, "
f"bandwidth={video_stream.get('bandwidth')}, "
f"codecs={video_stream.get('codecs')}")
print(f" baseUrl: {video_stream.get('baseUrl', '')[:100]}...")
print(f" backup_url: {len(video_stream.get('backup_url', []))} 个")
# 显示音频流信息
print(f"\n 音频流列表:")
for i, audio_stream in enumerate(dash.get('audio', [])[:3]):
print(f" [{i+1}] id={audio_stream.get('id')}, "
f"bandwidth={audio_stream.get('bandwidth')}, "
f"codecs={audio_stream.get('codecs')}")
print(f" baseUrl: {audio_stream.get('baseUrl', '')[:100]}...")
# 尝试下载一小段测试
if dash.get('video'):
first_video = dash['video'][0]
base_url = first_video.get('baseUrl', '')
if base_url:
print(f"\n3. 测试下载视频流...")
print(f" URL: {base_url}")
# 添加必要的请求头
headers = {
'Referer': 'https://www.bilibili.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
}
# 只下载前 1KB 测试
test_response = await page.request.get(
base_url,
headers=headers,
timeout=30000
)
print(f" 下载状态: {test_response.status}")
if test_response.ok:
content_length = len(await test_response.body())
print(f" ✅ 可以下载!获取到 {content_length} 字节")
print(f" Content-Type: {test_response.headers.get('content-type')}")
else:
print(f" ❌ 下载失败")
else:
print(f" ❌ API 错误: {playurl_data.get('message')}")
print(f" 完整响应: {json.dumps(playurl_data, indent=4, ensure_ascii=False)[:500]}")
else:
print(f" ❌ HTTP 错误: {playurl_response.status}")
except Exception as e:
print(f" ❌ 异常: {e}")
import traceback
traceback.print_exc()
# 尝试旧版 API
print("\n4. 尝试旧版 playurl API...")
try:
playurl_api = "https://api.bilibili.com/x/player/playurl"
params = {
'bvid': bvid,
'cid': cid,
'qn': 80,
'otype': 'json',
}
playurl_response = await page.request.get(playurl_api, params=params)
print(f" 状态码: {playurl_response.status}")
if playurl_response.ok:
playurl_data = await playurl_response.json()
print(f" 响应码: {playurl_data.get('code')}")
if playurl_data.get('code') == 0:
print(f" ✅ 旧版 API 可用")
print(f" 数据: {json.dumps(list(playurl_data.get('data', {}).keys()), indent=4, ensure_ascii=False)}")
else:
print(f" ❌ API 错误: {playurl_data.get('message')}")
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
if __name__ == '__main__':
bvid = "BV1KtFzzqEUG"
asyncio.run(test_video_stream(bvid))
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
测试 B站 WBI API 字幕提取
"""
import asyncio
import json
from playwright.async_api import async_playwright
async def test_wbi_subtitle(bvid):
"""测试 WBI API 字幕"""
url = f"https://www.bilibili.com/video/{bvid}"
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context()
page = await context.new_page()
print(f"正在测试视频: {url}")
print("=" * 80)
# 首先获取 aid 和 cid
print("\n1. 获取视频基本信息...")
try:
api_url = "https://api.bilibili.com/x/web-interface/view"
response = await page.request.get(api_url, params={'bvid': bvid})
data = await response.json()
if data.get('code') == 0:
video_data = data.get('data', {})
print(f" 标题: {video_data.get('title')}")
aid = video_data.get('aid')
cid = video_data.get('cid')
print(f" aid: {aid}")
print(f" cid: {cid}")
else:
print(f" ❌ 失败: {data.get('message')}")
await browser.close()
return
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
return
# 使用 WBI API 获取播放器信息(包含字幕)
print("\n2. 使用 WBI API 获取播放器信息...")
try:
wbi_url = "https://api.bilibili.com/x/player/wbi/v2"
wbi_response = await page.request.get(wbi_url, params={'aid': aid, 'cid': cid})
print(f" 状态码: {wbi_response.status}")
if wbi_response.ok:
wbi_data = await wbi_response.json()
print(f" 响应码: {wbi_data.get('code')}")
if wbi_data.get('code') == 0:
player_data = wbi_data.get('data', {})
print(f"\n 数据结构:")
print(f" {json.dumps(list(player_data.keys()), indent=4, ensure_ascii=False)}")
# 检查字幕
subtitle_info = player_data.get('subtitle', {})
print(f"\n 字幕信息:")
print(f" {json.dumps(subtitle_info, indent=4, ensure_ascii=False)}")
subtitles_list = subtitle_info.get('subtitles', [])
if subtitles_list:
print(f"\n ✅ 找到 {len(subtitles_list)} 个字幕!")
for i, sub in enumerate(subtitles_list):
print(f" [{i+1}] {sub.get('lan_doc')} (lan={sub.get('id')}, type={sub.get('id_type')})")
print(f" URL: {sub.get('subtitle_url')}")
# 获取字幕内容
subtitle_url = sub.get('subtitle_url', '')
if subtitle_url:
full_url = f"https:{subtitle_url}"
print(f"\n 正在获取字幕内容...")
sub_response = await page.request.get(full_url)
if sub_response.ok:
sub_data = await sub_response.json()
body = sub_data.get('body', [])
print(f" ✅ 获取成功,共 {len(body)} 条字幕")
print(f" 前3条字幕:")
for j, item in enumerate(body[:3]):
from_time = item.get('from', 0)
to_time = item.get('to', 0)
content = item.get('content', '')
print(f" [{j+1}] [{from_time:.1f}s - {to_time:.1f}s] {content}")
else:
print(f" ❌ 获取失败: {sub_response.status}")
else:
print(f"\n ❌ 没有找到字幕")
print(f" allow_submit: {subtitle_info.get('allow_submit')}")
else:
print(f" ❌ API 错误: {wbi_data.get('message')}")
else:
print(f" ❌ HTTP 错误: {wbi_response.status}")
except Exception as e:
print(f" ❌ 异常: {e}")
import traceback
traceback.print_exc()
# 尝试不使用 wbi,使用旧版本 API
print("\n3. 尝试旧版 API (v2)...")
try:
old_url = "https://api.bilibili.com/x/player/v2"
old_response = await page.request.get(old_url, params={'bvid': bvid, 'cid': cid})
print(f" 状态码: {old_response.status}")
if old_response.ok:
old_data = await old_response.json()
subtitle_info = old_data.get('data', {}).get('subtitle', {})
subtitles_list = subtitle_info.get('subtitles', [])
print(f" 字幕数量: {len(subtitles_list)}")
if subtitles_list:
print(f" ✅ 旧版 API 找到字幕!")
for sub in subtitles_list:
print(f" {sub.get('lan_doc')}")
except Exception as e:
print(f" ❌ 异常: {e}")
await browser.close()
if __name__ == '__main__':
bvid = "BV1KtFzzqEUG"
asyncio.run(test_wbi_subtitle(bvid))