Update obsidian-kb skill and add find-skills
- obsidian-kb: Add OCR processors (easyocr, tesseract) - obsidian-kb: Fix subtitle detection with test files - obsidian-kb: Add cookie verification script - Add find-skills skill for discovering capabilities - Add video processing scripts - Update clawhub lock file
This commit is contained in:
OpenClaw Agent
@@ -60,6 +60,10 @@
|
||||
"obsidian-kb": {
|
||||
"version": "1.0.0",
|
||||
"installedAt": 1738958400000
|
||||
},
|
||||
"find-skills": {
|
||||
"version": "0.1.0",
|
||||
"installedAt": 1770607549813
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
Binary file not shown.
|
After Width: | Height: | Size: 31 KiB |
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
使用旧版 API 获取并下载视频流
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def download_video_stream(bvid, output_dir="/tmp/bilibili_downloads"):
|
||||
"""下载视频流"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
# 创建输出目录
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
print(f"正在下载视频: {url}")
|
||||
print("=" * 80)
|
||||
|
||||
# 首先获取 aid 和 cid
|
||||
print("\n1. 获取视频信息...")
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') != 0:
|
||||
print(f" ❌ 失败: {data.get('message')}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
video_data = data.get('data', {})
|
||||
title = video_data.get('title')
|
||||
aid = video_data.get('aid')
|
||||
cid = video_data.get('cid')
|
||||
duration = video_data.get('duration')
|
||||
print(f" 标题: {title}")
|
||||
print(f" 时长: {duration}秒")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 获取视频流 URL
|
||||
print("\n2. 获取视频流 URL...")
|
||||
try:
|
||||
playurl_api = "https://api.bilibili.com/x/player/playurl"
|
||||
params = {
|
||||
'bvid': bvid,
|
||||
'cid': cid,
|
||||
'qn': 80, # 1080P
|
||||
'otype': 'json',
|
||||
'fnver': 0,
|
||||
'fnval': 0, # FLV 格式
|
||||
}
|
||||
|
||||
playurl_response = await page.request.get(playurl_api, params=params)
|
||||
|
||||
if not playurl_response.ok:
|
||||
print(f" ❌ HTTP 错误: {playurl_response.status}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
playurl_data = await playurl_response.json()
|
||||
|
||||
if playurl_data.get('code') != 0:
|
||||
print(f" ❌ API 错误: {playurl_data.get('message')}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
play_data = playurl_data.get('data', {})
|
||||
|
||||
# 检查支持的清晰度
|
||||
print(f"\n 支持的清晰度:")
|
||||
for i, (q, desc) in enumerate(zip(play_data.get('accept_quality', []), play_data.get('accept_description', []))):
|
||||
current = " (当前)" if q == play_data.get('quality') else ""
|
||||
print(f" {desc} ({q}){current}")
|
||||
|
||||
# 获取视频 URL
|
||||
durl = play_data.get('durl', [])
|
||||
if not durl:
|
||||
print(f" ❌ 没有找到视频流")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
print(f"\n 找到 {len(durl)} 个视频分段")
|
||||
video_urls = []
|
||||
for i, segment in enumerate(durl):
|
||||
segment_url = segment.get('url')
|
||||
segment_size = segment.get('size', 0)
|
||||
video_urls.append(segment_url)
|
||||
print(f" 分段 {i+1}: {segment_size / 1024 / 1024:.2f}MB")
|
||||
print(f" {segment_url[:100]}...")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 下载视频
|
||||
print(f"\n3. 开始下载视频...")
|
||||
all_segments = []
|
||||
total_size = 0
|
||||
|
||||
for i, video_url in enumerate(video_urls, 1):
|
||||
print(f"\n 下载分段 {i}/{len(video_urls)}...")
|
||||
|
||||
try:
|
||||
# 添加必要的请求头
|
||||
headers = {
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
}
|
||||
|
||||
# 下载分段
|
||||
segment_response = await page.request.get(
|
||||
video_url,
|
||||
headers=headers,
|
||||
timeout=60000
|
||||
)
|
||||
|
||||
if not segment_response.ok:
|
||||
print(f" ❌ 下载失败: {segment_response.status}")
|
||||
continue
|
||||
|
||||
segment_data = await segment_response.body()
|
||||
segment_size = len(segment_data)
|
||||
total_size += segment_size
|
||||
all_segments.append(segment_data)
|
||||
|
||||
print(f" ✅ 下载成功: {segment_size / 1024 / 1024:.2f}MB")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 下载异常: {e}")
|
||||
continue
|
||||
|
||||
if not all_segments:
|
||||
print(f"\n ❌ 没有成功下载任何分段")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 合并分段
|
||||
print(f"\n4. 合并视频分段...")
|
||||
video_data = b''.join(all_segments)
|
||||
|
||||
# 保存文件
|
||||
safe_title = "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
|
||||
output_path = os.path.join(output_dir, f"{safe_title}.flv")
|
||||
|
||||
with open(output_path, 'wb') as f:
|
||||
f.write(video_data)
|
||||
|
||||
print(f" ✅ 视频已保存: {output_path}")
|
||||
print(f" 文件大小: {total_size / 1024 / 1024:.2f}MB")
|
||||
|
||||
await browser.close()
|
||||
|
||||
return output_path
|
||||
|
||||
if __name__ == '__main__':
|
||||
bvid = "BV1KtFzzqEUG"
|
||||
result = asyncio.run(download_video_stream(bvid))
|
||||
if result:
|
||||
print(f"\n✅ 下载完成！")
|
||||
print(f"文件路径: {result}")
|
||||
else:
|
||||
print(f"\n❌ 下载失败")
|
||||
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
下载视频片段用于 OCR 提取硬字幕
|
||||
策略：每隔 N 秒截取一帧，而不是下载整个视频
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import cv2
|
||||
import numpy as np
|
||||
from playwright.async_api import async_playwright
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
async def extract_frames_for_ocr(bvid, output_dir="/tmp/bilibili_frames", interval_seconds=10, max_frames=200):
|
||||
"""
|
||||
提取视频帧用于 OCR
|
||||
|
||||
Args:
|
||||
bvid: B站视频 BV 号
|
||||
output_dir: 输出目录
|
||||
interval_seconds: 截帧间隔（秒）
|
||||
max_frames: 最大截取帧数
|
||||
|
||||
Returns:
|
||||
str: 输出目录路径
|
||||
"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
# 创建输出目录
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
print(f"正在提取视频帧: {url}")
|
||||
print(f"间隔: {interval_seconds}秒，最大帧数: {max_frames}")
|
||||
print("=" * 80)
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
# 获取视频信息
|
||||
print("\n1. 获取视频信息...")
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') != 0:
|
||||
print(f" ❌ 失败: {data.get('message')}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
video_data = data.get('data', {})
|
||||
title = video_data.get('title')
|
||||
aid = video_data.get('aid')
|
||||
cid = video_data.get('cid')
|
||||
duration = video_data.get('duration')
|
||||
print(f" 标题: {title}")
|
||||
print(f" 时长: {duration}秒")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 获取视频流 URL
|
||||
print("\n2. 获取视频流 URL...")
|
||||
try:
|
||||
playurl_api = "https://api.bilibili.com/x/player/playurl"
|
||||
params = {
|
||||
'bvid': bvid,
|
||||
'cid': cid,
|
||||
'qn': 80, # 1080P
|
||||
'otype': 'json',
|
||||
'fnver': 0,
|
||||
'fnval': 0, # FLV 格式
|
||||
}
|
||||
|
||||
playurl_response = await page.request.get(playurl_api, params=params)
|
||||
|
||||
if not playurl_response.ok or playurl_response.status != 200:
|
||||
print(f" ❌ HTTP 错误: {playurl_response.status}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
playurl_data = await playurl_response.json()
|
||||
|
||||
if playurl_data.get('code') != 0:
|
||||
print(f" ❌ API 错误: {playurl_data.get('message')}")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
play_data = playurl_data.get('data', {})
|
||||
durl = play_data.get('durl', [])
|
||||
|
||||
if not durl:
|
||||
print(f" ❌ 没有找到视频流")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 获取第一个分段的 URL
|
||||
first_segment_url = durl[0].get('url')
|
||||
print(f" ✅ 获取到视频流 URL")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 使用 OpenCV 直接从 URL 读取视频并提取帧
|
||||
print(f"\n3. 提取视频帧...")
|
||||
|
||||
try:
|
||||
# 添加请求头
|
||||
headers = {
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
}
|
||||
|
||||
# 使用 urllib 下载视频到临时文件（只下载前 50MB）
|
||||
print(f" 正在下载视频（前 50MB）...")
|
||||
|
||||
req = Request(first_segment_url, headers=headers)
|
||||
|
||||
# 下载视频数据（限制 50MB）
|
||||
chunk_size = 8192
|
||||
max_bytes = 50 * 1024 * 1024 # 50MB
|
||||
video_data = bytearray()
|
||||
|
||||
with urlopen(req, timeout=60) as response:
|
||||
while True:
|
||||
chunk = response.read(chunk_size)
|
||||
if not chunk or len(video_data) >= max_bytes:
|
||||
break
|
||||
video_data.extend(chunk)
|
||||
|
||||
print(f" 已下载 {len(video_data) / 1024 / 1024:.2f}MB")
|
||||
|
||||
# 保存为临时文件
|
||||
temp_video_path = os.path.join(output_dir, "temp_video.flv")
|
||||
with open(temp_video_path, 'wb') as f:
|
||||
f.write(bytes(video_data))
|
||||
|
||||
# 使用 OpenCV 打开视频
|
||||
cap = cv2.VideoCapture(temp_video_path)
|
||||
|
||||
if not cap.isOpened():
|
||||
print(f" ❌ 无法打开视频文件")
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
# 获取视频帧率
|
||||
fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
print(f" 帧率: {fps:.2f} fps")
|
||||
print(f" 总帧数: {total_frames}")
|
||||
|
||||
# 计算需要提取的帧
|
||||
interval_frames = int(fps * interval_seconds)
|
||||
frame_count = 0
|
||||
extracted_count = 0
|
||||
|
||||
print(f"\n 开始提取帧...")
|
||||
|
||||
while True:
|
||||
# 跳到目标帧
|
||||
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_count)
|
||||
|
||||
# 读取帧
|
||||
ret, frame = cap.read()
|
||||
|
||||
if not ret:
|
||||
break
|
||||
|
||||
# 保存帧
|
||||
frame_path = os.path.join(output_dir, f"frame_{extracted_count:04d}_{frame_count}.jpg")
|
||||
cv2.imwrite(frame_path, frame)
|
||||
extracted_count += 1
|
||||
|
||||
print(f" 提取第 {extracted_count} 帧 (时间: {frame_count / fps:.1f}s)")
|
||||
|
||||
# 检查是否达到最大帧数
|
||||
if extracted_count >= max_frames:
|
||||
print(f" 已达到最大帧数限制")
|
||||
break
|
||||
|
||||
# 跳到下一帧
|
||||
frame_count += interval_frames
|
||||
|
||||
cap.release()
|
||||
|
||||
# 删除临时视频文件
|
||||
os.remove(temp_video_path)
|
||||
|
||||
print(f"\n4. 完成！")
|
||||
print(f" 成功提取 {extracted_count} 帧图像")
|
||||
print(f" 输出目录: {output_dir}")
|
||||
|
||||
await browser.close()
|
||||
return output_dir
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 提取帧失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
await browser.close()
|
||||
return None
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
|
||||
bvid = "BV1KtFzzqEUG"
|
||||
|
||||
result = asyncio.run(extract_frames_for_ocr(
|
||||
bvid,
|
||||
interval_seconds=10, # 每 10 秒提取一帧
|
||||
max_frames=100 # 最多提取 100 帧
|
||||
))
|
||||
|
||||
if result:
|
||||
print(f"\n✅ 提取完成！")
|
||||
print(f"输出目录: {result}")
|
||||
print(f"\n下一步：使用 OCR 工具识别字幕")
|
||||
else:
|
||||
print(f"\n❌ 提取失败")
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,30 @@
|
||||
---
|
||||
title: "为什么现在咖啡比矿泉水还便宜？【食录】"
|
||||
source: "bilibili"
|
||||
url: "https://www.bilibili.com/video/BV1KtFzzqEUG"
|
||||
author: "赛博食录"
|
||||
duration: "12:48"
|
||||
date: "2026-02-06 11:44:59"
|
||||
bvid: "BV1KtFzzqEUG"
|
||||
created: "2026-02-08 23:32:02"
|
||||
stats:
|
||||
view: 977748
|
||||
like: 32085
|
||||
coin: 4368
|
||||
favorite: 7642
|
||||
tags:
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
## 视频简介
|
||||
你买过最便宜的咖啡要多少钱？（ps：本期全程无广
|
||||
|
||||
## 数据统计
|
||||
- 播放量: 977,748
|
||||
- 点赞: 32,085
|
||||
- 投币: 4,368
|
||||
- 收藏: 7,642
|
||||
- 分享: 1,653
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
处理B站视频并保存到知识库
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
# 添加项目路径
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb'))
|
||||
|
||||
from src.extractors.bilibili_extractor import BilibiliExtractor
|
||||
from src.outputs.obsidian_writer import ObsidianWriter
|
||||
from src.ai_client import AIClient
|
||||
|
||||
async def process_video(video_url):
|
||||
"""处理视频并生成 Obsidian 笔记"""
|
||||
|
||||
print(f"正在处理视频: {video_url}")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. 提取视频信息
|
||||
print("\n步骤 1/3: 提取视频信息...")
|
||||
extractor = BilibiliExtractor()
|
||||
try:
|
||||
result = await extractor._extract_async(video_url)
|
||||
print("✅ 视频信息提取成功")
|
||||
except Exception as e:
|
||||
print(f"❌ 提取失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return
|
||||
|
||||
# 显示提取的信息
|
||||
print("\n视频信息:")
|
||||
print("-" * 60)
|
||||
print(f"标题: {result['metadata']['title']}")
|
||||
print(f"UP主: {result['metadata']['author']}")
|
||||
print(f"时长: {result['metadata']['duration']}")
|
||||
print(f"发布时间: {result['metadata']['pubtime']}")
|
||||
print(f"播放量: {result['stats']['view']:,}")
|
||||
print(f"点赞: {result['stats']['like']:,}")
|
||||
print(f"BV号: {result['metadata']['bvid']}")
|
||||
|
||||
# 显示字幕信息
|
||||
subtitle = result['subtitle']
|
||||
if subtitle:
|
||||
lines = subtitle.split('\n')
|
||||
print(f"\n字幕: 找到 {len(lines)} 条字幕")
|
||||
else:
|
||||
print("\n字幕: 该视频没有官方字幕")
|
||||
|
||||
# 2. 生成 Obsidian 笔记
|
||||
print("\n步骤 2/3: 生成 Obsidian 笔记...")
|
||||
|
||||
# 创建笔记内容
|
||||
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 构建标签
|
||||
tags = result['metadata']['tags'].split(', ') if result['metadata']['tags'] else []
|
||||
tags_str = ' '.join([f"#{tag}" for tag in tags if tag])
|
||||
|
||||
# 构建笔记
|
||||
note_content = f"""---
|
||||
title: "{result['metadata']['title']}"
|
||||
source: "bilibili"
|
||||
url: "{result['metadata']['url']}"
|
||||
author: "{result['metadata']['author']}"
|
||||
duration: "{result['metadata']['duration']}"
|
||||
date: "{result['metadata']['pubtime']}"
|
||||
bvid: "{result['metadata']['bvid']}"
|
||||
created: "{now}"
|
||||
stats:
|
||||
view: {result['stats']['view']}
|
||||
like: {result['stats']['like']}
|
||||
coin: {result['stats']['coin']}
|
||||
favorite: {result['stats']['favorite']}
|
||||
tags:
|
||||
{chr(10).join([f' - {tag}' for tag in tags if tag])}
|
||||
---
|
||||
|
||||
{tags_str}
|
||||
|
||||
## 视频简介
|
||||
{result['description'][:500]}{'...' if len(result['description']) > 500 else ''}
|
||||
|
||||
## 数据统计
|
||||
- 播放量: {result['stats']['view']:,}
|
||||
- 点赞: {result['stats']['like']:,}
|
||||
- 投币: {result['stats']['coin']:,}
|
||||
- 收藏: {result['stats']['favorite']:,}
|
||||
- 分享: {result['stats']['share']:,}
|
||||
|
||||
"""
|
||||
|
||||
# 如果有字幕，添加到笔记
|
||||
if subtitle:
|
||||
note_content += "## 字幕内容\n\n"
|
||||
note_content += f"共 {len(subtitle.split(chr(10)))} 条字幕\n\n"
|
||||
|
||||
# 只显示前50条字幕，避免过长
|
||||
lines = subtitle.split('\n')
|
||||
if len(lines) > 50:
|
||||
note_content += '\n'.join(lines[:50])
|
||||
note_content += f"\n\n...(还有 {len(lines) - 50} 条字幕，已省略)\n"
|
||||
else:
|
||||
note_content += subtitle + '\n'
|
||||
|
||||
print("✅ 笔记生成成功")
|
||||
|
||||
# 3. 保存笔记文件
|
||||
print("\n步骤 3/3: 保存笔记文件...")
|
||||
|
||||
# 确定保存目录
|
||||
kb_dir = os.path.expanduser("~/ObsidianVault")
|
||||
if not os.path.exists(kb_dir):
|
||||
# 如果用户的主目录没有 ObsidianVault，使用本地目录
|
||||
kb_dir = os.path.join(os.path.dirname(__file__), 'kb_output')
|
||||
if not os.path.exists(kb_dir):
|
||||
os.makedirs(kb_dir)
|
||||
print(f"创建输出目录: {kb_dir}")
|
||||
|
||||
# 生成文件名（使用日期和标题）
|
||||
date_str = datetime.now().strftime('%Y%m%d')
|
||||
safe_title = result['metadata']['title'][:50].replace('/', '-').replace('\\', '-').replace(':', '-')
|
||||
filename = f"{date_str}-{safe_title}.md"
|
||||
|
||||
filepath = os.path.join(kb_dir, filename)
|
||||
|
||||
# 写入文件
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
f.write(note_content)
|
||||
|
||||
print(f"✅ 笔记已保存到: {filepath}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("🎉 处理完成！")
|
||||
print("=" * 60)
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 从URL中提取BV号
|
||||
import re
|
||||
|
||||
video_url = "https://www.bilibili.com/video/BV1KtFzzqEUG/?spm_id_from=333.1007.tianma.2-1-4.click&vd_source=d82ff6409be4a2d868295c58516354a2"
|
||||
|
||||
# 提取BV号
|
||||
bv_match = re.search(r'BV[a-zA-Z0-9]{10}', video_url)
|
||||
if bv_match:
|
||||
bvid = bv_match.group(0)
|
||||
print(f"识别到BV号: {bvid}\n")
|
||||
asyncio.run(process_video(bvid))
|
||||
else:
|
||||
print("❌ 无法从URL中提取BV号")
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"version": 1,
|
||||
"registry": "https://clawhub.ai",
|
||||
"slug": "find-skills",
|
||||
"installedVersion": "0.1.0",
|
||||
"installedAt": 1770607549810
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
---
|
||||
name: find-skills
|
||||
description: Helps users discover and install agent skills when they ask questions like "how do I do X", "find a skill for X", "is there a skill that can...", or express interest in extending capabilities. This skill should be used when the user is looking for functionality that might exist as an installable skill.
|
||||
---
|
||||
|
||||
# Find Skills
|
||||
|
||||
This skill helps you discover and install skills from the open agent skills ecosystem.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
Use this skill when the user:
|
||||
|
||||
- Asks "how do I do X" where X might be a common task with an existing skill
|
||||
- Says "find a skill for X" or "is there a skill for X"
|
||||
- Asks "can you do X" where X is a specialized capability
|
||||
- Expresses interest in extending agent capabilities
|
||||
- Wants to search for tools, templates, or workflows
|
||||
- Mentions they wish they had help with a specific domain (design, testing, deployment, etc.)
|
||||
|
||||
## What is the Skills CLI?
|
||||
|
||||
The Skills CLI (`npx skills`) is the package manager for the open agent skills ecosystem. Skills are modular packages that extend agent capabilities with specialized knowledge, workflows, and tools.
|
||||
|
||||
**Key commands:**
|
||||
|
||||
- `npx skills find [query]` - Search for skills interactively or by keyword
|
||||
- `npx skills add <package>` - Install a skill from GitHub or other sources
|
||||
- `npx skills check` - Check for skill updates
|
||||
- `npx skills update` - Update all installed skills
|
||||
|
||||
**Browse skills at:** https://skills.sh/
|
||||
|
||||
## How to Help Users Find Skills
|
||||
|
||||
### Step 1: Understand What They Need
|
||||
|
||||
When a user asks for help with something, identify:
|
||||
|
||||
1. The domain (e.g., React, testing, design, deployment)
|
||||
2. The specific task (e.g., writing tests, creating animations, reviewing PRs)
|
||||
3. Whether this is a common enough task that a skill likely exists
|
||||
|
||||
### Step 2: Search for Skills
|
||||
|
||||
Run the find command with a relevant query:
|
||||
|
||||
```bash
|
||||
npx skills find [query]
|
||||
```
|
||||
|
||||
For example:
|
||||
|
||||
- User asks "how do I make my React app faster?" → `npx skills find react performance`
|
||||
- User asks "can you help me with PR reviews?" → `npx skills find pr review`
|
||||
- User asks "I need to create a changelog" → `npx skills find changelog`
|
||||
|
||||
The command will return results like:
|
||||
|
||||
```
|
||||
Install with npx skills add <owner/repo@skill>
|
||||
|
||||
vercel-labs/agent-skills@vercel-react-best-practices
|
||||
└ https://skills.sh/vercel-labs/agent-skills/vercel-react-best-practices
|
||||
```
|
||||
|
||||
### Step 3: Present Options to the User
|
||||
|
||||
When you find relevant skills, present them to the user with:
|
||||
|
||||
1. The skill name and what it does
|
||||
2. The install command they can run
|
||||
3. A link to learn more at skills.sh
|
||||
|
||||
Example response:
|
||||
|
||||
```
|
||||
I found a skill that might help! The "vercel-react-best-practices" skill provides
|
||||
React and Next.js performance optimization guidelines from Vercel Engineering.
|
||||
|
||||
To install it:
|
||||
npx skills add vercel-labs/agent-skills@vercel-react-best-practices
|
||||
|
||||
Learn more: https://skills.sh/vercel-labs/agent-skills/vercel-react-best-practices
|
||||
```
|
||||
|
||||
### Step 4: Offer to Install
|
||||
|
||||
If the user wants to proceed, you can install the skill for them:
|
||||
|
||||
```bash
|
||||
npx skills add <owner/repo@skill> -g -y
|
||||
```
|
||||
|
||||
The `-g` flag installs globally (user-level) and `-y` skips confirmation prompts.
|
||||
|
||||
## Common Skill Categories
|
||||
|
||||
When searching, consider these common categories:
|
||||
|
||||
| Category | Example Queries |
|
||||
| --------------- | ---------------------------------------- |
|
||||
| Web Development | react, nextjs, typescript, css, tailwind |
|
||||
| Testing | testing, jest, playwright, e2e |
|
||||
| DevOps | deploy, docker, kubernetes, ci-cd |
|
||||
| Documentation | docs, readme, changelog, api-docs |
|
||||
| Code Quality | review, lint, refactor, best-practices |
|
||||
| Design | ui, ux, design-system, accessibility |
|
||||
| Productivity | workflow, automation, git |
|
||||
|
||||
## Tips for Effective Searches
|
||||
|
||||
1. **Use specific keywords**: "react testing" is better than just "testing"
|
||||
2. **Try alternative terms**: If "deploy" doesn't work, try "deployment" or "ci-cd"
|
||||
3. **Check popular sources**: Many skills come from `vercel-labs/agent-skills` or `ComposioHQ/awesome-claude-skills`
|
||||
|
||||
## When No Skills Are Found
|
||||
|
||||
If no relevant skills exist:
|
||||
|
||||
1. Acknowledge that no existing skill was found
|
||||
2. Offer to help with the task directly using your general capabilities
|
||||
3. Suggest the user could create their own skill with `npx skills init`
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
I searched for skills related to "xyz" but didn't find any matches.
|
||||
I can still help you with this task directly! Would you like me to proceed?
|
||||
|
||||
If this is something you do often, you could create your own skill:
|
||||
npx skills init my-xyz-skill
|
||||
```
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"ownerId": "kn77ajmmqw3cgnc3ay1x3e0ccd805hsw",
|
||||
"slug": "find-skills",
|
||||
"version": "0.1.0",
|
||||
"publishedAt": 1769698710765
|
||||
}
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
# 禁用 oneDNN 后端，解决 PaddleOCR 兼容性问题
|
||||
|
||||
echo "禁用 oneDNN..."
|
||||
|
||||
# 方法1：环境变量（临时）
|
||||
export FLAGS_use_mkldnn=0
|
||||
echo "✅ 环境变量已设置: FLAGS_use_mkldnn=0"
|
||||
|
||||
# 方法2：写入 ~/.bashrc（永久）
|
||||
if ! grep -q "FLAGS_use_mkldnn=0" ~/.bashrc; then
|
||||
echo "" >> ~/.bashrc
|
||||
echo "# 禁用 PaddleOCR 的 oneDNN 后端（解决兼容性问题）" >> ~/.bashrc
|
||||
echo "export FLAGS_use_mkldnn=0" >> ~/.bashrc
|
||||
echo "✅ 已写入 ~/.bashrc"
|
||||
else
|
||||
echo "ℹ️ ~/.bashrc 中已存在此配置"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "验证配置..."
|
||||
if [ "$FLAGS_use_mkldnn" = "0" ]; then
|
||||
echo "✅ 当前会话: FLAGS_use_mkldnn=$FLAGS_use_mkldnn"
|
||||
else
|
||||
echo "⚠️ 当前会话: FLAGS_use_mkldnn=${FLAGS_use_mkldnn:-未设置}"
|
||||
echo "💡 提示: 运行 'source ~/.bashrc' 使配置生效"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "完成！现在 PaddleOCR 将使用 CPU 原生计算，避免 oneDNN 兼容性问题。"
|
||||
@@ -0,0 +1,42 @@
|
||||
Collecting DrissionPage
|
||||
Using cached DrissionPage-4.1.1.2-py3-none-any.whl.metadata (4.7 kB)
|
||||
Requirement already satisfied: lxml in ./.venv/lib/python3.12/site-packages (from DrissionPage) (6.0.2)
|
||||
Requirement already satisfied: requests in ./.venv/lib/python3.12/site-packages (from DrissionPage) (2.32.5)
|
||||
Collecting cssselect (from DrissionPage)
|
||||
Using cached cssselect-1.4.0-py3-none-any.whl.metadata (2.4 kB)
|
||||
Collecting DownloadKit>=2.0.7 (from DrissionPage)
|
||||
Using cached DownloadKit-2.0.7-py3-none-any.whl.metadata (1.7 kB)
|
||||
Collecting websocket-client (from DrissionPage)
|
||||
Using cached websocket_client-1.9.0-py3-none-any.whl.metadata (8.3 kB)
|
||||
Requirement already satisfied: click in ./.venv/lib/python3.12/site-packages (from DrissionPage) (8.3.1)
|
||||
Collecting tldextract>=3.4.4 (from DrissionPage)
|
||||
Using cached tldextract-5.3.1-py3-none-any.whl.metadata (7.3 kB)
|
||||
Collecting psutil (from DrissionPage)
|
||||
Using cached psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl.metadata (22 kB)
|
||||
Collecting DataRecorder>=3.4.11 (from DownloadKit>=2.0.7->DrissionPage)
|
||||
Using cached DataRecorder-3.6.2-py3-none-any.whl.metadata (4.5 kB)
|
||||
Requirement already satisfied: idna in ./.venv/lib/python3.12/site-packages (from tldextract>=3.4.4->DrissionPage) (3.11)
|
||||
Collecting requests-file>=1.4 (from tldextract>=3.4.4->DrissionPage)
|
||||
Using cached requests_file-3.0.1-py2.py3-none-any.whl.metadata (1.7 kB)
|
||||
Collecting filelock>=3.0.8 (from tldextract>=3.4.4->DrissionPage)
|
||||
Using cached filelock-3.20.3-py3-none-any.whl.metadata (2.1 kB)
|
||||
Requirement already satisfied: charset_normalizer<4,>=2 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (3.4.4)
|
||||
Requirement already satisfied: urllib3<3,>=1.21.1 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (2.6.3)
|
||||
Requirement already satisfied: certifi>=2017.4.17 in ./.venv/lib/python3.12/site-packages (from requests->DrissionPage) (2026.1.4)
|
||||
Collecting openpyxl (from DataRecorder>=3.4.11->DownloadKit>=2.0.7->DrissionPage)
|
||||
Using cached openpyxl-3.1.5-py2.py3-none-any.whl.metadata (2.5 kB)
|
||||
Collecting et-xmlfile (from openpyxl->DataRecorder>=3.4.11->DownloadKit>=2.0.7->DrissionPage)
|
||||
Using cached et_xmlfile-2.0.0-py3-none-any.whl.metadata (2.7 kB)
|
||||
Using cached DrissionPage-4.1.1.2-py3-none-any.whl (257 kB)
|
||||
Using cached DownloadKit-2.0.7-py3-none-any.whl (21 kB)
|
||||
Using cached tldextract-5.3.1-py3-none-any.whl (105 kB)
|
||||
Using cached cssselect-1.4.0-py3-none-any.whl (18 kB)
|
||||
Using cached psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl (155 kB)
|
||||
Using cached websocket_client-1.9.0-py3-none-any.whl (82 kB)
|
||||
Using cached DataRecorder-3.6.2-py3-none-any.whl (37 kB)
|
||||
Using cached filelock-3.20.3-py3-none-any.whl (16 kB)
|
||||
Using cached requests_file-3.0.1-py2.py3-none-any.whl (4.5 kB)
|
||||
Using cached openpyxl-3.1.5-py2.py3-none-any.whl (250 kB)
|
||||
Using cached et_xmlfile-2.0.0-py3-none-any.whl (18 kB)
|
||||
Installing collected packages: websocket-client, psutil, filelock, et-xmlfile, cssselect, requests-file, openpyxl, tldextract, DataRecorder, DownloadKit, DrissionPage
|
||||
Successfully installed DataRecorder-3.6.2 DownloadKit-2.0.7 DrissionPage-4.1.1.2 cssselect-1.4.0 et-xmlfile-2.0.0 filelock-3.20.3 openpyxl-3.1.5 psutil-7.2.2 requests-file-3.0.1 tldextract-5.3.1 websocket-client-1.9.0
|
||||
@@ -208,6 +208,51 @@ bilibili_extraction:
|
||||
enable_danmaku: false # 启用弹幕提取
|
||||
timeout_seconds: 30 # 超时时间
|
||||
max_retries: 3 # 最大重试次数
|
||||
|
||||
# 硬字幕提取配置 (OCR 方式)
|
||||
hard_subtitle:
|
||||
# 是否启用硬字幕提取功能
|
||||
enabled: true
|
||||
|
||||
# OCR 引擎选择
|
||||
# 可选值: "paddleocr" (推荐), "tesseract"
|
||||
engine: "paddleocr"
|
||||
|
||||
# 字幕区域检测配置
|
||||
detection:
|
||||
# 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测)
|
||||
default_region: "bottom"
|
||||
# 当默认区域未检测到字幕时，是否自动回退到全屏模式
|
||||
fallback_auto: true
|
||||
# 是否缓存字幕区域检测结果（提升处理速度）
|
||||
cache_region: true
|
||||
|
||||
# OCR 识别配置
|
||||
ocr:
|
||||
# 置信度阈值: 低于此值的识别结果将被过滤
|
||||
confidence_threshold: 0.7
|
||||
# 批处理大小: 每次处理的帧数（影响内存使用和速度）
|
||||
batch_size: 32
|
||||
# 最大并发工作线程数
|
||||
max_workers: 4
|
||||
|
||||
# 视频帧提取配置
|
||||
video:
|
||||
# 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别
|
||||
interval_seconds: 5
|
||||
# 最大抽取帧数: 防止处理过长视频导致资源耗尽
|
||||
max_frames: 500
|
||||
# 流式处理模式: 逐步读取视频帧，避免一次性加载整个视频
|
||||
stream_mode: true
|
||||
|
||||
# 后处理配置
|
||||
postprocess:
|
||||
# 是否启用去重: 去除连续相同的字幕文本
|
||||
deduplicate: true
|
||||
# 最小文本长度: 少于此长度的识别结果将被视为噪声
|
||||
min_length: 2
|
||||
# 是否过滤噪声: 使用规则过滤明显错误的识别结果
|
||||
filter_noise: true
|
||||
```
|
||||
|
||||
### 环境变量
|
||||
@@ -319,6 +364,104 @@ else:
|
||||
content["subtitle_text"] = subtitle_text
|
||||
```
|
||||
|
||||
### 硬字幕提取功能
|
||||
|
||||
当视频没有内置字幕时，硬字幕提取功能可以通过 OCR（光学字符识别）技术从视频画面中识别字幕文本。
|
||||
|
||||
**功能介绍：**
|
||||
- 自动检测和识别视频画面中的字幕文本
|
||||
- 支持多种 OCR 引擎（PaddleOCR、Tesseract）
|
||||
- 智能字幕区域检测（底部、全屏、自动）
|
||||
- 自动去重和噪声过滤
|
||||
- 支持流式处理，降低内存占用
|
||||
|
||||
**使用场景：**
|
||||
- 视频没有提供 CC 字幕或 AI 字幕
|
||||
- 字幕嵌入在视频画面中（硬编码字幕）
|
||||
- 需要提取非标准格式的字幕
|
||||
|
||||
**配置说明：**
|
||||
|
||||
```yaml
|
||||
hard_subtitle:
|
||||
# 是否启用硬字幕提取功能
|
||||
enabled: true
|
||||
|
||||
# OCR 引擎选择
|
||||
# 可选值: "paddleocr" (推荐), "tesseract"
|
||||
engine: "paddleocr"
|
||||
|
||||
# 字幕区域检测配置
|
||||
detection:
|
||||
# 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测)
|
||||
default_region: "bottom"
|
||||
# 当默认区域未检测到字幕时，是否自动回退到全屏模式
|
||||
fallback_auto: true
|
||||
# 是否缓存字幕区域检测结果（提升处理速度）
|
||||
cache_region: true
|
||||
|
||||
# OCR 识别配置
|
||||
ocr:
|
||||
# 置信度阈值: 低于此值的识别结果将被过滤
|
||||
confidence_threshold: 0.7
|
||||
# 批处理大小: 每次处理的帧数（影响内存使用和速度）
|
||||
batch_size: 32
|
||||
# 最大并发工作线程数
|
||||
max_workers: 4
|
||||
|
||||
# 视频帧提取配置
|
||||
video:
|
||||
# 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别
|
||||
interval_seconds: 5
|
||||
# 最大抽取帧数: 防止处理过长视频导致资源耗尽
|
||||
max_frames: 500
|
||||
# 流式处理模式: 逐步读取视频帧，避免一次性加载整个视频
|
||||
stream_mode: true
|
||||
|
||||
# 后处理配置
|
||||
postprocess:
|
||||
# 是否启用去重: 去除连续相同的字幕文本
|
||||
deduplicate: true
|
||||
# 最小文本长度: 少于此长度的识别结果将被视为噪声
|
||||
min_length: 2
|
||||
# 是否过滤噪声: 使用规则过滤明显错误的识别结果
|
||||
filter_noise: true
|
||||
```
|
||||
|
||||
**使用示例：**
|
||||
|
||||
```python
|
||||
from obsidian_kb.extractors.bilibili import BilibiliExtractor
|
||||
|
||||
# 创建提取器实例（自动启用硬字幕提取）
|
||||
extractor = BilibiliExtractor(
|
||||
use_playwright=True,
|
||||
headless=True
|
||||
)
|
||||
|
||||
# 当视频没有内置字幕时，会自动尝试硬字幕提取
|
||||
subtitle = extractor.extract_subtitle("https://www.bilibili.com/video/BV1xx411c7mD")
|
||||
|
||||
print(f"识别到的字幕: {subtitle}")
|
||||
```
|
||||
|
||||
**注意事项：**
|
||||
|
||||
1. **性能考虑**：硬字幕提取需要较高的计算资源，建议在配置合理的机器上运行
|
||||
2. **识别准确度**：OCR 识别准确度受字幕字体、颜色、背景等因素影响，可能存在误差
|
||||
3. **处理时间**：硬字幕提取比软字幕提取慢，对于长视频需要较长时间处理
|
||||
4. **依赖安装**：使用前需要安装 OCR 引擎依赖：
|
||||
- PaddleOCR: `pip install paddleocr paddlepaddle`
|
||||
- Tesseract: 需要安装 Tesseract OCR 引擎和语言包
|
||||
5. **网络视频**：流式处理需要稳定的网络连接
|
||||
6. **字幕格式**：不同视频的字幕样式差异较大，可能需要调整配置参数以获得最佳效果
|
||||
|
||||
**最佳实践：**
|
||||
- 对于短视频（<30分钟），可以减小 `interval_seconds` 以提高识别精度
|
||||
- 对于长视频，增加 `interval_seconds` 和减少 `max_frames` 以平衡速度和质量
|
||||
- 如果字幕位置固定在底部，使用 `default_region: "bottom"` 可以提升速度
|
||||
- 首次使用建议先测试一段视频，根据结果调整配置参数
|
||||
|
||||
### ZhihuDrissionExtractor 类
|
||||
|
||||
知乎 DrissionPage 提取器提供知乎内容的完整提取功能：
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
# Obsidian KB - 待办事项
|
||||
|
||||
## 📌 当前任务：解决硬字幕 OCR 问题
|
||||
|
||||
### ✅ 已完成
|
||||
1. [x] 分析硬字幕提取问题
|
||||
- 发现 oneDNN 兼容性问题
|
||||
- 发现 PaddleOCR 参数错误
|
||||
|
||||
2. [x] 禁用 oneDNN
|
||||
- 设置环境变量 FLAGS_use_mkldnn=0
|
||||
- 写入 ~/.bashrc
|
||||
|
||||
3. [x] 修复 PaddleOCR 参数
|
||||
- 更新 `use_angle_cls` → `use_textline_orientation`
|
||||
- 移除不支持的 `show_log` 参数
|
||||
|
||||
4. [x] 安装 EasyOCR
|
||||
- 安装 EasyOCR 1.7.2
|
||||
- 测试 EasyOCR 初始化成功
|
||||
|
||||
5. [x] 创建 EasyOCR 处理器
|
||||
- 新文件：`src/extractors/easyocr_processor.py`
|
||||
- 支持简体中文识别
|
||||
- 避免所有 oneDNN 兼容性问题
|
||||
|
||||
### ⏳ 待完成
|
||||
- [ ] 1. 修改硬字幕提取器，支持双 OCR 引擎
|
||||
- 在 `hard_subtitle_extractor.py` 中添加引擎选择逻辑
|
||||
- 根据 `config.hard_subtitle.engine` 参数选择使用 PaddleOCR 或 EasyOCR
|
||||
- 保持向后兼容性
|
||||
|
||||
- [ ] 2. 更新配置文件
|
||||
- 添加 `hard_subtitle.engine` 配置项（`paddleocr` | `easyocr`）
|
||||
- 默认值设为 `easyocr`（更稳定）
|
||||
- 更新配置文档
|
||||
|
||||
- [ ] 3. 实际测试 EasyOCR 字幕提取
|
||||
- 使用真实视频测试硬字幕提取
|
||||
- 对比 EasyOCR 和 PaddleOCR 的识别效果
|
||||
- 测试处理速度和准确度
|
||||
|
||||
- [ ] 4. 性能优化（可选）
|
||||
- 优化帧提取逻辑（当前只提取 4 帧有问题）
|
||||
- 调整抽帧间隔和最大帧数
|
||||
- 考虑多线程/多进程加速
|
||||
|
||||
- [ ] 5. 文档更新
|
||||
- 更新 SKILL.md 中的硬字幕提取说明
|
||||
- 添加 OCR 引擎选择指南
|
||||
- 记录 EasyOCR 优势和限制
|
||||
|
||||
---
|
||||
|
||||
## 📋 备注
|
||||
|
||||
### 问题记录
|
||||
- **PaddleOCR oneDNN 问题**：即使设置环境变量，仍出现兼容性错误
|
||||
```
|
||||
(Unimplemented) ConvertPirAttribute2RuntimeAttribute not support
|
||||
[pir::ArrayAttribute<pir::DoubleAttribute>]
|
||||
```
|
||||
|
||||
- **帧提取异常**：12:48的视频只提取了4帧（应该150+帧）
|
||||
|
||||
### EasyOCR 优势
|
||||
- ✅ 不依赖 PaddlePaddle
|
||||
- ✅ 安装简单
|
||||
- ✅ 无底层框架兼容性问题
|
||||
- ✅ 支持 70+ 种语言
|
||||
- ✅ 活跃维护
|
||||
|
||||
### EasyOCR 限制
|
||||
- ⚠️ 中文准确度略低于 PaddleOCR
|
||||
- ⚠️ 首次初始化需要下载模型（约 40MB）
|
||||
- ⚠️ 处理速度可能略慢
|
||||
|
||||
---
|
||||
|
||||
## 🔗 相关文件
|
||||
|
||||
- `src/extractors/ocr_processor.py` - PaddleOCR 处理器
|
||||
- `src/extractors/easyocr_processor.py` - EasyOCR 处理器（新建）
|
||||
- `src/extractors/hard_subtitle_extractor.py` - 硬字幕提取器
|
||||
- `src/extractors/bilibili_extractor.py` - B站提取器（调用硬字幕提取）
|
||||
- `.disable_onednn.sh` - 禁用 oneDNN 脚本
|
||||
|
||||
---
|
||||
|
||||
创建时间：2026-02-09 08:02
|
||||
最后更新：2026-02-09 08:02
|
||||
@@ -1,38 +1,109 @@
|
||||
obsidian:
|
||||
vault_path: /home/admin/.openclaw/workspace/knowledge-base
|
||||
# Obsidian KB 配置文件
|
||||
# 优先级: ./config.yaml > ~/.obsidian-kb/config.yaml > ~/.config/obsidian-kb/config.yaml
|
||||
|
||||
git:
|
||||
enabled: false
|
||||
vault_path: /home/admin/.openclaw/workspace/knowledge-base
|
||||
remote: origin
|
||||
branch: main
|
||||
auto_push: false
|
||||
commit_message_template: "Add note: {title}"
|
||||
# AI 配置
|
||||
ai:
|
||||
provider: openai # AI 提供商
|
||||
model: gpt-4 # 模型名称
|
||||
api_key: sk-... # API Key (建议使用环境变量: OPENAI_API_KEY)
|
||||
|
||||
# 存储配置
|
||||
storage:
|
||||
inbox_dir: inbox # 收件箱目录
|
||||
topics_dir: topics # 主题目录
|
||||
data_dir: data # 数据目录
|
||||
|
||||
# 输出配置
|
||||
output:
|
||||
organize_by: author
|
||||
filename_format: "{date}_{title}.md"
|
||||
auto_tags: true
|
||||
default_tags:
|
||||
- video
|
||||
- bilibili
|
||||
format: markdown # 输出格式
|
||||
encoding: utf-8 # 文件编码
|
||||
include_timestamp: true # 包含时间戳
|
||||
include_source: true # 包含源链接
|
||||
|
||||
# Git 同步配置
|
||||
git:
|
||||
enabled: true # 启用 Git 同步
|
||||
vault_path: ~/ObsidianVault # Obsidian 知识库路径
|
||||
remote: origin # 远程仓库名
|
||||
branch: main # 分支名
|
||||
auto_commit: true # 自动提交
|
||||
commit_message_template: "Update from obsidian-kb: {count} items"
|
||||
|
||||
# 分段配置
|
||||
chunking:
|
||||
enabled: true # 启用长视频分段
|
||||
chunk_duration_seconds: 900 # 分段时长 (15分钟)
|
||||
max_chunk_duration_seconds: 1800 # 最大分段时长 (30分钟)
|
||||
long_video_threshold: 1800 # 长视频阈值 (30分钟)
|
||||
|
||||
# 数据源配置
|
||||
sources:
|
||||
bilibili:
|
||||
enabled: true
|
||||
extraction:
|
||||
headless: true
|
||||
timeout: 30
|
||||
cookie: '' # Bilibili Cookie (可选，环境变量: BILIBILI_COOKIE)
|
||||
youtube:
|
||||
enabled: false
|
||||
api_key: '' # YouTube API Key
|
||||
zhihu:
|
||||
enabled: true
|
||||
use_drissionpage: true # 使用 DrissionPage 绕过检测（推荐）
|
||||
cookie_file: ~/.obsidian-kb/zhihu_cookies.json # Cookie 文件路径
|
||||
cookie_str: "" # 可选：直接在配置中填入 Cookie JSON (环境变量: ZHIHU_COOKIE)
|
||||
timeout: 30
|
||||
headless: true
|
||||
|
||||
zhihu:
|
||||
enabled: true
|
||||
use_drissionpage: true
|
||||
cookie_file: ~/.obsidian-kb/zhihu_cookies.json
|
||||
cookie_str: "" # 可选：直接在配置中填入
|
||||
timeout: 30
|
||||
headless: true
|
||||
# Bilibili 提取器配置
|
||||
bilibili_extraction:
|
||||
use_playwright: true # 使用 Playwright 绕过检测
|
||||
headless: true # 无头模式
|
||||
cookie_file: '' # Cookie 文件路径 (可选)
|
||||
enable_subtitle: true # 启用字幕提取
|
||||
subtitle_language: zh-CN # 字幕语言
|
||||
enable_danmaku: false # 启用弹幕提取
|
||||
timeout_seconds: 30 # 超时时间
|
||||
max_retries: 3 # 最大重试次数
|
||||
|
||||
chunking:
|
||||
# 硬字幕提取配置 (OCR 方式)
|
||||
hard_subtitle:
|
||||
# 是否启用硬字幕提取功能
|
||||
enabled: true
|
||||
chunk_duration_seconds: 900
|
||||
long_video_threshold: 1800
|
||||
|
||||
# OCR 引擎选择
|
||||
# 可选值: "paddleocr" (推荐), "tesseract"
|
||||
engine: "paddleocr"
|
||||
|
||||
# 字幕区域检测配置
|
||||
detection:
|
||||
# 默认字幕区域: "bottom" (底部), "full" (全屏), "auto" (自动检测)
|
||||
default_region: "bottom"
|
||||
# 当默认区域未检测到字幕时，是否自动回退到全屏模式
|
||||
fallback_auto: true
|
||||
# 是否缓存字幕区域检测结果（提升处理速度）
|
||||
cache_region: true
|
||||
|
||||
# OCR 识别配置
|
||||
ocr:
|
||||
# 置信度阈值: 低于此值的识别结果将被过滤
|
||||
confidence_threshold: 0.7
|
||||
# 批处理大小: 每次处理的帧数（影响内存使用和速度）
|
||||
batch_size: 32
|
||||
# 最大并发工作线程数
|
||||
max_workers: 4
|
||||
|
||||
# 视频帧提取配置
|
||||
video:
|
||||
# 抽帧间隔 (秒): 每隔多少秒抽取一帧进行识别
|
||||
interval_seconds: 5
|
||||
# 最大抽取帧数: 防止处理过长视频导致资源耗尽
|
||||
max_frames: 500
|
||||
# 流式处理模式: 逐步读取视频帧，避免一次性加载整个视频
|
||||
stream_mode: true
|
||||
|
||||
# 后处理配置
|
||||
postprocess:
|
||||
# 是否启用去重: 去除连续相同的字幕文本
|
||||
deduplicate: true
|
||||
# 最小文本长度: 少于此长度的识别结果将被视为噪声
|
||||
min_length: 2
|
||||
# 是否过滤噪声: 使用规则过滤明显错误的识别结果
|
||||
filter_noise: true
|
||||
|
||||
@@ -14,6 +14,9 @@ anthropic>=0.7.0
|
||||
# undetected-playwright>=0.1.0
|
||||
beautifulsoup4>=4.12.0
|
||||
lxml>=4.9.0
|
||||
paddlepaddle>=2.5.0
|
||||
paddleocr>=2.7.0
|
||||
opencv-python>=4.8.0
|
||||
|
||||
# Zhihu support
|
||||
DrissionPage>=4.0.0
|
||||
|
||||
@@ -9,11 +9,14 @@ B站视频内容提取器
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
OCR 文字识别处理器（EasyOCR 版本）
|
||||
|
||||
使用 EasyOCR 实现视频帧文字识别功能，作为 PaddleOCR 的替代方案
|
||||
主要优势：不依赖 PaddlePaddle，避免 oneDNN 兼容性问题
|
||||
"""
|
||||
|
||||
import re
|
||||
import numpy as np
|
||||
from typing import List, Dict, Any, Tuple
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EasyOCRProcessor:
|
||||
"""
|
||||
EasyOCR 文字识别处理器
|
||||
|
||||
使用 EasyOCR 从视频帧中识别文字，支持批量识别和结果过滤
|
||||
|
||||
相比 PaddleOCR 的优势：
|
||||
- 不依赖 PaddlePaddle，避免 oneDNN 兼容性问题
|
||||
- 安装更简单，依赖更少
|
||||
- 支持多种语言（70+ 种）
|
||||
"""
|
||||
|
||||
# 相似度阈值：用于判断两个文本是否相似（基于编辑距离）
|
||||
SIMILARITY_THRESHOLD = 0.85
|
||||
|
||||
# 置信度阈值
|
||||
CONFIDENCE_THRESHOLD = 0.7
|
||||
|
||||
# 最小字幕长度（字符数）
|
||||
MIN_SUBTITLE_LENGTH = 2
|
||||
|
||||
def __init__(self, lang: str = 'ch_sim', use_gpu: bool = False, detail: int = 1):
|
||||
"""
|
||||
初始化 OCR 处理器
|
||||
|
||||
Args:
|
||||
lang: 语言支持，'ch_sim' 为简体中文，'ch_tra' 为繁体中文
|
||||
use_gpu: 是否使用 GPU 加速
|
||||
detail: 识别结果详细程度（0=只返回文本，1=返回文本+坐标+置信度）
|
||||
"""
|
||||
self.lang = lang
|
||||
self.use_gpu = use_gpu
|
||||
self.detail = detail
|
||||
|
||||
# 初始化 EasyOCR 实例
|
||||
self._ocr = None
|
||||
self._initialize_ocr()
|
||||
|
||||
def _initialize_ocr(self):
|
||||
"""初始化 EasyOCR 实例"""
|
||||
try:
|
||||
from easyocr import Reader
|
||||
|
||||
self._ocr = Reader(
|
||||
[self.lang],
|
||||
gpu=self.use_gpu,
|
||||
verbose=False # 不显示详细日志
|
||||
)
|
||||
logger.info("EasyOCR 初始化成功")
|
||||
except Exception as e:
|
||||
logger.error(f"EasyOCR 初始化失败: {e}")
|
||||
raise RuntimeError(f"无法初始化 EasyOCR: {e}")
|
||||
|
||||
def _initialize_ocr(self):
|
||||
"""初始化 EasyOCR 实例"""
|
||||
try:
|
||||
from easyocr import Reader
|
||||
|
||||
self._ocr = Reader(
|
||||
[self.lang],
|
||||
gpu=self.use_gpu,
|
||||
verbose=False # 不显示详细日志
|
||||
)
|
||||
logger.info("EasyOCR 初始化成功")
|
||||
except Exception as e:
|
||||
logger.error(f"EasyOCR 初始化失败: {e}")
|
||||
raise RuntimeError(f"无法初始化 EasyOCR: {e}")
|
||||
|
||||
def recognize_frames(self, frames: List[Tuple[np.ndarray, float]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
批量识别帧中的文字
|
||||
|
||||
Args:
|
||||
frames: 帧列表，每个元素为 (图像数组, 时间戳) 元组
|
||||
- 图像数组：numpy.ndarray，形状为 (H, W, 3) 的 RGB 图像
|
||||
- 时间戳：float，单位为秒
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 识别结果列表，每个结果包含：
|
||||
- time: float, 时间戳（秒）
|
||||
- text: str, 识别的文本
|
||||
- confidence: float, 置信度 (0-1)
|
||||
"""
|
||||
if not frames:
|
||||
logger.warning("输入帧列表为空")
|
||||
return []
|
||||
|
||||
results = []
|
||||
|
||||
for image, timestamp in frames:
|
||||
try:
|
||||
# 调用 EasyOCR 进行识别
|
||||
# EasyOCR 返回格式: [[[x1, y1], [x2, y2], [x3, y3], [x4, y4]], (text, confidence, ...)]
|
||||
# 根据 detail 参数，返回格式不同：
|
||||
# detail=0: 返回 [text1, text2, ...]
|
||||
# detail=1: 返回 [[bbox, (text, confidence)], ...]
|
||||
ocr_result = self._ocr.readtext(image, detail=self.detail)
|
||||
|
||||
if ocr_result:
|
||||
if self.detail == 0:
|
||||
# 简单模式：只返回文本列表
|
||||
for i, text in enumerate(ocr_result):
|
||||
if text:
|
||||
results.append({
|
||||
'time': timestamp,
|
||||
'text': str(text),
|
||||
'confidence': 1.0 # 简单模式下没有置信度
|
||||
})
|
||||
else:
|
||||
# 详细模式：返回边界框+文本+置信度
|
||||
for item in ocr_result:
|
||||
if len(item) >= 2:
|
||||
bbox = item[0]
|
||||
text_info = item[1]
|
||||
|
||||
if len(text_info) >= 2:
|
||||
text, confidence = text_info[0], text_info[1]
|
||||
|
||||
# 跳过空文本
|
||||
if not text:
|
||||
continue
|
||||
|
||||
# 处理置信度：可能是数字或特殊字符
|
||||
try:
|
||||
if isinstance(confidence, str):
|
||||
# 如果是 '?' 或其他特殊字符，使用默认置信度
|
||||
conf_float = 0.5
|
||||
else:
|
||||
conf_float = float(confidence)
|
||||
except (ValueError, TypeError):
|
||||
conf_float = 0.5
|
||||
|
||||
results.append({
|
||||
'time': timestamp,
|
||||
'text': text,
|
||||
'confidence': conf_float
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"帧识别失败 (时间戳 {timestamp}s): {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"原始识别结果数量: {len(results)}")
|
||||
|
||||
# 过滤和清洗结果
|
||||
filtered_results = self._filter_and_clean(results)
|
||||
logger.info(f"过滤后结果数量: {len(filtered_results)}")
|
||||
|
||||
return filtered_results
|
||||
|
||||
def _filter_and_clean(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
过滤和清洗识别结果
|
||||
|
||||
Args:
|
||||
results: 原始识别结果列表
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 过滤后的结果列表
|
||||
"""
|
||||
if not results:
|
||||
return []
|
||||
|
||||
# 步骤 1: 按置信度过滤
|
||||
filtered = self._filter_by_confidence(results)
|
||||
logger.debug(f"置信度过滤后: {len(filtered)}")
|
||||
|
||||
# 步骤 2: 过滤噪声（纯符号、单字符等）
|
||||
filtered = self._filter_noise(filtered)
|
||||
logger.debug(f"噪声过滤后: {len(filtered)}")
|
||||
|
||||
# 步骤 3: 去除重复或相似的文本
|
||||
filtered = self._remove_duplicates(filtered)
|
||||
logger.debug(f"去重后: {len(filtered)}")
|
||||
|
||||
# 步骤 4: 按时间戳排序
|
||||
filtered.sort(key=lambda x: x['time'])
|
||||
|
||||
return filtered
|
||||
|
||||
def _filter_by_confidence(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
按置信度过滤结果
|
||||
|
||||
Args:
|
||||
results: 识别结果列表
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 高置信度结果
|
||||
"""
|
||||
return [r for r in results if r['confidence'] >= self.CONFIDENCE_THRESHOLD]
|
||||
|
||||
def _filter_noise(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
过滤噪声结果
|
||||
|
||||
过滤条件：
|
||||
- 文本长度小于最小长度
|
||||
- 纯符号或空白字符
|
||||
|
||||
Args:
|
||||
results: 识别结果列表
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 过滤后的结果
|
||||
"""
|
||||
filtered = []
|
||||
|
||||
for result in results:
|
||||
text = result['text'].strip()
|
||||
|
||||
# 过滤过短的文本
|
||||
if len(text) < self.MIN_SUBTITLE_LENGTH:
|
||||
continue
|
||||
|
||||
# 过滤纯符号或空白字符
|
||||
if self._is_noise(text):
|
||||
continue
|
||||
|
||||
filtered.append(result)
|
||||
|
||||
return filtered
|
||||
|
||||
def _is_noise(self, text: str) -> bool:
|
||||
"""
|
||||
判断文本是否为噪声
|
||||
|
||||
Args:
|
||||
text: 要判断的文本
|
||||
|
||||
Returns:
|
||||
bool: 如果是噪声返回 True
|
||||
"""
|
||||
# 使用正则判断是否只包含符号和空白
|
||||
try:
|
||||
# 移除标点符号后检查是否为空
|
||||
# 先替换常见的中文标点
|
||||
cleaned = re.sub(r'[^\w\u4e00-\u9fff]', '', text)
|
||||
return len(cleaned) == 0
|
||||
except:
|
||||
# 如果正则出错，保守处理，不算噪声
|
||||
return False
|
||||
|
||||
def _remove_duplicates(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
去除重复或相似的文本
|
||||
|
||||
基于编辑距离（Levenshtein distance）判断文本相似度
|
||||
|
||||
Args:
|
||||
results: 识别结果列表（假设已按时间戳排序）
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 去重后的结果
|
||||
"""
|
||||
if not results:
|
||||
return []
|
||||
|
||||
deduped = [results[0]]
|
||||
|
||||
for current in results[1:]:
|
||||
last = deduped[-1]
|
||||
|
||||
# 计算相似度
|
||||
similarity = self._calculate_similarity(current['text'], last['text'])
|
||||
|
||||
# 如果相似度低于阈值，则保留
|
||||
if similarity < self.SIMILARITY_THRESHOLD:
|
||||
deduped.append(current)
|
||||
else:
|
||||
# 相似文本：保留置信度更高的那个
|
||||
if current['confidence'] > last['confidence']:
|
||||
deduped[-1] = current
|
||||
|
||||
return deduped
|
||||
|
||||
def _calculate_similarity(self, text1: str, text2: str) -> float:
|
||||
"""
|
||||
计算两个文本的相似度（基于归一化编辑距离）
|
||||
|
||||
Args:
|
||||
text1: 文本1
|
||||
text2: 文本2
|
||||
|
||||
Returns:
|
||||
float: 相似度 (0-1)，1 表示完全相同
|
||||
"""
|
||||
if text1 == text2:
|
||||
return 1.0
|
||||
|
||||
len1, len2 = len(text1), len(text2)
|
||||
|
||||
if len1 == 0 or len2 == 0:
|
||||
return 0.0
|
||||
|
||||
# 动态规划计算编辑距离
|
||||
dp = np.zeros((len1 + 1, len2 + 1))
|
||||
|
||||
for i in range(len1 + 1):
|
||||
dp[i, 0] = i
|
||||
|
||||
for j in range(len2 + 1):
|
||||
dp[0, j] = j
|
||||
|
||||
for i in range(1, len1 + 1):
|
||||
for j in range(1, len2 + 1):
|
||||
if text1[i-1] == text2[j-1]:
|
||||
dp[i, j] = dp[i-1, j-1]
|
||||
else:
|
||||
dp[i, j] = min(
|
||||
dp[i-1, j] + 1, # 删除
|
||||
dp[i, j-1] + 1, # 插入
|
||||
dp[i-1, j-1] + 1 # 替换
|
||||
)
|
||||
|
||||
edit_distance = dp[len1, len2]
|
||||
max_len = max(len1, len2)
|
||||
|
||||
# 归一化相似度：1 - (编辑距离 / 最大长度)
|
||||
similarity = 1.0 - (edit_distance / max_len)
|
||||
|
||||
return similarity
|
||||
|
||||
def get_ocr_info(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取 OCR 处理器信息
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: OCR 配置信息
|
||||
"""
|
||||
return {
|
||||
'engine': 'easyocr',
|
||||
'lang': self.lang,
|
||||
'use_gpu': self.use_gpu,
|
||||
'detail': self.detail,
|
||||
'confidence_threshold': self.CONFIDENCE_THRESHOLD,
|
||||
'similarity_threshold': self.SIMILARITY_THRESHOLD,
|
||||
'min_subtitle_length': self.MIN_SUBTITLE_LENGTH
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
"""
|
||||
OCR 文字识别处理器
|
||||
|
||||
使用 PaddleOCR 实现视频帧文字识别功能，主要用于提取硬字幕
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import numpy as np
|
||||
from typing import List, Dict, Any, Tuple
|
||||
from paddleocr import PaddleOCR
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OCRProcessor:
|
||||
"""
|
||||
OCR 文字识别处理器
|
||||
|
||||
使用 PaddleOCR 从视频帧中识别文字，支持批量识别和结果过滤
|
||||
"""
|
||||
|
||||
# 相似度阈值：用于判断两个文本是否相似（基于编辑距离）
|
||||
SIMILARITY_THRESHOLD = 0.85
|
||||
|
||||
# 置信度阈值
|
||||
CONFIDENCE_THRESHOLD = 0.7
|
||||
|
||||
# 最小字幕长度（字符数）
|
||||
MIN_SUBTITLE_LENGTH = 2
|
||||
|
||||
# 注意：Python re 模块不支持 \p{P} 等属性类，在 _is_noise 方法中使用自定义逻辑
|
||||
|
||||
def __init__(self, use_textline_orientation: bool = True, lang: str = 'ch',
|
||||
use_gpu: bool = False):
|
||||
"""
|
||||
初始化 OCR 处理器
|
||||
|
||||
Args:
|
||||
use_textline_orientation: 是否使用文本行方向分类器（PaddleOCR 3.x 参数名）
|
||||
lang: 语言支持，'ch' 为中英文混合
|
||||
use_gpu: 是否使用 GPU 加速（通过环境变量设置，如需启用请先设置 CUDA_VISIBLE_DEVICES）
|
||||
"""
|
||||
self.use_textline_orientation = use_textline_orientation
|
||||
self.lang = lang
|
||||
self.use_gpu = use_gpu
|
||||
|
||||
# 初始化 PaddleOCR 实例
|
||||
self._ocr = None
|
||||
self._initialize_ocr()
|
||||
|
||||
def _initialize_ocr(self):
|
||||
"""初始化 PaddleOCR 实例"""
|
||||
try:
|
||||
# 强制禁用 oneDNN 和 MKL
|
||||
os.environ['FLAGS_use_mkldnn'] = '0'
|
||||
os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
|
||||
os.environ['OMP_NUM_THREADS'] = '1'
|
||||
os.environ['PADDLE_ON_INFER'] = '1'
|
||||
|
||||
# PaddleOCR 3.x 版本参数
|
||||
self._ocr = PaddleOCR(
|
||||
use_textline_orientation=self.use_textline_orientation,
|
||||
lang=self.lang
|
||||
)
|
||||
logger.info("PaddleOCR 初始化成功（oneDNN 已禁用）")
|
||||
except Exception as e:
|
||||
logger.error(f"PaddleOCR 初始化失败: {e}")
|
||||
raise RuntimeError(f"无法初始化 PaddleOCR: {e}")
|
||||
|
||||
def recognize_frames(self, frames: List[Tuple[np.ndarray, float]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
批量识别帧中的文字
|
||||
|
||||
Args:
|
||||
frames: 帧列表，每个元素为 (图像数组, 时间戳) 元组
|
||||
- 图像数组：numpy.ndarray，形状为 (H, W, 3) 的 RGB 图像
|
||||
- 时间戳：float，单位为秒
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 识别结果列表，每个结果包含：
|
||||
- time: float, 时间戳（秒）
|
||||
- text: str, 识别的文本
|
||||
- confidence: float, 置信度 (0-1)
|
||||
"""
|
||||
if not frames:
|
||||
logger.warning("输入帧列表为空")
|
||||
return []
|
||||
|
||||
results = []
|
||||
|
||||
for image, timestamp in frames:
|
||||
try:
|
||||
# 调用 PaddleOCR 进行识别
|
||||
# 注意：PaddleOCR 3.x 版本的 ocr() 方法参数已简化
|
||||
ocr_result = self._ocr.ocr(image)
|
||||
|
||||
# PaddleOCR 返回格式: [[[box], (text, confidence)], ...]
|
||||
if ocr_result and ocr_result[0]:
|
||||
for line in ocr_result[0]:
|
||||
box, (text, confidence) = line
|
||||
if text and confidence:
|
||||
results.append({
|
||||
'time': timestamp,
|
||||
'text': text,
|
||||
'confidence': float(confidence)
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"帧识别失败 (时间戳 {timestamp}s): {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"原始识别结果数量: {len(results)}")
|
||||
|
||||
# 过滤和清洗结果
|
||||
filtered_results = self._filter_and_clean(results)
|
||||
logger.info(f"过滤后结果数量: {len(filtered_results)}")
|
||||
|
||||
return filtered_results
|
||||
|
||||
def _filter_and_clean(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
过滤和清洗识别结果
|
||||
|
||||
Args:
|
||||
results: 原始识别结果列表
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 过滤后的结果列表
|
||||
"""
|
||||
if not results:
|
||||
return []
|
||||
|
||||
# 步骤 1: 按置信度过滤
|
||||
filtered = self._filter_by_confidence(results)
|
||||
logger.debug(f"置信度过滤后: {len(filtered)}")
|
||||
|
||||
# 步骤 2: 过滤噪声（纯符号、单字符等）
|
||||
filtered = self._filter_noise(filtered)
|
||||
logger.debug(f"噪声过滤后: {len(filtered)}")
|
||||
|
||||
# 步骤 3: 去除重复或相似的文本
|
||||
filtered = self._remove_duplicates(filtered)
|
||||
logger.debug(f"去重后: {len(filtered)}")
|
||||
|
||||
# 步骤 4: 按时间戳排序
|
||||
filtered.sort(key=lambda x: x['time'])
|
||||
|
||||
return filtered
|
||||
|
||||
def _filter_by_confidence(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
按置信度过滤结果
|
||||
|
||||
Args:
|
||||
results: 识别结果列表
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 高置信度结果
|
||||
"""
|
||||
return [r for r in results if r['confidence'] >= self.CONFIDENCE_THRESHOLD]
|
||||
|
||||
def _filter_noise(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
过滤噪声结果
|
||||
|
||||
过滤条件：
|
||||
- 文本长度小于最小长度
|
||||
- 纯符号或空白字符
|
||||
|
||||
Args:
|
||||
results: 识别结果列表
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 过滤后的结果
|
||||
"""
|
||||
filtered = []
|
||||
|
||||
for result in results:
|
||||
text = result['text'].strip()
|
||||
|
||||
# 过滤过短的文本
|
||||
if len(text) < self.MIN_SUBTITLE_LENGTH:
|
||||
continue
|
||||
|
||||
# 过滤纯符号或空白字符
|
||||
if self._is_noise(text):
|
||||
continue
|
||||
|
||||
filtered.append(result)
|
||||
|
||||
return filtered
|
||||
|
||||
def _is_noise(self, text: str) -> bool:
|
||||
"""
|
||||
判断文本是否为噪声
|
||||
|
||||
Args:
|
||||
text: 要判断的文本
|
||||
|
||||
Returns:
|
||||
bool: 如果是噪声返回 True
|
||||
"""
|
||||
# 使用正则判断是否只包含符号和空白
|
||||
# 注意：这里用简单的正则，更复杂的判断可以后续优化
|
||||
try:
|
||||
# 移除标点符号后检查是否为空
|
||||
# 先替换常见的中文标点
|
||||
cleaned = re.sub(r'[^\w\u4e00-\u9fff]', '', text)
|
||||
return len(cleaned) == 0
|
||||
except:
|
||||
# 如果正则出错，保守处理，不算噪声
|
||||
return False
|
||||
|
||||
def _remove_duplicates(self, results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
去除重复或相似的文本
|
||||
|
||||
基于编辑距离（Levenshtein distance）判断文本相似度
|
||||
|
||||
Args:
|
||||
results: 识别结果列表（假设已按时间戳排序）
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 去重后的结果
|
||||
"""
|
||||
if not results:
|
||||
return []
|
||||
|
||||
deduped = [results[0]]
|
||||
|
||||
for current in results[1:]:
|
||||
last = deduped[-1]
|
||||
|
||||
# 计算相似度
|
||||
similarity = self._calculate_similarity(current['text'], last['text'])
|
||||
|
||||
# 如果相似度低于阈值，则保留
|
||||
if similarity < self.SIMILARITY_THRESHOLD:
|
||||
deduped.append(current)
|
||||
else:
|
||||
# 相似文本：保留置信度更高的那个
|
||||
if current['confidence'] > last['confidence']:
|
||||
deduped[-1] = current
|
||||
|
||||
return deduped
|
||||
|
||||
def _calculate_similarity(self, text1: str, text2: str) -> float:
|
||||
"""
|
||||
计算两个文本的相似度（基于归一化编辑距离）
|
||||
|
||||
Args:
|
||||
text1: 文本1
|
||||
text2: 文本2
|
||||
|
||||
Returns:
|
||||
float: 相似度 (0-1)，1 表示完全相同
|
||||
"""
|
||||
if text1 == text2:
|
||||
return 1.0
|
||||
|
||||
len1, len2 = len(text1), len(text2)
|
||||
|
||||
if len1 == 0 or len2 == 0:
|
||||
return 0.0
|
||||
|
||||
# 动态规划计算编辑距离
|
||||
dp = np.zeros((len1 + 1, len2 + 1))
|
||||
|
||||
for i in range(len1 + 1):
|
||||
dp[i, 0] = i
|
||||
|
||||
for j in range(len2 + 1):
|
||||
dp[0, j] = j
|
||||
|
||||
for i in range(1, len1 + 1):
|
||||
for j in range(1, len2 + 1):
|
||||
if text1[i-1] == text2[j-1]:
|
||||
dp[i, j] = dp[i-1, j-1]
|
||||
else:
|
||||
dp[i, j] = min(
|
||||
dp[i-1, j] + 1, # 删除
|
||||
dp[i, j-1] + 1, # 插入
|
||||
dp[i-1, j-1] + 1 # 替换
|
||||
)
|
||||
|
||||
edit_distance = dp[len1, len2]
|
||||
max_len = max(len1, len2)
|
||||
|
||||
# 归一化相似度：1 - (编辑距离 / 最大长度)
|
||||
similarity = 1.0 - (edit_distance / max_len)
|
||||
|
||||
return similarity
|
||||
|
||||
def get_ocr_info(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取 OCR 处理器信息
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: OCR 配置信息
|
||||
"""
|
||||
return {
|
||||
'use_textline_orientation': self.use_textline_orientation,
|
||||
'lang': self.lang,
|
||||
'use_gpu': self.use_gpu,
|
||||
'confidence_threshold': self.CONFIDENCE_THRESHOLD,
|
||||
'similarity_threshold': self.SIMILARITY_THRESHOLD,
|
||||
'min_subtitle_length': self.MIN_SUBTITLE_LENGTH
|
||||
}
|
||||
@@ -35,6 +35,10 @@ class DrissionBrowser:
|
||||
from DrissionPage import ChromiumOptions, ChromiumPage
|
||||
|
||||
options = ChromiumOptions()
|
||||
|
||||
# 设置浏览器路径（服务器环境需要）
|
||||
options.set_browser_path('/usr/bin/chromium-browser')
|
||||
|
||||
if self.headless:
|
||||
options.headless()
|
||||
options.set_user_agent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36')
|
||||
@@ -116,10 +120,17 @@ class ZhihuDrissionExtractor:
|
||||
|
||||
self.browser = DrissionBrowser(headless=self.headless, timeout=self.timeout)
|
||||
self.markdown_converter = MarkdownConverter()
|
||||
self.cookie_file = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
'data', 'cookies', 'zhihu.json'
|
||||
)
|
||||
# 从配置中读取 Cookie 文件路径，支持环境变量展开
|
||||
# 直接读取 cookie_file（因为 self.config 已经是 zhihu 配置）
|
||||
cookie_file = self.config.get('cookie_file')
|
||||
if cookie_file:
|
||||
self.cookie_file = os.path.expanduser(cookie_file)
|
||||
else:
|
||||
# 默认路径
|
||||
self.cookie_file = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||
'data', 'cookies', 'zhihu.json'
|
||||
)
|
||||
|
||||
def identify_url_type(self, url: str) -> Optional[str]:
|
||||
"""
|
||||
@@ -180,9 +191,11 @@ class ZhihuDrissionExtractor:
|
||||
cookies = []
|
||||
|
||||
# 1. 尝试从配置加载（优先级最高）
|
||||
if self.config.get('zhihu.cookie_str'):
|
||||
# 支持扁平和嵌套配置结构
|
||||
cookie_str = self.config.get('cookie_str')
|
||||
if cookie_str:
|
||||
try:
|
||||
cookies = json.loads(self.config.get('zhihu.cookie_str'))
|
||||
cookies = json.loads(cookie_str)
|
||||
print(f"从配置加载 Cookie，共 {len(cookies)} 个")
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"配置中的 Cookie JSON 格式错误: {e}")
|
||||
@@ -197,7 +210,16 @@ class ZhihuDrissionExtractor:
|
||||
|
||||
# 3. 尝试从文件加载（优先级最低）
|
||||
else:
|
||||
cookie_file = self.config.get('zhihu.cookie_file', self.cookie_file)
|
||||
# 直接从配置中读取 cookie_file（因为 self.config 已经是 zhihu 配置）
|
||||
cookie_file = self.config.get('cookie_file')
|
||||
|
||||
if not cookie_file:
|
||||
# 如果没有配置，使用默认路径
|
||||
cookie_file = self.cookie_file
|
||||
else:
|
||||
# 展开环境变量（如 ~）
|
||||
cookie_file = os.path.expanduser(cookie_file)
|
||||
|
||||
if cookie_file and Path(cookie_file).exists():
|
||||
try:
|
||||
with open(cookie_file, 'r', encoding='utf-8') as f:
|
||||
@@ -206,7 +228,8 @@ class ZhihuDrissionExtractor:
|
||||
except (json.JSONDecodeError, FileNotFoundError, IOError) as e:
|
||||
print(f"从文件加载 Cookie 失败: {e}")
|
||||
else:
|
||||
print("未找到 Cookie 配置，将使用未登录状态访问知乎")
|
||||
print(f"未找到 Cookie 配置，将使用未登录状态访问知乎")
|
||||
print(f" 期望路径: {cookie_file}")
|
||||
|
||||
return cookies
|
||||
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
字幕检测器测试脚本
|
||||
|
||||
验证：
|
||||
1. 预设区域定义
|
||||
2. 自动字幕区域检测
|
||||
3. 缓存机制
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
# 添加项目路径
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from src.extractors.subtitle_detector import SubtitleDetector
|
||||
|
||||
|
||||
def create_test_frame_with_text(width=1280, height=720):
|
||||
"""
|
||||
创建包含模拟字幕的测试帧
|
||||
|
||||
Args:
|
||||
width: 帧宽度
|
||||
height: 帧高度
|
||||
|
||||
Returns:
|
||||
numpy.ndarray: 测试帧
|
||||
"""
|
||||
# 创建黑色背景
|
||||
frame = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
|
||||
# 添加一些噪声背景
|
||||
noise = np.random.randint(0, 50, (height, width, 3), dtype=np.uint8)
|
||||
frame = cv2.add(frame, noise)
|
||||
|
||||
# 在底部添加白色文字（模拟B站字幕）
|
||||
font = cv2.FONT_HERSHEY_SIMPLEX
|
||||
font_scale = 1.0
|
||||
thickness = 2
|
||||
|
||||
# 添加多行字幕
|
||||
texts = [
|
||||
"这是测试字幕内容",
|
||||
"B站硬字幕提取测试",
|
||||
"00:01:23 第一行字幕",
|
||||
"00:01:25 第二行字幕"
|
||||
]
|
||||
|
||||
y_start = int(height * 0.85)
|
||||
line_height = 35
|
||||
|
||||
for i, text in enumerate(texts):
|
||||
y = y_start + i * line_height
|
||||
x = int(width * 0.15)
|
||||
|
||||
# 绘制文字阴影（黑色）
|
||||
cv2.putText(frame, text, (x + 2, y + 2), font, font_scale, (0, 0, 0), thickness + 2)
|
||||
# 绘制白色文字
|
||||
cv2.putText(frame, text, (x, y), font, font_scale, (255, 255, 255), thickness)
|
||||
|
||||
return frame
|
||||
|
||||
|
||||
def test_preset_regions():
|
||||
"""测试1: 预设区域定义"""
|
||||
print("\n" + "="*60)
|
||||
print("测试1: 预设区域定义")
|
||||
print("="*60)
|
||||
|
||||
detector = SubtitleDetector()
|
||||
frame_width, frame_height = 1280, 720
|
||||
|
||||
# 测试所有预设区域
|
||||
for region_name in [detector.REGION_BOTTOM, detector.REGION_CENTER, detector.REGION_TOP]:
|
||||
try:
|
||||
x, y, width, height = detector.get_preset_region(region_name, frame_height, frame_width)
|
||||
print(f"\n✓ 区域 '{region_name}':")
|
||||
print(f" 坐标: x={x}, y={y}")
|
||||
print(f" 尺寸: width={width}, height={height}")
|
||||
print(f" 相对位置: top={y/frame_height:.2f}, bottom={(y+height)/frame_height:.2f}")
|
||||
except Exception as e:
|
||||
print(f"✗ 区域 '{region_name}' 失败: {e}")
|
||||
|
||||
# 测试无效区域名称
|
||||
try:
|
||||
detector.get_preset_region("invalid", frame_height, frame_width)
|
||||
print("\n✗ 错误: 应该抛出异常但没有")
|
||||
except ValueError as e:
|
||||
print(f"\n✓ 无效区域名称正确抛出异常: {e}")
|
||||
|
||||
print("\n测试1完成!")
|
||||
|
||||
|
||||
def test_auto_detection():
|
||||
"""测试2: 自动字幕区域检测"""
|
||||
print("\n" + "="*60)
|
||||
print("测试2: 自动字幕区域检测")
|
||||
print("="*60)
|
||||
|
||||
detector = SubtitleDetector()
|
||||
|
||||
# 创建测试帧（带字幕）
|
||||
print("\n创建测试帧...")
|
||||
test_frame = create_test_frame_with_text()
|
||||
height, width = test_frame.shape[:2]
|
||||
print(f"帧尺寸: {width}x{height}")
|
||||
|
||||
# 保存测试帧用于可视化
|
||||
test_frame_path = "/tmp/test_subtitle_frame.png"
|
||||
cv2.imwrite(test_frame_path, test_frame)
|
||||
print(f"✓ 测试帧已保存到: {test_frame_path}")
|
||||
|
||||
# 执行自动检测
|
||||
print("\n执行自动检测...")
|
||||
try:
|
||||
x, y, w, h = detector.auto_detect_subtitle_region(test_frame, video_id=None, use_cache=False)
|
||||
print(f"✓ 检测到字幕区域:")
|
||||
print(f" 坐标: x={x}, y={y}")
|
||||
print(f" 尺寸: width={w}, height={h}")
|
||||
print(f" 相对位置: top={y/height:.2f}, bottom={(y+h)/height:.2f}")
|
||||
|
||||
# 在测试帧上绘制检测框
|
||||
frame_with_box = test_frame.copy()
|
||||
cv2.rectangle(frame_with_box, (x, y), (x+w, y+h), (0, 255, 0), 2)
|
||||
|
||||
# 保存结果
|
||||
result_path = "/tmp/test_subtitle_detected.png"
|
||||
cv2.imwrite(result_path, frame_with_box)
|
||||
print(f"✓ 检测结果已保存到: {result_path}")
|
||||
|
||||
# 验证检测区域是否合理（应该在底部）
|
||||
region_bottom = (y + h) / height
|
||||
if region_bottom > 0.7: # 底部30%
|
||||
print(f"✓ 检测区域位置合理（底部）")
|
||||
else:
|
||||
print(f"! 警告: 检测区域可能不在底部 (bottom={region_bottom:.2f})")
|
||||
|
||||
except Exception as e:
|
||||
print(f"✗ 自动检测失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n测试2完成!")
|
||||
|
||||
|
||||
def test_cache_mechanism():
|
||||
"""测试3: 缓存机制"""
|
||||
print("\n" + "="*60)
|
||||
print("测试3: 缓存机制")
|
||||
print("="*60)
|
||||
|
||||
detector = SubtitleDetector(cache_size=3) # 小缓存便于测试
|
||||
|
||||
# 创建测试帧
|
||||
test_frame = create_test_frame_with_text()
|
||||
|
||||
# 测试1: 首次检测并缓存
|
||||
print("\n1. 首次检测并缓存...")
|
||||
video_id1 = "BV1xx411c7mD"
|
||||
region1 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id1, use_cache=True)
|
||||
print(f"✓ 检测完成，区域: {region1}")
|
||||
print(f"✓ 缓存大小: {detector.get_cache_size()}")
|
||||
|
||||
# 测试2: 再次检测同一视频（应使用缓存）
|
||||
print("\n2. 再次检测同一视频（应使用缓存）...")
|
||||
region2 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id1, use_cache=True)
|
||||
print(f"✓ 缓存结果: {region2}")
|
||||
if region1 == region2:
|
||||
print(f"✓ 缓存命中，结果一致")
|
||||
else:
|
||||
print(f"✗ 警告: 缓存结果不一致")
|
||||
|
||||
# 测试3: 检测不同视频
|
||||
print("\n3. 检测不同视频...")
|
||||
video_id2 = "BV1yy411c7mE"
|
||||
region3 = detector.auto_detect_subtitle_region(test_frame, video_id=video_id2, use_cache=True)
|
||||
print(f"✓ 新视频检测完成，区域: {region3}")
|
||||
print(f"✓ 缓存大小: {detector.get_cache_size()}")
|
||||
|
||||
# 测试4: 查看所有缓存
|
||||
print("\n4. 查看所有缓存...")
|
||||
cached = detector.get_cached_regions()
|
||||
print(f"✓ 缓存内容:")
|
||||
for vid, reg in cached.items():
|
||||
print(f" {vid}: {reg}")
|
||||
|
||||
# 测试5: 清空指定视频缓存
|
||||
print("\n5. 清空指定视频缓存...")
|
||||
detector.clear_cache(video_id=video_id1)
|
||||
print(f"✓ 已删除 {video_id1}")
|
||||
print(f"✓ 缓存大小: {detector.get_cache_size()}")
|
||||
|
||||
# 测试6: 清空所有缓存
|
||||
print("\n6. 清空所有缓存...")
|
||||
detector.clear_cache()
|
||||
print(f"✓ 缓存已清空")
|
||||
print(f"✓ 缓存大小: {detector.get_cache_size()}")
|
||||
|
||||
# 测试7: 缓存容量限制（LRU）
|
||||
print("\n7. 测试缓存容量限制（LRU）...")
|
||||
for i in range(5):
|
||||
vid = f"BV{i:012d}"
|
||||
detector.auto_detect_subtitle_region(test_frame, video_id=vid, use_cache=True)
|
||||
print(f" 添加视频 {vid}，缓存大小: {detector.get_cache_size()}")
|
||||
|
||||
cached = detector.get_cached_regions()
|
||||
print(f"✓ 缓存中有 {len(cached)} 个视频（应为最大容量3）")
|
||||
print(f"✓ 保留的视频: {list(cached.keys())}")
|
||||
|
||||
print("\n测试3完成!")
|
||||
|
||||
|
||||
def test_integration():
|
||||
"""测试4: 集成测试"""
|
||||
print("\n" + "="*60)
|
||||
print("测试4: 集成测试")
|
||||
print("="*60)
|
||||
|
||||
detector = SubtitleDetector()
|
||||
test_frame = create_test_frame_with_text()
|
||||
height, width = test_frame.shape[:2]
|
||||
|
||||
print("\n模拟B站视频字幕检测流程...")
|
||||
|
||||
# 步骤1: 尝试自动检测
|
||||
print("\n步骤1: 自动检测字幕区域")
|
||||
video_id = "BV1xx411c7mD"
|
||||
region = detector.auto_detect_subtitle_region(test_frame, video_id=video_id)
|
||||
print(f"✓ 自动检测结果: {region}")
|
||||
|
||||
# 步骤2: 后续帧使用缓存
|
||||
print("\n步骤2: 后续帧使用缓存")
|
||||
for i in range(3):
|
||||
cached_region = detector.auto_detect_subtitle_region(test_frame, video_id=video_id)
|
||||
if cached_region == region:
|
||||
print(f"✓ 帧 {i+1}: 使用缓存，结果一致")
|
||||
else:
|
||||
print(f"✗ 帧 {i+1}: 缓存结果异常")
|
||||
|
||||
# 步骤3: 对比预设区域
|
||||
print("\n步骤3: 对比预设区域")
|
||||
bottom_region = detector.get_preset_region(detector.REGION_BOTTOM, height, width)
|
||||
print(f" 检测区域: y={region[1]}, height={region[3]}")
|
||||
print(f" 底部预设: y={bottom_region[1]}, height={bottom_region[3]}")
|
||||
|
||||
# 步骤4: 提取字幕区域
|
||||
print("\n步骤4: 提取字幕区域用于OCR")
|
||||
x, y, w, h = region
|
||||
subtitle_roi = test_frame[y:y+h, x:x+w]
|
||||
subtitle_path = "/tmp/subtitle_roi.png"
|
||||
cv2.imwrite(subtitle_path, subtitle_roi)
|
||||
print(f"✓ 字幕ROI已保存到: {subtitle_path}")
|
||||
print(f" ROI尺寸: {w}x{h}")
|
||||
|
||||
print("\n测试4完成!")
|
||||
|
||||
|
||||
def main():
|
||||
"""主测试函数"""
|
||||
print("\n" + "="*60)
|
||||
print("字幕检测器完整测试")
|
||||
print("="*60)
|
||||
|
||||
try:
|
||||
# 运行所有测试
|
||||
test_preset_regions()
|
||||
test_auto_detection()
|
||||
test_cache_mechanism()
|
||||
test_integration()
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("所有测试完成！")
|
||||
print("="*60)
|
||||
|
||||
print("\n生成的测试文件:")
|
||||
print(" - /tmp/test_subtitle_frame.png (原始测试帧)")
|
||||
print(" - /tmp/test_subtitle_detected.png (带检测框的结果)")
|
||||
print(" - /tmp/subtitle_roi.png (提取的字幕区域)")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n✗ 测试失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
验证知乎 Cookie 配置
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 添加 src 目录到 Python 路径
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
|
||||
|
||||
from sources.zhihu_drission_extractor import ZhihuDrissionExtractor
|
||||
|
||||
|
||||
def verify_cookie_config():
|
||||
"""验证 Cookie 配置"""
|
||||
print("=" * 60)
|
||||
print("知乎 Cookie 配置验证")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# 1. 检查 Cookie 文件路径
|
||||
cookie_file = os.path.expanduser('~/.obsidian-kb/zhihu_cookies.json')
|
||||
print(f"1. Cookie 文件路径")
|
||||
print(f" {cookie_file}")
|
||||
if os.path.exists(cookie_file):
|
||||
print(f" ✅ 文件存在")
|
||||
else:
|
||||
print(f" ❌ 文件不存在")
|
||||
print(f" 请上传 Cookie 文件到此路径")
|
||||
print()
|
||||
return False
|
||||
|
||||
print()
|
||||
|
||||
# 2. 检查 Cookie 文件格式
|
||||
print(f"2. Cookie 文件格式")
|
||||
try:
|
||||
with open(cookie_file, 'r', encoding='utf-8') as f:
|
||||
cookies = json.load(f)
|
||||
print(f" ✅ JSON 格式正确")
|
||||
print(f" Cookie 数量: {len(cookies)}")
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" ❌ JSON 格式错误: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ❌ 读取失败: {e}")
|
||||
return False
|
||||
|
||||
print()
|
||||
|
||||
# 3. 检查关键 Cookie
|
||||
print(f"3. 关键 Cookie 检查")
|
||||
cookie_names = [c['name'] for c in cookies]
|
||||
required_cookies = ['z_c0', 'd_c0']
|
||||
|
||||
for cookie_name in required_cookies:
|
||||
if cookie_name in cookie_names:
|
||||
cookie = next(c for c in cookies if c['name'] == cookie_name)
|
||||
value_preview = cookie['value'][:10] + '...' if len(cookie['value']) > 10 else cookie['value']
|
||||
print(f" ✅ {cookie_name}: {value_preview}")
|
||||
else:
|
||||
print(f" ⚠️ {cookie_name}: 未找到（可能影响登录状态）")
|
||||
|
||||
print()
|
||||
|
||||
# 4. 检查配置文件
|
||||
print(f"4. 配置文件检查")
|
||||
config_file = os.path.join(os.path.dirname(__file__), 'config.yaml')
|
||||
if os.path.exists(config_file):
|
||||
print(f" ✅ config.yaml 存在")
|
||||
with open(config_file, 'r') as f:
|
||||
content = f.read()
|
||||
if 'zhihu_cookies.json' in content:
|
||||
print(f" ✅ Cookie 路径已配置")
|
||||
else:
|
||||
print(f" ⚠️ Cookie 路径未找到（可能是使用了其他配置）")
|
||||
else:
|
||||
print(f" ❌ config.yaml 不存在")
|
||||
|
||||
print()
|
||||
|
||||
# 5. 测试提取器初始化
|
||||
print(f"5. 提取器初始化测试")
|
||||
try:
|
||||
# 加载配置文件并传递给提取器
|
||||
import yaml
|
||||
with open(config_file, 'r', encoding='utf-8') as f:
|
||||
full_config = yaml.safe_load(f)
|
||||
|
||||
print(f" 配置文件: {config_file}")
|
||||
print(f" zhihu 配置: {full_config.get('zhihu', {})}")
|
||||
|
||||
# 提取 zhihu 配置并传递
|
||||
zhihu_config = full_config.get('zhihu', {})
|
||||
extractor = ZhihuDrissionExtractor(config=zhihu_config)
|
||||
print(f" ✅ ZhihuDrissionExtractor 初始化成功")
|
||||
|
||||
# 测试 Cookie 加载
|
||||
loaded_cookies = extractor._load_cookies()
|
||||
print(f" ✅ 成功加载 {len(loaded_cookies)} 个 Cookie")
|
||||
|
||||
if len(loaded_cookies) == 0:
|
||||
print(f" ⚠️ 未加载到 Cookie，将使用未登录状态访问")
|
||||
print(f" 这可能导致提取失败或内容不完整")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ 初始化失败: {e}")
|
||||
return False
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("✅ 配置验证完成！")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# 6. 下一步建议
|
||||
print("下一步：")
|
||||
print("1. 如果 Cookie 文件格式正确，可以直接开始提取")
|
||||
print("2. 建议先测试一个简单的知乎回答链接")
|
||||
print()
|
||||
print("示例命令：")
|
||||
print(" python3 -c \"from src.sources.zhihu_drission_extractor import ZhihuDrissionExtractor; print(ZhihuDrissionExtractor().extract('https://www.zhihu.com/question/266309957/answer/2003427891925509507')['title'])\"")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
success = verify_cookie_config()
|
||||
sys.exit(0 if success else 1)
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
深度测试B站字幕提取 - 尝试所有可能的方法
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from playwright.async_api import async_playwright
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
async def test_all_methods(bvid):
|
||||
"""尝试所有方法获取字幕"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
print(f"正在测试视频: {url}")
|
||||
print("=" * 80)
|
||||
|
||||
# 等待页面完全加载
|
||||
print("\n1. 访问页面...")
|
||||
await page.goto(url, wait_until='networkidle', timeout=60000)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# 获取基本信息
|
||||
print("\n2. 获取视频基本信息...")
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') == 0:
|
||||
video_data = data.get('data', {})
|
||||
print(f" 标题: {video_data.get('title')}")
|
||||
print(f" aid: {video_data.get('aid')}")
|
||||
print(f" cid: {video_data.get('cid')}")
|
||||
cid = video_data.get('cid')
|
||||
aid = video_data.get('aid')
|
||||
else:
|
||||
print(f" ❌ 获取基本信息失败: {data.get('message')}")
|
||||
await browser.close()
|
||||
return
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
await browser.close()
|
||||
return
|
||||
|
||||
# 方法1: 使用标准 API（官方 CC 字幕）
|
||||
print("\n3. 方法1: 官方 CC 字幕 API...")
|
||||
try:
|
||||
player_url = "https://api.bilibili.com/x/player/v2"
|
||||
player_response = await page.request.get(player_url, params={'bvid': bvid, 'cid': cid})
|
||||
player_data = await player_response.json()
|
||||
|
||||
if player_data.get('code') == 0:
|
||||
subtitle_info = player_data.get('data', {}).get('subtitle', {})
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
|
||||
if subtitles_list:
|
||||
print(f" ✅ 找到 {len(subtitles_list)} 个官方字幕")
|
||||
for i, sub in enumerate(subtitles_list):
|
||||
print(f" [{i+1}] {sub.get('lan_doc')} (id={sub.get('id')})")
|
||||
|
||||
# 获取第一个字幕内容
|
||||
subtitle_url = subtitles_list[0].get('subtitle_url', '')
|
||||
if subtitle_url:
|
||||
full_url = f"https:{subtitle_url}"
|
||||
print(f" URL: {subtitle_url}")
|
||||
sub_response = await page.request.get(full_url)
|
||||
if sub_response.ok:
|
||||
sub_data = await sub_response.json()
|
||||
body = sub_data.get('body', [])
|
||||
print(f" ✅ 获取成功，共 {len(body)} 条字幕")
|
||||
print(f" 示例: {body[0] if body else '无'}")
|
||||
else:
|
||||
print(" ❌ 没有找到官方 CC 字幕")
|
||||
print(f" Subtitle info: {json.dumps(subtitle_info, indent=8, ensure_ascii=False)}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
|
||||
# 方法2: 尝试 AI 字幕 API
|
||||
print("\n4. 方法2: AI 自动生成字幕...")
|
||||
try:
|
||||
# B站 AI 字幕可能在另一个 API
|
||||
ai_subtitle_url = "https://api.bilibili.com/x/player/wbi/v2/playurl"
|
||||
# 这个通常用于视频流，但也可能包含字幕信息
|
||||
ai_params = {
|
||||
'bvid': bvid,
|
||||
'cid': cid,
|
||||
'qn': 80, # 清晰度
|
||||
'fnval': 16, # 包含字幕的标志
|
||||
}
|
||||
ai_response = await page.request.get(ai_subtitle_url, params=ai_params)
|
||||
ai_data = await ai_response.json()
|
||||
|
||||
if ai_data.get('code') == 0:
|
||||
print(f" AI API 响应: {json.dumps(list(ai_data.get('data', {}).keys()), indent=8, ensure_ascii=False)}")
|
||||
|
||||
# 检查是否有字幕信息
|
||||
if 'subtitle' in ai_data.get('data', {}):
|
||||
subtitle = ai_data['data']['subtitle']
|
||||
print(f" ✅ 找到 AI 字幕信息")
|
||||
print(f" {json.dumps(subtitle, indent=8, ensure_ascii=False)}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
|
||||
# 方法3: 从页面脚本标签中提取（可能包含字幕配置）
|
||||
print("\n5. 方法3: 从页面 HTML 中提取字幕配置...")
|
||||
try:
|
||||
content = await page.content()
|
||||
soup = BeautifulSoup(content, 'lxml')
|
||||
script_tags = soup.find_all('script')
|
||||
|
||||
found_subtitle = False
|
||||
for script in script_tags:
|
||||
if script.string and ('subtitle' in script.string or 'cc_subtitle' in script.string):
|
||||
# 提取包含字幕配置的 JSON
|
||||
matches = re.findall(r'window\.__(?:playinfo|INITIAL_STATE|subtitle__)\s*=\s*({.+?});', script.string)
|
||||
for match in matches:
|
||||
try:
|
||||
data = json.loads(match)
|
||||
# 查找字幕相关字段
|
||||
if 'subtitle' in str(data) or 'subtitles' in str(data):
|
||||
print(f" ✅ 找到字幕配置")
|
||||
print(f" {json.dumps(data, indent=8, ensure_ascii=False)[:500]}")
|
||||
found_subtitle = True
|
||||
break
|
||||
except:
|
||||
pass
|
||||
if found_subtitle:
|
||||
break
|
||||
|
||||
if not found_subtitle:
|
||||
print(" ❌ 页面中没有找到字幕配置")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
|
||||
# 方法4: 尝试不同的播放器 API 端点
|
||||
print("\n6. 方法4: 尝试其他 API 端点...")
|
||||
endpoints = [
|
||||
f"https://api.bilibili.com/x/player/wbi/v2/player?bvid={bvid}&cid={cid}",
|
||||
f"https://api.bilibili.com/x/player/v2?bvid={bvid}&cid={cid}",
|
||||
f"https://api.bilibili.com/x/player/playurl?bvid={bvid}&cid={cid}&qn=80&fnval=16",
|
||||
]
|
||||
|
||||
for endpoint in endpoints:
|
||||
try:
|
||||
print(f" 测试: {endpoint}")
|
||||
response = await page.request.get(endpoint)
|
||||
if response.ok:
|
||||
data = await response.json()
|
||||
# 检查响应中是否有字幕信息
|
||||
json_str = json.dumps(data, ensure_ascii=False)
|
||||
if 'subtitle' in json_str.lower():
|
||||
print(f" ✅ 可能包含字幕信息")
|
||||
# 只显示相关部分
|
||||
if 'data' in data and 'subtitle' in str(data['data']):
|
||||
print(f" {json.dumps(data.get('data', {}).get('subtitle', {}), indent=12, ensure_ascii=False)}[:500]")
|
||||
else:
|
||||
print(f" 状态码: {response.status}")
|
||||
except Exception as e:
|
||||
print(f" 异常: {e}")
|
||||
|
||||
# 方法5: 尝试使用 aid 而不是 cid
|
||||
print("\n7. 方法5: 尝试使用 aid 参数...")
|
||||
try:
|
||||
if aid:
|
||||
player_url = "https://api.bilibili.com/x/player/v2"
|
||||
response = await page.request.get(player_url, params={'aid': aid, 'bvid': bvid})
|
||||
data = await response.json()
|
||||
print(f" 响应码: {data.get('code')}")
|
||||
if data.get('code') == 0:
|
||||
subtitle_info = data.get('data', {}).get('subtitle', {})
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
if subtitles_list:
|
||||
print(f" ✅ 使用 aid 找到 {len(subtitles_list)} 个字幕")
|
||||
else:
|
||||
print(f" ❌ 使用 aid 也没有找到字幕")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
|
||||
# 方法6: 检查视频是否有多 P（分集）
|
||||
print("\n8. 方法6: 检查是否为分集视频...")
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') == 0:
|
||||
pages = data.get('data', {}).get('pages', [])
|
||||
print(f" 共 {len(pages)} 个分集")
|
||||
for i, p in enumerate(pages):
|
||||
print(f" [{i+1}] cid={p.get('cid')}, page={p.get('page')}, title={p.get('part')}")
|
||||
|
||||
# 如果有多个分集，尝试每个分集的字幕
|
||||
if len(pages) > 1:
|
||||
print(f"\n 尝试获取第1个分集的字幕...")
|
||||
first_cid = pages[0].get('cid')
|
||||
player_url = "https://api.bilibili.com/x/player/v2"
|
||||
response = await page.request.get(player_url, params={'bvid': bvid, 'cid': first_cid})
|
||||
data = await response.json()
|
||||
subtitle_info = data.get('data', {}).get('subtitle', {})
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
if subtitles_list:
|
||||
print(f" ✅ 第1个分集有 {len(subtitles_list)} 个字幕")
|
||||
else:
|
||||
print(f" ❌ 第1个分集也没有字幕")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
bvid = "BV1KtFzzqEUG"
|
||||
asyncio.run(test_all_methods(bvid))
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
测试修复后的 B 站字幕提取
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
|
||||
# 添加项目路径
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'skills', 'obsidian-kb'))
|
||||
|
||||
from src.extractors.bilibili_extractor import BilibiliExtractor
|
||||
|
||||
async def test_extract():
|
||||
"""测试提取器"""
|
||||
# 创建提取器实例
|
||||
extractor = BilibiliExtractor()
|
||||
|
||||
# 测试视频
|
||||
bvid = "BV1yx411s7Jq"
|
||||
|
||||
print(f"测试视频: {bvid}")
|
||||
print("=" * 60)
|
||||
|
||||
try:
|
||||
# 直接调用异步方法
|
||||
result = await extractor._extract_async(bvid)
|
||||
|
||||
print("\n✅ 提取成功！\n")
|
||||
print("=" * 60)
|
||||
print("元数据:")
|
||||
print("=" * 60)
|
||||
for key, value in result['metadata'].items():
|
||||
print(f" {key}: {value}")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("字幕:")
|
||||
print("=" * 60)
|
||||
subtitle = result['subtitle']
|
||||
if subtitle:
|
||||
lines = subtitle.split('\n')
|
||||
print(f"共 {len(lines)} 条字幕")
|
||||
print("\n前 10 条字幕:")
|
||||
print("\n".join(lines[:10]))
|
||||
if len(lines) > 10:
|
||||
print(f"\n...(还有 {len(lines) - 10} 条)")
|
||||
else:
|
||||
print(" 该视频没有字幕")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ 测试失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(test_extract())
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
播放视频并截屏，查看字幕
|
||||
"""
|
||||
import asyncio
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def play_and_screenshot(bvid):
|
||||
"""播放视频并截屏"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=False) # 使用有头模式
|
||||
context = await browser.new_context(viewport={'width': 1920, 'height': 1080})
|
||||
page = await context.new_page()
|
||||
|
||||
print(f"正在访问视频: {url}")
|
||||
print("=" * 80)
|
||||
|
||||
print("\n1. 访问页面...")
|
||||
await page.goto(url, wait_until='networkidle', timeout=60000)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
print("2. 截取初始状态...")
|
||||
await page.screenshot(path="/tmp/bilibili_1_initial.png")
|
||||
|
||||
print("3. 尝试点击播放按钮...")
|
||||
try:
|
||||
# 尝试多个可能的播放按钮选择器
|
||||
selectors = [
|
||||
'.bpx-player-ctrl-play',
|
||||
'.bpx-player-shadow-play-area',
|
||||
'video',
|
||||
]
|
||||
|
||||
for selector in selectors:
|
||||
element = await page.query_selector(selector)
|
||||
if element:
|
||||
await element.click()
|
||||
print(f" ✅ 已点击 {selector}")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 点击播放失败: {e}")
|
||||
|
||||
print("4. 等待视频播放...")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
print("5. 截取播放中状态...")
|
||||
await page.screenshot(path="/tmp/bilibili_2_playing.png")
|
||||
|
||||
print("6. 尝试点击字幕按钮...")
|
||||
try:
|
||||
# 查找字幕按钮
|
||||
subtitle_btn = await page.query_selector('.bpx-player-ctrl-subtitle')
|
||||
if subtitle_btn:
|
||||
is_enabled = await subtitle_btn.is_enabled()
|
||||
print(f" 字幕按钮状态: {'可用' if is_enabled else '不可用'}")
|
||||
if is_enabled:
|
||||
await subtitle_btn.click()
|
||||
print(" ✅ 已点击字幕按钮")
|
||||
await asyncio.sleep(2)
|
||||
await page.screenshot(path="/tmp/bilibili_3_subtitle_on.png")
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 点击字幕按钮失败: {e}")
|
||||
|
||||
print("7. 获取页面文本内容（包含字幕）...")
|
||||
try:
|
||||
# 获取所有文本
|
||||
body_text = await page.evaluate('() => document.body.innerText')
|
||||
# 查找可能包含字幕的元素
|
||||
subtitle_elements = await page.query_selector_all('[class*="subtitle"], [class*="caption"]')
|
||||
print(f" 找到 {len(subtitle_elements)} 个可能的字幕元素")
|
||||
for i, elem in enumerate(subtitle_elements[:5]):
|
||||
text = await elem.inner_text()
|
||||
if text.strip():
|
||||
print(f" [{i+1}] {text[:100]}")
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 获取文本失败: {e}")
|
||||
|
||||
print("\n8. 检查页面中是否有动态字幕数据...")
|
||||
try:
|
||||
script_result = await page.evaluate('''
|
||||
() => {
|
||||
const results = {
|
||||
hasSubtitlePlugin: false,
|
||||
subtitleText: '',
|
||||
captions: [],
|
||||
};
|
||||
|
||||
// 检查播放器实例
|
||||
const playerContainer = document.querySelector('.bpx-player-container');
|
||||
if (playerContainer) {
|
||||
results.playerExists = true;
|
||||
}
|
||||
|
||||
// 查找字幕元素
|
||||
const captionElements = document.querySelectorAll('.bpx-player-caption');
|
||||
captionElements.forEach(el => {
|
||||
results.captions.push(el.innerText);
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
''')
|
||||
print(f" 检查结果: {script_result}")
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 检查失败: {e}")
|
||||
|
||||
print("\n9. 等待30秒查看效果...")
|
||||
await asyncio.sleep(30)
|
||||
|
||||
print("10. 截取最终状态...")
|
||||
await page.screenshot(path="/tmp/bilibili_4_final.png")
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("测试完成！请检查以下截图：")
|
||||
print(" /tmp/bilibili_1_initial.png - 初始状态")
|
||||
print(" /tmp/bilibili_2_playing.png - 播放中")
|
||||
print(" /tmp/bilibili_3_subtitle_on.png - 开启字幕后（如果有）")
|
||||
print(" /tmp/bilibili_4_final.png - 最终状态")
|
||||
print("=" * 80)
|
||||
|
||||
print("\n按回车键关闭浏览器...")
|
||||
# 等待用户确认（但在自动化环境中会自动继续）
|
||||
await asyncio.sleep(5)
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
bvid = "BV1KtFzzqEUG"
|
||||
asyncio.run(play_and_screenshot(bvid))
|
||||
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
测试动态字幕加载 - 播放视频并监听字幕请求
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def test_dynamic_subtitle(bvid):
|
||||
"""测试动态字幕加载"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
print(f"正在测试视频: {url}")
|
||||
print("=" * 80)
|
||||
|
||||
# 监听所有网络请求，查找字幕相关的请求
|
||||
subtitle_requests = []
|
||||
|
||||
def handle_request(request):
|
||||
url = request.url
|
||||
# 记录可能包含字幕的请求
|
||||
if 'subtitle' in url.lower() or 'cc' in url.lower():
|
||||
subtitle_requests.append({
|
||||
'url': url,
|
||||
'method': request.method,
|
||||
'resource_type': request.resource_type
|
||||
})
|
||||
print(f"📡 发现字幕请求: {url[:100]}...")
|
||||
|
||||
page.on('request', handle_request)
|
||||
|
||||
print("\n1. 访问页面并等待加载...")
|
||||
await page.goto(url, wait_until='networkidle', timeout=60000)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# 模拟播放视频（点击播放按钮）
|
||||
print("\n2. 尝试播放视频...")
|
||||
try:
|
||||
# 尝试点击播放按钮
|
||||
play_button = await page.query_selector('.bpx-player-ctrl-play')
|
||||
if play_button:
|
||||
await play_button.click()
|
||||
await asyncio.sleep(5) # 等待视频播放
|
||||
print(" ✅ 已点击播放按钮")
|
||||
else:
|
||||
print(" ⚠️ 未找到播放按钮")
|
||||
await asyncio.sleep(5) # 无论如何等待一下
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 播放异常: {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
# 检查是否有字幕按钮
|
||||
print("\n3. 检查字幕按钮...")
|
||||
try:
|
||||
subtitle_button = await page.query_selector('.bpx-player-ctrl-subtitle')
|
||||
if subtitle_button:
|
||||
print(" ✅ 找到字幕按钮")
|
||||
is_enabled = await subtitle_button.is_enabled()
|
||||
print(f" 字幕按钮状态: {'可用' if is_enabled else '不可用'}")
|
||||
|
||||
# 尝试点击字幕按钮
|
||||
if is_enabled:
|
||||
await subtitle_button.click()
|
||||
await asyncio.sleep(2)
|
||||
print(" ✅ 已点击字幕按钮")
|
||||
else:
|
||||
print(" ❌ 未找到字幕按钮")
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 检查字幕按钮异常: {e}")
|
||||
|
||||
# 等待更多网络请求
|
||||
print("\n4. 等待字幕请求...")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# 显示捕获的字幕请求
|
||||
print(f"\n5. 捕获的字幕请求 (共 {len(subtitle_requests)} 个):")
|
||||
if subtitle_requests:
|
||||
for i, req in enumerate(subtitle_requests):
|
||||
print(f"\n [{i+1}] {req['method']} {req['resource_type']}")
|
||||
print(f" {req['url']}")
|
||||
else:
|
||||
print(" ❌ 未捕获到字幕请求")
|
||||
|
||||
# 尝试从页面中提取字幕文本
|
||||
print("\n6. 尝试从页面提取字幕文本...")
|
||||
try:
|
||||
# B站字幕通常在 .bpx-player-subtitle-item 元素中
|
||||
subtitle_items = await page.query_selector_all('.bpx-player-subtitle-item')
|
||||
|
||||
if subtitle_items:
|
||||
print(f" ✅ 找到 {len(subtitle_items)} 个字幕元素")
|
||||
# 获取前几个字幕
|
||||
for i, item in enumerate(subtitle_items[:5]):
|
||||
text = await item.inner_text()
|
||||
print(f" [{i+1}] {text}")
|
||||
else:
|
||||
print(" ❌ 未找到字幕元素")
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 提取字幕文本异常: {e}")
|
||||
|
||||
# 尝试获取所有文本内容（包含动态加载的字幕）
|
||||
print("\n7. 检查页面中是否有字幕相关的数据...")
|
||||
try:
|
||||
# 尝试从 window 对象中获取字幕数据
|
||||
script_result = await page.evaluate('''
|
||||
() => {
|
||||
const results = {};
|
||||
|
||||
// 检查不同的位置
|
||||
if (window.__playinfo__?.data?.subtitle) {
|
||||
results.playinfo_subtitle = window.__playinfo__.data.subtitle;
|
||||
}
|
||||
|
||||
if (window.__subtitle__ || window.__subtitles__) {
|
||||
results.subtitle_var = window.__subtitle__ || window.__subtitles__;
|
||||
}
|
||||
|
||||
// 检查播放器实例
|
||||
const player = document.querySelector('.bpx-player-container')?.__player__;
|
||||
if (player?.subtitlePlugin) {
|
||||
results.player_subtitle = Object.keys(player.subtitlePlugin);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
''')
|
||||
|
||||
if script_result:
|
||||
print(f" 找到的字幕数据: {json.dumps(script_result, indent=8, ensure_ascii=False)}")
|
||||
else:
|
||||
print(" 未找到字幕相关的数据")
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 检查页面数据异常: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("测试完成")
|
||||
print("=" * 80)
|
||||
|
||||
if __name__ == '__main__':
|
||||
bvid = "BV1KtFzzqEUG"
|
||||
asyncio.run(test_dynamic_subtitle(bvid))
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
测试B站字幕提取问题
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def test_subtitle_api(bvid):
|
||||
"""测试字幕API"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
await page.goto(url, wait_until='domcontentloaded')
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# 方法1: 获取 aid
|
||||
print("=== 测试方法1: 使用 aid ===")
|
||||
try:
|
||||
aid = await page.evaluate('() => window.__INITIAL_STATE__?.videoData?.aid')
|
||||
print(f"aid: {aid}")
|
||||
if aid:
|
||||
api_url = "https://api.bilibili.com/x/player/v2"
|
||||
response = await page.request.get(api_url, params={'aid': aid, 'bvid': bvid})
|
||||
data = await response.json()
|
||||
print(f"Response code: {data.get('code')}")
|
||||
print(f"Subtitle info: {json.dumps(data.get('data', {}).get('subtitle', {}), indent=2, ensure_ascii=False)}")
|
||||
except Exception as e:
|
||||
print(f"方法1失败: {e}")
|
||||
|
||||
# 方法2: 获取 cid
|
||||
print("\n=== 测试方法2: 使用 cid ===")
|
||||
try:
|
||||
cid = await page.evaluate('() => window.__INITIAL_STATE__?.videoData?.cid')
|
||||
print(f"cid: {cid}")
|
||||
if cid:
|
||||
api_url = "https://api.bilibili.com/x/player/v2"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid, 'cid': cid})
|
||||
data = await response.json()
|
||||
print(f"Response code: {data.get('code')}")
|
||||
print(f"Subtitle info: {json.dumps(data.get('data', {}).get('subtitle', {}), indent=2, ensure_ascii=False)}")
|
||||
|
||||
# 如果有字幕，尝试获取字幕内容
|
||||
subtitle_info = data.get('data', {}).get('subtitle', {})
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
if subtitles_list:
|
||||
subtitle_url = subtitles_list[0].get('subtitle_url', '')
|
||||
print(f"\nSubtitle URL: {subtitle_url}")
|
||||
full_url = f"https:{subtitle_url}"
|
||||
response = await page.request.get(full_url)
|
||||
if response.ok:
|
||||
sub_data = await response.json()
|
||||
print(f"First 3 subtitles:")
|
||||
for i, item in enumerate(sub_data.get('body', [])[:3]):
|
||||
print(f" [{i+1}] {item}")
|
||||
except Exception as e:
|
||||
print(f"方法2失败: {e}")
|
||||
|
||||
# 方法3: 从页面直接提取
|
||||
print("\n=== 测试方法3: 从页面直接提取 ===")
|
||||
try:
|
||||
subtitle_data = await page.evaluate('() => window.__INITIAL_STATE__?.videoData?.subtitle')
|
||||
print(f"Subtitle data from page: {json.dumps(subtitle_data, indent=2, ensure_ascii=False)}")
|
||||
except Exception as e:
|
||||
print(f"方法3失败: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 测试一个有字幕的视频
|
||||
bvid = "BV1MnFNz5EDL" # 可以换成其他视频
|
||||
asyncio.run(test_subtitle_api(bvid))
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
测试B站字幕提取问题 - 改进版
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from playwright.async_api import async_playwright
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
async def test_subtitle_api(bvid):
|
||||
"""测试字幕API"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
print("正在访问页面...")
|
||||
await page.goto(url, wait_until='networkidle', timeout=60000)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# 方法1: 检查 __INITIAL_STATE__ 是否存在
|
||||
print("\n=== 测试方法1: 检查 __INITIAL_STATE__ 结构 ===")
|
||||
try:
|
||||
initial_state = await page.evaluate('() => window.__INITIAL_STATE__')
|
||||
if initial_state:
|
||||
print(f"__INITIAL_STATE__ keys: {list(initial_state.keys())}")
|
||||
video_data = initial_state.get('videoData', {})
|
||||
print(f"videoData keys: {list(video_data.keys())}")
|
||||
print(f"aid: {video_data.get('aid')}")
|
||||
print(f"cid: {video_data.get('cid')}")
|
||||
else:
|
||||
print("__INITIAL_STATE__ 不存在")
|
||||
except Exception as e:
|
||||
print(f"方法1失败: {e}")
|
||||
|
||||
# 方法2: 从 HTML 中提取
|
||||
print("\n=== 测试方法2: 从 HTML script 标签提取 ===")
|
||||
try:
|
||||
content = await page.content()
|
||||
soup = BeautifulSoup(content, 'lxml')
|
||||
script_tags = soup.find_all('script')
|
||||
|
||||
for script in script_tags:
|
||||
if script.string and '__INITIAL_STATE__' in script.string:
|
||||
match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.+?});', script.string)
|
||||
if match:
|
||||
data = json.loads(match.group(1))
|
||||
video_data = data.get('videoData', {})
|
||||
print(f"aid: {video_data.get('aid')}")
|
||||
print(f"cid: {video_data.get('cid')}")
|
||||
print(f"subtitle: {video_data.get('subtitle')}")
|
||||
|
||||
# 如果找到了 cid，尝试获取字幕
|
||||
cid = video_data.get('cid')
|
||||
if cid:
|
||||
await test_subtitle_api_with_cid(page, bvid, cid)
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"方法2失败: {e}")
|
||||
|
||||
# 方法3: 使用 API 获取视频信息
|
||||
print("\n=== 测试方法3: 使用 API 获取视频信息 ===")
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') == 0:
|
||||
video_data = data.get('data', {})
|
||||
print(f"aid: {video_data.get('aid')}")
|
||||
print(f"cid: {video_data.get('cid')}")
|
||||
print(f"title: {video_data.get('title')}")
|
||||
|
||||
cid = video_data.get('cid')
|
||||
if cid:
|
||||
await test_subtitle_api_with_cid(page, bvid, cid)
|
||||
except Exception as e:
|
||||
print(f"方法3失败: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
async def test_subtitle_api_with_cid(page, bvid, cid):
|
||||
"""使用 cid 测试字幕 API"""
|
||||
print(f"\n=== 使用 cid={cid} 获取字幕 ===")
|
||||
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/player/v2"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid, 'cid': cid})
|
||||
data = await response.json()
|
||||
|
||||
print(f"Response code: {data.get('code')}")
|
||||
|
||||
if data.get('code') == 0:
|
||||
player_data = data.get('data', {})
|
||||
subtitle_info = player_data.get('subtitle', {})
|
||||
print(f"Subtitle info:")
|
||||
print(f" allow_submit: {subtitle_info.get('allow_submit')}")
|
||||
print(f" subtitles count: {len(subtitle_info.get('subtitles', []))}")
|
||||
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
if subtitles_list:
|
||||
print(f"\nFound {len(subtitles_list)} subtitles:")
|
||||
for i, sub in enumerate(subtitles_list[:3]):
|
||||
print(f" [{i+1}] {sub.get('id')} - {sub.get('lan_doc')}")
|
||||
|
||||
# 获取第一个字幕内容
|
||||
subtitle_url = subtitles_list[0].get('subtitle_url', '')
|
||||
print(f"\nFetching subtitle from: {subtitle_url}")
|
||||
full_url = f"https:{subtitle_url}"
|
||||
response = await page.request.get(full_url)
|
||||
if response.ok:
|
||||
sub_data = await response.json()
|
||||
print(f"First 3 subtitle items:")
|
||||
for i, item in enumerate(sub_data.get('body', [])[:3]):
|
||||
print(f" [{i+1}] [{item.get('from')} - {item.get('to')}] {item.get('content')}")
|
||||
else:
|
||||
print(f"Failed to fetch subtitle: {response.status}")
|
||||
else:
|
||||
print("No subtitles found")
|
||||
else:
|
||||
print(f"API error: {data.get('message')}")
|
||||
except Exception as e:
|
||||
print(f"Failed: {e}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 测试一个有字幕的视频
|
||||
bvid = "BV1MnFNz5EDL"
|
||||
asyncio.run(test_subtitle_api(bvid))
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
测试B站字幕提取 - 测试多个视频
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def test_video(bvid):
|
||||
"""测试单个视频"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
# 使用 API 获取视频信息
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') == 0:
|
||||
video_data = data.get('data', {})
|
||||
title = video_data.get('title', '')
|
||||
cid = video_data.get('cid', '')
|
||||
print(f"\n视频: {title}")
|
||||
print(f"BV号: {bvid}")
|
||||
print(f"cid: {cid}")
|
||||
|
||||
# 获取字幕
|
||||
if cid:
|
||||
player_url = "https://api.bilibili.com/x/player/v2"
|
||||
player_response = await page.request.get(player_url, params={'bvid': bvid, 'cid': cid})
|
||||
player_data = await player_response.json()
|
||||
|
||||
if player_data.get('code') == 0:
|
||||
subtitle_info = player_data.get('data', {}).get('subtitle', {})
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
|
||||
if subtitles_list:
|
||||
print(f"✅ 找到 {len(subtitles_list)} 个字幕")
|
||||
for i, sub in enumerate(subtitles_list[:3]):
|
||||
print(f" [{i+1}] {sub.get('lan_doc')} - {sub.get('subtitle_url')}")
|
||||
|
||||
# 获取第一个字幕内容
|
||||
subtitle_url = subtitles_list[0].get('subtitle_url', '')
|
||||
if subtitle_url:
|
||||
full_url = f"https:{subtitle_url}"
|
||||
sub_response = await page.request.get(full_url)
|
||||
if sub_response.ok:
|
||||
sub_data = await sub_response.json()
|
||||
body = sub_data.get('body', [])
|
||||
print(f"\n字幕示例 (前3条):")
|
||||
for i, item in enumerate(body[:3]):
|
||||
from_time = item.get('from', 0)
|
||||
to_time = item.get('to', 0)
|
||||
content = item.get('content', '')
|
||||
print(f" [{i+1}] [{from_time:.1f} - {to_time:.1f}] {content}")
|
||||
else:
|
||||
print(f"❌ 获取字幕内容失败: {sub_response.status}")
|
||||
else:
|
||||
print("❌ 该视频没有字幕")
|
||||
else:
|
||||
print(f"❌ 获取视频信息失败: {data.get('message')}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
async def main():
|
||||
"""测试多个视频"""
|
||||
# 测试视频列表（包含不同类型的视频）
|
||||
test_videos = [
|
||||
"BV1XX4y1C7tF", # 教程类视频，可能有字幕
|
||||
"BV1GJ411x7h7", # TED演讲，通常有字幕
|
||||
"BV1uA411H7K4", # 科技类视频
|
||||
"BV1MnFNz5EDL", # 之前的测试视频
|
||||
]
|
||||
|
||||
for bvid in test_videos:
|
||||
await test_video(bvid)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
测试B站字幕提取 - 使用确定有字幕的视频
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def test_with_cookie(bvid, cookie=None):
|
||||
"""测试带 Cookie 的字幕提取"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
# 如果有 cookie，添加到上下文
|
||||
if cookie:
|
||||
await context.add_cookies([cookie])
|
||||
print("使用已登录 Cookie")
|
||||
|
||||
# 使用 API 获取视频信息
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') == 0:
|
||||
video_data = data.get('data', {})
|
||||
title = video_data.get('title', '')
|
||||
cid = video_data.get('cid', '')
|
||||
print(f"\n视频: {title}")
|
||||
print(f"BV号: {bvid}")
|
||||
print(f"cid: {cid}")
|
||||
|
||||
# 获取字幕
|
||||
if cid:
|
||||
player_url = "https://api.bilibili.com/x/player/v2"
|
||||
player_response = await page.request.get(player_url, params={'bvid': bvid, 'cid': cid})
|
||||
player_data = await player_response.json()
|
||||
|
||||
print(f"Player API 响应码: {player_data.get('code')}")
|
||||
|
||||
if player_data.get('code') == 0:
|
||||
subtitle_info = player_data.get('data', {}).get('subtitle', {})
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
|
||||
if subtitles_list:
|
||||
print(f"✅ 找到 {len(subtitles_list)} 个字幕")
|
||||
for i, sub in enumerate(subtitles_list):
|
||||
print(f" [{i+1}] 语言: {sub.get('lan_doc')}, ID: {sub.get('id')}")
|
||||
print(f" URL: {sub.get('subtitle_url')}")
|
||||
|
||||
# 获取第一个字幕内容
|
||||
subtitle_url = subtitles_list[0].get('subtitle_url', '')
|
||||
if subtitle_url:
|
||||
full_url = f"https:{subtitle_url}"
|
||||
print(f"\n正在获取字幕内容: {full_url}")
|
||||
sub_response = await page.request.get(full_url)
|
||||
print(f"字幕请求状态: {sub_response.status}")
|
||||
|
||||
if sub_response.ok:
|
||||
sub_data = await sub_response.json()
|
||||
body = sub_data.get('body', [])
|
||||
print(f"\n字幕内容示例 (前5条):")
|
||||
for i, item in enumerate(body[:5]):
|
||||
from_time = item.get('from', 0)
|
||||
to_time = item.get('to', 0)
|
||||
content = item.get('content', '')
|
||||
print(f" [{i+1}] [{from_time:.1f}s - {to_time:.1f}s] {content}")
|
||||
print(f"\n✅ 字幕获取成功！共 {len(body)} 条字幕")
|
||||
else:
|
||||
print(f"❌ 获取字幕内容失败: {sub_response.status}")
|
||||
print(f"响应头: {sub_response.headers}")
|
||||
else:
|
||||
print("❌ 该视频没有字幕")
|
||||
print(f"Subtitle info: {json.dumps(subtitle_info, indent=2, ensure_ascii=False)}")
|
||||
else:
|
||||
print(f"❌ Player API 失败: {player_data.get('message')}")
|
||||
print(f"完整响应: {json.dumps(player_data, indent=2, ensure_ascii=False)}")
|
||||
else:
|
||||
print(f"❌ 获取视频信息失败: {data.get('message')}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
async def main():
|
||||
"""测试多个视频"""
|
||||
|
||||
# 罗翔的视频通常有官方字幕
|
||||
test_videos = [
|
||||
"BV1GJ411x7h7", # 罗翔 - 刑法
|
||||
"BV1yx411s7Jq", # 可能的官方视频
|
||||
"BV1zv4y1m7Qp", # B站官方视频
|
||||
]
|
||||
|
||||
for bvid in test_videos:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"测试视频: {bvid}")
|
||||
print(f"{'='*60}")
|
||||
try:
|
||||
await test_with_cookie(bvid)
|
||||
except Exception as e:
|
||||
print(f"❌ 测试失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
await asyncio.sleep(2)
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
测试 B站 API 获取视频流
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def test_video_stream(bvid):
|
||||
"""测试获取视频流"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
print(f"正在测试视频: {url}")
|
||||
print("=" * 80)
|
||||
|
||||
# 首先获取 aid 和 cid
|
||||
print("\n1. 获取视频基本信息...")
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') == 0:
|
||||
video_data = data.get('data', {})
|
||||
print(f" 标题: {video_data.get('title')}")
|
||||
aid = video_data.get('aid')
|
||||
cid = video_data.get('cid')
|
||||
print(f" aid: {aid}")
|
||||
print(f" cid: {cid}")
|
||||
else:
|
||||
print(f" ❌ 失败: {data.get('message')}")
|
||||
await browser.close()
|
||||
return
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
await browser.close()
|
||||
return
|
||||
|
||||
# 尝试获取视频流 URL
|
||||
print("\n2. 尝试获取视频流 URL...")
|
||||
try:
|
||||
playurl_api = "https://api.bilibili.com/x/player/wbi/playurl"
|
||||
params = {
|
||||
'bvid': bvid,
|
||||
'cid': cid,
|
||||
'qn': 80, # 清晰度：80=1080P
|
||||
'fnval': 16, # 包含 DASH 格式
|
||||
'fnver': 0,
|
||||
'fourk': 1,
|
||||
}
|
||||
|
||||
playurl_response = await page.request.get(playurl_api, params=params)
|
||||
print(f" 状态码: {playurl_response.status}")
|
||||
|
||||
if playurl_response.ok:
|
||||
playurl_data = await playurl_response.json()
|
||||
print(f" 响应码: {playurl_data.get('code')}")
|
||||
|
||||
if playurl_data.get('code') == 0:
|
||||
play_data = playurl_data.get('data', {})
|
||||
print(f"\n 数据类型:")
|
||||
print(f" {json.dumps(list(play_data.keys()), indent=4, ensure_ascii=False)}")
|
||||
|
||||
# 检查 DASH 格式
|
||||
if 'dash' in play_data:
|
||||
dash = play_data['dash']
|
||||
print(f"\n DASH 格式:")
|
||||
print(f" duration: {dash.get('duration')}s")
|
||||
print(f" video streams: {len(dash.get('video', []))}")
|
||||
print(f" audio streams: {len(dash.get('audio', []))}")
|
||||
|
||||
# 显示视频流信息
|
||||
print(f"\n 视频流列表:")
|
||||
for i, video_stream in enumerate(dash.get('video', [])[:3]):
|
||||
print(f" [{i+1}] id={video_stream.get('id')}, "
|
||||
f"bandwidth={video_stream.get('bandwidth')}, "
|
||||
f"codecs={video_stream.get('codecs')}")
|
||||
print(f" baseUrl: {video_stream.get('baseUrl', '')[:100]}...")
|
||||
print(f" backup_url: {len(video_stream.get('backup_url', []))} 个")
|
||||
|
||||
# 显示音频流信息
|
||||
print(f"\n 音频流列表:")
|
||||
for i, audio_stream in enumerate(dash.get('audio', [])[:3]):
|
||||
print(f" [{i+1}] id={audio_stream.get('id')}, "
|
||||
f"bandwidth={audio_stream.get('bandwidth')}, "
|
||||
f"codecs={audio_stream.get('codecs')}")
|
||||
print(f" baseUrl: {audio_stream.get('baseUrl', '')[:100]}...")
|
||||
|
||||
# 尝试下载一小段测试
|
||||
if dash.get('video'):
|
||||
first_video = dash['video'][0]
|
||||
base_url = first_video.get('baseUrl', '')
|
||||
if base_url:
|
||||
print(f"\n3. 测试下载视频流...")
|
||||
print(f" URL: {base_url}")
|
||||
|
||||
# 添加必要的请求头
|
||||
headers = {
|
||||
'Referer': 'https://www.bilibili.com/',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
}
|
||||
|
||||
# 只下载前 1KB 测试
|
||||
test_response = await page.request.get(
|
||||
base_url,
|
||||
headers=headers,
|
||||
timeout=30000
|
||||
)
|
||||
print(f" 下载状态: {test_response.status}")
|
||||
if test_response.ok:
|
||||
content_length = len(await test_response.body())
|
||||
print(f" ✅ 可以下载！获取到 {content_length} 字节")
|
||||
print(f" Content-Type: {test_response.headers.get('content-type')}")
|
||||
else:
|
||||
print(f" ❌ 下载失败")
|
||||
else:
|
||||
print(f" ❌ API 错误: {playurl_data.get('message')}")
|
||||
print(f" 完整响应: {json.dumps(playurl_data, indent=4, ensure_ascii=False)[:500]}")
|
||||
else:
|
||||
print(f" ❌ HTTP 错误: {playurl_response.status}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# 尝试旧版 API
|
||||
print("\n4. 尝试旧版 playurl API...")
|
||||
try:
|
||||
playurl_api = "https://api.bilibili.com/x/player/playurl"
|
||||
params = {
|
||||
'bvid': bvid,
|
||||
'cid': cid,
|
||||
'qn': 80,
|
||||
'otype': 'json',
|
||||
}
|
||||
|
||||
playurl_response = await page.request.get(playurl_api, params=params)
|
||||
print(f" 状态码: {playurl_response.status}")
|
||||
|
||||
if playurl_response.ok:
|
||||
playurl_data = await playurl_response.json()
|
||||
print(f" 响应码: {playurl_data.get('code')}")
|
||||
|
||||
if playurl_data.get('code') == 0:
|
||||
print(f" ✅ 旧版 API 可用")
|
||||
print(f" 数据: {json.dumps(list(playurl_data.get('data', {}).keys()), indent=4, ensure_ascii=False)}")
|
||||
else:
|
||||
print(f" ❌ API 错误: {playurl_data.get('message')}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
bvid = "BV1KtFzzqEUG"
|
||||
asyncio.run(test_video_stream(bvid))
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
测试 B站 WBI API 字幕提取
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async def test_wbi_subtitle(bvid):
|
||||
"""测试 WBI API 字幕"""
|
||||
url = f"https://www.bilibili.com/video/{bvid}"
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context()
|
||||
page = await context.new_page()
|
||||
|
||||
print(f"正在测试视频: {url}")
|
||||
print("=" * 80)
|
||||
|
||||
# 首先获取 aid 和 cid
|
||||
print("\n1. 获取视频基本信息...")
|
||||
try:
|
||||
api_url = "https://api.bilibili.com/x/web-interface/view"
|
||||
response = await page.request.get(api_url, params={'bvid': bvid})
|
||||
data = await response.json()
|
||||
|
||||
if data.get('code') == 0:
|
||||
video_data = data.get('data', {})
|
||||
print(f" 标题: {video_data.get('title')}")
|
||||
aid = video_data.get('aid')
|
||||
cid = video_data.get('cid')
|
||||
print(f" aid: {aid}")
|
||||
print(f" cid: {cid}")
|
||||
else:
|
||||
print(f" ❌ 失败: {data.get('message')}")
|
||||
await browser.close()
|
||||
return
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
await browser.close()
|
||||
return
|
||||
|
||||
# 使用 WBI API 获取播放器信息（包含字幕）
|
||||
print("\n2. 使用 WBI API 获取播放器信息...")
|
||||
try:
|
||||
wbi_url = "https://api.bilibili.com/x/player/wbi/v2"
|
||||
wbi_response = await page.request.get(wbi_url, params={'aid': aid, 'cid': cid})
|
||||
|
||||
print(f" 状态码: {wbi_response.status}")
|
||||
if wbi_response.ok:
|
||||
wbi_data = await wbi_response.json()
|
||||
print(f" 响应码: {wbi_data.get('code')}")
|
||||
|
||||
if wbi_data.get('code') == 0:
|
||||
player_data = wbi_data.get('data', {})
|
||||
print(f"\n 数据结构:")
|
||||
print(f" {json.dumps(list(player_data.keys()), indent=4, ensure_ascii=False)}")
|
||||
|
||||
# 检查字幕
|
||||
subtitle_info = player_data.get('subtitle', {})
|
||||
print(f"\n 字幕信息:")
|
||||
print(f" {json.dumps(subtitle_info, indent=4, ensure_ascii=False)}")
|
||||
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
if subtitles_list:
|
||||
print(f"\n ✅ 找到 {len(subtitles_list)} 个字幕!")
|
||||
for i, sub in enumerate(subtitles_list):
|
||||
print(f" [{i+1}] {sub.get('lan_doc')} (lan={sub.get('id')}, type={sub.get('id_type')})")
|
||||
print(f" URL: {sub.get('subtitle_url')}")
|
||||
|
||||
# 获取字幕内容
|
||||
subtitle_url = sub.get('subtitle_url', '')
|
||||
if subtitle_url:
|
||||
full_url = f"https:{subtitle_url}"
|
||||
print(f"\n 正在获取字幕内容...")
|
||||
sub_response = await page.request.get(full_url)
|
||||
if sub_response.ok:
|
||||
sub_data = await sub_response.json()
|
||||
body = sub_data.get('body', [])
|
||||
print(f" ✅ 获取成功，共 {len(body)} 条字幕")
|
||||
print(f" 前3条字幕:")
|
||||
for j, item in enumerate(body[:3]):
|
||||
from_time = item.get('from', 0)
|
||||
to_time = item.get('to', 0)
|
||||
content = item.get('content', '')
|
||||
print(f" [{j+1}] [{from_time:.1f}s - {to_time:.1f}s] {content}")
|
||||
else:
|
||||
print(f" ❌ 获取失败: {sub_response.status}")
|
||||
else:
|
||||
print(f"\n ❌ 没有找到字幕")
|
||||
print(f" allow_submit: {subtitle_info.get('allow_submit')}")
|
||||
else:
|
||||
print(f" ❌ API 错误: {wbi_data.get('message')}")
|
||||
else:
|
||||
print(f" ❌ HTTP 错误: {wbi_response.status}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# 尝试不使用 wbi，使用旧版本 API
|
||||
print("\n3. 尝试旧版 API (v2)...")
|
||||
try:
|
||||
old_url = "https://api.bilibili.com/x/player/v2"
|
||||
old_response = await page.request.get(old_url, params={'bvid': bvid, 'cid': cid})
|
||||
print(f" 状态码: {old_response.status}")
|
||||
if old_response.ok:
|
||||
old_data = await old_response.json()
|
||||
subtitle_info = old_data.get('data', {}).get('subtitle', {})
|
||||
subtitles_list = subtitle_info.get('subtitles', [])
|
||||
print(f" 字幕数量: {len(subtitles_list)}")
|
||||
if subtitles_list:
|
||||
print(f" ✅ 旧版 API 找到字幕!")
|
||||
for sub in subtitles_list:
|
||||
print(f" {sub.get('lan_doc')}")
|
||||
except Exception as e:
|
||||
print(f" ❌ 异常: {e}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
bvid = "BV1KtFzzqEUG"
|
||||
asyncio.run(test_wbi_subtitle(bvid))
|
||||
Reference in New Issue
Block a user