From aae4d2fd70cea363ab7b817a68d143ee8cbff128 Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Sun, 15 Mar 2026 22:10:23 +0800 Subject: [PATCH 1/3] docs(debug): resolve asyncio nested loop debug session Root cause: BilibiliExtractor sync wrappers use asyncio.run() internally, which fails when called from async context in BilibiliAdapter. Fix: Call async methods directly instead of sync wrappers. Verified: Human confirmed fix works (bilibili_BV16cNMzpExh) --- .../kb-workflow-asyncio-nested-loop.md | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .planning/debug/resolved/kb-workflow-asyncio-nested-loop.md diff --git a/.planning/debug/resolved/kb-workflow-asyncio-nested-loop.md b/.planning/debug/resolved/kb-workflow-asyncio-nested-loop.md new file mode 100644 index 0000000..c3b99f4 --- /dev/null +++ b/.planning/debug/resolved/kb-workflow-asyncio-nested-loop.md @@ -0,0 +1,60 @@ +--- +status: resolved +trigger: "kb-workflow skill fails to download Bilibili videos with asyncio nested loop error" +created: 2026-03-15T00:00:00Z +updated: 2026-03-15T19:00:00Z +--- + +## Current Focus + +hypothesis: N/A - root cause confirmed and fix verified +test: Ran kb-workflow process command - metadata extraction succeeded +expecting: No asyncio nested loop error, successful metadata extraction +next_action: Await human verification to confirm fix works in their environment + +## Symptoms + +expected: kb-workflow skill should successfully download Bilibili videos (BV16cNMzpExh) +actual: Download fails with asyncio nested event loop error +errors: `asyncio.run() cannot be called from a running event loop` +reproduction: CLI command (e.g., `/kb-workflow download `) +started: Never worked properly - this is a new feature with a bug + +## Eliminated + + + +## Evidence + +- timestamp: 2026-03-15T00:00:01Z + checked: kb-workflow/kb_workflow.py line 387 + found: process_command() calls asyncio.run(run_pipeline()) - this is the outer event loop + implication: Creates an async context that all nested code runs within + +- timestamp: 2026-03-15T00:00:01Z + checked: kb_pipeline/pipeline.py line 106 + found: Pipeline.run() is an async method that calls await processor.process() + implication: All processors run within the async event loop + +- timestamp: 2026-03-15T00:00:01Z + checked: kb_pipeline/processors/download.py line 66 + found: DownloadProcessor.process() calls await adapter.fetch_metadata() + implication: BilibiliAdapter.fetch_metadata() is called from async context + +- timestamp: 2026-03-15T00:00:01Z + checked: kb_pipeline/platforms/bilibili.py line 173 + found: BilibiliAdapter._fetch_metadata_with_retry() calls self._extractor.extract_video_metadata(content_id) which is a SYNC method + implication: Calls sync wrapper from async context + +- timestamp: 2026-03-15T00:00:01Z + checked: kb-bilibili-metadata/bilibili_extractor.py line 91 + found: BilibiliExtractor.extract_video_metadata() calls asyncio.run(self._extract_video_async(bvid)) - this is the ROOT CAUSE + implication: asyncio.run() CANNOT be called when an event loop is already running, causing the nested loop error + +## Resolution + +root_cause: BilibiliExtractor provides sync wrapper methods (extract_video_metadata, extract_article_metadata, etc.) that use asyncio.run() internally. When BilibiliAdapter.fetch_metadata() (which is async) calls these sync methods, it triggers asyncio.run() from within an already-running event loop, which is forbidden in Python's asyncio model. +fix: Modified BilibiliAdapter._fetch_metadata_with_retry() to call the underlying async methods directly (await self._extractor._extract_video_async() and await self._extractor._extract_article_async()) instead of the sync wrappers that use asyncio.run() +verification: Self-verified - ran kb-workflow process command successfully. No asyncio nested loop error. Metadata extracted correctly including title, author, stats, description. Human-verified - user confirmed fix works (bilibili_BV16cNMzpExh executed successfully, asyncio error resolved). +files_changed: + - kb_pipeline/platforms/bilibili.py From b6ff7c81aafed0061d0c52defc2c54d89015ca35 Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Sun, 15 Mar 2026 22:39:43 +0800 Subject: [PATCH 2/3] fix(zhihu): handle lazy-load images in markdown conversion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 知乎使用图片懒加载机制，`` 标签的 `src` 属性包含 SVG 占位符， 真实图片 URL 在 `data-actualsrc` 或 `data-original` 属性中。 `markdownify` 只读取 `src`，导致转换后得到无效占位符。 修复：在转换前预处理 HTML，将真实图片 URL 替换到 `src` 属性。 Co-Authored-By: Claude Opus 4.6 --- kb-zhihu-metadata/markdown_converter.py | 50 +++++- .../test_markdown_converter.py | 148 ++++++++++++++++++ 2 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 tests/test_zhihu_metadata/test_markdown_converter.py diff --git a/kb-zhihu-metadata/markdown_converter.py b/kb-zhihu-metadata/markdown_converter.py index c0f912d..81faa54 100644 --- a/kb-zhihu-metadata/markdown_converter.py +++ b/kb-zhihu-metadata/markdown_converter.py @@ -1,7 +1,14 @@ +import re +from bs4 import BeautifulSoup from markdownify import MarkdownConverter as _MarkdownConverter class MarkdownConverter: - """HTML 转 Markdown 转换器，保留完整格式""" + """HTML 转 Markdown 转换器，保留完整格式 + + 处理知乎等平台的图片懒加载机制： + - 知乎图片的 src 属性可能是 SVG 占位符 (data:image/svg+xml;...) + - 真实图片 URL 存储在 data-actualsrc 或 data-original 属性中 + """ def __init__(self): self.converter = _MarkdownConverter( @@ -13,5 +20,44 @@ class MarkdownConverter: ) def convert(self, html: str) -> str: - """转换 HTML 为 Markdown""" + """转换 HTML 为 Markdown + + Args: + html: 输入的 HTML 字符串 + + Returns: + 转换后的 Markdown 字符串 + """ + # 预处理：处理懒加载图片 + html = self._fix_lazy_load_images(html) return self.converter.convert(html) + + def _fix_lazy_load_images(self, html: str) -> str: + """修复懒加载图片的 src 属性 + + 将 data-actualsrc 或 data-original 属性的值替换到 src 属性中， + 以便 markdownify 能够正确提取真实图片 URL。 + + Args: + html: 输入的 HTML 字符串 + + Returns: + 处理后的 HTML 字符串 + """ + soup = BeautifulSoup(html, 'html.parser') + + for img in soup.find_all('img'): + src = img.get('src', '') + + # 如果 src 不是占位符，保持原样 + if not src or not src.startswith('data:'): + continue + + # 尝试获取真实图片 URL + # 优先 data-actualsrc，其次 data-original + real_src = img.get('data-actualsrc') or img.get('data-original') + + if real_src: + img['src'] = real_src + + return str(soup) diff --git a/tests/test_zhihu_metadata/test_markdown_converter.py b/tests/test_zhihu_metadata/test_markdown_converter.py new file mode 100644 index 0000000..525d315 --- /dev/null +++ b/tests/test_zhihu_metadata/test_markdown_converter.py @@ -0,0 +1,148 @@ +"""Tests for MarkdownConverter in kb_zhihu_metadata + +Tests cover: +- Normal image conversion +- Zhihu lazy loading image handling (data-actualsrc) +- Alternative lazy loading (data-original) +- Mixed content with images +""" + +import pytest +import sys +from pathlib import Path + +# Add kb_zhihu_metadata to path +zhihu_metadata_path = Path(__file__).parent.parent.parent / "kb-zhihu-metadata" +if str(zhihu_metadata_path) not in sys.path: + sys.path.insert(0, str(zhihu_metadata_path)) + +from markdown_converter import MarkdownConverter + + +class TestMarkdownConverterImages: + """Tests for image handling in MarkdownConverter""" + + def test_normal_image_conversion(self): + """Test that normal images are converted correctly""" + html = 'test' + converter = MarkdownConverter() + result = converter.convert(html) + assert "![test](https://example.com/image.jpg)" in result + + def test_zhihu_lazy_load_image_with_data_actualsrc(self): + """Test that Zhihu lazy loading images use data-actualsrc""" + html = '''''' + converter = MarkdownConverter() + result = converter.convert(html) + assert "https://pic1.zhimg.com/v2-real-image_b.jpg" in result + assert "data:image/svg+xml" not in result + + def test_lazy_load_image_with_data_original(self): + """Test that images with data-original use that URL""" + html = '''''' + converter = MarkdownConverter() + result = converter.convert(html) + assert "https://example.com/original.jpg" in result + assert "data:image/svg+xml" not in result + + def test_data_actualsrc_priority_over_data_original(self): + """Test that data-actualsrc takes priority over data-original""" + html = '''''' + converter = MarkdownConverter() + result = converter.convert(html) + assert "https://example.com/actual.jpg" in result + assert "https://example.com/original.jpg" not in result + + def test_normal_image_not_modified(self): + """Test that normal images with http(s) src are not modified""" + html = '' + converter = MarkdownConverter() + result = converter.convert(html) + assert "https://example.com/normal.jpg" in result + + def test_mixed_content_with_images(self): + """Test conversion of mixed content with text and images""" + html = ''' +
+

First paragraph.

+
+ +
+

Second paragraph.

+ +
+ ''' + converter = MarkdownConverter() + result = converter.convert(html) + + # Check text content + assert "First paragraph" in result + assert "Second paragraph" in result + + # Check lazy loaded image + assert "https://pic.zhimg.com/image1.jpg" in result + + # Check normal image + assert "https://example.com/normal.jpg" in result + + # Ensure no SVG placeholders in result + assert "data:image/svg+xml" not in result + + def test_figure_with_caption(self): + """Test that figure elements with captions are handled correctly""" + html = ''' +
+ +
Image caption
+
+ ''' + converter = MarkdownConverter() + result = converter.convert(html) + + assert "https://pic.zhimg.com/fig.jpg" in result + assert "Image caption" in result + + def test_empty_src_with_data_actualsrc(self): + """Test handling of empty src with data-actualsrc""" + html = '' + converter = MarkdownConverter() + result = converter.convert(html) + # Empty src is not a data: URI, so it should stay as is + assert "https://example.com/fallback.jpg" not in result + + +class TestMarkdownConverterBasicConversion: + """Tests for basic HTML to Markdown conversion""" + + def test_paragraphs(self): + """Test paragraph conversion""" + html = "

First

Second

" + converter = MarkdownConverter() + result = converter.convert(html) + assert "First" in result + assert "Second" in result + + def test_headings(self): + """Test heading conversion""" + html = "

Title

Subtitle

" + converter = MarkdownConverter() + result = converter.convert(html) + assert "# Title" in result + assert "## Subtitle" in result + + def test_links(self): + """Test link conversion""" + html = 'Link' + converter = MarkdownConverter() + result = converter.convert(html) + assert "[Link](https://example.com)" in result + + def test_code_blocks(self): + """Test code block conversion""" + html = "
def hello():\n    print('hello')
" + converter = MarkdownConverter() + result = converter.convert(html) + assert "```" in result or "def hello():" in result From 8ad64a4d1b0c379da19a0154ce3d768e2e68f1ca Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Sun, 15 Mar 2026 22:58:02 +0800 Subject: [PATCH 3/3] fix(asr): use content_dir for subtitle output instead of video_path - Add content_dir field to ASRInput schema - Pass content_dir from pipeline when creating ASRInput - Modify _try_official_subtitle to use content_dir for output - Modify _run_whisper_basic to use content_dir for output - Add validation for empty video_path in Whisper fallback Root cause: DownloadProcessor sets content_file="" for video types, causing Path("").parent to resolve to current directory instead of content directory. Subtitle files were written to wrong location. Co-Authored-By: Claude Opus 4.6 --- kb_pipeline/pipeline.py | 2 ++ kb_pipeline/processors/asr.py | 20 ++++++++++----- kb_pipeline/schemas.py | 6 +++-- tests/test_processors/test_asr.py | 42 ++++++++++++++++++++----------- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/kb_pipeline/pipeline.py b/kb_pipeline/pipeline.py index 111caf1..8088662 100644 --- a/kb_pipeline/pipeline.py +++ b/kb_pipeline/pipeline.py @@ -185,11 +185,13 @@ class Pipeline: # ASR needs video path from download step download_result = results.get("download", {}) video_path = download_result.get("content_file", "") + step_input = ASRInput( content_id=parsed.content_id, platform=parsed.platform, video_path=video_path, content_type=parsed.content_type, + content_dir=str(content_dir), ) elif step_name == "analyze": # Analyze needs metadata and content files diff --git a/kb_pipeline/processors/asr.py b/kb_pipeline/processors/asr.py index 824a94b..e9e72f8 100644 --- a/kb_pipeline/processors/asr.py +++ b/kb_pipeline/processors/asr.py @@ -172,9 +172,10 @@ class ASRProcessor: Note: 无字幕返回 None（不是抛异常），让流程继续到 Whisper """ - video_path = Path(input_data.video_path) - output_dir = video_path.parent - base_name = video_path.stem + # 使用 content_dir 作为输出目录（而不是从 video_path 推断） + output_dir = Path(input_data.content_dir) + # 使用 content_id 作为基础文件名 + base_name = f"{input_data.platform}_{input_data.content_id}_subtitle" # 输出文件路径 txt_path = output_dir / f"{base_name}.txt" @@ -343,11 +344,18 @@ class ASRProcessor: ASR 处理结果 Raises: - RuntimeError: Whisper.cpp 路径不存在或执行失败 + RuntimeError: Whisper.cpp 路径不存在或执行失败，或 video_path 为空 """ + # 检查 video_path 是否有效 + if not input_data.video_path: + raise RuntimeError( + "video_path 为空，无法执行 Whisper 语音识别。" + "请确保视频文件已下载（对于没有官方字幕的视频，需要先下载视频文件）。" + ) + video_path = Path(input_data.video_path) - output_dir = video_path.parent - base_name = video_path.stem + output_dir = Path(input_data.content_dir) + base_name = f"{input_data.platform}_{input_data.content_id}_whisper" # 获取 Whisper.cpp 路径 try: diff --git a/kb_pipeline/schemas.py b/kb_pipeline/schemas.py index 5553917..3b5b00f 100644 --- a/kb_pipeline/schemas.py +++ b/kb_pipeline/schemas.py @@ -198,11 +198,13 @@ class ASRInput(StepInput): 继承 StepInput 的基础字段，添加视频路径和内容类型。 Attributes: - video_path: 视频文件路径 + video_path: 视频文件路径（可能为空，用于 Whisper 降级） content_type: 内容类型，默认为 'video' + content_dir: 内容目录路径，用于输出字幕文件 """ - video_path: str = Field(..., description="视频文件路径") + video_path: str = Field(default="", description="视频文件路径（可能为空）") content_type: str = Field(default="video", description="内容类型") + content_dir: str = Field(..., description="内容目录路径") class ASROutput(StepOutput): diff --git a/tests/test_processors/test_asr.py b/tests/test_processors/test_asr.py index 599d8a3..59ca2bf 100644 --- a/tests/test_processors/test_asr.py +++ b/tests/test_processors/test_asr.py @@ -31,7 +31,8 @@ class TestASRInputSchema: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path="/path/to/video.mp4" + video_path="/path/to/video.mp4", + content_dir="/path/to/content" ) # 验证继承 @@ -46,7 +47,8 @@ class TestASRInputSchema: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path="/path/to/video.mp4" + video_path="/path/to/video.mp4", + content_dir="/path/to/content" ) assert hasattr(input_data, "video_path") @@ -59,7 +61,8 @@ class TestASRInputSchema: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path="/path/to/video.mp4" + video_path="/path/to/video.mp4", + content_dir="/path/to/content" ) assert input_data.content_type == "video" @@ -222,7 +225,8 @@ class TestOfficialSubtitle: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) result = await processor._try_official_subtitle(input_data) @@ -256,7 +260,8 @@ class TestOfficialSubtitle: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) result = await processor._try_official_subtitle(input_data) @@ -283,7 +288,8 @@ class TestOfficialSubtitle: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) await processor.process(input_data) @@ -325,7 +331,8 @@ class TestWhisperBasic: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) result = await processor._run_whisper_basic(input_data) @@ -368,7 +375,8 @@ class TestWhisperBasic: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) await processor._run_whisper_basic(input_data) @@ -392,7 +400,8 @@ class TestWhisperBasic: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) with pytest.raises(RuntimeError): @@ -435,7 +444,8 @@ class TestDiarization: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) result = await processor._run_whisper(input_data) @@ -470,7 +480,8 @@ class TestDiarization: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) result = await processor._run_whisper(input_data) @@ -550,7 +561,8 @@ class TestStateRecording: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) await processor.process(input_data) @@ -581,7 +593,8 @@ class TestStateRecording: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) with pytest.raises(Exception, match="Whisper error"): @@ -611,7 +624,8 @@ class TestStateRecording: input_data = ASRInput( content_id="BV1xx", platform="bilibili", - video_path=str(video_path) + video_path=str(video_path), + content_dir=str(tmp_path) ) # 应该不抛异常