Merge pull request 'fix/ci-deploy-reset' (#11) from fix/ci-deploy-reset into main

Reviewed-on: http://192.168.1.22:8418/starmen1265/kb-skills/pulls/11

.planning/debug/resolved/kb-workflow-asyncio-nested-loop.md

@@ -0,0 +1,60 @@
+---
+status: resolved
+trigger: "kb-workflow skill fails to download Bilibili videos with asyncio nested loop error"
+created: 2026-03-15T00:00:00Z
+updated: 2026-03-15T19:00:00Z
+---
+
+## Current Focus
+
+hypothesis: N/A - root cause confirmed and fix verified
+test: Ran kb-workflow process command - metadata extraction succeeded
+expecting: No asyncio nested loop error, successful metadata extraction
+next_action: Await human verification to confirm fix works in their environment
+
+## Symptoms
+
+expected: kb-workflow skill should successfully download Bilibili videos (BV16cNMzpExh)
+actual: Download fails with asyncio nested event loop error
+errors: `asyncio.run() cannot be called from a running event loop`
+reproduction: CLI command (e.g., `/kb-workflow download <url>`)
+started: Never worked properly - this is a new feature with a bug
+
+## Eliminated
+
+<!-- APPEND only -->
+
+## Evidence
+
+- timestamp: 2026-03-15T00:00:01Z
+  checked: kb-workflow/kb_workflow.py line 387
+  found: process_command() calls asyncio.run(run_pipeline()) - this is the outer event loop
+  implication: Creates an async context that all nested code runs within
+
+- timestamp: 2026-03-15T00:00:01Z
+  checked: kb_pipeline/pipeline.py line 106
+  found: Pipeline.run() is an async method that calls await processor.process()
+  implication: All processors run within the async event loop
+
+- timestamp: 2026-03-15T00:00:01Z
+  checked: kb_pipeline/processors/download.py line 66
+  found: DownloadProcessor.process() calls await adapter.fetch_metadata()
+  implication: BilibiliAdapter.fetch_metadata() is called from async context
+
+- timestamp: 2026-03-15T00:00:01Z
+  checked: kb_pipeline/platforms/bilibili.py line 173
+  found: BilibiliAdapter._fetch_metadata_with_retry() calls self._extractor.extract_video_metadata(content_id) which is a SYNC method
+  implication: Calls sync wrapper from async context
+
+- timestamp: 2026-03-15T00:00:01Z
+  checked: kb-bilibili-metadata/bilibili_extractor.py line 91
+  found: BilibiliExtractor.extract_video_metadata() calls asyncio.run(self._extract_video_async(bvid)) - this is the ROOT CAUSE
+  implication: asyncio.run() CANNOT be called when an event loop is already running, causing the nested loop error
+
+## Resolution
+
+root_cause: BilibiliExtractor provides sync wrapper methods (extract_video_metadata, extract_article_metadata, etc.) that use asyncio.run() internally. When BilibiliAdapter.fetch_metadata() (which is async) calls these sync methods, it triggers asyncio.run() from within an already-running event loop, which is forbidden in Python's asyncio model.
+fix: Modified BilibiliAdapter._fetch_metadata_with_retry() to call the underlying async methods directly (await self._extractor._extract_video_async() and await self._extractor._extract_article_async()) instead of the sync wrappers that use asyncio.run()
+verification: Self-verified - ran kb-workflow process command successfully. No asyncio nested loop error. Metadata extracted correctly including title, author, stats, description. Human-verified - user confirmed fix works (bilibili_BV16cNMzpExh executed successfully, asyncio error resolved).
+files_changed:
+  - kb_pipeline/platforms/bilibili.py

kb-zhihu-metadata/markdown_converter.py

@@ -1,7 +1,14 @@
+import re
+from bs4 import BeautifulSoup
 from markdownify import MarkdownConverter as _MarkdownConverter
 
 class MarkdownConverter:
-    """HTML 转 Markdown 转换器，保留完整格式"""
+    """HTML 转 Markdown 转换器，保留完整格式
+
+    处理知乎等平台的图片懒加载机制：
+    - 知乎图片的 src 属性可能是 SVG 占位符 (data:image/svg+xml;...)
+    - 真实图片 URL 存储在 data-actualsrc 或 data-original 属性中
+    """
 
     def __init__(self):
         self.converter = _MarkdownConverter(
@@ -13,5 +20,44 @@ class MarkdownConverter:
         )
 
     def convert(self, html: str) -> str:
-        """转换 HTML 为 Markdown"""
+        """转换 HTML 为 Markdown
+
+        Args:
+            html: 输入的 HTML 字符串
+
+        Returns:
+            转换后的 Markdown 字符串
+        """
+        # 预处理：处理懒加载图片
+        html = self._fix_lazy_load_images(html)
         return self.converter.convert(html)
+
+    def _fix_lazy_load_images(self, html: str) -> str:
+        """修复懒加载图片的 src 属性
+
+        将 data-actualsrc 或 data-original 属性的值替换到 src 属性中，
+        以便 markdownify 能够正确提取真实图片 URL。
+
+        Args:
+            html: 输入的 HTML 字符串
+
+        Returns:
+            处理后的 HTML 字符串
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+
+        for img in soup.find_all('img'):
+            src = img.get('src', '')
+
+            # 如果 src 不是占位符，保持原样
+            if not src or not src.startswith('data:'):
+                continue
+
+            # 尝试获取真实图片 URL
+            # 优先 data-actualsrc，其次 data-original
+            real_src = img.get('data-actualsrc') or img.get('data-original')
+
+            if real_src:
+                img['src'] = real_src
+
+        return str(soup)

kb_pipeline/pipeline.py

@@ -185,11 +185,13 @@ class Pipeline:
                     # ASR needs video path from download step
                     download_result = results.get("download", {})
                     video_path = download_result.get("content_file", "")
+
                     step_input = ASRInput(
                         content_id=parsed.content_id,
                         platform=parsed.platform,
                         video_path=video_path,
                         content_type=parsed.content_type,
+                        content_dir=str(content_dir),
                     )
                 elif step_name == "analyze":
                     # Analyze needs metadata and content files

kb_pipeline/processors/asr.py

@@ -172,9 +172,10 @@ class ASRProcessor:
         Note:
             无字幕返回 None（不是抛异常），让流程继续到 Whisper
         """
-        video_path = Path(input_data.video_path)
+        # 使用 content_dir 作为输出目录（而不是从 video_path 推断）
-        output_dir = video_path.parent
+        output_dir = Path(input_data.content_dir)
-        base_name = video_path.stem
+        # 使用 content_id 作为基础文件名
+        base_name = f"{input_data.platform}_{input_data.content_id}_subtitle"
 
         # 输出文件路径
         txt_path = output_dir / f"{base_name}.txt"
@@ -343,11 +344,18 @@ class ASRProcessor:
             ASR 处理结果
 
         Raises:
-            RuntimeError: Whisper.cpp 路径不存在或执行失败
+            RuntimeError: Whisper.cpp 路径不存在或执行失败，或 video_path 为空
         """
+        # 检查 video_path 是否有效
+        if not input_data.video_path:
+            raise RuntimeError(
+                "video_path 为空，无法执行 Whisper 语音识别。"
+                "请确保视频文件已下载（对于没有官方字幕的视频，需要先下载视频文件）。"
+            )
+
         video_path = Path(input_data.video_path)
-        output_dir = video_path.parent
+        output_dir = Path(input_data.content_dir)
-        base_name = video_path.stem
+        base_name = f"{input_data.platform}_{input_data.content_id}_whisper"
 
         # 获取 Whisper.cpp 路径
         try:

kb_pipeline/schemas.py

@@ -198,11 +198,13 @@ class ASRInput(StepInput):
     继承 StepInput 的基础字段，添加视频路径和内容类型。
 
     Attributes:
-        video_path: 视频文件路径
+        video_path: 视频文件路径（可能为空，用于 Whisper 降级）
         content_type: 内容类型，默认为 'video'
+        content_dir: 内容目录路径，用于输出字幕文件
     """
-    video_path: str = Field(..., description="视频文件路径")
+    video_path: str = Field(default="", description="视频文件路径（可能为空）")
     content_type: str = Field(default="video", description="内容类型")
+    content_dir: str = Field(..., description="内容目录路径")
 
 
 class ASROutput(StepOutput):

tests/test_processors/test_asr.py

@@ -31,7 +31,8 @@ class TestASRInputSchema:
         input_data = ASRInput(
             content_id="BV1xx",
             platform="bilibili",
-            video_path="/path/to/video.mp4"
+            video_path="/path/to/video.mp4",
+            content_dir="/path/to/content"
         )
 
         # 验证继承
@@ -46,7 +47,8 @@ class TestASRInputSchema:
         input_data = ASRInput(
             content_id="BV1xx",
             platform="bilibili",
-            video_path="/path/to/video.mp4"
+            video_path="/path/to/video.mp4",
+            content_dir="/path/to/content"
         )
 
         assert hasattr(input_data, "video_path")
@@ -59,7 +61,8 @@ class TestASRInputSchema:
         input_data = ASRInput(
             content_id="BV1xx",
             platform="bilibili",
-            video_path="/path/to/video.mp4"
+            video_path="/path/to/video.mp4",
+            content_dir="/path/to/content"
         )
 
         assert input_data.content_type == "video"
@@ -222,7 +225,8 @@ class TestOfficialSubtitle:
                 input_data = ASRInput(
                     content_id="BV1xx",
                     platform="bilibili",
-                    video_path=str(video_path)
+                    video_path=str(video_path),
+                    content_dir=str(tmp_path)
                 )
 
                 result = await processor._try_official_subtitle(input_data)
@@ -256,7 +260,8 @@ class TestOfficialSubtitle:
                 input_data = ASRInput(
                     content_id="BV1xx",
                     platform="bilibili",
-                    video_path=str(video_path)
+                    video_path=str(video_path),
+                    content_dir=str(tmp_path)
                 )
 
                 result = await processor._try_official_subtitle(input_data)
@@ -283,7 +288,8 @@ class TestOfficialSubtitle:
                 input_data = ASRInput(
                     content_id="BV1xx",
                     platform="bilibili",
-                    video_path=str(video_path)
+                    video_path=str(video_path),
+                    content_dir=str(tmp_path)
                 )
 
                 await processor.process(input_data)
@@ -325,7 +331,8 @@ class TestWhisperBasic:
                         input_data = ASRInput(
                             content_id="BV1xx",
                             platform="bilibili",
-                            video_path=str(video_path)
+                            video_path=str(video_path),
+                            content_dir=str(tmp_path)
                         )
 
                         result = await processor._run_whisper_basic(input_data)
@@ -368,7 +375,8 @@ class TestWhisperBasic:
                             input_data = ASRInput(
                                 content_id="BV1xx",
                                 platform="bilibili",
-                                video_path=str(video_path)
+                                video_path=str(video_path),
+                                content_dir=str(tmp_path)
                             )
 
                             await processor._run_whisper_basic(input_data)
@@ -392,7 +400,8 @@ class TestWhisperBasic:
                 input_data = ASRInput(
                     content_id="BV1xx",
                     platform="bilibili",
-                    video_path=str(video_path)
+                    video_path=str(video_path),
+                    content_dir=str(tmp_path)
                 )
 
                 with pytest.raises(RuntimeError):
@@ -435,7 +444,8 @@ class TestDiarization:
                             input_data = ASRInput(
                                 content_id="BV1xx",
                                 platform="bilibili",
-                                video_path=str(video_path)
+                                video_path=str(video_path),
+                                content_dir=str(tmp_path)
                             )
 
                             result = await processor._run_whisper(input_data)
@@ -470,7 +480,8 @@ class TestDiarization:
                 input_data = ASRInput(
                     content_id="BV1xx",
                     platform="bilibili",
-                    video_path=str(video_path)
+                    video_path=str(video_path),
+                    content_dir=str(tmp_path)
                 )
 
                 result = await processor._run_whisper(input_data)
@@ -550,7 +561,8 @@ class TestStateRecording:
                 input_data = ASRInput(
                     content_id="BV1xx",
                     platform="bilibili",
-                    video_path=str(video_path)
+                    video_path=str(video_path),
+                    content_dir=str(tmp_path)
                 )
 
                 await processor.process(input_data)
@@ -581,7 +593,8 @@ class TestStateRecording:
                     input_data = ASRInput(
                         content_id="BV1xx",
                         platform="bilibili",
-                        video_path=str(video_path)
+                        video_path=str(video_path),
+                        content_dir=str(tmp_path)
                     )
 
                     with pytest.raises(Exception, match="Whisper error"):
@@ -611,7 +624,8 @@ class TestStateRecording:
                 input_data = ASRInput(
                     content_id="BV1xx",
                     platform="bilibili",
-                    video_path=str(video_path)
+                    video_path=str(video_path),
+                    content_dir=str(tmp_path)
                 )
 
                 # 应该不抛异常

tests/test_zhihu_metadata/test_markdown_converter.py

@@ -0,0 +1,148 @@
+"""Tests for MarkdownConverter in kb_zhihu_metadata
+
+Tests cover:
+- Normal image conversion
+- Zhihu lazy loading image handling (data-actualsrc)
+- Alternative lazy loading (data-original)
+- Mixed content with images
+"""
+
+import pytest
+import sys
+from pathlib import Path
+
+# Add kb_zhihu_metadata to path
+zhihu_metadata_path = Path(__file__).parent.parent.parent / "kb-zhihu-metadata"
+if str(zhihu_metadata_path) not in sys.path:
+    sys.path.insert(0, str(zhihu_metadata_path))
+
+from markdown_converter import MarkdownConverter
+
+
+class TestMarkdownConverterImages:
+    """Tests for image handling in MarkdownConverter"""
+
+    def test_normal_image_conversion(self):
+        """Test that normal images are converted correctly"""
+        html = '<img src="https://example.com/image.jpg" alt="test">'
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "![test](https://example.com/image.jpg)" in result
+
+    def test_zhihu_lazy_load_image_with_data_actualsrc(self):
+        """Test that Zhihu lazy loading images use data-actualsrc"""
+        html = '''<img src="data:image/svg+xml;utf8,<svg></svg>"
+                   data-actualsrc="https://pic1.zhimg.com/v2-real-image_b.jpg">'''
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "https://pic1.zhimg.com/v2-real-image_b.jpg" in result
+        assert "data:image/svg+xml" not in result
+
+    def test_lazy_load_image_with_data_original(self):
+        """Test that images with data-original use that URL"""
+        html = '''<img src="data:image/svg+xml;..." data-original="https://example.com/original.jpg">'''
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "https://example.com/original.jpg" in result
+        assert "data:image/svg+xml" not in result
+
+    def test_data_actualsrc_priority_over_data_original(self):
+        """Test that data-actualsrc takes priority over data-original"""
+        html = '''<img src="data:image/svg+xml;..."
+                   data-actualsrc="https://example.com/actual.jpg"
+                   data-original="https://example.com/original.jpg">'''
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "https://example.com/actual.jpg" in result
+        assert "https://example.com/original.jpg" not in result
+
+    def test_normal_image_not_modified(self):
+        """Test that normal images with http(s) src are not modified"""
+        html = '<img src="https://example.com/normal.jpg">'
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "https://example.com/normal.jpg" in result
+
+    def test_mixed_content_with_images(self):
+        """Test conversion of mixed content with text and images"""
+        html = '''
+        <div class="RichText">
+            <p>First paragraph.</p>
+            <figure>
+                <img src="data:image/svg+xml;..." data-actualsrc="https://pic.zhimg.com/image1.jpg">
+            </figure>
+            <p>Second paragraph.</p>
+            <img src="https://example.com/normal.jpg">
+        </div>
+        '''
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+
+        # Check text content
+        assert "First paragraph" in result
+        assert "Second paragraph" in result
+
+        # Check lazy loaded image
+        assert "https://pic.zhimg.com/image1.jpg" in result
+
+        # Check normal image
+        assert "https://example.com/normal.jpg" in result
+
+        # Ensure no SVG placeholders in result
+        assert "data:image/svg+xml" not in result
+
+    def test_figure_with_caption(self):
+        """Test that figure elements with captions are handled correctly"""
+        html = '''
+        <figure>
+            <img src="data:image/svg+xml;..." data-actualsrc="https://pic.zhimg.com/fig.jpg">
+            <figcaption>Image caption</figcaption>
+        </figure>
+        '''
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+
+        assert "https://pic.zhimg.com/fig.jpg" in result
+        assert "Image caption" in result
+
+    def test_empty_src_with_data_actualsrc(self):
+        """Test handling of empty src with data-actualsrc"""
+        html = '<img src="" data-actualsrc="https://example.com/fallback.jpg">'
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        # Empty src is not a data: URI, so it should stay as is
+        assert "https://example.com/fallback.jpg" not in result
+
+
+class TestMarkdownConverterBasicConversion:
+    """Tests for basic HTML to Markdown conversion"""
+
+    def test_paragraphs(self):
+        """Test paragraph conversion"""
+        html = "<p>First</p><p>Second</p>"
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "First" in result
+        assert "Second" in result
+
+    def test_headings(self):
+        """Test heading conversion"""
+        html = "<h1>Title</h1><h2>Subtitle</h2>"
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "# Title" in result
+        assert "## Subtitle" in result
+
+    def test_links(self):
+        """Test link conversion"""
+        html = '<a href="https://example.com">Link</a>'
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "[Link](https://example.com)" in result
+
+    def test_code_blocks(self):
+        """Test code block conversion"""
+        html = "<pre><code>def hello():\n    print('hello')</code></pre>"
+        converter = MarkdownConverter()
+        result = converter.convert(html)
+        assert "```" in result or "def hello():" in result