fix(zhihu): handle lazy-load images in markdown conversion
知乎使用图片懒加载机制,`<img>` 标签的 `src` 属性包含 SVG 占位符, 真实图片 URL 在 `data-actualsrc` 或 `data-original` 属性中。 `markdownify` 只读取 `src`,导致转换后得到无效占位符。 修复:在转换前预处理 HTML,将真实图片 URL 替换到 `src` 属性。 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,14 @@
|
||||
import re
|
||||
from bs4 import BeautifulSoup
|
||||
from markdownify import MarkdownConverter as _MarkdownConverter
|
||||
|
||||
class MarkdownConverter:
|
||||
"""HTML 转 Markdown 转换器,保留完整格式"""
|
||||
"""HTML 转 Markdown 转换器,保留完整格式
|
||||
|
||||
处理知乎等平台的图片懒加载机制:
|
||||
- 知乎图片的 src 属性可能是 SVG 占位符 (data:image/svg+xml;...)
|
||||
- 真实图片 URL 存储在 data-actualsrc 或 data-original 属性中
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.converter = _MarkdownConverter(
|
||||
@@ -13,5 +20,44 @@ class MarkdownConverter:
|
||||
)
|
||||
|
||||
def convert(self, html: str) -> str:
|
||||
"""转换 HTML 为 Markdown"""
|
||||
"""转换 HTML 为 Markdown
|
||||
|
||||
Args:
|
||||
html: 输入的 HTML 字符串
|
||||
|
||||
Returns:
|
||||
转换后的 Markdown 字符串
|
||||
"""
|
||||
# 预处理:处理懒加载图片
|
||||
html = self._fix_lazy_load_images(html)
|
||||
return self.converter.convert(html)
|
||||
|
||||
def _fix_lazy_load_images(self, html: str) -> str:
|
||||
"""修复懒加载图片的 src 属性
|
||||
|
||||
将 data-actualsrc 或 data-original 属性的值替换到 src 属性中,
|
||||
以便 markdownify 能够正确提取真实图片 URL。
|
||||
|
||||
Args:
|
||||
html: 输入的 HTML 字符串
|
||||
|
||||
Returns:
|
||||
处理后的 HTML 字符串
|
||||
"""
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
for img in soup.find_all('img'):
|
||||
src = img.get('src', '')
|
||||
|
||||
# 如果 src 不是占位符,保持原样
|
||||
if not src or not src.startswith('data:'):
|
||||
continue
|
||||
|
||||
# 尝试获取真实图片 URL
|
||||
# 优先 data-actualsrc,其次 data-original
|
||||
real_src = img.get('data-actualsrc') or img.get('data-original')
|
||||
|
||||
if real_src:
|
||||
img['src'] = real_src
|
||||
|
||||
return str(soup)
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Tests for MarkdownConverter in kb_zhihu_metadata
|
||||
|
||||
Tests cover:
|
||||
- Normal image conversion
|
||||
- Zhihu lazy loading image handling (data-actualsrc)
|
||||
- Alternative lazy loading (data-original)
|
||||
- Mixed content with images
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add kb_zhihu_metadata to path
|
||||
zhihu_metadata_path = Path(__file__).parent.parent.parent / "kb-zhihu-metadata"
|
||||
if str(zhihu_metadata_path) not in sys.path:
|
||||
sys.path.insert(0, str(zhihu_metadata_path))
|
||||
|
||||
from markdown_converter import MarkdownConverter
|
||||
|
||||
|
||||
class TestMarkdownConverterImages:
|
||||
"""Tests for image handling in MarkdownConverter"""
|
||||
|
||||
def test_normal_image_conversion(self):
|
||||
"""Test that normal images are converted correctly"""
|
||||
html = '<img src="https://example.com/image.jpg" alt="test">'
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "" in result
|
||||
|
||||
def test_zhihu_lazy_load_image_with_data_actualsrc(self):
|
||||
"""Test that Zhihu lazy loading images use data-actualsrc"""
|
||||
html = '''<img src="data:image/svg+xml;utf8,<svg></svg>"
|
||||
data-actualsrc="https://pic1.zhimg.com/v2-real-image_b.jpg">'''
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "https://pic1.zhimg.com/v2-real-image_b.jpg" in result
|
||||
assert "data:image/svg+xml" not in result
|
||||
|
||||
def test_lazy_load_image_with_data_original(self):
|
||||
"""Test that images with data-original use that URL"""
|
||||
html = '''<img src="data:image/svg+xml;..." data-original="https://example.com/original.jpg">'''
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "https://example.com/original.jpg" in result
|
||||
assert "data:image/svg+xml" not in result
|
||||
|
||||
def test_data_actualsrc_priority_over_data_original(self):
|
||||
"""Test that data-actualsrc takes priority over data-original"""
|
||||
html = '''<img src="data:image/svg+xml;..."
|
||||
data-actualsrc="https://example.com/actual.jpg"
|
||||
data-original="https://example.com/original.jpg">'''
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "https://example.com/actual.jpg" in result
|
||||
assert "https://example.com/original.jpg" not in result
|
||||
|
||||
def test_normal_image_not_modified(self):
|
||||
"""Test that normal images with http(s) src are not modified"""
|
||||
html = '<img src="https://example.com/normal.jpg">'
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "https://example.com/normal.jpg" in result
|
||||
|
||||
def test_mixed_content_with_images(self):
|
||||
"""Test conversion of mixed content with text and images"""
|
||||
html = '''
|
||||
<div class="RichText">
|
||||
<p>First paragraph.</p>
|
||||
<figure>
|
||||
<img src="data:image/svg+xml;..." data-actualsrc="https://pic.zhimg.com/image1.jpg">
|
||||
</figure>
|
||||
<p>Second paragraph.</p>
|
||||
<img src="https://example.com/normal.jpg">
|
||||
</div>
|
||||
'''
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
|
||||
# Check text content
|
||||
assert "First paragraph" in result
|
||||
assert "Second paragraph" in result
|
||||
|
||||
# Check lazy loaded image
|
||||
assert "https://pic.zhimg.com/image1.jpg" in result
|
||||
|
||||
# Check normal image
|
||||
assert "https://example.com/normal.jpg" in result
|
||||
|
||||
# Ensure no SVG placeholders in result
|
||||
assert "data:image/svg+xml" not in result
|
||||
|
||||
def test_figure_with_caption(self):
|
||||
"""Test that figure elements with captions are handled correctly"""
|
||||
html = '''
|
||||
<figure>
|
||||
<img src="data:image/svg+xml;..." data-actualsrc="https://pic.zhimg.com/fig.jpg">
|
||||
<figcaption>Image caption</figcaption>
|
||||
</figure>
|
||||
'''
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
|
||||
assert "https://pic.zhimg.com/fig.jpg" in result
|
||||
assert "Image caption" in result
|
||||
|
||||
def test_empty_src_with_data_actualsrc(self):
|
||||
"""Test handling of empty src with data-actualsrc"""
|
||||
html = '<img src="" data-actualsrc="https://example.com/fallback.jpg">'
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
# Empty src is not a data: URI, so it should stay as is
|
||||
assert "https://example.com/fallback.jpg" not in result
|
||||
|
||||
|
||||
class TestMarkdownConverterBasicConversion:
|
||||
"""Tests for basic HTML to Markdown conversion"""
|
||||
|
||||
def test_paragraphs(self):
|
||||
"""Test paragraph conversion"""
|
||||
html = "<p>First</p><p>Second</p>"
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "First" in result
|
||||
assert "Second" in result
|
||||
|
||||
def test_headings(self):
|
||||
"""Test heading conversion"""
|
||||
html = "<h1>Title</h1><h2>Subtitle</h2>"
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "# Title" in result
|
||||
assert "## Subtitle" in result
|
||||
|
||||
def test_links(self):
|
||||
"""Test link conversion"""
|
||||
html = '<a href="https://example.com">Link</a>'
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "[Link](https://example.com)" in result
|
||||
|
||||
def test_code_blocks(self):
|
||||
"""Test code block conversion"""
|
||||
html = "<pre><code>def hello():\n print('hello')</code></pre>"
|
||||
converter = MarkdownConverter()
|
||||
result = converter.convert(html)
|
||||
assert "```" in result or "def hello():" in result
|
||||
Reference in New Issue
Block a user