fix/crawler-proxy-config #42

Merged
starmen1265 merged 74 commits from fix/crawler-proxy-config into main 2026-03-24 00:21:02 +08:00
Showing only changes of commit faff7903a4 - Show all commits
+29 -9
View File
@@ -567,24 +567,44 @@ class UnifiedSegmentationStrategy:
return all_segments
def _fallback_chunk(self, chunk_text: str, offset: int) -> List[VideoSegment]:
"""对单个分块使用规则分段
"""对单个分块使用规则分段(降级)
Args:
chunk_text: 分块文本
offset: 该块在原文中的起始位置
offset: 该分块在原文中的起始位置
Returns:
VideoSegment 对象列表
"""
# 简单实现:将整个块作为一个分段
return [
VideoSegment(
# 创建虚拟的分段对象用于规则分段器
# 规则分段器需要 Segment 对象列表,我们创建一个包含整个文本的虚拟分段
from kb_workflow.srt.parser import Segment
virtual_segment = Segment(
index=1,
start_time=0.0,
end_time=1.0,
text=chunk_text
)
# 使用规则分段器
paragraphs = self.fallback_segmenter.segment([virtual_segment])
# 转换为 VideoSegment,并设置正确的偏移量
segments = []
current_pos = offset
for para in paragraphs:
# 计算该分段在原文中的位置
seg = VideoSegment(
title=None,
content=chunk_text,
start_index=offset,
end_index=offset + len(chunk_text)
content=para,
start_index=current_pos,
end_index=current_pos + len(para)
)
]
segments.append(seg)
current_pos += len(para)
return segments
# ============================================================================