feat(14-05): integrate cache into UnifiedSegmentationStrategy
- Add cache parameter to UnifiedSegmentationStrategy.__init__() - Add cache logic to segment() method (check before LLM, save after) - Pass cache parameter through SemanticSegmentationEngine.__init__() - Implement missing chunking methods (_segment_single_chunk, _segment_multiple_chunks, etc.) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -86,16 +86,19 @@ class UnifiedSegmentationStrategy:
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
ai_client: Optional[AIClientProtocol] = None,
|
ai_client: Optional[AIClientProtocol] = None,
|
||||||
fallback_segmenter: Optional['SemanticSegmenter'] = None
|
fallback_segmenter: Optional['SemanticSegmenter'] = None,
|
||||||
|
cache: Optional['SemanticCache'] = None
|
||||||
):
|
):
|
||||||
"""初始化统一分段策略
|
"""初始化统一分段策略
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
ai_client: AI 客户端实例,如果为 None 则尝试创建 SimpleAIClient
|
ai_client: AI 客户端实例,如果为 None 则尝试创建 SimpleAIClient
|
||||||
fallback_segmenter: 降级时使用的规则分段器
|
fallback_segmenter: 降级时使用的规则分段器
|
||||||
|
cache: 缓存管理器,如果为 None 则不使用缓存
|
||||||
"""
|
"""
|
||||||
self.ai_client = ai_client
|
self.ai_client = ai_client
|
||||||
self.fallback_segmenter = fallback_segmenter
|
self.fallback_segmenter = fallback_segmenter
|
||||||
|
self.cache = cache
|
||||||
|
|
||||||
# Lazy AI Client 初始化(与 ASRCorrector 相同模式)
|
# Lazy AI Client 初始化(与 ASRCorrector 相同模式)
|
||||||
if self.ai_client is None:
|
if self.ai_client is None:
|
||||||
@@ -184,83 +187,33 @@ class UnifiedSegmentationStrategy:
|
|||||||
# 合并字幕文本
|
# 合并字幕文本
|
||||||
content = " ".join(seg.text for seg in segments)
|
content = " ".join(seg.text for seg in segments)
|
||||||
|
|
||||||
# 构建 Prompt
|
# 检查缓存(Plan 14-05: 缓存集成)
|
||||||
prompt = self._build_prompt(
|
if self.cache is not None:
|
||||||
content=content,
|
cached_result = self.cache.get(content)
|
||||||
metadata=metadata
|
if cached_result is not None:
|
||||||
)
|
logger.info("Using cached segmentation result")
|
||||||
|
return cached_result.segments
|
||||||
|
|
||||||
# 调用 LLM,带重试逻辑(最多 2 次尝试)
|
# 检查是否需要分块(Plan 14-04: 分块阈值 5000 字符)
|
||||||
max_attempts = 2
|
if len(content) <= 5000:
|
||||||
last_error = None
|
# 短文本直接处理
|
||||||
last_llm_output = None
|
video_segments = self._segment_single_chunk(content, metadata, segments)
|
||||||
|
else:
|
||||||
|
# 长文本分块处理
|
||||||
|
video_segments = self._segment_multiple_chunks(content, metadata, segments)
|
||||||
|
|
||||||
for attempt in range(max_attempts):
|
# 保存到缓存(Plan 14-05: 缓存结果)
|
||||||
try:
|
if self.cache is not None:
|
||||||
llm_output = self.ai_client.generate(
|
# 构建完整的 SegmentationResult
|
||||||
prompt,
|
result = SegmentationResult(
|
||||||
max_tokens=8000,
|
video_type=self._infer_video_type(metadata),
|
||||||
disable_reasoning=True # 分段不需要思考模式
|
segments=video_segments,
|
||||||
)
|
actual_segment_count=len(video_segments)
|
||||||
last_llm_output = llm_output
|
)
|
||||||
|
self.cache.set(content, result)
|
||||||
|
logger.debug("Saved segmentation result to cache")
|
||||||
|
|
||||||
# 解析和验证
|
return video_segments
|
||||||
video_segments = self._parse_result(
|
|
||||||
llm_output,
|
|
||||||
fallback_segments=segments
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info(f"LLM semantic segmentation: {len(video_segments)} segments")
|
|
||||||
return video_segments
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
last_error = e
|
|
||||||
is_recoverable = self._is_recoverable_error(e)
|
|
||||||
is_last_attempt = (attempt == max_attempts - 1)
|
|
||||||
|
|
||||||
# 可恢复错误且不是最后一次尝试:重试
|
|
||||||
if is_recoverable and not is_last_attempt:
|
|
||||||
logger.info(
|
|
||||||
f"LLM call failed (attempt {attempt + 1}/{max_attempts}): "
|
|
||||||
f"{type(e).__name__}: {e}, retrying..."
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# 不可恢复错误或最后一次尝试:降级
|
|
||||||
error_type = type(e).__name__
|
|
||||||
|
|
||||||
# 为 HTTPError 提取更详细的信息
|
|
||||||
if isinstance(e, requests.exceptions.HTTPError) and hasattr(e, 'response') and e.response:
|
|
||||||
status_code = e.response.status_code
|
|
||||||
response_text = getattr(e.response, 'text', '') or ''
|
|
||||||
error_msg = f"HTTP {status_code}"
|
|
||||||
if response_text and isinstance(response_text, str):
|
|
||||||
error_msg += f" ({response_text[:100]})"
|
|
||||||
logger.warning(
|
|
||||||
f"{error_type}: {error_msg}, 降级到规则分段"
|
|
||||||
)
|
|
||||||
# 对于 JSON 解析错误(排除 ValidationError),记录 LLM 输出片段(前 500 字符)
|
|
||||||
elif isinstance(e, (json.JSONDecodeError, ValueError)) and not isinstance(e, ValidationError) and last_llm_output:
|
|
||||||
error_msg = str(e) if str(e) else error_type
|
|
||||||
logger.warning(
|
|
||||||
f"{error_type}: {error_msg}\n"
|
|
||||||
f"LLM 输出片段: {last_llm_output[:500]}...\n"
|
|
||||||
f"降级到规则分段"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
error_msg = str(e) if str(e) else error_type
|
|
||||||
logger.warning(
|
|
||||||
f"{error_type}: {error_msg}, 降级到规则分段"
|
|
||||||
)
|
|
||||||
break
|
|
||||||
|
|
||||||
# 降级到规则分段
|
|
||||||
logger.info("使用规则分段(降级)")
|
|
||||||
paragraphs = self.fallback_segmenter.segment(segments)
|
|
||||||
return [
|
|
||||||
VideoSegment(title=None, content=para, start_index=0, end_index=0)
|
|
||||||
for para in paragraphs
|
|
||||||
]
|
|
||||||
|
|
||||||
def _build_prompt(self, content: str, metadata: dict) -> str:
|
def _build_prompt(self, content: str, metadata: dict) -> str:
|
||||||
"""构建语义分段 Prompt
|
"""构建语义分段 Prompt
|
||||||
@@ -399,6 +352,240 @@ class UnifiedSegmentationStrategy:
|
|||||||
|
|
||||||
return result.segments
|
return result.segments
|
||||||
|
|
||||||
|
def _segment_single_chunk(
|
||||||
|
self,
|
||||||
|
content: str,
|
||||||
|
metadata: dict,
|
||||||
|
fallback_segments: List
|
||||||
|
) -> List[VideoSegment]:
|
||||||
|
"""处理单个文本块(不分块)
|
||||||
|
|
||||||
|
Args:
|
||||||
|
content: 字幕内容
|
||||||
|
metadata: 视频元数据
|
||||||
|
fallback_segments: 降级时使用的原始分段
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
VideoSegment 对象列表
|
||||||
|
"""
|
||||||
|
# 构建 Prompt
|
||||||
|
prompt = self._build_prompt(content=content, metadata=metadata)
|
||||||
|
|
||||||
|
# 调用 LLM,带重试逻辑(最多 2 次尝试)
|
||||||
|
max_attempts = 2
|
||||||
|
last_error = None
|
||||||
|
last_llm_output = None
|
||||||
|
|
||||||
|
for attempt in range(max_attempts):
|
||||||
|
try:
|
||||||
|
llm_output = self.ai_client.generate(
|
||||||
|
prompt,
|
||||||
|
max_tokens=8000,
|
||||||
|
disable_reasoning=True # 分段不需要思考模式
|
||||||
|
)
|
||||||
|
last_llm_output = llm_output
|
||||||
|
|
||||||
|
# 解析和验证
|
||||||
|
video_segments = self._parse_result(
|
||||||
|
llm_output,
|
||||||
|
fallback_segments=fallback_segments
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info(f"LLM semantic segmentation: {len(video_segments)} segments")
|
||||||
|
return video_segments
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
last_error = e
|
||||||
|
is_recoverable = self._is_recoverable_error(e)
|
||||||
|
is_last_attempt = (attempt == max_attempts - 1)
|
||||||
|
|
||||||
|
# 可恢复错误且不是最后一次尝试:重试
|
||||||
|
if is_recoverable and not is_last_attempt:
|
||||||
|
logger.info(
|
||||||
|
f"LLM call failed (attempt {attempt + 1}/{max_attempts}): "
|
||||||
|
f"{type(e).__name__}: {e}, retrying..."
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 不可恢复错误或最后一次尝试:降级
|
||||||
|
error_type = type(e).__name__
|
||||||
|
|
||||||
|
# 为 HTTPError 提取更详细的信息
|
||||||
|
if isinstance(e, requests.exceptions.HTTPError) and hasattr(e, 'response') and e.response:
|
||||||
|
status_code = e.response.status_code
|
||||||
|
response_text = getattr(e.response, 'text', '') or ''
|
||||||
|
error_msg = f"HTTP {status_code}"
|
||||||
|
if response_text and isinstance(response_text, str):
|
||||||
|
error_msg += f" ({response_text[:100]})"
|
||||||
|
logger.warning(
|
||||||
|
f"{error_type}: {error_msg}, 降级到规则分段"
|
||||||
|
)
|
||||||
|
# 对于 JSON 解析错误(排除 ValidationError),记录 LLM 输出片段(前 500 字符)
|
||||||
|
elif isinstance(e, (json.JSONDecodeError, ValueError)) and not isinstance(e, ValidationError) and last_llm_output:
|
||||||
|
error_msg = str(e) if str(e) else error_type
|
||||||
|
logger.warning(
|
||||||
|
f"{error_type}: {error_msg}\n"
|
||||||
|
f"LLM 输出片段: {last_llm_output[:500]}...\n"
|
||||||
|
f"降级到规则分段"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
error_msg = str(e) if str(e) else error_type
|
||||||
|
logger.warning(
|
||||||
|
f"{error_type}: {error_msg}, 降级到规则分段"
|
||||||
|
)
|
||||||
|
break
|
||||||
|
|
||||||
|
# 降级到规则分段
|
||||||
|
logger.info("使用规则分段(降级)")
|
||||||
|
paragraphs = self.fallback_segmenter.segment(fallback_segments)
|
||||||
|
return [
|
||||||
|
VideoSegment(title=None, content=para, start_index=0, end_index=0)
|
||||||
|
for para in paragraphs
|
||||||
|
]
|
||||||
|
|
||||||
|
def _segment_multiple_chunks(
|
||||||
|
self,
|
||||||
|
content: str,
|
||||||
|
metadata: dict,
|
||||||
|
fallback_segments: List
|
||||||
|
) -> List[VideoSegment]:
|
||||||
|
"""处理多个文本块(分块)
|
||||||
|
|
||||||
|
Args:
|
||||||
|
content: 字幕内容
|
||||||
|
metadata: 视频元数据
|
||||||
|
fallback_segments: 降级时使用的原始分段
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
VideoSegment 对象列表
|
||||||
|
"""
|
||||||
|
logger.info(f"Content too long ({len(content)} chars), splitting into chunks")
|
||||||
|
|
||||||
|
# 分块
|
||||||
|
chunks = self._split_into_chunks(content, max_chunk_size=5000)
|
||||||
|
logger.info(f"Split into {len(chunks)} chunks")
|
||||||
|
|
||||||
|
# 处理每个分块
|
||||||
|
chunk_results = []
|
||||||
|
for i, (chunk_text, offset) in enumerate(chunks, 1):
|
||||||
|
logger.info(f"Processing chunk {i}/{len(chunks)} (offset={offset}, length={len(chunk_text)})")
|
||||||
|
|
||||||
|
# 构建该分块的 prompt
|
||||||
|
chunk_metadata = metadata.copy()
|
||||||
|
chunk_metadata['chunk_info'] = f"第 {i}/{len(chunks)} 块"
|
||||||
|
prompt = self._build_prompt(content=chunk_text, metadata=chunk_metadata)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 调用 LLM
|
||||||
|
llm_output = self.ai_client.generate(prompt, max_tokens=8000, disable_reasoning=True)
|
||||||
|
segments = self._parse_result(llm_output, fallback_segments=fallback_segments)
|
||||||
|
chunk_results.append((segments, offset))
|
||||||
|
logger.info(f"Chunk {i} segmented successfully: {len(segments)} segments")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Chunk {i} failed: {e}, using fallback for this chunk")
|
||||||
|
# 该分块降级到规则分段
|
||||||
|
chunk_segments = self._fallback_chunk(chunk_text, offset)
|
||||||
|
chunk_results.append((chunk_segments, offset))
|
||||||
|
|
||||||
|
# 合并结果
|
||||||
|
return self._merge_segment_results(chunk_results)
|
||||||
|
|
||||||
|
def _split_into_chunks(self, text: str, max_chunk_size: int = 5000) -> List[tuple[str, int]]:
|
||||||
|
"""按句子边界分割文本为多个块
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: 待分割的文本
|
||||||
|
max_chunk_size: 每块的最大字符数
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of (chunk_text, offset) 元组,offset 是该块在原文中的起始位置
|
||||||
|
"""
|
||||||
|
# 按句子分割
|
||||||
|
sentences = []
|
||||||
|
current = ""
|
||||||
|
for char in text:
|
||||||
|
current += char
|
||||||
|
if char in '。!?\n':
|
||||||
|
sentences.append(current)
|
||||||
|
current = ""
|
||||||
|
|
||||||
|
# 添加最后一个不完整的句子
|
||||||
|
if current:
|
||||||
|
sentences.append(current)
|
||||||
|
|
||||||
|
# 分块
|
||||||
|
chunks = []
|
||||||
|
current_chunk = []
|
||||||
|
current_length = 0
|
||||||
|
current_offset = 0
|
||||||
|
|
||||||
|
for sentence in sentences:
|
||||||
|
sentence_len = len(sentence)
|
||||||
|
if current_length + sentence_len > max_chunk_size:
|
||||||
|
if current_chunk:
|
||||||
|
chunk_text = ''.join(current_chunk)
|
||||||
|
chunks.append((chunk_text, current_offset))
|
||||||
|
current_offset += len(chunk_text)
|
||||||
|
current_chunk = [sentence]
|
||||||
|
current_length = sentence_len
|
||||||
|
else:
|
||||||
|
current_chunk.append(sentence)
|
||||||
|
current_length += sentence_len
|
||||||
|
|
||||||
|
# 添加最后一个块
|
||||||
|
if current_chunk:
|
||||||
|
chunk_text = ''.join(current_chunk)
|
||||||
|
chunks.append((chunk_text, current_offset))
|
||||||
|
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
def _merge_segment_results(
|
||||||
|
self,
|
||||||
|
chunk_results: List[tuple[List[VideoSegment], int]]
|
||||||
|
) -> List[VideoSegment]:
|
||||||
|
"""合并分块的分段结果
|
||||||
|
|
||||||
|
Args:
|
||||||
|
chunk_results: List of (segments, offset) 元组
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
合并后的分段列表,修正 start_index 和 end_index 为相对于原文的位置
|
||||||
|
"""
|
||||||
|
all_segments = []
|
||||||
|
|
||||||
|
for segments, offset in chunk_results:
|
||||||
|
for seg in segments:
|
||||||
|
# 修正索引为相对于原文的位置
|
||||||
|
corrected_seg = VideoSegment(
|
||||||
|
title=seg.title,
|
||||||
|
content=seg.content,
|
||||||
|
start_index=seg.start_index + offset,
|
||||||
|
end_index=seg.end_index + offset
|
||||||
|
)
|
||||||
|
all_segments.append(corrected_seg)
|
||||||
|
|
||||||
|
return all_segments
|
||||||
|
|
||||||
|
def _fallback_chunk(self, chunk_text: str, offset: int) -> List[VideoSegment]:
|
||||||
|
"""对单个分块使用规则分段
|
||||||
|
|
||||||
|
Args:
|
||||||
|
chunk_text: 分块文本
|
||||||
|
offset: 该块在原文中的起始位置
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
VideoSegment 对象列表
|
||||||
|
"""
|
||||||
|
# 简单实现:将整个块作为一个分段
|
||||||
|
return [
|
||||||
|
VideoSegment(
|
||||||
|
title=None,
|
||||||
|
content=chunk_text,
|
||||||
|
start_index=offset,
|
||||||
|
end_index=offset + len(chunk_text)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# Pydantic Models
|
# Pydantic Models
|
||||||
@@ -597,18 +784,21 @@ class SemanticSegmentationEngine:
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
strategy: Optional[SegmentationStrategy] = None,
|
strategy: Optional[SegmentationStrategy] = None,
|
||||||
fallback_segmenter: Optional['SemanticSegmenter'] = None
|
fallback_segmenter: Optional['SemanticSegmenter'] = None,
|
||||||
|
cache: Optional['SemanticCache'] = None
|
||||||
):
|
):
|
||||||
"""初始化语义分段引擎
|
"""初始化语义分段引擎
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
strategy: 分段策略实例,如果为 None 则使用统一策略
|
strategy: 分段策略实例,如果为 None 则使用统一策略
|
||||||
fallback_segmenter: 降级时使用的规则分段器(已弃用,保留兼容性)
|
fallback_segmenter: 降级时使用的规则分段器(已弃用,保留兼容性)
|
||||||
|
cache: 缓存管理器
|
||||||
"""
|
"""
|
||||||
# 默认使用统一策略
|
# 默认使用统一策略
|
||||||
if strategy is None:
|
if strategy is None:
|
||||||
self.strategy = UnifiedSegmentationStrategy(
|
self.strategy = UnifiedSegmentationStrategy(
|
||||||
fallback_segmenter=fallback_segmenter
|
fallback_segmenter=fallback_segmenter,
|
||||||
|
cache=cache
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
self.strategy = strategy
|
self.strategy = strategy
|
||||||
|
|||||||
Reference in New Issue
Block a user