"""测试语义分段的鲁棒性功能 (Phase 14) 测试 LLM 调用失败时的重试和降级策略。 Plan 14-01: 实现可恢复错误检测和重试逻辑 """ import json import pytest from unittest.mock import Mock from requests.exceptions import ConnectionError, Timeout, HTTPError from kb_workflow.srt.parser import Segment from kb_workflow.srt.semantic_engine import ( UnifiedSegmentationStrategy, VideoSegment ) class TestRecoverableErrorDetection: """测试可恢复错误检测功能""" def test_is_recoverable_error_connection_error(self): """Test: ConnectionError 被识别为可恢复错误""" strategy = UnifiedSegmentationStrategy() # 创建模拟客户端,抛出 ConnectionError mock_client = Mock() mock_client.api_key = "test_key" mock_client.generate.side_effect = ConnectionError("Connection refused") strategy = UnifiedSegmentationStrategy(ai_client=mock_client) # 准备测试数据 segments = [ Segment(1, 0.0, 1.0, "第一句"), Segment(2, 1.0, 2.0, "第二句") ] # 应该降级到规则分段(ConnectionError 可恢复,重试 1 次后降级) result = strategy.segment(segments, {"title": "测试"}) # 验证降级结果 assert len(result) > 0 # 验证调用了 2 次(初始 + 1 次重试) assert mock_client.generate.call_count == 2 def test_is_recoverable_error_timeout(self): """Test: Timeout 被识别为可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" mock_client.generate.side_effect = Timeout("Request timeout") strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 应该重试 1 次 assert mock_client.generate.call_count == 2 assert len(result) > 0 def test_is_recoverable_error_rate_limit_429(self): """Test: HTTP 429 (限流) 被识别为可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" # 创建 429 响应 response = Mock() response.status_code = 429 response.text = "Rate limit exceeded" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 应该重试 1 次 assert mock_client.generate.call_count == 2 assert len(result) > 0 def test_is_recoverable_error_server_error_500(self): """Test: HTTP 500 (服务器错误) 被识别为可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 500 response.text = "Internal server error" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 应该重试 1 次 assert mock_client.generate.call_count == 2 assert len(result) > 0 def test_is_recoverable_error_server_error_502(self): """Test: HTTP 502 (网关错误) 被识别为可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 502 response.text = "Bad gateway" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 应该重试 1 次 assert mock_client.generate.call_count == 2 assert len(result) > 0 def test_is_recoverable_error_server_error_503(self): """Test: HTTP 503 (服务不可用) 被识别为可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 503 response.text = "Service unavailable" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 应该重试 1 次 assert mock_client.generate.call_count == 2 assert len(result) > 0 def test_is_recoverable_error_server_error_504(self): """Test: HTTP 504 (网关超时) 被识别为可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 504 response.text = "Gateway timeout" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 应该重试 1 次 assert mock_client.generate.call_count == 2 assert len(result) > 0 class TestNonRecoverableErrorDetection: """测试不可恢复错误检测功能""" def test_non_recoverable_error_400(self): """Test: HTTP 400 (错误请求) 被识别为不可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 400 response.text = "Bad request" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 不应该重试,直接降级 assert mock_client.generate.call_count == 1 assert len(result) > 0 def test_non_recoverable_error_401(self): """Test: HTTP 401 (未授权) 被识别为不可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 401 response.text = "Unauthorized" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 不应该重试 assert mock_client.generate.call_count == 1 assert len(result) > 0 def test_non_recoverable_error_403(self): """Test: HTTP 403 (禁止访问) 被识别为不可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 403 response.text = "Forbidden" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 不应该重试 assert mock_client.generate.call_count == 1 assert len(result) > 0 def test_non_recoverable_error_404(self): """Test: HTTP 404 (未找到) 被识别为不可恢复错误""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 404 response.text = "Not found" http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 不应该重试 assert mock_client.generate.call_count == 1 assert len(result) > 0 class TestRetryMechanism: """测试重试机制""" def test_retry_once_on_recoverable_error(self): """Test: 可恢复错误重试 1 次""" mock_client = Mock() mock_client.api_key = "test_key" mock_client.generate.side_effect = ConnectionError("Network error") strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 验证调用次数:初始 1 次 + 重试 1 次 = 2 次 assert mock_client.generate.call_count == 2 def test_no_retry_on_non_recoverable_error(self): """Test: 不可恢复错误不重试""" mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 401 http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] result = strategy.segment(segments, {"title": "测试"}) # 验证只调用 1 次(不重试) assert mock_client.generate.call_count == 1 def test_immediate_retry_no_delay(self): """Test: 立即重试,无延迟""" import time mock_client = Mock() mock_client.api_key = "test_key" mock_client.generate.side_effect = ConnectionError("Network error") strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] start_time = time.time() result = strategy.segment(segments, {"title": "测试"}) elapsed_time = time.time() - start_time # 验证重试是立即的(无延迟) # 正常重试应该有延迟,立即重试应该很快(< 0.1 秒) assert elapsed_time < 0.1 class TestFallbackToRuleBased: """测试降级到规则分段""" def test_fallback_on_llm_failure(self): """Test: LLM 调用失败时降级到规则分段""" mock_client = Mock() mock_client.api_key = "test_key" mock_client.generate.side_effect = Exception("LLM error") strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [ Segment(1, 0.0, 1.0, "第一句"), Segment(2, 1.0, 2.0, "第二句") ] result = strategy.segment(segments, {"title": "测试"}) # 验证返回了规则分段结果 assert len(result) > 0 # 验证返回的是 VideoSegment 对象(规则分段转换后的格式) assert all(isinstance(seg, VideoSegment) for seg in result) # 验证规则分段没有标题(降级分段标题为 null) assert all(seg.title is None for seg in result) def test_fallback_when_ai_client_unavailable(self): """Test: AI 客户端不可用时直接使用规则分段""" # 创建没有 AI 客户端的策略 strategy = UnifiedSegmentationStrategy(ai_client=None) segments = [ Segment(1, 0.0, 1.0, "第一句"), Segment(2, 1.0, 2.0, "第二句") ] result = strategy.segment(segments, {"title": "测试"}) # 验证返回了规则分段结果 assert len(result) > 0 assert all(isinstance(seg, VideoSegment) for seg in result) assert all(seg.title is None for seg in result) class TestLoggingBehavior: """测试日志记录行为""" def test_retry_logging(self, caplog): """Test: 重试时记录 info 级别日志""" import logging mock_client = Mock() mock_client.api_key = "test_key" mock_client.generate.side_effect = ConnectionError("Network error") strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] with caplog.at_level(logging.INFO): result = strategy.segment(segments, {"title": "测试"}) # 验证日志包含重试信息 assert any("重试" in record.message or "retry" in record.message.lower() for record in caplog.records if record.levelno == logging.INFO) def test_fallback_warning_logging(self, caplog): """Test: 降级时记录 warning 级别日志""" import logging mock_client = Mock() mock_client.api_key = "test_key" response = Mock() response.status_code = 401 http_error = HTTPError(response=response) mock_client.generate.side_effect = http_error strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] with caplog.at_level(logging.WARNING): result = strategy.segment(segments, {"title": "测试"}) # 验证日志包含降级信息和错误类型 assert any("降级" in record.message or "fallback" in record.message.lower() for record in caplog.records if record.levelno == logging.WARNING) # 验证日志包含错误类型(401 或 Unauthorized) assert any("401" in record.message or "Unauthorized" in record.message for record in caplog.records if record.levelno == logging.WARNING) # ============================================================================ # Test: 解析失败重试 (Plan 14-02 - Task 14-02-01) # ============================================================================ class TestParseFailureRetry: """测试解析失败时的重试逻辑(Plan 14-02)""" def test_json_decode_error_triggers_retry(self, caplog): """测试 JSONDecodeError 触发重试 1 次,仍失败则降级""" import logging # 创建模拟 AI 客户端,始终返回无效 JSON mock_client = Mock() mock_client.api_key = "test-key" invalid_response = "这不是有效的 JSON" mock_client.generate.side_effect = [invalid_response, invalid_response] strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] with caplog.at_level(logging.WARNING): result = strategy.segment(segments, {"title": "测试"}) # 应该降级到规则分段 assert len(result) > 0 assert all(isinstance(seg, VideoSegment) for seg in result) # 验证调用了两次 LLM(初始 + 1 次重试) assert mock_client.generate.call_count == 2 # 验证日志包含解析失败和降级信息 # extract_json_from_llm_response 抛出 ValueError,不是 JSONDecodeError assert any( ("ValueError" in record.message or "JSON" in record.message) and "解析失败" in record.message or "未找到 JSON 内容" in record.message for record in caplog.records ) assert any("降级到规则分段" in record.message for record in caplog.records) def test_json_decode_error_success_on_retry(self, caplog): """测试 JSONDecodeError 重试后成功""" import logging import json mock_client = Mock() mock_client.api_key = "test-key" # 第一次失败,第二次成功 invalid_response = "无效 JSON" valid_response = json.dumps({ "video_type": "知识类", "segments": [ { "title": "测试标题", "content": "测试内容", "start_index": 0, "end_index": 10 } ], "actual_segment_count": 1 }, ensure_ascii=False) mock_client.generate.side_effect = [invalid_response, valid_response] strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] with caplog.at_level(logging.INFO): result = strategy.segment(segments, {"title": "测试"}) # 应该返回 LLM 分段结果 assert len(result) == 1 assert result[0].title == "测试标题" # 验证调用了两次 LLM(初始 + 1 次重试) assert mock_client.generate.call_count == 2 # 验证日志包含重试信息(可能是"重试"或"retrying") assert any( "重试" in record.message or "retrying" in record.message.lower() for record in caplog.records ) def test_validation_error_no_retry(self, caplog): """测试 ValidationError(Pydantic 验证失败)直接降级,不重试""" import logging import json mock_client = Mock() mock_client.api_key = "test-key" # 返回 Pydantic 验证失败的响应(缺少必需字段) invalid_response = json.dumps({ "video_type": "知识类", "segments": [ { "title": "测试", # 缺少 content, start_index, end_index } ], "actual_segment_count": 1 }, ensure_ascii=False) mock_client.generate.return_value = invalid_response strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] with caplog.at_level(logging.WARNING): result = strategy.segment(segments, {"title": "测试"}) # 应该降级到规则分段 assert len(result) > 0 # 验证只调用了 1 次 LLM(不重试) assert mock_client.generate.call_count == 1 # 验证日志包含验证失败信息 assert any( "ValidationError" in record.message or "验证失败" in record.message for record in caplog.records ) def test_parse_error_logs_llm_output(self, caplog): """测试解析失败时记录 LLM 输出片段(前 500 字符)""" import logging mock_client = Mock() mock_client.api_key = "test-key" # 创建一个超过 500 字符的无效响应 long_invalid_response = "这是一个很长的无效响应。" * 100 mock_client.generate.side_effect = [long_invalid_response, long_invalid_response] strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] with caplog.at_level(logging.WARNING): result = strategy.segment(segments, {"title": "测试"}) # 验证日志包含 LLM 输出片段(使用 WARNING 级别) warning_logs = [record for record in caplog.records if record.levelno == logging.WARNING] assert len(warning_logs) > 0 # 验证日志包含 LLM 输出片段信息 assert any( "LLM 输出片段" in record.message for record in warning_logs ) class TestFallbackLogging: """测试降级时的日志记录(Plan 14-02 - Task 14-02-02)""" def test_fallback_logs_error_type_and_message(self, caplog): """测试降级时记录错误类型和消息""" import logging mock_client = Mock() mock_client.api_key = "test-key" invalid_response = "无效 JSON" mock_client.generate.side_effect = [invalid_response, invalid_response] strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] with caplog.at_level(logging.WARNING): result = strategy.segment(segments, {"title": "测试"}) # 验证有 WARNING 级别的日志 warning_logs = [record for record in caplog.records if record.levelno == logging.WARNING] assert len(warning_logs) > 0 # 验证日志包含具体的错误信息 assert any( "降级到规则分段" in record.message for record in warning_logs ) def test_retry_logs_info_level(self, caplog): """测试重试时使用 INFO 级别日志""" import logging import json mock_client = Mock() mock_client.api_key = "test-key" invalid_response = "无效 JSON" valid_response = json.dumps({ "video_type": "知识类", "segments": [ { "title": "测试", "content": "内容", "start_index": 0, "end_index": 10 } ], "actual_segment_count": 1 }, ensure_ascii=False) mock_client.generate.side_effect = [invalid_response, valid_response] strategy = UnifiedSegmentationStrategy(ai_client=mock_client) segments = [Segment(1, 0.0, 1.0, "测试")] with caplog.at_level(logging.INFO): result = strategy.segment(segments, {"title": "测试"}) # 验证有 INFO 级别的重试日志 info_logs = [record for record in caplog.records if record.levelno == logging.INFO] assert len(info_logs) > 0 # 验证日志包含重试信息(可能是"重试"或"retrying") assert any( "重试" in record.message or "retrying" in record.message.lower() for record in info_logs ) class TestExtractJSON: """测试 JSON 提取函数(Plan 14-02)""" def test_extract_pure_json(self): """测试提取纯 JSON""" from kb_workflow.srt.semantic_engine import extract_json_from_llm_response response = '{"key": "value"}' result = extract_json_from_llm_response(response) assert result == {"key": "value"} def test_extract_json_from_markdown_block(self): """测试从 markdown 代码块中提取 JSON""" from kb_workflow.srt.semantic_engine import extract_json_from_llm_response response = '''```json { "key": "value" } ```''' result = extract_json_from_llm_response(response) assert result == {"key": "value"} def test_extract_json_from_text(self): """测试从包含额外文本的响应中提取 JSON""" from kb_workflow.srt.semantic_engine import extract_json_from_llm_response response = '这是一些说明文本。{"key": "value"}还有更多文本。' result = extract_json_from_llm_response(response) assert result == {"key": "value"} def test_extract_json_raises_value_error_on_failure(self): """测试无法提取 JSON 时抛出 ValueError""" from kb_workflow.srt.semantic_engine import extract_json_from_llm_response import pytest response = '这不包含任何 JSON 内容。' with pytest.raises(ValueError, match="未找到 JSON 内容"): extract_json_from_llm_response(response)