kb-skills/tests/test_semantic_robustness.py

"""测试语义分段的鲁棒性功能 (Phase 14)

测试 LLM 调用失败时的重试和降级策略。

Plan 14-01: 实现可恢复错误检测和重试逻辑
"""

import json
import pytest
from unittest.mock import Mock
from requests.exceptions import ConnectionError, Timeout, HTTPError

from kb_workflow.srt.parser import Segment
from kb_workflow.srt.semantic_engine import (
    UnifiedSegmentationStrategy,
    VideoSegment
)


class TestRecoverableErrorDetection:
    """测试可恢复错误检测功能"""

    def test_is_recoverable_error_connection_error(self):
        """Test: ConnectionError 被识别为可恢复错误"""
        strategy = UnifiedSegmentationStrategy()

        # 创建模拟客户端，抛出 ConnectionError
        mock_client = Mock()
        mock_client.api_key = "test_key"
        mock_client.generate.side_effect = ConnectionError("Connection refused")

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        # 准备测试数据
        segments = [
            Segment(1, 0.0, 1.0, "第一句"),
            Segment(2, 1.0, 2.0, "第二句")
        ]

        # 应该降级到规则分段（ConnectionError 可恢复，重试 1 次后降级）
        result = strategy.segment(segments, {"title": "测试"})

        # 验证降级结果
        assert len(result) > 0
        # 验证调用了 2 次（初始 + 1 次重试）
        assert mock_client.generate.call_count == 2

    def test_is_recoverable_error_timeout(self):
        """Test: Timeout 被识别为可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"
        mock_client.generate.side_effect = Timeout("Request timeout")

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 应该重试 1 次
        assert mock_client.generate.call_count == 2
        assert len(result) > 0

    def test_is_recoverable_error_rate_limit_429(self):
        """Test: HTTP 429 (限流) 被识别为可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        # 创建 429 响应
        response = Mock()
        response.status_code = 429
        response.text = "Rate limit exceeded"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 应该重试 1 次
        assert mock_client.generate.call_count == 2
        assert len(result) > 0

    def test_is_recoverable_error_server_error_500(self):
        """Test: HTTP 500 (服务器错误) 被识别为可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 500
        response.text = "Internal server error"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 应该重试 1 次
        assert mock_client.generate.call_count == 2
        assert len(result) > 0

    def test_is_recoverable_error_server_error_502(self):
        """Test: HTTP 502 (网关错误) 被识别为可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 502
        response.text = "Bad gateway"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 应该重试 1 次
        assert mock_client.generate.call_count == 2
        assert len(result) > 0

    def test_is_recoverable_error_server_error_503(self):
        """Test: HTTP 503 (服务不可用) 被识别为可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 503
        response.text = "Service unavailable"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 应该重试 1 次
        assert mock_client.generate.call_count == 2
        assert len(result) > 0

    def test_is_recoverable_error_server_error_504(self):
        """Test: HTTP 504 (网关超时) 被识别为可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 504
        response.text = "Gateway timeout"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 应该重试 1 次
        assert mock_client.generate.call_count == 2
        assert len(result) > 0


class TestNonRecoverableErrorDetection:
    """测试不可恢复错误检测功能"""

    def test_non_recoverable_error_400(self):
        """Test: HTTP 400 (错误请求) 被识别为不可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 400
        response.text = "Bad request"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 不应该重试，直接降级
        assert mock_client.generate.call_count == 1
        assert len(result) > 0

    def test_non_recoverable_error_401(self):
        """Test: HTTP 401 (未授权) 被识别为不可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 401
        response.text = "Unauthorized"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 不应该重试
        assert mock_client.generate.call_count == 1
        assert len(result) > 0

    def test_non_recoverable_error_403(self):
        """Test: HTTP 403 (禁止访问) 被识别为不可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 403
        response.text = "Forbidden"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 不应该重试
        assert mock_client.generate.call_count == 1
        assert len(result) > 0

    def test_non_recoverable_error_404(self):
        """Test: HTTP 404 (未找到) 被识别为不可恢复错误"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 404
        response.text = "Not found"
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 不应该重试
        assert mock_client.generate.call_count == 1
        assert len(result) > 0


class TestRetryMechanism:
    """测试重试机制"""

    def test_retry_once_on_recoverable_error(self):
        """Test: 可恢复错误重试 1 次"""
        mock_client = Mock()
        mock_client.api_key = "test_key"
        mock_client.generate.side_effect = ConnectionError("Network error")

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 验证调用次数：初始 1 次 + 重试 1 次 = 2 次
        assert mock_client.generate.call_count == 2

    def test_no_retry_on_non_recoverable_error(self):
        """Test: 不可恢复错误不重试"""
        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 401
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]
        result = strategy.segment(segments, {"title": "测试"})

        # 验证只调用 1 次（不重试）
        assert mock_client.generate.call_count == 1

    def test_immediate_retry_no_delay(self):
        """Test: 立即重试，无延迟"""
        import time

        mock_client = Mock()
        mock_client.api_key = "test_key"
        mock_client.generate.side_effect = ConnectionError("Network error")

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        start_time = time.time()
        result = strategy.segment(segments, {"title": "测试"})
        elapsed_time = time.time() - start_time

        # 验证重试是立即的（无延迟）
        # 正常重试应该有延迟，立即重试应该很快（< 0.1 秒）
        assert elapsed_time < 0.1


class TestFallbackToRuleBased:
    """测试降级到规则分段"""

    def test_fallback_on_llm_failure(self):
        """Test: LLM 调用失败时降级到规则分段"""
        mock_client = Mock()
        mock_client.api_key = "test_key"
        mock_client.generate.side_effect = Exception("LLM error")

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [
            Segment(1, 0.0, 1.0, "第一句"),
            Segment(2, 1.0, 2.0, "第二句")
        ]
        result = strategy.segment(segments, {"title": "测试"})

        # 验证返回了规则分段结果
        assert len(result) > 0
        # 验证返回的是 VideoSegment 对象（规则分段转换后的格式）
        assert all(isinstance(seg, VideoSegment) for seg in result)
        # 验证规则分段没有标题（降级分段标题为 null）
        assert all(seg.title is None for seg in result)

    def test_fallback_when_ai_client_unavailable(self):
        """Test: AI 客户端不可用时直接使用规则分段"""
        # 创建没有 AI 客户端的策略
        strategy = UnifiedSegmentationStrategy(ai_client=None)

        segments = [
            Segment(1, 0.0, 1.0, "第一句"),
            Segment(2, 1.0, 2.0, "第二句")
        ]
        result = strategy.segment(segments, {"title": "测试"})

        # 验证返回了规则分段结果
        assert len(result) > 0
        assert all(isinstance(seg, VideoSegment) for seg in result)
        assert all(seg.title is None for seg in result)


class TestLoggingBehavior:
    """测试日志记录行为"""

    def test_retry_logging(self, caplog):
        """Test: 重试时记录 info 级别日志"""
        import logging

        mock_client = Mock()
        mock_client.api_key = "test_key"
        mock_client.generate.side_effect = ConnectionError("Network error")

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        with caplog.at_level(logging.INFO):
            result = strategy.segment(segments, {"title": "测试"})

        # 验证日志包含重试信息
        assert any("重试" in record.message or "retry" in record.message.lower()
                   for record in caplog.records if record.levelno == logging.INFO)

    def test_fallback_warning_logging(self, caplog):
        """Test: 降级时记录 warning 级别日志"""
        import logging

        mock_client = Mock()
        mock_client.api_key = "test_key"

        response = Mock()
        response.status_code = 401
        http_error = HTTPError(response=response)
        mock_client.generate.side_effect = http_error

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        with caplog.at_level(logging.WARNING):
            result = strategy.segment(segments, {"title": "测试"})

        # 验证日志包含降级信息和错误类型
        assert any("降级" in record.message or "fallback" in record.message.lower()
                   for record in caplog.records if record.levelno == logging.WARNING)
        # 验证日志包含错误类型（401 或 Unauthorized）
        assert any("401" in record.message or "Unauthorized" in record.message
                   for record in caplog.records if record.levelno == logging.WARNING)


# ============================================================================
# Test: 解析失败重试 (Plan 14-02 - Task 14-02-01)
# ============================================================================

class TestParseFailureRetry:
    """测试解析失败时的重试逻辑（Plan 14-02）"""

    def test_json_decode_error_triggers_retry(self, caplog):
        """测试 JSONDecodeError 触发重试 1 次，仍失败则降级"""
        import logging

        # 创建模拟 AI 客户端，始终返回无效 JSON
        mock_client = Mock()
        mock_client.api_key = "test-key"
        invalid_response = "这不是有效的 JSON"
        mock_client.generate.side_effect = [invalid_response, invalid_response]

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        with caplog.at_level(logging.WARNING):
            result = strategy.segment(segments, {"title": "测试"})

        # 应该降级到规则分段
        assert len(result) > 0
        assert all(isinstance(seg, VideoSegment) for seg in result)

        # 验证调用了两次 LLM（初始 + 1 次重试）
        assert mock_client.generate.call_count == 2

        # 验证日志包含解析失败和降级信息
        # extract_json_from_llm_response 抛出 ValueError，不是 JSONDecodeError
        assert any(
            ("ValueError" in record.message or "JSON" in record.message) and
            "解析失败" in record.message or "未找到 JSON 内容" in record.message
            for record in caplog.records
        )
        assert any("降级到规则分段" in record.message for record in caplog.records)

    def test_json_decode_error_success_on_retry(self, caplog):
        """测试 JSONDecodeError 重试后成功"""
        import logging
        import json

        mock_client = Mock()
        mock_client.api_key = "test-key"

        # 第一次失败，第二次成功
        invalid_response = "无效 JSON"
        valid_response = json.dumps({
            "video_type": "知识类",
            "segments": [
                {
                    "title": "测试标题",
                    "content": "测试内容",
                    "start_index": 0,
                    "end_index": 10
                }
            ],
            "actual_segment_count": 1
        }, ensure_ascii=False)

        mock_client.generate.side_effect = [invalid_response, valid_response]

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        with caplog.at_level(logging.INFO):
            result = strategy.segment(segments, {"title": "测试"})

        # 应该返回 LLM 分段结果
        assert len(result) == 1
        assert result[0].title == "测试标题"

        # 验证调用了两次 LLM（初始 + 1 次重试）
        assert mock_client.generate.call_count == 2

        # 验证日志包含重试信息
        assert any("重试" in record.message for record in caplog.records)

    def test_validation_error_no_retry(self, caplog):
        """测试 ValidationError（Pydantic 验证失败）直接降级，不重试"""
        import logging
        import json

        mock_client = Mock()
        mock_client.api_key = "test-key"

        # 返回 Pydantic 验证失败的响应（缺少必需字段）
        invalid_response = json.dumps({
            "video_type": "知识类",
            "segments": [
                {
                    "title": "测试",
                    # 缺少 content, start_index, end_index
                }
            ],
            "actual_segment_count": 1
        }, ensure_ascii=False)

        mock_client.generate.return_value = invalid_response

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        with caplog.at_level(logging.WARNING):
            result = strategy.segment(segments, {"title": "测试"})

        # 应该降级到规则分段
        assert len(result) > 0

        # 验证只调用了 1 次 LLM（不重试）
        assert mock_client.generate.call_count == 1

        # 验证日志包含验证失败信息
        assert any("Pydantic 验证失败" in record.message or "验证失败" in record.message
                   for record in caplog.records)

    def test_parse_error_logs_llm_output(self, caplog):
        """测试解析失败时记录 LLM 输出片段（前 500 字符）"""
        import logging

        mock_client = Mock()
        mock_client.api_key = "test-key"

        # 创建一个超过 500 字符的无效响应
        long_invalid_response = "这是一个很长的无效响应。" * 100
        mock_client.generate.side_effect = [long_invalid_response, long_invalid_response]

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        with caplog.at_level(logging.ERROR):
            result = strategy.segment(segments, {"title": "测试"})

        # 验证日志包含 LLM 输出片段
        error_logs = [record for record in caplog.records if record.levelno == logging.ERROR]
        assert len(error_logs) > 0

        # 验证日志包含 LLM 输出片段信息
        assert any(
            "LLM 输出片段" in record.message or "原始 LLM 输出" in record.message
            for record in error_logs
        )


class TestFallbackLogging:
    """测试降级时的日志记录（Plan 14-02 - Task 14-02-02）"""

    def test_fallback_logs_error_type_and_message(self, caplog):
        """测试降级时记录错误类型和消息"""
        import logging

        mock_client = Mock()
        mock_client.api_key = "test-key"
        invalid_response = "无效 JSON"
        mock_client.generate.side_effect = [invalid_response, invalid_response]

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        with caplog.at_level(logging.WARNING):
            result = strategy.segment(segments, {"title": "测试"})

        # 验证有 WARNING 级别的日志
        warning_logs = [record for record in caplog.records if record.levelno == logging.WARNING]
        assert len(warning_logs) > 0

        # 验证日志包含具体的错误信息
        assert any(
            "降级到规则分段" in record.message
            for record in warning_logs
        )

    def test_retry_logs_info_level(self, caplog):
        """测试重试时使用 INFO 级别日志"""
        import logging
        import json

        mock_client = Mock()
        mock_client.api_key = "test-key"

        invalid_response = "无效 JSON"
        valid_response = json.dumps({
            "video_type": "知识类",
            "segments": [
                {
                    "title": "测试",
                    "content": "内容",
                    "start_index": 0,
                    "end_index": 10
                }
            ],
            "actual_segment_count": 1
        }, ensure_ascii=False)

        mock_client.generate.side_effect = [invalid_response, valid_response]

        strategy = UnifiedSegmentationStrategy(ai_client=mock_client)

        segments = [Segment(1, 0.0, 1.0, "测试")]

        with caplog.at_level(logging.INFO):
            result = strategy.segment(segments, {"title": "测试"})

        # 验证有 INFO 级别的重试日志
        info_logs = [record for record in caplog.records if record.levelno == logging.INFO]
        assert len(info_logs) > 0

        # 验证日志包含重试信息（可能是"重试"或"retrying"）
        assert any(
            "重试" in record.message or "retrying" in record.message.lower()
            for record in info_logs
        )


class TestExtractJSON:
    """测试 JSON 提取函数（Plan 14-02）"""

    def test_extract_pure_json(self):
        """测试提取纯 JSON"""
        from kb_workflow.srt.semantic_engine import extract_json_from_llm_response

        response = '{"key": "value"}'
        result = extract_json_from_llm_response(response)
        assert result == {"key": "value"}

    def test_extract_json_from_markdown_block(self):
        """测试从 markdown 代码块中提取 JSON"""
        from kb_workflow.srt.semantic_engine import extract_json_from_llm_response

        response = '''```json
{
  "key": "value"
}
```'''
        result = extract_json_from_llm_response(response)
        assert result == {"key": "value"}

    def test_extract_json_from_text(self):
        """测试从包含额外文本的响应中提取 JSON"""
        from kb_workflow.srt.semantic_engine import extract_json_from_llm_response

        response = '这是一些说明文本。{"key": "value"}还有更多文本。'
        result = extract_json_from_llm_response(response)
        assert result == {"key": "value"}

    def test_extract_json_raises_value_error_on_failure(self):
        """测试无法提取 JSON 时抛出 ValueError"""
        from kb_workflow.srt.semantic_engine import extract_json_from_llm_response
        import pytest

        response = '这不包含任何 JSON 内容。'
        with pytest.raises(ValueError, match="未找到 JSON 内容"):
            extract_json_from_llm_response(response)