diff --git a/kb_pipeline/processors/analyze.py b/kb_pipeline/processors/analyze.py index c71dead..8d46e01 100644 --- a/kb_pipeline/processors/analyze.py +++ b/kb_pipeline/processors/analyze.py @@ -81,6 +81,15 @@ class AnalyzeProcessor(BaseProcessor): metadata = self._load_json(input_data.metadata_file) content = self._load_json(input_data.content_file) + # 处理 Whisper 字幕格式 - 提取完整文本 + if content and 'transcription' in content: + # Whisper JSON 格式,提取文本 + text_parts = [] + for item in content.get('transcription', []): + text_parts.append(item.get('text', '')) + content = {'text': ' '.join(text_parts)} + logger.info(f"Extracted {len(content['text'])} chars from Whisper transcription") + # 对于文章类型,内容可能是纯文本文件 if not content and input_data.content_type in ("article", "answer", "zhuanlan"): content_path = Path(input_data.content_file) diff --git a/kb_pipeline/processors/asr.py b/kb_pipeline/processors/asr.py index cb15951..b94bbaf 100644 --- a/kb_pipeline/processors/asr.py +++ b/kb_pipeline/processors/asr.py @@ -167,6 +167,92 @@ class ASRProcessor: self.state_manager.save(state) raise + def _check_existing_subtitle_files(self, input_data: ASRInput) -> Optional[ASROutput]: + """检查是否已存在字幕文件(幂等性检查) + + Args: + input_data: ASR 输入数据 + + Returns: + 如果找到现有字幕文件返回 ASROutput,否则返回 None + """ + output_dir = Path(input_data.content_dir) + + # 尝试两种命名格式:新格式(带 _subtitle 后缀)和旧格式 + base_names = [ + f"{input_data.platform}_{input_data.content_id}_subtitle", + f"{input_data.platform}_{input_data.content_id}", + ] + + for base_name in base_names: + json_path = output_dir / f"{base_name}.json" + srt_path = output_dir / f"{base_name}.srt" + txt_path = output_dir / f"{base_name}.txt" + + if json_path.exists(): + # 读取现有字幕数据 + segments = [] + text = "" + duration = 0.0 + + try: + with open(json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # 解析字幕格式 + if 'transcription' in data: + # Whisper 格式 + for i, item in enumerate(data['transcription']): + offsets = item.get('offsets', {}) + if isinstance(offsets, dict): + # 新格式:{"from": 0, "to": 5320} + start = offsets.get('from', 0) / 1000.0 + end = offsets.get('to', 0) / 1000.0 + else: + # 旧格式:[0, 5320] + start = offsets[0] / 1000.0 if len(offsets) > 0 else 0 + end = offsets[1] / 1000.0 if len(offsets) > 1 else 0 + segments.append({ + 'id': i, + 'start': start, + 'end': end, + 'text': item.get('text', '') + }) + elif 'body' in data: + # B站官方字幕格式 + for i, item in enumerate(data['body']): + segments.append({ + 'id': i, + 'start': item.get('from', 0), + 'end': item.get('to', 0), + 'text': item.get('content', '') + }) + + text = ' '.join(s['text'] for s in segments) + if segments: + duration = segments[-1]['end'] + + output_files = {'json': str(json_path)} + if srt_path.exists(): + output_files['srt'] = str(srt_path) + if txt_path.exists(): + output_files['txt'] = str(txt_path) + + return ASROutput( + success=True, + tool="existing", + has_subtitle=True, + output_files=output_files, + segments=segments, + text=text, + duration_seconds=duration + ) + except Exception as e: + logger.warning(f"Failed to read existing subtitle file: {e}") + continue + + return None + async def _try_official_subtitle(self, input_data: ASRInput) -> Optional[ASROutput]: """尝试下载 B站官方字幕 @@ -433,10 +519,19 @@ class ASRProcessor: # Whisper.cpp JSON 格式 if 'transcription' in data: for i, item in enumerate(data['transcription']): + offsets = item.get('offsets', {}) + if isinstance(offsets, dict): + # 新格式:{"from": 0, "to": 5320} + start = offsets.get('from', 0) / 1000.0 + end = offsets.get('to', 0) / 1000.0 + else: + # 旧格式:[0, 5320] + start = offsets[0] / 1000.0 if len(offsets) > 0 else 0 + end = offsets[1] / 1000.0 if len(offsets) > 1 else 0 segments.append({ 'id': i, - 'start': item.get('offsets', [0, 0])[0] / 1000.0, - 'end': item.get('offsets', [0, 0])[1] / 1000.0, + 'start': start, + 'end': end, 'text': item.get('text', '') }) text = ' '.join(s['text'] for s in segments)