diff --git a/.env.example b/.env.example index 21fda8a..033b571 100644 --- a/.env.example +++ b/.env.example @@ -50,6 +50,8 @@ # vLLM 实时/离线解码微批处理:最多等待 10ms 收集请求,单批最多 8 个音频输入。 # QWEN_VLLM_DECODE_BATCH_WAIT_MS=10 # QWEN_VLLM_DECODE_MAX_BATCH_SIZE=8 +# 实时 partial 仅重复解码最近 8 秒窗口;定稿仍使用整段音频。 +# REALTIME_STREAM_WINDOW_SEC=8.0 # ASR_ENABLE_WORD_TIMESTAMPS=false # ----------------------------------------------------------------------------- diff --git a/app/core/config.py b/app/core/config.py index 61ef705..130bf1f 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -108,10 +108,11 @@ class Settings: REALTIME_MIN_PARTIAL_SEC: float = 0.45 REALTIME_PARTIAL_EMIT_INTERVAL_SEC: float = 0.25 REALTIME_PARTIAL_WINDOW_SEC: float = 8.0 - # Match FunASR's roughly 600 ms online decode cadence; Qwen re-decodes the - # accumulated stream, so this stays configurable for server-side tuning. + # Match FunASR's roughly 600 ms online decode cadence. Qwen partial only + # re-decodes the recent rolling window; final transcription still uses full audio. REALTIME_STREAM_CHUNK_SEC: float = 0.6 REALTIME_STREAM_MAX_PENDING_CHUNKS: int = 3 + # 实时 partial 重解码窗口长度;完整定稿音频保存在 WebSocket turn buffer 中。 REALTIME_STREAM_WINDOW_SEC: float = 8.0 REALTIME_STREAM_STABLE_TAIL_CHARS: int = 8 REALTIME_STREAM_STABLE_MIN_GROW_CHARS: int = 2 diff --git a/app/services/asr/qwen3_vllm.py b/app/services/asr/qwen3_vllm.py index a854ee4..fe18103 100644 --- a/app/services/asr/qwen3_vllm.py +++ b/app/services/asr/qwen3_vllm.py @@ -12,6 +12,7 @@ import re import threading import time from dataclasses import dataclass, field +from difflib import SequenceMatcher from typing import Any, Optional import librosa @@ -173,6 +174,7 @@ class _GenerateRequest: prompts: list[dict[str, Any]] sampling_params: Any batch_key: tuple[Any, ...] + queued_at: float = field(default_factory=time.monotonic) done: threading.Event = field(default_factory=threading.Event) outputs: Optional[list[Any]] = None error: Optional[Exception] = None @@ -258,8 +260,11 @@ class _VLLMGenerateBatcher: def _generate_group(self, requests: list[_GenerateRequest]) -> None: prompts = [prompt for request in requests for prompt in request.prompts] + queued_at = min(request.queued_at for request in requests) + lock_wait_started = time.monotonic() try: with self._engine_lock: + generate_started = time.monotonic() outputs = self._llm.generate( prompts, sampling_params=requests[0].sampling_params, @@ -269,6 +274,19 @@ class _VLLMGenerateBatcher: raise RuntimeError( f"vLLM returned {len(outputs)} outputs for {len(prompts)} prompts" ) + generate_ms = (time.monotonic() - generate_started) * 1000.0 + lock_wait_ms = (generate_started - lock_wait_started) * 1000.0 + if len(requests) > 1 or generate_ms >= 500.0 or lock_wait_ms >= 100.0: + logger.info( + "vLLM decode batch completed: requests=%d prompts=%d max_tokens=%s " + "queue_wait_ms=%.1f engine_lock_wait_ms=%.1f generate_ms=%.1f", + len(requests), + len(prompts), + getattr(requests[0].sampling_params, "max_tokens", None), + (lock_wait_started - queued_at) * 1000.0, + lock_wait_ms, + generate_ms, + ) cursor = 0 for request in requests: request.outputs = list(outputs[cursor:cursor + len(request.prompts)]) @@ -311,13 +329,71 @@ class VLLMRealtimeState: unfixed_chunk_num: int unfixed_token_num: int max_new_tokens: int + max_audio_window_sec: float = 8.0 chunk_id: int = 0 text: str = "" raw_decoded: str = "" + window_text: str = "" + # 记录滚动窗口是否已经丢弃旧音频,供文本快照重叠拼接使用。 + audio_window_start_samples: int = 0 + window_rolled: bool = False audio_buffer: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) audio_accum: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) +def _transcript_char_views(text: str) -> list[tuple[str, int]]: + """Return normalized speech characters with their original string offsets.""" + return [ + (char.lower(), index) + for index, char in enumerate(text) + if char.isalnum() or "\u4e00" <= char <= "\u9fff" + ] + + +def _merge_rolling_window_text( + assembled_text: str, + previous_window_text: str, + current_window_text: str, +) -> tuple[str, bool]: + """Append only the new tail after matching adjacent partial-window overlap.""" + if not assembled_text: + return current_window_text.strip(), True + previous_views = _transcript_char_views(previous_window_text) + current_views = _transcript_char_views(current_window_text) + if not previous_views or not current_views: + return assembled_text, False + + match = SequenceMatcher( + None, + [char for char, _ in previous_views], + [char for char, _ in current_views], + autojunk=False, + ).find_longest_match(0, len(previous_views), 0, len(current_views)) + min_overlap = min(8, len(previous_views), len(current_views)) + max_edge_gap = max(12, int(min(len(previous_views), len(current_views)) * 0.25)) + previous_tail_gap = len(previous_views) - (match.a + match.size) + if ( + match.size < min_overlap + or previous_tail_gap > max_edge_gap + or match.b > max_edge_gap + ): + return assembled_text, False + + overlap_end = match.b + match.size + delta_start = ( + current_views[overlap_end][1] + if overlap_end < len(current_views) + else len(current_window_text) + ) + delta = current_window_text[delta_start:].lstrip() + if not delta: + return assembled_text, True + + # Chinese text joins directly; add a separator only between adjacent Latin words. + separator = " " if assembled_text[-1].isalnum() and delta[0].isalnum() else "" + return f"{assembled_text.rstrip()}{separator}{delta}", True + + class Qwen3VLLMBackend: """Thin adapter over official vLLM APIs for Qwen3-ASR.""" @@ -366,7 +442,8 @@ class Qwen3VLLMBackend: temperature=0.01, max_tokens=max_new_tokens, ) - self._max_inference_batch_size = max_inference_batch_size + self._max_inference_batch_size = max(1, max_inference_batch_size) + self._decode_max_batch_size = max(1, decode_max_batch_size) self._gpu_memory_utilization = gpu_memory_utilization self._enforce_eager = enforce_eager self._tensor_parallel_size = tensor_parallel_size @@ -380,12 +457,12 @@ class Qwen3VLLMBackend: self._llm, self._engine_lock, batch_wait_ms=decode_batch_wait_ms, - max_batch_size=max(decode_max_batch_size, max_inference_batch_size), + max_batch_size=self._decode_max_batch_size, ) logger.info( "Enabled vLLM decode micro-batching: wait_ms=%s, max_batch_size=%s", max(0, decode_batch_wait_ms), - max(decode_max_batch_size, max_inference_batch_size), + self._decode_max_batch_size, ) def _get_forced_aligner_gpu_memory_utilization(self) -> float: @@ -445,11 +522,17 @@ class Qwen3VLLMBackend: } ) - outputs = self._generate_batcher.generate( - prompts, - sampling_params=self._sampling_params, - batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature), - ) + outputs: list[Any] = [] + # Keep one offline request within the same hard batch cap used for + # combining concurrent realtime sessions. + for start in range(0, len(prompts), self._decode_max_batch_size): + outputs.extend( + self._generate_batcher.generate( + prompts[start:start + self._decode_max_batch_size], + sampling_params=self._sampling_params, + batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature), + ) + ) transcripts: list[_GeneratedTranscript] = [] for output, (_audio, _context, language) in zip(outputs, audio_items): @@ -612,6 +695,7 @@ class Qwen3VLLMBackend: unfixed_chunk_num: int = 2, unfixed_token_num: int = 5, max_new_tokens: int = 32, + max_audio_window_sec: float = 8.0, ) -> VLLMRealtimeState: normalized_language = _normalize_language_name(language) or "" return VLLMRealtimeState( @@ -621,13 +705,16 @@ class Qwen3VLLMBackend: unfixed_chunk_num=unfixed_chunk_num, unfixed_token_num=unfixed_token_num, max_new_tokens=max_new_tokens, + max_audio_window_sec=max(1.0, max_audio_window_sec), audio_buffer=np.array([], dtype=np.float32), audio_accum=np.array([], dtype=np.float32), ) def _decode_stream(self, state: VLLMRealtimeState) -> VLLMRealtimeState: prefix = "" - if state.chunk_id >= state.unfixed_chunk_num and state.raw_decoded: + # After the audio window rolls, decode a fresh window snapshot and stitch + # its text overlap; old token prefixes would pin the model to stale audio. + if not state.window_rolled and state.chunk_id >= state.unfixed_chunk_num and state.raw_decoded: token_ids = self._tokenizer.encode(state.raw_decoded, add_special_tokens=False) rollback = token_ids[-state.unfixed_token_num:] if state.unfixed_token_num > 0 else [] if rollback: @@ -650,7 +737,19 @@ class Qwen3VLLMBackend: generated = str(output.outputs[0].text if output.outputs else "") parsed_language, parsed_text = _parse_asr_output(prefix + generated, state.language or None) state.raw_decoded = prefix + generated - state.text = parsed_text + if state.window_rolled and parsed_text: + state.text, overlap_matched = _merge_rolling_window_text( + state.text, + state.window_text, + parsed_text, + ) + # Keep the older anchor when a model revision breaks overlap matching; + # the next window still has another chance to find the shared span. + if overlap_matched: + state.window_text = parsed_text + elif not state.window_rolled: + state.text = parsed_text + state.window_text = parsed_text state.language = parsed_language or state.language state.chunk_id += 1 return state @@ -662,6 +761,13 @@ class Qwen3VLLMBackend: segment = state.audio_buffer[:segment_size].copy() state.audio_buffer = state.audio_buffer[segment_size:] state.audio_accum = np.concatenate([state.audio_accum, segment]) + max_window_samples = int(state.max_audio_window_sec * _DEFAULT_SAMPLE_RATE) + if len(state.audio_accum) > max_window_samples: + samples_to_drop = len(state.audio_accum) - max_window_samples + # Partial 只保留最近窗口,完整音频仍由 WebSocket 侧保留用于定稿。 + state.audio_accum = state.audio_accum[samples_to_drop:] + state.audio_window_start_samples += samples_to_drop + state.window_rolled = True state = self._decode_stream(state) return state @@ -669,6 +775,13 @@ class Qwen3VLLMBackend: if len(state.audio_buffer) > 0: state.audio_accum = np.concatenate([state.audio_accum, state.audio_buffer]) state.audio_buffer = np.array([], dtype=np.float32) + max_window_samples = int(state.max_audio_window_sec * _DEFAULT_SAMPLE_RATE) + if len(state.audio_accum) > max_window_samples: + samples_to_drop = len(state.audio_accum) - max_window_samples + # 收尾 partial 仍按窗口处理;最终整段转写使用外层保留的完整音频。 + state.audio_accum = state.audio_accum[samples_to_drop:] + state.audio_window_start_samples += samples_to_drop + state.window_rolled = True state = self._decode_stream(state) elif state.chunk_id == 0 and len(state.audio_accum) > 0: state = self._decode_stream(state) diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index 947124d..c73551c 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -1414,6 +1414,10 @@ class Qwen3ASRService: unfixed_chunk_num=self._stream_unfixed_chunk_num(ctx), unfixed_token_num=self._stream_unfixed_token_num(ctx), max_new_tokens=48, + max_audio_window_sec=max( + float(getattr(settings, "REALTIME_STREAM_WINDOW_SEC", 8.0) or 8.0), + 1.0, + ), ) except Exception as exc: logger.debug("Init realtime partial stream failed: %s", exc)