调整实时partial超窗口重解压逻辑,减少长句持续增长时的重复推理

main
Bifang 2026-10-09 13:54:47 +08:00
parent f1074ce878
commit 5842c25251
4 changed files with 132 additions and 12 deletions

View File

@ -50,6 +50,8 @@
# vLLM 实时/离线解码微批处理:最多等待 10ms 收集请求,单批最多 8 个音频输入。
# QWEN_VLLM_DECODE_BATCH_WAIT_MS=10
# QWEN_VLLM_DECODE_MAX_BATCH_SIZE=8
# 实时 partial 仅重复解码最近 8 秒窗口;定稿仍使用整段音频。
# REALTIME_STREAM_WINDOW_SEC=8.0
# ASR_ENABLE_WORD_TIMESTAMPS=false
# -----------------------------------------------------------------------------

View File

@ -108,10 +108,11 @@ class Settings:
REALTIME_MIN_PARTIAL_SEC: float = 0.45
REALTIME_PARTIAL_EMIT_INTERVAL_SEC: float = 0.25
REALTIME_PARTIAL_WINDOW_SEC: float = 8.0
# Match FunASR's roughly 600 ms online decode cadence; Qwen re-decodes the
# accumulated stream, so this stays configurable for server-side tuning.
# Match FunASR's roughly 600 ms online decode cadence. Qwen partial only
# re-decodes the recent rolling window; final transcription still uses full audio.
REALTIME_STREAM_CHUNK_SEC: float = 0.6
REALTIME_STREAM_MAX_PENDING_CHUNKS: int = 3
# 实时 partial 重解码窗口长度;完整定稿音频保存在 WebSocket turn buffer 中。
REALTIME_STREAM_WINDOW_SEC: float = 8.0
REALTIME_STREAM_STABLE_TAIL_CHARS: int = 8
REALTIME_STREAM_STABLE_MIN_GROW_CHARS: int = 2

View File

@ -12,6 +12,7 @@ import re
import threading
import time
from dataclasses import dataclass, field
from difflib import SequenceMatcher
from typing import Any, Optional
import librosa
@ -173,6 +174,7 @@ class _GenerateRequest:
prompts: list[dict[str, Any]]
sampling_params: Any
batch_key: tuple[Any, ...]
queued_at: float = field(default_factory=time.monotonic)
done: threading.Event = field(default_factory=threading.Event)
outputs: Optional[list[Any]] = None
error: Optional[Exception] = None
@ -258,8 +260,11 @@ class _VLLMGenerateBatcher:
def _generate_group(self, requests: list[_GenerateRequest]) -> None:
prompts = [prompt for request in requests for prompt in request.prompts]
queued_at = min(request.queued_at for request in requests)
lock_wait_started = time.monotonic()
try:
with self._engine_lock:
generate_started = time.monotonic()
outputs = self._llm.generate(
prompts,
sampling_params=requests[0].sampling_params,
@ -269,6 +274,19 @@ class _VLLMGenerateBatcher:
raise RuntimeError(
f"vLLM returned {len(outputs)} outputs for {len(prompts)} prompts"
)
generate_ms = (time.monotonic() - generate_started) * 1000.0
lock_wait_ms = (generate_started - lock_wait_started) * 1000.0
if len(requests) > 1 or generate_ms >= 500.0 or lock_wait_ms >= 100.0:
logger.info(
"vLLM decode batch completed: requests=%d prompts=%d max_tokens=%s "
"queue_wait_ms=%.1f engine_lock_wait_ms=%.1f generate_ms=%.1f",
len(requests),
len(prompts),
getattr(requests[0].sampling_params, "max_tokens", None),
(lock_wait_started - queued_at) * 1000.0,
lock_wait_ms,
generate_ms,
)
cursor = 0
for request in requests:
request.outputs = list(outputs[cursor:cursor + len(request.prompts)])
@ -311,13 +329,71 @@ class VLLMRealtimeState:
unfixed_chunk_num: int
unfixed_token_num: int
max_new_tokens: int
max_audio_window_sec: float = 8.0
chunk_id: int = 0
text: str = ""
raw_decoded: str = ""
window_text: str = ""
# 记录滚动窗口是否已经丢弃旧音频,供文本快照重叠拼接使用。
audio_window_start_samples: int = 0
window_rolled: bool = False
audio_buffer: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
audio_accum: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
def _transcript_char_views(text: str) -> list[tuple[str, int]]:
"""Return normalized speech characters with their original string offsets."""
return [
(char.lower(), index)
for index, char in enumerate(text)
if char.isalnum() or "\u4e00" <= char <= "\u9fff"
]
def _merge_rolling_window_text(
assembled_text: str,
previous_window_text: str,
current_window_text: str,
) -> tuple[str, bool]:
"""Append only the new tail after matching adjacent partial-window overlap."""
if not assembled_text:
return current_window_text.strip(), True
previous_views = _transcript_char_views(previous_window_text)
current_views = _transcript_char_views(current_window_text)
if not previous_views or not current_views:
return assembled_text, False
match = SequenceMatcher(
None,
[char for char, _ in previous_views],
[char for char, _ in current_views],
autojunk=False,
).find_longest_match(0, len(previous_views), 0, len(current_views))
min_overlap = min(8, len(previous_views), len(current_views))
max_edge_gap = max(12, int(min(len(previous_views), len(current_views)) * 0.25))
previous_tail_gap = len(previous_views) - (match.a + match.size)
if (
match.size < min_overlap
or previous_tail_gap > max_edge_gap
or match.b > max_edge_gap
):
return assembled_text, False
overlap_end = match.b + match.size
delta_start = (
current_views[overlap_end][1]
if overlap_end < len(current_views)
else len(current_window_text)
)
delta = current_window_text[delta_start:].lstrip()
if not delta:
return assembled_text, True
# Chinese text joins directly; add a separator only between adjacent Latin words.
separator = " " if assembled_text[-1].isalnum() and delta[0].isalnum() else ""
return f"{assembled_text.rstrip()}{separator}{delta}", True
class Qwen3VLLMBackend:
"""Thin adapter over official vLLM APIs for Qwen3-ASR."""
@ -366,7 +442,8 @@ class Qwen3VLLMBackend:
temperature=0.01,
max_tokens=max_new_tokens,
)
self._max_inference_batch_size = max_inference_batch_size
self._max_inference_batch_size = max(1, max_inference_batch_size)
self._decode_max_batch_size = max(1, decode_max_batch_size)
self._gpu_memory_utilization = gpu_memory_utilization
self._enforce_eager = enforce_eager
self._tensor_parallel_size = tensor_parallel_size
@ -380,12 +457,12 @@ class Qwen3VLLMBackend:
self._llm,
self._engine_lock,
batch_wait_ms=decode_batch_wait_ms,
max_batch_size=max(decode_max_batch_size, max_inference_batch_size),
max_batch_size=self._decode_max_batch_size,
)
logger.info(
"Enabled vLLM decode micro-batching: wait_ms=%s, max_batch_size=%s",
max(0, decode_batch_wait_ms),
max(decode_max_batch_size, max_inference_batch_size),
self._decode_max_batch_size,
)
def _get_forced_aligner_gpu_memory_utilization(self) -> float:
@ -445,11 +522,17 @@ class Qwen3VLLMBackend:
}
)
outputs = self._generate_batcher.generate(
prompts,
sampling_params=self._sampling_params,
batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature),
)
outputs: list[Any] = []
# Keep one offline request within the same hard batch cap used for
# combining concurrent realtime sessions.
for start in range(0, len(prompts), self._decode_max_batch_size):
outputs.extend(
self._generate_batcher.generate(
prompts[start:start + self._decode_max_batch_size],
sampling_params=self._sampling_params,
batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature),
)
)
transcripts: list[_GeneratedTranscript] = []
for output, (_audio, _context, language) in zip(outputs, audio_items):
@ -612,6 +695,7 @@ class Qwen3VLLMBackend:
unfixed_chunk_num: int = 2,
unfixed_token_num: int = 5,
max_new_tokens: int = 32,
max_audio_window_sec: float = 8.0,
) -> VLLMRealtimeState:
normalized_language = _normalize_language_name(language) or ""
return VLLMRealtimeState(
@ -621,13 +705,16 @@ class Qwen3VLLMBackend:
unfixed_chunk_num=unfixed_chunk_num,
unfixed_token_num=unfixed_token_num,
max_new_tokens=max_new_tokens,
max_audio_window_sec=max(1.0, max_audio_window_sec),
audio_buffer=np.array([], dtype=np.float32),
audio_accum=np.array([], dtype=np.float32),
)
def _decode_stream(self, state: VLLMRealtimeState) -> VLLMRealtimeState:
prefix = ""
if state.chunk_id >= state.unfixed_chunk_num and state.raw_decoded:
# After the audio window rolls, decode a fresh window snapshot and stitch
# its text overlap; old token prefixes would pin the model to stale audio.
if not state.window_rolled and state.chunk_id >= state.unfixed_chunk_num and state.raw_decoded:
token_ids = self._tokenizer.encode(state.raw_decoded, add_special_tokens=False)
rollback = token_ids[-state.unfixed_token_num:] if state.unfixed_token_num > 0 else []
if rollback:
@ -650,7 +737,19 @@ class Qwen3VLLMBackend:
generated = str(output.outputs[0].text if output.outputs else "")
parsed_language, parsed_text = _parse_asr_output(prefix + generated, state.language or None)
state.raw_decoded = prefix + generated
state.text = parsed_text
if state.window_rolled and parsed_text:
state.text, overlap_matched = _merge_rolling_window_text(
state.text,
state.window_text,
parsed_text,
)
# Keep the older anchor when a model revision breaks overlap matching;
# the next window still has another chance to find the shared span.
if overlap_matched:
state.window_text = parsed_text
elif not state.window_rolled:
state.text = parsed_text
state.window_text = parsed_text
state.language = parsed_language or state.language
state.chunk_id += 1
return state
@ -662,6 +761,13 @@ class Qwen3VLLMBackend:
segment = state.audio_buffer[:segment_size].copy()
state.audio_buffer = state.audio_buffer[segment_size:]
state.audio_accum = np.concatenate([state.audio_accum, segment])
max_window_samples = int(state.max_audio_window_sec * _DEFAULT_SAMPLE_RATE)
if len(state.audio_accum) > max_window_samples:
samples_to_drop = len(state.audio_accum) - max_window_samples
# Partial 只保留最近窗口,完整音频仍由 WebSocket 侧保留用于定稿。
state.audio_accum = state.audio_accum[samples_to_drop:]
state.audio_window_start_samples += samples_to_drop
state.window_rolled = True
state = self._decode_stream(state)
return state
@ -669,6 +775,13 @@ class Qwen3VLLMBackend:
if len(state.audio_buffer) > 0:
state.audio_accum = np.concatenate([state.audio_accum, state.audio_buffer])
state.audio_buffer = np.array([], dtype=np.float32)
max_window_samples = int(state.max_audio_window_sec * _DEFAULT_SAMPLE_RATE)
if len(state.audio_accum) > max_window_samples:
samples_to_drop = len(state.audio_accum) - max_window_samples
# 收尾 partial 仍按窗口处理;最终整段转写使用外层保留的完整音频。
state.audio_accum = state.audio_accum[samples_to_drop:]
state.audio_window_start_samples += samples_to_drop
state.window_rolled = True
state = self._decode_stream(state)
elif state.chunk_id == 0 and len(state.audio_accum) > 0:
state = self._decode_stream(state)

View File

@ -1414,6 +1414,10 @@ class Qwen3ASRService:
unfixed_chunk_num=self._stream_unfixed_chunk_num(ctx),
unfixed_token_num=self._stream_unfixed_token_num(ctx),
max_new_tokens=48,
max_audio_window_sec=max(
float(getattr(settings, "REALTIME_STREAM_WINDOW_SEC", 8.0) or 8.0),
1.0,
),
)
except Exception as exc:
logger.debug("Init realtime partial stream failed: %s", exc)