调整实时partial超窗口重解压逻辑,减少长句持续增长时的重复推理
parent
f1074ce878
commit
5842c25251
|
|
@ -50,6 +50,8 @@
|
|||
# vLLM 实时/离线解码微批处理:最多等待 10ms 收集请求,单批最多 8 个音频输入。
|
||||
# QWEN_VLLM_DECODE_BATCH_WAIT_MS=10
|
||||
# QWEN_VLLM_DECODE_MAX_BATCH_SIZE=8
|
||||
# 实时 partial 仅重复解码最近 8 秒窗口;定稿仍使用整段音频。
|
||||
# REALTIME_STREAM_WINDOW_SEC=8.0
|
||||
# ASR_ENABLE_WORD_TIMESTAMPS=false
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -108,10 +108,11 @@ class Settings:
|
|||
REALTIME_MIN_PARTIAL_SEC: float = 0.45
|
||||
REALTIME_PARTIAL_EMIT_INTERVAL_SEC: float = 0.25
|
||||
REALTIME_PARTIAL_WINDOW_SEC: float = 8.0
|
||||
# Match FunASR's roughly 600 ms online decode cadence; Qwen re-decodes the
|
||||
# accumulated stream, so this stays configurable for server-side tuning.
|
||||
# Match FunASR's roughly 600 ms online decode cadence. Qwen partial only
|
||||
# re-decodes the recent rolling window; final transcription still uses full audio.
|
||||
REALTIME_STREAM_CHUNK_SEC: float = 0.6
|
||||
REALTIME_STREAM_MAX_PENDING_CHUNKS: int = 3
|
||||
# 实时 partial 重解码窗口长度;完整定稿音频保存在 WebSocket turn buffer 中。
|
||||
REALTIME_STREAM_WINDOW_SEC: float = 8.0
|
||||
REALTIME_STREAM_STABLE_TAIL_CHARS: int = 8
|
||||
REALTIME_STREAM_STABLE_MIN_GROW_CHARS: int = 2
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import re
|
|||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Any, Optional
|
||||
|
||||
import librosa
|
||||
|
|
@ -173,6 +174,7 @@ class _GenerateRequest:
|
|||
prompts: list[dict[str, Any]]
|
||||
sampling_params: Any
|
||||
batch_key: tuple[Any, ...]
|
||||
queued_at: float = field(default_factory=time.monotonic)
|
||||
done: threading.Event = field(default_factory=threading.Event)
|
||||
outputs: Optional[list[Any]] = None
|
||||
error: Optional[Exception] = None
|
||||
|
|
@ -258,8 +260,11 @@ class _VLLMGenerateBatcher:
|
|||
|
||||
def _generate_group(self, requests: list[_GenerateRequest]) -> None:
|
||||
prompts = [prompt for request in requests for prompt in request.prompts]
|
||||
queued_at = min(request.queued_at for request in requests)
|
||||
lock_wait_started = time.monotonic()
|
||||
try:
|
||||
with self._engine_lock:
|
||||
generate_started = time.monotonic()
|
||||
outputs = self._llm.generate(
|
||||
prompts,
|
||||
sampling_params=requests[0].sampling_params,
|
||||
|
|
@ -269,6 +274,19 @@ class _VLLMGenerateBatcher:
|
|||
raise RuntimeError(
|
||||
f"vLLM returned {len(outputs)} outputs for {len(prompts)} prompts"
|
||||
)
|
||||
generate_ms = (time.monotonic() - generate_started) * 1000.0
|
||||
lock_wait_ms = (generate_started - lock_wait_started) * 1000.0
|
||||
if len(requests) > 1 or generate_ms >= 500.0 or lock_wait_ms >= 100.0:
|
||||
logger.info(
|
||||
"vLLM decode batch completed: requests=%d prompts=%d max_tokens=%s "
|
||||
"queue_wait_ms=%.1f engine_lock_wait_ms=%.1f generate_ms=%.1f",
|
||||
len(requests),
|
||||
len(prompts),
|
||||
getattr(requests[0].sampling_params, "max_tokens", None),
|
||||
(lock_wait_started - queued_at) * 1000.0,
|
||||
lock_wait_ms,
|
||||
generate_ms,
|
||||
)
|
||||
cursor = 0
|
||||
for request in requests:
|
||||
request.outputs = list(outputs[cursor:cursor + len(request.prompts)])
|
||||
|
|
@ -311,13 +329,71 @@ class VLLMRealtimeState:
|
|||
unfixed_chunk_num: int
|
||||
unfixed_token_num: int
|
||||
max_new_tokens: int
|
||||
max_audio_window_sec: float = 8.0
|
||||
chunk_id: int = 0
|
||||
text: str = ""
|
||||
raw_decoded: str = ""
|
||||
window_text: str = ""
|
||||
# 记录滚动窗口是否已经丢弃旧音频,供文本快照重叠拼接使用。
|
||||
audio_window_start_samples: int = 0
|
||||
window_rolled: bool = False
|
||||
audio_buffer: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
|
||||
audio_accum: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
|
||||
|
||||
|
||||
def _transcript_char_views(text: str) -> list[tuple[str, int]]:
|
||||
"""Return normalized speech characters with their original string offsets."""
|
||||
return [
|
||||
(char.lower(), index)
|
||||
for index, char in enumerate(text)
|
||||
if char.isalnum() or "\u4e00" <= char <= "\u9fff"
|
||||
]
|
||||
|
||||
|
||||
def _merge_rolling_window_text(
|
||||
assembled_text: str,
|
||||
previous_window_text: str,
|
||||
current_window_text: str,
|
||||
) -> tuple[str, bool]:
|
||||
"""Append only the new tail after matching adjacent partial-window overlap."""
|
||||
if not assembled_text:
|
||||
return current_window_text.strip(), True
|
||||
previous_views = _transcript_char_views(previous_window_text)
|
||||
current_views = _transcript_char_views(current_window_text)
|
||||
if not previous_views or not current_views:
|
||||
return assembled_text, False
|
||||
|
||||
match = SequenceMatcher(
|
||||
None,
|
||||
[char for char, _ in previous_views],
|
||||
[char for char, _ in current_views],
|
||||
autojunk=False,
|
||||
).find_longest_match(0, len(previous_views), 0, len(current_views))
|
||||
min_overlap = min(8, len(previous_views), len(current_views))
|
||||
max_edge_gap = max(12, int(min(len(previous_views), len(current_views)) * 0.25))
|
||||
previous_tail_gap = len(previous_views) - (match.a + match.size)
|
||||
if (
|
||||
match.size < min_overlap
|
||||
or previous_tail_gap > max_edge_gap
|
||||
or match.b > max_edge_gap
|
||||
):
|
||||
return assembled_text, False
|
||||
|
||||
overlap_end = match.b + match.size
|
||||
delta_start = (
|
||||
current_views[overlap_end][1]
|
||||
if overlap_end < len(current_views)
|
||||
else len(current_window_text)
|
||||
)
|
||||
delta = current_window_text[delta_start:].lstrip()
|
||||
if not delta:
|
||||
return assembled_text, True
|
||||
|
||||
# Chinese text joins directly; add a separator only between adjacent Latin words.
|
||||
separator = " " if assembled_text[-1].isalnum() and delta[0].isalnum() else ""
|
||||
return f"{assembled_text.rstrip()}{separator}{delta}", True
|
||||
|
||||
|
||||
class Qwen3VLLMBackend:
|
||||
"""Thin adapter over official vLLM APIs for Qwen3-ASR."""
|
||||
|
||||
|
|
@ -366,7 +442,8 @@ class Qwen3VLLMBackend:
|
|||
temperature=0.01,
|
||||
max_tokens=max_new_tokens,
|
||||
)
|
||||
self._max_inference_batch_size = max_inference_batch_size
|
||||
self._max_inference_batch_size = max(1, max_inference_batch_size)
|
||||
self._decode_max_batch_size = max(1, decode_max_batch_size)
|
||||
self._gpu_memory_utilization = gpu_memory_utilization
|
||||
self._enforce_eager = enforce_eager
|
||||
self._tensor_parallel_size = tensor_parallel_size
|
||||
|
|
@ -380,12 +457,12 @@ class Qwen3VLLMBackend:
|
|||
self._llm,
|
||||
self._engine_lock,
|
||||
batch_wait_ms=decode_batch_wait_ms,
|
||||
max_batch_size=max(decode_max_batch_size, max_inference_batch_size),
|
||||
max_batch_size=self._decode_max_batch_size,
|
||||
)
|
||||
logger.info(
|
||||
"Enabled vLLM decode micro-batching: wait_ms=%s, max_batch_size=%s",
|
||||
max(0, decode_batch_wait_ms),
|
||||
max(decode_max_batch_size, max_inference_batch_size),
|
||||
self._decode_max_batch_size,
|
||||
)
|
||||
|
||||
def _get_forced_aligner_gpu_memory_utilization(self) -> float:
|
||||
|
|
@ -445,11 +522,17 @@ class Qwen3VLLMBackend:
|
|||
}
|
||||
)
|
||||
|
||||
outputs = self._generate_batcher.generate(
|
||||
prompts,
|
||||
sampling_params=self._sampling_params,
|
||||
batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature),
|
||||
)
|
||||
outputs: list[Any] = []
|
||||
# Keep one offline request within the same hard batch cap used for
|
||||
# combining concurrent realtime sessions.
|
||||
for start in range(0, len(prompts), self._decode_max_batch_size):
|
||||
outputs.extend(
|
||||
self._generate_batcher.generate(
|
||||
prompts[start:start + self._decode_max_batch_size],
|
||||
sampling_params=self._sampling_params,
|
||||
batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature),
|
||||
)
|
||||
)
|
||||
|
||||
transcripts: list[_GeneratedTranscript] = []
|
||||
for output, (_audio, _context, language) in zip(outputs, audio_items):
|
||||
|
|
@ -612,6 +695,7 @@ class Qwen3VLLMBackend:
|
|||
unfixed_chunk_num: int = 2,
|
||||
unfixed_token_num: int = 5,
|
||||
max_new_tokens: int = 32,
|
||||
max_audio_window_sec: float = 8.0,
|
||||
) -> VLLMRealtimeState:
|
||||
normalized_language = _normalize_language_name(language) or ""
|
||||
return VLLMRealtimeState(
|
||||
|
|
@ -621,13 +705,16 @@ class Qwen3VLLMBackend:
|
|||
unfixed_chunk_num=unfixed_chunk_num,
|
||||
unfixed_token_num=unfixed_token_num,
|
||||
max_new_tokens=max_new_tokens,
|
||||
max_audio_window_sec=max(1.0, max_audio_window_sec),
|
||||
audio_buffer=np.array([], dtype=np.float32),
|
||||
audio_accum=np.array([], dtype=np.float32),
|
||||
)
|
||||
|
||||
def _decode_stream(self, state: VLLMRealtimeState) -> VLLMRealtimeState:
|
||||
prefix = ""
|
||||
if state.chunk_id >= state.unfixed_chunk_num and state.raw_decoded:
|
||||
# After the audio window rolls, decode a fresh window snapshot and stitch
|
||||
# its text overlap; old token prefixes would pin the model to stale audio.
|
||||
if not state.window_rolled and state.chunk_id >= state.unfixed_chunk_num and state.raw_decoded:
|
||||
token_ids = self._tokenizer.encode(state.raw_decoded, add_special_tokens=False)
|
||||
rollback = token_ids[-state.unfixed_token_num:] if state.unfixed_token_num > 0 else []
|
||||
if rollback:
|
||||
|
|
@ -650,7 +737,19 @@ class Qwen3VLLMBackend:
|
|||
generated = str(output.outputs[0].text if output.outputs else "")
|
||||
parsed_language, parsed_text = _parse_asr_output(prefix + generated, state.language or None)
|
||||
state.raw_decoded = prefix + generated
|
||||
state.text = parsed_text
|
||||
if state.window_rolled and parsed_text:
|
||||
state.text, overlap_matched = _merge_rolling_window_text(
|
||||
state.text,
|
||||
state.window_text,
|
||||
parsed_text,
|
||||
)
|
||||
# Keep the older anchor when a model revision breaks overlap matching;
|
||||
# the next window still has another chance to find the shared span.
|
||||
if overlap_matched:
|
||||
state.window_text = parsed_text
|
||||
elif not state.window_rolled:
|
||||
state.text = parsed_text
|
||||
state.window_text = parsed_text
|
||||
state.language = parsed_language or state.language
|
||||
state.chunk_id += 1
|
||||
return state
|
||||
|
|
@ -662,6 +761,13 @@ class Qwen3VLLMBackend:
|
|||
segment = state.audio_buffer[:segment_size].copy()
|
||||
state.audio_buffer = state.audio_buffer[segment_size:]
|
||||
state.audio_accum = np.concatenate([state.audio_accum, segment])
|
||||
max_window_samples = int(state.max_audio_window_sec * _DEFAULT_SAMPLE_RATE)
|
||||
if len(state.audio_accum) > max_window_samples:
|
||||
samples_to_drop = len(state.audio_accum) - max_window_samples
|
||||
# Partial 只保留最近窗口,完整音频仍由 WebSocket 侧保留用于定稿。
|
||||
state.audio_accum = state.audio_accum[samples_to_drop:]
|
||||
state.audio_window_start_samples += samples_to_drop
|
||||
state.window_rolled = True
|
||||
state = self._decode_stream(state)
|
||||
return state
|
||||
|
||||
|
|
@ -669,6 +775,13 @@ class Qwen3VLLMBackend:
|
|||
if len(state.audio_buffer) > 0:
|
||||
state.audio_accum = np.concatenate([state.audio_accum, state.audio_buffer])
|
||||
state.audio_buffer = np.array([], dtype=np.float32)
|
||||
max_window_samples = int(state.max_audio_window_sec * _DEFAULT_SAMPLE_RATE)
|
||||
if len(state.audio_accum) > max_window_samples:
|
||||
samples_to_drop = len(state.audio_accum) - max_window_samples
|
||||
# 收尾 partial 仍按窗口处理;最终整段转写使用外层保留的完整音频。
|
||||
state.audio_accum = state.audio_accum[samples_to_drop:]
|
||||
state.audio_window_start_samples += samples_to_drop
|
||||
state.window_rolled = True
|
||||
state = self._decode_stream(state)
|
||||
elif state.chunk_id == 0 and len(state.audio_accum) > 0:
|
||||
state = self._decode_stream(state)
|
||||
|
|
|
|||
|
|
@ -1414,6 +1414,10 @@ class Qwen3ASRService:
|
|||
unfixed_chunk_num=self._stream_unfixed_chunk_num(ctx),
|
||||
unfixed_token_num=self._stream_unfixed_token_num(ctx),
|
||||
max_new_tokens=48,
|
||||
max_audio_window_sec=max(
|
||||
float(getattr(settings, "REALTIME_STREAM_WINDOW_SEC", 8.0) or 8.0),
|
||||
1.0,
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Init realtime partial stream failed: %s", exc)
|
||||
|
|
|
|||
Loading…
Reference in New Issue