调整并发解码请求逻辑,每批最多8个音频输入

main
Bifang 2026-10-09 13:31:37 +08:00
parent 1c0d3d145b
commit f1074ce878
4 changed files with 190 additions and 22 deletions

View File

@ -47,6 +47,9 @@
# QWEN_VLLM_ENFORCE_EAGER=true
# ASR_BATCH_SIZE=4
# QWEN_VLLM_SHARED_CONCURRENCY=8
# vLLM 实时/离线解码微批处理:最多等待 10ms 收集请求,单批最多 8 个音频输入。
# QWEN_VLLM_DECODE_BATCH_WAIT_MS=10
# QWEN_VLLM_DECODE_MAX_BATCH_SIZE=8
# ASR_ENABLE_WORD_TIMESTAMPS=false
# -----------------------------------------------------------------------------

View File

@ -81,6 +81,9 @@ class Settings:
# Runtime 并发配置(按 backend 独立控制)
QWEN_VLLM_SHARED_CONCURRENCY: int = 8
# 多个实时会话的解码请求在该时间窗内合批;批量上限包含离线和实时请求。
QWEN_VLLM_DECODE_BATCH_WAIT_MS: int = 10
QWEN_VLLM_DECODE_MAX_BATCH_SIZE: int = 8
QWEN_VLLM_ENFORCE_EAGER: bool = True
QWEN_RUST_CPU_WORKERS: int = 4
QWEN_RUST_ASR_CONCURRENCY: int = 0
@ -231,6 +234,18 @@ class Settings:
str(self.QWEN_VLLM_SHARED_CONCURRENCY),
)
)
self.QWEN_VLLM_DECODE_BATCH_WAIT_MS = int(
os.getenv(
"QWEN_VLLM_DECODE_BATCH_WAIT_MS",
str(self.QWEN_VLLM_DECODE_BATCH_WAIT_MS),
)
)
self.QWEN_VLLM_DECODE_MAX_BATCH_SIZE = int(
os.getenv(
"QWEN_VLLM_DECODE_MAX_BATCH_SIZE",
str(self.QWEN_VLLM_DECODE_MAX_BATCH_SIZE),
)
)
self.QWEN_VLLM_ENFORCE_EAGER = (
os.getenv(
"QWEN_VLLM_ENFORCE_EAGER",

View File

@ -413,6 +413,8 @@ class Qwen3ASREngine(BaseASREngine):
enforce_eager=settings.QWEN_VLLM_ENFORCE_EAGER,
max_model_len=max_model_len,
tensor_parallel_size=tensor_parallel_size,
decode_batch_wait_ms=settings.QWEN_VLLM_DECODE_BATCH_WAIT_MS,
decode_max_batch_size=settings.QWEN_VLLM_DECODE_MAX_BATCH_SIZE,
)
@_handle_asr_error("转写")

View File

@ -7,8 +7,10 @@ import importlib
import importlib.util
import logging
import os
import queue
import re
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Optional
@ -166,6 +168,141 @@ class _GeneratedTranscript:
language: str
@dataclass
class _GenerateRequest:
prompts: list[dict[str, Any]]
sampling_params: Any
batch_key: tuple[Any, ...]
done: threading.Event = field(default_factory=threading.Event)
outputs: Optional[list[Any]] = None
error: Optional[Exception] = None
class _VLLMGenerateBatcher:
"""Collect concurrent session requests into small compatible vLLM batches."""
def __init__(
self,
llm: Any,
engine_lock: threading.RLock,
*,
batch_wait_ms: int,
max_batch_size: int,
) -> None:
self._llm = llm
self._engine_lock = engine_lock
self._batch_wait_sec = max(0, batch_wait_ms) / 1000.0
self._max_batch_size = max(1, max_batch_size)
self._requests: queue.Queue[_GenerateRequest] = queue.Queue()
self._pending: Optional[_GenerateRequest] = None
self._worker = threading.Thread(
target=self._run,
name="qwen-vllm-realtime-batcher",
daemon=True,
)
self._worker.start()
def generate(
self,
prompts: list[dict[str, Any]],
*,
sampling_params: Any,
batch_key: tuple[Any, ...],
) -> list[Any]:
request = _GenerateRequest(
prompts=prompts,
sampling_params=sampling_params,
batch_key=batch_key,
)
self._requests.put(request)
request.done.wait()
if request.error is not None:
raise request.error
return request.outputs or []
def _take_request(self, timeout: Optional[float] = None) -> _GenerateRequest:
if self._pending is not None:
request, self._pending = self._pending, None
return request
return self._requests.get(timeout=timeout)
def _run(self) -> None:
while True:
first = self._take_request()
collected = [first]
prompt_count = len(first.prompts)
deadline = time.monotonic() + self._batch_wait_sec
# A short collection window lets independently scheduled WebSocket
# sessions share one GPU launch, as FunASR's realtime batcher does.
while prompt_count < self._max_batch_size and self._batch_wait_sec > 0:
remaining = deadline - time.monotonic()
if remaining <= 0:
break
try:
request = self._requests.get(timeout=remaining)
except queue.Empty:
break
if prompt_count + len(request.prompts) > self._max_batch_size:
self._pending = request
break
collected.append(request)
prompt_count += len(request.prompts)
compatible: dict[tuple[Any, ...], list[_GenerateRequest]] = {}
for request in collected:
compatible.setdefault(request.batch_key, []).append(request)
for group in compatible.values():
self._generate_group(group)
def _generate_group(self, requests: list[_GenerateRequest]) -> None:
prompts = [prompt for request in requests for prompt in request.prompts]
try:
with self._engine_lock:
outputs = self._llm.generate(
prompts,
sampling_params=requests[0].sampling_params,
use_tqdm=False,
)
if len(outputs) != len(prompts):
raise RuntimeError(
f"vLLM returned {len(outputs)} outputs for {len(prompts)} prompts"
)
cursor = 0
for request in requests:
request.outputs = list(outputs[cursor:cursor + len(request.prompts)])
cursor += len(request.prompts)
request.done.set()
except Exception as exc:
# A backend may reject a combined call for shape or memory reasons.
# Retry each request separately so one session cannot fail its peers.
memory_error = any(
marker in str(exc).lower()
for marker in ("out of memory", "no_memory", "cuda out of memory")
)
if len(requests) > 1 and not memory_error:
logger.warning("Combined vLLM request failed; retrying requests separately: %s", exc)
for request in requests:
try:
with self._engine_lock:
request.outputs = list(
self._llm.generate(
request.prompts,
sampling_params=request.sampling_params,
use_tqdm=False,
)
)
except Exception as request_exc:
request.error = request_exc
finally:
request.done.set()
return
for request in requests:
request.error = exc
request.done.set()
@dataclass
class VLLMRealtimeState:
prompt_raw: str
@ -194,6 +331,8 @@ class Qwen3VLLMBackend:
enforce_eager: bool = True,
max_model_len: Optional[int] = None,
tensor_parallel_size: int = 1,
decode_batch_wait_ms: int = 10,
decode_max_batch_size: int = 8,
) -> None:
try:
vllm_module = importlib.import_module("vllm")
@ -235,9 +374,19 @@ class Qwen3VLLMBackend:
self._forced_aligner: Any | None = None
self._timestamp_token_id: int | None = None
self._timestamp_segment_time: float | None = None
# Share one backend instance across multiple offline tasks, but serialize
# direct vLLM engine calls to avoid cross-request state corruption/hangs.
# Keep generation and forced alignment mutually exclusive on this engine.
self._engine_lock = threading.RLock()
self._generate_batcher = _VLLMGenerateBatcher(
self._llm,
self._engine_lock,
batch_wait_ms=decode_batch_wait_ms,
max_batch_size=max(decode_max_batch_size, max_inference_batch_size),
)
logger.info(
"Enabled vLLM decode micro-batching: wait_ms=%s, max_batch_size=%s",
max(0, decode_batch_wait_ms),
max(decode_max_batch_size, max_inference_batch_size),
)
def _get_forced_aligner_gpu_memory_utilization(self) -> float:
configured = _resolve_forced_aligner_gpu_memory_utilization(self._gpu_memory_utilization)
@ -296,12 +445,11 @@ class Qwen3VLLMBackend:
}
)
with self._engine_lock:
outputs = self._llm.generate(
prompts,
sampling_params=self._sampling_params,
use_tqdm=False,
)
outputs = self._generate_batcher.generate(
prompts,
sampling_params=self._sampling_params,
batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature),
)
transcripts: list[_GeneratedTranscript] = []
for output, (_audio, _context, language) in zip(outputs, audio_items):
@ -485,20 +633,20 @@ class Qwen3VLLMBackend:
if rollback:
prefix = self._tokenizer.decode(rollback, skip_special_tokens=False).replace("\ufffd", "")
with self._engine_lock:
output = self._llm.generate(
[
{
"prompt": state.prompt_raw + prefix,
"multi_modal_data": {"audio": [state.audio_accum]},
}
],
sampling_params=self._sampling_params_cls(
temperature=0.01,
max_tokens=state.max_new_tokens,
),
use_tqdm=False,
)[0]
sampling_params = self._sampling_params_cls(
temperature=0.01,
max_tokens=state.max_new_tokens,
)
output = self._generate_batcher.generate(
[
{
"prompt": state.prompt_raw + prefix,
"multi_modal_data": {"audio": [state.audio_accum]},
}
],
sampling_params=sampling_params,
batch_key=(state.max_new_tokens, 0.01),
)[0]
generated = str(output.outputs[0].text if output.outputs else "")
parsed_language, parsed_text = _parse_asr_output(prefix + generated, state.language or None)
state.raw_decoded = prefix + generated