调整并发解码请求逻辑,每批最多8个音频输入
parent
1c0d3d145b
commit
f1074ce878
|
|
@ -47,6 +47,9 @@
|
|||
# QWEN_VLLM_ENFORCE_EAGER=true
|
||||
# ASR_BATCH_SIZE=4
|
||||
# QWEN_VLLM_SHARED_CONCURRENCY=8
|
||||
# vLLM 实时/离线解码微批处理:最多等待 10ms 收集请求,单批最多 8 个音频输入。
|
||||
# QWEN_VLLM_DECODE_BATCH_WAIT_MS=10
|
||||
# QWEN_VLLM_DECODE_MAX_BATCH_SIZE=8
|
||||
# ASR_ENABLE_WORD_TIMESTAMPS=false
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -81,6 +81,9 @@ class Settings:
|
|||
|
||||
# Runtime 并发配置(按 backend 独立控制)
|
||||
QWEN_VLLM_SHARED_CONCURRENCY: int = 8
|
||||
# 多个实时会话的解码请求在该时间窗内合批;批量上限包含离线和实时请求。
|
||||
QWEN_VLLM_DECODE_BATCH_WAIT_MS: int = 10
|
||||
QWEN_VLLM_DECODE_MAX_BATCH_SIZE: int = 8
|
||||
QWEN_VLLM_ENFORCE_EAGER: bool = True
|
||||
QWEN_RUST_CPU_WORKERS: int = 4
|
||||
QWEN_RUST_ASR_CONCURRENCY: int = 0
|
||||
|
|
@ -231,6 +234,18 @@ class Settings:
|
|||
str(self.QWEN_VLLM_SHARED_CONCURRENCY),
|
||||
)
|
||||
)
|
||||
self.QWEN_VLLM_DECODE_BATCH_WAIT_MS = int(
|
||||
os.getenv(
|
||||
"QWEN_VLLM_DECODE_BATCH_WAIT_MS",
|
||||
str(self.QWEN_VLLM_DECODE_BATCH_WAIT_MS),
|
||||
)
|
||||
)
|
||||
self.QWEN_VLLM_DECODE_MAX_BATCH_SIZE = int(
|
||||
os.getenv(
|
||||
"QWEN_VLLM_DECODE_MAX_BATCH_SIZE",
|
||||
str(self.QWEN_VLLM_DECODE_MAX_BATCH_SIZE),
|
||||
)
|
||||
)
|
||||
self.QWEN_VLLM_ENFORCE_EAGER = (
|
||||
os.getenv(
|
||||
"QWEN_VLLM_ENFORCE_EAGER",
|
||||
|
|
|
|||
|
|
@ -413,6 +413,8 @@ class Qwen3ASREngine(BaseASREngine):
|
|||
enforce_eager=settings.QWEN_VLLM_ENFORCE_EAGER,
|
||||
max_model_len=max_model_len,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
decode_batch_wait_ms=settings.QWEN_VLLM_DECODE_BATCH_WAIT_MS,
|
||||
decode_max_batch_size=settings.QWEN_VLLM_DECODE_MAX_BATCH_SIZE,
|
||||
)
|
||||
|
||||
@_handle_asr_error("转写")
|
||||
|
|
|
|||
|
|
@ -7,8 +7,10 @@ import importlib
|
|||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
|
||||
|
|
@ -166,6 +168,141 @@ class _GeneratedTranscript:
|
|||
language: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _GenerateRequest:
|
||||
prompts: list[dict[str, Any]]
|
||||
sampling_params: Any
|
||||
batch_key: tuple[Any, ...]
|
||||
done: threading.Event = field(default_factory=threading.Event)
|
||||
outputs: Optional[list[Any]] = None
|
||||
error: Optional[Exception] = None
|
||||
|
||||
|
||||
class _VLLMGenerateBatcher:
|
||||
"""Collect concurrent session requests into small compatible vLLM batches."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
llm: Any,
|
||||
engine_lock: threading.RLock,
|
||||
*,
|
||||
batch_wait_ms: int,
|
||||
max_batch_size: int,
|
||||
) -> None:
|
||||
self._llm = llm
|
||||
self._engine_lock = engine_lock
|
||||
self._batch_wait_sec = max(0, batch_wait_ms) / 1000.0
|
||||
self._max_batch_size = max(1, max_batch_size)
|
||||
self._requests: queue.Queue[_GenerateRequest] = queue.Queue()
|
||||
self._pending: Optional[_GenerateRequest] = None
|
||||
self._worker = threading.Thread(
|
||||
target=self._run,
|
||||
name="qwen-vllm-realtime-batcher",
|
||||
daemon=True,
|
||||
)
|
||||
self._worker.start()
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompts: list[dict[str, Any]],
|
||||
*,
|
||||
sampling_params: Any,
|
||||
batch_key: tuple[Any, ...],
|
||||
) -> list[Any]:
|
||||
request = _GenerateRequest(
|
||||
prompts=prompts,
|
||||
sampling_params=sampling_params,
|
||||
batch_key=batch_key,
|
||||
)
|
||||
self._requests.put(request)
|
||||
request.done.wait()
|
||||
if request.error is not None:
|
||||
raise request.error
|
||||
return request.outputs or []
|
||||
|
||||
def _take_request(self, timeout: Optional[float] = None) -> _GenerateRequest:
|
||||
if self._pending is not None:
|
||||
request, self._pending = self._pending, None
|
||||
return request
|
||||
return self._requests.get(timeout=timeout)
|
||||
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
first = self._take_request()
|
||||
collected = [first]
|
||||
prompt_count = len(first.prompts)
|
||||
deadline = time.monotonic() + self._batch_wait_sec
|
||||
|
||||
# A short collection window lets independently scheduled WebSocket
|
||||
# sessions share one GPU launch, as FunASR's realtime batcher does.
|
||||
while prompt_count < self._max_batch_size and self._batch_wait_sec > 0:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
try:
|
||||
request = self._requests.get(timeout=remaining)
|
||||
except queue.Empty:
|
||||
break
|
||||
if prompt_count + len(request.prompts) > self._max_batch_size:
|
||||
self._pending = request
|
||||
break
|
||||
collected.append(request)
|
||||
prompt_count += len(request.prompts)
|
||||
|
||||
compatible: dict[tuple[Any, ...], list[_GenerateRequest]] = {}
|
||||
for request in collected:
|
||||
compatible.setdefault(request.batch_key, []).append(request)
|
||||
|
||||
for group in compatible.values():
|
||||
self._generate_group(group)
|
||||
|
||||
def _generate_group(self, requests: list[_GenerateRequest]) -> None:
|
||||
prompts = [prompt for request in requests for prompt in request.prompts]
|
||||
try:
|
||||
with self._engine_lock:
|
||||
outputs = self._llm.generate(
|
||||
prompts,
|
||||
sampling_params=requests[0].sampling_params,
|
||||
use_tqdm=False,
|
||||
)
|
||||
if len(outputs) != len(prompts):
|
||||
raise RuntimeError(
|
||||
f"vLLM returned {len(outputs)} outputs for {len(prompts)} prompts"
|
||||
)
|
||||
cursor = 0
|
||||
for request in requests:
|
||||
request.outputs = list(outputs[cursor:cursor + len(request.prompts)])
|
||||
cursor += len(request.prompts)
|
||||
request.done.set()
|
||||
except Exception as exc:
|
||||
# A backend may reject a combined call for shape or memory reasons.
|
||||
# Retry each request separately so one session cannot fail its peers.
|
||||
memory_error = any(
|
||||
marker in str(exc).lower()
|
||||
for marker in ("out of memory", "no_memory", "cuda out of memory")
|
||||
)
|
||||
if len(requests) > 1 and not memory_error:
|
||||
logger.warning("Combined vLLM request failed; retrying requests separately: %s", exc)
|
||||
for request in requests:
|
||||
try:
|
||||
with self._engine_lock:
|
||||
request.outputs = list(
|
||||
self._llm.generate(
|
||||
request.prompts,
|
||||
sampling_params=request.sampling_params,
|
||||
use_tqdm=False,
|
||||
)
|
||||
)
|
||||
except Exception as request_exc:
|
||||
request.error = request_exc
|
||||
finally:
|
||||
request.done.set()
|
||||
return
|
||||
for request in requests:
|
||||
request.error = exc
|
||||
request.done.set()
|
||||
|
||||
|
||||
@dataclass
|
||||
class VLLMRealtimeState:
|
||||
prompt_raw: str
|
||||
|
|
@ -194,6 +331,8 @@ class Qwen3VLLMBackend:
|
|||
enforce_eager: bool = True,
|
||||
max_model_len: Optional[int] = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
decode_batch_wait_ms: int = 10,
|
||||
decode_max_batch_size: int = 8,
|
||||
) -> None:
|
||||
try:
|
||||
vllm_module = importlib.import_module("vllm")
|
||||
|
|
@ -235,9 +374,19 @@ class Qwen3VLLMBackend:
|
|||
self._forced_aligner: Any | None = None
|
||||
self._timestamp_token_id: int | None = None
|
||||
self._timestamp_segment_time: float | None = None
|
||||
# Share one backend instance across multiple offline tasks, but serialize
|
||||
# direct vLLM engine calls to avoid cross-request state corruption/hangs.
|
||||
# Keep generation and forced alignment mutually exclusive on this engine.
|
||||
self._engine_lock = threading.RLock()
|
||||
self._generate_batcher = _VLLMGenerateBatcher(
|
||||
self._llm,
|
||||
self._engine_lock,
|
||||
batch_wait_ms=decode_batch_wait_ms,
|
||||
max_batch_size=max(decode_max_batch_size, max_inference_batch_size),
|
||||
)
|
||||
logger.info(
|
||||
"Enabled vLLM decode micro-batching: wait_ms=%s, max_batch_size=%s",
|
||||
max(0, decode_batch_wait_ms),
|
||||
max(decode_max_batch_size, max_inference_batch_size),
|
||||
)
|
||||
|
||||
def _get_forced_aligner_gpu_memory_utilization(self) -> float:
|
||||
configured = _resolve_forced_aligner_gpu_memory_utilization(self._gpu_memory_utilization)
|
||||
|
|
@ -296,12 +445,11 @@ class Qwen3VLLMBackend:
|
|||
}
|
||||
)
|
||||
|
||||
with self._engine_lock:
|
||||
outputs = self._llm.generate(
|
||||
prompts,
|
||||
sampling_params=self._sampling_params,
|
||||
use_tqdm=False,
|
||||
)
|
||||
outputs = self._generate_batcher.generate(
|
||||
prompts,
|
||||
sampling_params=self._sampling_params,
|
||||
batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature),
|
||||
)
|
||||
|
||||
transcripts: list[_GeneratedTranscript] = []
|
||||
for output, (_audio, _context, language) in zip(outputs, audio_items):
|
||||
|
|
@ -485,20 +633,20 @@ class Qwen3VLLMBackend:
|
|||
if rollback:
|
||||
prefix = self._tokenizer.decode(rollback, skip_special_tokens=False).replace("\ufffd", "")
|
||||
|
||||
with self._engine_lock:
|
||||
output = self._llm.generate(
|
||||
[
|
||||
{
|
||||
"prompt": state.prompt_raw + prefix,
|
||||
"multi_modal_data": {"audio": [state.audio_accum]},
|
||||
}
|
||||
],
|
||||
sampling_params=self._sampling_params_cls(
|
||||
temperature=0.01,
|
||||
max_tokens=state.max_new_tokens,
|
||||
),
|
||||
use_tqdm=False,
|
||||
)[0]
|
||||
sampling_params = self._sampling_params_cls(
|
||||
temperature=0.01,
|
||||
max_tokens=state.max_new_tokens,
|
||||
)
|
||||
output = self._generate_batcher.generate(
|
||||
[
|
||||
{
|
||||
"prompt": state.prompt_raw + prefix,
|
||||
"multi_modal_data": {"audio": [state.audio_accum]},
|
||||
}
|
||||
],
|
||||
sampling_params=sampling_params,
|
||||
batch_key=(state.max_new_tokens, 0.01),
|
||||
)[0]
|
||||
generated = str(output.outputs[0].text if output.outputs else "")
|
||||
parsed_language, parsed_text = _parse_asr_output(prefix + generated, state.language or None)
|
||||
state.raw_decoded = prefix + generated
|
||||
|
|
|
|||
Loading…
Reference in New Issue