diff --git a/.env.example b/.env.example index a5334f1..21fda8a 100644 --- a/.env.example +++ b/.env.example @@ -47,6 +47,9 @@ # QWEN_VLLM_ENFORCE_EAGER=true # ASR_BATCH_SIZE=4 # QWEN_VLLM_SHARED_CONCURRENCY=8 +# vLLM 实时/离线解码微批处理:最多等待 10ms 收集请求,单批最多 8 个音频输入。 +# QWEN_VLLM_DECODE_BATCH_WAIT_MS=10 +# QWEN_VLLM_DECODE_MAX_BATCH_SIZE=8 # ASR_ENABLE_WORD_TIMESTAMPS=false # ----------------------------------------------------------------------------- diff --git a/app/core/config.py b/app/core/config.py index dc2fa7e..61ef705 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -81,6 +81,9 @@ class Settings: # Runtime 并发配置(按 backend 独立控制) QWEN_VLLM_SHARED_CONCURRENCY: int = 8 + # 多个实时会话的解码请求在该时间窗内合批;批量上限包含离线和实时请求。 + QWEN_VLLM_DECODE_BATCH_WAIT_MS: int = 10 + QWEN_VLLM_DECODE_MAX_BATCH_SIZE: int = 8 QWEN_VLLM_ENFORCE_EAGER: bool = True QWEN_RUST_CPU_WORKERS: int = 4 QWEN_RUST_ASR_CONCURRENCY: int = 0 @@ -231,6 +234,18 @@ class Settings: str(self.QWEN_VLLM_SHARED_CONCURRENCY), ) ) + self.QWEN_VLLM_DECODE_BATCH_WAIT_MS = int( + os.getenv( + "QWEN_VLLM_DECODE_BATCH_WAIT_MS", + str(self.QWEN_VLLM_DECODE_BATCH_WAIT_MS), + ) + ) + self.QWEN_VLLM_DECODE_MAX_BATCH_SIZE = int( + os.getenv( + "QWEN_VLLM_DECODE_MAX_BATCH_SIZE", + str(self.QWEN_VLLM_DECODE_MAX_BATCH_SIZE), + ) + ) self.QWEN_VLLM_ENFORCE_EAGER = ( os.getenv( "QWEN_VLLM_ENFORCE_EAGER", diff --git a/app/services/asr/qwen3_engine.py b/app/services/asr/qwen3_engine.py index fa8b79d..2bca0bd 100644 --- a/app/services/asr/qwen3_engine.py +++ b/app/services/asr/qwen3_engine.py @@ -413,6 +413,8 @@ class Qwen3ASREngine(BaseASREngine): enforce_eager=settings.QWEN_VLLM_ENFORCE_EAGER, max_model_len=max_model_len, tensor_parallel_size=tensor_parallel_size, + decode_batch_wait_ms=settings.QWEN_VLLM_DECODE_BATCH_WAIT_MS, + decode_max_batch_size=settings.QWEN_VLLM_DECODE_MAX_BATCH_SIZE, ) @_handle_asr_error("转写") diff --git a/app/services/asr/qwen3_vllm.py b/app/services/asr/qwen3_vllm.py index b07d553..a854ee4 100644 --- a/app/services/asr/qwen3_vllm.py +++ b/app/services/asr/qwen3_vllm.py @@ -7,8 +7,10 @@ import importlib import importlib.util import logging import os +import queue import re import threading +import time from dataclasses import dataclass, field from typing import Any, Optional @@ -166,6 +168,141 @@ class _GeneratedTranscript: language: str +@dataclass +class _GenerateRequest: + prompts: list[dict[str, Any]] + sampling_params: Any + batch_key: tuple[Any, ...] + done: threading.Event = field(default_factory=threading.Event) + outputs: Optional[list[Any]] = None + error: Optional[Exception] = None + + +class _VLLMGenerateBatcher: + """Collect concurrent session requests into small compatible vLLM batches.""" + + def __init__( + self, + llm: Any, + engine_lock: threading.RLock, + *, + batch_wait_ms: int, + max_batch_size: int, + ) -> None: + self._llm = llm + self._engine_lock = engine_lock + self._batch_wait_sec = max(0, batch_wait_ms) / 1000.0 + self._max_batch_size = max(1, max_batch_size) + self._requests: queue.Queue[_GenerateRequest] = queue.Queue() + self._pending: Optional[_GenerateRequest] = None + self._worker = threading.Thread( + target=self._run, + name="qwen-vllm-realtime-batcher", + daemon=True, + ) + self._worker.start() + + def generate( + self, + prompts: list[dict[str, Any]], + *, + sampling_params: Any, + batch_key: tuple[Any, ...], + ) -> list[Any]: + request = _GenerateRequest( + prompts=prompts, + sampling_params=sampling_params, + batch_key=batch_key, + ) + self._requests.put(request) + request.done.wait() + if request.error is not None: + raise request.error + return request.outputs or [] + + def _take_request(self, timeout: Optional[float] = None) -> _GenerateRequest: + if self._pending is not None: + request, self._pending = self._pending, None + return request + return self._requests.get(timeout=timeout) + + def _run(self) -> None: + while True: + first = self._take_request() + collected = [first] + prompt_count = len(first.prompts) + deadline = time.monotonic() + self._batch_wait_sec + + # A short collection window lets independently scheduled WebSocket + # sessions share one GPU launch, as FunASR's realtime batcher does. + while prompt_count < self._max_batch_size and self._batch_wait_sec > 0: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + request = self._requests.get(timeout=remaining) + except queue.Empty: + break + if prompt_count + len(request.prompts) > self._max_batch_size: + self._pending = request + break + collected.append(request) + prompt_count += len(request.prompts) + + compatible: dict[tuple[Any, ...], list[_GenerateRequest]] = {} + for request in collected: + compatible.setdefault(request.batch_key, []).append(request) + + for group in compatible.values(): + self._generate_group(group) + + def _generate_group(self, requests: list[_GenerateRequest]) -> None: + prompts = [prompt for request in requests for prompt in request.prompts] + try: + with self._engine_lock: + outputs = self._llm.generate( + prompts, + sampling_params=requests[0].sampling_params, + use_tqdm=False, + ) + if len(outputs) != len(prompts): + raise RuntimeError( + f"vLLM returned {len(outputs)} outputs for {len(prompts)} prompts" + ) + cursor = 0 + for request in requests: + request.outputs = list(outputs[cursor:cursor + len(request.prompts)]) + cursor += len(request.prompts) + request.done.set() + except Exception as exc: + # A backend may reject a combined call for shape or memory reasons. + # Retry each request separately so one session cannot fail its peers. + memory_error = any( + marker in str(exc).lower() + for marker in ("out of memory", "no_memory", "cuda out of memory") + ) + if len(requests) > 1 and not memory_error: + logger.warning("Combined vLLM request failed; retrying requests separately: %s", exc) + for request in requests: + try: + with self._engine_lock: + request.outputs = list( + self._llm.generate( + request.prompts, + sampling_params=request.sampling_params, + use_tqdm=False, + ) + ) + except Exception as request_exc: + request.error = request_exc + finally: + request.done.set() + return + for request in requests: + request.error = exc + request.done.set() + + @dataclass class VLLMRealtimeState: prompt_raw: str @@ -194,6 +331,8 @@ class Qwen3VLLMBackend: enforce_eager: bool = True, max_model_len: Optional[int] = None, tensor_parallel_size: int = 1, + decode_batch_wait_ms: int = 10, + decode_max_batch_size: int = 8, ) -> None: try: vllm_module = importlib.import_module("vllm") @@ -235,9 +374,19 @@ class Qwen3VLLMBackend: self._forced_aligner: Any | None = None self._timestamp_token_id: int | None = None self._timestamp_segment_time: float | None = None - # Share one backend instance across multiple offline tasks, but serialize - # direct vLLM engine calls to avoid cross-request state corruption/hangs. + # Keep generation and forced alignment mutually exclusive on this engine. self._engine_lock = threading.RLock() + self._generate_batcher = _VLLMGenerateBatcher( + self._llm, + self._engine_lock, + batch_wait_ms=decode_batch_wait_ms, + max_batch_size=max(decode_max_batch_size, max_inference_batch_size), + ) + logger.info( + "Enabled vLLM decode micro-batching: wait_ms=%s, max_batch_size=%s", + max(0, decode_batch_wait_ms), + max(decode_max_batch_size, max_inference_batch_size), + ) def _get_forced_aligner_gpu_memory_utilization(self) -> float: configured = _resolve_forced_aligner_gpu_memory_utilization(self._gpu_memory_utilization) @@ -296,12 +445,11 @@ class Qwen3VLLMBackend: } ) - with self._engine_lock: - outputs = self._llm.generate( - prompts, - sampling_params=self._sampling_params, - use_tqdm=False, - ) + outputs = self._generate_batcher.generate( + prompts, + sampling_params=self._sampling_params, + batch_key=(self._sampling_params.max_tokens, self._sampling_params.temperature), + ) transcripts: list[_GeneratedTranscript] = [] for output, (_audio, _context, language) in zip(outputs, audio_items): @@ -485,20 +633,20 @@ class Qwen3VLLMBackend: if rollback: prefix = self._tokenizer.decode(rollback, skip_special_tokens=False).replace("\ufffd", "") - with self._engine_lock: - output = self._llm.generate( - [ - { - "prompt": state.prompt_raw + prefix, - "multi_modal_data": {"audio": [state.audio_accum]}, - } - ], - sampling_params=self._sampling_params_cls( - temperature=0.01, - max_tokens=state.max_new_tokens, - ), - use_tqdm=False, - )[0] + sampling_params = self._sampling_params_cls( + temperature=0.01, + max_tokens=state.max_new_tokens, + ) + output = self._generate_batcher.generate( + [ + { + "prompt": state.prompt_raw + prefix, + "multi_modal_data": {"audio": [state.audio_accum]}, + } + ], + sampling_params=sampling_params, + batch_key=(state.max_new_tokens, 0.01), + )[0] generated = str(output.outputs[0].text if output.outputs else "") parsed_language, parsed_text = _parse_asr_output(prefix + generated, state.language or None) state.raw_decoded = prefix + generated