diff --git a/app/api/v1/websocket_asr.py b/app/api/v1/websocket_asr.py index ce82187..10b80f7 100644 --- a/app/api/v1/websocket_asr.py +++ b/app/api/v1/websocket_asr.py @@ -6,9 +6,10 @@ import time import uuid from typing import Optional -from fastapi import APIRouter, WebSocket +from fastapi import APIRouter, HTTPException, Request, WebSocket from ...services.qwen3_websocket_asr import Qwen3ASRService +from ...core.security import validate_token logger = logging.getLogger(__name__) router = APIRouter(prefix="/ws/v1/asr", tags=["WebSocket ASR"]) @@ -45,3 +46,22 @@ async def qwen_asr_websocket( if task_id is None: task_id = str(uuid.uuid4())[:8] await _qwen3_service.handle_connection(websocket, task_id) + + +@router.post("/model") +async def prepare_realtime_model(request: Request) -> dict[str, str]: + """Preload a selected realtime model before the browser opens its audio stream.""" + authorized, message = validate_token(request) + if not authorized: + raise HTTPException(status_code=401, detail=message) + try: + payload = await request.json() + model_id = str(payload.get("model_id") or "").strip() + return await _qwen3_service.prepare_realtime_model(model_id) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + except Exception as exc: + logger.exception("Realtime model preparation failed") + raise HTTPException(status_code=500, detail=f"模型切换失败:{exc}") from exc diff --git a/app/services/asr/engines/global_models.py b/app/services/asr/engines/global_models.py index 382e5c4..41346b0 100644 --- a/app/services/asr/engines/global_models.py +++ b/app/services/asr/engines/global_models.py @@ -157,3 +157,10 @@ def get_global_punc_realtime_model(device: str): def get_punc_realtime_inference_lock(): """获取实时PUNC模型推理锁(线程安全)""" return _punc_realtime_inference_lock + + +def unload_global_punc_realtime_model() -> None: + """Release the auxiliary realtime punctuation model during route switching.""" + global _global_punc_realtime_model + with _punc_realtime_model_lock: + _global_punc_realtime_model = None diff --git a/app/services/asr/funasr_streaming.py b/app/services/asr/funasr_streaming.py index 69cebfe..d5dd6c1 100644 --- a/app/services/asr/funasr_streaming.py +++ b/app/services/asr/funasr_streaming.py @@ -28,6 +28,14 @@ _offline_model_lock = threading.Lock() _offline_inference_lock = threading.Lock() +def unload_models() -> None: + """Release shared Paraformer instances before switching realtime routes.""" + global _model, _offline_model + with _model_lock, _offline_model_lock: + _model = None + _offline_model = None + + def get_streaming_model() -> Any: """Load one process-wide model instance; per-stream caches live in the caller.""" global _model diff --git a/app/services/asr/qwen3_engine.py b/app/services/asr/qwen3_engine.py index 367de7c..7cfec72 100644 --- a/app/services/asr/qwen3_engine.py +++ b/app/services/asr/qwen3_engine.py @@ -212,6 +212,14 @@ class Qwen3ASREngine(BaseASREngine): "GPU runtimes and CPU QwenASR Rust." ) + def close(self) -> None: + """Release backend-owned workers and model references when the engine is evicted.""" + model = getattr(self, "model", None) + close = getattr(model, "close", None) + if callable(close): + close() + self.model = None + def _load_rust_backend( self, model_path: str, diff --git a/app/services/asr/qwen3_vllm.py b/app/services/asr/qwen3_vllm.py index 00a0d97..a250156 100644 --- a/app/services/asr/qwen3_vllm.py +++ b/app/services/asr/qwen3_vllm.py @@ -197,7 +197,7 @@ class _VLLMGenerateBatcher: self._engine_lock = engine_lock self._batch_wait_sec = max(0, batch_wait_ms) / 1000.0 self._max_batch_size = max(1, max_batch_size) - self._requests: queue.Queue[_GenerateRequest] = queue.Queue() + self._requests: queue.Queue[Optional[_GenerateRequest]] = queue.Queue() self._pending: Optional[_GenerateRequest] = None self._worker = threading.Thread( target=self._run, @@ -237,6 +237,8 @@ class _VLLMGenerateBatcher: def _run(self) -> None: while True: first = self._take_request() + if first is None: + return collected = [first] prompt_count = len(first.prompts) deadline = time.monotonic() + self._batch_wait_sec @@ -251,6 +253,9 @@ class _VLLMGenerateBatcher: request = self._requests.get(timeout=remaining) except queue.Empty: break + if request is None: + self._pending = None + return if prompt_count + len(request.prompts) > self._max_batch_size: self._pending = request break @@ -264,6 +269,14 @@ class _VLLMGenerateBatcher: for group in compatible.values(): self._generate_group(group) + def close(self) -> None: + """Stop the daemon batcher so its model references can be collected.""" + self._requests.put(None) + if self._worker.is_alive() and threading.current_thread() is not self._worker: + self._worker.join(timeout=5) + if not self._worker.is_alive(): + self._llm = None + def _generate_group(self, requests: list[_GenerateRequest]) -> None: prompts = [prompt for request in requests for prompt in request.prompts] queued_at = min(request.queued_at for request in requests) @@ -476,6 +489,16 @@ class Qwen3VLLMBackend: self._decode_max_batch_size, ) + def close(self) -> None: + """Stop decode work and release vLLM model references before route changes.""" + self._generate_batcher.close() + aligner_close = getattr(self._forced_aligner, "close", None) + if callable(aligner_close): + aligner_close() + self._forced_aligner = None + self._llm = None + self._tokenizer = None + def _get_forced_aligner_gpu_memory_utilization(self) -> float: configured = _resolve_forced_aligner_gpu_memory_utilization(self._gpu_memory_utilization) logger.info( diff --git a/app/services/asr/runtime/local_pool.py b/app/services/asr/runtime/local_pool.py index 6f8b9bf..2c94969 100644 --- a/app/services/asr/runtime/local_pool.py +++ b/app/services/asr/runtime/local_pool.py @@ -39,6 +39,26 @@ class LocalEnginePool(Generic[T]): def warmup(self) -> None: self._ensure_state() + @property + def is_idle(self) -> bool: + """A pool is idle when every initialized engine is back in its queue.""" + state = self._state + return state is None or state.queue.qsize() == self._size + + def close(self) -> None: + """Close all idle engines before dropping this pool.""" + state = self._state + if state is None: + return + if not self.is_idle: + raise RuntimeError("Cannot close an engine pool while an engine is leased") + for _ in range(self._size): + engine = state.queue.get_nowait() + close = getattr(engine, "close", None) + if callable(close): + close() + self._state = None + async def acquire(self) -> T: state = self._ensure_state() return await state.queue.get() diff --git a/app/services/asr/runtime/router.py b/app/services/asr/runtime/router.py index eaf9125..0f87f0f 100644 --- a/app/services/asr/runtime/router.py +++ b/app/services/asr/runtime/router.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import gc import threading from dataclasses import dataclass from enum import Enum @@ -75,6 +76,22 @@ class RuntimeRouter: self._shared_limits: dict[tuple[RuntimeFamily, str], asyncio.Semaphore] = {} self._pool_lock = threading.Lock() self._loaded_model_ids: set[str] = set() + self._active_model_leases: dict[str, int] = {} + self._model_switching = False + self._model_switch_ready = threading.Event() + self._model_switch_ready.set() + + def begin_model_switch(self) -> None: + """Temporarily gate new offline/Qwen leases while a realtime model is replaced.""" + with self._pool_lock: + self._model_switching = True + self._model_switch_ready.clear() + + def end_model_switch(self) -> None: + """Resume queued engine acquisitions after the replacement is stable.""" + with self._pool_lock: + self._model_switching = False + self._model_switch_ready.set() def resolve_model_id(self, model_id: Optional[str]) -> str: if model_id: @@ -151,6 +168,37 @@ class RuntimeRouter: def get_loaded_model_ids(self) -> list[str]: return sorted(self._loaded_model_ids) + def unload_model(self, model_id: str) -> bool: + """Drop an idle cached engine to release memory during realtime switching.""" + family = self._resolve_family(model_id) + key = (family, model_id) + with self._pool_lock: + if self._active_model_leases.get(model_id, 0): + return False + if family == RuntimeFamily.QWEN_VLLM: + semaphore = self._shared_limits.get(key) + limit = max(1, int(settings.QWEN_VLLM_SHARED_CONCURRENCY)) + if semaphore is not None and semaphore._value < limit: + return False + engine = self._shared_engines.get(key) + close = getattr(engine, "close", None) + if callable(close): + close() + self._shared_limits.pop(key, None) + self._shared_engines.pop(key, None) + else: + pool = self._pools.get(key) + if pool is not None and not pool.is_idle: + return False + if pool is not None: + pool.close() + self._pools.pop(key, None) + self._loaded_model_ids.discard(model_id) + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return True + def get_memory_usage(self) -> dict[str, object]: memory_info: dict[str, object] = { "model_list": self.get_loaded_model_ids(), @@ -172,19 +220,47 @@ class RuntimeRouter: async def acquire_engine(self, model_id: Optional[str] = None) -> RuntimeEngineLease: resolved_model_id = self.resolve_model_id(model_id) family = self._resolve_family(resolved_model_id) - if family == RuntimeFamily.QWEN_VLLM: - engine, semaphore = self._get_shared_engine(family, resolved_model_id) - await semaphore.acquire() - return RuntimeEngineLease( - engine=engine, - release_callback=semaphore.release, - ) - pool = self._create_pool(family, resolved_model_id) - engine = await pool.acquire() - return RuntimeEngineLease( - engine=engine, - release_callback=lambda: pool.release(engine), - ) + # Reserve before awaiting a queue/semaphore so unload cannot race a + # request that has selected the engine but has not acquired it yet. + while True: + with self._pool_lock: + if not self._model_switching: + self._active_model_leases[resolved_model_id] = ( + self._active_model_leases.get(resolved_model_id, 0) + 1 + ) + break + await asyncio.to_thread(self._model_switch_ready.wait) + + def release_reservation() -> None: + with self._pool_lock: + active = self._active_model_leases.get(resolved_model_id, 0) + if active <= 1: + self._active_model_leases.pop(resolved_model_id, None) + else: + self._active_model_leases[resolved_model_id] = active - 1 + + try: + if family == RuntimeFamily.QWEN_VLLM: + engine, semaphore = self._get_shared_engine(family, resolved_model_id) + await semaphore.acquire() + + async def release_shared() -> None: + semaphore.release() + release_reservation() + + return RuntimeEngineLease(engine=engine, release_callback=release_shared) + + pool = self._create_pool(family, resolved_model_id) + engine = await pool.acquire() + + async def release_pooled() -> None: + await pool.release(engine) + release_reservation() + + return RuntimeEngineLease(engine=engine, release_callback=release_pooled) + except Exception: + release_reservation() + raise async def run_offline(self, request: OfflineASRRequest) -> ASRFullResult: async with await self.acquire_engine(request.model_id) as engine: diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index 70f63a4..83982d8 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import gc import io import json import logging @@ -17,6 +18,7 @@ from typing import Any, Dict, List, Optional import numpy as np import soundfile as sf +import torch from fastapi import WebSocket, WebSocketDisconnect from app.core.config import settings @@ -37,11 +39,13 @@ from app.services.asr.funasr_streaming import ( get_offline_inference_lock as get_funasr_offline_inference_lock, get_offline_model as get_funasr_offline_model, get_streaming_model as get_funasr_streaming_model, + unload_models as unload_funasr_models, ) from app.services.asr.engines.global_models import ( get_global_punc_realtime_model, get_global_vad_model, get_punc_realtime_inference_lock, + unload_global_punc_realtime_model, get_vad_inference_lock, ) from app.services.realtime_speaker_clusterer import get_realtime_speaker_clusterer @@ -356,6 +360,89 @@ class Qwen3ASRService: def __init__(self) -> None: self._sessions: Dict[str, SessionEntry] = {} self._session_lock = asyncio.Lock() + self._model_switch_lock = asyncio.Lock() + self._active_websocket_connections = 0 + self._switching_realtime_model = False + self._current_realtime_model = "qwen3-asr" + + async def prepare_realtime_model(self, model_id: str) -> Dict[str, str]: + """Swap and warm the selected realtime ASR route before capture begins.""" + if model_id not in {"qwen3-asr", "funasr-paraformer-streaming"}: + raise ValueError(f"Unsupported realtime model: {model_id}") + async with self._model_switch_lock: + if self._active_websocket_connections and self._current_realtime_model != model_id: + raise RuntimeError("请先结束当前实时会议,再切换识别模型。") + self._switching_realtime_model = True + previous_model = self._current_realtime_model + runtime_router = get_runtime_router() + runtime_router.begin_model_switch() + try: + if previous_model != model_id: + # A disconnected resumable session may still retain the old + # model lease; changing models invalidates that resume context. + async with self._session_lock: + resumable_contexts = [session.ctx for session in self._sessions.values()] + self._sessions.clear() + for session_ctx in resumable_contexts: + await self._release_session_resources(session_ctx) + resumable_contexts.clear() + session_ctx = None + if previous_model == model_id: + # A same-model request still verifies that lazy model loading + # completed before the frontend enables microphone capture. + if model_id == "funasr-paraformer-streaming": + await run_sync(get_funasr_streaming_model) + await run_sync(get_funasr_offline_model) + if settings.ASR_ENABLE_REALTIME_PUNC: + await run_sync(get_global_punc_realtime_model, settings.DEVICE) + await run_sync(get_global_vad_model, settings.DEVICE) + else: + qwen_model_id = validate_realtime_model_id("qwen3-asr") + await run_sync(runtime_router.warmup_model, qwen_model_id) + elif previous_model == "funasr-paraformer-streaming": + unload_funasr_models() + unload_global_punc_realtime_model() + else: + qwen_model_id = validate_realtime_model_id("qwen3-asr") + if not runtime_router.unload_model(qwen_model_id): + raise RuntimeError("Qwen 模型仍有任务占用,暂时无法释放;请稍后重试。") + + if model_id == "funasr-paraformer-streaming": + await run_sync(get_funasr_streaming_model) + await run_sync(get_funasr_offline_model) + if settings.ASR_ENABLE_REALTIME_PUNC: + await run_sync(get_global_punc_realtime_model, settings.DEVICE) + await run_sync(get_global_vad_model, settings.DEVICE) + else: + qwen_model_id = validate_realtime_model_id("qwen3-asr") + await run_sync(runtime_router.warmup_model, qwen_model_id) + + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + self._current_realtime_model = model_id + logger.info("Realtime ASR model switch stable: previous=%s current=%s", previous_model, model_id) + return {"model_id": model_id, "status": "ready"} + except Exception: + # Restore the old route if loading the replacement failed. + try: + if model_id == "funasr-paraformer-streaming": + unload_funasr_models() + unload_global_punc_realtime_model() + if previous_model == "funasr-paraformer-streaming": + await run_sync(get_funasr_streaming_model) + await run_sync(get_funasr_offline_model) + if settings.ASR_ENABLE_REALTIME_PUNC: + await run_sync(get_global_punc_realtime_model, settings.DEVICE) + else: + qwen_model_id = validate_realtime_model_id("qwen3-asr") + await run_sync(get_runtime_router().warmup_model, qwen_model_id) + except Exception: + logger.exception("Failed to restore realtime ASR model after switch failure") + raise + finally: + runtime_router.end_model_switch() + self._switching_realtime_model = False async def _cleanup_expired_sessions(self) -> None: ttl_sec = max(int(getattr(settings, "REALTIME_SESSION_RESUME_TTL_SEC", 120) or 0), 0) @@ -3312,6 +3399,7 @@ class Qwen3ASRService: async def handle_connection(self, websocket: WebSocket, task_id: str) -> None: await websocket.accept() logger.info("[%s] Qwen3 websocket connected", task_id) + self._active_websocket_connections += 1 ctx = ConnectionContext() session_id = task_id @@ -3326,6 +3414,14 @@ class Qwen3ASRService: msg_type = data.get("type", "") if msg_type == "start": + if self._switching_realtime_model: + await self._send_error( + websocket, + "识别模型正在切换,请等待切换完成后再开始。", + task_id, + "MODEL_SWITCHING", + ) + continue if ctx.state != ConnectionState.READY: await self._send_error(websocket, "识别已在进行中", task_id, "INVALID_STATE") continue @@ -3376,6 +3472,7 @@ class Qwen3ASRService: "UNSUPPORTED_MODEL", ) continue + self._current_realtime_model = model_id context, hotwords = _split_realtime_hotwords( payload.get("context", "") ) @@ -3801,6 +3898,7 @@ class Qwen3ASRService: await self._send_error(websocket, str(exc), task_id) finally: await self._detach_session(session_id, keep_for_resume=keep_for_resume) + self._active_websocket_connections = max(0, self._active_websocket_connections - 1) logger.info("[%s] Connection closed", task_id) async def _stop( diff --git a/test_web/index.html b/test_web/index.html index adf62f8..90905d2 100644 --- a/test_web/index.html +++ b/test_web/index.html @@ -3916,13 +3916,66 @@ ${transcript} function syncLiveProtocolOptions() { state.liveAsrModel = $("liveAsrModel").value || "qwen3-asr"; const useFunASR = state.liveAsrModel === "funasr-paraformer-streaming"; + // Keep protocol-specific controls visible only when the selected backend reads them. + $("liveMinPartialSec").closest(".field").hidden = useFunASR; + ["liveStreamChunkSec", "liveUnfixedChunkNum", "liveUnfixedTokenNum"].forEach((id) => { + $(id).closest(".field").hidden = useFunASR; + }); + $("liveEnableNativePartialStream").closest(".switch").hidden = useFunASR; + $("liveEnableRealtimeLongform").closest(".switch").hidden = useFunASR; + $("livePreRollMs").closest(".field").querySelector("label").textContent = + "项目 VAD 句首预读缓冲(pre_roll_ms)"; $("liveProtocolBadge").textContent = useFunASR ? "FunASR 流式识别" : "Qwen3-ASR"; $("liveProtocolNote").textContent = useFunASR - ? "使用 Paraformer 在线模型的 600ms 分块与会话 cache;句子定稿后仍通过现有 sentences 事件返回。" - : "使用现有 Qwen3-ASR 实时路线;模型切换只影响 ASR 推理,WebSocket 返回结构保持一致。"; + ? "Paraformer 固定按 600ms 分块;静音收段、项目 VAD 预读和说话人选项仍生效。" + : "Qwen3-ASR 使用可调解码块和 partial 参数;静音收段、VAD 预读和说话人选项仍生效。"; $("liveQwenParams").hidden = false; } + async function switchLiveAsrModel() { + const selector = $("liveAsrModel"); + const previousModel = selector.dataset.readyModel || "qwen3-asr"; + const nextModel = selector.value || "qwen3-asr"; + if (nextModel === previousModel) { + syncLiveProtocolOptions(); + return; + } + if (state.liveSocket || state.liveStartedAt) { + selector.value = previousModel; + syncLiveProtocolOptions(); + setStatus($("liveStatus"), "请先结束实时会议再切换模型", "warn"); + return; + } + + state.liveModelSwitching = true; + selector.disabled = true; + $("liveStartBtn").disabled = true; + $("liveProtocolNote").textContent = "正在释放旧模型并加载、预热新模型,请稍候……"; + try { + const result = await requestJson("/ws/v1/asr/model", { + method: "POST", + headers: headers(), + body: JSON.stringify({ model_id: nextModel }), + }); + if (result.status !== "ready" || result.model_id !== nextModel) { + throw new Error("服务端未确认模型已就绪。"); + } + selector.dataset.readyModel = nextModel; + state.liveAsrModel = nextModel; + syncLiveProtocolOptions(); + setStatus($("liveStatus"), "模型已切换并预热完成", "ok"); + } catch (error) { + selector.value = previousModel; + state.liveAsrModel = previousModel; + syncLiveProtocolOptions(); + setStatus($("liveStatus"), error.message || "模型切换失败", "err"); + } finally { + state.liveModelSwitching = false; + selector.disabled = false; + $("liveStartBtn").disabled = Boolean(state.liveSocket); + } + } + function buildQwenRealtimeStartPayload(sampleRate) { const manualSessionId = $("liveSessionId").value.trim(); const sessionId = manualSessionId @@ -4282,10 +4335,10 @@ ${transcript} } function setLiveRunning(running) { - $("liveStartBtn").disabled = running; + $("liveStartBtn").disabled = running || Boolean(state.liveModelSwitching); $("liveStopBtn").disabled = !running; $("liveSimulateDisconnectBtn").disabled = !running; - $("liveAsrModel").disabled = running; + $("liveAsrModel").disabled = running || Boolean(state.liveModelSwitching); $("liveMicState").textContent = running ? "麦克风采集中" : "麦克风空闲"; $("liveMicStateSummary").textContent = running ? "采集中" : "空闲"; } @@ -4448,6 +4501,7 @@ ${transcript} } async function startRealtimeMeeting() { + if (state.liveModelSwitching) throw new Error("模型正在切换和预热,请稍后开始测试。"); if (state.liveSocket) throw new Error("实时会议已经在进行中。"); clearLiveTranscript(); @@ -4956,7 +5010,7 @@ ${transcript} renderLiveSessions(); }); $("liveWsPath").addEventListener("input", syncLiveProtocolOptions); - $("liveAsrModel").addEventListener("change", syncLiveProtocolOptions); + $("liveAsrModel").addEventListener("change", () => switchLiveAsrModel()); $("copyLiveJsonBtn").addEventListener("click", async () => { await navigator.clipboard.writeText(JSON.stringify(state.liveMessages || [], null, 2)); }); @@ -5046,6 +5100,7 @@ ${transcript} await navigator.clipboard.writeText(JSON.stringify(state.lastJson || {}, null, 2)); }); syncLiveProtocolOptions(); + $("liveAsrModel").dataset.readyModel = $("liveAsrModel").value || "qwen3-asr"; syncSpeakerOptions(); clearLiveTranscript(); renderLiveSessions();