调整页面,选择 FunASR 时,页面会隐藏 Qwen 专属的 partial、解码块和长段重转写参数;公共的静音收段、VAD 预读、说话人选项仍显示。页面也明确标出 Paraformer 固定使用 600ms 分块

main
Bifang 2026-10-13 09:58:36 +08:00
parent 629a9217e6
commit 3d0a7f7705
9 changed files with 335 additions and 20 deletions

View File

@ -6,9 +6,10 @@ import time
import uuid
from typing import Optional
from fastapi import APIRouter, WebSocket
from fastapi import APIRouter, HTTPException, Request, WebSocket
from ...services.qwen3_websocket_asr import Qwen3ASRService
from ...core.security import validate_token
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/ws/v1/asr", tags=["WebSocket ASR"])
@ -45,3 +46,22 @@ async def qwen_asr_websocket(
if task_id is None:
task_id = str(uuid.uuid4())[:8]
await _qwen3_service.handle_connection(websocket, task_id)
@router.post("/model")
async def prepare_realtime_model(request: Request) -> dict[str, str]:
"""Preload a selected realtime model before the browser opens its audio stream."""
authorized, message = validate_token(request)
if not authorized:
raise HTTPException(status_code=401, detail=message)
try:
payload = await request.json()
model_id = str(payload.get("model_id") or "").strip()
return await _qwen3_service.prepare_realtime_model(model_id)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except RuntimeError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except Exception as exc:
logger.exception("Realtime model preparation failed")
raise HTTPException(status_code=500, detail=f"模型切换失败:{exc}") from exc

View File

@ -157,3 +157,10 @@ def get_global_punc_realtime_model(device: str):
def get_punc_realtime_inference_lock():
"""获取实时PUNC模型推理锁(线程安全)"""
return _punc_realtime_inference_lock
def unload_global_punc_realtime_model() -> None:
"""Release the auxiliary realtime punctuation model during route switching."""
global _global_punc_realtime_model
with _punc_realtime_model_lock:
_global_punc_realtime_model = None

View File

@ -28,6 +28,14 @@ _offline_model_lock = threading.Lock()
_offline_inference_lock = threading.Lock()
def unload_models() -> None:
"""Release shared Paraformer instances before switching realtime routes."""
global _model, _offline_model
with _model_lock, _offline_model_lock:
_model = None
_offline_model = None
def get_streaming_model() -> Any:
"""Load one process-wide model instance; per-stream caches live in the caller."""
global _model

View File

@ -212,6 +212,14 @@ class Qwen3ASREngine(BaseASREngine):
"GPU runtimes and CPU QwenASR Rust."
)
def close(self) -> None:
"""Release backend-owned workers and model references when the engine is evicted."""
model = getattr(self, "model", None)
close = getattr(model, "close", None)
if callable(close):
close()
self.model = None
def _load_rust_backend(
self,
model_path: str,

View File

@ -197,7 +197,7 @@ class _VLLMGenerateBatcher:
self._engine_lock = engine_lock
self._batch_wait_sec = max(0, batch_wait_ms) / 1000.0
self._max_batch_size = max(1, max_batch_size)
self._requests: queue.Queue[_GenerateRequest] = queue.Queue()
self._requests: queue.Queue[Optional[_GenerateRequest]] = queue.Queue()
self._pending: Optional[_GenerateRequest] = None
self._worker = threading.Thread(
target=self._run,
@ -237,6 +237,8 @@ class _VLLMGenerateBatcher:
def _run(self) -> None:
while True:
first = self._take_request()
if first is None:
return
collected = [first]
prompt_count = len(first.prompts)
deadline = time.monotonic() + self._batch_wait_sec
@ -251,6 +253,9 @@ class _VLLMGenerateBatcher:
request = self._requests.get(timeout=remaining)
except queue.Empty:
break
if request is None:
self._pending = None
return
if prompt_count + len(request.prompts) > self._max_batch_size:
self._pending = request
break
@ -264,6 +269,14 @@ class _VLLMGenerateBatcher:
for group in compatible.values():
self._generate_group(group)
def close(self) -> None:
"""Stop the daemon batcher so its model references can be collected."""
self._requests.put(None)
if self._worker.is_alive() and threading.current_thread() is not self._worker:
self._worker.join(timeout=5)
if not self._worker.is_alive():
self._llm = None
def _generate_group(self, requests: list[_GenerateRequest]) -> None:
prompts = [prompt for request in requests for prompt in request.prompts]
queued_at = min(request.queued_at for request in requests)
@ -476,6 +489,16 @@ class Qwen3VLLMBackend:
self._decode_max_batch_size,
)
def close(self) -> None:
"""Stop decode work and release vLLM model references before route changes."""
self._generate_batcher.close()
aligner_close = getattr(self._forced_aligner, "close", None)
if callable(aligner_close):
aligner_close()
self._forced_aligner = None
self._llm = None
self._tokenizer = None
def _get_forced_aligner_gpu_memory_utilization(self) -> float:
configured = _resolve_forced_aligner_gpu_memory_utilization(self._gpu_memory_utilization)
logger.info(

View File

@ -39,6 +39,26 @@ class LocalEnginePool(Generic[T]):
def warmup(self) -> None:
self._ensure_state()
@property
def is_idle(self) -> bool:
"""A pool is idle when every initialized engine is back in its queue."""
state = self._state
return state is None or state.queue.qsize() == self._size
def close(self) -> None:
"""Close all idle engines before dropping this pool."""
state = self._state
if state is None:
return
if not self.is_idle:
raise RuntimeError("Cannot close an engine pool while an engine is leased")
for _ in range(self._size):
engine = state.queue.get_nowait()
close = getattr(engine, "close", None)
if callable(close):
close()
self._state = None
async def acquire(self) -> T:
state = self._ensure_state()
return await state.queue.get()

View File

@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
import gc
import threading
from dataclasses import dataclass
from enum import Enum
@ -75,6 +76,22 @@ class RuntimeRouter:
self._shared_limits: dict[tuple[RuntimeFamily, str], asyncio.Semaphore] = {}
self._pool_lock = threading.Lock()
self._loaded_model_ids: set[str] = set()
self._active_model_leases: dict[str, int] = {}
self._model_switching = False
self._model_switch_ready = threading.Event()
self._model_switch_ready.set()
def begin_model_switch(self) -> None:
"""Temporarily gate new offline/Qwen leases while a realtime model is replaced."""
with self._pool_lock:
self._model_switching = True
self._model_switch_ready.clear()
def end_model_switch(self) -> None:
"""Resume queued engine acquisitions after the replacement is stable."""
with self._pool_lock:
self._model_switching = False
self._model_switch_ready.set()
def resolve_model_id(self, model_id: Optional[str]) -> str:
if model_id:
@ -151,6 +168,37 @@ class RuntimeRouter:
def get_loaded_model_ids(self) -> list[str]:
return sorted(self._loaded_model_ids)
def unload_model(self, model_id: str) -> bool:
"""Drop an idle cached engine to release memory during realtime switching."""
family = self._resolve_family(model_id)
key = (family, model_id)
with self._pool_lock:
if self._active_model_leases.get(model_id, 0):
return False
if family == RuntimeFamily.QWEN_VLLM:
semaphore = self._shared_limits.get(key)
limit = max(1, int(settings.QWEN_VLLM_SHARED_CONCURRENCY))
if semaphore is not None and semaphore._value < limit:
return False
engine = self._shared_engines.get(key)
close = getattr(engine, "close", None)
if callable(close):
close()
self._shared_limits.pop(key, None)
self._shared_engines.pop(key, None)
else:
pool = self._pools.get(key)
if pool is not None and not pool.is_idle:
return False
if pool is not None:
pool.close()
self._pools.pop(key, None)
self._loaded_model_ids.discard(model_id)
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
return True
def get_memory_usage(self) -> dict[str, object]:
memory_info: dict[str, object] = {
"model_list": self.get_loaded_model_ids(),
@ -172,19 +220,47 @@ class RuntimeRouter:
async def acquire_engine(self, model_id: Optional[str] = None) -> RuntimeEngineLease:
resolved_model_id = self.resolve_model_id(model_id)
family = self._resolve_family(resolved_model_id)
if family == RuntimeFamily.QWEN_VLLM:
engine, semaphore = self._get_shared_engine(family, resolved_model_id)
await semaphore.acquire()
return RuntimeEngineLease(
engine=engine,
release_callback=semaphore.release,
)
pool = self._create_pool(family, resolved_model_id)
engine = await pool.acquire()
return RuntimeEngineLease(
engine=engine,
release_callback=lambda: pool.release(engine),
)
# Reserve before awaiting a queue/semaphore so unload cannot race a
# request that has selected the engine but has not acquired it yet.
while True:
with self._pool_lock:
if not self._model_switching:
self._active_model_leases[resolved_model_id] = (
self._active_model_leases.get(resolved_model_id, 0) + 1
)
break
await asyncio.to_thread(self._model_switch_ready.wait)
def release_reservation() -> None:
with self._pool_lock:
active = self._active_model_leases.get(resolved_model_id, 0)
if active <= 1:
self._active_model_leases.pop(resolved_model_id, None)
else:
self._active_model_leases[resolved_model_id] = active - 1
try:
if family == RuntimeFamily.QWEN_VLLM:
engine, semaphore = self._get_shared_engine(family, resolved_model_id)
await semaphore.acquire()
async def release_shared() -> None:
semaphore.release()
release_reservation()
return RuntimeEngineLease(engine=engine, release_callback=release_shared)
pool = self._create_pool(family, resolved_model_id)
engine = await pool.acquire()
async def release_pooled() -> None:
await pool.release(engine)
release_reservation()
return RuntimeEngineLease(engine=engine, release_callback=release_pooled)
except Exception:
release_reservation()
raise
async def run_offline(self, request: OfflineASRRequest) -> ASRFullResult:
async with await self.acquire_engine(request.model_id) as engine:

View File

@ -4,6 +4,7 @@
from __future__ import annotations
import asyncio
import gc
import io
import json
import logging
@ -17,6 +18,7 @@ from typing import Any, Dict, List, Optional
import numpy as np
import soundfile as sf
import torch
from fastapi import WebSocket, WebSocketDisconnect
from app.core.config import settings
@ -37,11 +39,13 @@ from app.services.asr.funasr_streaming import (
get_offline_inference_lock as get_funasr_offline_inference_lock,
get_offline_model as get_funasr_offline_model,
get_streaming_model as get_funasr_streaming_model,
unload_models as unload_funasr_models,
)
from app.services.asr.engines.global_models import (
get_global_punc_realtime_model,
get_global_vad_model,
get_punc_realtime_inference_lock,
unload_global_punc_realtime_model,
get_vad_inference_lock,
)
from app.services.realtime_speaker_clusterer import get_realtime_speaker_clusterer
@ -356,6 +360,89 @@ class Qwen3ASRService:
def __init__(self) -> None:
self._sessions: Dict[str, SessionEntry] = {}
self._session_lock = asyncio.Lock()
self._model_switch_lock = asyncio.Lock()
self._active_websocket_connections = 0
self._switching_realtime_model = False
self._current_realtime_model = "qwen3-asr"
async def prepare_realtime_model(self, model_id: str) -> Dict[str, str]:
"""Swap and warm the selected realtime ASR route before capture begins."""
if model_id not in {"qwen3-asr", "funasr-paraformer-streaming"}:
raise ValueError(f"Unsupported realtime model: {model_id}")
async with self._model_switch_lock:
if self._active_websocket_connections and self._current_realtime_model != model_id:
raise RuntimeError("请先结束当前实时会议,再切换识别模型。")
self._switching_realtime_model = True
previous_model = self._current_realtime_model
runtime_router = get_runtime_router()
runtime_router.begin_model_switch()
try:
if previous_model != model_id:
# A disconnected resumable session may still retain the old
# model lease; changing models invalidates that resume context.
async with self._session_lock:
resumable_contexts = [session.ctx for session in self._sessions.values()]
self._sessions.clear()
for session_ctx in resumable_contexts:
await self._release_session_resources(session_ctx)
resumable_contexts.clear()
session_ctx = None
if previous_model == model_id:
# A same-model request still verifies that lazy model loading
# completed before the frontend enables microphone capture.
if model_id == "funasr-paraformer-streaming":
await run_sync(get_funasr_streaming_model)
await run_sync(get_funasr_offline_model)
if settings.ASR_ENABLE_REALTIME_PUNC:
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
await run_sync(get_global_vad_model, settings.DEVICE)
else:
qwen_model_id = validate_realtime_model_id("qwen3-asr")
await run_sync(runtime_router.warmup_model, qwen_model_id)
elif previous_model == "funasr-paraformer-streaming":
unload_funasr_models()
unload_global_punc_realtime_model()
else:
qwen_model_id = validate_realtime_model_id("qwen3-asr")
if not runtime_router.unload_model(qwen_model_id):
raise RuntimeError("Qwen 模型仍有任务占用,暂时无法释放;请稍后重试。")
if model_id == "funasr-paraformer-streaming":
await run_sync(get_funasr_streaming_model)
await run_sync(get_funasr_offline_model)
if settings.ASR_ENABLE_REALTIME_PUNC:
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
await run_sync(get_global_vad_model, settings.DEVICE)
else:
qwen_model_id = validate_realtime_model_id("qwen3-asr")
await run_sync(runtime_router.warmup_model, qwen_model_id)
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
self._current_realtime_model = model_id
logger.info("Realtime ASR model switch stable: previous=%s current=%s", previous_model, model_id)
return {"model_id": model_id, "status": "ready"}
except Exception:
# Restore the old route if loading the replacement failed.
try:
if model_id == "funasr-paraformer-streaming":
unload_funasr_models()
unload_global_punc_realtime_model()
if previous_model == "funasr-paraformer-streaming":
await run_sync(get_funasr_streaming_model)
await run_sync(get_funasr_offline_model)
if settings.ASR_ENABLE_REALTIME_PUNC:
await run_sync(get_global_punc_realtime_model, settings.DEVICE)
else:
qwen_model_id = validate_realtime_model_id("qwen3-asr")
await run_sync(get_runtime_router().warmup_model, qwen_model_id)
except Exception:
logger.exception("Failed to restore realtime ASR model after switch failure")
raise
finally:
runtime_router.end_model_switch()
self._switching_realtime_model = False
async def _cleanup_expired_sessions(self) -> None:
ttl_sec = max(int(getattr(settings, "REALTIME_SESSION_RESUME_TTL_SEC", 120) or 0), 0)
@ -3312,6 +3399,7 @@ class Qwen3ASRService:
async def handle_connection(self, websocket: WebSocket, task_id: str) -> None:
await websocket.accept()
logger.info("[%s] Qwen3 websocket connected", task_id)
self._active_websocket_connections += 1
ctx = ConnectionContext()
session_id = task_id
@ -3326,6 +3414,14 @@ class Qwen3ASRService:
msg_type = data.get("type", "")
if msg_type == "start":
if self._switching_realtime_model:
await self._send_error(
websocket,
"识别模型正在切换,请等待切换完成后再开始。",
task_id,
"MODEL_SWITCHING",
)
continue
if ctx.state != ConnectionState.READY:
await self._send_error(websocket, "识别已在进行中", task_id, "INVALID_STATE")
continue
@ -3376,6 +3472,7 @@ class Qwen3ASRService:
"UNSUPPORTED_MODEL",
)
continue
self._current_realtime_model = model_id
context, hotwords = _split_realtime_hotwords(
payload.get("context", "")
)
@ -3801,6 +3898,7 @@ class Qwen3ASRService:
await self._send_error(websocket, str(exc), task_id)
finally:
await self._detach_session(session_id, keep_for_resume=keep_for_resume)
self._active_websocket_connections = max(0, self._active_websocket_connections - 1)
logger.info("[%s] Connection closed", task_id)
async def _stop(

View File

@ -3916,13 +3916,66 @@ ${transcript}
function syncLiveProtocolOptions() {
state.liveAsrModel = $("liveAsrModel").value || "qwen3-asr";
const useFunASR = state.liveAsrModel === "funasr-paraformer-streaming";
// Keep protocol-specific controls visible only when the selected backend reads them.
$("liveMinPartialSec").closest(".field").hidden = useFunASR;
["liveStreamChunkSec", "liveUnfixedChunkNum", "liveUnfixedTokenNum"].forEach((id) => {
$(id).closest(".field").hidden = useFunASR;
});
$("liveEnableNativePartialStream").closest(".switch").hidden = useFunASR;
$("liveEnableRealtimeLongform").closest(".switch").hidden = useFunASR;
$("livePreRollMs").closest(".field").querySelector("label").textContent =
"项目 VAD 句首预读缓冲(pre_roll_ms)";
$("liveProtocolBadge").textContent = useFunASR ? "FunASR 流式识别" : "Qwen3-ASR";
$("liveProtocolNote").textContent = useFunASR
? "使用 Paraformer 在线模型的 600ms 分块与会话 cache;句子定稿后仍通过现有 sentences 事件返回。"
: "使用现有 Qwen3-ASR 实时路线;模型切换只影响 ASR 推理,WebSocket 返回结构保持一致。";
? "Paraformer 固定按 600ms 分块;静音收段、项目 VAD 预读和说话人选项仍生效。"
: "Qwen3-ASR 使用可调解码块和 partial 参数;静音收段、VAD 预读和说话人选项仍生效。";
$("liveQwenParams").hidden = false;
}
async function switchLiveAsrModel() {
const selector = $("liveAsrModel");
const previousModel = selector.dataset.readyModel || "qwen3-asr";
const nextModel = selector.value || "qwen3-asr";
if (nextModel === previousModel) {
syncLiveProtocolOptions();
return;
}
if (state.liveSocket || state.liveStartedAt) {
selector.value = previousModel;
syncLiveProtocolOptions();
setStatus($("liveStatus"), "请先结束实时会议再切换模型", "warn");
return;
}
state.liveModelSwitching = true;
selector.disabled = true;
$("liveStartBtn").disabled = true;
$("liveProtocolNote").textContent = "正在释放旧模型并加载、预热新模型,请稍候……";
try {
const result = await requestJson("/ws/v1/asr/model", {
method: "POST",
headers: headers(),
body: JSON.stringify({ model_id: nextModel }),
});
if (result.status !== "ready" || result.model_id !== nextModel) {
throw new Error("服务端未确认模型已就绪。");
}
selector.dataset.readyModel = nextModel;
state.liveAsrModel = nextModel;
syncLiveProtocolOptions();
setStatus($("liveStatus"), "模型已切换并预热完成", "ok");
} catch (error) {
selector.value = previousModel;
state.liveAsrModel = previousModel;
syncLiveProtocolOptions();
setStatus($("liveStatus"), error.message || "模型切换失败", "err");
} finally {
state.liveModelSwitching = false;
selector.disabled = false;
$("liveStartBtn").disabled = Boolean(state.liveSocket);
}
}
function buildQwenRealtimeStartPayload(sampleRate) {
const manualSessionId = $("liveSessionId").value.trim();
const sessionId = manualSessionId
@ -4282,10 +4335,10 @@ ${transcript}
}
function setLiveRunning(running) {
$("liveStartBtn").disabled = running;
$("liveStartBtn").disabled = running || Boolean(state.liveModelSwitching);
$("liveStopBtn").disabled = !running;
$("liveSimulateDisconnectBtn").disabled = !running;
$("liveAsrModel").disabled = running;
$("liveAsrModel").disabled = running || Boolean(state.liveModelSwitching);
$("liveMicState").textContent = running ? "麦克风采集中" : "麦克风空闲";
$("liveMicStateSummary").textContent = running ? "采集中" : "空闲";
}
@ -4448,6 +4501,7 @@ ${transcript}
}
async function startRealtimeMeeting() {
if (state.liveModelSwitching) throw new Error("模型正在切换和预热,请稍后开始测试。");
if (state.liveSocket) throw new Error("实时会议已经在进行中。");
clearLiveTranscript();
@ -4956,7 +5010,7 @@ ${transcript}
renderLiveSessions();
});
$("liveWsPath").addEventListener("input", syncLiveProtocolOptions);
$("liveAsrModel").addEventListener("change", syncLiveProtocolOptions);
$("liveAsrModel").addEventListener("change", () => switchLiveAsrModel());
$("copyLiveJsonBtn").addEventListener("click", async () => {
await navigator.clipboard.writeText(JSON.stringify(state.liveMessages || [], null, 2));
});
@ -5046,6 +5100,7 @@ ${transcript}
await navigator.clipboard.writeText(JSON.stringify(state.lastJson || {}, null, 2));
});
syncLiveProtocolOptions();
$("liveAsrModel").dataset.readyModel = $("liveAsrModel").value || "qwen3-asr";
syncSpeakerOptions();
clearLiveTranscript();
renderLiveSessions();