133 lines
4.2 KiB
Python
133 lines
4.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""Lazy lifecycle wrapper for the R2T2 vLLM streaming model."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import gc
|
|
import logging
|
|
import threading
|
|
from pathlib import Path
|
|
from typing import Any, Optional
|
|
|
|
import numpy as np
|
|
|
|
from app.core.accelerator import get_accelerator_info
|
|
from app.core.config import settings
|
|
from app.core.device import detect_device
|
|
from app.services.asr.qwen3_engine import Qwen3ASREngine
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
R2T2_MODEL_ID = "netease-youdao/Confucius4-R2T2"
|
|
_model: Optional[Any] = None
|
|
_engine: Optional[Qwen3ASREngine] = None
|
|
_model_lock = threading.RLock()
|
|
_inference_lock = threading.Lock()
|
|
|
|
|
|
def get_r2t2_model_path() -> Path:
|
|
"""Resolve the downloaded ModelScope checkpoint for the selected route."""
|
|
return Path(settings.MODELSCOPE_PATH) / R2T2_MODEL_ID
|
|
|
|
|
|
def get_r2t2_model() -> Any:
|
|
"""Load R2T2 weights with the app's vLLM backend and upstream stream decoder."""
|
|
global _model, _engine
|
|
with _model_lock:
|
|
if _model is not None:
|
|
return _model
|
|
|
|
accelerator = get_accelerator_info()
|
|
device = detect_device(settings.DEVICE)
|
|
if not accelerator.is_gpu or not device.startswith("cuda"):
|
|
raise RuntimeError("Confucius4-R2T2 streaming requires a CUDA GPU and vLLM")
|
|
|
|
model_path = get_r2t2_model_path()
|
|
if not model_path.is_dir():
|
|
raise FileNotFoundError(
|
|
f"Confucius4-R2T2 checkpoint is missing: {model_path}; "
|
|
"run the model download check before starting the service"
|
|
)
|
|
|
|
# Keep R2T2's vLLM-only decoder lazy so CPU-only installations do not
|
|
# import its runtime until this route is selected.
|
|
from app.services.asr.r2t2_model import R2T2ASRModel
|
|
|
|
logger.info(
|
|
"Loading Confucius4-R2T2: model=%s device=%s",
|
|
model_path,
|
|
device,
|
|
)
|
|
_engine = Qwen3ASREngine(
|
|
model_path=str(model_path),
|
|
device=device,
|
|
max_model_len=16384,
|
|
max_new_tokens=64,
|
|
)
|
|
backend = _engine.model
|
|
from types import SimpleNamespace
|
|
|
|
_model = R2T2ASRModel(
|
|
backend=_engine.backend,
|
|
model=backend._llm,
|
|
processor=SimpleNamespace(tokenizer=backend._tokenizer),
|
|
sampling_params=backend._sampling_params,
|
|
max_new_tokens=64,
|
|
)
|
|
# Retain the engine so its batching worker and vLLM references close on switch.
|
|
_model._engine_owner = _engine
|
|
# Match the upstream server's eager chunk warmup so the first live
|
|
# utterance does not pay vLLM's initial scheduler/model setup cost.
|
|
warmup_state = _model.init_streaming_state(
|
|
context="",
|
|
language=None,
|
|
unfixed_chunk_num=0,
|
|
unfixed_token_num=1,
|
|
chunk_size_sec=0.16,
|
|
)
|
|
with _inference_lock:
|
|
_model.streaming_transcribe_no_reset(
|
|
np.zeros((5120,), dtype=np.float32),
|
|
warmup_state,
|
|
4,
|
|
)
|
|
_model.finish_streaming_transcribe_no_reset(warmup_state, 4)
|
|
logger.info("Confucius4-R2T2 streaming model loaded")
|
|
return _model
|
|
|
|
|
|
def unload_r2t2_model() -> bool:
|
|
"""Release the R2T2 vLLM instance when switching realtime model routes."""
|
|
global _model, _engine
|
|
# Use the same lock order as get_r2t2_model (model, then inference) to
|
|
# prevent a model switch from deadlocking with a first-use warmup.
|
|
with _model_lock:
|
|
if not _inference_lock.acquire(blocking=False):
|
|
return False
|
|
try:
|
|
model = _model
|
|
if model is None:
|
|
return True
|
|
engine = _engine
|
|
if engine is not None:
|
|
engine.close()
|
|
_model = None
|
|
_engine = None
|
|
finally:
|
|
_inference_lock.release()
|
|
del model
|
|
gc.collect()
|
|
try:
|
|
import torch
|
|
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
except ImportError:
|
|
pass
|
|
return True
|
|
|
|
|
|
def get_r2t2_inference_lock() -> threading.Lock:
|
|
"""Serialize calls because each R2T2 stream mutates shared vLLM model state."""
|
|
return _inference_lock
|