test/app/services/asr/r2t2_runtime.py

133 lines
4.2 KiB
Python

# -*- coding: utf-8 -*-
"""Lazy lifecycle wrapper for the R2T2 vLLM streaming model."""
from __future__ import annotations
import gc
import logging
import threading
from pathlib import Path
from typing import Any, Optional
import numpy as np
from app.core.accelerator import get_accelerator_info
from app.core.config import settings
from app.core.device import detect_device
from app.services.asr.qwen3_engine import Qwen3ASREngine
logger = logging.getLogger(__name__)
R2T2_MODEL_ID = "netease-youdao/Confucius4-R2T2"
_model: Optional[Any] = None
_engine: Optional[Qwen3ASREngine] = None
_model_lock = threading.RLock()
_inference_lock = threading.Lock()
def get_r2t2_model_path() -> Path:
"""Resolve the downloaded ModelScope checkpoint for the selected route."""
return Path(settings.MODELSCOPE_PATH) / R2T2_MODEL_ID
def get_r2t2_model() -> Any:
"""Load R2T2 weights with the app's vLLM backend and upstream stream decoder."""
global _model, _engine
with _model_lock:
if _model is not None:
return _model
accelerator = get_accelerator_info()
device = detect_device(settings.DEVICE)
if not accelerator.is_gpu or not device.startswith("cuda"):
raise RuntimeError("Confucius4-R2T2 streaming requires a CUDA GPU and vLLM")
model_path = get_r2t2_model_path()
if not model_path.is_dir():
raise FileNotFoundError(
f"Confucius4-R2T2 checkpoint is missing: {model_path}; "
"run the model download check before starting the service"
)
# Keep R2T2's vLLM-only decoder lazy so CPU-only installations do not
# import its runtime until this route is selected.
from app.services.asr.r2t2_model import R2T2ASRModel
logger.info(
"Loading Confucius4-R2T2: model=%s device=%s",
model_path,
device,
)
_engine = Qwen3ASREngine(
model_path=str(model_path),
device=device,
max_model_len=16384,
max_new_tokens=64,
)
backend = _engine.model
from types import SimpleNamespace
_model = R2T2ASRModel(
backend=_engine.backend,
model=backend._llm,
processor=SimpleNamespace(tokenizer=backend._tokenizer),
sampling_params=backend._sampling_params,
max_new_tokens=64,
)
# Retain the engine so its batching worker and vLLM references close on switch.
_model._engine_owner = _engine
# Match the upstream server's eager chunk warmup so the first live
# utterance does not pay vLLM's initial scheduler/model setup cost.
warmup_state = _model.init_streaming_state(
context="",
language=None,
unfixed_chunk_num=0,
unfixed_token_num=1,
chunk_size_sec=0.16,
)
with _inference_lock:
_model.streaming_transcribe_no_reset(
np.zeros((5120,), dtype=np.float32),
warmup_state,
4,
)
_model.finish_streaming_transcribe_no_reset(warmup_state, 4)
logger.info("Confucius4-R2T2 streaming model loaded")
return _model
def unload_r2t2_model() -> bool:
"""Release the R2T2 vLLM instance when switching realtime model routes."""
global _model, _engine
# Use the same lock order as get_r2t2_model (model, then inference) to
# prevent a model switch from deadlocking with a first-use warmup.
with _model_lock:
if not _inference_lock.acquire(blocking=False):
return False
try:
model = _model
if model is None:
return True
engine = _engine
if engine is not None:
engine.close()
_model = None
_engine = None
finally:
_inference_lock.release()
del model
gc.collect()
try:
import torch
if torch.cuda.is_available():
torch.cuda.empty_cache()
except ImportError:
pass
return True
def get_r2t2_inference_lock() -> threading.Lock:
"""Serialize calls because each R2T2 stream mutates shared vLLM model state."""
return _inference_lock