diff --git a/app/services/asr/funasr_streaming.py b/app/services/asr/funasr_streaming.py index af7c824..69cebfe 100644 --- a/app/services/asr/funasr_streaming.py +++ b/app/services/asr/funasr_streaming.py @@ -16,10 +16,16 @@ logger = logging.getLogger(__name__) FUNASR_STREAMING_MODEL_ID = ( "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online" ) +FUNASR_OFFLINE_MODEL_ID = ( + "iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404" +) _model: Any = None _model_lock = threading.Lock() _inference_lock = threading.Lock() +_offline_model: Any = None +_offline_model_lock = threading.Lock() +_offline_inference_lock = threading.Lock() def get_streaming_model() -> Any: @@ -46,3 +52,29 @@ def get_streaming_model() -> Any: def get_inference_lock() -> threading.Lock: """Protect mutable AutoModel generate configuration across WebSocket tasks.""" return _inference_lock + + +def get_offline_model() -> Any: + """Load FunASR's contextual second-pass model for final text and hotwords.""" + global _offline_model + if _offline_model is None: + with _offline_model_lock: + if _offline_model is None: + model_path = resolve_model_path(FUNASR_OFFLINE_MODEL_ID) + device = detect_device(settings.DEVICE) + kwargs = dict(settings.FUNASR_AUTOMODEL_KWARGS) + kwargs["disable_update"] = settings.FUNASR_DISABLE_UPDATE + logger.info( + "Loading FunASR offline final ASR model: model=%s device=%s disable_update=%s", + model_path, + device, + settings.FUNASR_DISABLE_UPDATE, + ) + _offline_model = AutoModel(model=model_path, device=device, **kwargs) + logger.info("FunASR offline final ASR model loaded: %s", model_path) + return _offline_model + + +def get_offline_inference_lock() -> threading.Lock: + """Serialize access to the shared offline AutoModel instance.""" + return _offline_inference_lock diff --git a/app/services/asr/model_capabilities.py b/app/services/asr/model_capabilities.py index 6f53847..4a355e1 100644 --- a/app/services/asr/model_capabilities.py +++ b/app/services/asr/model_capabilities.py @@ -55,6 +55,32 @@ _REALTIME_ASR_ASSETS = ( ), ) +_REALTIME_FINAL_ASR_ASSETS = ( + ModelAsset( + source="modelscope", + model_id="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404", + description="FunASR Contextual Offline Final ASR", + required_patterns=( + "configuration.json", + "config.yaml", + "model.pt", + "tokens.json", + "am.mvn", + ), + min_total_size_bytes=800_000_000, + ), +) + +_REALTIME_PUNC_ASSETS = ( + ModelAsset( + source="modelscope", + model_id=settings.PUNC_REALTIME_MODEL, + description="FunASR Realtime Punctuation", + required_patterns=("configuration.json", "config.yaml", "model.pt", "tokens.json"), + min_total_size_bytes=200_000_000, + ), +) + _DIARIZATION_ASSETS = ( ModelAsset( source="modelscope", @@ -105,6 +131,8 @@ def get_download_modelscope_assets() -> list[ModelAsset]: return _dedupe_assets([ *_VAD_ASSETS, *_REALTIME_ASR_ASSETS, + *_REALTIME_FINAL_ASR_ASSETS, + *_REALTIME_PUNC_ASSETS, *_DIARIZATION_ASSETS, ]) @@ -114,8 +142,15 @@ def get_runtime_required_modelscope_assets( include_realtime_punc: bool, ) -> list[ModelAsset]: """Return ModelScope assets required by the current runtime plan.""" - _ = include_realtime_punc - return _dedupe_assets([*_VAD_ASSETS, *_REALTIME_ASR_ASSETS, *_DIARIZATION_ASSETS]) + assets = [ + *_VAD_ASSETS, + *_REALTIME_ASR_ASSETS, + *_REALTIME_FINAL_ASR_ASSETS, + *_DIARIZATION_ASSETS, + ] + if include_realtime_punc: + assets.extend(_REALTIME_PUNC_ASSETS) + return _dedupe_assets(assets) def _dedupe_assets(assets: list[ModelAsset]) -> list[ModelAsset]: diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index 845092b..357f9df 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -23,14 +23,27 @@ from app.core.config import settings from app.core.exceptions import create_error_response from app.core.executor import run_sync from app.core.text_cleanup import deduplicate_asr_text +from app.core.hotword_resolver import ( + apply_hotword_rules, + format_hotword_prompt_context, + parse_hotwords, +) from app.services.asr.model_selection import validate_realtime_model_id from app.services.asr.qwen3_engine import Qwen3ASREngine, Qwen3StreamingState from app.services.asr.funasr_streaming import ( + FUNASR_OFFLINE_MODEL_ID, FUNASR_STREAMING_MODEL_ID, get_inference_lock as get_funasr_streaming_inference_lock, + get_offline_inference_lock as get_funasr_offline_inference_lock, + get_offline_model as get_funasr_offline_model, get_streaming_model as get_funasr_streaming_model, ) -from app.services.asr.engines.global_models import get_global_vad_model, get_vad_inference_lock +from app.services.asr.engines.global_models import ( + get_global_punc_realtime_model, + get_global_vad_model, + get_punc_realtime_inference_lock, + get_vad_inference_lock, +) from app.services.realtime_speaker_clusterer import get_realtime_speaker_clusterer from app.services.asr.runtime import RuntimeEngineLease, get_runtime_router from app.services.speaker_registry import get_speaker_registry_service @@ -61,6 +74,16 @@ def _normalize_transcript_chunk_text(text: str) -> str: return _remove_repeated_chars(collapsed, 8) +def _split_realtime_hotwords(context: Any) -> tuple[str, list[dict[str, Any]]]: + """Read test-page hotwords from the established WebSocket context field.""" + raw_context = str(context or "") + marker = "[[ASR_HOTWORDS]]\n" + if marker not in raw_context: + return raw_context, [] + prompt_context, _, hotword_text = raw_context.partition(marker) + return prompt_context.strip(), parse_hotwords(hotword_text) + + def _is_opening_punctuation(ch: str) -> bool: return ch in '([{"\'“‘' @@ -252,7 +275,11 @@ class ConnectionContext: # FunASR uses the same cached streaming model but isolates mutable decoder # cache and not-yet-complete audio chunks per WebSocket turn. funasr_model: Any = None + funasr_offline_model: Any = None + funasr_punc_model: Any = None funasr_cache: Dict[str, Any] = field(default_factory=dict) + # The punctuation model keeps context across finalized utterances in a session. + funasr_punc_cache: Dict[str, Any] = field(default_factory=dict) funasr_pending_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) funasr_transcript: RealtimeTranscriptAssembler = field(default_factory=RealtimeTranscriptAssembler) pre_roll_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) @@ -1020,6 +1047,18 @@ class Qwen3ASRService: ctx.diagnostic_session_id, FUNASR_STREAMING_MODEL_ID, ) + if ctx.funasr_offline_model is None: + ctx.funasr_offline_model = await run_sync(get_funasr_offline_model) + logger.info( + "[%s] Using FunASR final model: %s", + ctx.diagnostic_session_id, + FUNASR_OFFLINE_MODEL_ID, + ) + if settings.ASR_ENABLE_REALTIME_PUNC and ctx.funasr_punc_model is None: + ctx.funasr_punc_model = await run_sync( + get_global_punc_realtime_model, + settings.DEVICE, + ) return None if ctx.engine is not None: @@ -1102,6 +1141,8 @@ class Qwen3ASRService: decoder_chunk_look_back=1, batch_size=1, fs=16000, + # Paraformer's contextual decoding accepts words, not Qwen prompt text. + hotword=self._funasr_hotword_text(ctx), ) if not results: return "" @@ -1116,6 +1157,66 @@ class Qwen3ASRService: parts.append(text) return "".join(parts) + @staticmethod + def _funasr_hotword_text(ctx: ConnectionContext) -> str: + """Convert the shared hotword schema to FunASR's whitespace word list.""" + return " ".join( + str(entry.get("text", "")).strip() + for entry in parse_hotwords(ctx.params.get("hotwords")) + if str(entry.get("text", "")).strip() + ) + + def _finalize_funasr_text(self, ctx: ConnectionContext, text: str) -> str: + """Match FunASR's final-text order: punctuation, then hotword correction.""" + finalized = str(text or "").strip() + if not finalized: + return finalized + + if settings.ASR_ENABLE_REALTIME_PUNC and ctx.funasr_punc_model is not None: + try: + with get_punc_realtime_inference_lock(): + results = ctx.funasr_punc_model.generate( + input=finalized, + cache=ctx.funasr_punc_cache, + ) + result = results[0] if isinstance(results, (list, tuple)) and results else results + if isinstance(result, dict) and result.get("text"): + finalized = str(result["text"]).strip() + except Exception: + # Keep the recognized words if punctuation inference fails for this segment. + logger.exception("[%s] FunASR punctuation inference failed", ctx.diagnostic_session_id) + + hotwords = ctx.params.get("hotwords") + if hotwords: + finalized, was_applied, matched = apply_hotword_rules(finalized, hotwords) + if was_applied: + logger.info( + "[%s] FunASR hotword correction applied: matched=%s", + ctx.diagnostic_session_id, + matched, + ) + return finalized + + def _transcribe_funasr_final(self, ctx: ConnectionContext, audio: np.ndarray) -> str: + """Run FunASR's offline contextual second pass on the VAD-clipped turn.""" + if ctx.funasr_offline_model is None or audio.size == 0: + return "" + with get_funasr_offline_inference_lock(): + results = ctx.funasr_offline_model.generate( + input=np.asarray(audio, dtype=np.float32), + hotword=self._funasr_hotword_text(ctx) or None, + ) + if not results: + return "" + result_items = results if isinstance(results, (list, tuple)) else [results] + texts: List[str] = [] + for item in result_items: + text = item.get("text", "") if isinstance(item, dict) else getattr(item, "text", "") + text = str(text or "").strip() + if text: + texts.append(text) + return "".join(texts) + async def _push_funasr_stream_audio( self, ctx: ConnectionContext, @@ -2619,10 +2720,14 @@ class Qwen3ASRService: ) funasr_final_inference_ms = 0.0 if funasr_streaming: - # Replay only the VAD-clipped region so trailing silence or a next - # utterance already present in the network frame cannot enter the final text. + # FunASR uses the streaming model for previews and the contextual + # offline model for final text; both operate on the VAD-clipped turn. funasr_final_started = time.perf_counter() - stream_text = await self._finish_funasr_stream(ctx, finalized_audio) + stream_text = await run_sync(self._transcribe_funasr_final, ctx, finalized_audio) + if not stream_text.strip(): + # Preserve the online model's final replay as a fallback if the + # contextual second pass returns no usable text. + stream_text = await self._finish_funasr_stream(ctx, finalized_audio) funasr_final_inference_ms = (time.perf_counter() - funasr_final_started) * 1000.0 stream_language = self._infer_text_language(stream_text) elif carry_audio.size == 0: @@ -2726,6 +2831,10 @@ class Qwen3ASRService: language=segment_language, ) segment_text = self._trim_previous_segment_overlap(ctx, segment_text) + if funasr_streaming: + # Keep CT-Transformer inference off the asyncio event loop so other + # WebSocket sessions can continue sending and decoding audio. + segment_text = await run_sync(self._finalize_funasr_text, ctx, segment_text) is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec) logger.info( @@ -2974,12 +3083,23 @@ class Qwen3ASRService: "UNSUPPORTED_MODEL", ) continue + context, hotwords = _split_realtime_hotwords( + payload.get("context", "") + ) + hotword_context = format_hotword_prompt_context( + hotwords + ) + if model_id == "qwen3-asr" and hotword_context: + # Keep Qwen's existing prompt-based hotword behavior while + # exposing the same hotword field for both realtime models. + context = "\n".join(part for part in (context, hotword_context) if part) ctx.params = { "model_id": model_id, "format": payload.get("format", "pcm"), "sample_rate": payload.get("sample_rate", 16000), "language": payload.get("language"), - "context": payload.get("context", ""), + "context": context, + "hotwords": hotwords, "enable_inverse_text_normalization": payload.get( "enable_inverse_text_normalization", True, diff --git a/scripts/download_models_standalone.py b/scripts/download_models_standalone.py index 8498a38..c6423a9 100644 --- a/scripts/download_models_standalone.py +++ b/scripts/download_models_standalone.py @@ -29,6 +29,24 @@ class ModelAsset: MODEL_ASSETS: tuple[ModelAsset, ...] = ( + ModelAsset( + model_id="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404", + description="FunASR Contextual Offline Final ASR", + required_patterns=( + "configuration.json", + "config.yaml", + "model.pt", + "tokens.json", + "am.mvn", + ), + min_total_size_bytes=800_000_000, + ), + ModelAsset( + model_id="iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727", + description="FunASR Realtime Punctuation", + required_patterns=("configuration.json", "config.yaml", "model.pt", "tokens.json"), + min_total_size_bytes=200_000_000, + ), ModelAsset( model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online", description="FunASR Paraformer Streaming ASR", diff --git a/test_web/index.html b/test_web/index.html index 65284f4..1b59684 100644 --- a/test_web/index.html +++ b/test_web/index.html @@ -2229,6 +2229,11 @@ +
Qwen 将热词加入上下文提示;FunASR 用热词增强离线定稿,并执行热词纠正。
+