From a0f2ae100091ba96a3f179a2b763b405f1e1cf39 Mon Sep 17 00:00:00 2001 From: Bifang <915779419@qq.com> Date: Sat, 10 Oct 2026 16:24:05 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=20FunASR=20=E5=AE=9E?= =?UTF-8?q?=E6=97=B6=E8=AF=86=E5=88=AB=E8=B7=AF=E7=BA=BF=EF=BC=8C=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E9=A1=B5=E7=9A=84=E2=80=9C=E5=AE=9E=E6=97=B6=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E6=A8=A1=E5=9E=8B=E2=80=9D=E4=B8=8B=E6=8B=89=E6=A1=86?= =?UTF-8?q?=E7=8E=B0=E5=9C=A8=E5=8F=AF=E4=BB=A5=E9=80=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/asr/funasr_streaming.py | 48 +++++ app/services/asr/model_capabilities.py | 19 +- app/services/qwen3_websocket_asr.py | 256 +++++++++++++++++++++++-- app/utils/download_models.py | 36 +++- scripts/download_models_standalone.py | 12 ++ test_web/index.html | 29 ++- 6 files changed, 371 insertions(+), 29 deletions(-) create mode 100644 app/services/asr/funasr_streaming.py diff --git a/app/services/asr/funasr_streaming.py b/app/services/asr/funasr_streaming.py new file mode 100644 index 0000000..af7c824 --- /dev/null +++ b/app/services/asr/funasr_streaming.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +"""Shared model loader for FunASR's native Paraformer streaming checkpoint.""" + +import logging +import threading +from typing import Any + +from funasr import AutoModel + +from app.core.config import settings +from app.core.device import detect_device +from app.infrastructure import resolve_model_path + +logger = logging.getLogger(__name__) + +FUNASR_STREAMING_MODEL_ID = ( + "iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online" +) + +_model: Any = None +_model_lock = threading.Lock() +_inference_lock = threading.Lock() + + +def get_streaming_model() -> Any: + """Load one process-wide model instance; per-stream caches live in the caller.""" + global _model + if _model is None: + with _model_lock: + if _model is None: + model_path = resolve_model_path(FUNASR_STREAMING_MODEL_ID) + device = detect_device(settings.DEVICE) + kwargs = dict(settings.FUNASR_AUTOMODEL_KWARGS) + kwargs["disable_update"] = settings.FUNASR_DISABLE_UPDATE + logger.info( + "Loading FunASR streaming ASR model: model=%s device=%s disable_update=%s", + model_path, + device, + settings.FUNASR_DISABLE_UPDATE, + ) + _model = AutoModel(model=model_path, device=device, **kwargs) + logger.info("FunASR streaming ASR model loaded: %s", model_path) + return _model + + +def get_inference_lock() -> threading.Lock: + """Protect mutable AutoModel generate configuration across WebSocket tasks.""" + return _inference_lock diff --git a/app/services/asr/model_capabilities.py b/app/services/asr/model_capabilities.py index 8800128..6f53847 100644 --- a/app/services/asr/model_capabilities.py +++ b/app/services/asr/model_capabilities.py @@ -39,6 +39,22 @@ _VAD_ASSETS = ( ), ) +_REALTIME_ASR_ASSETS = ( + ModelAsset( + source="modelscope", + model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online", + description="FunASR Paraformer Streaming ASR", + required_patterns=( + "configuration.json", + "config.yaml", + "model.pt", + "tokens.json", + "am.mvn", + ), + min_total_size_bytes=800_000_000, + ), +) + _DIARIZATION_ASSETS = ( ModelAsset( source="modelscope", @@ -88,6 +104,7 @@ def get_download_modelscope_assets() -> list[ModelAsset]: """Return the full static ModelScope export set used by predownload/export.""" return _dedupe_assets([ *_VAD_ASSETS, + *_REALTIME_ASR_ASSETS, *_DIARIZATION_ASSETS, ]) @@ -98,7 +115,7 @@ def get_runtime_required_modelscope_assets( ) -> list[ModelAsset]: """Return ModelScope assets required by the current runtime plan.""" _ = include_realtime_punc - return _dedupe_assets([*_VAD_ASSETS, *_DIARIZATION_ASSETS]) + return _dedupe_assets([*_VAD_ASSETS, *_REALTIME_ASR_ASSETS, *_DIARIZATION_ASSETS]) def _dedupe_assets(assets: list[ModelAsset]) -> list[ModelAsset]: diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index 703d7aa..845092b 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -25,6 +25,11 @@ from app.core.executor import run_sync from app.core.text_cleanup import deduplicate_asr_text from app.services.asr.model_selection import validate_realtime_model_id from app.services.asr.qwen3_engine import Qwen3ASREngine, Qwen3StreamingState +from app.services.asr.funasr_streaming import ( + FUNASR_STREAMING_MODEL_ID, + get_inference_lock as get_funasr_streaming_inference_lock, + get_streaming_model as get_funasr_streaming_model, +) from app.services.asr.engines.global_models import get_global_vad_model, get_vad_inference_lock from app.services.realtime_speaker_clusterer import get_realtime_speaker_clusterer from app.services.asr.runtime import RuntimeEngineLease, get_runtime_router @@ -244,6 +249,12 @@ class ConnectionContext: params: Dict[str, Any] = field(default_factory=dict) engine_lease: Optional[RuntimeEngineLease] = None engine: Optional[Qwen3ASREngine] = None + # FunASR uses the same cached streaming model but isolates mutable decoder + # cache and not-yet-complete audio chunks per WebSocket turn. + funasr_model: Any = None + funasr_cache: Dict[str, Any] = field(default_factory=dict) + funasr_pending_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) + funasr_transcript: RealtimeTranscriptAssembler = field(default_factory=RealtimeTranscriptAssembler) pre_roll_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32)) pre_roll_start_sample: int = 0 audio_samples_received: int = 0 @@ -997,7 +1008,20 @@ class Qwen3ASRService: ) -> tuple[List[Dict[str, Any]], List[int]]: return get_realtime_speaker_clusterer().cluster_records(records) - async def _ensure_engine(self, ctx: ConnectionContext) -> Qwen3ASREngine: + def _uses_funasr_streaming(self, ctx: ConnectionContext) -> bool: + return ctx.params.get("model_id", "qwen3-asr") == "funasr-paraformer-streaming" + + async def _ensure_engine(self, ctx: ConnectionContext) -> Optional[Qwen3ASREngine]: + if self._uses_funasr_streaming(ctx): + if ctx.funasr_model is None: + ctx.funasr_model = await run_sync(get_funasr_streaming_model) + logger.info( + "[%s] Using FunASR streaming model: %s", + ctx.diagnostic_session_id, + FUNASR_STREAMING_MODEL_ID, + ) + return None + if ctx.engine is not None: return ctx.engine @@ -1018,6 +1042,135 @@ class Qwen3ASRService: ctx.engine = engine return engine + async def _emit_funasr_partial( + self, + websocket: WebSocket, + ctx: ConnectionContext, + task_id: str, + text: str, + ) -> None: + """Publish FunASR's newly decoded text using the existing sentence schema.""" + if not text.strip(): + return + start_sample = int(ctx.segment_buffer_start_sample) + end_sample = int(ctx.segment_buffer_start_sample + ctx.segment_audio_buffer.size) + sentence_payload = self._build_tencent_sentence( + { + "index": ctx.segment_index, + "start_ms": int(start_sample / 16), + "end_ms": int(end_sample / 16), + "text": text, + "speaker_id": -1, + "sentence_type": 0, + }, + sentence_type=0, + sentence_text=text, + ) + await self._send_json_safe( + websocket, + ctx, + { + "type": "sentences", + "code": 0, + "voice_id": task_id, + "final": 0, + "result": { + "slice_type": 2, + "index": int(ctx.segment_index), + "voice_text_str": text, + }, + "sentences": [sentence_payload], + }, + ) + + def _generate_funasr_chunk( + self, + ctx: ConnectionContext, + audio: np.ndarray, + *, + is_final: bool, + ) -> str: + """Run one native Paraformer chunk using this session's cache.""" + chunk_size = [0, 10, 5] + with get_funasr_streaming_inference_lock(): + results = ctx.funasr_model.generate( + input=np.asarray(audio, dtype=np.float32), + cache=ctx.funasr_cache, + is_final=is_final, + chunk_size=chunk_size, + encoder_chunk_look_back=4, + decoder_chunk_look_back=1, + batch_size=1, + fs=16000, + ) + if not results: + return "" + parts: List[str] = [] + for result in results if isinstance(results, (list, tuple)) else [results]: + if isinstance(result, dict): + text = result.get("text", "") + else: + text = getattr(result, "text", "") + text = str(text or "").strip() + if text: + parts.append(text) + return "".join(parts) + + async def _push_funasr_stream_audio( + self, + ctx: ConnectionContext, + audio: np.ndarray, + ) -> str: + """Feed 600 ms chunks and return newly appended provisional text.""" + if audio.size: + ctx.funasr_pending_audio = np.concatenate( + (ctx.funasr_pending_audio, np.asarray(audio, dtype=np.float32)) + ) + + # Decode each complete chunk immediately for low-latency provisional text. + # At commit, replay the VAD-clipped segment once to mark its true last + # chunk is_final=True and produce the authoritative final transcript. + stride = 10 * 960 + new_text = "" + while ctx.funasr_pending_audio.size >= stride: + chunk = ctx.funasr_pending_audio[:stride] + ctx.funasr_pending_audio = ctx.funasr_pending_audio[stride:] + chunk_text = await run_sync(self._generate_funasr_chunk, ctx, chunk, is_final=False) + if chunk_text: + before = ctx.funasr_transcript.text() + ctx.funasr_transcript.push(chunk_text) + after = ctx.funasr_transcript.text() + if after.startswith(before): + new_text += after[len(before):] + elif after != before: + # A model-side revision is displayed as the latest suffix; + # the final event replaces the entire provisional segment. + new_text += after + return new_text + + async def _finish_funasr_stream( + self, + ctx: ConnectionContext, + finalized_audio: np.ndarray, + ) -> str: + """Replay the clipped segment with the final flag on its true last chunk.""" + audio = np.asarray(finalized_audio, dtype=np.float32) + ctx.funasr_cache = {} + ctx.funasr_pending_audio = np.array([], dtype=np.float32) + ctx.funasr_transcript = RealtimeTranscriptAssembler() + stride = 10 * 960 + for start in range(0, int(audio.size), stride): + end = min(start + stride, int(audio.size)) + chunk_text = await run_sync( + self._generate_funasr_chunk, + ctx, + audio[start:end], + is_final=end == int(audio.size), + ) + if chunk_text: + ctx.funasr_transcript.push(chunk_text) + return ctx.funasr_transcript.text() + def _has_voice(self, audio: np.ndarray) -> bool: if audio.size == 0: return False @@ -1420,12 +1573,21 @@ class Qwen3ASRService: async def _push_realtime_stream_audio( self, ctx: ConnectionContext, - engine: Qwen3ASREngine, + engine: Optional[Qwen3ASREngine], audio: np.ndarray, ) -> None: ctx.last_stream_push_ms = 0.0 if audio.size == 0: return + if self._uses_funasr_streaming(ctx): + push_started = time.perf_counter() + try: + await self._push_funasr_stream_audio(ctx, audio) + finally: + ctx.last_stream_push_ms = (time.perf_counter() - push_started) * 1000.0 + return + if engine is None: + return if not self._should_use_native_partial_stream(ctx, engine): return @@ -1618,6 +1780,9 @@ class Qwen3ASRService: ctx.last_partial_decode_samples = 0 ctx.realtime_stream_state = None ctx.realtime_stream_failure_logged = False + ctx.funasr_cache = {} + ctx.funasr_pending_audio = np.array([], dtype=np.float32) + ctx.funasr_transcript = RealtimeTranscriptAssembler() def _prefer_segment_text( self, @@ -2432,6 +2597,7 @@ class Qwen3ASRService: ) -> None: commit_started = time.perf_counter() engine = await self._ensure_engine(ctx) + funasr_streaming = self._uses_funasr_streaming(ctx) current_audio = np.asarray(ctx.segment_audio_buffer, dtype=np.float32) finalized_audio = current_audio @@ -2446,11 +2612,20 @@ class Qwen3ASRService: elif reason == "silence" and self._enable_realtime_vad_split(ctx): finalized_audio, carry_audio = await self._split_silence_audio_by_vad(current_audio) has_native_stream_snapshot = ( - carry_audio.size == 0 + not funasr_streaming + and carry_audio.size == 0 and self._should_use_native_partial_stream(ctx, engine) and ctx.realtime_stream_state is not None ) - if carry_audio.size == 0: + funasr_final_inference_ms = 0.0 + if funasr_streaming: + # Replay only the VAD-clipped region so trailing silence or a next + # utterance already present in the network frame cannot enter the final text. + funasr_final_started = time.perf_counter() + stream_text = await self._finish_funasr_stream(ctx, finalized_audio) + funasr_final_inference_ms = (time.perf_counter() - funasr_final_started) * 1000.0 + stream_language = self._infer_text_language(stream_text) + elif carry_audio.size == 0: stream_text, stream_language = await self._finish_realtime_stream_text(ctx, engine) if stream_text.strip(): update_observed_text = ( @@ -2471,7 +2646,13 @@ class Qwen3ASRService: ctx.realtime_stream_state = None final_transcribe_started = time.perf_counter() - if self._enable_realtime_longform(ctx): + if funasr_streaming: + segment_text = self._normalize_output_text( + stream_text, + ctx, + enable_itn=False, + ) + elif self._enable_realtime_longform(ctx): segment_text = await self._transcribe_audio_longform_text( engine, ctx, @@ -2485,7 +2666,10 @@ class Qwen3ASRService: finalized_audio, final_pass=True, ) - final_transcribe_ms = (time.perf_counter() - final_transcribe_started) * 1000.0 + final_transcribe_ms = ( + (time.perf_counter() - final_transcribe_started) * 1000.0 + + funasr_final_inference_ms + ) segment_text = self._sanitize_candidate_text(segment_text) if not segment_text.strip(): segment_text = self._prefer_segment_text( @@ -2508,7 +2692,13 @@ class Qwen3ASRService: duration_sec=max(duration_sec, 0.1), explicit_language=ctx.params.get("language"), ) - if suspicious and finalized_audio.size > 0 and self._enable_realtime_refine(ctx): + if ( + not funasr_streaming + and engine is not None + and suspicious + and finalized_audio.size > 0 + and self._enable_realtime_refine(ctx) + ): refined_text = await self._refine_segment_text( engine, ctx, @@ -2638,11 +2828,18 @@ class Qwen3ASRService: self._reset_partial_state(ctx) if ctx.sentence_active and carry_audio.size > 0: - await self._push_realtime_stream_audio( - ctx, - engine, - np.asarray(carry_audio, dtype=np.float32), - ) + if funasr_streaming: + funasr_delta = await self._push_funasr_stream_audio( + ctx, + np.asarray(carry_audio, dtype=np.float32), + ) + await self._emit_funasr_partial(websocket, ctx, task_id, funasr_delta) + else: + await self._push_realtime_stream_audio( + ctx, + engine, + np.asarray(carry_audio, dtype=np.float32), + ) async def _commit_vad_boundary( self, @@ -2768,7 +2965,17 @@ class Qwen3ASRService: payload.get("enable_speaker_identification", False), ) ) + model_id = str(payload.get("model_id", "qwen3-asr") or "qwen3-asr").strip() + if model_id not in {"qwen3-asr", "funasr-paraformer-streaming"}: + await self._send_error( + websocket, + f"Unsupported realtime model: {model_id}", + task_id, + "UNSUPPORTED_MODEL", + ) + continue ctx.params = { + "model_id": model_id, "format": payload.get("format", "pcm"), "sample_rate": payload.get("sample_rate", 16000), "language": payload.get("language"), @@ -2942,20 +3149,31 @@ class Qwen3ASRService: start_sample=audio_start_sample, ) engine = await self._ensure_engine(ctx) - await self._push_realtime_stream_audio( - ctx, - engine, - np.asarray(ctx.segment_audio_buffer, dtype=np.float32), - ) + if self._uses_funasr_streaming(ctx): + funasr_delta = await self._push_funasr_stream_audio( + ctx, + np.asarray(ctx.segment_audio_buffer, dtype=np.float32), + ) + await self._emit_funasr_partial(websocket, ctx, task_id, funasr_delta) + else: + await self._push_realtime_stream_audio( + ctx, + engine, + np.asarray(ctx.segment_audio_buffer, dtype=np.float32), + ) else: self._append_turn_audio(ctx, audio, has_voice=energy_voice) engine = await self._ensure_engine(ctx) - await self._push_realtime_stream_audio(ctx, engine, audio) + if self._uses_funasr_streaming(ctx): + funasr_delta = await self._push_funasr_stream_audio(ctx, audio) + await self._emit_funasr_partial(websocket, ctx, task_id, funasr_delta) + else: + await self._push_realtime_stream_audio(ctx, engine, audio) if not ctx.sentence_active: continue - if self._should_decode_turn_partial(ctx): + if not self._uses_funasr_streaming(ctx) and self._should_decode_turn_partial(ctx): partial_decode_started = time.perf_counter() has_native_stream_snapshot = ( self._should_use_native_partial_stream(ctx, engine) diff --git a/app/utils/download_models.py b/app/utils/download_models.py index 0d7b2d1..865310d 100644 --- a/app/utils/download_models.py +++ b/app/utils/download_models.py @@ -13,6 +13,7 @@ from typing import Optional from modelscope.hub.snapshot_download import snapshot_download as ms_snapshot_download from app.core.config import settings from app.services.asr.model_capabilities import ( + ModelAsset, get_all_qwen_modelscope_assets, get_camplusplus_replacement_paths, get_download_modelscope_assets, @@ -49,6 +50,37 @@ def check_model_exists(model_id: str, source: str = "modelscope") -> tuple[bool, return False, "" +def _check_model_asset_complete(asset: ModelAsset) -> tuple[bool, str]: + """Check declared files and minimum size so partial caches get re-downloaded.""" + model_path = _get_cache_path(asset.model_id, source="modelscope") + if not model_path.is_dir(): + return False, "directory_missing" + + missing = [pattern for pattern in asset.required_patterns if not any(model_path.glob(pattern))] + if missing: + return False, f"missing_files={','.join(missing)}" + + alternatives = asset.alternative_required_patterns + if alternatives and not any( + all(any(model_path.glob(pattern)) for pattern in group) + for group in alternatives + ): + return False, "alternative_model_files_missing" + + if asset.min_total_size_bytes: + total_size = sum( + item.stat().st_size + for item in model_path.rglob("*") + if item.is_file() + ) + if total_size < asset.min_total_size_bytes: + return False, f"too_small={total_size}" + + if not asset.required_patterns and not alternatives and not any(model_path.iterdir()): + return False, "directory_empty" + return True, str(model_path) + + def check_all_models() -> list[tuple[str, str, str, Optional[str]]]: """检查所有模型是否存在 @@ -60,12 +92,12 @@ def check_all_models() -> list[tuple[str, str, str, Optional[str]]]: qwen_assets = _get_qwen_modelscope_assets() for asset in ms_assets: - exists, _ = check_model_exists(asset.model_id, source="modelscope") + exists, _ = _check_model_asset_complete(asset) if not exists: missing.append((asset.model_id, asset.description, "modelscope", asset.revision)) for asset in qwen_assets: - exists, _ = check_model_exists(asset.model_id, source="modelscope") + exists, _ = _check_model_asset_complete(asset) if not exists: missing.append((asset.model_id, asset.description, "modelscope", asset.revision)) diff --git a/scripts/download_models_standalone.py b/scripts/download_models_standalone.py index 1f6f2c9..8498a38 100644 --- a/scripts/download_models_standalone.py +++ b/scripts/download_models_standalone.py @@ -29,6 +29,18 @@ class ModelAsset: MODEL_ASSETS: tuple[ModelAsset, ...] = ( + ModelAsset( + model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online", + description="FunASR Paraformer Streaming ASR", + required_patterns=( + "configuration.json", + "config.yaml", + "model.pt", + "tokens.json", + "am.mvn", + ), + min_total_size_bytes=800_000_000, + ), ModelAsset( model_id="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", description="VAD", diff --git a/test_web/index.html b/test_web/index.html index 8ea46c5..65284f4 100644 --- a/test_web/index.html +++ b/test_web/index.html @@ -2178,6 +2178,13 @@ +
+ + +
@@ -2749,6 +2756,7 @@ liveStartedAt: null, liveTaskId: "", liveSessionId: "", + liveAsrModel: "qwen3-asr", liveTranscript: [], liveMessages: [], liveChunkCount: 0, @@ -3901,9 +3909,12 @@ ${transcript} } function syncLiveProtocolOptions() { - $("liveProtocolBadge").textContent = "Tencent-like 协议"; - $("liveProtocolNote").textContent = - "当前返回结构对齐腾讯实时 speaker demo,主消费 voice_id/start/sentences/end/error。"; + state.liveAsrModel = $("liveAsrModel").value || "qwen3-asr"; + const useFunASR = state.liveAsrModel === "funasr-paraformer-streaming"; + $("liveProtocolBadge").textContent = useFunASR ? "FunASR 流式识别" : "Qwen3-ASR"; + $("liveProtocolNote").textContent = useFunASR + ? "使用 Paraformer 在线模型的 600ms 分块与会话 cache;句子定稿后仍通过现有 sentences 事件返回。" + : "使用现有 Qwen3-ASR 实时路线;模型切换只影响 ASR 推理,WebSocket 返回结构保持一致。"; $("liveQwenParams").hidden = false; } @@ -3919,6 +3930,7 @@ ${transcript} payload: { format: "pcm", sample_rate: sampleRate, + model_id: state.liveAsrModel, session_id: sessionId, language: $("liveLanguage").value.trim() || null, context: $("liveContext").value.trim(), @@ -4253,6 +4265,7 @@ ${transcript} $("liveStartBtn").disabled = running; $("liveStopBtn").disabled = !running; $("liveSimulateDisconnectBtn").disabled = !running; + $("liveAsrModel").disabled = running; $("liveMicState").textContent = running ? "麦克风采集中" : "麦克风空闲"; $("liveMicStateSummary").textContent = running ? "采集中" : "空闲"; } @@ -4283,10 +4296,11 @@ ${transcript} if (existingPartialIndex >= 0) { const existingPartial = state.liveTranscript[existingPartialIndex]; existingPartial.label = label; - // Backend partial events are complete, revisable snapshots for this segment. - // Replace the draft wholesale so a changed hypothesis cannot leave stale text - // frozen in the bubble; the matching final event replaces this same segment. - existingPartial.text = safeText; + // Qwen sends revisable snapshots; native FunASR sends chunk deltas. + // Both are reconciled under the segment id and replaced by its final sentence. + existingPartial.text = state.liveAsrModel === "funasr-paraformer-streaming" + ? joinLiveTranscriptText(existingPartial.text, safeText) + : safeText; existingPartial.speaker = speaker; existingPartial.speakerPending = speakerPending; existingPartial.segmentIndex = segmentIndex; @@ -4922,6 +4936,7 @@ ${transcript} renderLiveSessions(); }); $("liveWsPath").addEventListener("input", syncLiveProtocolOptions); + $("liveAsrModel").addEventListener("change", syncLiveProtocolOptions); $("copyLiveJsonBtn").addEventListener("click", async () => { await navigator.clipboard.writeText(JSON.stringify(state.liveMessages || [], null, 2)); });