加入 FunASR 实时识别路线,测试页的“实时识别模型”下拉框现在可以选

main
Bifang 2026-10-10 16:24:05 +08:00
parent ca1ec42ea8
commit a0f2ae1000
6 changed files with 371 additions and 29 deletions

View File

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
"""Shared model loader for FunASR's native Paraformer streaming checkpoint."""
import logging
import threading
from typing import Any
from funasr import AutoModel
from app.core.config import settings
from app.core.device import detect_device
from app.infrastructure import resolve_model_path
logger = logging.getLogger(__name__)
FUNASR_STREAMING_MODEL_ID = (
"iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online"
)
_model: Any = None
_model_lock = threading.Lock()
_inference_lock = threading.Lock()
def get_streaming_model() -> Any:
"""Load one process-wide model instance; per-stream caches live in the caller."""
global _model
if _model is None:
with _model_lock:
if _model is None:
model_path = resolve_model_path(FUNASR_STREAMING_MODEL_ID)
device = detect_device(settings.DEVICE)
kwargs = dict(settings.FUNASR_AUTOMODEL_KWARGS)
kwargs["disable_update"] = settings.FUNASR_DISABLE_UPDATE
logger.info(
"Loading FunASR streaming ASR model: model=%s device=%s disable_update=%s",
model_path,
device,
settings.FUNASR_DISABLE_UPDATE,
)
_model = AutoModel(model=model_path, device=device, **kwargs)
logger.info("FunASR streaming ASR model loaded: %s", model_path)
return _model
def get_inference_lock() -> threading.Lock:
"""Protect mutable AutoModel generate configuration across WebSocket tasks."""
return _inference_lock

View File

@ -39,6 +39,22 @@ _VAD_ASSETS = (
),
)
_REALTIME_ASR_ASSETS = (
ModelAsset(
source="modelscope",
model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online",
description="FunASR Paraformer Streaming ASR",
required_patterns=(
"configuration.json",
"config.yaml",
"model.pt",
"tokens.json",
"am.mvn",
),
min_total_size_bytes=800_000_000,
),
)
_DIARIZATION_ASSETS = (
ModelAsset(
source="modelscope",
@ -88,6 +104,7 @@ def get_download_modelscope_assets() -> list[ModelAsset]:
"""Return the full static ModelScope export set used by predownload/export."""
return _dedupe_assets([
*_VAD_ASSETS,
*_REALTIME_ASR_ASSETS,
*_DIARIZATION_ASSETS,
])
@ -98,7 +115,7 @@ def get_runtime_required_modelscope_assets(
) -> list[ModelAsset]:
"""Return ModelScope assets required by the current runtime plan."""
_ = include_realtime_punc
return _dedupe_assets([*_VAD_ASSETS, *_DIARIZATION_ASSETS])
return _dedupe_assets([*_VAD_ASSETS, *_REALTIME_ASR_ASSETS, *_DIARIZATION_ASSETS])
def _dedupe_assets(assets: list[ModelAsset]) -> list[ModelAsset]:

View File

@ -25,6 +25,11 @@ from app.core.executor import run_sync
from app.core.text_cleanup import deduplicate_asr_text
from app.services.asr.model_selection import validate_realtime_model_id
from app.services.asr.qwen3_engine import Qwen3ASREngine, Qwen3StreamingState
from app.services.asr.funasr_streaming import (
FUNASR_STREAMING_MODEL_ID,
get_inference_lock as get_funasr_streaming_inference_lock,
get_streaming_model as get_funasr_streaming_model,
)
from app.services.asr.engines.global_models import get_global_vad_model, get_vad_inference_lock
from app.services.realtime_speaker_clusterer import get_realtime_speaker_clusterer
from app.services.asr.runtime import RuntimeEngineLease, get_runtime_router
@ -244,6 +249,12 @@ class ConnectionContext:
params: Dict[str, Any] = field(default_factory=dict)
engine_lease: Optional[RuntimeEngineLease] = None
engine: Optional[Qwen3ASREngine] = None
# FunASR uses the same cached streaming model but isolates mutable decoder
# cache and not-yet-complete audio chunks per WebSocket turn.
funasr_model: Any = None
funasr_cache: Dict[str, Any] = field(default_factory=dict)
funasr_pending_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
funasr_transcript: RealtimeTranscriptAssembler = field(default_factory=RealtimeTranscriptAssembler)
pre_roll_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
pre_roll_start_sample: int = 0
audio_samples_received: int = 0
@ -997,7 +1008,20 @@ class Qwen3ASRService:
) -> tuple[List[Dict[str, Any]], List[int]]:
return get_realtime_speaker_clusterer().cluster_records(records)
async def _ensure_engine(self, ctx: ConnectionContext) -> Qwen3ASREngine:
def _uses_funasr_streaming(self, ctx: ConnectionContext) -> bool:
return ctx.params.get("model_id", "qwen3-asr") == "funasr-paraformer-streaming"
async def _ensure_engine(self, ctx: ConnectionContext) -> Optional[Qwen3ASREngine]:
if self._uses_funasr_streaming(ctx):
if ctx.funasr_model is None:
ctx.funasr_model = await run_sync(get_funasr_streaming_model)
logger.info(
"[%s] Using FunASR streaming model: %s",
ctx.diagnostic_session_id,
FUNASR_STREAMING_MODEL_ID,
)
return None
if ctx.engine is not None:
return ctx.engine
@ -1018,6 +1042,135 @@ class Qwen3ASRService:
ctx.engine = engine
return engine
async def _emit_funasr_partial(
self,
websocket: WebSocket,
ctx: ConnectionContext,
task_id: str,
text: str,
) -> None:
"""Publish FunASR's newly decoded text using the existing sentence schema."""
if not text.strip():
return
start_sample = int(ctx.segment_buffer_start_sample)
end_sample = int(ctx.segment_buffer_start_sample + ctx.segment_audio_buffer.size)
sentence_payload = self._build_tencent_sentence(
{
"index": ctx.segment_index,
"start_ms": int(start_sample / 16),
"end_ms": int(end_sample / 16),
"text": text,
"speaker_id": -1,
"sentence_type": 0,
},
sentence_type=0,
sentence_text=text,
)
await self._send_json_safe(
websocket,
ctx,
{
"type": "sentences",
"code": 0,
"voice_id": task_id,
"final": 0,
"result": {
"slice_type": 2,
"index": int(ctx.segment_index),
"voice_text_str": text,
},
"sentences": [sentence_payload],
},
)
def _generate_funasr_chunk(
self,
ctx: ConnectionContext,
audio: np.ndarray,
*,
is_final: bool,
) -> str:
"""Run one native Paraformer chunk using this session's cache."""
chunk_size = [0, 10, 5]
with get_funasr_streaming_inference_lock():
results = ctx.funasr_model.generate(
input=np.asarray(audio, dtype=np.float32),
cache=ctx.funasr_cache,
is_final=is_final,
chunk_size=chunk_size,
encoder_chunk_look_back=4,
decoder_chunk_look_back=1,
batch_size=1,
fs=16000,
)
if not results:
return ""
parts: List[str] = []
for result in results if isinstance(results, (list, tuple)) else [results]:
if isinstance(result, dict):
text = result.get("text", "")
else:
text = getattr(result, "text", "")
text = str(text or "").strip()
if text:
parts.append(text)
return "".join(parts)
async def _push_funasr_stream_audio(
self,
ctx: ConnectionContext,
audio: np.ndarray,
) -> str:
"""Feed 600 ms chunks and return newly appended provisional text."""
if audio.size:
ctx.funasr_pending_audio = np.concatenate(
(ctx.funasr_pending_audio, np.asarray(audio, dtype=np.float32))
)
# Decode each complete chunk immediately for low-latency provisional text.
# At commit, replay the VAD-clipped segment once to mark its true last
# chunk is_final=True and produce the authoritative final transcript.
stride = 10 * 960
new_text = ""
while ctx.funasr_pending_audio.size >= stride:
chunk = ctx.funasr_pending_audio[:stride]
ctx.funasr_pending_audio = ctx.funasr_pending_audio[stride:]
chunk_text = await run_sync(self._generate_funasr_chunk, ctx, chunk, is_final=False)
if chunk_text:
before = ctx.funasr_transcript.text()
ctx.funasr_transcript.push(chunk_text)
after = ctx.funasr_transcript.text()
if after.startswith(before):
new_text += after[len(before):]
elif after != before:
# A model-side revision is displayed as the latest suffix;
# the final event replaces the entire provisional segment.
new_text += after
return new_text
async def _finish_funasr_stream(
self,
ctx: ConnectionContext,
finalized_audio: np.ndarray,
) -> str:
"""Replay the clipped segment with the final flag on its true last chunk."""
audio = np.asarray(finalized_audio, dtype=np.float32)
ctx.funasr_cache = {}
ctx.funasr_pending_audio = np.array([], dtype=np.float32)
ctx.funasr_transcript = RealtimeTranscriptAssembler()
stride = 10 * 960
for start in range(0, int(audio.size), stride):
end = min(start + stride, int(audio.size))
chunk_text = await run_sync(
self._generate_funasr_chunk,
ctx,
audio[start:end],
is_final=end == int(audio.size),
)
if chunk_text:
ctx.funasr_transcript.push(chunk_text)
return ctx.funasr_transcript.text()
def _has_voice(self, audio: np.ndarray) -> bool:
if audio.size == 0:
return False
@ -1420,12 +1573,21 @@ class Qwen3ASRService:
async def _push_realtime_stream_audio(
self,
ctx: ConnectionContext,
engine: Qwen3ASREngine,
engine: Optional[Qwen3ASREngine],
audio: np.ndarray,
) -> None:
ctx.last_stream_push_ms = 0.0
if audio.size == 0:
return
if self._uses_funasr_streaming(ctx):
push_started = time.perf_counter()
try:
await self._push_funasr_stream_audio(ctx, audio)
finally:
ctx.last_stream_push_ms = (time.perf_counter() - push_started) * 1000.0
return
if engine is None:
return
if not self._should_use_native_partial_stream(ctx, engine):
return
@ -1618,6 +1780,9 @@ class Qwen3ASRService:
ctx.last_partial_decode_samples = 0
ctx.realtime_stream_state = None
ctx.realtime_stream_failure_logged = False
ctx.funasr_cache = {}
ctx.funasr_pending_audio = np.array([], dtype=np.float32)
ctx.funasr_transcript = RealtimeTranscriptAssembler()
def _prefer_segment_text(
self,
@ -2432,6 +2597,7 @@ class Qwen3ASRService:
) -> None:
commit_started = time.perf_counter()
engine = await self._ensure_engine(ctx)
funasr_streaming = self._uses_funasr_streaming(ctx)
current_audio = np.asarray(ctx.segment_audio_buffer, dtype=np.float32)
finalized_audio = current_audio
@ -2446,11 +2612,20 @@ class Qwen3ASRService:
elif reason == "silence" and self._enable_realtime_vad_split(ctx):
finalized_audio, carry_audio = await self._split_silence_audio_by_vad(current_audio)
has_native_stream_snapshot = (
carry_audio.size == 0
not funasr_streaming
and carry_audio.size == 0
and self._should_use_native_partial_stream(ctx, engine)
and ctx.realtime_stream_state is not None
)
if carry_audio.size == 0:
funasr_final_inference_ms = 0.0
if funasr_streaming:
# Replay only the VAD-clipped region so trailing silence or a next
# utterance already present in the network frame cannot enter the final text.
funasr_final_started = time.perf_counter()
stream_text = await self._finish_funasr_stream(ctx, finalized_audio)
funasr_final_inference_ms = (time.perf_counter() - funasr_final_started) * 1000.0
stream_language = self._infer_text_language(stream_text)
elif carry_audio.size == 0:
stream_text, stream_language = await self._finish_realtime_stream_text(ctx, engine)
if stream_text.strip():
update_observed_text = (
@ -2471,7 +2646,13 @@ class Qwen3ASRService:
ctx.realtime_stream_state = None
final_transcribe_started = time.perf_counter()
if self._enable_realtime_longform(ctx):
if funasr_streaming:
segment_text = self._normalize_output_text(
stream_text,
ctx,
enable_itn=False,
)
elif self._enable_realtime_longform(ctx):
segment_text = await self._transcribe_audio_longform_text(
engine,
ctx,
@ -2485,7 +2666,10 @@ class Qwen3ASRService:
finalized_audio,
final_pass=True,
)
final_transcribe_ms = (time.perf_counter() - final_transcribe_started) * 1000.0
final_transcribe_ms = (
(time.perf_counter() - final_transcribe_started) * 1000.0
+ funasr_final_inference_ms
)
segment_text = self._sanitize_candidate_text(segment_text)
if not segment_text.strip():
segment_text = self._prefer_segment_text(
@ -2508,7 +2692,13 @@ class Qwen3ASRService:
duration_sec=max(duration_sec, 0.1),
explicit_language=ctx.params.get("language"),
)
if suspicious and finalized_audio.size > 0 and self._enable_realtime_refine(ctx):
if (
not funasr_streaming
and engine is not None
and suspicious
and finalized_audio.size > 0
and self._enable_realtime_refine(ctx)
):
refined_text = await self._refine_segment_text(
engine,
ctx,
@ -2638,11 +2828,18 @@ class Qwen3ASRService:
self._reset_partial_state(ctx)
if ctx.sentence_active and carry_audio.size > 0:
await self._push_realtime_stream_audio(
ctx,
engine,
np.asarray(carry_audio, dtype=np.float32),
)
if funasr_streaming:
funasr_delta = await self._push_funasr_stream_audio(
ctx,
np.asarray(carry_audio, dtype=np.float32),
)
await self._emit_funasr_partial(websocket, ctx, task_id, funasr_delta)
else:
await self._push_realtime_stream_audio(
ctx,
engine,
np.asarray(carry_audio, dtype=np.float32),
)
async def _commit_vad_boundary(
self,
@ -2768,7 +2965,17 @@ class Qwen3ASRService:
payload.get("enable_speaker_identification", False),
)
)
model_id = str(payload.get("model_id", "qwen3-asr") or "qwen3-asr").strip()
if model_id not in {"qwen3-asr", "funasr-paraformer-streaming"}:
await self._send_error(
websocket,
f"Unsupported realtime model: {model_id}",
task_id,
"UNSUPPORTED_MODEL",
)
continue
ctx.params = {
"model_id": model_id,
"format": payload.get("format", "pcm"),
"sample_rate": payload.get("sample_rate", 16000),
"language": payload.get("language"),
@ -2942,20 +3149,31 @@ class Qwen3ASRService:
start_sample=audio_start_sample,
)
engine = await self._ensure_engine(ctx)
await self._push_realtime_stream_audio(
ctx,
engine,
np.asarray(ctx.segment_audio_buffer, dtype=np.float32),
)
if self._uses_funasr_streaming(ctx):
funasr_delta = await self._push_funasr_stream_audio(
ctx,
np.asarray(ctx.segment_audio_buffer, dtype=np.float32),
)
await self._emit_funasr_partial(websocket, ctx, task_id, funasr_delta)
else:
await self._push_realtime_stream_audio(
ctx,
engine,
np.asarray(ctx.segment_audio_buffer, dtype=np.float32),
)
else:
self._append_turn_audio(ctx, audio, has_voice=energy_voice)
engine = await self._ensure_engine(ctx)
await self._push_realtime_stream_audio(ctx, engine, audio)
if self._uses_funasr_streaming(ctx):
funasr_delta = await self._push_funasr_stream_audio(ctx, audio)
await self._emit_funasr_partial(websocket, ctx, task_id, funasr_delta)
else:
await self._push_realtime_stream_audio(ctx, engine, audio)
if not ctx.sentence_active:
continue
if self._should_decode_turn_partial(ctx):
if not self._uses_funasr_streaming(ctx) and self._should_decode_turn_partial(ctx):
partial_decode_started = time.perf_counter()
has_native_stream_snapshot = (
self._should_use_native_partial_stream(ctx, engine)

View File

@ -13,6 +13,7 @@ from typing import Optional
from modelscope.hub.snapshot_download import snapshot_download as ms_snapshot_download
from app.core.config import settings
from app.services.asr.model_capabilities import (
ModelAsset,
get_all_qwen_modelscope_assets,
get_camplusplus_replacement_paths,
get_download_modelscope_assets,
@ -49,6 +50,37 @@ def check_model_exists(model_id: str, source: str = "modelscope") -> tuple[bool,
return False, ""
def _check_model_asset_complete(asset: ModelAsset) -> tuple[bool, str]:
"""Check declared files and minimum size so partial caches get re-downloaded."""
model_path = _get_cache_path(asset.model_id, source="modelscope")
if not model_path.is_dir():
return False, "directory_missing"
missing = [pattern for pattern in asset.required_patterns if not any(model_path.glob(pattern))]
if missing:
return False, f"missing_files={','.join(missing)}"
alternatives = asset.alternative_required_patterns
if alternatives and not any(
all(any(model_path.glob(pattern)) for pattern in group)
for group in alternatives
):
return False, "alternative_model_files_missing"
if asset.min_total_size_bytes:
total_size = sum(
item.stat().st_size
for item in model_path.rglob("*")
if item.is_file()
)
if total_size < asset.min_total_size_bytes:
return False, f"too_small={total_size}"
if not asset.required_patterns and not alternatives and not any(model_path.iterdir()):
return False, "directory_empty"
return True, str(model_path)
def check_all_models() -> list[tuple[str, str, str, Optional[str]]]:
"""检查所有模型是否存在
@ -60,12 +92,12 @@ def check_all_models() -> list[tuple[str, str, str, Optional[str]]]:
qwen_assets = _get_qwen_modelscope_assets()
for asset in ms_assets:
exists, _ = check_model_exists(asset.model_id, source="modelscope")
exists, _ = _check_model_asset_complete(asset)
if not exists:
missing.append((asset.model_id, asset.description, "modelscope", asset.revision))
for asset in qwen_assets:
exists, _ = check_model_exists(asset.model_id, source="modelscope")
exists, _ = _check_model_asset_complete(asset)
if not exists:
missing.append((asset.model_id, asset.description, "modelscope", asset.revision))

View File

@ -29,6 +29,18 @@ class ModelAsset:
MODEL_ASSETS: tuple[ModelAsset, ...] = (
ModelAsset(
model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online",
description="FunASR Paraformer Streaming ASR",
required_patterns=(
"configuration.json",
"config.yaml",
"model.pt",
"tokens.json",
"am.mvn",
),
min_total_size_bytes=800_000_000,
),
ModelAsset(
model_id="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",
description="VAD",

View File

@ -2178,6 +2178,13 @@
<label for="liveLanguage">识别语言(language)</label>
<input id="liveLanguage" type="text" placeholder="留空自动识别,例如 zh / en">
</div>
<div class="field">
<label for="liveAsrModel">实时识别模型</label>
<select id="liveAsrModel">
<option value="qwen3-asr" selected>Qwen3-ASR(现有实时路线)</option>
<option value="funasr-paraformer-streaming">FunASR Paraformer Streaming(在线模型)</option>
</select>
</div>
<label class="switch">数字归一化 <input id="liveEnableItn" type="checkbox" checked></label>
<label class="switch">复用上次会话 <input id="liveReuseSessionId" type="checkbox" checked></label>
</div>
@ -2749,6 +2756,7 @@
liveStartedAt: null,
liveTaskId: "",
liveSessionId: "",
liveAsrModel: "qwen3-asr",
liveTranscript: [],
liveMessages: [],
liveChunkCount: 0,
@ -3901,9 +3909,12 @@ ${transcript}
}
function syncLiveProtocolOptions() {
$("liveProtocolBadge").textContent = "Tencent-like 协议";
$("liveProtocolNote").textContent =
"当前返回结构对齐腾讯实时 speaker demo,主消费 voice_id/start/sentences/end/error。";
state.liveAsrModel = $("liveAsrModel").value || "qwen3-asr";
const useFunASR = state.liveAsrModel === "funasr-paraformer-streaming";
$("liveProtocolBadge").textContent = useFunASR ? "FunASR 流式识别" : "Qwen3-ASR";
$("liveProtocolNote").textContent = useFunASR
? "使用 Paraformer 在线模型的 600ms 分块与会话 cache;句子定稿后仍通过现有 sentences 事件返回。"
: "使用现有 Qwen3-ASR 实时路线;模型切换只影响 ASR 推理,WebSocket 返回结构保持一致。";
$("liveQwenParams").hidden = false;
}
@ -3919,6 +3930,7 @@ ${transcript}
payload: {
format: "pcm",
sample_rate: sampleRate,
model_id: state.liveAsrModel,
session_id: sessionId,
language: $("liveLanguage").value.trim() || null,
context: $("liveContext").value.trim(),
@ -4253,6 +4265,7 @@ ${transcript}
$("liveStartBtn").disabled = running;
$("liveStopBtn").disabled = !running;
$("liveSimulateDisconnectBtn").disabled = !running;
$("liveAsrModel").disabled = running;
$("liveMicState").textContent = running ? "麦克风采集中" : "麦克风空闲";
$("liveMicStateSummary").textContent = running ? "采集中" : "空闲";
}
@ -4283,10 +4296,11 @@ ${transcript}
if (existingPartialIndex >= 0) {
const existingPartial = state.liveTranscript[existingPartialIndex];
existingPartial.label = label;
// Backend partial events are complete, revisable snapshots for this segment.
// Replace the draft wholesale so a changed hypothesis cannot leave stale text
// frozen in the bubble; the matching final event replaces this same segment.
existingPartial.text = safeText;
// Qwen sends revisable snapshots; native FunASR sends chunk deltas.
// Both are reconciled under the segment id and replaced by its final sentence.
existingPartial.text = state.liveAsrModel === "funasr-paraformer-streaming"
? joinLiveTranscriptText(existingPartial.text, safeText)
: safeText;
existingPartial.speaker = speaker;
existingPartial.speakerPending = speakerPending;
existingPartial.segmentIndex = segmentIndex;
@ -4922,6 +4936,7 @@ ${transcript}
renderLiveSessions();
});
$("liveWsPath").addEventListener("input", syncLiveProtocolOptions);
$("liveAsrModel").addEventListener("change", syncLiveProtocolOptions);
$("copyLiveJsonBtn").addEventListener("click", async () => {
await navigator.clipboard.writeText(JSON.stringify(state.liveMessages || [], null, 2));
});