补齐 FunASR 的定稿流程:在线 Paraformer 输出 partial,上下文离线 Paraformer 负责定稿,随后 CT-Transformer 补标点

main
Bifang 2026-10-10 17:12:08 +08:00
parent a0f2ae1000
commit 826c65c96f
5 changed files with 226 additions and 8 deletions

View File

@ -16,10 +16,16 @@ logger = logging.getLogger(__name__)
FUNASR_STREAMING_MODEL_ID = (
"iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online"
)
FUNASR_OFFLINE_MODEL_ID = (
"iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404"
)
_model: Any = None
_model_lock = threading.Lock()
_inference_lock = threading.Lock()
_offline_model: Any = None
_offline_model_lock = threading.Lock()
_offline_inference_lock = threading.Lock()
def get_streaming_model() -> Any:
@ -46,3 +52,29 @@ def get_streaming_model() -> Any:
def get_inference_lock() -> threading.Lock:
"""Protect mutable AutoModel generate configuration across WebSocket tasks."""
return _inference_lock
def get_offline_model() -> Any:
"""Load FunASR's contextual second-pass model for final text and hotwords."""
global _offline_model
if _offline_model is None:
with _offline_model_lock:
if _offline_model is None:
model_path = resolve_model_path(FUNASR_OFFLINE_MODEL_ID)
device = detect_device(settings.DEVICE)
kwargs = dict(settings.FUNASR_AUTOMODEL_KWARGS)
kwargs["disable_update"] = settings.FUNASR_DISABLE_UPDATE
logger.info(
"Loading FunASR offline final ASR model: model=%s device=%s disable_update=%s",
model_path,
device,
settings.FUNASR_DISABLE_UPDATE,
)
_offline_model = AutoModel(model=model_path, device=device, **kwargs)
logger.info("FunASR offline final ASR model loaded: %s", model_path)
return _offline_model
def get_offline_inference_lock() -> threading.Lock:
"""Serialize access to the shared offline AutoModel instance."""
return _offline_inference_lock

View File

@ -55,6 +55,32 @@ _REALTIME_ASR_ASSETS = (
),
)
_REALTIME_FINAL_ASR_ASSETS = (
ModelAsset(
source="modelscope",
model_id="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404",
description="FunASR Contextual Offline Final ASR",
required_patterns=(
"configuration.json",
"config.yaml",
"model.pt",
"tokens.json",
"am.mvn",
),
min_total_size_bytes=800_000_000,
),
)
_REALTIME_PUNC_ASSETS = (
ModelAsset(
source="modelscope",
model_id=settings.PUNC_REALTIME_MODEL,
description="FunASR Realtime Punctuation",
required_patterns=("configuration.json", "config.yaml", "model.pt", "tokens.json"),
min_total_size_bytes=200_000_000,
),
)
_DIARIZATION_ASSETS = (
ModelAsset(
source="modelscope",
@ -105,6 +131,8 @@ def get_download_modelscope_assets() -> list[ModelAsset]:
return _dedupe_assets([
*_VAD_ASSETS,
*_REALTIME_ASR_ASSETS,
*_REALTIME_FINAL_ASR_ASSETS,
*_REALTIME_PUNC_ASSETS,
*_DIARIZATION_ASSETS,
])
@ -114,8 +142,15 @@ def get_runtime_required_modelscope_assets(
include_realtime_punc: bool,
) -> list[ModelAsset]:
"""Return ModelScope assets required by the current runtime plan."""
_ = include_realtime_punc
return _dedupe_assets([*_VAD_ASSETS, *_REALTIME_ASR_ASSETS, *_DIARIZATION_ASSETS])
assets = [
*_VAD_ASSETS,
*_REALTIME_ASR_ASSETS,
*_REALTIME_FINAL_ASR_ASSETS,
*_DIARIZATION_ASSETS,
]
if include_realtime_punc:
assets.extend(_REALTIME_PUNC_ASSETS)
return _dedupe_assets(assets)
def _dedupe_assets(assets: list[ModelAsset]) -> list[ModelAsset]:

View File

@ -23,14 +23,27 @@ from app.core.config import settings
from app.core.exceptions import create_error_response
from app.core.executor import run_sync
from app.core.text_cleanup import deduplicate_asr_text
from app.core.hotword_resolver import (
apply_hotword_rules,
format_hotword_prompt_context,
parse_hotwords,
)
from app.services.asr.model_selection import validate_realtime_model_id
from app.services.asr.qwen3_engine import Qwen3ASREngine, Qwen3StreamingState
from app.services.asr.funasr_streaming import (
FUNASR_OFFLINE_MODEL_ID,
FUNASR_STREAMING_MODEL_ID,
get_inference_lock as get_funasr_streaming_inference_lock,
get_offline_inference_lock as get_funasr_offline_inference_lock,
get_offline_model as get_funasr_offline_model,
get_streaming_model as get_funasr_streaming_model,
)
from app.services.asr.engines.global_models import get_global_vad_model, get_vad_inference_lock
from app.services.asr.engines.global_models import (
get_global_punc_realtime_model,
get_global_vad_model,
get_punc_realtime_inference_lock,
get_vad_inference_lock,
)
from app.services.realtime_speaker_clusterer import get_realtime_speaker_clusterer
from app.services.asr.runtime import RuntimeEngineLease, get_runtime_router
from app.services.speaker_registry import get_speaker_registry_service
@ -61,6 +74,16 @@ def _normalize_transcript_chunk_text(text: str) -> str:
return _remove_repeated_chars(collapsed, 8)
def _split_realtime_hotwords(context: Any) -> tuple[str, list[dict[str, Any]]]:
"""Read test-page hotwords from the established WebSocket context field."""
raw_context = str(context or "")
marker = "[[ASR_HOTWORDS]]\n"
if marker not in raw_context:
return raw_context, []
prompt_context, _, hotword_text = raw_context.partition(marker)
return prompt_context.strip(), parse_hotwords(hotword_text)
def _is_opening_punctuation(ch: str) -> bool:
return ch in '([{"\'“‘'
@ -252,7 +275,11 @@ class ConnectionContext:
# FunASR uses the same cached streaming model but isolates mutable decoder
# cache and not-yet-complete audio chunks per WebSocket turn.
funasr_model: Any = None
funasr_offline_model: Any = None
funasr_punc_model: Any = None
funasr_cache: Dict[str, Any] = field(default_factory=dict)
# The punctuation model keeps context across finalized utterances in a session.
funasr_punc_cache: Dict[str, Any] = field(default_factory=dict)
funasr_pending_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
funasr_transcript: RealtimeTranscriptAssembler = field(default_factory=RealtimeTranscriptAssembler)
pre_roll_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
@ -1020,6 +1047,18 @@ class Qwen3ASRService:
ctx.diagnostic_session_id,
FUNASR_STREAMING_MODEL_ID,
)
if ctx.funasr_offline_model is None:
ctx.funasr_offline_model = await run_sync(get_funasr_offline_model)
logger.info(
"[%s] Using FunASR final model: %s",
ctx.diagnostic_session_id,
FUNASR_OFFLINE_MODEL_ID,
)
if settings.ASR_ENABLE_REALTIME_PUNC and ctx.funasr_punc_model is None:
ctx.funasr_punc_model = await run_sync(
get_global_punc_realtime_model,
settings.DEVICE,
)
return None
if ctx.engine is not None:
@ -1102,6 +1141,8 @@ class Qwen3ASRService:
decoder_chunk_look_back=1,
batch_size=1,
fs=16000,
# Paraformer's contextual decoding accepts words, not Qwen prompt text.
hotword=self._funasr_hotword_text(ctx),
)
if not results:
return ""
@ -1116,6 +1157,66 @@ class Qwen3ASRService:
parts.append(text)
return "".join(parts)
@staticmethod
def _funasr_hotword_text(ctx: ConnectionContext) -> str:
"""Convert the shared hotword schema to FunASR's whitespace word list."""
return " ".join(
str(entry.get("text", "")).strip()
for entry in parse_hotwords(ctx.params.get("hotwords"))
if str(entry.get("text", "")).strip()
)
def _finalize_funasr_text(self, ctx: ConnectionContext, text: str) -> str:
"""Match FunASR's final-text order: punctuation, then hotword correction."""
finalized = str(text or "").strip()
if not finalized:
return finalized
if settings.ASR_ENABLE_REALTIME_PUNC and ctx.funasr_punc_model is not None:
try:
with get_punc_realtime_inference_lock():
results = ctx.funasr_punc_model.generate(
input=finalized,
cache=ctx.funasr_punc_cache,
)
result = results[0] if isinstance(results, (list, tuple)) and results else results
if isinstance(result, dict) and result.get("text"):
finalized = str(result["text"]).strip()
except Exception:
# Keep the recognized words if punctuation inference fails for this segment.
logger.exception("[%s] FunASR punctuation inference failed", ctx.diagnostic_session_id)
hotwords = ctx.params.get("hotwords")
if hotwords:
finalized, was_applied, matched = apply_hotword_rules(finalized, hotwords)
if was_applied:
logger.info(
"[%s] FunASR hotword correction applied: matched=%s",
ctx.diagnostic_session_id,
matched,
)
return finalized
def _transcribe_funasr_final(self, ctx: ConnectionContext, audio: np.ndarray) -> str:
"""Run FunASR's offline contextual second pass on the VAD-clipped turn."""
if ctx.funasr_offline_model is None or audio.size == 0:
return ""
with get_funasr_offline_inference_lock():
results = ctx.funasr_offline_model.generate(
input=np.asarray(audio, dtype=np.float32),
hotword=self._funasr_hotword_text(ctx) or None,
)
if not results:
return ""
result_items = results if isinstance(results, (list, tuple)) else [results]
texts: List[str] = []
for item in result_items:
text = item.get("text", "") if isinstance(item, dict) else getattr(item, "text", "")
text = str(text or "").strip()
if text:
texts.append(text)
return "".join(texts)
async def _push_funasr_stream_audio(
self,
ctx: ConnectionContext,
@ -2619,10 +2720,14 @@ class Qwen3ASRService:
)
funasr_final_inference_ms = 0.0
if funasr_streaming:
# Replay only the VAD-clipped region so trailing silence or a next
# utterance already present in the network frame cannot enter the final text.
# FunASR uses the streaming model for previews and the contextual
# offline model for final text; both operate on the VAD-clipped turn.
funasr_final_started = time.perf_counter()
stream_text = await self._finish_funasr_stream(ctx, finalized_audio)
stream_text = await run_sync(self._transcribe_funasr_final, ctx, finalized_audio)
if not stream_text.strip():
# Preserve the online model's final replay as a fallback if the
# contextual second pass returns no usable text.
stream_text = await self._finish_funasr_stream(ctx, finalized_audio)
funasr_final_inference_ms = (time.perf_counter() - funasr_final_started) * 1000.0
stream_language = self._infer_text_language(stream_text)
elif carry_audio.size == 0:
@ -2726,6 +2831,10 @@ class Qwen3ASRService:
language=segment_language,
)
segment_text = self._trim_previous_segment_overlap(ctx, segment_text)
if funasr_streaming:
# Keep CT-Transformer inference off the asyncio event loop so other
# WebSocket sessions can continue sending and decoding audio.
segment_text = await run_sync(self._finalize_funasr_text, ctx, segment_text)
is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec)
logger.info(
@ -2974,12 +3083,23 @@ class Qwen3ASRService:
"UNSUPPORTED_MODEL",
)
continue
context, hotwords = _split_realtime_hotwords(
payload.get("context", "")
)
hotword_context = format_hotword_prompt_context(
hotwords
)
if model_id == "qwen3-asr" and hotword_context:
# Keep Qwen's existing prompt-based hotword behavior while
# exposing the same hotword field for both realtime models.
context = "\n".join(part for part in (context, hotword_context) if part)
ctx.params = {
"model_id": model_id,
"format": payload.get("format", "pcm"),
"sample_rate": payload.get("sample_rate", 16000),
"language": payload.get("language"),
"context": payload.get("context", ""),
"context": context,
"hotwords": hotwords,
"enable_inverse_text_normalization": payload.get(
"enable_inverse_text_normalization",
True,

View File

@ -29,6 +29,24 @@ class ModelAsset:
MODEL_ASSETS: tuple[ModelAsset, ...] = (
ModelAsset(
model_id="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404",
description="FunASR Contextual Offline Final ASR",
required_patterns=(
"configuration.json",
"config.yaml",
"model.pt",
"tokens.json",
"am.mvn",
),
min_total_size_bytes=800_000_000,
),
ModelAsset(
model_id="iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727",
description="FunASR Realtime Punctuation",
required_patterns=("configuration.json", "config.yaml", "model.pt", "tokens.json"),
min_total_size_bytes=200_000_000,
),
ModelAsset(
model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online",
description="FunASR Paraformer Streaming ASR",

View File

@ -2229,6 +2229,11 @@
<label for="liveContext">上下文提示(context)</label>
<textarea id="liveContext" placeholder="可填会议主题、术语、项目名,帮助实时测试更贴近实际场景。"></textarea>
</div>
<div class="field" style="margin-top: 10px;">
<label for="liveHotwords">实时热词</label>
<textarea id="liveHotwords" placeholder="一行一个,可附权重,例如:\n通义千问 2.0\nModelScope 1.5"></textarea>
<p class="compact-note">Qwen 将热词加入上下文提示;FunASR 用热词增强离线定稿,并执行热词纠正。</p>
</div>
<div class="mini-grid" style="margin-top: 10px;">
<div class="field">
<label for="liveStreamChunkSec">在线解码块长(FunASR 约 600 ms)</label>
@ -3925,6 +3930,13 @@ ${transcript}
|| createClientMessageId().slice(0, 16);
$("liveSessionId").value = sessionId;
state.liveSessionId = sessionId;
const hotwords = parseHotwords($("liveHotwords").value);
const contextParts = [$("liveContext").value.trim()].filter(Boolean);
if (hotwords.length) {
contextParts.push(
`[[ASR_HOTWORDS]]\n${hotwords.map((item) => `${item.hotword} ${item.weight}`).join(" ")}`,
);
}
return {
type: "start",
payload: {
@ -3933,7 +3945,8 @@ ${transcript}
model_id: state.liveAsrModel,
session_id: sessionId,
language: $("liveLanguage").value.trim() || null,
context: $("liveContext").value.trim(),
// Preserve the existing WebSocket payload contract by reusing context.
context: contextParts.join("\n"),
enable_inverse_text_normalization: $("liveEnableItn").checked,
// Favor a quick provisional preview; the final segment still replaces it with re-transcribed text.
chunk_size_sec: Math.max(Number($("liveStreamChunkSec").value || 0.6), 0.6),