补齐 FunASR 的定稿流程:在线 Paraformer 输出 partial,上下文离线 Paraformer 负责定稿,随后 CT-Transformer 补标点
parent
a0f2ae1000
commit
826c65c96f
|
|
@ -16,10 +16,16 @@ logger = logging.getLogger(__name__)
|
|||
FUNASR_STREAMING_MODEL_ID = (
|
||||
"iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online"
|
||||
)
|
||||
FUNASR_OFFLINE_MODEL_ID = (
|
||||
"iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404"
|
||||
)
|
||||
|
||||
_model: Any = None
|
||||
_model_lock = threading.Lock()
|
||||
_inference_lock = threading.Lock()
|
||||
_offline_model: Any = None
|
||||
_offline_model_lock = threading.Lock()
|
||||
_offline_inference_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_streaming_model() -> Any:
|
||||
|
|
@ -46,3 +52,29 @@ def get_streaming_model() -> Any:
|
|||
def get_inference_lock() -> threading.Lock:
|
||||
"""Protect mutable AutoModel generate configuration across WebSocket tasks."""
|
||||
return _inference_lock
|
||||
|
||||
|
||||
def get_offline_model() -> Any:
|
||||
"""Load FunASR's contextual second-pass model for final text and hotwords."""
|
||||
global _offline_model
|
||||
if _offline_model is None:
|
||||
with _offline_model_lock:
|
||||
if _offline_model is None:
|
||||
model_path = resolve_model_path(FUNASR_OFFLINE_MODEL_ID)
|
||||
device = detect_device(settings.DEVICE)
|
||||
kwargs = dict(settings.FUNASR_AUTOMODEL_KWARGS)
|
||||
kwargs["disable_update"] = settings.FUNASR_DISABLE_UPDATE
|
||||
logger.info(
|
||||
"Loading FunASR offline final ASR model: model=%s device=%s disable_update=%s",
|
||||
model_path,
|
||||
device,
|
||||
settings.FUNASR_DISABLE_UPDATE,
|
||||
)
|
||||
_offline_model = AutoModel(model=model_path, device=device, **kwargs)
|
||||
logger.info("FunASR offline final ASR model loaded: %s", model_path)
|
||||
return _offline_model
|
||||
|
||||
|
||||
def get_offline_inference_lock() -> threading.Lock:
|
||||
"""Serialize access to the shared offline AutoModel instance."""
|
||||
return _offline_inference_lock
|
||||
|
|
|
|||
|
|
@ -55,6 +55,32 @@ _REALTIME_ASR_ASSETS = (
|
|||
),
|
||||
)
|
||||
|
||||
_REALTIME_FINAL_ASR_ASSETS = (
|
||||
ModelAsset(
|
||||
source="modelscope",
|
||||
model_id="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404",
|
||||
description="FunASR Contextual Offline Final ASR",
|
||||
required_patterns=(
|
||||
"configuration.json",
|
||||
"config.yaml",
|
||||
"model.pt",
|
||||
"tokens.json",
|
||||
"am.mvn",
|
||||
),
|
||||
min_total_size_bytes=800_000_000,
|
||||
),
|
||||
)
|
||||
|
||||
_REALTIME_PUNC_ASSETS = (
|
||||
ModelAsset(
|
||||
source="modelscope",
|
||||
model_id=settings.PUNC_REALTIME_MODEL,
|
||||
description="FunASR Realtime Punctuation",
|
||||
required_patterns=("configuration.json", "config.yaml", "model.pt", "tokens.json"),
|
||||
min_total_size_bytes=200_000_000,
|
||||
),
|
||||
)
|
||||
|
||||
_DIARIZATION_ASSETS = (
|
||||
ModelAsset(
|
||||
source="modelscope",
|
||||
|
|
@ -105,6 +131,8 @@ def get_download_modelscope_assets() -> list[ModelAsset]:
|
|||
return _dedupe_assets([
|
||||
*_VAD_ASSETS,
|
||||
*_REALTIME_ASR_ASSETS,
|
||||
*_REALTIME_FINAL_ASR_ASSETS,
|
||||
*_REALTIME_PUNC_ASSETS,
|
||||
*_DIARIZATION_ASSETS,
|
||||
])
|
||||
|
||||
|
|
@ -114,8 +142,15 @@ def get_runtime_required_modelscope_assets(
|
|||
include_realtime_punc: bool,
|
||||
) -> list[ModelAsset]:
|
||||
"""Return ModelScope assets required by the current runtime plan."""
|
||||
_ = include_realtime_punc
|
||||
return _dedupe_assets([*_VAD_ASSETS, *_REALTIME_ASR_ASSETS, *_DIARIZATION_ASSETS])
|
||||
assets = [
|
||||
*_VAD_ASSETS,
|
||||
*_REALTIME_ASR_ASSETS,
|
||||
*_REALTIME_FINAL_ASR_ASSETS,
|
||||
*_DIARIZATION_ASSETS,
|
||||
]
|
||||
if include_realtime_punc:
|
||||
assets.extend(_REALTIME_PUNC_ASSETS)
|
||||
return _dedupe_assets(assets)
|
||||
|
||||
|
||||
def _dedupe_assets(assets: list[ModelAsset]) -> list[ModelAsset]:
|
||||
|
|
|
|||
|
|
@ -23,14 +23,27 @@ from app.core.config import settings
|
|||
from app.core.exceptions import create_error_response
|
||||
from app.core.executor import run_sync
|
||||
from app.core.text_cleanup import deduplicate_asr_text
|
||||
from app.core.hotword_resolver import (
|
||||
apply_hotword_rules,
|
||||
format_hotword_prompt_context,
|
||||
parse_hotwords,
|
||||
)
|
||||
from app.services.asr.model_selection import validate_realtime_model_id
|
||||
from app.services.asr.qwen3_engine import Qwen3ASREngine, Qwen3StreamingState
|
||||
from app.services.asr.funasr_streaming import (
|
||||
FUNASR_OFFLINE_MODEL_ID,
|
||||
FUNASR_STREAMING_MODEL_ID,
|
||||
get_inference_lock as get_funasr_streaming_inference_lock,
|
||||
get_offline_inference_lock as get_funasr_offline_inference_lock,
|
||||
get_offline_model as get_funasr_offline_model,
|
||||
get_streaming_model as get_funasr_streaming_model,
|
||||
)
|
||||
from app.services.asr.engines.global_models import get_global_vad_model, get_vad_inference_lock
|
||||
from app.services.asr.engines.global_models import (
|
||||
get_global_punc_realtime_model,
|
||||
get_global_vad_model,
|
||||
get_punc_realtime_inference_lock,
|
||||
get_vad_inference_lock,
|
||||
)
|
||||
from app.services.realtime_speaker_clusterer import get_realtime_speaker_clusterer
|
||||
from app.services.asr.runtime import RuntimeEngineLease, get_runtime_router
|
||||
from app.services.speaker_registry import get_speaker_registry_service
|
||||
|
|
@ -61,6 +74,16 @@ def _normalize_transcript_chunk_text(text: str) -> str:
|
|||
return _remove_repeated_chars(collapsed, 8)
|
||||
|
||||
|
||||
def _split_realtime_hotwords(context: Any) -> tuple[str, list[dict[str, Any]]]:
|
||||
"""Read test-page hotwords from the established WebSocket context field."""
|
||||
raw_context = str(context or "")
|
||||
marker = "[[ASR_HOTWORDS]]\n"
|
||||
if marker not in raw_context:
|
||||
return raw_context, []
|
||||
prompt_context, _, hotword_text = raw_context.partition(marker)
|
||||
return prompt_context.strip(), parse_hotwords(hotword_text)
|
||||
|
||||
|
||||
def _is_opening_punctuation(ch: str) -> bool:
|
||||
return ch in '([{"\'“‘'
|
||||
|
||||
|
|
@ -252,7 +275,11 @@ class ConnectionContext:
|
|||
# FunASR uses the same cached streaming model but isolates mutable decoder
|
||||
# cache and not-yet-complete audio chunks per WebSocket turn.
|
||||
funasr_model: Any = None
|
||||
funasr_offline_model: Any = None
|
||||
funasr_punc_model: Any = None
|
||||
funasr_cache: Dict[str, Any] = field(default_factory=dict)
|
||||
# The punctuation model keeps context across finalized utterances in a session.
|
||||
funasr_punc_cache: Dict[str, Any] = field(default_factory=dict)
|
||||
funasr_pending_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
|
||||
funasr_transcript: RealtimeTranscriptAssembler = field(default_factory=RealtimeTranscriptAssembler)
|
||||
pre_roll_audio: np.ndarray = field(default_factory=lambda: np.array([], dtype=np.float32))
|
||||
|
|
@ -1020,6 +1047,18 @@ class Qwen3ASRService:
|
|||
ctx.diagnostic_session_id,
|
||||
FUNASR_STREAMING_MODEL_ID,
|
||||
)
|
||||
if ctx.funasr_offline_model is None:
|
||||
ctx.funasr_offline_model = await run_sync(get_funasr_offline_model)
|
||||
logger.info(
|
||||
"[%s] Using FunASR final model: %s",
|
||||
ctx.diagnostic_session_id,
|
||||
FUNASR_OFFLINE_MODEL_ID,
|
||||
)
|
||||
if settings.ASR_ENABLE_REALTIME_PUNC and ctx.funasr_punc_model is None:
|
||||
ctx.funasr_punc_model = await run_sync(
|
||||
get_global_punc_realtime_model,
|
||||
settings.DEVICE,
|
||||
)
|
||||
return None
|
||||
|
||||
if ctx.engine is not None:
|
||||
|
|
@ -1102,6 +1141,8 @@ class Qwen3ASRService:
|
|||
decoder_chunk_look_back=1,
|
||||
batch_size=1,
|
||||
fs=16000,
|
||||
# Paraformer's contextual decoding accepts words, not Qwen prompt text.
|
||||
hotword=self._funasr_hotword_text(ctx),
|
||||
)
|
||||
if not results:
|
||||
return ""
|
||||
|
|
@ -1116,6 +1157,66 @@ class Qwen3ASRService:
|
|||
parts.append(text)
|
||||
return "".join(parts)
|
||||
|
||||
@staticmethod
|
||||
def _funasr_hotword_text(ctx: ConnectionContext) -> str:
|
||||
"""Convert the shared hotword schema to FunASR's whitespace word list."""
|
||||
return " ".join(
|
||||
str(entry.get("text", "")).strip()
|
||||
for entry in parse_hotwords(ctx.params.get("hotwords"))
|
||||
if str(entry.get("text", "")).strip()
|
||||
)
|
||||
|
||||
def _finalize_funasr_text(self, ctx: ConnectionContext, text: str) -> str:
|
||||
"""Match FunASR's final-text order: punctuation, then hotword correction."""
|
||||
finalized = str(text or "").strip()
|
||||
if not finalized:
|
||||
return finalized
|
||||
|
||||
if settings.ASR_ENABLE_REALTIME_PUNC and ctx.funasr_punc_model is not None:
|
||||
try:
|
||||
with get_punc_realtime_inference_lock():
|
||||
results = ctx.funasr_punc_model.generate(
|
||||
input=finalized,
|
||||
cache=ctx.funasr_punc_cache,
|
||||
)
|
||||
result = results[0] if isinstance(results, (list, tuple)) and results else results
|
||||
if isinstance(result, dict) and result.get("text"):
|
||||
finalized = str(result["text"]).strip()
|
||||
except Exception:
|
||||
# Keep the recognized words if punctuation inference fails for this segment.
|
||||
logger.exception("[%s] FunASR punctuation inference failed", ctx.diagnostic_session_id)
|
||||
|
||||
hotwords = ctx.params.get("hotwords")
|
||||
if hotwords:
|
||||
finalized, was_applied, matched = apply_hotword_rules(finalized, hotwords)
|
||||
if was_applied:
|
||||
logger.info(
|
||||
"[%s] FunASR hotword correction applied: matched=%s",
|
||||
ctx.diagnostic_session_id,
|
||||
matched,
|
||||
)
|
||||
return finalized
|
||||
|
||||
def _transcribe_funasr_final(self, ctx: ConnectionContext, audio: np.ndarray) -> str:
|
||||
"""Run FunASR's offline contextual second pass on the VAD-clipped turn."""
|
||||
if ctx.funasr_offline_model is None or audio.size == 0:
|
||||
return ""
|
||||
with get_funasr_offline_inference_lock():
|
||||
results = ctx.funasr_offline_model.generate(
|
||||
input=np.asarray(audio, dtype=np.float32),
|
||||
hotword=self._funasr_hotword_text(ctx) or None,
|
||||
)
|
||||
if not results:
|
||||
return ""
|
||||
result_items = results if isinstance(results, (list, tuple)) else [results]
|
||||
texts: List[str] = []
|
||||
for item in result_items:
|
||||
text = item.get("text", "") if isinstance(item, dict) else getattr(item, "text", "")
|
||||
text = str(text or "").strip()
|
||||
if text:
|
||||
texts.append(text)
|
||||
return "".join(texts)
|
||||
|
||||
async def _push_funasr_stream_audio(
|
||||
self,
|
||||
ctx: ConnectionContext,
|
||||
|
|
@ -2619,10 +2720,14 @@ class Qwen3ASRService:
|
|||
)
|
||||
funasr_final_inference_ms = 0.0
|
||||
if funasr_streaming:
|
||||
# Replay only the VAD-clipped region so trailing silence or a next
|
||||
# utterance already present in the network frame cannot enter the final text.
|
||||
# FunASR uses the streaming model for previews and the contextual
|
||||
# offline model for final text; both operate on the VAD-clipped turn.
|
||||
funasr_final_started = time.perf_counter()
|
||||
stream_text = await self._finish_funasr_stream(ctx, finalized_audio)
|
||||
stream_text = await run_sync(self._transcribe_funasr_final, ctx, finalized_audio)
|
||||
if not stream_text.strip():
|
||||
# Preserve the online model's final replay as a fallback if the
|
||||
# contextual second pass returns no usable text.
|
||||
stream_text = await self._finish_funasr_stream(ctx, finalized_audio)
|
||||
funasr_final_inference_ms = (time.perf_counter() - funasr_final_started) * 1000.0
|
||||
stream_language = self._infer_text_language(stream_text)
|
||||
elif carry_audio.size == 0:
|
||||
|
|
@ -2726,6 +2831,10 @@ class Qwen3ASRService:
|
|||
language=segment_language,
|
||||
)
|
||||
segment_text = self._trim_previous_segment_overlap(ctx, segment_text)
|
||||
if funasr_streaming:
|
||||
# Keep CT-Transformer inference off the asyncio event loop so other
|
||||
# WebSocket sessions can continue sending and decoding audio.
|
||||
segment_text = await run_sync(self._finalize_funasr_text, ctx, segment_text)
|
||||
|
||||
is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec)
|
||||
logger.info(
|
||||
|
|
@ -2974,12 +3083,23 @@ class Qwen3ASRService:
|
|||
"UNSUPPORTED_MODEL",
|
||||
)
|
||||
continue
|
||||
context, hotwords = _split_realtime_hotwords(
|
||||
payload.get("context", "")
|
||||
)
|
||||
hotword_context = format_hotword_prompt_context(
|
||||
hotwords
|
||||
)
|
||||
if model_id == "qwen3-asr" and hotword_context:
|
||||
# Keep Qwen's existing prompt-based hotword behavior while
|
||||
# exposing the same hotword field for both realtime models.
|
||||
context = "\n".join(part for part in (context, hotword_context) if part)
|
||||
ctx.params = {
|
||||
"model_id": model_id,
|
||||
"format": payload.get("format", "pcm"),
|
||||
"sample_rate": payload.get("sample_rate", 16000),
|
||||
"language": payload.get("language"),
|
||||
"context": payload.get("context", ""),
|
||||
"context": context,
|
||||
"hotwords": hotwords,
|
||||
"enable_inverse_text_normalization": payload.get(
|
||||
"enable_inverse_text_normalization",
|
||||
True,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,24 @@ class ModelAsset:
|
|||
|
||||
|
||||
MODEL_ASSETS: tuple[ModelAsset, ...] = (
|
||||
ModelAsset(
|
||||
model_id="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404",
|
||||
description="FunASR Contextual Offline Final ASR",
|
||||
required_patterns=(
|
||||
"configuration.json",
|
||||
"config.yaml",
|
||||
"model.pt",
|
||||
"tokens.json",
|
||||
"am.mvn",
|
||||
),
|
||||
min_total_size_bytes=800_000_000,
|
||||
),
|
||||
ModelAsset(
|
||||
model_id="iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727",
|
||||
description="FunASR Realtime Punctuation",
|
||||
required_patterns=("configuration.json", "config.yaml", "model.pt", "tokens.json"),
|
||||
min_total_size_bytes=200_000_000,
|
||||
),
|
||||
ModelAsset(
|
||||
model_id="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online",
|
||||
description="FunASR Paraformer Streaming ASR",
|
||||
|
|
|
|||
|
|
@ -2229,6 +2229,11 @@
|
|||
<label for="liveContext">上下文提示(context)</label>
|
||||
<textarea id="liveContext" placeholder="可填会议主题、术语、项目名,帮助实时测试更贴近实际场景。"></textarea>
|
||||
</div>
|
||||
<div class="field" style="margin-top: 10px;">
|
||||
<label for="liveHotwords">实时热词</label>
|
||||
<textarea id="liveHotwords" placeholder="一行一个,可附权重,例如:\n通义千问 2.0\nModelScope 1.5"></textarea>
|
||||
<p class="compact-note">Qwen 将热词加入上下文提示;FunASR 用热词增强离线定稿,并执行热词纠正。</p>
|
||||
</div>
|
||||
<div class="mini-grid" style="margin-top: 10px;">
|
||||
<div class="field">
|
||||
<label for="liveStreamChunkSec">在线解码块长(FunASR 约 600 ms)</label>
|
||||
|
|
@ -3925,6 +3930,13 @@ ${transcript}
|
|||
|| createClientMessageId().slice(0, 16);
|
||||
$("liveSessionId").value = sessionId;
|
||||
state.liveSessionId = sessionId;
|
||||
const hotwords = parseHotwords($("liveHotwords").value);
|
||||
const contextParts = [$("liveContext").value.trim()].filter(Boolean);
|
||||
if (hotwords.length) {
|
||||
contextParts.push(
|
||||
`[[ASR_HOTWORDS]]\n${hotwords.map((item) => `${item.hotword} ${item.weight}`).join(" ")}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
type: "start",
|
||||
payload: {
|
||||
|
|
@ -3933,7 +3945,8 @@ ${transcript}
|
|||
model_id: state.liveAsrModel,
|
||||
session_id: sessionId,
|
||||
language: $("liveLanguage").value.trim() || null,
|
||||
context: $("liveContext").value.trim(),
|
||||
// Preserve the existing WebSocket payload contract by reusing context.
|
||||
context: contextParts.join("\n"),
|
||||
enable_inverse_text_normalization: $("liveEnableItn").checked,
|
||||
// Favor a quick provisional preview; the final segment still replaces it with re-transcribed text.
|
||||
chunk_size_sec: Math.max(Number($("liveStreamChunkSec").value || 0.6), 0.6),
|
||||
|
|
|
|||
Loading…
Reference in New Issue