From fdd9ae20a4ab15ecacadc3ec5280f53e3d7ce8ab Mon Sep 17 00:00:00 2001 From: Bifang <915779419@qq.com> Date: Wed, 14 Oct 2026 10:31:27 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AF=B9=E9=BD=90FunASAR=E5=AE=9E=E6=97=B6?= =?UTF-8?q?=E6=B5=81=E5=A3=B0=E7=BA=B9=E8=81=9A=E7=B1=BB=E8=B7=AF=E7=BA=BF?= =?UTF-8?q?=EF=BC=8C=E6=94=B9=E4=B8=BA=E9=80=9A=E8=BF=87AutoModel.generate?= =?UTF-8?q?=E9=A2=84=E5=A4=84=E7=90=86=E9=93=BE=EF=BC=8C=E9=81=97=E5=BC=83?= =?UTF-8?q?=E5=8E=9F=E5=A7=8B=E9=87=87=E7=94=A8ModelScope=20pipeline?= =?UTF-8?q?=E7=9A=84=E5=A4=84=E7=90=86=E9=93=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/asr/funasr_speaker.py | 84 ++++++++++++++++++++++ app/services/qwen3_websocket_asr.py | 16 +++-- app/services/realtime_speaker_clusterer.py | 19 +++-- 3 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 app/services/asr/funasr_speaker.py diff --git a/app/services/asr/funasr_speaker.py b/app/services/asr/funasr_speaker.py new file mode 100644 index 0000000..25b1b94 --- /dev/null +++ b/app/services/asr/funasr_speaker.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +"""FunASR-native speaker embedding extraction for realtime diarization.""" + +from __future__ import annotations + +import logging +import threading +from typing import Any, Iterable + +import numpy as np + +from funasr import AutoModel + +from app.core.config import settings +from app.core.device import detect_device +from app.infrastructure import resolve_model_path + +logger = logging.getLogger(__name__) + +_model: Any = None +_model_lock = threading.Lock() +_inference_lock = threading.Lock() + + +def _get_model() -> Any: + """Load the configured realtime SV checkpoint through FunASR AutoModel.""" + global _model + if _model is None: + with _model_lock: + if _model is None: + model_path = resolve_model_path(settings.REALTIME_SV_MODEL) + device = detect_device(settings.DEVICE) + kwargs = dict(settings.FUNASR_AUTOMODEL_KWARGS) + kwargs["disable_update"] = settings.FUNASR_DISABLE_UPDATE + logger.info( + "Loading FunASR realtime speaker model: model=%s device=%s disable_update=%s", + model_path, + device, + settings.FUNASR_DISABLE_UPDATE, + ) + _model = AutoModel(model=model_path, device=device, **kwargs) + logger.info("FunASR realtime speaker model loaded: %s", model_path) + return _model + + +def extract_realtime_speaker_embeddings(audio_chunks: Iterable[np.ndarray]) -> list[np.ndarray]: + """Extract normalized embeddings using FunASR's generate/preprocess path. + + FunASR's ERes2Net inference converts waveform chunks to normalized fbank + features before calling the network. Calling the model's inner ``forward`` + directly skips that preprocessing and can make cosine speaker matching + unreliable, so realtime diarization must go through ``AutoModel.generate``. + """ + chunks = [np.asarray(audio, dtype=np.float32).reshape(-1) for audio in audio_chunks] + if not chunks: + return [] + if any(chunk.size == 0 for chunk in chunks): + raise ValueError("speaker audio chunks must not be empty") + + model = _get_model() + # Match FunASR's realtime tracker: infer all 1.5 s windows together through + # generate(), which runs the model's own audio loading and feature frontend. + with _inference_lock: + results = model.generate(input=chunks, cache={}, is_final=True) + + if not isinstance(results, (list, tuple)) or len(results) != len(chunks): + raise RuntimeError( + "FunASR speaker model returned an unexpected result count: " + f"expected={len(chunks)} actual={len(results) if hasattr(results, '__len__') else 'unknown'}" + ) + + embeddings: list[np.ndarray] = [] + for result in results: + value = result.get("spk_embedding") if isinstance(result, dict) else None + if value is None: + raise RuntimeError("FunASR speaker model result has no spk_embedding") + if hasattr(value, "detach"): + value = value.detach().cpu().numpy() + embedding = np.asarray(value, dtype=np.float32).reshape(-1) + norm = float(np.linalg.norm(embedding)) + if norm <= 1e-12: + raise RuntimeError("FunASR speaker model returned an empty embedding") + embeddings.append((embedding / norm).astype(np.float32)) + return embeddings diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index a8015dd..d0a5640 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -371,14 +371,16 @@ class SessionEntry: class Qwen3ASRService: - # Use the same 1.5 s SV window / 0.75 s hop as the realtime clusterer. - # Four consecutive changed windows (spanning >=2.25 s) reject brief - # acknowledgements such as "嗯" before they can split a transcript turn. + # 实时声纹探测沿用聚类器的 1.5 秒窗口和 0.75 秒步长,因此相邻窗口有一半重叠. _SPEAKER_CHANGE_WINDOW_SAMPLES = 24000 _SPEAKER_CHANGE_HOP_SAMPLES = 12000 + # 候选声纹需连续命中 4 次,且首尾窗口中心至少相隔 2.25 秒,才确认切换. + # 注意:这是探测窗口中心的时间跨度,不代表新说话人实际发言至少 2.25 秒. _SPEAKER_CHANGE_CONFIRM_HITS = 4 _SPEAKER_CHANGE_CONFIRM_SPAN_SAMPLES = 36000 + # 限制上一段的最短长度为 0.75 秒,防止边界落在当前语音段开头. _SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES = 12000 + # 参考声纹或候选声纹与当前窗口的余弦相似度阈值. _SPEAKER_CHANGE_COSINE_THRESHOLD = 0.60 _MAX_SPEAKER_HISTORY_RECORDS = 128 @@ -393,7 +395,7 @@ class Qwen3ASRService: self._model_switch_lock = asyncio.Lock() self._active_websocket_connections = 0 self._switching_realtime_model = False - self._current_realtime_model = "qwen3-asr" + self._current_realtime_model = "funasr-paraformer-streaming" async def prepare_realtime_model(self, model_id: str) -> Dict[str, str]: """Swap and warm the selected realtime ASR route before capture begins.""" @@ -1724,6 +1726,7 @@ class Qwen3ASRService: if audio.size != self._SPEAKER_CHANGE_WINDOW_SAMPLES or not self._has_voice(audio): return + # 记录本次窗口终点;下次至少前进一个 hop 才再次提取声纹. ctx.speaker_change_last_probe_end_sample = end_sample task = asyncio.create_task(self._extract_realtime_speaker_probe(audio, start_sample)) ctx.speaker_change_probe_task = task @@ -1759,6 +1762,7 @@ class Qwen3ASRService: ctx.speaker_change_reference_center_sample = center_sample return None + # 窗口与稳定参考相似:缓慢更新参考声纹,并清除未确认的候选. reference_score = float(np.dot(reference, embedding)) if reference_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD: # Slowly adapt to normal voice and prosody variation from one speaker. @@ -1771,6 +1775,7 @@ class Qwen3ASRService: ctx.speaker_change_candidate_hits = 0 return None + # 与旧声纹不相似还不足以切段;还要检查多个窗口是否彼此像同一个新说话人. candidate = ctx.speaker_change_candidate_embedding if candidate is not None and candidate.shape == embedding.shape: candidate_score = float(np.dot(candidate, embedding)) @@ -1778,15 +1783,18 @@ class Qwen3ASRService: candidate_score = -1.0 if candidate_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD: + # 本窗口不像旧声纹、但像候选:累加连续命中次数,更新候选声纹. updated = (candidate * ctx.speaker_change_candidate_hits) + embedding norm = max(float(np.linalg.norm(updated)), 1e-12) ctx.speaker_change_candidate_embedding = (updated / norm).astype(np.float32) ctx.speaker_change_candidate_hits += 1 else: + # 候选声纹不连续:从本窗口重新计数,本窗口为第 1 次命中. ctx.speaker_change_candidate_embedding = embedding.copy() ctx.speaker_change_candidate_start_center_sample = center_sample ctx.speaker_change_candidate_hits = 1 + # 命中次数和首尾窗口中心跨度都达标后,才认定为持续切换. candidate_start = ctx.speaker_change_candidate_start_center_sample if ( candidate_start is None diff --git a/app/services/realtime_speaker_clusterer.py b/app/services/realtime_speaker_clusterer.py index 11e8441..2d11610 100644 --- a/app/services/realtime_speaker_clusterer.py +++ b/app/services/realtime_speaker_clusterer.py @@ -13,6 +13,7 @@ from sklearn.cluster._kmeans import k_means from app.core.config import settings from app.core.executor import run_sync +from app.services.asr.funasr_speaker import extract_realtime_speaker_embeddings from app.services.speaker_registry import get_speaker_registry_service logger = logging.getLogger(__name__) @@ -288,21 +289,19 @@ class RealtimeSpeakerClusterer: } ] - registry = get_speaker_registry_service() + # 与 FunASR 实时跟踪器保持一致:整批声纹窗口通过 AutoModel.generate + # 提取,确保先执行 ERes2Net 所需的音频特征预处理,再进入模型。 + embeddings = await run_sync( + extract_realtime_speaker_embeddings, + [item["audio"] for item in chunk_items], + ) results: List[Dict[str, Any]] = [] - for item in chunk_items: - embedding = await run_sync( - registry.extract_embedding_from_audio, - item["audio"], - 16000, - model_id=settings.REALTIME_SV_MODEL, - model_revision=settings.REALTIME_SV_MODEL_REVISION or None, - ) + for item, embedding in zip(chunk_items, embeddings): results.append( { "start_ms": int(item["start_ms"]), "end_ms": int(item["end_ms"]), - "embedding": self._normalize_embedding(np.asarray(embedding, dtype=np.float32)), + "embedding": self._normalize_embedding(embedding), } ) return results