对齐FunASAR实时流声纹聚类路线,改为通过AutoModel.generate预处理链,遗弃原始采用ModelScope pipeline的处理链
parent
541c800fb4
commit
fdd9ae20a4
|
|
@ -0,0 +1,84 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""FunASR-native speaker embedding extraction for realtime diarization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Any, Iterable
|
||||
|
||||
import numpy as np
|
||||
|
||||
from funasr import AutoModel
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.device import detect_device
|
||||
from app.infrastructure import resolve_model_path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_model: Any = None
|
||||
_model_lock = threading.Lock()
|
||||
_inference_lock = threading.Lock()
|
||||
|
||||
|
||||
def _get_model() -> Any:
|
||||
"""Load the configured realtime SV checkpoint through FunASR AutoModel."""
|
||||
global _model
|
||||
if _model is None:
|
||||
with _model_lock:
|
||||
if _model is None:
|
||||
model_path = resolve_model_path(settings.REALTIME_SV_MODEL)
|
||||
device = detect_device(settings.DEVICE)
|
||||
kwargs = dict(settings.FUNASR_AUTOMODEL_KWARGS)
|
||||
kwargs["disable_update"] = settings.FUNASR_DISABLE_UPDATE
|
||||
logger.info(
|
||||
"Loading FunASR realtime speaker model: model=%s device=%s disable_update=%s",
|
||||
model_path,
|
||||
device,
|
||||
settings.FUNASR_DISABLE_UPDATE,
|
||||
)
|
||||
_model = AutoModel(model=model_path, device=device, **kwargs)
|
||||
logger.info("FunASR realtime speaker model loaded: %s", model_path)
|
||||
return _model
|
||||
|
||||
|
||||
def extract_realtime_speaker_embeddings(audio_chunks: Iterable[np.ndarray]) -> list[np.ndarray]:
|
||||
"""Extract normalized embeddings using FunASR's generate/preprocess path.
|
||||
|
||||
FunASR's ERes2Net inference converts waveform chunks to normalized fbank
|
||||
features before calling the network. Calling the model's inner ``forward``
|
||||
directly skips that preprocessing and can make cosine speaker matching
|
||||
unreliable, so realtime diarization must go through ``AutoModel.generate``.
|
||||
"""
|
||||
chunks = [np.asarray(audio, dtype=np.float32).reshape(-1) for audio in audio_chunks]
|
||||
if not chunks:
|
||||
return []
|
||||
if any(chunk.size == 0 for chunk in chunks):
|
||||
raise ValueError("speaker audio chunks must not be empty")
|
||||
|
||||
model = _get_model()
|
||||
# Match FunASR's realtime tracker: infer all 1.5 s windows together through
|
||||
# generate(), which runs the model's own audio loading and feature frontend.
|
||||
with _inference_lock:
|
||||
results = model.generate(input=chunks, cache={}, is_final=True)
|
||||
|
||||
if not isinstance(results, (list, tuple)) or len(results) != len(chunks):
|
||||
raise RuntimeError(
|
||||
"FunASR speaker model returned an unexpected result count: "
|
||||
f"expected={len(chunks)} actual={len(results) if hasattr(results, '__len__') else 'unknown'}"
|
||||
)
|
||||
|
||||
embeddings: list[np.ndarray] = []
|
||||
for result in results:
|
||||
value = result.get("spk_embedding") if isinstance(result, dict) else None
|
||||
if value is None:
|
||||
raise RuntimeError("FunASR speaker model result has no spk_embedding")
|
||||
if hasattr(value, "detach"):
|
||||
value = value.detach().cpu().numpy()
|
||||
embedding = np.asarray(value, dtype=np.float32).reshape(-1)
|
||||
norm = float(np.linalg.norm(embedding))
|
||||
if norm <= 1e-12:
|
||||
raise RuntimeError("FunASR speaker model returned an empty embedding")
|
||||
embeddings.append((embedding / norm).astype(np.float32))
|
||||
return embeddings
|
||||
|
|
@ -371,14 +371,16 @@ class SessionEntry:
|
|||
|
||||
|
||||
class Qwen3ASRService:
|
||||
# Use the same 1.5 s SV window / 0.75 s hop as the realtime clusterer.
|
||||
# Four consecutive changed windows (spanning >=2.25 s) reject brief
|
||||
# acknowledgements such as "嗯" before they can split a transcript turn.
|
||||
# 实时声纹探测沿用聚类器的 1.5 秒窗口和 0.75 秒步长,因此相邻窗口有一半重叠.
|
||||
_SPEAKER_CHANGE_WINDOW_SAMPLES = 24000
|
||||
_SPEAKER_CHANGE_HOP_SAMPLES = 12000
|
||||
# 候选声纹需连续命中 4 次,且首尾窗口中心至少相隔 2.25 秒,才确认切换.
|
||||
# 注意:这是探测窗口中心的时间跨度,不代表新说话人实际发言至少 2.25 秒.
|
||||
_SPEAKER_CHANGE_CONFIRM_HITS = 4
|
||||
_SPEAKER_CHANGE_CONFIRM_SPAN_SAMPLES = 36000
|
||||
# 限制上一段的最短长度为 0.75 秒,防止边界落在当前语音段开头.
|
||||
_SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES = 12000
|
||||
# 参考声纹或候选声纹与当前窗口的余弦相似度阈值.
|
||||
_SPEAKER_CHANGE_COSINE_THRESHOLD = 0.60
|
||||
|
||||
_MAX_SPEAKER_HISTORY_RECORDS = 128
|
||||
|
|
@ -393,7 +395,7 @@ class Qwen3ASRService:
|
|||
self._model_switch_lock = asyncio.Lock()
|
||||
self._active_websocket_connections = 0
|
||||
self._switching_realtime_model = False
|
||||
self._current_realtime_model = "qwen3-asr"
|
||||
self._current_realtime_model = "funasr-paraformer-streaming"
|
||||
|
||||
async def prepare_realtime_model(self, model_id: str) -> Dict[str, str]:
|
||||
"""Swap and warm the selected realtime ASR route before capture begins."""
|
||||
|
|
@ -1724,6 +1726,7 @@ class Qwen3ASRService:
|
|||
if audio.size != self._SPEAKER_CHANGE_WINDOW_SAMPLES or not self._has_voice(audio):
|
||||
return
|
||||
|
||||
# 记录本次窗口终点;下次至少前进一个 hop 才再次提取声纹.
|
||||
ctx.speaker_change_last_probe_end_sample = end_sample
|
||||
task = asyncio.create_task(self._extract_realtime_speaker_probe(audio, start_sample))
|
||||
ctx.speaker_change_probe_task = task
|
||||
|
|
@ -1759,6 +1762,7 @@ class Qwen3ASRService:
|
|||
ctx.speaker_change_reference_center_sample = center_sample
|
||||
return None
|
||||
|
||||
# 窗口与稳定参考相似:缓慢更新参考声纹,并清除未确认的候选.
|
||||
reference_score = float(np.dot(reference, embedding))
|
||||
if reference_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD:
|
||||
# Slowly adapt to normal voice and prosody variation from one speaker.
|
||||
|
|
@ -1771,6 +1775,7 @@ class Qwen3ASRService:
|
|||
ctx.speaker_change_candidate_hits = 0
|
||||
return None
|
||||
|
||||
# 与旧声纹不相似还不足以切段;还要检查多个窗口是否彼此像同一个新说话人.
|
||||
candidate = ctx.speaker_change_candidate_embedding
|
||||
if candidate is not None and candidate.shape == embedding.shape:
|
||||
candidate_score = float(np.dot(candidate, embedding))
|
||||
|
|
@ -1778,15 +1783,18 @@ class Qwen3ASRService:
|
|||
candidate_score = -1.0
|
||||
|
||||
if candidate_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD:
|
||||
# 本窗口不像旧声纹、但像候选:累加连续命中次数,更新候选声纹.
|
||||
updated = (candidate * ctx.speaker_change_candidate_hits) + embedding
|
||||
norm = max(float(np.linalg.norm(updated)), 1e-12)
|
||||
ctx.speaker_change_candidate_embedding = (updated / norm).astype(np.float32)
|
||||
ctx.speaker_change_candidate_hits += 1
|
||||
else:
|
||||
# 候选声纹不连续:从本窗口重新计数,本窗口为第 1 次命中.
|
||||
ctx.speaker_change_candidate_embedding = embedding.copy()
|
||||
ctx.speaker_change_candidate_start_center_sample = center_sample
|
||||
ctx.speaker_change_candidate_hits = 1
|
||||
|
||||
# 命中次数和首尾窗口中心跨度都达标后,才认定为持续切换.
|
||||
candidate_start = ctx.speaker_change_candidate_start_center_sample
|
||||
if (
|
||||
candidate_start is None
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from sklearn.cluster._kmeans import k_means
|
|||
|
||||
from app.core.config import settings
|
||||
from app.core.executor import run_sync
|
||||
from app.services.asr.funasr_speaker import extract_realtime_speaker_embeddings
|
||||
from app.services.speaker_registry import get_speaker_registry_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -288,21 +289,19 @@ class RealtimeSpeakerClusterer:
|
|||
}
|
||||
]
|
||||
|
||||
registry = get_speaker_registry_service()
|
||||
# 与 FunASR 实时跟踪器保持一致:整批声纹窗口通过 AutoModel.generate
|
||||
# 提取,确保先执行 ERes2Net 所需的音频特征预处理,再进入模型。
|
||||
embeddings = await run_sync(
|
||||
extract_realtime_speaker_embeddings,
|
||||
[item["audio"] for item in chunk_items],
|
||||
)
|
||||
results: List[Dict[str, Any]] = []
|
||||
for item in chunk_items:
|
||||
embedding = await run_sync(
|
||||
registry.extract_embedding_from_audio,
|
||||
item["audio"],
|
||||
16000,
|
||||
model_id=settings.REALTIME_SV_MODEL,
|
||||
model_revision=settings.REALTIME_SV_MODEL_REVISION or None,
|
||||
)
|
||||
for item, embedding in zip(chunk_items, embeddings):
|
||||
results.append(
|
||||
{
|
||||
"start_ms": int(item["start_ms"]),
|
||||
"end_ms": int(item["end_ms"]),
|
||||
"embedding": self._normalize_embedding(np.asarray(embedding, dtype=np.float32)),
|
||||
"embedding": self._normalize_embedding(embedding),
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
|
|
|||
Loading…
Reference in New Issue