From 3af8f168f6fd1b6034e9bb522c4315e5e9924c57 Mon Sep 17 00:00:00 2001 From: Bifang <915779419@qq.com> Date: Thu, 8 Oct 2026 14:28:27 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AF=B9=E9=BD=90FunASR=E6=BB=91=E5=8A=A8?= =?UTF-8?q?=E7=AA=97=E5=8F=A3=E8=81=9A=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/qwen3_websocket_asr.py | 235 ++++++--- app/services/realtime_speaker_clusterer.py | 523 +++++++++------------ 2 files changed, 389 insertions(+), 369 deletions(-) diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index b6b04d8..3a6dd78 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -264,6 +264,11 @@ class ConnectionContext: total_samples: int = 0 confirmed_segments: List[Dict[str, Any]] = field(default_factory=list) speaker_records: List[Dict[str, Any]] = field(default_factory=list) + # FunASR-style persistent speaker centroids, indexed by stable speaker ID. + speaker_centers: List[np.ndarray] = field(default_factory=list) + speaker_center_updates: List[int] = field(default_factory=list) + speaker_timeline: List[Dict[str, Any]] = field(default_factory=list) + speaker_cluster_labels: Dict[int, Dict[str, Any]] = field(default_factory=dict) speaker_display_map: Dict[int, str] = field(default_factory=dict) speaker_display_profiles: Dict[str, np.ndarray] = field(default_factory=dict) speaker_display_counts: Dict[str, int] = field(default_factory=dict) @@ -304,11 +309,8 @@ class SessionEntry: class Qwen3ASRService: - _MAX_SPEAKER_HISTORY_RECORDS = 64 - _MAX_SPEAKER_MATCH_CONTEXT = 24 - _MAX_SPEAKER_FINAL_RECLUSTER = 48 - _MAX_SPEAKER_REALTIME_RECLUSTER = 12 - _MIN_REALTIME_RECLUSTER_SEGMENTS = 5 + _MAX_SPEAKER_HISTORY_RECORDS = 128 + _MAX_SPEAKER_FINAL_RECLUSTER = 128 _MIN_HARD_SEGMENT_SEC = 12.0 _SEGMENT_EVENT_TEXT_TAIL_CHARS = 2000 _SEGMENT_EVENT_TAIL_COUNT = 8 @@ -633,6 +635,8 @@ class Qwen3ASRService: strategy = str(speaker_info.get("speaker_strategy") or "") if strategy == "registry_match": return True + if strategy == "chunk_cluster": + return True if strategy in {"recent_named_attach", "recent_named_inherit"}: confidence = float(speaker_info.get("speaker_confidence") or 0.0) return confidence >= 0.7 @@ -701,6 +705,8 @@ class Qwen3ASRService: ctx: ConnectionContext, speaker_info: Dict[str, Any], ) -> Optional[Dict[str, Any]]: + if speaker_info.get("speaker_strategy") == "chunk_cluster": + return None if speaker_info.get("registry_speaker_id") or speaker_info.get("user_id"): return None if speaker_info.get("_matched_existing"): @@ -728,6 +734,8 @@ class Qwen3ASRService: ctx: ConnectionContext, speaker_info: Dict[str, Any], ) -> Optional[Dict[str, Any]]: + if speaker_info.get("speaker_strategy") == "chunk_cluster": + return None if speaker_info.get("registry_speaker_id") or speaker_info.get("user_id"): return speaker_info if speaker_info.get("_matched_existing"): @@ -755,34 +763,37 @@ class Qwen3ASRService: self, ctx: ConnectionContext, ) -> List[Dict[str, Any]]: - if len(ctx.speaker_records) <= self._MAX_SPEAKER_MATCH_CONTEXT: - return ctx.speaker_records - return ctx.speaker_records[-self._MAX_SPEAKER_MATCH_CONTEXT:] + # The clusterer applies its limit by chunk count (128), not by the + # variable number of ASR segments those chunks happen to span. + return ctx.speaker_records def _recluster_confirmed_segments( self, ctx: ConnectionContext, - *, - max_records: Optional[int] = None, - only_stable_updates: bool = False, - only_pending_updates: bool = False, ) -> List[Dict[str, Any]]: - if len(ctx.speaker_records) < 2: + if not ctx.speaker_records: return [] - record_limit = max_records or self._MAX_SPEAKER_FINAL_RECLUSTER + record_limit = self._MAX_SPEAKER_FINAL_RECLUSTER records = ( ctx.speaker_records if len(ctx.speaker_records) <= record_limit else ctx.speaker_records[-record_limit:] ) - clusters, assignments = self._cluster_speaker_records(records) + clusterer = get_realtime_speaker_clusterer() + clusters, assignments, timeline = clusterer.cluster_records_with_stable_history( + records, + ctx.speaker_centers, + ctx.speaker_center_updates, + update_centers=False, + ) if not clusters: return [] + ctx.speaker_timeline = timeline cluster_labels: dict[int, Dict[str, Any]] = {} - unknown_display_index = 1 - for idx, cluster in enumerate(clusters): + for cluster in clusters: + idx = int(cluster["cluster_index"]) named_records = [ item for item in cluster["record_refs"] if item.get("registry_speaker_id") @@ -802,9 +813,9 @@ class Qwen3ASRService: ), ) speaker_id = preferred.get("registry_speaker_id") or preferred.get("speaker_id") - speaker_name = preferred.get("speaker_name") or speaker_id or f"Speaker{unknown_display_index:02d}" + speaker_name = preferred.get("speaker_name") or speaker_id or f"Speaker{idx + 1:02d}" cluster_labels[idx] = { - "speaker_id": speaker_id or f"Speaker{unknown_display_index:02d}", + "speaker_id": speaker_id or f"Speaker{idx + 1:02d}", "speaker_name": speaker_name, "user_id": preferred.get("user_id"), "registry_speaker_id": preferred.get("registry_speaker_id"), @@ -812,7 +823,7 @@ class Qwen3ASRService: "speaker_confidence": 1.0, } else: - speaker_id = f"Speaker{unknown_display_index:02d}" + speaker_id = f"Speaker{idx + 1:02d}" cluster_labels[idx] = { "speaker_id": speaker_id, "speaker_name": speaker_id, @@ -820,10 +831,13 @@ class Qwen3ASRService: "speaker_strategy": "final_recluster", "speaker_confidence": 1.0, } - unknown_display_index += 1 + + ctx.speaker_cluster_labels = cluster_labels updates: List[Dict[str, Any]] = [] for record, cluster_idx in zip(records, assignments): + if cluster_idx < 0 or cluster_idx not in cluster_labels: + continue segment_index = int(record["index"]) segment = next( (item for item in ctx.confirmed_segments if int(item.get("index", -1)) == segment_index), @@ -831,15 +845,7 @@ class Qwen3ASRService: ) if segment is None: continue - if only_pending_updates and not segment.get("speaker_pending"): - continue new_info = cluster_labels[cluster_idx] - if only_stable_updates: - has_named_identity = bool(new_info.get("registry_speaker_id") or new_info.get("user_id")) - generic_name = str(new_info.get("speaker_name") or "").strip() - if not has_named_identity and re.fullmatch(r"Speaker\d+", generic_name): - if not segment.get("speaker_pending"): - continue changed = any(segment.get(key) != value for key, value in new_info.items()) if not changed and not segment.get("speaker_pending"): continue @@ -854,18 +860,128 @@ class Qwen3ASRService: return updates - def _maybe_recluster_recent_segments( + def _build_final_speaker_segments( self, ctx: ConnectionContext, + *, + min_split_sec: float = 3.0, ) -> List[Dict[str, Any]]: - if len(ctx.speaker_records) < self._MIN_REALTIME_RECLUSTER_SEGMENTS: - return [] - return self._recluster_confirmed_segments( - ctx, - max_records=self._MAX_SPEAKER_REALTIME_RECLUSTER, - only_stable_updates=True, - only_pending_updates=True, - ) + """Split final ASR text where the FunASR speaker timeline changes. + + ASR does not provide character timestamps, so text is apportioned by + each speaker's share of the overlapping audio duration, matching the + approximation used by FunASR's realtime tracker. + """ + if not ctx.speaker_timeline: + return [dict(item) for item in ctx.confirmed_segments] + history_start_ms = min(int(item["start_ms"]) for item in ctx.speaker_timeline) + + def speaker_info(cluster_index: int) -> Dict[str, Any]: + label = ctx.speaker_cluster_labels.get(cluster_index) + if label: + return label + generic_id = f"Speaker{cluster_index + 1:02d}" + return {"speaker_id": generic_id, "speaker_name": generic_id} + + final_segments: List[Dict[str, Any]] = [] + for source_segment in ctx.confirmed_segments: + original = dict(source_segment) + seg_start = int(original.get("start_ms", 0)) + seg_end = int(original.get("end_ms", seg_start)) + if seg_start < history_start_ms < seg_end: + # FunASR keeps any text before the 128-chunk history window + # intact and only reassigns the portion covered by diarization. + text = str(original.get("text") or "") + ratio = (history_start_ms - seg_start) / max(seg_end - seg_start, 1) + split_index = min(len(text) - 1, max(1, round(len(text) * ratio))) + prefix_text = text[:split_index].strip() + suffix_text = text[split_index:].strip() + if prefix_text and suffix_text: + prefix = dict(original) + prefix.update(index=len(final_segments), text=prefix_text, end_ms=history_start_ms) + final_segments.append(prefix) + original["text"] = suffix_text + original["start_ms"] = history_start_ms + seg_start = history_start_ms + overlaps: List[List[int]] = [] + for item in ctx.speaker_timeline: + start = max(seg_start, int(item["start_ms"])) + end = min(seg_end, int(item["end_ms"])) + if end <= start: + continue + cluster_index = int(item["cluster_index"]) + if overlaps and overlaps[-1][2] == cluster_index and start <= overlaps[-1][1]: + overlaps[-1][1] = end + else: + overlaps.append([start, end, cluster_index]) + + if not overlaps: + segment = dict(original) + segment["index"] = len(final_segments) + final_segments.append(segment) + continue + + # FunASR merges same-speaker regions and absorbs regions shorter + # than three seconds into the neighboring speaker. + filtered: List[List[int]] = [overlaps[0][:]] + for current in overlaps[1:]: + previous = filtered[-1] + if current[2] == previous[2]: + previous[1] = current[1] + elif (current[1] - current[0]) < min_split_sec * 1000: + previous[1] = current[1] + else: + filtered.append(current[:]) + + merged: List[List[int]] = [filtered[0][:]] + for current in filtered[1:]: + previous = merged[-1] + if (previous[1] - previous[0]) < min_split_sec * 1000: + merged[-1] = [previous[0], current[1], current[2]] + else: + merged.append(current[:]) + if len(merged) > 1 and (merged[-1][1] - merged[-1][0]) < min_split_sec * 1000: + merged[-2][1] = merged[-1][1] + merged.pop() + + if len(merged) <= 1: + segment = dict(original) + segment.update(speaker_info(merged[0][2])) + segment["index"] = len(final_segments) + final_segments.append(segment) + continue + + total_duration = sum(end - start for start, end, _ in merged) + text = str(original.get("text") or "") + if len(text) < len(merged): + longest = max(merged, key=lambda item: item[1] - item[0]) + segment = dict(original) + segment.update(speaker_info(longest[2])) + segment["index"] = len(final_segments) + final_segments.append(segment) + continue + char_pos = 0 + for part_index, (start, end, cluster_index) in enumerate(merged): + if part_index == len(merged) - 1: + part_text = text[char_pos:] + else: + remaining_parts = len(merged) - part_index - 1 + proposed = int(len(text) * (end - start) / max(total_duration, 1)) + char_count = max(1, min(proposed, len(text) - char_pos - remaining_parts)) + part_text = text[char_pos:char_pos + char_count] + char_pos += char_count + if not part_text.strip(): + continue + fragment = dict(original) + fragment.update(speaker_info(cluster_index)) + fragment.update( + index=len(final_segments), + text=part_text.strip(), + start_ms=start, + end_ms=end, + ) + final_segments.append(fragment) + return final_segments def _build_sv_chunks(self, audio: np.ndarray) -> List[np.ndarray]: return get_realtime_speaker_clusterer().build_sv_chunks(audio) @@ -2427,6 +2543,8 @@ class Qwen3ASRService: segment_end_ms=segment_end_ms, enable_registry_match=bool(ctx.params.get("enable_speaker_identification", True)), speaker_threshold=ctx.params.get("speaker_threshold"), + speaker_centers=ctx.speaker_centers, + speaker_center_updates=ctx.speaker_center_updates, ) if not speaker_info: return None @@ -2448,11 +2566,14 @@ class Qwen3ASRService: and not speaker_info.get("user_id") ): safe_cluster_index = self._safe_int(cluster_index) - display_id = self._match_display_speaker_id( - ctx, - cluster_index=safe_cluster_index, - embedding=speaker_info.get("_embedding"), - ) + if speaker_info.get("speaker_strategy") == "chunk_cluster": + display_id = str(speaker_info.get("speaker_id")) + else: + display_id = self._match_display_speaker_id( + ctx, + cluster_index=safe_cluster_index, + embedding=speaker_info.get("_embedding"), + ) if display_id: speaker_info["speaker_id"] = display_id speaker_info["speaker_name"] = display_id @@ -2507,21 +2628,6 @@ class Qwen3ASRService: segment_end_ms=segment_end_ms, speaker_info=speaker_info, ) - queue_backlog = ctx.speaker_job_queue.qsize() - updates: List[Dict[str, Any]] = [] - if queue_backlog <= 1: - updates = self._maybe_recluster_recent_segments(ctx) - if updates: - for updated in updates: - await self._emit_speaker_update( - websocket, - ctx, - task_id, - int(updated["segment_index"]), - ) - return - # Re-clustering may resolve pending IDs even if this segment's - # immediate speaker fields did not change. if before_view != after_view: await self._emit_speaker_update(websocket, ctx, task_id, segment_index) except Exception as exc: @@ -3419,15 +3525,14 @@ class Qwen3ASRService: } ) - all_texts = [ - segment["text"] - for segment in ctx.confirmed_segments - if segment["text"].strip() - ] + # Finalize the same timeline used by FunASR: a transcript sentence + # can become multiple speaker-specific entries at session end. + final_segments = self._build_final_speaker_segments(ctx) + all_texts = [segment["text"] for segment in final_segments if segment["text"].strip()] full_text = "\n".join(all_texts) public_segments = [ self._public_payload(segment) or {} - for segment in ctx.confirmed_segments + for segment in final_segments ] await websocket.send_json( @@ -3440,7 +3545,7 @@ class Qwen3ASRService: "final": 1, "result": { "slice_type": 2, - "index": max(len(ctx.confirmed_segments) - 1, 0), + "index": max(len(public_segments) - 1, 0), "voice_text_str": full_text, }, "sentences": [ diff --git a/app/services/realtime_speaker_clusterer.py b/app/services/realtime_speaker_clusterer.py index 25bbae6..11e8441 100644 --- a/app/services/realtime_speaker_clusterer.py +++ b/app/services/realtime_speaker_clusterer.py @@ -4,8 +4,6 @@ from __future__ import annotations import logging -import re -from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple import numpy as np @@ -24,128 +22,15 @@ class RealtimeSpeakerClusterer: """Sentence-level speaker attribution using chunk embeddings and session clustering.""" FUNASR_SMOOTH_MINDUR_SEC = 0.7 - _FAST_ATTACH_MAX_SEC = 1.6 + FUNASR_MAX_HISTORY_CHUNKS = 128 + FUNASR_CENTER_MATCH_THRESHOLD = 0.6 _REGISTRY_MATCH_MIN_SEC = 2.4 - def _is_generic_speaker_name(self, value: Any) -> bool: - text = str(value or "").strip() - return bool(re.fullmatch(r"Speaker\d+", text)) - - def _has_named_identity(self, record: Optional[Dict[str, Any]]) -> bool: - if not record: - return False - return bool( - record.get("registry_speaker_id") - or record.get("user_id") - or ( - record.get("speaker_name") - and not self._is_generic_speaker_name(record.get("speaker_name")) - ) - ) - def _normalize_embedding(self, embedding: np.ndarray) -> np.ndarray: emb = np.asarray(embedding, dtype=np.float32).reshape(-1) norm = max(float(np.linalg.norm(emb)), 1e-12) return (emb / norm).astype(np.float32) - @staticmethod - def _coerce_cluster_index(value: Any, fallback: int) -> int: - try: - if value is None: - return int(fallback) - return int(value) - except (TypeError, ValueError): - return int(fallback) - - def _match_existing_speaker( - self, - speaker_records: List[Dict[str, Any]], - mean_embedding: np.ndarray, - ) -> Optional[Dict[str, Any]]: - if not speaker_records: - return None - - best_record: Optional[Dict[str, Any]] = None - best_score = -1.0 - for record in speaker_records: - record_embedding = record.get("embedding") - if record_embedding is None: - continue - score = float(np.dot(self._normalize_embedding(record_embedding), mean_embedding)) - if score > best_score: - best_score = score - best_record = record - - if best_record is None: - return None - - base_threshold = max( - float(getattr(settings, "REALTIME_UNKNOWN_SPK_CLUSTER_THRESHOLD", 0.58) or 0.58), - 0.3, - ) - has_named_identity = bool( - best_record.get("registry_speaker_id") - or best_record.get("user_id") - or ( - best_record.get("speaker_name") - and not self._is_generic_speaker_name(best_record.get("speaker_name")) - ) - ) - threshold = max(base_threshold, 0.62 if has_named_identity else 0.72) - if best_score < threshold: - return None - - matched = dict(best_record) - matched["_match_score"] = best_score - return matched - - def _next_generic_speaker_id( - self, - speaker_records: List[Dict[str, Any]], - ) -> str: - seen: set[int] = set() - for record in speaker_records: - for value in ( - record.get("speaker_id"), - record.get("speaker_name"), - ): - text = str(value or "").strip() - match = re.fullmatch(r"Speaker(\d+)", text) - if match: - seen.add(int(match.group(1))) - next_index = max(seen, default=0) + 1 - return f"Speaker{next_index:02d}" - - def _is_mixed_speaker_segment( - self, - speaker_records: List[Dict[str, Any]], - current_chunks: List[Dict[str, Any]], - ) -> bool: - if len(current_chunks) < 2 or len(speaker_records) < 2: - return False - assigned_labels: List[str] = [] - for chunk in current_chunks: - embedding = chunk.get("embedding") - if embedding is None: - continue - matched = self._match_existing_speaker( - speaker_records, - self._normalize_embedding(np.asarray(embedding, dtype=np.float32)), - ) - if not matched: - continue - label = str( - matched.get("registry_speaker_id") - or matched.get("speaker_name") - or matched.get("speaker_id") - or "" - ).strip() - if label: - assigned_labels.append(label) - if len(assigned_labels) < 2: - return False - return len(set(assigned_labels)) >= 2 - def _correct_labels(self, labels: np.ndarray) -> np.ndarray: labels_id = 0 id2id: dict[int, int] = {} @@ -216,6 +101,144 @@ class RealtimeSpeakerClusterer: labels = self._spectral_cluster(X, oracle_num=oracle_num) return self._merge_by_cos(labels, X, cos_thr=0.78 if oracle_num is None else 1.0) + def _map_cluster_centers( + self, + cluster_centers: np.ndarray, + speaker_centers: List[np.ndarray], + speaker_center_updates: List[int], + *, + update_centers: bool, + ) -> List[int]: + """Map each window's temporary cluster label to a persistent speaker ID. + + FunASR's ClusterBackend labels are local to each clustering pass. Its + realtime tracker maps those labels through cosine-matched centroids so + speakers keep the same identity as the 128-chunk window moves. + """ + centers = np.stack( + [self._normalize_embedding(center) for center in np.asarray(cluster_centers)], + axis=0, + ) + stable_ids: List[int] = [] + used_ids: set[int] = set() + for center in centers: + best_id: Optional[int] = None + best_score = -1.0 + for candidate, known_center in enumerate(speaker_centers): + if candidate in used_ids: + continue + score = float(np.dot(center, self._normalize_embedding(known_center))) + if score > best_score: + best_id, best_score = candidate, score + + matched = best_id is not None and best_score >= self.FUNASR_CENTER_MATCH_THRESHOLD + created = False + if not matched and update_centers and len(speaker_centers) < 15: + best_id = len(speaker_centers) + speaker_centers.append(center.copy()) + speaker_center_updates.append(1) + matched = True + created = True + elif best_id is None: + # Match FunASR's max-speaker fallback when all old IDs were used. + if not speaker_centers: + best_id = 0 + else: + scores = [float(np.dot(center, self._normalize_embedding(item))) for item in speaker_centers] + best_id = int(np.argmax(scores)) + + if update_centers and matched and not created: + assert best_id is not None + count = speaker_center_updates[best_id] + weight = 1.0 / min(count + 1, 20) + updated = (1.0 - weight) * speaker_centers[best_id] + weight * center + speaker_centers[best_id] = self._normalize_embedding(updated) + speaker_center_updates[best_id] = count + 1 + + assert best_id is not None + stable_ids.append(best_id) + used_ids.add(best_id) + return stable_ids + + def cluster_chunk_history( + self, + chunks: List[Dict[str, Any]], + speaker_centers: List[np.ndarray], + speaker_center_updates: List[int], + *, + update_centers: bool, + ) -> Tuple[List[Dict[str, Any]], List[int]]: + """Cluster a fixed-size rolling window of timestamped speaker chunks.""" + valid_chunks = [item for item in chunks if item.get("embedding") is not None] + valid_chunks = valid_chunks[-self.FUNASR_MAX_HISTORY_CHUNKS:] + if not valid_chunks: + return [], [] + + embeddings = np.stack( + [self._normalize_embedding(np.asarray(item["embedding"], dtype=np.float32)) for item in valid_chunks], + axis=0, + ) + labels = self._cluster_embeddings(embeddings, oracle_num=None) + temporary_ids = sorted({int(value) for value in labels.tolist()}) + centers = np.stack( + [self._normalize_embedding(embeddings[labels == label].mean(axis=0)) for label in temporary_ids], + axis=0, + ) + stable_ids = self._map_cluster_centers( + centers, + speaker_centers, + speaker_center_updates, + update_centers=update_centers, + ) + temp_to_stable = dict(zip(temporary_ids, stable_ids)) + + timeline = self._postprocess_timeline(valid_chunks, labels, embeddings) + for item in timeline: + item["cluster_index"] = int(temp_to_stable[int(item["cluster_index"])]) + return timeline, stable_ids + + def flatten_chunk_history(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Return the most recent FunASR-sized history from segment records.""" + chunks: List[Dict[str, Any]] = [] + for record in records: + entries = record.get("chunks") or [] + if entries: + chunks.extend(entries) + continue + embeddings = record.get("embeddings") or [] + if not embeddings and record.get("embedding") is not None: + embeddings = [record["embedding"]] + for embedding in embeddings: + chunks.append({ + "start_ms": int(record.get("start_ms", 0)), + "end_ms": int(record.get("end_ms", record.get("start_ms", 0))), + "embedding": embedding, + }) + return chunks[-self.FUNASR_MAX_HISTORY_CHUNKS:] + + @staticmethod + def pick_timeline_speaker( + timeline: List[Dict[str, Any]], + *, + start_ms: int, + end_ms: int, + ) -> Optional[int]: + """Assign a sentence to its dominant overlapping speaker, as FunASR does.""" + sentence_speaker = 0 + max_overlap = 0 + for item in timeline: + overlap = min(end_ms, int(item["end_ms"])) - max(start_ms, int(item["start_ms"])) + if overlap > 0: + speaker = int(item["cluster_index"]) + if overlap > max_overlap: + max_overlap = overlap + sentence_speaker = speaker + if sentence_speaker == speaker: + max_overlap += overlap + if max_overlap <= 0: + return None + return sentence_speaker + def build_sv_chunks( self, audio: np.ndarray, @@ -300,42 +323,6 @@ class RealtimeSpeakerClusterer: logger.warning("Realtime registry speaker embedding failed: %s", exc) return None - def _flatten_chunk_records( - self, - records: List[Dict[str, Any]], - ) -> Tuple[List[Dict[str, Any]], np.ndarray]: - flat_chunks: List[Dict[str, Any]] = [] - flat_embeddings: List[np.ndarray] = [] - for record_idx, record in enumerate(records): - chunk_entries = record.get("chunks") or [] - if not chunk_entries: - embeddings = record.get("embeddings") or [] - if not embeddings and record.get("embedding") is not None: - embeddings = [record["embedding"]] - start_ms = int(record.get("start_ms", 0)) - end_ms = int(record.get("end_ms", start_ms)) - for embedding in embeddings: - chunk_entries.append( - { - "start_ms": start_ms, - "end_ms": end_ms, - "embedding": embedding, - } - ) - for chunk in chunk_entries: - emb = self._normalize_embedding(np.asarray(chunk["embedding"], dtype=np.float32)) - flat_chunks.append( - { - "record_index": record_idx, - "start_ms": int(chunk.get("start_ms", record.get("start_ms", 0))), - "end_ms": int(chunk.get("end_ms", record.get("end_ms", 0))), - } - ) - flat_embeddings.append(emb) - if not flat_embeddings: - return [], np.zeros((0, 0), dtype=np.float32) - return flat_chunks, np.stack(flat_embeddings, axis=0).astype(np.float32) - def _merge_seque(self, distribute_res: List[List[float]]) -> List[List[float]]: if not distribute_res: return [] @@ -406,82 +393,66 @@ class RealtimeSpeakerClusterer: for item in distribute_res ] - def _pick_segment_cluster( - self, - cluster_ranges: List[Dict[str, Any]], - *, - segment_start_ms: int, - segment_end_ms: int, - ) -> int: - overlaps: Dict[int, int] = defaultdict(int) - for item in cluster_ranges: - overlap = min(segment_end_ms, int(item["end_ms"])) - max(segment_start_ms, int(item["start_ms"])) - if overlap > 0: - overlaps[int(item["cluster_index"])] += int(overlap) - if overlaps: - return max(overlaps.items(), key=lambda kv: (kv[1], -kv[0]))[0] - - centers = [ - item for item in cluster_ranges - if int(item["start_ms"]) <= segment_end_ms and int(item["end_ms"]) >= segment_start_ms - ] - if centers: - return int(centers[0]["cluster_index"]) - return 0 - def cluster_records_with_ranges( self, records: List[Dict[str, Any]], ) -> Tuple[List[Dict[str, Any]], List[int], List[Dict[str, Any]]]: - flat_chunks, X = self._flatten_chunk_records(records) - if X.size == 0: - return [], [], [] + # Preserve the existing helper API while routing it through the same + # 128-chunk policy used by live sessions. + return self.cluster_records_with_stable_history(records, [], [], update_centers=True) - labels = self._cluster_embeddings(X, oracle_num=None) - cluster_ranges = self._postprocess_timeline(flat_chunks, labels, X) - - cluster_ids = sorted({int(item["cluster_index"]) for item in cluster_ranges}) - if not cluster_ids: - cluster_ids = sorted({int(label) for label in labels.tolist()}) - - clusters: List[Dict[str, Any]] = [] - for cluster_idx in cluster_ids: - member_mask = labels == int(cluster_idx) - member_embeddings = X[member_mask] - centroid = self._normalize_embedding(member_embeddings.mean(0)) - record_refs = [] - seen_record_indices = set() - for emb_idx, chunk in enumerate(flat_chunks): - record_idx = int(chunk["record_index"]) - if not member_mask[emb_idx] or record_idx in seen_record_indices: - continue - seen_record_indices.add(record_idx) - record_refs.append(records[record_idx]) - clusters.append( - { - "centroid": centroid, - "count": int(member_mask.sum()), - "record_refs": record_refs, - } + def cluster_records_with_stable_history( + self, + records: List[Dict[str, Any]], + speaker_centers: List[np.ndarray], + speaker_center_updates: List[int], + *, + update_centers: bool, + ) -> Tuple[List[Dict[str, Any]], List[int], List[Dict[str, Any]]]: + """Cluster only the last 128 chunks and return persistent speaker IDs.""" + chunks = self.flatten_chunk_history(records) + timeline, _ = self.cluster_chunk_history( + chunks, + speaker_centers, + speaker_center_updates, + update_centers=update_centers, + ) + assignments = [ + self.pick_timeline_speaker( + timeline, + start_ms=int(record.get("start_ms", 0)), + end_ms=int(record.get("end_ms", record.get("start_ms", 0))), ) - - record_to_labels: List[List[int]] = [[] for _ in records] - for emb_idx, chunk in enumerate(flat_chunks): - record_to_labels[int(chunk["record_index"])].append(int(labels[emb_idx])) - - record_assignments: List[int] = [] - for record, local_assignments in zip(records, record_to_labels): - if local_assignments: - cluster_idx = self._pick_segment_cluster( - cluster_ranges, - segment_start_ms=int(record.get("start_ms", 0)), - segment_end_ms=int(record.get("end_ms", record.get("start_ms", 0))), - ) - record_assignments.append(cluster_idx) + for record in records + ] + cluster_ids = sorted({int(item["cluster_index"]) for item in timeline}) + clusters: List[Dict[str, Any]] = [] + for cluster_id in cluster_ids: + refs = [ + record + for record, assigned_id in zip(records, assignments) + if assigned_id == cluster_id + ] + if cluster_id < len(speaker_centers): + center = speaker_centers[cluster_id] else: - record_assignments.append(-1) - - return clusters, record_assignments, cluster_ranges + matching_embeddings = [ + item["embedding"] + for item in chunks + if self.pick_timeline_speaker( + timeline, + start_ms=int(item["start_ms"]), + end_ms=int(item["end_ms"]), + ) == cluster_id + ] + center = np.mean(matching_embeddings or [chunks[0]["embedding"]], axis=0) + clusters.append({ + "cluster_index": cluster_id, + "centroid": self._normalize_embedding(np.asarray(center, dtype=np.float32)), + "count": sum(1 for item in timeline if int(item["cluster_index"]) == cluster_id), + "record_refs": refs, + }) + return clusters, [-1 if value is None else int(value) for value in assignments], timeline def cluster_records( self, @@ -499,104 +470,47 @@ class RealtimeSpeakerClusterer: segment_end_ms: int, enable_registry_match: bool, speaker_threshold: Optional[float], + speaker_centers: List[np.ndarray], + speaker_center_updates: List[int], ) -> Optional[Dict[str, Any]]: duration_sec = float(len(audio)) / 16000.0 - last_record = speaker_records[-1] if speaker_records else None - if duration_sec < self._FAST_ATTACH_MAX_SEC and speaker_records: - if self._has_named_identity(last_record): - speaker_id = last_record.get("registry_speaker_id") or last_record.get("speaker_id") or "Speaker01" - speaker_name = last_record.get("speaker_name") or speaker_id - return { - "speaker_id": speaker_id, - "speaker_name": speaker_name, - "user_id": last_record.get("user_id"), - "registry_speaker_id": last_record.get("registry_speaker_id"), - "speaker_confidence": 0.0, - "speaker_strategy": "short_attach", - "_cluster_index": last_record.get("cluster_index"), - "_embedding": last_record.get("embedding"), - "_chunk_embeddings": last_record.get("embeddings") or [], - "_chunks": last_record.get("chunks") or [], - } - current_chunks = await self.extract_chunk_embeddings( np.asarray(audio, dtype=np.float32), segment_start_ms=segment_start_ms, ) if not current_chunks: - if speaker_records: - if self._has_named_identity(last_record): - speaker_id = last_record.get("registry_speaker_id") or last_record.get("speaker_id") or "Speaker01" - speaker_name = last_record.get("speaker_name") or speaker_id - return { - "speaker_id": speaker_id, - "speaker_name": speaker_name, - "user_id": last_record.get("user_id"), - "registry_speaker_id": last_record.get("registry_speaker_id"), - "speaker_confidence": 0.0, - "speaker_strategy": "embedding_attach", - "_cluster_index": last_record.get("cluster_index"), - "_embedding": last_record.get("embedding"), - "_chunk_embeddings": last_record.get("embeddings") or [], - "_chunks": last_record.get("chunks") or [], - } return None mean_embedding = self._normalize_embedding( np.mean(np.stack([chunk["embedding"] for chunk in current_chunks], axis=0), axis=0) ) - if duration_sec >= 4.0 and self._is_mixed_speaker_segment(speaker_records, current_chunks): - return { - "speaker_id": -1, - "speaker_name": "", - "user_id": None, - "registry_speaker_id": None, - "speaker_confidence": 0.0, - "speaker_strategy": "mixed_segment", - "_cluster_index": None, - "_embedding": mean_embedding, - "_chunk_embeddings": [np.asarray(chunk["embedding"], dtype=np.float32) for chunk in current_chunks], - "_chunks": current_chunks, - } - matched_record = self._match_existing_speaker(speaker_records, mean_embedding) - speaker_id = self._next_generic_speaker_id(speaker_records) + + # Re-cluster the same fixed number of recent chunks as FunASR. Chunk + # labels are temporary, so the session-owned centers keep IDs stable. + history_chunks = self.flatten_chunk_history(speaker_records) + timeline, _ = self.cluster_chunk_history( + history_chunks + current_chunks, + speaker_centers, + speaker_center_updates, + update_centers=True, + ) + cluster_index = self.pick_timeline_speaker( + timeline, + start_ms=segment_start_ms, + end_ms=segment_end_ms, + ) + if cluster_index is None: + cluster_index = max(len(speaker_centers) - 1, 0) + speaker_id = f"Speaker{cluster_index + 1:02d}" speaker_name = speaker_id user_id = None registry_speaker_id = None - strategy = "new_speaker" - cluster_index = max( - [ - int(record.get("cluster_index", -1)) - for record in speaker_records - if record.get("cluster_index") is not None - ] or [-1] - ) + 1 - confidence = 0.0 - - if matched_record is not None: - speaker_id = ( - matched_record.get("registry_speaker_id") - or matched_record.get("speaker_id") - or speaker_id - ) - speaker_name = matched_record.get("speaker_name") or speaker_id - user_id = matched_record.get("user_id") - registry_speaker_id = matched_record.get("registry_speaker_id") - cluster_index = self._coerce_cluster_index( - matched_record.get("cluster_index"), - cluster_index, - ) - confidence = float(matched_record.get("_match_score", 0.0)) - strategy = "embedding_match" + confidence = 1.0 + strategy = "chunk_cluster" if ( - not registry_speaker_id - and enable_registry_match + enable_registry_match and duration_sec >= self._REGISTRY_MATCH_MIN_SEC - and ( - matched_record is None - or confidence >= max(float(getattr(settings, "REALTIME_SPEAKER_CONFIRM_THRESHOLD", 0.62) or 0.62), 0.7) - ) ): # Realtime clustering can use a dedicated fast model, while the # registry stores embeddings from the registration model. Match the @@ -622,11 +536,12 @@ class RealtimeSpeakerClusterer: "registry_speaker_id": registry_speaker_id, "speaker_confidence": round(confidence, 4), "speaker_strategy": strategy, - "_cluster_index": self._coerce_cluster_index(cluster_index, 0), + "_cluster_index": int(cluster_index), "_embedding": mean_embedding, "_chunk_embeddings": [np.asarray(chunk["embedding"], dtype=np.float32) for chunk in current_chunks], "_chunks": current_chunks, - "_matched_existing": matched_record is not None, + "_matched_existing": True, + "_speaker_timeline": timeline, }