对齐FunASR滑动窗口聚类

main
Bifang 2026-10-08 14:28:27 +08:00
parent e905aefc94
commit 3af8f168f6
2 changed files with 389 additions and 369 deletions

View File

@ -264,6 +264,11 @@ class ConnectionContext:
total_samples: int = 0
confirmed_segments: List[Dict[str, Any]] = field(default_factory=list)
speaker_records: List[Dict[str, Any]] = field(default_factory=list)
# FunASR-style persistent speaker centroids, indexed by stable speaker ID.
speaker_centers: List[np.ndarray] = field(default_factory=list)
speaker_center_updates: List[int] = field(default_factory=list)
speaker_timeline: List[Dict[str, Any]] = field(default_factory=list)
speaker_cluster_labels: Dict[int, Dict[str, Any]] = field(default_factory=dict)
speaker_display_map: Dict[int, str] = field(default_factory=dict)
speaker_display_profiles: Dict[str, np.ndarray] = field(default_factory=dict)
speaker_display_counts: Dict[str, int] = field(default_factory=dict)
@ -304,11 +309,8 @@ class SessionEntry:
class Qwen3ASRService:
_MAX_SPEAKER_HISTORY_RECORDS = 64
_MAX_SPEAKER_MATCH_CONTEXT = 24
_MAX_SPEAKER_FINAL_RECLUSTER = 48
_MAX_SPEAKER_REALTIME_RECLUSTER = 12
_MIN_REALTIME_RECLUSTER_SEGMENTS = 5
_MAX_SPEAKER_HISTORY_RECORDS = 128
_MAX_SPEAKER_FINAL_RECLUSTER = 128
_MIN_HARD_SEGMENT_SEC = 12.0
_SEGMENT_EVENT_TEXT_TAIL_CHARS = 2000
_SEGMENT_EVENT_TAIL_COUNT = 8
@ -633,6 +635,8 @@ class Qwen3ASRService:
strategy = str(speaker_info.get("speaker_strategy") or "")
if strategy == "registry_match":
return True
if strategy == "chunk_cluster":
return True
if strategy in {"recent_named_attach", "recent_named_inherit"}:
confidence = float(speaker_info.get("speaker_confidence") or 0.0)
return confidence >= 0.7
@ -701,6 +705,8 @@ class Qwen3ASRService:
ctx: ConnectionContext,
speaker_info: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
if speaker_info.get("speaker_strategy") == "chunk_cluster":
return None
if speaker_info.get("registry_speaker_id") or speaker_info.get("user_id"):
return None
if speaker_info.get("_matched_existing"):
@ -728,6 +734,8 @@ class Qwen3ASRService:
ctx: ConnectionContext,
speaker_info: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
if speaker_info.get("speaker_strategy") == "chunk_cluster":
return None
if speaker_info.get("registry_speaker_id") or speaker_info.get("user_id"):
return speaker_info
if speaker_info.get("_matched_existing"):
@ -755,34 +763,37 @@ class Qwen3ASRService:
self,
ctx: ConnectionContext,
) -> List[Dict[str, Any]]:
if len(ctx.speaker_records) <= self._MAX_SPEAKER_MATCH_CONTEXT:
return ctx.speaker_records
return ctx.speaker_records[-self._MAX_SPEAKER_MATCH_CONTEXT:]
# The clusterer applies its limit by chunk count (128), not by the
# variable number of ASR segments those chunks happen to span.
return ctx.speaker_records
def _recluster_confirmed_segments(
self,
ctx: ConnectionContext,
*,
max_records: Optional[int] = None,
only_stable_updates: bool = False,
only_pending_updates: bool = False,
) -> List[Dict[str, Any]]:
if len(ctx.speaker_records) < 2:
if not ctx.speaker_records:
return []
record_limit = max_records or self._MAX_SPEAKER_FINAL_RECLUSTER
record_limit = self._MAX_SPEAKER_FINAL_RECLUSTER
records = (
ctx.speaker_records
if len(ctx.speaker_records) <= record_limit
else ctx.speaker_records[-record_limit:]
)
clusters, assignments = self._cluster_speaker_records(records)
clusterer = get_realtime_speaker_clusterer()
clusters, assignments, timeline = clusterer.cluster_records_with_stable_history(
records,
ctx.speaker_centers,
ctx.speaker_center_updates,
update_centers=False,
)
if not clusters:
return []
ctx.speaker_timeline = timeline
cluster_labels: dict[int, Dict[str, Any]] = {}
unknown_display_index = 1
for idx, cluster in enumerate(clusters):
for cluster in clusters:
idx = int(cluster["cluster_index"])
named_records = [
item for item in cluster["record_refs"]
if item.get("registry_speaker_id")
@ -802,9 +813,9 @@ class Qwen3ASRService:
),
)
speaker_id = preferred.get("registry_speaker_id") or preferred.get("speaker_id")
speaker_name = preferred.get("speaker_name") or speaker_id or f"Speaker{unknown_display_index:02d}"
speaker_name = preferred.get("speaker_name") or speaker_id or f"Speaker{idx + 1:02d}"
cluster_labels[idx] = {
"speaker_id": speaker_id or f"Speaker{unknown_display_index:02d}",
"speaker_id": speaker_id or f"Speaker{idx + 1:02d}",
"speaker_name": speaker_name,
"user_id": preferred.get("user_id"),
"registry_speaker_id": preferred.get("registry_speaker_id"),
@ -812,7 +823,7 @@ class Qwen3ASRService:
"speaker_confidence": 1.0,
}
else:
speaker_id = f"Speaker{unknown_display_index:02d}"
speaker_id = f"Speaker{idx + 1:02d}"
cluster_labels[idx] = {
"speaker_id": speaker_id,
"speaker_name": speaker_id,
@ -820,10 +831,13 @@ class Qwen3ASRService:
"speaker_strategy": "final_recluster",
"speaker_confidence": 1.0,
}
unknown_display_index += 1
ctx.speaker_cluster_labels = cluster_labels
updates: List[Dict[str, Any]] = []
for record, cluster_idx in zip(records, assignments):
if cluster_idx < 0 or cluster_idx not in cluster_labels:
continue
segment_index = int(record["index"])
segment = next(
(item for item in ctx.confirmed_segments if int(item.get("index", -1)) == segment_index),
@ -831,15 +845,7 @@ class Qwen3ASRService:
)
if segment is None:
continue
if only_pending_updates and not segment.get("speaker_pending"):
continue
new_info = cluster_labels[cluster_idx]
if only_stable_updates:
has_named_identity = bool(new_info.get("registry_speaker_id") or new_info.get("user_id"))
generic_name = str(new_info.get("speaker_name") or "").strip()
if not has_named_identity and re.fullmatch(r"Speaker\d+", generic_name):
if not segment.get("speaker_pending"):
continue
changed = any(segment.get(key) != value for key, value in new_info.items())
if not changed and not segment.get("speaker_pending"):
continue
@ -854,18 +860,128 @@ class Qwen3ASRService:
return updates
def _maybe_recluster_recent_segments(
def _build_final_speaker_segments(
self,
ctx: ConnectionContext,
*,
min_split_sec: float = 3.0,
) -> List[Dict[str, Any]]:
if len(ctx.speaker_records) < self._MIN_REALTIME_RECLUSTER_SEGMENTS:
return []
return self._recluster_confirmed_segments(
ctx,
max_records=self._MAX_SPEAKER_REALTIME_RECLUSTER,
only_stable_updates=True,
only_pending_updates=True,
)
"""Split final ASR text where the FunASR speaker timeline changes.
ASR does not provide character timestamps, so text is apportioned by
each speaker's share of the overlapping audio duration, matching the
approximation used by FunASR's realtime tracker.
"""
if not ctx.speaker_timeline:
return [dict(item) for item in ctx.confirmed_segments]
history_start_ms = min(int(item["start_ms"]) for item in ctx.speaker_timeline)
def speaker_info(cluster_index: int) -> Dict[str, Any]:
label = ctx.speaker_cluster_labels.get(cluster_index)
if label:
return label
generic_id = f"Speaker{cluster_index + 1:02d}"
return {"speaker_id": generic_id, "speaker_name": generic_id}
final_segments: List[Dict[str, Any]] = []
for source_segment in ctx.confirmed_segments:
original = dict(source_segment)
seg_start = int(original.get("start_ms", 0))
seg_end = int(original.get("end_ms", seg_start))
if seg_start < history_start_ms < seg_end:
# FunASR keeps any text before the 128-chunk history window
# intact and only reassigns the portion covered by diarization.
text = str(original.get("text") or "")
ratio = (history_start_ms - seg_start) / max(seg_end - seg_start, 1)
split_index = min(len(text) - 1, max(1, round(len(text) * ratio)))
prefix_text = text[:split_index].strip()
suffix_text = text[split_index:].strip()
if prefix_text and suffix_text:
prefix = dict(original)
prefix.update(index=len(final_segments), text=prefix_text, end_ms=history_start_ms)
final_segments.append(prefix)
original["text"] = suffix_text
original["start_ms"] = history_start_ms
seg_start = history_start_ms
overlaps: List[List[int]] = []
for item in ctx.speaker_timeline:
start = max(seg_start, int(item["start_ms"]))
end = min(seg_end, int(item["end_ms"]))
if end <= start:
continue
cluster_index = int(item["cluster_index"])
if overlaps and overlaps[-1][2] == cluster_index and start <= overlaps[-1][1]:
overlaps[-1][1] = end
else:
overlaps.append([start, end, cluster_index])
if not overlaps:
segment = dict(original)
segment["index"] = len(final_segments)
final_segments.append(segment)
continue
# FunASR merges same-speaker regions and absorbs regions shorter
# than three seconds into the neighboring speaker.
filtered: List[List[int]] = [overlaps[0][:]]
for current in overlaps[1:]:
previous = filtered[-1]
if current[2] == previous[2]:
previous[1] = current[1]
elif (current[1] - current[0]) < min_split_sec * 1000:
previous[1] = current[1]
else:
filtered.append(current[:])
merged: List[List[int]] = [filtered[0][:]]
for current in filtered[1:]:
previous = merged[-1]
if (previous[1] - previous[0]) < min_split_sec * 1000:
merged[-1] = [previous[0], current[1], current[2]]
else:
merged.append(current[:])
if len(merged) > 1 and (merged[-1][1] - merged[-1][0]) < min_split_sec * 1000:
merged[-2][1] = merged[-1][1]
merged.pop()
if len(merged) <= 1:
segment = dict(original)
segment.update(speaker_info(merged[0][2]))
segment["index"] = len(final_segments)
final_segments.append(segment)
continue
total_duration = sum(end - start for start, end, _ in merged)
text = str(original.get("text") or "")
if len(text) < len(merged):
longest = max(merged, key=lambda item: item[1] - item[0])
segment = dict(original)
segment.update(speaker_info(longest[2]))
segment["index"] = len(final_segments)
final_segments.append(segment)
continue
char_pos = 0
for part_index, (start, end, cluster_index) in enumerate(merged):
if part_index == len(merged) - 1:
part_text = text[char_pos:]
else:
remaining_parts = len(merged) - part_index - 1
proposed = int(len(text) * (end - start) / max(total_duration, 1))
char_count = max(1, min(proposed, len(text) - char_pos - remaining_parts))
part_text = text[char_pos:char_pos + char_count]
char_pos += char_count
if not part_text.strip():
continue
fragment = dict(original)
fragment.update(speaker_info(cluster_index))
fragment.update(
index=len(final_segments),
text=part_text.strip(),
start_ms=start,
end_ms=end,
)
final_segments.append(fragment)
return final_segments
def _build_sv_chunks(self, audio: np.ndarray) -> List[np.ndarray]:
return get_realtime_speaker_clusterer().build_sv_chunks(audio)
@ -2427,6 +2543,8 @@ class Qwen3ASRService:
segment_end_ms=segment_end_ms,
enable_registry_match=bool(ctx.params.get("enable_speaker_identification", True)),
speaker_threshold=ctx.params.get("speaker_threshold"),
speaker_centers=ctx.speaker_centers,
speaker_center_updates=ctx.speaker_center_updates,
)
if not speaker_info:
return None
@ -2448,11 +2566,14 @@ class Qwen3ASRService:
and not speaker_info.get("user_id")
):
safe_cluster_index = self._safe_int(cluster_index)
display_id = self._match_display_speaker_id(
ctx,
cluster_index=safe_cluster_index,
embedding=speaker_info.get("_embedding"),
)
if speaker_info.get("speaker_strategy") == "chunk_cluster":
display_id = str(speaker_info.get("speaker_id"))
else:
display_id = self._match_display_speaker_id(
ctx,
cluster_index=safe_cluster_index,
embedding=speaker_info.get("_embedding"),
)
if display_id:
speaker_info["speaker_id"] = display_id
speaker_info["speaker_name"] = display_id
@ -2507,21 +2628,6 @@ class Qwen3ASRService:
segment_end_ms=segment_end_ms,
speaker_info=speaker_info,
)
queue_backlog = ctx.speaker_job_queue.qsize()
updates: List[Dict[str, Any]] = []
if queue_backlog <= 1:
updates = self._maybe_recluster_recent_segments(ctx)
if updates:
for updated in updates:
await self._emit_speaker_update(
websocket,
ctx,
task_id,
int(updated["segment_index"]),
)
return
# Re-clustering may resolve pending IDs even if this segment's
# immediate speaker fields did not change.
if before_view != after_view:
await self._emit_speaker_update(websocket, ctx, task_id, segment_index)
except Exception as exc:
@ -3419,15 +3525,14 @@ class Qwen3ASRService:
}
)
all_texts = [
segment["text"]
for segment in ctx.confirmed_segments
if segment["text"].strip()
]
# Finalize the same timeline used by FunASR: a transcript sentence
# can become multiple speaker-specific entries at session end.
final_segments = self._build_final_speaker_segments(ctx)
all_texts = [segment["text"] for segment in final_segments if segment["text"].strip()]
full_text = "\n".join(all_texts)
public_segments = [
self._public_payload(segment) or {}
for segment in ctx.confirmed_segments
for segment in final_segments
]
await websocket.send_json(
@ -3440,7 +3545,7 @@ class Qwen3ASRService:
"final": 1,
"result": {
"slice_type": 2,
"index": max(len(ctx.confirmed_segments) - 1, 0),
"index": max(len(public_segments) - 1, 0),
"voice_text_str": full_text,
},
"sentences": [

View File

@ -4,8 +4,6 @@
from __future__ import annotations
import logging
import re
from collections import defaultdict
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
@ -24,128 +22,15 @@ class RealtimeSpeakerClusterer:
"""Sentence-level speaker attribution using chunk embeddings and session clustering."""
FUNASR_SMOOTH_MINDUR_SEC = 0.7
_FAST_ATTACH_MAX_SEC = 1.6
FUNASR_MAX_HISTORY_CHUNKS = 128
FUNASR_CENTER_MATCH_THRESHOLD = 0.6
_REGISTRY_MATCH_MIN_SEC = 2.4
def _is_generic_speaker_name(self, value: Any) -> bool:
text = str(value or "").strip()
return bool(re.fullmatch(r"Speaker\d+", text))
def _has_named_identity(self, record: Optional[Dict[str, Any]]) -> bool:
if not record:
return False
return bool(
record.get("registry_speaker_id")
or record.get("user_id")
or (
record.get("speaker_name")
and not self._is_generic_speaker_name(record.get("speaker_name"))
)
)
def _normalize_embedding(self, embedding: np.ndarray) -> np.ndarray:
emb = np.asarray(embedding, dtype=np.float32).reshape(-1)
norm = max(float(np.linalg.norm(emb)), 1e-12)
return (emb / norm).astype(np.float32)
@staticmethod
def _coerce_cluster_index(value: Any, fallback: int) -> int:
try:
if value is None:
return int(fallback)
return int(value)
except (TypeError, ValueError):
return int(fallback)
def _match_existing_speaker(
self,
speaker_records: List[Dict[str, Any]],
mean_embedding: np.ndarray,
) -> Optional[Dict[str, Any]]:
if not speaker_records:
return None
best_record: Optional[Dict[str, Any]] = None
best_score = -1.0
for record in speaker_records:
record_embedding = record.get("embedding")
if record_embedding is None:
continue
score = float(np.dot(self._normalize_embedding(record_embedding), mean_embedding))
if score > best_score:
best_score = score
best_record = record
if best_record is None:
return None
base_threshold = max(
float(getattr(settings, "REALTIME_UNKNOWN_SPK_CLUSTER_THRESHOLD", 0.58) or 0.58),
0.3,
)
has_named_identity = bool(
best_record.get("registry_speaker_id")
or best_record.get("user_id")
or (
best_record.get("speaker_name")
and not self._is_generic_speaker_name(best_record.get("speaker_name"))
)
)
threshold = max(base_threshold, 0.62 if has_named_identity else 0.72)
if best_score < threshold:
return None
matched = dict(best_record)
matched["_match_score"] = best_score
return matched
def _next_generic_speaker_id(
self,
speaker_records: List[Dict[str, Any]],
) -> str:
seen: set[int] = set()
for record in speaker_records:
for value in (
record.get("speaker_id"),
record.get("speaker_name"),
):
text = str(value or "").strip()
match = re.fullmatch(r"Speaker(\d+)", text)
if match:
seen.add(int(match.group(1)))
next_index = max(seen, default=0) + 1
return f"Speaker{next_index:02d}"
def _is_mixed_speaker_segment(
self,
speaker_records: List[Dict[str, Any]],
current_chunks: List[Dict[str, Any]],
) -> bool:
if len(current_chunks) < 2 or len(speaker_records) < 2:
return False
assigned_labels: List[str] = []
for chunk in current_chunks:
embedding = chunk.get("embedding")
if embedding is None:
continue
matched = self._match_existing_speaker(
speaker_records,
self._normalize_embedding(np.asarray(embedding, dtype=np.float32)),
)
if not matched:
continue
label = str(
matched.get("registry_speaker_id")
or matched.get("speaker_name")
or matched.get("speaker_id")
or ""
).strip()
if label:
assigned_labels.append(label)
if len(assigned_labels) < 2:
return False
return len(set(assigned_labels)) >= 2
def _correct_labels(self, labels: np.ndarray) -> np.ndarray:
labels_id = 0
id2id: dict[int, int] = {}
@ -216,6 +101,144 @@ class RealtimeSpeakerClusterer:
labels = self._spectral_cluster(X, oracle_num=oracle_num)
return self._merge_by_cos(labels, X, cos_thr=0.78 if oracle_num is None else 1.0)
def _map_cluster_centers(
self,
cluster_centers: np.ndarray,
speaker_centers: List[np.ndarray],
speaker_center_updates: List[int],
*,
update_centers: bool,
) -> List[int]:
"""Map each window's temporary cluster label to a persistent speaker ID.
FunASR's ClusterBackend labels are local to each clustering pass. Its
realtime tracker maps those labels through cosine-matched centroids so
speakers keep the same identity as the 128-chunk window moves.
"""
centers = np.stack(
[self._normalize_embedding(center) for center in np.asarray(cluster_centers)],
axis=0,
)
stable_ids: List[int] = []
used_ids: set[int] = set()
for center in centers:
best_id: Optional[int] = None
best_score = -1.0
for candidate, known_center in enumerate(speaker_centers):
if candidate in used_ids:
continue
score = float(np.dot(center, self._normalize_embedding(known_center)))
if score > best_score:
best_id, best_score = candidate, score
matched = best_id is not None and best_score >= self.FUNASR_CENTER_MATCH_THRESHOLD
created = False
if not matched and update_centers and len(speaker_centers) < 15:
best_id = len(speaker_centers)
speaker_centers.append(center.copy())
speaker_center_updates.append(1)
matched = True
created = True
elif best_id is None:
# Match FunASR's max-speaker fallback when all old IDs were used.
if not speaker_centers:
best_id = 0
else:
scores = [float(np.dot(center, self._normalize_embedding(item))) for item in speaker_centers]
best_id = int(np.argmax(scores))
if update_centers and matched and not created:
assert best_id is not None
count = speaker_center_updates[best_id]
weight = 1.0 / min(count + 1, 20)
updated = (1.0 - weight) * speaker_centers[best_id] + weight * center
speaker_centers[best_id] = self._normalize_embedding(updated)
speaker_center_updates[best_id] = count + 1
assert best_id is not None
stable_ids.append(best_id)
used_ids.add(best_id)
return stable_ids
def cluster_chunk_history(
self,
chunks: List[Dict[str, Any]],
speaker_centers: List[np.ndarray],
speaker_center_updates: List[int],
*,
update_centers: bool,
) -> Tuple[List[Dict[str, Any]], List[int]]:
"""Cluster a fixed-size rolling window of timestamped speaker chunks."""
valid_chunks = [item for item in chunks if item.get("embedding") is not None]
valid_chunks = valid_chunks[-self.FUNASR_MAX_HISTORY_CHUNKS:]
if not valid_chunks:
return [], []
embeddings = np.stack(
[self._normalize_embedding(np.asarray(item["embedding"], dtype=np.float32)) for item in valid_chunks],
axis=0,
)
labels = self._cluster_embeddings(embeddings, oracle_num=None)
temporary_ids = sorted({int(value) for value in labels.tolist()})
centers = np.stack(
[self._normalize_embedding(embeddings[labels == label].mean(axis=0)) for label in temporary_ids],
axis=0,
)
stable_ids = self._map_cluster_centers(
centers,
speaker_centers,
speaker_center_updates,
update_centers=update_centers,
)
temp_to_stable = dict(zip(temporary_ids, stable_ids))
timeline = self._postprocess_timeline(valid_chunks, labels, embeddings)
for item in timeline:
item["cluster_index"] = int(temp_to_stable[int(item["cluster_index"])])
return timeline, stable_ids
def flatten_chunk_history(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Return the most recent FunASR-sized history from segment records."""
chunks: List[Dict[str, Any]] = []
for record in records:
entries = record.get("chunks") or []
if entries:
chunks.extend(entries)
continue
embeddings = record.get("embeddings") or []
if not embeddings and record.get("embedding") is not None:
embeddings = [record["embedding"]]
for embedding in embeddings:
chunks.append({
"start_ms": int(record.get("start_ms", 0)),
"end_ms": int(record.get("end_ms", record.get("start_ms", 0))),
"embedding": embedding,
})
return chunks[-self.FUNASR_MAX_HISTORY_CHUNKS:]
@staticmethod
def pick_timeline_speaker(
timeline: List[Dict[str, Any]],
*,
start_ms: int,
end_ms: int,
) -> Optional[int]:
"""Assign a sentence to its dominant overlapping speaker, as FunASR does."""
sentence_speaker = 0
max_overlap = 0
for item in timeline:
overlap = min(end_ms, int(item["end_ms"])) - max(start_ms, int(item["start_ms"]))
if overlap > 0:
speaker = int(item["cluster_index"])
if overlap > max_overlap:
max_overlap = overlap
sentence_speaker = speaker
if sentence_speaker == speaker:
max_overlap += overlap
if max_overlap <= 0:
return None
return sentence_speaker
def build_sv_chunks(
self,
audio: np.ndarray,
@ -300,42 +323,6 @@ class RealtimeSpeakerClusterer:
logger.warning("Realtime registry speaker embedding failed: %s", exc)
return None
def _flatten_chunk_records(
self,
records: List[Dict[str, Any]],
) -> Tuple[List[Dict[str, Any]], np.ndarray]:
flat_chunks: List[Dict[str, Any]] = []
flat_embeddings: List[np.ndarray] = []
for record_idx, record in enumerate(records):
chunk_entries = record.get("chunks") or []
if not chunk_entries:
embeddings = record.get("embeddings") or []
if not embeddings and record.get("embedding") is not None:
embeddings = [record["embedding"]]
start_ms = int(record.get("start_ms", 0))
end_ms = int(record.get("end_ms", start_ms))
for embedding in embeddings:
chunk_entries.append(
{
"start_ms": start_ms,
"end_ms": end_ms,
"embedding": embedding,
}
)
for chunk in chunk_entries:
emb = self._normalize_embedding(np.asarray(chunk["embedding"], dtype=np.float32))
flat_chunks.append(
{
"record_index": record_idx,
"start_ms": int(chunk.get("start_ms", record.get("start_ms", 0))),
"end_ms": int(chunk.get("end_ms", record.get("end_ms", 0))),
}
)
flat_embeddings.append(emb)
if not flat_embeddings:
return [], np.zeros((0, 0), dtype=np.float32)
return flat_chunks, np.stack(flat_embeddings, axis=0).astype(np.float32)
def _merge_seque(self, distribute_res: List[List[float]]) -> List[List[float]]:
if not distribute_res:
return []
@ -406,82 +393,66 @@ class RealtimeSpeakerClusterer:
for item in distribute_res
]
def _pick_segment_cluster(
self,
cluster_ranges: List[Dict[str, Any]],
*,
segment_start_ms: int,
segment_end_ms: int,
) -> int:
overlaps: Dict[int, int] = defaultdict(int)
for item in cluster_ranges:
overlap = min(segment_end_ms, int(item["end_ms"])) - max(segment_start_ms, int(item["start_ms"]))
if overlap > 0:
overlaps[int(item["cluster_index"])] += int(overlap)
if overlaps:
return max(overlaps.items(), key=lambda kv: (kv[1], -kv[0]))[0]
centers = [
item for item in cluster_ranges
if int(item["start_ms"]) <= segment_end_ms and int(item["end_ms"]) >= segment_start_ms
]
if centers:
return int(centers[0]["cluster_index"])
return 0
def cluster_records_with_ranges(
self,
records: List[Dict[str, Any]],
) -> Tuple[List[Dict[str, Any]], List[int], List[Dict[str, Any]]]:
flat_chunks, X = self._flatten_chunk_records(records)
if X.size == 0:
return [], [], []
# Preserve the existing helper API while routing it through the same
# 128-chunk policy used by live sessions.
return self.cluster_records_with_stable_history(records, [], [], update_centers=True)
labels = self._cluster_embeddings(X, oracle_num=None)
cluster_ranges = self._postprocess_timeline(flat_chunks, labels, X)
cluster_ids = sorted({int(item["cluster_index"]) for item in cluster_ranges})
if not cluster_ids:
cluster_ids = sorted({int(label) for label in labels.tolist()})
clusters: List[Dict[str, Any]] = []
for cluster_idx in cluster_ids:
member_mask = labels == int(cluster_idx)
member_embeddings = X[member_mask]
centroid = self._normalize_embedding(member_embeddings.mean(0))
record_refs = []
seen_record_indices = set()
for emb_idx, chunk in enumerate(flat_chunks):
record_idx = int(chunk["record_index"])
if not member_mask[emb_idx] or record_idx in seen_record_indices:
continue
seen_record_indices.add(record_idx)
record_refs.append(records[record_idx])
clusters.append(
{
"centroid": centroid,
"count": int(member_mask.sum()),
"record_refs": record_refs,
}
def cluster_records_with_stable_history(
self,
records: List[Dict[str, Any]],
speaker_centers: List[np.ndarray],
speaker_center_updates: List[int],
*,
update_centers: bool,
) -> Tuple[List[Dict[str, Any]], List[int], List[Dict[str, Any]]]:
"""Cluster only the last 128 chunks and return persistent speaker IDs."""
chunks = self.flatten_chunk_history(records)
timeline, _ = self.cluster_chunk_history(
chunks,
speaker_centers,
speaker_center_updates,
update_centers=update_centers,
)
assignments = [
self.pick_timeline_speaker(
timeline,
start_ms=int(record.get("start_ms", 0)),
end_ms=int(record.get("end_ms", record.get("start_ms", 0))),
)
record_to_labels: List[List[int]] = [[] for _ in records]
for emb_idx, chunk in enumerate(flat_chunks):
record_to_labels[int(chunk["record_index"])].append(int(labels[emb_idx]))
record_assignments: List[int] = []
for record, local_assignments in zip(records, record_to_labels):
if local_assignments:
cluster_idx = self._pick_segment_cluster(
cluster_ranges,
segment_start_ms=int(record.get("start_ms", 0)),
segment_end_ms=int(record.get("end_ms", record.get("start_ms", 0))),
)
record_assignments.append(cluster_idx)
for record in records
]
cluster_ids = sorted({int(item["cluster_index"]) for item in timeline})
clusters: List[Dict[str, Any]] = []
for cluster_id in cluster_ids:
refs = [
record
for record, assigned_id in zip(records, assignments)
if assigned_id == cluster_id
]
if cluster_id < len(speaker_centers):
center = speaker_centers[cluster_id]
else:
record_assignments.append(-1)
return clusters, record_assignments, cluster_ranges
matching_embeddings = [
item["embedding"]
for item in chunks
if self.pick_timeline_speaker(
timeline,
start_ms=int(item["start_ms"]),
end_ms=int(item["end_ms"]),
) == cluster_id
]
center = np.mean(matching_embeddings or [chunks[0]["embedding"]], axis=0)
clusters.append({
"cluster_index": cluster_id,
"centroid": self._normalize_embedding(np.asarray(center, dtype=np.float32)),
"count": sum(1 for item in timeline if int(item["cluster_index"]) == cluster_id),
"record_refs": refs,
})
return clusters, [-1 if value is None else int(value) for value in assignments], timeline
def cluster_records(
self,
@ -499,104 +470,47 @@ class RealtimeSpeakerClusterer:
segment_end_ms: int,
enable_registry_match: bool,
speaker_threshold: Optional[float],
speaker_centers: List[np.ndarray],
speaker_center_updates: List[int],
) -> Optional[Dict[str, Any]]:
duration_sec = float(len(audio)) / 16000.0
last_record = speaker_records[-1] if speaker_records else None
if duration_sec < self._FAST_ATTACH_MAX_SEC and speaker_records:
if self._has_named_identity(last_record):
speaker_id = last_record.get("registry_speaker_id") or last_record.get("speaker_id") or "Speaker01"
speaker_name = last_record.get("speaker_name") or speaker_id
return {
"speaker_id": speaker_id,
"speaker_name": speaker_name,
"user_id": last_record.get("user_id"),
"registry_speaker_id": last_record.get("registry_speaker_id"),
"speaker_confidence": 0.0,
"speaker_strategy": "short_attach",
"_cluster_index": last_record.get("cluster_index"),
"_embedding": last_record.get("embedding"),
"_chunk_embeddings": last_record.get("embeddings") or [],
"_chunks": last_record.get("chunks") or [],
}
current_chunks = await self.extract_chunk_embeddings(
np.asarray(audio, dtype=np.float32),
segment_start_ms=segment_start_ms,
)
if not current_chunks:
if speaker_records:
if self._has_named_identity(last_record):
speaker_id = last_record.get("registry_speaker_id") or last_record.get("speaker_id") or "Speaker01"
speaker_name = last_record.get("speaker_name") or speaker_id
return {
"speaker_id": speaker_id,
"speaker_name": speaker_name,
"user_id": last_record.get("user_id"),
"registry_speaker_id": last_record.get("registry_speaker_id"),
"speaker_confidence": 0.0,
"speaker_strategy": "embedding_attach",
"_cluster_index": last_record.get("cluster_index"),
"_embedding": last_record.get("embedding"),
"_chunk_embeddings": last_record.get("embeddings") or [],
"_chunks": last_record.get("chunks") or [],
}
return None
mean_embedding = self._normalize_embedding(
np.mean(np.stack([chunk["embedding"] for chunk in current_chunks], axis=0), axis=0)
)
if duration_sec >= 4.0 and self._is_mixed_speaker_segment(speaker_records, current_chunks):
return {
"speaker_id": -1,
"speaker_name": "",
"user_id": None,
"registry_speaker_id": None,
"speaker_confidence": 0.0,
"speaker_strategy": "mixed_segment",
"_cluster_index": None,
"_embedding": mean_embedding,
"_chunk_embeddings": [np.asarray(chunk["embedding"], dtype=np.float32) for chunk in current_chunks],
"_chunks": current_chunks,
}
matched_record = self._match_existing_speaker(speaker_records, mean_embedding)
speaker_id = self._next_generic_speaker_id(speaker_records)
# Re-cluster the same fixed number of recent chunks as FunASR. Chunk
# labels are temporary, so the session-owned centers keep IDs stable.
history_chunks = self.flatten_chunk_history(speaker_records)
timeline, _ = self.cluster_chunk_history(
history_chunks + current_chunks,
speaker_centers,
speaker_center_updates,
update_centers=True,
)
cluster_index = self.pick_timeline_speaker(
timeline,
start_ms=segment_start_ms,
end_ms=segment_end_ms,
)
if cluster_index is None:
cluster_index = max(len(speaker_centers) - 1, 0)
speaker_id = f"Speaker{cluster_index + 1:02d}"
speaker_name = speaker_id
user_id = None
registry_speaker_id = None
strategy = "new_speaker"
cluster_index = max(
[
int(record.get("cluster_index", -1))
for record in speaker_records
if record.get("cluster_index") is not None
] or [-1]
) + 1
confidence = 0.0
if matched_record is not None:
speaker_id = (
matched_record.get("registry_speaker_id")
or matched_record.get("speaker_id")
or speaker_id
)
speaker_name = matched_record.get("speaker_name") or speaker_id
user_id = matched_record.get("user_id")
registry_speaker_id = matched_record.get("registry_speaker_id")
cluster_index = self._coerce_cluster_index(
matched_record.get("cluster_index"),
cluster_index,
)
confidence = float(matched_record.get("_match_score", 0.0))
strategy = "embedding_match"
confidence = 1.0
strategy = "chunk_cluster"
if (
not registry_speaker_id
and enable_registry_match
enable_registry_match
and duration_sec >= self._REGISTRY_MATCH_MIN_SEC
and (
matched_record is None
or confidence >= max(float(getattr(settings, "REALTIME_SPEAKER_CONFIRM_THRESHOLD", 0.62) or 0.62), 0.7)
)
):
# Realtime clustering can use a dedicated fast model, while the
# registry stores embeddings from the registration model. Match the
@ -622,11 +536,12 @@ class RealtimeSpeakerClusterer:
"registry_speaker_id": registry_speaker_id,
"speaker_confidence": round(confidence, 4),
"speaker_strategy": strategy,
"_cluster_index": self._coerce_cluster_index(cluster_index, 0),
"_cluster_index": int(cluster_index),
"_embedding": mean_embedding,
"_chunk_embeddings": [np.asarray(chunk["embedding"], dtype=np.float32) for chunk in current_chunks],
"_chunks": current_chunks,
"_matched_existing": matched_record is not None,
"_matched_existing": True,
"_speaker_timeline": timeline,
}