对齐FunASR滑动窗口聚类
parent
e905aefc94
commit
3af8f168f6
|
|
@ -264,6 +264,11 @@ class ConnectionContext:
|
|||
total_samples: int = 0
|
||||
confirmed_segments: List[Dict[str, Any]] = field(default_factory=list)
|
||||
speaker_records: List[Dict[str, Any]] = field(default_factory=list)
|
||||
# FunASR-style persistent speaker centroids, indexed by stable speaker ID.
|
||||
speaker_centers: List[np.ndarray] = field(default_factory=list)
|
||||
speaker_center_updates: List[int] = field(default_factory=list)
|
||||
speaker_timeline: List[Dict[str, Any]] = field(default_factory=list)
|
||||
speaker_cluster_labels: Dict[int, Dict[str, Any]] = field(default_factory=dict)
|
||||
speaker_display_map: Dict[int, str] = field(default_factory=dict)
|
||||
speaker_display_profiles: Dict[str, np.ndarray] = field(default_factory=dict)
|
||||
speaker_display_counts: Dict[str, int] = field(default_factory=dict)
|
||||
|
|
@ -304,11 +309,8 @@ class SessionEntry:
|
|||
|
||||
|
||||
class Qwen3ASRService:
|
||||
_MAX_SPEAKER_HISTORY_RECORDS = 64
|
||||
_MAX_SPEAKER_MATCH_CONTEXT = 24
|
||||
_MAX_SPEAKER_FINAL_RECLUSTER = 48
|
||||
_MAX_SPEAKER_REALTIME_RECLUSTER = 12
|
||||
_MIN_REALTIME_RECLUSTER_SEGMENTS = 5
|
||||
_MAX_SPEAKER_HISTORY_RECORDS = 128
|
||||
_MAX_SPEAKER_FINAL_RECLUSTER = 128
|
||||
_MIN_HARD_SEGMENT_SEC = 12.0
|
||||
_SEGMENT_EVENT_TEXT_TAIL_CHARS = 2000
|
||||
_SEGMENT_EVENT_TAIL_COUNT = 8
|
||||
|
|
@ -633,6 +635,8 @@ class Qwen3ASRService:
|
|||
strategy = str(speaker_info.get("speaker_strategy") or "")
|
||||
if strategy == "registry_match":
|
||||
return True
|
||||
if strategy == "chunk_cluster":
|
||||
return True
|
||||
if strategy in {"recent_named_attach", "recent_named_inherit"}:
|
||||
confidence = float(speaker_info.get("speaker_confidence") or 0.0)
|
||||
return confidence >= 0.7
|
||||
|
|
@ -701,6 +705,8 @@ class Qwen3ASRService:
|
|||
ctx: ConnectionContext,
|
||||
speaker_info: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if speaker_info.get("speaker_strategy") == "chunk_cluster":
|
||||
return None
|
||||
if speaker_info.get("registry_speaker_id") or speaker_info.get("user_id"):
|
||||
return None
|
||||
if speaker_info.get("_matched_existing"):
|
||||
|
|
@ -728,6 +734,8 @@ class Qwen3ASRService:
|
|||
ctx: ConnectionContext,
|
||||
speaker_info: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if speaker_info.get("speaker_strategy") == "chunk_cluster":
|
||||
return None
|
||||
if speaker_info.get("registry_speaker_id") or speaker_info.get("user_id"):
|
||||
return speaker_info
|
||||
if speaker_info.get("_matched_existing"):
|
||||
|
|
@ -755,34 +763,37 @@ class Qwen3ASRService:
|
|||
self,
|
||||
ctx: ConnectionContext,
|
||||
) -> List[Dict[str, Any]]:
|
||||
if len(ctx.speaker_records) <= self._MAX_SPEAKER_MATCH_CONTEXT:
|
||||
return ctx.speaker_records
|
||||
return ctx.speaker_records[-self._MAX_SPEAKER_MATCH_CONTEXT:]
|
||||
# The clusterer applies its limit by chunk count (128), not by the
|
||||
# variable number of ASR segments those chunks happen to span.
|
||||
return ctx.speaker_records
|
||||
|
||||
def _recluster_confirmed_segments(
|
||||
self,
|
||||
ctx: ConnectionContext,
|
||||
*,
|
||||
max_records: Optional[int] = None,
|
||||
only_stable_updates: bool = False,
|
||||
only_pending_updates: bool = False,
|
||||
) -> List[Dict[str, Any]]:
|
||||
if len(ctx.speaker_records) < 2:
|
||||
if not ctx.speaker_records:
|
||||
return []
|
||||
|
||||
record_limit = max_records or self._MAX_SPEAKER_FINAL_RECLUSTER
|
||||
record_limit = self._MAX_SPEAKER_FINAL_RECLUSTER
|
||||
records = (
|
||||
ctx.speaker_records
|
||||
if len(ctx.speaker_records) <= record_limit
|
||||
else ctx.speaker_records[-record_limit:]
|
||||
)
|
||||
clusters, assignments = self._cluster_speaker_records(records)
|
||||
clusterer = get_realtime_speaker_clusterer()
|
||||
clusters, assignments, timeline = clusterer.cluster_records_with_stable_history(
|
||||
records,
|
||||
ctx.speaker_centers,
|
||||
ctx.speaker_center_updates,
|
||||
update_centers=False,
|
||||
)
|
||||
if not clusters:
|
||||
return []
|
||||
ctx.speaker_timeline = timeline
|
||||
|
||||
cluster_labels: dict[int, Dict[str, Any]] = {}
|
||||
unknown_display_index = 1
|
||||
for idx, cluster in enumerate(clusters):
|
||||
for cluster in clusters:
|
||||
idx = int(cluster["cluster_index"])
|
||||
named_records = [
|
||||
item for item in cluster["record_refs"]
|
||||
if item.get("registry_speaker_id")
|
||||
|
|
@ -802,9 +813,9 @@ class Qwen3ASRService:
|
|||
),
|
||||
)
|
||||
speaker_id = preferred.get("registry_speaker_id") or preferred.get("speaker_id")
|
||||
speaker_name = preferred.get("speaker_name") or speaker_id or f"Speaker{unknown_display_index:02d}"
|
||||
speaker_name = preferred.get("speaker_name") or speaker_id or f"Speaker{idx + 1:02d}"
|
||||
cluster_labels[idx] = {
|
||||
"speaker_id": speaker_id or f"Speaker{unknown_display_index:02d}",
|
||||
"speaker_id": speaker_id or f"Speaker{idx + 1:02d}",
|
||||
"speaker_name": speaker_name,
|
||||
"user_id": preferred.get("user_id"),
|
||||
"registry_speaker_id": preferred.get("registry_speaker_id"),
|
||||
|
|
@ -812,7 +823,7 @@ class Qwen3ASRService:
|
|||
"speaker_confidence": 1.0,
|
||||
}
|
||||
else:
|
||||
speaker_id = f"Speaker{unknown_display_index:02d}"
|
||||
speaker_id = f"Speaker{idx + 1:02d}"
|
||||
cluster_labels[idx] = {
|
||||
"speaker_id": speaker_id,
|
||||
"speaker_name": speaker_id,
|
||||
|
|
@ -820,10 +831,13 @@ class Qwen3ASRService:
|
|||
"speaker_strategy": "final_recluster",
|
||||
"speaker_confidence": 1.0,
|
||||
}
|
||||
unknown_display_index += 1
|
||||
|
||||
ctx.speaker_cluster_labels = cluster_labels
|
||||
|
||||
updates: List[Dict[str, Any]] = []
|
||||
for record, cluster_idx in zip(records, assignments):
|
||||
if cluster_idx < 0 or cluster_idx not in cluster_labels:
|
||||
continue
|
||||
segment_index = int(record["index"])
|
||||
segment = next(
|
||||
(item for item in ctx.confirmed_segments if int(item.get("index", -1)) == segment_index),
|
||||
|
|
@ -831,15 +845,7 @@ class Qwen3ASRService:
|
|||
)
|
||||
if segment is None:
|
||||
continue
|
||||
if only_pending_updates and not segment.get("speaker_pending"):
|
||||
continue
|
||||
new_info = cluster_labels[cluster_idx]
|
||||
if only_stable_updates:
|
||||
has_named_identity = bool(new_info.get("registry_speaker_id") or new_info.get("user_id"))
|
||||
generic_name = str(new_info.get("speaker_name") or "").strip()
|
||||
if not has_named_identity and re.fullmatch(r"Speaker\d+", generic_name):
|
||||
if not segment.get("speaker_pending"):
|
||||
continue
|
||||
changed = any(segment.get(key) != value for key, value in new_info.items())
|
||||
if not changed and not segment.get("speaker_pending"):
|
||||
continue
|
||||
|
|
@ -854,18 +860,128 @@ class Qwen3ASRService:
|
|||
|
||||
return updates
|
||||
|
||||
def _maybe_recluster_recent_segments(
|
||||
def _build_final_speaker_segments(
|
||||
self,
|
||||
ctx: ConnectionContext,
|
||||
*,
|
||||
min_split_sec: float = 3.0,
|
||||
) -> List[Dict[str, Any]]:
|
||||
if len(ctx.speaker_records) < self._MIN_REALTIME_RECLUSTER_SEGMENTS:
|
||||
return []
|
||||
return self._recluster_confirmed_segments(
|
||||
ctx,
|
||||
max_records=self._MAX_SPEAKER_REALTIME_RECLUSTER,
|
||||
only_stable_updates=True,
|
||||
only_pending_updates=True,
|
||||
)
|
||||
"""Split final ASR text where the FunASR speaker timeline changes.
|
||||
|
||||
ASR does not provide character timestamps, so text is apportioned by
|
||||
each speaker's share of the overlapping audio duration, matching the
|
||||
approximation used by FunASR's realtime tracker.
|
||||
"""
|
||||
if not ctx.speaker_timeline:
|
||||
return [dict(item) for item in ctx.confirmed_segments]
|
||||
history_start_ms = min(int(item["start_ms"]) for item in ctx.speaker_timeline)
|
||||
|
||||
def speaker_info(cluster_index: int) -> Dict[str, Any]:
|
||||
label = ctx.speaker_cluster_labels.get(cluster_index)
|
||||
if label:
|
||||
return label
|
||||
generic_id = f"Speaker{cluster_index + 1:02d}"
|
||||
return {"speaker_id": generic_id, "speaker_name": generic_id}
|
||||
|
||||
final_segments: List[Dict[str, Any]] = []
|
||||
for source_segment in ctx.confirmed_segments:
|
||||
original = dict(source_segment)
|
||||
seg_start = int(original.get("start_ms", 0))
|
||||
seg_end = int(original.get("end_ms", seg_start))
|
||||
if seg_start < history_start_ms < seg_end:
|
||||
# FunASR keeps any text before the 128-chunk history window
|
||||
# intact and only reassigns the portion covered by diarization.
|
||||
text = str(original.get("text") or "")
|
||||
ratio = (history_start_ms - seg_start) / max(seg_end - seg_start, 1)
|
||||
split_index = min(len(text) - 1, max(1, round(len(text) * ratio)))
|
||||
prefix_text = text[:split_index].strip()
|
||||
suffix_text = text[split_index:].strip()
|
||||
if prefix_text and suffix_text:
|
||||
prefix = dict(original)
|
||||
prefix.update(index=len(final_segments), text=prefix_text, end_ms=history_start_ms)
|
||||
final_segments.append(prefix)
|
||||
original["text"] = suffix_text
|
||||
original["start_ms"] = history_start_ms
|
||||
seg_start = history_start_ms
|
||||
overlaps: List[List[int]] = []
|
||||
for item in ctx.speaker_timeline:
|
||||
start = max(seg_start, int(item["start_ms"]))
|
||||
end = min(seg_end, int(item["end_ms"]))
|
||||
if end <= start:
|
||||
continue
|
||||
cluster_index = int(item["cluster_index"])
|
||||
if overlaps and overlaps[-1][2] == cluster_index and start <= overlaps[-1][1]:
|
||||
overlaps[-1][1] = end
|
||||
else:
|
||||
overlaps.append([start, end, cluster_index])
|
||||
|
||||
if not overlaps:
|
||||
segment = dict(original)
|
||||
segment["index"] = len(final_segments)
|
||||
final_segments.append(segment)
|
||||
continue
|
||||
|
||||
# FunASR merges same-speaker regions and absorbs regions shorter
|
||||
# than three seconds into the neighboring speaker.
|
||||
filtered: List[List[int]] = [overlaps[0][:]]
|
||||
for current in overlaps[1:]:
|
||||
previous = filtered[-1]
|
||||
if current[2] == previous[2]:
|
||||
previous[1] = current[1]
|
||||
elif (current[1] - current[0]) < min_split_sec * 1000:
|
||||
previous[1] = current[1]
|
||||
else:
|
||||
filtered.append(current[:])
|
||||
|
||||
merged: List[List[int]] = [filtered[0][:]]
|
||||
for current in filtered[1:]:
|
||||
previous = merged[-1]
|
||||
if (previous[1] - previous[0]) < min_split_sec * 1000:
|
||||
merged[-1] = [previous[0], current[1], current[2]]
|
||||
else:
|
||||
merged.append(current[:])
|
||||
if len(merged) > 1 and (merged[-1][1] - merged[-1][0]) < min_split_sec * 1000:
|
||||
merged[-2][1] = merged[-1][1]
|
||||
merged.pop()
|
||||
|
||||
if len(merged) <= 1:
|
||||
segment = dict(original)
|
||||
segment.update(speaker_info(merged[0][2]))
|
||||
segment["index"] = len(final_segments)
|
||||
final_segments.append(segment)
|
||||
continue
|
||||
|
||||
total_duration = sum(end - start for start, end, _ in merged)
|
||||
text = str(original.get("text") or "")
|
||||
if len(text) < len(merged):
|
||||
longest = max(merged, key=lambda item: item[1] - item[0])
|
||||
segment = dict(original)
|
||||
segment.update(speaker_info(longest[2]))
|
||||
segment["index"] = len(final_segments)
|
||||
final_segments.append(segment)
|
||||
continue
|
||||
char_pos = 0
|
||||
for part_index, (start, end, cluster_index) in enumerate(merged):
|
||||
if part_index == len(merged) - 1:
|
||||
part_text = text[char_pos:]
|
||||
else:
|
||||
remaining_parts = len(merged) - part_index - 1
|
||||
proposed = int(len(text) * (end - start) / max(total_duration, 1))
|
||||
char_count = max(1, min(proposed, len(text) - char_pos - remaining_parts))
|
||||
part_text = text[char_pos:char_pos + char_count]
|
||||
char_pos += char_count
|
||||
if not part_text.strip():
|
||||
continue
|
||||
fragment = dict(original)
|
||||
fragment.update(speaker_info(cluster_index))
|
||||
fragment.update(
|
||||
index=len(final_segments),
|
||||
text=part_text.strip(),
|
||||
start_ms=start,
|
||||
end_ms=end,
|
||||
)
|
||||
final_segments.append(fragment)
|
||||
return final_segments
|
||||
|
||||
def _build_sv_chunks(self, audio: np.ndarray) -> List[np.ndarray]:
|
||||
return get_realtime_speaker_clusterer().build_sv_chunks(audio)
|
||||
|
|
@ -2427,6 +2543,8 @@ class Qwen3ASRService:
|
|||
segment_end_ms=segment_end_ms,
|
||||
enable_registry_match=bool(ctx.params.get("enable_speaker_identification", True)),
|
||||
speaker_threshold=ctx.params.get("speaker_threshold"),
|
||||
speaker_centers=ctx.speaker_centers,
|
||||
speaker_center_updates=ctx.speaker_center_updates,
|
||||
)
|
||||
if not speaker_info:
|
||||
return None
|
||||
|
|
@ -2448,11 +2566,14 @@ class Qwen3ASRService:
|
|||
and not speaker_info.get("user_id")
|
||||
):
|
||||
safe_cluster_index = self._safe_int(cluster_index)
|
||||
display_id = self._match_display_speaker_id(
|
||||
ctx,
|
||||
cluster_index=safe_cluster_index,
|
||||
embedding=speaker_info.get("_embedding"),
|
||||
)
|
||||
if speaker_info.get("speaker_strategy") == "chunk_cluster":
|
||||
display_id = str(speaker_info.get("speaker_id"))
|
||||
else:
|
||||
display_id = self._match_display_speaker_id(
|
||||
ctx,
|
||||
cluster_index=safe_cluster_index,
|
||||
embedding=speaker_info.get("_embedding"),
|
||||
)
|
||||
if display_id:
|
||||
speaker_info["speaker_id"] = display_id
|
||||
speaker_info["speaker_name"] = display_id
|
||||
|
|
@ -2507,21 +2628,6 @@ class Qwen3ASRService:
|
|||
segment_end_ms=segment_end_ms,
|
||||
speaker_info=speaker_info,
|
||||
)
|
||||
queue_backlog = ctx.speaker_job_queue.qsize()
|
||||
updates: List[Dict[str, Any]] = []
|
||||
if queue_backlog <= 1:
|
||||
updates = self._maybe_recluster_recent_segments(ctx)
|
||||
if updates:
|
||||
for updated in updates:
|
||||
await self._emit_speaker_update(
|
||||
websocket,
|
||||
ctx,
|
||||
task_id,
|
||||
int(updated["segment_index"]),
|
||||
)
|
||||
return
|
||||
# Re-clustering may resolve pending IDs even if this segment's
|
||||
# immediate speaker fields did not change.
|
||||
if before_view != after_view:
|
||||
await self._emit_speaker_update(websocket, ctx, task_id, segment_index)
|
||||
except Exception as exc:
|
||||
|
|
@ -3419,15 +3525,14 @@ class Qwen3ASRService:
|
|||
}
|
||||
)
|
||||
|
||||
all_texts = [
|
||||
segment["text"]
|
||||
for segment in ctx.confirmed_segments
|
||||
if segment["text"].strip()
|
||||
]
|
||||
# Finalize the same timeline used by FunASR: a transcript sentence
|
||||
# can become multiple speaker-specific entries at session end.
|
||||
final_segments = self._build_final_speaker_segments(ctx)
|
||||
all_texts = [segment["text"] for segment in final_segments if segment["text"].strip()]
|
||||
full_text = "\n".join(all_texts)
|
||||
public_segments = [
|
||||
self._public_payload(segment) or {}
|
||||
for segment in ctx.confirmed_segments
|
||||
for segment in final_segments
|
||||
]
|
||||
|
||||
await websocket.send_json(
|
||||
|
|
@ -3440,7 +3545,7 @@ class Qwen3ASRService:
|
|||
"final": 1,
|
||||
"result": {
|
||||
"slice_type": 2,
|
||||
"index": max(len(ctx.confirmed_segments) - 1, 0),
|
||||
"index": max(len(public_segments) - 1, 0),
|
||||
"voice_text_str": full_text,
|
||||
},
|
||||
"sentences": [
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -24,128 +22,15 @@ class RealtimeSpeakerClusterer:
|
|||
"""Sentence-level speaker attribution using chunk embeddings and session clustering."""
|
||||
|
||||
FUNASR_SMOOTH_MINDUR_SEC = 0.7
|
||||
_FAST_ATTACH_MAX_SEC = 1.6
|
||||
FUNASR_MAX_HISTORY_CHUNKS = 128
|
||||
FUNASR_CENTER_MATCH_THRESHOLD = 0.6
|
||||
_REGISTRY_MATCH_MIN_SEC = 2.4
|
||||
|
||||
def _is_generic_speaker_name(self, value: Any) -> bool:
|
||||
text = str(value or "").strip()
|
||||
return bool(re.fullmatch(r"Speaker\d+", text))
|
||||
|
||||
def _has_named_identity(self, record: Optional[Dict[str, Any]]) -> bool:
|
||||
if not record:
|
||||
return False
|
||||
return bool(
|
||||
record.get("registry_speaker_id")
|
||||
or record.get("user_id")
|
||||
or (
|
||||
record.get("speaker_name")
|
||||
and not self._is_generic_speaker_name(record.get("speaker_name"))
|
||||
)
|
||||
)
|
||||
|
||||
def _normalize_embedding(self, embedding: np.ndarray) -> np.ndarray:
|
||||
emb = np.asarray(embedding, dtype=np.float32).reshape(-1)
|
||||
norm = max(float(np.linalg.norm(emb)), 1e-12)
|
||||
return (emb / norm).astype(np.float32)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_cluster_index(value: Any, fallback: int) -> int:
|
||||
try:
|
||||
if value is None:
|
||||
return int(fallback)
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return int(fallback)
|
||||
|
||||
def _match_existing_speaker(
|
||||
self,
|
||||
speaker_records: List[Dict[str, Any]],
|
||||
mean_embedding: np.ndarray,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not speaker_records:
|
||||
return None
|
||||
|
||||
best_record: Optional[Dict[str, Any]] = None
|
||||
best_score = -1.0
|
||||
for record in speaker_records:
|
||||
record_embedding = record.get("embedding")
|
||||
if record_embedding is None:
|
||||
continue
|
||||
score = float(np.dot(self._normalize_embedding(record_embedding), mean_embedding))
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_record = record
|
||||
|
||||
if best_record is None:
|
||||
return None
|
||||
|
||||
base_threshold = max(
|
||||
float(getattr(settings, "REALTIME_UNKNOWN_SPK_CLUSTER_THRESHOLD", 0.58) or 0.58),
|
||||
0.3,
|
||||
)
|
||||
has_named_identity = bool(
|
||||
best_record.get("registry_speaker_id")
|
||||
or best_record.get("user_id")
|
||||
or (
|
||||
best_record.get("speaker_name")
|
||||
and not self._is_generic_speaker_name(best_record.get("speaker_name"))
|
||||
)
|
||||
)
|
||||
threshold = max(base_threshold, 0.62 if has_named_identity else 0.72)
|
||||
if best_score < threshold:
|
||||
return None
|
||||
|
||||
matched = dict(best_record)
|
||||
matched["_match_score"] = best_score
|
||||
return matched
|
||||
|
||||
def _next_generic_speaker_id(
|
||||
self,
|
||||
speaker_records: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
seen: set[int] = set()
|
||||
for record in speaker_records:
|
||||
for value in (
|
||||
record.get("speaker_id"),
|
||||
record.get("speaker_name"),
|
||||
):
|
||||
text = str(value or "").strip()
|
||||
match = re.fullmatch(r"Speaker(\d+)", text)
|
||||
if match:
|
||||
seen.add(int(match.group(1)))
|
||||
next_index = max(seen, default=0) + 1
|
||||
return f"Speaker{next_index:02d}"
|
||||
|
||||
def _is_mixed_speaker_segment(
|
||||
self,
|
||||
speaker_records: List[Dict[str, Any]],
|
||||
current_chunks: List[Dict[str, Any]],
|
||||
) -> bool:
|
||||
if len(current_chunks) < 2 or len(speaker_records) < 2:
|
||||
return False
|
||||
assigned_labels: List[str] = []
|
||||
for chunk in current_chunks:
|
||||
embedding = chunk.get("embedding")
|
||||
if embedding is None:
|
||||
continue
|
||||
matched = self._match_existing_speaker(
|
||||
speaker_records,
|
||||
self._normalize_embedding(np.asarray(embedding, dtype=np.float32)),
|
||||
)
|
||||
if not matched:
|
||||
continue
|
||||
label = str(
|
||||
matched.get("registry_speaker_id")
|
||||
or matched.get("speaker_name")
|
||||
or matched.get("speaker_id")
|
||||
or ""
|
||||
).strip()
|
||||
if label:
|
||||
assigned_labels.append(label)
|
||||
if len(assigned_labels) < 2:
|
||||
return False
|
||||
return len(set(assigned_labels)) >= 2
|
||||
|
||||
def _correct_labels(self, labels: np.ndarray) -> np.ndarray:
|
||||
labels_id = 0
|
||||
id2id: dict[int, int] = {}
|
||||
|
|
@ -216,6 +101,144 @@ class RealtimeSpeakerClusterer:
|
|||
labels = self._spectral_cluster(X, oracle_num=oracle_num)
|
||||
return self._merge_by_cos(labels, X, cos_thr=0.78 if oracle_num is None else 1.0)
|
||||
|
||||
def _map_cluster_centers(
|
||||
self,
|
||||
cluster_centers: np.ndarray,
|
||||
speaker_centers: List[np.ndarray],
|
||||
speaker_center_updates: List[int],
|
||||
*,
|
||||
update_centers: bool,
|
||||
) -> List[int]:
|
||||
"""Map each window's temporary cluster label to a persistent speaker ID.
|
||||
|
||||
FunASR's ClusterBackend labels are local to each clustering pass. Its
|
||||
realtime tracker maps those labels through cosine-matched centroids so
|
||||
speakers keep the same identity as the 128-chunk window moves.
|
||||
"""
|
||||
centers = np.stack(
|
||||
[self._normalize_embedding(center) for center in np.asarray(cluster_centers)],
|
||||
axis=0,
|
||||
)
|
||||
stable_ids: List[int] = []
|
||||
used_ids: set[int] = set()
|
||||
for center in centers:
|
||||
best_id: Optional[int] = None
|
||||
best_score = -1.0
|
||||
for candidate, known_center in enumerate(speaker_centers):
|
||||
if candidate in used_ids:
|
||||
continue
|
||||
score = float(np.dot(center, self._normalize_embedding(known_center)))
|
||||
if score > best_score:
|
||||
best_id, best_score = candidate, score
|
||||
|
||||
matched = best_id is not None and best_score >= self.FUNASR_CENTER_MATCH_THRESHOLD
|
||||
created = False
|
||||
if not matched and update_centers and len(speaker_centers) < 15:
|
||||
best_id = len(speaker_centers)
|
||||
speaker_centers.append(center.copy())
|
||||
speaker_center_updates.append(1)
|
||||
matched = True
|
||||
created = True
|
||||
elif best_id is None:
|
||||
# Match FunASR's max-speaker fallback when all old IDs were used.
|
||||
if not speaker_centers:
|
||||
best_id = 0
|
||||
else:
|
||||
scores = [float(np.dot(center, self._normalize_embedding(item))) for item in speaker_centers]
|
||||
best_id = int(np.argmax(scores))
|
||||
|
||||
if update_centers and matched and not created:
|
||||
assert best_id is not None
|
||||
count = speaker_center_updates[best_id]
|
||||
weight = 1.0 / min(count + 1, 20)
|
||||
updated = (1.0 - weight) * speaker_centers[best_id] + weight * center
|
||||
speaker_centers[best_id] = self._normalize_embedding(updated)
|
||||
speaker_center_updates[best_id] = count + 1
|
||||
|
||||
assert best_id is not None
|
||||
stable_ids.append(best_id)
|
||||
used_ids.add(best_id)
|
||||
return stable_ids
|
||||
|
||||
def cluster_chunk_history(
|
||||
self,
|
||||
chunks: List[Dict[str, Any]],
|
||||
speaker_centers: List[np.ndarray],
|
||||
speaker_center_updates: List[int],
|
||||
*,
|
||||
update_centers: bool,
|
||||
) -> Tuple[List[Dict[str, Any]], List[int]]:
|
||||
"""Cluster a fixed-size rolling window of timestamped speaker chunks."""
|
||||
valid_chunks = [item for item in chunks if item.get("embedding") is not None]
|
||||
valid_chunks = valid_chunks[-self.FUNASR_MAX_HISTORY_CHUNKS:]
|
||||
if not valid_chunks:
|
||||
return [], []
|
||||
|
||||
embeddings = np.stack(
|
||||
[self._normalize_embedding(np.asarray(item["embedding"], dtype=np.float32)) for item in valid_chunks],
|
||||
axis=0,
|
||||
)
|
||||
labels = self._cluster_embeddings(embeddings, oracle_num=None)
|
||||
temporary_ids = sorted({int(value) for value in labels.tolist()})
|
||||
centers = np.stack(
|
||||
[self._normalize_embedding(embeddings[labels == label].mean(axis=0)) for label in temporary_ids],
|
||||
axis=0,
|
||||
)
|
||||
stable_ids = self._map_cluster_centers(
|
||||
centers,
|
||||
speaker_centers,
|
||||
speaker_center_updates,
|
||||
update_centers=update_centers,
|
||||
)
|
||||
temp_to_stable = dict(zip(temporary_ids, stable_ids))
|
||||
|
||||
timeline = self._postprocess_timeline(valid_chunks, labels, embeddings)
|
||||
for item in timeline:
|
||||
item["cluster_index"] = int(temp_to_stable[int(item["cluster_index"])])
|
||||
return timeline, stable_ids
|
||||
|
||||
def flatten_chunk_history(self, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
"""Return the most recent FunASR-sized history from segment records."""
|
||||
chunks: List[Dict[str, Any]] = []
|
||||
for record in records:
|
||||
entries = record.get("chunks") or []
|
||||
if entries:
|
||||
chunks.extend(entries)
|
||||
continue
|
||||
embeddings = record.get("embeddings") or []
|
||||
if not embeddings and record.get("embedding") is not None:
|
||||
embeddings = [record["embedding"]]
|
||||
for embedding in embeddings:
|
||||
chunks.append({
|
||||
"start_ms": int(record.get("start_ms", 0)),
|
||||
"end_ms": int(record.get("end_ms", record.get("start_ms", 0))),
|
||||
"embedding": embedding,
|
||||
})
|
||||
return chunks[-self.FUNASR_MAX_HISTORY_CHUNKS:]
|
||||
|
||||
@staticmethod
|
||||
def pick_timeline_speaker(
|
||||
timeline: List[Dict[str, Any]],
|
||||
*,
|
||||
start_ms: int,
|
||||
end_ms: int,
|
||||
) -> Optional[int]:
|
||||
"""Assign a sentence to its dominant overlapping speaker, as FunASR does."""
|
||||
sentence_speaker = 0
|
||||
max_overlap = 0
|
||||
for item in timeline:
|
||||
overlap = min(end_ms, int(item["end_ms"])) - max(start_ms, int(item["start_ms"]))
|
||||
if overlap > 0:
|
||||
speaker = int(item["cluster_index"])
|
||||
if overlap > max_overlap:
|
||||
max_overlap = overlap
|
||||
sentence_speaker = speaker
|
||||
if sentence_speaker == speaker:
|
||||
max_overlap += overlap
|
||||
if max_overlap <= 0:
|
||||
return None
|
||||
return sentence_speaker
|
||||
|
||||
def build_sv_chunks(
|
||||
self,
|
||||
audio: np.ndarray,
|
||||
|
|
@ -300,42 +323,6 @@ class RealtimeSpeakerClusterer:
|
|||
logger.warning("Realtime registry speaker embedding failed: %s", exc)
|
||||
return None
|
||||
|
||||
def _flatten_chunk_records(
|
||||
self,
|
||||
records: List[Dict[str, Any]],
|
||||
) -> Tuple[List[Dict[str, Any]], np.ndarray]:
|
||||
flat_chunks: List[Dict[str, Any]] = []
|
||||
flat_embeddings: List[np.ndarray] = []
|
||||
for record_idx, record in enumerate(records):
|
||||
chunk_entries = record.get("chunks") or []
|
||||
if not chunk_entries:
|
||||
embeddings = record.get("embeddings") or []
|
||||
if not embeddings and record.get("embedding") is not None:
|
||||
embeddings = [record["embedding"]]
|
||||
start_ms = int(record.get("start_ms", 0))
|
||||
end_ms = int(record.get("end_ms", start_ms))
|
||||
for embedding in embeddings:
|
||||
chunk_entries.append(
|
||||
{
|
||||
"start_ms": start_ms,
|
||||
"end_ms": end_ms,
|
||||
"embedding": embedding,
|
||||
}
|
||||
)
|
||||
for chunk in chunk_entries:
|
||||
emb = self._normalize_embedding(np.asarray(chunk["embedding"], dtype=np.float32))
|
||||
flat_chunks.append(
|
||||
{
|
||||
"record_index": record_idx,
|
||||
"start_ms": int(chunk.get("start_ms", record.get("start_ms", 0))),
|
||||
"end_ms": int(chunk.get("end_ms", record.get("end_ms", 0))),
|
||||
}
|
||||
)
|
||||
flat_embeddings.append(emb)
|
||||
if not flat_embeddings:
|
||||
return [], np.zeros((0, 0), dtype=np.float32)
|
||||
return flat_chunks, np.stack(flat_embeddings, axis=0).astype(np.float32)
|
||||
|
||||
def _merge_seque(self, distribute_res: List[List[float]]) -> List[List[float]]:
|
||||
if not distribute_res:
|
||||
return []
|
||||
|
|
@ -406,82 +393,66 @@ class RealtimeSpeakerClusterer:
|
|||
for item in distribute_res
|
||||
]
|
||||
|
||||
def _pick_segment_cluster(
|
||||
self,
|
||||
cluster_ranges: List[Dict[str, Any]],
|
||||
*,
|
||||
segment_start_ms: int,
|
||||
segment_end_ms: int,
|
||||
) -> int:
|
||||
overlaps: Dict[int, int] = defaultdict(int)
|
||||
for item in cluster_ranges:
|
||||
overlap = min(segment_end_ms, int(item["end_ms"])) - max(segment_start_ms, int(item["start_ms"]))
|
||||
if overlap > 0:
|
||||
overlaps[int(item["cluster_index"])] += int(overlap)
|
||||
if overlaps:
|
||||
return max(overlaps.items(), key=lambda kv: (kv[1], -kv[0]))[0]
|
||||
|
||||
centers = [
|
||||
item for item in cluster_ranges
|
||||
if int(item["start_ms"]) <= segment_end_ms and int(item["end_ms"]) >= segment_start_ms
|
||||
]
|
||||
if centers:
|
||||
return int(centers[0]["cluster_index"])
|
||||
return 0
|
||||
|
||||
def cluster_records_with_ranges(
|
||||
self,
|
||||
records: List[Dict[str, Any]],
|
||||
) -> Tuple[List[Dict[str, Any]], List[int], List[Dict[str, Any]]]:
|
||||
flat_chunks, X = self._flatten_chunk_records(records)
|
||||
if X.size == 0:
|
||||
return [], [], []
|
||||
# Preserve the existing helper API while routing it through the same
|
||||
# 128-chunk policy used by live sessions.
|
||||
return self.cluster_records_with_stable_history(records, [], [], update_centers=True)
|
||||
|
||||
labels = self._cluster_embeddings(X, oracle_num=None)
|
||||
cluster_ranges = self._postprocess_timeline(flat_chunks, labels, X)
|
||||
|
||||
cluster_ids = sorted({int(item["cluster_index"]) for item in cluster_ranges})
|
||||
if not cluster_ids:
|
||||
cluster_ids = sorted({int(label) for label in labels.tolist()})
|
||||
|
||||
clusters: List[Dict[str, Any]] = []
|
||||
for cluster_idx in cluster_ids:
|
||||
member_mask = labels == int(cluster_idx)
|
||||
member_embeddings = X[member_mask]
|
||||
centroid = self._normalize_embedding(member_embeddings.mean(0))
|
||||
record_refs = []
|
||||
seen_record_indices = set()
|
||||
for emb_idx, chunk in enumerate(flat_chunks):
|
||||
record_idx = int(chunk["record_index"])
|
||||
if not member_mask[emb_idx] or record_idx in seen_record_indices:
|
||||
continue
|
||||
seen_record_indices.add(record_idx)
|
||||
record_refs.append(records[record_idx])
|
||||
clusters.append(
|
||||
{
|
||||
"centroid": centroid,
|
||||
"count": int(member_mask.sum()),
|
||||
"record_refs": record_refs,
|
||||
}
|
||||
def cluster_records_with_stable_history(
|
||||
self,
|
||||
records: List[Dict[str, Any]],
|
||||
speaker_centers: List[np.ndarray],
|
||||
speaker_center_updates: List[int],
|
||||
*,
|
||||
update_centers: bool,
|
||||
) -> Tuple[List[Dict[str, Any]], List[int], List[Dict[str, Any]]]:
|
||||
"""Cluster only the last 128 chunks and return persistent speaker IDs."""
|
||||
chunks = self.flatten_chunk_history(records)
|
||||
timeline, _ = self.cluster_chunk_history(
|
||||
chunks,
|
||||
speaker_centers,
|
||||
speaker_center_updates,
|
||||
update_centers=update_centers,
|
||||
)
|
||||
assignments = [
|
||||
self.pick_timeline_speaker(
|
||||
timeline,
|
||||
start_ms=int(record.get("start_ms", 0)),
|
||||
end_ms=int(record.get("end_ms", record.get("start_ms", 0))),
|
||||
)
|
||||
|
||||
record_to_labels: List[List[int]] = [[] for _ in records]
|
||||
for emb_idx, chunk in enumerate(flat_chunks):
|
||||
record_to_labels[int(chunk["record_index"])].append(int(labels[emb_idx]))
|
||||
|
||||
record_assignments: List[int] = []
|
||||
for record, local_assignments in zip(records, record_to_labels):
|
||||
if local_assignments:
|
||||
cluster_idx = self._pick_segment_cluster(
|
||||
cluster_ranges,
|
||||
segment_start_ms=int(record.get("start_ms", 0)),
|
||||
segment_end_ms=int(record.get("end_ms", record.get("start_ms", 0))),
|
||||
)
|
||||
record_assignments.append(cluster_idx)
|
||||
for record in records
|
||||
]
|
||||
cluster_ids = sorted({int(item["cluster_index"]) for item in timeline})
|
||||
clusters: List[Dict[str, Any]] = []
|
||||
for cluster_id in cluster_ids:
|
||||
refs = [
|
||||
record
|
||||
for record, assigned_id in zip(records, assignments)
|
||||
if assigned_id == cluster_id
|
||||
]
|
||||
if cluster_id < len(speaker_centers):
|
||||
center = speaker_centers[cluster_id]
|
||||
else:
|
||||
record_assignments.append(-1)
|
||||
|
||||
return clusters, record_assignments, cluster_ranges
|
||||
matching_embeddings = [
|
||||
item["embedding"]
|
||||
for item in chunks
|
||||
if self.pick_timeline_speaker(
|
||||
timeline,
|
||||
start_ms=int(item["start_ms"]),
|
||||
end_ms=int(item["end_ms"]),
|
||||
) == cluster_id
|
||||
]
|
||||
center = np.mean(matching_embeddings or [chunks[0]["embedding"]], axis=0)
|
||||
clusters.append({
|
||||
"cluster_index": cluster_id,
|
||||
"centroid": self._normalize_embedding(np.asarray(center, dtype=np.float32)),
|
||||
"count": sum(1 for item in timeline if int(item["cluster_index"]) == cluster_id),
|
||||
"record_refs": refs,
|
||||
})
|
||||
return clusters, [-1 if value is None else int(value) for value in assignments], timeline
|
||||
|
||||
def cluster_records(
|
||||
self,
|
||||
|
|
@ -499,104 +470,47 @@ class RealtimeSpeakerClusterer:
|
|||
segment_end_ms: int,
|
||||
enable_registry_match: bool,
|
||||
speaker_threshold: Optional[float],
|
||||
speaker_centers: List[np.ndarray],
|
||||
speaker_center_updates: List[int],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
duration_sec = float(len(audio)) / 16000.0
|
||||
last_record = speaker_records[-1] if speaker_records else None
|
||||
if duration_sec < self._FAST_ATTACH_MAX_SEC and speaker_records:
|
||||
if self._has_named_identity(last_record):
|
||||
speaker_id = last_record.get("registry_speaker_id") or last_record.get("speaker_id") or "Speaker01"
|
||||
speaker_name = last_record.get("speaker_name") or speaker_id
|
||||
return {
|
||||
"speaker_id": speaker_id,
|
||||
"speaker_name": speaker_name,
|
||||
"user_id": last_record.get("user_id"),
|
||||
"registry_speaker_id": last_record.get("registry_speaker_id"),
|
||||
"speaker_confidence": 0.0,
|
||||
"speaker_strategy": "short_attach",
|
||||
"_cluster_index": last_record.get("cluster_index"),
|
||||
"_embedding": last_record.get("embedding"),
|
||||
"_chunk_embeddings": last_record.get("embeddings") or [],
|
||||
"_chunks": last_record.get("chunks") or [],
|
||||
}
|
||||
|
||||
current_chunks = await self.extract_chunk_embeddings(
|
||||
np.asarray(audio, dtype=np.float32),
|
||||
segment_start_ms=segment_start_ms,
|
||||
)
|
||||
if not current_chunks:
|
||||
if speaker_records:
|
||||
if self._has_named_identity(last_record):
|
||||
speaker_id = last_record.get("registry_speaker_id") or last_record.get("speaker_id") or "Speaker01"
|
||||
speaker_name = last_record.get("speaker_name") or speaker_id
|
||||
return {
|
||||
"speaker_id": speaker_id,
|
||||
"speaker_name": speaker_name,
|
||||
"user_id": last_record.get("user_id"),
|
||||
"registry_speaker_id": last_record.get("registry_speaker_id"),
|
||||
"speaker_confidence": 0.0,
|
||||
"speaker_strategy": "embedding_attach",
|
||||
"_cluster_index": last_record.get("cluster_index"),
|
||||
"_embedding": last_record.get("embedding"),
|
||||
"_chunk_embeddings": last_record.get("embeddings") or [],
|
||||
"_chunks": last_record.get("chunks") or [],
|
||||
}
|
||||
return None
|
||||
|
||||
mean_embedding = self._normalize_embedding(
|
||||
np.mean(np.stack([chunk["embedding"] for chunk in current_chunks], axis=0), axis=0)
|
||||
)
|
||||
if duration_sec >= 4.0 and self._is_mixed_speaker_segment(speaker_records, current_chunks):
|
||||
return {
|
||||
"speaker_id": -1,
|
||||
"speaker_name": "",
|
||||
"user_id": None,
|
||||
"registry_speaker_id": None,
|
||||
"speaker_confidence": 0.0,
|
||||
"speaker_strategy": "mixed_segment",
|
||||
"_cluster_index": None,
|
||||
"_embedding": mean_embedding,
|
||||
"_chunk_embeddings": [np.asarray(chunk["embedding"], dtype=np.float32) for chunk in current_chunks],
|
||||
"_chunks": current_chunks,
|
||||
}
|
||||
matched_record = self._match_existing_speaker(speaker_records, mean_embedding)
|
||||
speaker_id = self._next_generic_speaker_id(speaker_records)
|
||||
|
||||
# Re-cluster the same fixed number of recent chunks as FunASR. Chunk
|
||||
# labels are temporary, so the session-owned centers keep IDs stable.
|
||||
history_chunks = self.flatten_chunk_history(speaker_records)
|
||||
timeline, _ = self.cluster_chunk_history(
|
||||
history_chunks + current_chunks,
|
||||
speaker_centers,
|
||||
speaker_center_updates,
|
||||
update_centers=True,
|
||||
)
|
||||
cluster_index = self.pick_timeline_speaker(
|
||||
timeline,
|
||||
start_ms=segment_start_ms,
|
||||
end_ms=segment_end_ms,
|
||||
)
|
||||
if cluster_index is None:
|
||||
cluster_index = max(len(speaker_centers) - 1, 0)
|
||||
speaker_id = f"Speaker{cluster_index + 1:02d}"
|
||||
speaker_name = speaker_id
|
||||
user_id = None
|
||||
registry_speaker_id = None
|
||||
strategy = "new_speaker"
|
||||
cluster_index = max(
|
||||
[
|
||||
int(record.get("cluster_index", -1))
|
||||
for record in speaker_records
|
||||
if record.get("cluster_index") is not None
|
||||
] or [-1]
|
||||
) + 1
|
||||
confidence = 0.0
|
||||
|
||||
if matched_record is not None:
|
||||
speaker_id = (
|
||||
matched_record.get("registry_speaker_id")
|
||||
or matched_record.get("speaker_id")
|
||||
or speaker_id
|
||||
)
|
||||
speaker_name = matched_record.get("speaker_name") or speaker_id
|
||||
user_id = matched_record.get("user_id")
|
||||
registry_speaker_id = matched_record.get("registry_speaker_id")
|
||||
cluster_index = self._coerce_cluster_index(
|
||||
matched_record.get("cluster_index"),
|
||||
cluster_index,
|
||||
)
|
||||
confidence = float(matched_record.get("_match_score", 0.0))
|
||||
strategy = "embedding_match"
|
||||
confidence = 1.0
|
||||
strategy = "chunk_cluster"
|
||||
|
||||
if (
|
||||
not registry_speaker_id
|
||||
and enable_registry_match
|
||||
enable_registry_match
|
||||
and duration_sec >= self._REGISTRY_MATCH_MIN_SEC
|
||||
and (
|
||||
matched_record is None
|
||||
or confidence >= max(float(getattr(settings, "REALTIME_SPEAKER_CONFIRM_THRESHOLD", 0.62) or 0.62), 0.7)
|
||||
)
|
||||
):
|
||||
# Realtime clustering can use a dedicated fast model, while the
|
||||
# registry stores embeddings from the registration model. Match the
|
||||
|
|
@ -622,11 +536,12 @@ class RealtimeSpeakerClusterer:
|
|||
"registry_speaker_id": registry_speaker_id,
|
||||
"speaker_confidence": round(confidence, 4),
|
||||
"speaker_strategy": strategy,
|
||||
"_cluster_index": self._coerce_cluster_index(cluster_index, 0),
|
||||
"_cluster_index": int(cluster_index),
|
||||
"_embedding": mean_embedding,
|
||||
"_chunk_embeddings": [np.asarray(chunk["embedding"], dtype=np.float32) for chunk in current_chunks],
|
||||
"_chunks": current_chunks,
|
||||
"_matched_existing": matched_record is not None,
|
||||
"_matched_existing": True,
|
||||
"_speaker_timeline": timeline,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue