测试页面新加实时说话人切换逻辑

main
Bifang 2026-10-13 10:48:21 +08:00
parent 3d0a7f7705
commit 539e699915
1 changed files with 231 additions and 0 deletions

View File

@ -320,6 +320,15 @@ class ConnectionContext:
speaker_display_profiles: Dict[str, np.ndarray] = field(default_factory=dict)
speaker_display_counts: Dict[str, int] = field(default_factory=dict)
next_speaker_display_id: int = 1
# Live speaker-change probes run off the receive loop so SV inference does
# not stall PCM handling. A change is accepted only after several windows.
speaker_change_probe_task: Optional[asyncio.Task[Any]] = None
speaker_change_last_probe_end_sample: int = 0
speaker_change_reference_embedding: Optional[np.ndarray] = None
speaker_change_reference_center_sample: Optional[int] = None
speaker_change_candidate_embedding: Optional[np.ndarray] = None
speaker_change_candidate_start_center_sample: Optional[int] = None
speaker_change_candidate_hits: int = 0
timeline_cursor_ms: int = 0
segment_index: int = 0
last_partial_text: str = ""
@ -351,6 +360,16 @@ class SessionEntry:
class Qwen3ASRService:
# Use the same 1.5 s SV window / 0.75 s hop as the realtime clusterer.
# Four consecutive changed windows (spanning >=2.25 s) reject brief
# acknowledgements such as "嗯" before they can split a transcript turn.
_SPEAKER_CHANGE_WINDOW_SAMPLES = 24000
_SPEAKER_CHANGE_HOP_SAMPLES = 12000
_SPEAKER_CHANGE_CONFIRM_HITS = 4
_SPEAKER_CHANGE_CONFIRM_SPAN_SAMPLES = 36000
_SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES = 12000
_SPEAKER_CHANGE_COSINE_THRESHOLD = 0.60
_MAX_SPEAKER_HISTORY_RECORDS = 128
_MAX_SPEAKER_FINAL_RECLUSTER = 128
_SEGMENT_EVENT_TEXT_TAIL_CHARS = 2000
@ -1568,6 +1587,197 @@ class Qwen3ASRService:
new_text += after
return new_text
def _reset_realtime_speaker_change_state(self, ctx: ConnectionContext) -> None:
"""Discard pending online speaker evidence when an ASR turn is cut."""
task = ctx.speaker_change_probe_task
if task is not None and not task.done():
task.cancel()
ctx.speaker_change_probe_task = None
ctx.speaker_change_last_probe_end_sample = 0
ctx.speaker_change_reference_embedding = None
ctx.speaker_change_reference_center_sample = None
ctx.speaker_change_candidate_embedding = None
ctx.speaker_change_candidate_start_center_sample = None
ctx.speaker_change_candidate_hits = 0
async def _extract_realtime_speaker_probe(
self,
audio: np.ndarray,
start_sample: int,
) -> Optional[Dict[str, Any]]:
"""Extract one SV embedding asynchronously from a copied live window."""
try:
chunks = await get_realtime_speaker_clusterer().extract_chunk_embeddings(
np.asarray(audio, dtype=np.float32),
segment_start_ms=int(start_sample / 16),
)
if not chunks or chunks[0].get("embedding") is None:
return None
return {
"embedding": np.asarray(chunks[0]["embedding"], dtype=np.float32),
"center_sample": int(start_sample + audio.size // 2),
}
except Exception as exc:
logger.debug("Realtime speaker-change probe failed: %s", exc)
return None
def _schedule_realtime_speaker_probe(self, ctx: ConnectionContext) -> None:
"""Queue a rolling SV probe without blocking the WebSocket receive loop."""
if (
not ctx.sentence_active
or not bool(ctx.params.get("enable_speaker", True))
or ctx.speaker_change_probe_task is not None
):
return
end_sample = int(ctx.segment_buffer_start_sample + ctx.segment_audio_buffer.size)
if end_sample - ctx.segment_buffer_start_sample < self._SPEAKER_CHANGE_WINDOW_SAMPLES:
return
if (
end_sample - ctx.speaker_change_last_probe_end_sample
< self._SPEAKER_CHANGE_HOP_SAMPLES
):
return
start_sample = end_sample - self._SPEAKER_CHANGE_WINDOW_SAMPLES
audio = np.asarray(
ctx.segment_audio_buffer[-self._SPEAKER_CHANGE_WINDOW_SAMPLES :],
dtype=np.float32,
).copy()
# Avoid embeddings dominated by silence or an isolated low-energy sound.
if audio.size != self._SPEAKER_CHANGE_WINDOW_SAMPLES or not self._has_voice(audio):
return
ctx.speaker_change_last_probe_end_sample = end_sample
task = asyncio.create_task(self._extract_realtime_speaker_probe(audio, start_sample))
ctx.speaker_change_probe_task = task
self._track_background_task(ctx, task)
def _consume_realtime_speaker_probe(
self,
ctx: ConnectionContext,
) -> Optional[int]:
"""Update stable/candidate speaker evidence and return a confirmed cut."""
task = ctx.speaker_change_probe_task
if task is None or not task.done():
return None
ctx.speaker_change_probe_task = None
try:
result = task.result()
except asyncio.CancelledError:
return None
except Exception as exc:
logger.debug("Realtime speaker-change probe task failed: %s", exc)
return None
if not result:
ctx.speaker_change_candidate_embedding = None
ctx.speaker_change_candidate_start_center_sample = None
ctx.speaker_change_candidate_hits = 0
return None
embedding = np.asarray(result["embedding"], dtype=np.float32).reshape(-1)
center_sample = int(result["center_sample"])
reference = ctx.speaker_change_reference_embedding
if reference is None or reference.shape != embedding.shape:
ctx.speaker_change_reference_embedding = embedding.copy()
ctx.speaker_change_reference_center_sample = center_sample
return None
reference_score = float(np.dot(reference, embedding))
if reference_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD:
# Slowly adapt to normal voice and prosody variation from one speaker.
updated = (0.8 * reference) + (0.2 * embedding)
norm = max(float(np.linalg.norm(updated)), 1e-12)
ctx.speaker_change_reference_embedding = (updated / norm).astype(np.float32)
ctx.speaker_change_reference_center_sample = center_sample
ctx.speaker_change_candidate_embedding = None
ctx.speaker_change_candidate_start_center_sample = None
ctx.speaker_change_candidate_hits = 0
return None
candidate = ctx.speaker_change_candidate_embedding
if candidate is not None and candidate.shape == embedding.shape:
candidate_score = float(np.dot(candidate, embedding))
else:
candidate_score = -1.0
if candidate_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD:
updated = (candidate * ctx.speaker_change_candidate_hits) + embedding
norm = max(float(np.linalg.norm(updated)), 1e-12)
ctx.speaker_change_candidate_embedding = (updated / norm).astype(np.float32)
ctx.speaker_change_candidate_hits += 1
else:
ctx.speaker_change_candidate_embedding = embedding.copy()
ctx.speaker_change_candidate_start_center_sample = center_sample
ctx.speaker_change_candidate_hits = 1
candidate_start = ctx.speaker_change_candidate_start_center_sample
if (
candidate_start is None
or ctx.speaker_change_candidate_hits < self._SPEAKER_CHANGE_CONFIRM_HITS
or center_sample - candidate_start < self._SPEAKER_CHANGE_CONFIRM_SPAN_SAMPLES
):
return None
reference_center = ctx.speaker_change_reference_center_sample
if reference_center is None:
return None
# Place the cut halfway between the last stable voice window and the
# first sustained new-speaker window; the confirmation delay stays in
# the new speaker's carry buffer and is not lost.
turn_start_sample = int(ctx.segment_buffer_start_sample)
boundary_sample = max(
(int(reference_center) + int(candidate_start)) // 2,
turn_start_sample + self._SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES,
)
logger.info(
"[%s] Sustained speaker change confirmed: candidate_hits=%d "
"candidate_span_sec=%.2f boundary_offset_sec=%.2f",
ctx.diagnostic_session_id,
ctx.speaker_change_candidate_hits,
(center_sample - candidate_start) / 16000.0,
(boundary_sample - turn_start_sample) / 16000.0,
)
return boundary_sample
async def _commit_realtime_speaker_change(
self,
websocket: WebSocket,
ctx: ConnectionContext,
task_id: str,
boundary_sample: int,
) -> bool:
"""Finalize the previous speaker and retain post-boundary audio as carry."""
current_audio = np.asarray(ctx.segment_audio_buffer, dtype=np.float32)
turn_start_sample = int(ctx.segment_buffer_start_sample)
boundary_offset = int(boundary_sample) - turn_start_sample
if (
boundary_offset < self._SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES
or boundary_offset >= current_audio.size
):
return False
finalized_audio = np.asarray(current_audio[:boundary_offset], dtype=np.float32)
carry_audio = np.asarray(current_audio[boundary_offset:], dtype=np.float32)
if carry_audio.size == 0:
return False
await self._commit_retranscribe_turn(
websocket,
ctx,
task_id,
"speaker_change",
finalized_audio_override=finalized_audio,
carry_audio_override=carry_audio,
segment_start_ms_override=int(turn_start_sample / 16),
segment_end_ms_override=int(boundary_sample / 16),
)
# VAD still sees continuous speech across this diarization boundary.
ctx.vad_speech_active = True
ctx.vad_speech_start_sample = int(boundary_sample)
return True
async def _finish_funasr_stream(
self,
ctx: ConnectionContext,
@ -3205,6 +3415,12 @@ class Qwen3ASRService:
ctx,
segment_text,
)
if funasr_streaming and reason == "speaker_change":
# A confirmed speaker transition is a sentence boundary even when
# the punctuation model did not emit a terminal mark.
boundary_segment = {"text": segment_text}
self._close_segment_at_speaker_boundary(boundary_segment)
segment_text = str(boundary_segment.get("text") or "")
is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec)
logger.info(
@ -3314,6 +3530,7 @@ class Qwen3ASRService:
ctx.sentence_active = bool(carry_audio.size > 0)
ctx.silence_samples = 0
ctx.total_samples = int(carry_audio.size)
self._reset_realtime_speaker_change_state(ctx)
self._reset_partial_state(ctx)
if ctx.sentence_active and carry_audio.size > 0:
@ -3840,6 +4057,20 @@ class Qwen3ASRService:
ctx.state = ConnectionState.STREAMING
continue
if self._uses_funasr_streaming(ctx) and not bulk_audio_frame:
speaker_boundary = self._consume_realtime_speaker_probe(ctx)
if speaker_boundary is not None and await self._commit_realtime_speaker_change(
websocket,
ctx,
task_id,
speaker_boundary,
):
ctx.state = ConnectionState.STREAMING
continue
# Enqueue the next copied 1.5 s window after consuming
# the prior result; diarization must never block PCM I/O.
self._schedule_realtime_speaker_probe(ctx)
silence_threshold = self._get_dynamic_silence_threshold_samples(ctx)
if self._enable_realtime_vad(ctx) and ctx.vad_speech_active:
# Give FSMN-VAD time to confirm an endpoint before the