测试页面新加实时说话人切换逻辑
parent
3d0a7f7705
commit
539e699915
|
|
@ -320,6 +320,15 @@ class ConnectionContext:
|
|||
speaker_display_profiles: Dict[str, np.ndarray] = field(default_factory=dict)
|
||||
speaker_display_counts: Dict[str, int] = field(default_factory=dict)
|
||||
next_speaker_display_id: int = 1
|
||||
# Live speaker-change probes run off the receive loop so SV inference does
|
||||
# not stall PCM handling. A change is accepted only after several windows.
|
||||
speaker_change_probe_task: Optional[asyncio.Task[Any]] = None
|
||||
speaker_change_last_probe_end_sample: int = 0
|
||||
speaker_change_reference_embedding: Optional[np.ndarray] = None
|
||||
speaker_change_reference_center_sample: Optional[int] = None
|
||||
speaker_change_candidate_embedding: Optional[np.ndarray] = None
|
||||
speaker_change_candidate_start_center_sample: Optional[int] = None
|
||||
speaker_change_candidate_hits: int = 0
|
||||
timeline_cursor_ms: int = 0
|
||||
segment_index: int = 0
|
||||
last_partial_text: str = ""
|
||||
|
|
@ -351,6 +360,16 @@ class SessionEntry:
|
|||
|
||||
|
||||
class Qwen3ASRService:
|
||||
# Use the same 1.5 s SV window / 0.75 s hop as the realtime clusterer.
|
||||
# Four consecutive changed windows (spanning >=2.25 s) reject brief
|
||||
# acknowledgements such as "嗯" before they can split a transcript turn.
|
||||
_SPEAKER_CHANGE_WINDOW_SAMPLES = 24000
|
||||
_SPEAKER_CHANGE_HOP_SAMPLES = 12000
|
||||
_SPEAKER_CHANGE_CONFIRM_HITS = 4
|
||||
_SPEAKER_CHANGE_CONFIRM_SPAN_SAMPLES = 36000
|
||||
_SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES = 12000
|
||||
_SPEAKER_CHANGE_COSINE_THRESHOLD = 0.60
|
||||
|
||||
_MAX_SPEAKER_HISTORY_RECORDS = 128
|
||||
_MAX_SPEAKER_FINAL_RECLUSTER = 128
|
||||
_SEGMENT_EVENT_TEXT_TAIL_CHARS = 2000
|
||||
|
|
@ -1568,6 +1587,197 @@ class Qwen3ASRService:
|
|||
new_text += after
|
||||
return new_text
|
||||
|
||||
def _reset_realtime_speaker_change_state(self, ctx: ConnectionContext) -> None:
|
||||
"""Discard pending online speaker evidence when an ASR turn is cut."""
|
||||
task = ctx.speaker_change_probe_task
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
ctx.speaker_change_probe_task = None
|
||||
ctx.speaker_change_last_probe_end_sample = 0
|
||||
ctx.speaker_change_reference_embedding = None
|
||||
ctx.speaker_change_reference_center_sample = None
|
||||
ctx.speaker_change_candidate_embedding = None
|
||||
ctx.speaker_change_candidate_start_center_sample = None
|
||||
ctx.speaker_change_candidate_hits = 0
|
||||
|
||||
async def _extract_realtime_speaker_probe(
|
||||
self,
|
||||
audio: np.ndarray,
|
||||
start_sample: int,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Extract one SV embedding asynchronously from a copied live window."""
|
||||
try:
|
||||
chunks = await get_realtime_speaker_clusterer().extract_chunk_embeddings(
|
||||
np.asarray(audio, dtype=np.float32),
|
||||
segment_start_ms=int(start_sample / 16),
|
||||
)
|
||||
if not chunks or chunks[0].get("embedding") is None:
|
||||
return None
|
||||
return {
|
||||
"embedding": np.asarray(chunks[0]["embedding"], dtype=np.float32),
|
||||
"center_sample": int(start_sample + audio.size // 2),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.debug("Realtime speaker-change probe failed: %s", exc)
|
||||
return None
|
||||
|
||||
def _schedule_realtime_speaker_probe(self, ctx: ConnectionContext) -> None:
|
||||
"""Queue a rolling SV probe without blocking the WebSocket receive loop."""
|
||||
if (
|
||||
not ctx.sentence_active
|
||||
or not bool(ctx.params.get("enable_speaker", True))
|
||||
or ctx.speaker_change_probe_task is not None
|
||||
):
|
||||
return
|
||||
|
||||
end_sample = int(ctx.segment_buffer_start_sample + ctx.segment_audio_buffer.size)
|
||||
if end_sample - ctx.segment_buffer_start_sample < self._SPEAKER_CHANGE_WINDOW_SAMPLES:
|
||||
return
|
||||
if (
|
||||
end_sample - ctx.speaker_change_last_probe_end_sample
|
||||
< self._SPEAKER_CHANGE_HOP_SAMPLES
|
||||
):
|
||||
return
|
||||
|
||||
start_sample = end_sample - self._SPEAKER_CHANGE_WINDOW_SAMPLES
|
||||
audio = np.asarray(
|
||||
ctx.segment_audio_buffer[-self._SPEAKER_CHANGE_WINDOW_SAMPLES :],
|
||||
dtype=np.float32,
|
||||
).copy()
|
||||
# Avoid embeddings dominated by silence or an isolated low-energy sound.
|
||||
if audio.size != self._SPEAKER_CHANGE_WINDOW_SAMPLES or not self._has_voice(audio):
|
||||
return
|
||||
|
||||
ctx.speaker_change_last_probe_end_sample = end_sample
|
||||
task = asyncio.create_task(self._extract_realtime_speaker_probe(audio, start_sample))
|
||||
ctx.speaker_change_probe_task = task
|
||||
self._track_background_task(ctx, task)
|
||||
|
||||
def _consume_realtime_speaker_probe(
|
||||
self,
|
||||
ctx: ConnectionContext,
|
||||
) -> Optional[int]:
|
||||
"""Update stable/candidate speaker evidence and return a confirmed cut."""
|
||||
task = ctx.speaker_change_probe_task
|
||||
if task is None or not task.done():
|
||||
return None
|
||||
ctx.speaker_change_probe_task = None
|
||||
try:
|
||||
result = task.result()
|
||||
except asyncio.CancelledError:
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.debug("Realtime speaker-change probe task failed: %s", exc)
|
||||
return None
|
||||
if not result:
|
||||
ctx.speaker_change_candidate_embedding = None
|
||||
ctx.speaker_change_candidate_start_center_sample = None
|
||||
ctx.speaker_change_candidate_hits = 0
|
||||
return None
|
||||
|
||||
embedding = np.asarray(result["embedding"], dtype=np.float32).reshape(-1)
|
||||
center_sample = int(result["center_sample"])
|
||||
reference = ctx.speaker_change_reference_embedding
|
||||
if reference is None or reference.shape != embedding.shape:
|
||||
ctx.speaker_change_reference_embedding = embedding.copy()
|
||||
ctx.speaker_change_reference_center_sample = center_sample
|
||||
return None
|
||||
|
||||
reference_score = float(np.dot(reference, embedding))
|
||||
if reference_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD:
|
||||
# Slowly adapt to normal voice and prosody variation from one speaker.
|
||||
updated = (0.8 * reference) + (0.2 * embedding)
|
||||
norm = max(float(np.linalg.norm(updated)), 1e-12)
|
||||
ctx.speaker_change_reference_embedding = (updated / norm).astype(np.float32)
|
||||
ctx.speaker_change_reference_center_sample = center_sample
|
||||
ctx.speaker_change_candidate_embedding = None
|
||||
ctx.speaker_change_candidate_start_center_sample = None
|
||||
ctx.speaker_change_candidate_hits = 0
|
||||
return None
|
||||
|
||||
candidate = ctx.speaker_change_candidate_embedding
|
||||
if candidate is not None and candidate.shape == embedding.shape:
|
||||
candidate_score = float(np.dot(candidate, embedding))
|
||||
else:
|
||||
candidate_score = -1.0
|
||||
|
||||
if candidate_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD:
|
||||
updated = (candidate * ctx.speaker_change_candidate_hits) + embedding
|
||||
norm = max(float(np.linalg.norm(updated)), 1e-12)
|
||||
ctx.speaker_change_candidate_embedding = (updated / norm).astype(np.float32)
|
||||
ctx.speaker_change_candidate_hits += 1
|
||||
else:
|
||||
ctx.speaker_change_candidate_embedding = embedding.copy()
|
||||
ctx.speaker_change_candidate_start_center_sample = center_sample
|
||||
ctx.speaker_change_candidate_hits = 1
|
||||
|
||||
candidate_start = ctx.speaker_change_candidate_start_center_sample
|
||||
if (
|
||||
candidate_start is None
|
||||
or ctx.speaker_change_candidate_hits < self._SPEAKER_CHANGE_CONFIRM_HITS
|
||||
or center_sample - candidate_start < self._SPEAKER_CHANGE_CONFIRM_SPAN_SAMPLES
|
||||
):
|
||||
return None
|
||||
|
||||
reference_center = ctx.speaker_change_reference_center_sample
|
||||
if reference_center is None:
|
||||
return None
|
||||
# Place the cut halfway between the last stable voice window and the
|
||||
# first sustained new-speaker window; the confirmation delay stays in
|
||||
# the new speaker's carry buffer and is not lost.
|
||||
turn_start_sample = int(ctx.segment_buffer_start_sample)
|
||||
boundary_sample = max(
|
||||
(int(reference_center) + int(candidate_start)) // 2,
|
||||
turn_start_sample + self._SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"[%s] Sustained speaker change confirmed: candidate_hits=%d "
|
||||
"candidate_span_sec=%.2f boundary_offset_sec=%.2f",
|
||||
ctx.diagnostic_session_id,
|
||||
ctx.speaker_change_candidate_hits,
|
||||
(center_sample - candidate_start) / 16000.0,
|
||||
(boundary_sample - turn_start_sample) / 16000.0,
|
||||
)
|
||||
return boundary_sample
|
||||
|
||||
async def _commit_realtime_speaker_change(
|
||||
self,
|
||||
websocket: WebSocket,
|
||||
ctx: ConnectionContext,
|
||||
task_id: str,
|
||||
boundary_sample: int,
|
||||
) -> bool:
|
||||
"""Finalize the previous speaker and retain post-boundary audio as carry."""
|
||||
current_audio = np.asarray(ctx.segment_audio_buffer, dtype=np.float32)
|
||||
turn_start_sample = int(ctx.segment_buffer_start_sample)
|
||||
boundary_offset = int(boundary_sample) - turn_start_sample
|
||||
if (
|
||||
boundary_offset < self._SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES
|
||||
or boundary_offset >= current_audio.size
|
||||
):
|
||||
return False
|
||||
|
||||
finalized_audio = np.asarray(current_audio[:boundary_offset], dtype=np.float32)
|
||||
carry_audio = np.asarray(current_audio[boundary_offset:], dtype=np.float32)
|
||||
if carry_audio.size == 0:
|
||||
return False
|
||||
|
||||
await self._commit_retranscribe_turn(
|
||||
websocket,
|
||||
ctx,
|
||||
task_id,
|
||||
"speaker_change",
|
||||
finalized_audio_override=finalized_audio,
|
||||
carry_audio_override=carry_audio,
|
||||
segment_start_ms_override=int(turn_start_sample / 16),
|
||||
segment_end_ms_override=int(boundary_sample / 16),
|
||||
)
|
||||
# VAD still sees continuous speech across this diarization boundary.
|
||||
ctx.vad_speech_active = True
|
||||
ctx.vad_speech_start_sample = int(boundary_sample)
|
||||
return True
|
||||
|
||||
async def _finish_funasr_stream(
|
||||
self,
|
||||
ctx: ConnectionContext,
|
||||
|
|
@ -3205,6 +3415,12 @@ class Qwen3ASRService:
|
|||
ctx,
|
||||
segment_text,
|
||||
)
|
||||
if funasr_streaming and reason == "speaker_change":
|
||||
# A confirmed speaker transition is a sentence boundary even when
|
||||
# the punctuation model did not emit a terminal mark.
|
||||
boundary_segment = {"text": segment_text}
|
||||
self._close_segment_at_speaker_boundary(boundary_segment)
|
||||
segment_text = str(boundary_segment.get("text") or "")
|
||||
|
||||
is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec)
|
||||
logger.info(
|
||||
|
|
@ -3314,6 +3530,7 @@ class Qwen3ASRService:
|
|||
ctx.sentence_active = bool(carry_audio.size > 0)
|
||||
ctx.silence_samples = 0
|
||||
ctx.total_samples = int(carry_audio.size)
|
||||
self._reset_realtime_speaker_change_state(ctx)
|
||||
self._reset_partial_state(ctx)
|
||||
|
||||
if ctx.sentence_active and carry_audio.size > 0:
|
||||
|
|
@ -3840,6 +4057,20 @@ class Qwen3ASRService:
|
|||
ctx.state = ConnectionState.STREAMING
|
||||
continue
|
||||
|
||||
if self._uses_funasr_streaming(ctx) and not bulk_audio_frame:
|
||||
speaker_boundary = self._consume_realtime_speaker_probe(ctx)
|
||||
if speaker_boundary is not None and await self._commit_realtime_speaker_change(
|
||||
websocket,
|
||||
ctx,
|
||||
task_id,
|
||||
speaker_boundary,
|
||||
):
|
||||
ctx.state = ConnectionState.STREAMING
|
||||
continue
|
||||
# Enqueue the next copied 1.5 s window after consuming
|
||||
# the prior result; diarization must never block PCM I/O.
|
||||
self._schedule_realtime_speaker_probe(ctx)
|
||||
|
||||
silence_threshold = self._get_dynamic_silence_threshold_samples(ctx)
|
||||
if self._enable_realtime_vad(ctx) and ctx.vad_speech_active:
|
||||
# Give FSMN-VAD time to confirm an endpoint before the
|
||||
|
|
|
|||
Loading…
Reference in New Issue