From 539e699915ad3e2a51552164c98fa5c1f832a886 Mon Sep 17 00:00:00 2001 From: Bifang <915779419@qq.com> Date: Tue, 13 Oct 2026 10:48:21 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E9=A1=B5=E9=9D=A2=E6=96=B0?= =?UTF-8?q?=E5=8A=A0=E5=AE=9E=E6=97=B6=E8=AF=B4=E8=AF=9D=E4=BA=BA=E5=88=87?= =?UTF-8?q?=E6=8D=A2=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/qwen3_websocket_asr.py | 231 ++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index 83982d8..08d6b9b 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -320,6 +320,15 @@ class ConnectionContext: speaker_display_profiles: Dict[str, np.ndarray] = field(default_factory=dict) speaker_display_counts: Dict[str, int] = field(default_factory=dict) next_speaker_display_id: int = 1 + # Live speaker-change probes run off the receive loop so SV inference does + # not stall PCM handling. A change is accepted only after several windows. + speaker_change_probe_task: Optional[asyncio.Task[Any]] = None + speaker_change_last_probe_end_sample: int = 0 + speaker_change_reference_embedding: Optional[np.ndarray] = None + speaker_change_reference_center_sample: Optional[int] = None + speaker_change_candidate_embedding: Optional[np.ndarray] = None + speaker_change_candidate_start_center_sample: Optional[int] = None + speaker_change_candidate_hits: int = 0 timeline_cursor_ms: int = 0 segment_index: int = 0 last_partial_text: str = "" @@ -351,6 +360,16 @@ class SessionEntry: class Qwen3ASRService: + # Use the same 1.5 s SV window / 0.75 s hop as the realtime clusterer. + # Four consecutive changed windows (spanning >=2.25 s) reject brief + # acknowledgements such as "嗯" before they can split a transcript turn. + _SPEAKER_CHANGE_WINDOW_SAMPLES = 24000 + _SPEAKER_CHANGE_HOP_SAMPLES = 12000 + _SPEAKER_CHANGE_CONFIRM_HITS = 4 + _SPEAKER_CHANGE_CONFIRM_SPAN_SAMPLES = 36000 + _SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES = 12000 + _SPEAKER_CHANGE_COSINE_THRESHOLD = 0.60 + _MAX_SPEAKER_HISTORY_RECORDS = 128 _MAX_SPEAKER_FINAL_RECLUSTER = 128 _SEGMENT_EVENT_TEXT_TAIL_CHARS = 2000 @@ -1568,6 +1587,197 @@ class Qwen3ASRService: new_text += after return new_text + def _reset_realtime_speaker_change_state(self, ctx: ConnectionContext) -> None: + """Discard pending online speaker evidence when an ASR turn is cut.""" + task = ctx.speaker_change_probe_task + if task is not None and not task.done(): + task.cancel() + ctx.speaker_change_probe_task = None + ctx.speaker_change_last_probe_end_sample = 0 + ctx.speaker_change_reference_embedding = None + ctx.speaker_change_reference_center_sample = None + ctx.speaker_change_candidate_embedding = None + ctx.speaker_change_candidate_start_center_sample = None + ctx.speaker_change_candidate_hits = 0 + + async def _extract_realtime_speaker_probe( + self, + audio: np.ndarray, + start_sample: int, + ) -> Optional[Dict[str, Any]]: + """Extract one SV embedding asynchronously from a copied live window.""" + try: + chunks = await get_realtime_speaker_clusterer().extract_chunk_embeddings( + np.asarray(audio, dtype=np.float32), + segment_start_ms=int(start_sample / 16), + ) + if not chunks or chunks[0].get("embedding") is None: + return None + return { + "embedding": np.asarray(chunks[0]["embedding"], dtype=np.float32), + "center_sample": int(start_sample + audio.size // 2), + } + except Exception as exc: + logger.debug("Realtime speaker-change probe failed: %s", exc) + return None + + def _schedule_realtime_speaker_probe(self, ctx: ConnectionContext) -> None: + """Queue a rolling SV probe without blocking the WebSocket receive loop.""" + if ( + not ctx.sentence_active + or not bool(ctx.params.get("enable_speaker", True)) + or ctx.speaker_change_probe_task is not None + ): + return + + end_sample = int(ctx.segment_buffer_start_sample + ctx.segment_audio_buffer.size) + if end_sample - ctx.segment_buffer_start_sample < self._SPEAKER_CHANGE_WINDOW_SAMPLES: + return + if ( + end_sample - ctx.speaker_change_last_probe_end_sample + < self._SPEAKER_CHANGE_HOP_SAMPLES + ): + return + + start_sample = end_sample - self._SPEAKER_CHANGE_WINDOW_SAMPLES + audio = np.asarray( + ctx.segment_audio_buffer[-self._SPEAKER_CHANGE_WINDOW_SAMPLES :], + dtype=np.float32, + ).copy() + # Avoid embeddings dominated by silence or an isolated low-energy sound. + if audio.size != self._SPEAKER_CHANGE_WINDOW_SAMPLES or not self._has_voice(audio): + return + + ctx.speaker_change_last_probe_end_sample = end_sample + task = asyncio.create_task(self._extract_realtime_speaker_probe(audio, start_sample)) + ctx.speaker_change_probe_task = task + self._track_background_task(ctx, task) + + def _consume_realtime_speaker_probe( + self, + ctx: ConnectionContext, + ) -> Optional[int]: + """Update stable/candidate speaker evidence and return a confirmed cut.""" + task = ctx.speaker_change_probe_task + if task is None or not task.done(): + return None + ctx.speaker_change_probe_task = None + try: + result = task.result() + except asyncio.CancelledError: + return None + except Exception as exc: + logger.debug("Realtime speaker-change probe task failed: %s", exc) + return None + if not result: + ctx.speaker_change_candidate_embedding = None + ctx.speaker_change_candidate_start_center_sample = None + ctx.speaker_change_candidate_hits = 0 + return None + + embedding = np.asarray(result["embedding"], dtype=np.float32).reshape(-1) + center_sample = int(result["center_sample"]) + reference = ctx.speaker_change_reference_embedding + if reference is None or reference.shape != embedding.shape: + ctx.speaker_change_reference_embedding = embedding.copy() + ctx.speaker_change_reference_center_sample = center_sample + return None + + reference_score = float(np.dot(reference, embedding)) + if reference_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD: + # Slowly adapt to normal voice and prosody variation from one speaker. + updated = (0.8 * reference) + (0.2 * embedding) + norm = max(float(np.linalg.norm(updated)), 1e-12) + ctx.speaker_change_reference_embedding = (updated / norm).astype(np.float32) + ctx.speaker_change_reference_center_sample = center_sample + ctx.speaker_change_candidate_embedding = None + ctx.speaker_change_candidate_start_center_sample = None + ctx.speaker_change_candidate_hits = 0 + return None + + candidate = ctx.speaker_change_candidate_embedding + if candidate is not None and candidate.shape == embedding.shape: + candidate_score = float(np.dot(candidate, embedding)) + else: + candidate_score = -1.0 + + if candidate_score >= self._SPEAKER_CHANGE_COSINE_THRESHOLD: + updated = (candidate * ctx.speaker_change_candidate_hits) + embedding + norm = max(float(np.linalg.norm(updated)), 1e-12) + ctx.speaker_change_candidate_embedding = (updated / norm).astype(np.float32) + ctx.speaker_change_candidate_hits += 1 + else: + ctx.speaker_change_candidate_embedding = embedding.copy() + ctx.speaker_change_candidate_start_center_sample = center_sample + ctx.speaker_change_candidate_hits = 1 + + candidate_start = ctx.speaker_change_candidate_start_center_sample + if ( + candidate_start is None + or ctx.speaker_change_candidate_hits < self._SPEAKER_CHANGE_CONFIRM_HITS + or center_sample - candidate_start < self._SPEAKER_CHANGE_CONFIRM_SPAN_SAMPLES + ): + return None + + reference_center = ctx.speaker_change_reference_center_sample + if reference_center is None: + return None + # Place the cut halfway between the last stable voice window and the + # first sustained new-speaker window; the confirmation delay stays in + # the new speaker's carry buffer and is not lost. + turn_start_sample = int(ctx.segment_buffer_start_sample) + boundary_sample = max( + (int(reference_center) + int(candidate_start)) // 2, + turn_start_sample + self._SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES, + ) + + logger.info( + "[%s] Sustained speaker change confirmed: candidate_hits=%d " + "candidate_span_sec=%.2f boundary_offset_sec=%.2f", + ctx.diagnostic_session_id, + ctx.speaker_change_candidate_hits, + (center_sample - candidate_start) / 16000.0, + (boundary_sample - turn_start_sample) / 16000.0, + ) + return boundary_sample + + async def _commit_realtime_speaker_change( + self, + websocket: WebSocket, + ctx: ConnectionContext, + task_id: str, + boundary_sample: int, + ) -> bool: + """Finalize the previous speaker and retain post-boundary audio as carry.""" + current_audio = np.asarray(ctx.segment_audio_buffer, dtype=np.float32) + turn_start_sample = int(ctx.segment_buffer_start_sample) + boundary_offset = int(boundary_sample) - turn_start_sample + if ( + boundary_offset < self._SPEAKER_CHANGE_MIN_SEGMENT_SAMPLES + or boundary_offset >= current_audio.size + ): + return False + + finalized_audio = np.asarray(current_audio[:boundary_offset], dtype=np.float32) + carry_audio = np.asarray(current_audio[boundary_offset:], dtype=np.float32) + if carry_audio.size == 0: + return False + + await self._commit_retranscribe_turn( + websocket, + ctx, + task_id, + "speaker_change", + finalized_audio_override=finalized_audio, + carry_audio_override=carry_audio, + segment_start_ms_override=int(turn_start_sample / 16), + segment_end_ms_override=int(boundary_sample / 16), + ) + # VAD still sees continuous speech across this diarization boundary. + ctx.vad_speech_active = True + ctx.vad_speech_start_sample = int(boundary_sample) + return True + async def _finish_funasr_stream( self, ctx: ConnectionContext, @@ -3205,6 +3415,12 @@ class Qwen3ASRService: ctx, segment_text, ) + if funasr_streaming and reason == "speaker_change": + # A confirmed speaker transition is a sentence boundary even when + # the punctuation model did not emit a terminal mark. + boundary_segment = {"text": segment_text} + self._close_segment_at_speaker_boundary(boundary_segment) + segment_text = str(boundary_segment.get("text") or "") is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec) logger.info( @@ -3314,6 +3530,7 @@ class Qwen3ASRService: ctx.sentence_active = bool(carry_audio.size > 0) ctx.silence_samples = 0 ctx.total_samples = int(carry_audio.size) + self._reset_realtime_speaker_change_state(ctx) self._reset_partial_state(ctx) if ctx.sentence_active and carry_audio.size > 0: @@ -3840,6 +4057,20 @@ class Qwen3ASRService: ctx.state = ConnectionState.STREAMING continue + if self._uses_funasr_streaming(ctx) and not bulk_audio_frame: + speaker_boundary = self._consume_realtime_speaker_probe(ctx) + if speaker_boundary is not None and await self._commit_realtime_speaker_change( + websocket, + ctx, + task_id, + speaker_boundary, + ): + ctx.state = ConnectionState.STREAMING + continue + # Enqueue the next copied 1.5 s window after consuming + # the prior result; diarization must never block PCM I/O. + self._schedule_realtime_speaker_probe(ctx) + silence_threshold = self._get_dynamic_silence_threshold_samples(ctx) if self._enable_realtime_vad(ctx) and ctx.vad_speech_active: # Give FSMN-VAD time to confirm an endpoint before the