diff --git a/app/services/qwen3_websocket_asr.py b/app/services/qwen3_websocket_asr.py index 7fe7362..7a2c9c7 100644 --- a/app/services/qwen3_websocket_asr.py +++ b/app/services/qwen3_websocket_asr.py @@ -473,6 +473,165 @@ class Qwen3ASRService: sentence[key] = payload.get(key) return sentence + @staticmethod + def _split_text_at_punctuation(text: str, durations: List[int]) -> List[str]: + """Split text near timed speaker boundaries without orphaning punctuation.""" + cleaned = str(text or "").strip() + if not cleaned or len(durations) < 2: + return [cleaned] if cleaned else [] + + total_duration = sum(max(int(duration), 0) for duration in durations) + if total_duration <= 0 or len(cleaned) < len(durations): + return [cleaned] + + boundaries: List[int] = [] + elapsed = 0 + previous_boundary = 0 + for duration in durations[:-1]: + elapsed += max(int(duration), 0) + target = round(len(cleaned) * elapsed / total_duration) + min_boundary = previous_boundary + 1 + max_boundary = len(cleaned) - (len(durations) - len(boundaries) - 1) + target = min(max(target, min_boundary), max_boundary) + + # Prefer a sentence boundary close to the acoustic boundary so a + # comma or full stop stays with the preceding speaker's words. + radius = max(8, min(48, len(cleaned) // 5)) + candidates = [ + idx + 1 + for idx, char in enumerate(cleaned) + if min_boundary <= idx + 1 <= max_boundary + and abs((idx + 1) - target) <= radius + and unicodedata.category(char).startswith("P") + and not _is_opening_punctuation(char) + ] + if candidates: + boundary = min(candidates, key=lambda idx: abs(idx - target)) + else: + boundary = target + # A fallback character cut can land immediately before a mark; + # move that mark left so the next bubble never begins with it. + while ( + boundary < max_boundary + and unicodedata.category(cleaned[boundary]).startswith("P") + and not _is_opening_punctuation(cleaned[boundary]) + ): + boundary += 1 + boundaries.append(boundary) + previous_boundary = boundary + + cuts = [0, *boundaries, len(cleaned)] + return [cleaned[cuts[idx]:cuts[idx + 1]].strip() for idx in range(len(cuts) - 1)] + + def _build_live_speaker_fragments( + self, + ctx: ConnectionContext, + segment: Dict[str, Any], + speaker_info: Dict[str, Any], + *, + segment_index: int, + ) -> List[Dict[str, Any]]: + """Create live bubble updates from speaker changes inside one ASR segment.""" + timeline = speaker_info.get("_speaker_timeline") or [] + start_ms = int(segment.get("start_ms", 0)) + end_ms = int(segment.get("end_ms", start_ms)) + spans: List[List[int]] = [] + for item in timeline: + start = max(start_ms, int(item.get("start_ms", start_ms))) + end = min(end_ms, int(item.get("end_ms", end_ms))) + if end <= start: + continue + cluster_index = int(item.get("cluster_index", 0)) + if spans and spans[-1][2] == cluster_index and start <= spans[-1][1]: + spans[-1][1] = max(spans[-1][1], end) + else: + spans.append([start, end, cluster_index]) + + # Very short label flips are usually caused by a window straddling a + # turn boundary; absorb them before creating visible bubbles. + min_turn_ms = 1500 + changed = True + while changed and len(spans) > 1: + changed = False + for idx, span in enumerate(spans): + if span[1] - span[0] >= min_turn_ms: + continue + if idx == 0: + spans[1][0] = span[0] + else: + spans[idx - 1][1] = span[1] + spans.pop(idx) + changed = True + break + + if len(spans) <= 1: + return [dict(segment)] + + durations = [max(span[1] - span[0], 1) for span in spans] + text_parts = self._split_text_at_punctuation(str(segment.get("text") or ""), durations) + if len(text_parts) != len(spans) or any(not part for part in text_parts): + return [dict(segment)] + + result: List[Dict[str, Any]] = [] + for idx, (span, part_text) in enumerate(zip(spans, text_parts)): + cluster_index = span[2] + mapped_speaker = ctx.speaker_display_map.get(cluster_index) + if mapped_speaker: + label = mapped_speaker + else: + label = f"Speaker{cluster_index + 1:02d}" + fragment = dict(segment) + fragment.update( + text=part_text, + start_ms=span[0], + end_ms=span[1], + speaker_id=label, + speaker_name=label, + ) + # Keep the original sentence ID for the first bubble. Extra IDs + # occupy a reserved range and are replaced by normal IDs at end. + fragment["index"] = ( + segment_index + if idx == 0 + else 1_000_000_000 + segment_index * 1000 + idx + ) + if cluster_index == speaker_info.get("_cluster_index"): + for key in ("user_id", "registry_speaker_id", "speaker_name", "speaker_id"): + if speaker_info.get(key) is not None: + fragment[key] = speaker_info[key] + result.append(fragment) + return result + + def _move_leading_punctuation_to_previous( + self, + ctx: ConnectionContext, + text: str, + ) -> tuple[str, Optional[int]]: + """Keep punctuation attached to the phrase before an ASR segment cut.""" + current = str(text or "").lstrip() + if not current or not ctx.confirmed_segments: + return current, None + punctuation_end = 0 + while punctuation_end < len(current): + char = current[punctuation_end] + if not unicodedata.category(char).startswith("P") or _is_opening_punctuation(char): + break + punctuation_end += 1 + if punctuation_end == 0: + return current, None + + previous = ctx.confirmed_segments[-1] + previous_text = str(previous.get("text") or "").rstrip() + if not previous_text: + return current, None + leading = current[:punctuation_end] + # If the prior pass already closed this phrase, don't stack another mark. + if unicodedata.category(previous_text[-1]).startswith("P"): + leading = "" + if leading: + previous["text"] = previous_text + leading + return current[punctuation_end:].lstrip(), int(previous.get("index", -1)) + async def _send_json_safe( self, websocket: WebSocket, @@ -552,6 +711,7 @@ class Qwen3ASRService: ctx: ConnectionContext, task_id: str, segment_index: int, + speaker_info: Optional[Dict[str, Any]] = None, ) -> None: segment = next( (item for item in ctx.confirmed_segments if int(item.get("index", -1)) == int(segment_index)), @@ -559,7 +719,26 @@ class Qwen3ASRService: ) if segment is None: return - sentence_payload = self._build_tencent_sentence(segment, sentence_type=1) + effective_speaker_info = speaker_info or ( + segment if segment.get("_speaker_timeline") else None + ) + fragments = ( + self._build_live_speaker_fragments( + ctx, + segment, + effective_speaker_info, + segment_index=segment_index, + ) + if effective_speaker_info + else [dict(segment)] + ) + sentence_payloads = [ + self._build_tencent_sentence(fragment, sentence_type=1) + for fragment in fragments + if str(fragment.get("text") or "").strip() + ] + if not sentence_payloads: + return await self._send_json_safe( websocket, ctx, @@ -573,7 +752,7 @@ class Qwen3ASRService: "index": int(segment_index), "voice_text_str": str(segment.get("text") or ""), }, - "sentences": [sentence_payload], + "sentences": sentence_payloads, }, ) @@ -901,7 +1080,7 @@ class Qwen3ASRService: self, ctx: ConnectionContext, *, - min_split_sec: float = 3.0, + min_split_sec: float = 1.5, ) -> List[Dict[str, Any]]: """Split final ASR text where the FunASR speaker timeline changes. @@ -929,10 +1108,10 @@ class Qwen3ASRService: # FunASR keeps any text before the 128-chunk history window # intact and only reassigns the portion covered by diarization. text = str(original.get("text") or "") - ratio = (history_start_ms - seg_start) / max(seg_end - seg_start, 1) - split_index = min(len(text) - 1, max(1, round(len(text) * ratio))) - prefix_text = text[:split_index].strip() - suffix_text = text[split_index:].strip() + prefix_text, suffix_text = self._split_text_at_punctuation( + text, + [history_start_ms - seg_start, seg_end - history_start_ms], + ) if text else ("", "") if prefix_text and suffix_text: prefix = dict(original) prefix.update(index=len(final_segments), text=prefix_text, end_ms=history_start_ms) @@ -958,8 +1137,8 @@ class Qwen3ASRService: final_segments.append(segment) continue - # FunASR merges same-speaker regions and absorbs regions shorter - # than three seconds into the neighboring speaker. + # Use the same minimum turn length as live rendering so a speaker + # change does not disappear when the session is finalized. filtered: List[List[int]] = [overlaps[0][:]] for current in overlaps[1:]: previous = filtered[-1] @@ -988,7 +1167,6 @@ class Qwen3ASRService: final_segments.append(segment) continue - total_duration = sum(end - start for start, end, _ in merged) text = str(original.get("text") or "") if len(text) < len(merged): longest = max(merged, key=lambda item: item[1] - item[0]) @@ -997,16 +1175,18 @@ class Qwen3ASRService: segment["index"] = len(final_segments) final_segments.append(segment) continue - char_pos = 0 - for part_index, (start, end, cluster_index) in enumerate(merged): - if part_index == len(merged) - 1: - part_text = text[char_pos:] - else: - remaining_parts = len(merged) - part_index - 1 - proposed = int(len(text) * (end - start) / max(total_duration, 1)) - char_count = max(1, min(proposed, len(text) - char_pos - remaining_parts)) - part_text = text[char_pos:char_pos + char_count] - char_pos += char_count + text_parts = self._split_text_at_punctuation( + text, + [end - start for start, end, _ in merged], + ) + if len(text_parts) != len(merged) or any(not part for part in text_parts): + longest = max(merged, key=lambda item: item[1] - item[0]) + segment = dict(original) + segment.update(speaker_info(longest[2])) + segment["index"] = len(final_segments) + final_segments.append(segment) + continue + for part_text, (start, end, cluster_index) in zip(text_parts, merged): if not part_text.strip(): continue fragment = dict(original) @@ -2037,6 +2217,8 @@ class Qwen3ASRService: self, ctx: ConnectionContext, text: str, + *, + preserve_boundary_punctuation: bool = False, ) -> str: current = str(text or "").strip() if not current or not ctx.confirmed_segments: @@ -2048,6 +2230,10 @@ class Qwen3ASRService: return current trimmed = current[cut_index:].lstrip(" \t\r\n,,。.!!??;;::、") + if preserve_boundary_punctuation: + # Final text is normalized after overlap trimming; let the boundary + # handler attach any surviving mark to the prior confirmed segment. + trimmed = current[cut_index:].lstrip() if trimmed != current: logger.debug( "Trimmed realtime segment overlap: removed=%s remaining=%s", @@ -2623,7 +2809,13 @@ class Qwen3ASRService: speaker_info=speaker_info, ) if before_view != after_view: - await self._emit_speaker_update(websocket, ctx, task_id, segment_index) + await self._emit_speaker_update( + websocket, + ctx, + task_id, + segment_index, + speaker_info=speaker_info, + ) except Exception as exc: logger.warning("[%s] Async speaker resolution failed for segment %s: %s", task_id, segment_index, exc) @@ -2832,11 +3024,19 @@ class Qwen3ASRService: duration_sec=duration_sec, language=segment_language, ) - segment_text = self._trim_previous_segment_overlap(ctx, segment_text) + segment_text = self._trim_previous_segment_overlap( + ctx, + segment_text, + preserve_boundary_punctuation=True, + ) if funasr_streaming: # Keep CT-Transformer inference off the asyncio event loop so other # WebSocket sessions can continue sending and decoding audio. segment_text = await run_sync(self._finalize_funasr_text, ctx, segment_text) + segment_text, punctuation_previous_index = self._move_leading_punctuation_to_previous( + ctx, + segment_text, + ) is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec) logger.info( @@ -2923,6 +3123,16 @@ class Qwen3ASRService: ) ctx.segment_index += 1 + if punctuation_previous_index is not None and punctuation_previous_index >= 0: + # The prior bubble was already sent; patch it with the boundary mark + # so punctuation never appears detached at the next bubble's start. + await self._emit_speaker_update( + websocket, + ctx, + task_id, + punctuation_previous_index, + ) + ctx.timeline_cursor_ms = segment_end_ms if carry_audio.size > 0: ctx.segment_buffer_start_sample += max( diff --git a/test_web/index.html b/test_web/index.html index 1b59684..adf62f8 100644 --- a/test_web/index.html +++ b/test_web/index.html @@ -4148,6 +4148,13 @@ ${transcript} const orderedItems = items .map((item, index) => ({ item, index })) .sort((left, right) => { + const leftStart = Number(left.item.startTimeMs || 0); + const rightStart = Number(right.item.startTimeMs || 0); + // Speaker subsegments can arrive after later ASR segments; order by + // their audio timestamps while retaining ID order for equal/unknown times. + if (leftStart > 0 && rightStart > 0 && leftStart !== rightStart) { + return leftStart - rightStart; + } const leftId = left.item.segmentIndex; const rightId = right.item.segmentIndex; const leftHasId = leftId !== null && leftId !== undefined && Number.isFinite(Number(leftId));