气泡切割调整,将窗口内多个说话人分割为多个窗口。修改标点边界问题

main
Bifang 2026-10-12 11:27:43 +08:00
parent 93e58b95bc
commit 3084f3b79a
2 changed files with 239 additions and 22 deletions

View File

@ -473,6 +473,165 @@ class Qwen3ASRService:
sentence[key] = payload.get(key)
return sentence
@staticmethod
def _split_text_at_punctuation(text: str, durations: List[int]) -> List[str]:
"""Split text near timed speaker boundaries without orphaning punctuation."""
cleaned = str(text or "").strip()
if not cleaned or len(durations) < 2:
return [cleaned] if cleaned else []
total_duration = sum(max(int(duration), 0) for duration in durations)
if total_duration <= 0 or len(cleaned) < len(durations):
return [cleaned]
boundaries: List[int] = []
elapsed = 0
previous_boundary = 0
for duration in durations[:-1]:
elapsed += max(int(duration), 0)
target = round(len(cleaned) * elapsed / total_duration)
min_boundary = previous_boundary + 1
max_boundary = len(cleaned) - (len(durations) - len(boundaries) - 1)
target = min(max(target, min_boundary), max_boundary)
# Prefer a sentence boundary close to the acoustic boundary so a
# comma or full stop stays with the preceding speaker's words.
radius = max(8, min(48, len(cleaned) // 5))
candidates = [
idx + 1
for idx, char in enumerate(cleaned)
if min_boundary <= idx + 1 <= max_boundary
and abs((idx + 1) - target) <= radius
and unicodedata.category(char).startswith("P")
and not _is_opening_punctuation(char)
]
if candidates:
boundary = min(candidates, key=lambda idx: abs(idx - target))
else:
boundary = target
# A fallback character cut can land immediately before a mark;
# move that mark left so the next bubble never begins with it.
while (
boundary < max_boundary
and unicodedata.category(cleaned[boundary]).startswith("P")
and not _is_opening_punctuation(cleaned[boundary])
):
boundary += 1
boundaries.append(boundary)
previous_boundary = boundary
cuts = [0, *boundaries, len(cleaned)]
return [cleaned[cuts[idx]:cuts[idx + 1]].strip() for idx in range(len(cuts) - 1)]
def _build_live_speaker_fragments(
self,
ctx: ConnectionContext,
segment: Dict[str, Any],
speaker_info: Dict[str, Any],
*,
segment_index: int,
) -> List[Dict[str, Any]]:
"""Create live bubble updates from speaker changes inside one ASR segment."""
timeline = speaker_info.get("_speaker_timeline") or []
start_ms = int(segment.get("start_ms", 0))
end_ms = int(segment.get("end_ms", start_ms))
spans: List[List[int]] = []
for item in timeline:
start = max(start_ms, int(item.get("start_ms", start_ms)))
end = min(end_ms, int(item.get("end_ms", end_ms)))
if end <= start:
continue
cluster_index = int(item.get("cluster_index", 0))
if spans and spans[-1][2] == cluster_index and start <= spans[-1][1]:
spans[-1][1] = max(spans[-1][1], end)
else:
spans.append([start, end, cluster_index])
# Very short label flips are usually caused by a window straddling a
# turn boundary; absorb them before creating visible bubbles.
min_turn_ms = 1500
changed = True
while changed and len(spans) > 1:
changed = False
for idx, span in enumerate(spans):
if span[1] - span[0] >= min_turn_ms:
continue
if idx == 0:
spans[1][0] = span[0]
else:
spans[idx - 1][1] = span[1]
spans.pop(idx)
changed = True
break
if len(spans) <= 1:
return [dict(segment)]
durations = [max(span[1] - span[0], 1) for span in spans]
text_parts = self._split_text_at_punctuation(str(segment.get("text") or ""), durations)
if len(text_parts) != len(spans) or any(not part for part in text_parts):
return [dict(segment)]
result: List[Dict[str, Any]] = []
for idx, (span, part_text) in enumerate(zip(spans, text_parts)):
cluster_index = span[2]
mapped_speaker = ctx.speaker_display_map.get(cluster_index)
if mapped_speaker:
label = mapped_speaker
else:
label = f"Speaker{cluster_index + 1:02d}"
fragment = dict(segment)
fragment.update(
text=part_text,
start_ms=span[0],
end_ms=span[1],
speaker_id=label,
speaker_name=label,
)
# Keep the original sentence ID for the first bubble. Extra IDs
# occupy a reserved range and are replaced by normal IDs at end.
fragment["index"] = (
segment_index
if idx == 0
else 1_000_000_000 + segment_index * 1000 + idx
)
if cluster_index == speaker_info.get("_cluster_index"):
for key in ("user_id", "registry_speaker_id", "speaker_name", "speaker_id"):
if speaker_info.get(key) is not None:
fragment[key] = speaker_info[key]
result.append(fragment)
return result
def _move_leading_punctuation_to_previous(
self,
ctx: ConnectionContext,
text: str,
) -> tuple[str, Optional[int]]:
"""Keep punctuation attached to the phrase before an ASR segment cut."""
current = str(text or "").lstrip()
if not current or not ctx.confirmed_segments:
return current, None
punctuation_end = 0
while punctuation_end < len(current):
char = current[punctuation_end]
if not unicodedata.category(char).startswith("P") or _is_opening_punctuation(char):
break
punctuation_end += 1
if punctuation_end == 0:
return current, None
previous = ctx.confirmed_segments[-1]
previous_text = str(previous.get("text") or "").rstrip()
if not previous_text:
return current, None
leading = current[:punctuation_end]
# If the prior pass already closed this phrase, don't stack another mark.
if unicodedata.category(previous_text[-1]).startswith("P"):
leading = ""
if leading:
previous["text"] = previous_text + leading
return current[punctuation_end:].lstrip(), int(previous.get("index", -1))
async def _send_json_safe(
self,
websocket: WebSocket,
@ -552,6 +711,7 @@ class Qwen3ASRService:
ctx: ConnectionContext,
task_id: str,
segment_index: int,
speaker_info: Optional[Dict[str, Any]] = None,
) -> None:
segment = next(
(item for item in ctx.confirmed_segments if int(item.get("index", -1)) == int(segment_index)),
@ -559,7 +719,26 @@ class Qwen3ASRService:
)
if segment is None:
return
sentence_payload = self._build_tencent_sentence(segment, sentence_type=1)
effective_speaker_info = speaker_info or (
segment if segment.get("_speaker_timeline") else None
)
fragments = (
self._build_live_speaker_fragments(
ctx,
segment,
effective_speaker_info,
segment_index=segment_index,
)
if effective_speaker_info
else [dict(segment)]
)
sentence_payloads = [
self._build_tencent_sentence(fragment, sentence_type=1)
for fragment in fragments
if str(fragment.get("text") or "").strip()
]
if not sentence_payloads:
return
await self._send_json_safe(
websocket,
ctx,
@ -573,7 +752,7 @@ class Qwen3ASRService:
"index": int(segment_index),
"voice_text_str": str(segment.get("text") or ""),
},
"sentences": [sentence_payload],
"sentences": sentence_payloads,
},
)
@ -901,7 +1080,7 @@ class Qwen3ASRService:
self,
ctx: ConnectionContext,
*,
min_split_sec: float = 3.0,
min_split_sec: float = 1.5,
) -> List[Dict[str, Any]]:
"""Split final ASR text where the FunASR speaker timeline changes.
@ -929,10 +1108,10 @@ class Qwen3ASRService:
# FunASR keeps any text before the 128-chunk history window
# intact and only reassigns the portion covered by diarization.
text = str(original.get("text") or "")
ratio = (history_start_ms - seg_start) / max(seg_end - seg_start, 1)
split_index = min(len(text) - 1, max(1, round(len(text) * ratio)))
prefix_text = text[:split_index].strip()
suffix_text = text[split_index:].strip()
prefix_text, suffix_text = self._split_text_at_punctuation(
text,
[history_start_ms - seg_start, seg_end - history_start_ms],
) if text else ("", "")
if prefix_text and suffix_text:
prefix = dict(original)
prefix.update(index=len(final_segments), text=prefix_text, end_ms=history_start_ms)
@ -958,8 +1137,8 @@ class Qwen3ASRService:
final_segments.append(segment)
continue
# FunASR merges same-speaker regions and absorbs regions shorter
# than three seconds into the neighboring speaker.
# Use the same minimum turn length as live rendering so a speaker
# change does not disappear when the session is finalized.
filtered: List[List[int]] = [overlaps[0][:]]
for current in overlaps[1:]:
previous = filtered[-1]
@ -988,7 +1167,6 @@ class Qwen3ASRService:
final_segments.append(segment)
continue
total_duration = sum(end - start for start, end, _ in merged)
text = str(original.get("text") or "")
if len(text) < len(merged):
longest = max(merged, key=lambda item: item[1] - item[0])
@ -997,16 +1175,18 @@ class Qwen3ASRService:
segment["index"] = len(final_segments)
final_segments.append(segment)
continue
char_pos = 0
for part_index, (start, end, cluster_index) in enumerate(merged):
if part_index == len(merged) - 1:
part_text = text[char_pos:]
else:
remaining_parts = len(merged) - part_index - 1
proposed = int(len(text) * (end - start) / max(total_duration, 1))
char_count = max(1, min(proposed, len(text) - char_pos - remaining_parts))
part_text = text[char_pos:char_pos + char_count]
char_pos += char_count
text_parts = self._split_text_at_punctuation(
text,
[end - start for start, end, _ in merged],
)
if len(text_parts) != len(merged) or any(not part for part in text_parts):
longest = max(merged, key=lambda item: item[1] - item[0])
segment = dict(original)
segment.update(speaker_info(longest[2]))
segment["index"] = len(final_segments)
final_segments.append(segment)
continue
for part_text, (start, end, cluster_index) in zip(text_parts, merged):
if not part_text.strip():
continue
fragment = dict(original)
@ -2037,6 +2217,8 @@ class Qwen3ASRService:
self,
ctx: ConnectionContext,
text: str,
*,
preserve_boundary_punctuation: bool = False,
) -> str:
current = str(text or "").strip()
if not current or not ctx.confirmed_segments:
@ -2048,6 +2230,10 @@ class Qwen3ASRService:
return current
trimmed = current[cut_index:].lstrip(" \t\r\n,,。.!!??;;::、")
if preserve_boundary_punctuation:
# Final text is normalized after overlap trimming; let the boundary
# handler attach any surviving mark to the prior confirmed segment.
trimmed = current[cut_index:].lstrip()
if trimmed != current:
logger.debug(
"Trimmed realtime segment overlap: removed=%s remaining=%s",
@ -2623,7 +2809,13 @@ class Qwen3ASRService:
speaker_info=speaker_info,
)
if before_view != after_view:
await self._emit_speaker_update(websocket, ctx, task_id, segment_index)
await self._emit_speaker_update(
websocket,
ctx,
task_id,
segment_index,
speaker_info=speaker_info,
)
except Exception as exc:
logger.warning("[%s] Async speaker resolution failed for segment %s: %s", task_id, segment_index, exc)
@ -2832,11 +3024,19 @@ class Qwen3ASRService:
duration_sec=duration_sec,
language=segment_language,
)
segment_text = self._trim_previous_segment_overlap(ctx, segment_text)
segment_text = self._trim_previous_segment_overlap(
ctx,
segment_text,
preserve_boundary_punctuation=True,
)
if funasr_streaming:
# Keep CT-Transformer inference off the asyncio event loop so other
# WebSocket sessions can continue sending and decoding audio.
segment_text = await run_sync(self._finalize_funasr_text, ctx, segment_text)
segment_text, punctuation_previous_index = self._move_leading_punctuation_to_previous(
ctx,
segment_text,
)
is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec)
logger.info(
@ -2923,6 +3123,16 @@ class Qwen3ASRService:
)
ctx.segment_index += 1
if punctuation_previous_index is not None and punctuation_previous_index >= 0:
# The prior bubble was already sent; patch it with the boundary mark
# so punctuation never appears detached at the next bubble's start.
await self._emit_speaker_update(
websocket,
ctx,
task_id,
punctuation_previous_index,
)
ctx.timeline_cursor_ms = segment_end_ms
if carry_audio.size > 0:
ctx.segment_buffer_start_sample += max(

View File

@ -4148,6 +4148,13 @@ ${transcript}
const orderedItems = items
.map((item, index) => ({ item, index }))
.sort((left, right) => {
const leftStart = Number(left.item.startTimeMs || 0);
const rightStart = Number(right.item.startTimeMs || 0);
// Speaker subsegments can arrive after later ASR segments; order by
// their audio timestamps while retaining ID order for equal/unknown times.
if (leftStart > 0 && rightStart > 0 && leftStart !== rightStart) {
return leftStart - rightStart;
}
const leftId = left.item.segmentIndex;
const rightId = right.item.segmentIndex;
const leftHasId = leftId !== null && leftId !== undefined && Number.isFinite(Number(leftId));