气泡切割调整,将窗口内多个说话人分割为多个窗口。修改标点边界问题
parent
93e58b95bc
commit
3084f3b79a
|
|
@ -473,6 +473,165 @@ class Qwen3ASRService:
|
|||
sentence[key] = payload.get(key)
|
||||
return sentence
|
||||
|
||||
@staticmethod
|
||||
def _split_text_at_punctuation(text: str, durations: List[int]) -> List[str]:
|
||||
"""Split text near timed speaker boundaries without orphaning punctuation."""
|
||||
cleaned = str(text or "").strip()
|
||||
if not cleaned or len(durations) < 2:
|
||||
return [cleaned] if cleaned else []
|
||||
|
||||
total_duration = sum(max(int(duration), 0) for duration in durations)
|
||||
if total_duration <= 0 or len(cleaned) < len(durations):
|
||||
return [cleaned]
|
||||
|
||||
boundaries: List[int] = []
|
||||
elapsed = 0
|
||||
previous_boundary = 0
|
||||
for duration in durations[:-1]:
|
||||
elapsed += max(int(duration), 0)
|
||||
target = round(len(cleaned) * elapsed / total_duration)
|
||||
min_boundary = previous_boundary + 1
|
||||
max_boundary = len(cleaned) - (len(durations) - len(boundaries) - 1)
|
||||
target = min(max(target, min_boundary), max_boundary)
|
||||
|
||||
# Prefer a sentence boundary close to the acoustic boundary so a
|
||||
# comma or full stop stays with the preceding speaker's words.
|
||||
radius = max(8, min(48, len(cleaned) // 5))
|
||||
candidates = [
|
||||
idx + 1
|
||||
for idx, char in enumerate(cleaned)
|
||||
if min_boundary <= idx + 1 <= max_boundary
|
||||
and abs((idx + 1) - target) <= radius
|
||||
and unicodedata.category(char).startswith("P")
|
||||
and not _is_opening_punctuation(char)
|
||||
]
|
||||
if candidates:
|
||||
boundary = min(candidates, key=lambda idx: abs(idx - target))
|
||||
else:
|
||||
boundary = target
|
||||
# A fallback character cut can land immediately before a mark;
|
||||
# move that mark left so the next bubble never begins with it.
|
||||
while (
|
||||
boundary < max_boundary
|
||||
and unicodedata.category(cleaned[boundary]).startswith("P")
|
||||
and not _is_opening_punctuation(cleaned[boundary])
|
||||
):
|
||||
boundary += 1
|
||||
boundaries.append(boundary)
|
||||
previous_boundary = boundary
|
||||
|
||||
cuts = [0, *boundaries, len(cleaned)]
|
||||
return [cleaned[cuts[idx]:cuts[idx + 1]].strip() for idx in range(len(cuts) - 1)]
|
||||
|
||||
def _build_live_speaker_fragments(
|
||||
self,
|
||||
ctx: ConnectionContext,
|
||||
segment: Dict[str, Any],
|
||||
speaker_info: Dict[str, Any],
|
||||
*,
|
||||
segment_index: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Create live bubble updates from speaker changes inside one ASR segment."""
|
||||
timeline = speaker_info.get("_speaker_timeline") or []
|
||||
start_ms = int(segment.get("start_ms", 0))
|
||||
end_ms = int(segment.get("end_ms", start_ms))
|
||||
spans: List[List[int]] = []
|
||||
for item in timeline:
|
||||
start = max(start_ms, int(item.get("start_ms", start_ms)))
|
||||
end = min(end_ms, int(item.get("end_ms", end_ms)))
|
||||
if end <= start:
|
||||
continue
|
||||
cluster_index = int(item.get("cluster_index", 0))
|
||||
if spans and spans[-1][2] == cluster_index and start <= spans[-1][1]:
|
||||
spans[-1][1] = max(spans[-1][1], end)
|
||||
else:
|
||||
spans.append([start, end, cluster_index])
|
||||
|
||||
# Very short label flips are usually caused by a window straddling a
|
||||
# turn boundary; absorb them before creating visible bubbles.
|
||||
min_turn_ms = 1500
|
||||
changed = True
|
||||
while changed and len(spans) > 1:
|
||||
changed = False
|
||||
for idx, span in enumerate(spans):
|
||||
if span[1] - span[0] >= min_turn_ms:
|
||||
continue
|
||||
if idx == 0:
|
||||
spans[1][0] = span[0]
|
||||
else:
|
||||
spans[idx - 1][1] = span[1]
|
||||
spans.pop(idx)
|
||||
changed = True
|
||||
break
|
||||
|
||||
if len(spans) <= 1:
|
||||
return [dict(segment)]
|
||||
|
||||
durations = [max(span[1] - span[0], 1) for span in spans]
|
||||
text_parts = self._split_text_at_punctuation(str(segment.get("text") or ""), durations)
|
||||
if len(text_parts) != len(spans) or any(not part for part in text_parts):
|
||||
return [dict(segment)]
|
||||
|
||||
result: List[Dict[str, Any]] = []
|
||||
for idx, (span, part_text) in enumerate(zip(spans, text_parts)):
|
||||
cluster_index = span[2]
|
||||
mapped_speaker = ctx.speaker_display_map.get(cluster_index)
|
||||
if mapped_speaker:
|
||||
label = mapped_speaker
|
||||
else:
|
||||
label = f"Speaker{cluster_index + 1:02d}"
|
||||
fragment = dict(segment)
|
||||
fragment.update(
|
||||
text=part_text,
|
||||
start_ms=span[0],
|
||||
end_ms=span[1],
|
||||
speaker_id=label,
|
||||
speaker_name=label,
|
||||
)
|
||||
# Keep the original sentence ID for the first bubble. Extra IDs
|
||||
# occupy a reserved range and are replaced by normal IDs at end.
|
||||
fragment["index"] = (
|
||||
segment_index
|
||||
if idx == 0
|
||||
else 1_000_000_000 + segment_index * 1000 + idx
|
||||
)
|
||||
if cluster_index == speaker_info.get("_cluster_index"):
|
||||
for key in ("user_id", "registry_speaker_id", "speaker_name", "speaker_id"):
|
||||
if speaker_info.get(key) is not None:
|
||||
fragment[key] = speaker_info[key]
|
||||
result.append(fragment)
|
||||
return result
|
||||
|
||||
def _move_leading_punctuation_to_previous(
|
||||
self,
|
||||
ctx: ConnectionContext,
|
||||
text: str,
|
||||
) -> tuple[str, Optional[int]]:
|
||||
"""Keep punctuation attached to the phrase before an ASR segment cut."""
|
||||
current = str(text or "").lstrip()
|
||||
if not current or not ctx.confirmed_segments:
|
||||
return current, None
|
||||
punctuation_end = 0
|
||||
while punctuation_end < len(current):
|
||||
char = current[punctuation_end]
|
||||
if not unicodedata.category(char).startswith("P") or _is_opening_punctuation(char):
|
||||
break
|
||||
punctuation_end += 1
|
||||
if punctuation_end == 0:
|
||||
return current, None
|
||||
|
||||
previous = ctx.confirmed_segments[-1]
|
||||
previous_text = str(previous.get("text") or "").rstrip()
|
||||
if not previous_text:
|
||||
return current, None
|
||||
leading = current[:punctuation_end]
|
||||
# If the prior pass already closed this phrase, don't stack another mark.
|
||||
if unicodedata.category(previous_text[-1]).startswith("P"):
|
||||
leading = ""
|
||||
if leading:
|
||||
previous["text"] = previous_text + leading
|
||||
return current[punctuation_end:].lstrip(), int(previous.get("index", -1))
|
||||
|
||||
async def _send_json_safe(
|
||||
self,
|
||||
websocket: WebSocket,
|
||||
|
|
@ -552,6 +711,7 @@ class Qwen3ASRService:
|
|||
ctx: ConnectionContext,
|
||||
task_id: str,
|
||||
segment_index: int,
|
||||
speaker_info: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
segment = next(
|
||||
(item for item in ctx.confirmed_segments if int(item.get("index", -1)) == int(segment_index)),
|
||||
|
|
@ -559,7 +719,26 @@ class Qwen3ASRService:
|
|||
)
|
||||
if segment is None:
|
||||
return
|
||||
sentence_payload = self._build_tencent_sentence(segment, sentence_type=1)
|
||||
effective_speaker_info = speaker_info or (
|
||||
segment if segment.get("_speaker_timeline") else None
|
||||
)
|
||||
fragments = (
|
||||
self._build_live_speaker_fragments(
|
||||
ctx,
|
||||
segment,
|
||||
effective_speaker_info,
|
||||
segment_index=segment_index,
|
||||
)
|
||||
if effective_speaker_info
|
||||
else [dict(segment)]
|
||||
)
|
||||
sentence_payloads = [
|
||||
self._build_tencent_sentence(fragment, sentence_type=1)
|
||||
for fragment in fragments
|
||||
if str(fragment.get("text") or "").strip()
|
||||
]
|
||||
if not sentence_payloads:
|
||||
return
|
||||
await self._send_json_safe(
|
||||
websocket,
|
||||
ctx,
|
||||
|
|
@ -573,7 +752,7 @@ class Qwen3ASRService:
|
|||
"index": int(segment_index),
|
||||
"voice_text_str": str(segment.get("text") or ""),
|
||||
},
|
||||
"sentences": [sentence_payload],
|
||||
"sentences": sentence_payloads,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -901,7 +1080,7 @@ class Qwen3ASRService:
|
|||
self,
|
||||
ctx: ConnectionContext,
|
||||
*,
|
||||
min_split_sec: float = 3.0,
|
||||
min_split_sec: float = 1.5,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Split final ASR text where the FunASR speaker timeline changes.
|
||||
|
||||
|
|
@ -929,10 +1108,10 @@ class Qwen3ASRService:
|
|||
# FunASR keeps any text before the 128-chunk history window
|
||||
# intact and only reassigns the portion covered by diarization.
|
||||
text = str(original.get("text") or "")
|
||||
ratio = (history_start_ms - seg_start) / max(seg_end - seg_start, 1)
|
||||
split_index = min(len(text) - 1, max(1, round(len(text) * ratio)))
|
||||
prefix_text = text[:split_index].strip()
|
||||
suffix_text = text[split_index:].strip()
|
||||
prefix_text, suffix_text = self._split_text_at_punctuation(
|
||||
text,
|
||||
[history_start_ms - seg_start, seg_end - history_start_ms],
|
||||
) if text else ("", "")
|
||||
if prefix_text and suffix_text:
|
||||
prefix = dict(original)
|
||||
prefix.update(index=len(final_segments), text=prefix_text, end_ms=history_start_ms)
|
||||
|
|
@ -958,8 +1137,8 @@ class Qwen3ASRService:
|
|||
final_segments.append(segment)
|
||||
continue
|
||||
|
||||
# FunASR merges same-speaker regions and absorbs regions shorter
|
||||
# than three seconds into the neighboring speaker.
|
||||
# Use the same minimum turn length as live rendering so a speaker
|
||||
# change does not disappear when the session is finalized.
|
||||
filtered: List[List[int]] = [overlaps[0][:]]
|
||||
for current in overlaps[1:]:
|
||||
previous = filtered[-1]
|
||||
|
|
@ -988,7 +1167,6 @@ class Qwen3ASRService:
|
|||
final_segments.append(segment)
|
||||
continue
|
||||
|
||||
total_duration = sum(end - start for start, end, _ in merged)
|
||||
text = str(original.get("text") or "")
|
||||
if len(text) < len(merged):
|
||||
longest = max(merged, key=lambda item: item[1] - item[0])
|
||||
|
|
@ -997,16 +1175,18 @@ class Qwen3ASRService:
|
|||
segment["index"] = len(final_segments)
|
||||
final_segments.append(segment)
|
||||
continue
|
||||
char_pos = 0
|
||||
for part_index, (start, end, cluster_index) in enumerate(merged):
|
||||
if part_index == len(merged) - 1:
|
||||
part_text = text[char_pos:]
|
||||
else:
|
||||
remaining_parts = len(merged) - part_index - 1
|
||||
proposed = int(len(text) * (end - start) / max(total_duration, 1))
|
||||
char_count = max(1, min(proposed, len(text) - char_pos - remaining_parts))
|
||||
part_text = text[char_pos:char_pos + char_count]
|
||||
char_pos += char_count
|
||||
text_parts = self._split_text_at_punctuation(
|
||||
text,
|
||||
[end - start for start, end, _ in merged],
|
||||
)
|
||||
if len(text_parts) != len(merged) or any(not part for part in text_parts):
|
||||
longest = max(merged, key=lambda item: item[1] - item[0])
|
||||
segment = dict(original)
|
||||
segment.update(speaker_info(longest[2]))
|
||||
segment["index"] = len(final_segments)
|
||||
final_segments.append(segment)
|
||||
continue
|
||||
for part_text, (start, end, cluster_index) in zip(text_parts, merged):
|
||||
if not part_text.strip():
|
||||
continue
|
||||
fragment = dict(original)
|
||||
|
|
@ -2037,6 +2217,8 @@ class Qwen3ASRService:
|
|||
self,
|
||||
ctx: ConnectionContext,
|
||||
text: str,
|
||||
*,
|
||||
preserve_boundary_punctuation: bool = False,
|
||||
) -> str:
|
||||
current = str(text or "").strip()
|
||||
if not current or not ctx.confirmed_segments:
|
||||
|
|
@ -2048,6 +2230,10 @@ class Qwen3ASRService:
|
|||
return current
|
||||
|
||||
trimmed = current[cut_index:].lstrip(" \t\r\n,,。.!!??;;::、")
|
||||
if preserve_boundary_punctuation:
|
||||
# Final text is normalized after overlap trimming; let the boundary
|
||||
# handler attach any surviving mark to the prior confirmed segment.
|
||||
trimmed = current[cut_index:].lstrip()
|
||||
if trimmed != current:
|
||||
logger.debug(
|
||||
"Trimmed realtime segment overlap: removed=%s remaining=%s",
|
||||
|
|
@ -2623,7 +2809,13 @@ class Qwen3ASRService:
|
|||
speaker_info=speaker_info,
|
||||
)
|
||||
if before_view != after_view:
|
||||
await self._emit_speaker_update(websocket, ctx, task_id, segment_index)
|
||||
await self._emit_speaker_update(
|
||||
websocket,
|
||||
ctx,
|
||||
task_id,
|
||||
segment_index,
|
||||
speaker_info=speaker_info,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("[%s] Async speaker resolution failed for segment %s: %s", task_id, segment_index, exc)
|
||||
|
||||
|
|
@ -2832,11 +3024,19 @@ class Qwen3ASRService:
|
|||
duration_sec=duration_sec,
|
||||
language=segment_language,
|
||||
)
|
||||
segment_text = self._trim_previous_segment_overlap(ctx, segment_text)
|
||||
segment_text = self._trim_previous_segment_overlap(
|
||||
ctx,
|
||||
segment_text,
|
||||
preserve_boundary_punctuation=True,
|
||||
)
|
||||
if funasr_streaming:
|
||||
# Keep CT-Transformer inference off the asyncio event loop so other
|
||||
# WebSocket sessions can continue sending and decoding audio.
|
||||
segment_text = await run_sync(self._finalize_funasr_text, ctx, segment_text)
|
||||
segment_text, punctuation_previous_index = self._move_leading_punctuation_to_previous(
|
||||
ctx,
|
||||
segment_text,
|
||||
)
|
||||
|
||||
is_valid = self._is_valid_committed_segment_text(segment_text, duration_sec)
|
||||
logger.info(
|
||||
|
|
@ -2923,6 +3123,16 @@ class Qwen3ASRService:
|
|||
)
|
||||
ctx.segment_index += 1
|
||||
|
||||
if punctuation_previous_index is not None and punctuation_previous_index >= 0:
|
||||
# The prior bubble was already sent; patch it with the boundary mark
|
||||
# so punctuation never appears detached at the next bubble's start.
|
||||
await self._emit_speaker_update(
|
||||
websocket,
|
||||
ctx,
|
||||
task_id,
|
||||
punctuation_previous_index,
|
||||
)
|
||||
|
||||
ctx.timeline_cursor_ms = segment_end_ms
|
||||
if carry_audio.size > 0:
|
||||
ctx.segment_buffer_start_sample += max(
|
||||
|
|
|
|||
|
|
@ -4148,6 +4148,13 @@ ${transcript}
|
|||
const orderedItems = items
|
||||
.map((item, index) => ({ item, index }))
|
||||
.sort((left, right) => {
|
||||
const leftStart = Number(left.item.startTimeMs || 0);
|
||||
const rightStart = Number(right.item.startTimeMs || 0);
|
||||
// Speaker subsegments can arrive after later ASR segments; order by
|
||||
// their audio timestamps while retaining ID order for equal/unknown times.
|
||||
if (leftStart > 0 && rightStart > 0 && leftStart !== rightStart) {
|
||||
return leftStart - rightStart;
|
||||
}
|
||||
const leftId = left.item.segmentIndex;
|
||||
const rightId = right.item.segmentIndex;
|
||||
const leftHasId = leftId !== null && leftId !== undefined && Number.isFinite(Number(leftId));
|
||||
|
|
|
|||
Loading…
Reference in New Issue