935 lines
50 KiB
Python
935 lines
50 KiB
Python
"""配音(TTS)与台词时间轴:把台词变成可对齐的音频与字幕。
|
||
|
||
业务流程中的「台词配音」在这里落到统一任务编排(TaskKind.VOICEOVER),
|
||
语音时长会回写镜头,供时间线决定片段长度与字幕入出点。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from pathlib import Path
|
||
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.config import settings
|
||
from app.models import Character, Project, Shot, VoiceAsset
|
||
from app.providers import registry as model_registry
|
||
from app.providers.base import VoiceGenRequest
|
||
from app.services import costing, events, storage
|
||
from app.services import ffmpeg as ff
|
||
|
||
# 多角色对白逐行合成时,换人之间留的静音(毫秒)
|
||
LINE_GAP_MS = 220
|
||
|
||
# 中文约每秒 4.3 字,英文约每秒 2.6 词,句读额外停顿
|
||
CPS_CJK = 4.3
|
||
WPS_LATIN = 2.6
|
||
PAUSE_SECONDS = 0.28
|
||
|
||
|
||
GENDER_LABELS = {"male": "男声", "female": "女声", "neutral": "中性声"}
|
||
|
||
AUDIO_SUFFIXES = {".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".opus"}
|
||
|
||
# 性别关键词:中文文本里性别往往只写在名字或设定里(「男性工程师」「25 岁女性」)。
|
||
# 双字/多字词整权重 1,单字「男」「女」只算 0.5,避免「男朋友」「女朋友」互相抵消后
|
||
# 反而判成无性别。
|
||
_GENDER_MARKERS = {
|
||
"female": ("女性", "女子", "女士", "小姐", "姑娘", "女孩", "女生", "女主", "母亲", "妈妈",
|
||
"姐姐", "妹妹", "阿姨", "奶奶", "女声", "她"),
|
||
"male": ("男性", "男子", "先生", "小哥", "大叔", "男主", "父亲", "爸爸", "哥哥", "弟弟",
|
||
"叔叔", "爷爷", "男声", "男生", "他"),
|
||
}
|
||
|
||
# 姓名里的性别线索:国产剧本里「俊杰」「紫小慧」这种名字本身携带性别,
|
||
# 但 characters.gender 常常是空的(平台人物母版压根没有这一列),
|
||
# 只靠「男性/女士」这类关键词就推不出性别,声线只能瞎选。
|
||
_NAME_FEMALE_MARKERS = ("慧", "娟", "婷", "燕", "琳", "莉", "娜", "玲", "芳", "秀", "云", "霞",
|
||
"梅", "兰", "凤", "姐", "妹", "娘", "婆", "妞", " Girl".strip().lower())
|
||
_NAME_MALE_MARKERS = ("杰", "军", "强", "磊", "涛", "鹏", "飞", "刚", "伟", "建", "国", "明", "辉",
|
||
"斌", "波", "哥", "弟", "爷", "叔", "伯", "兄")
|
||
|
||
# 年龄档:只有性别还不够——「25 岁年轻女员工」配到「偏低的中国老年女声」,
|
||
# 性别对了,人物依旧不对(这正是上一版成片被吐槽的原因)。
|
||
AGE_BANDS = ("child", "young", "middle", "old")
|
||
AGE_BAND_LABELS = {"child": "儿童声", "young": "青年声", "middle": "中年声", "old": "老年声"}
|
||
_AGE_ORDER = {"child": 0, "young": 1, "middle": 2, "old": 3}
|
||
_AGE_MARKERS = {
|
||
"child": ("女童", "男童", "儿童", "小孩", "孩子", "萝莉", "正太", "娃娃音", "学龄", "童声"),
|
||
"young": ("年轻", "青年", "少女", "少男", "少年", "小伙", "姑娘", "女孩", "女生", "男生",
|
||
"大学生", "二十", "25 岁", "25岁", "20 岁", "20岁", "三十", "30 岁", "30岁"),
|
||
"middle": ("中年", "四十", "五十", "大叔", "大妈", "阿姨", "姐夫", "嫂子", "母亲", "父亲",
|
||
"妈妈", "爸爸", "已婚"),
|
||
"old": ("老年", "老者", "老人", "苍老", "年迈", "奶奶", "外婆", "姥姥", "爷爷", "外公",
|
||
"姥爷", "老太", "退休金", "拄拐"),
|
||
}
|
||
|
||
|
||
def infer_age_band(*texts: str) -> str:
|
||
"""从设定/声线描述里推断年龄档;判不出来返回空串。
|
||
|
||
按「老 → 中 → 青 → 童」顺序取第一个命中的档位:描述文本往往同时出现
|
||
「中年男声,语速很慢」这类混合表述,取跨度最大的一端(更老)比取中间值安全,
|
||
因为「老年声配年轻角色」是最容易被耳朵听出来的错误。
|
||
"""
|
||
blob = " ".join(str(text or "") for text in texts)
|
||
if not blob.strip():
|
||
return ""
|
||
for band in ("old", "middle", "young", "child"):
|
||
if any(marker in blob for marker in _AGE_MARKERS[band]):
|
||
return band
|
||
return ""
|
||
|
||
|
||
def voice_age_band(voice: VoiceAsset | None) -> str:
|
||
if voice is None:
|
||
return ""
|
||
return infer_age_band(voice.name, voice.description, voice.tuning.get("prompt")
|
||
if isinstance(voice.tuning, dict) else "")
|
||
|
||
|
||
def age_band_distance(a: str, b: str) -> int:
|
||
if a not in _AGE_ORDER or b not in _AGE_ORDER:
|
||
return -1
|
||
return abs(_AGE_ORDER[a] - _AGE_ORDER[b])
|
||
|
||
|
||
def current_voice_audio(db: Session, voice: VoiceAsset) -> str:
|
||
"""该声线当前真正可用的音色文件(优先 reference,其次最新的音色样本)。
|
||
|
||
平台声线母版经常只带一段「试听样本」而没有单独的参考音频;IndexTTS2 需要的
|
||
恰好就是一段人声样本,所以两者都能当克隆参考,不能让「没有 reference」= 没有音色。
|
||
"""
|
||
from app.constants import AssetKind
|
||
|
||
rows = [row for row in storage.assets_of(db, "voice", voice.id)
|
||
if row.kind in (AssetKind.REFERENCE, AssetKind.VOICE_SAMPLE)
|
||
and storage.exists(row.path)
|
||
and storage.belongs_to_owner(db, "voice", voice.id, row)]
|
||
if not rows:
|
||
return ""
|
||
ranked = [row for row in rows if row.kind == AssetKind.REFERENCE] or rows
|
||
return str(sorted(ranked, key=lambda row: (row.id, row.current_version or 1))[-1].path)
|
||
|
||
|
||
def repair_voice_reference(db: Session, voice: VoiceAsset | None, *, reason: str = "") -> str:
|
||
"""把声线的克隆参考音频指向它自己当前最新的音色素材,返回变更说明(无变更返回空串)。
|
||
|
||
两个必须自愈的场景:
|
||
1. 用户往声线里补录/新引入了音色素材,但 ``provider_voice_id`` 还停在旧文件上,
|
||
于是「声音明明换了,配音听起来一点没变」;
|
||
2. ``provider_voice_id`` 指的是已被删除/替换的文件,适配器会静默换成通用音色。
|
||
"""
|
||
if voice is None or "comfyui" not in str(voice.provider or ""):
|
||
return ""
|
||
latest = current_voice_audio(db, voice)
|
||
if not latest or latest == str(voice.provider_voice_id or "").strip():
|
||
return ""
|
||
previous = str(voice.provider_voice_id or "").strip()
|
||
voice.provider_voice_id = latest
|
||
label = "音色样本" if latest.endswith((".wav", ".mp3")) and "reference" not in latest else "参考音频"
|
||
return (f"声线「{voice.name}」的克隆{label}已更新为 {latest}"
|
||
+ (f"(原 {previous})" if previous else "(此前没有可用参考音频)")
|
||
+ (f":{reason}" if reason else ""))
|
||
|
||
|
||
def voice_reference_outdated(db: Session, voice: VoiceAsset | None) -> str:
|
||
"""该声线有更新的音色素材、但克隆参考还没跟上时返回那个新文件,否则空串。"""
|
||
if voice is None or "comfyui" not in str(voice.provider or ""):
|
||
return ""
|
||
latest = current_voice_audio(db, voice)
|
||
if not latest or latest == str(voice.provider_voice_id or "").strip():
|
||
return ""
|
||
return latest
|
||
|
||
|
||
def voice_reference_ready(db: Session, voice: VoiceAsset | None) -> bool:
|
||
"""声线是否真的能决定音色。
|
||
|
||
IndexTTS2 这类 zero-shot 声线靠参考音频克隆:``provider_voice_id`` 为空时适配器
|
||
会静默回落到模型配置里的通用参考音频,全片听起来都是同一个陌生人。
|
||
「项目里有声线记录」不等于「这个角色有自己的音色」,这里连文件是否真实存在一起判定。
|
||
"""
|
||
if voice is None:
|
||
return False
|
||
candidate = str(voice.provider_voice_id or "").strip()
|
||
if not candidate:
|
||
return False
|
||
if "comfyui" not in str(voice.provider or ""):
|
||
# 云端音色库用的是供应商音色 ID,不是本地参考音频文件
|
||
return True
|
||
if Path(candidate).suffix.lower() not in AUDIO_SUFFIXES and "/" not in candidate:
|
||
return False
|
||
return storage.exists(candidate)
|
||
|
||
|
||
def infer_gender(*texts: str) -> str:
|
||
"""从人物姓名与设定文本里推断性别;判不出来返回空串。"""
|
||
blob = " ".join(str(text or "") for text in texts)
|
||
if not blob.strip():
|
||
return ""
|
||
scores = {"female": 0.0, "male": 0.0}
|
||
for gender, markers in _GENDER_MARKERS.items():
|
||
scores[gender] += sum(blob.count(marker) for marker in markers)
|
||
scores[gender] += 0.5 * blob.count("女" if gender == "female" else "男")
|
||
# 姓名只作为弱线索参与(权重 0.4/字),文本里一旦有明确的「男性/女性」表述,
|
||
# 关键词得分一定更高,不会被名字翻盘。
|
||
name = str(texts[0] or "") if texts else ""
|
||
scores["female"] += 0.4 * sum(name.count(m) for m in _NAME_FEMALE_MARKERS)
|
||
scores["male"] += 0.4 * sum(name.count(m) for m in _NAME_MALE_MARKERS)
|
||
if scores["female"] == scores["male"]:
|
||
return ""
|
||
return "female" if scores["female"] > scores["male"] else "male"
|
||
|
||
|
||
def character_gender(db: Session, char: Character | None) -> str:
|
||
"""人物性别:结构化字段 → 已绑声线的性别 → 姓名与外观设定文本推断。"""
|
||
if char is None:
|
||
return ""
|
||
gender = str(getattr(char, "gender", "") or "").strip().lower()
|
||
if gender in ("male", "female", "neutral"):
|
||
return gender
|
||
if char.default_voice_id:
|
||
voice = db.get(VoiceAsset, char.default_voice_id)
|
||
voice_gender = str(getattr(voice, "gender", "") or "").strip().lower()
|
||
if voice_gender in ("male", "female", "neutral"):
|
||
return voice_gender
|
||
snapshot = getattr(char, "platform_snapshot_json", None) or {}
|
||
if isinstance(snapshot, dict):
|
||
snap_gender = str(snapshot.get("gender") or "").strip().lower()
|
||
if snap_gender in ("male", "female", "neutral"):
|
||
return snap_gender
|
||
return infer_gender(char.name, char.appearance, char.persona_prompt,
|
||
char.voice_profile, char.personality)
|
||
|
||
|
||
def character_age_band(db: Session, char: Character | None) -> str:
|
||
"""人物年龄档。
|
||
|
||
必须以人物自己的设定为准:绑定声线的描述经常写着「25 岁女性」,看起来很好用,
|
||
但它正是我们要审计的对象——把「老年女声」当成角色的年龄依据,错配就永远查不出来
|
||
(紫小慧 25 岁配老年女声,第一次就是因为这个自检不出问题)。
|
||
人物文本判不出档位时,才退回已绑声线的描述兜底。
|
||
"""
|
||
if char is None:
|
||
return ""
|
||
snapshot_from_char = _character_age_band_from_text(char)
|
||
if snapshot_from_char:
|
||
return snapshot_from_char
|
||
if char.default_voice_id:
|
||
band = voice_age_band(db.get(VoiceAsset, char.default_voice_id))
|
||
if band:
|
||
return band
|
||
return ""
|
||
|
||
|
||
def _character_age_band_from_text(char: Character) -> str:
|
||
snapshot = getattr(char, "platform_snapshot_json", None) or {}
|
||
extra = ""
|
||
if isinstance(snapshot, dict):
|
||
extra = str(snapshot.get("appearance") or "") + " " + str(snapshot.get("role") or "")
|
||
return infer_age_band(char.name, char.appearance, char.persona_prompt,
|
||
char.voice_profile, char.personality, char.role, extra)
|
||
|
||
|
||
def character_age_band_text(char: Character | None) -> str:
|
||
"""只读人物文本判年龄档(不查声线),供自检与用例复用。"""
|
||
return "" if char is None else _character_age_band_from_text(char)
|
||
|
||
|
||
def backfill_gender(db: Session, project: Project) -> list[str]:
|
||
"""把推断出来的性别写回 characters.gender,让「配音为什么选这条声线」可复核。
|
||
|
||
只填空字段,绝不覆盖用户手工设置过的值;返回被补齐的人物名,供事件流说明改了什么。
|
||
"""
|
||
filled: list[str] = []
|
||
for char in db.query(Character).filter(Character.project_id == project.id).all():
|
||
if str(char.gender or "").strip():
|
||
continue
|
||
gender = infer_gender(char.name, char.appearance, char.persona_prompt,
|
||
char.voice_profile, char.personality)
|
||
if gender in ("male", "female"):
|
||
char.gender = gender
|
||
filled.append(f"{char.name}={gender}")
|
||
return filled
|
||
|
||
|
||
def speaker_character(db: Session, shot: Shot, *, speaker_id: int | None = None) -> Character | None:
|
||
"""本镜到底是谁在说话:显式说话角色 → 画面描述里最早点名的人 → 出镜名单第一位。
|
||
|
||
项目 9《自检-异步调度验证》实测:镜头 02 的画面描述是「俊杰坐在服务台后,
|
||
面带轻松微笑,微微抬手示意紫小慧不用担心」,台词「别慌,我们公司用的是紫光汇智
|
||
的云桌面……」明显是俊杰说的。但俊杰是用户**导入脚本之后**才补进项目的人物,
|
||
`speaker_character_id` 一直是空的,旧逻辑直接退回 `shot.characters[0]`(紫小慧),
|
||
于是男主的台词配成了女声,第二次配音起这条女声线还被永久粘在镜头上——
|
||
这就是用户说的「配音接错人」。
|
||
|
||
判定口径:画面描述/梗概里**最早被点名**的项目人物就是这一镜的动作主语。
|
||
这是可解释的启发式,不是硬猜——推断出来的说话人会写回镜头并留一条事件,
|
||
用户在分镜里一眼就能看到、可以直接改。
|
||
"""
|
||
if speaker_id:
|
||
# 逐行配音/逐条字幕:这一行是谁说的由调用方给出,不用再从画面描述里猜。
|
||
char = db.get(Character, int(speaker_id))
|
||
if char is not None:
|
||
return char
|
||
if shot.speaker_character_id:
|
||
char = db.get(Character, shot.speaker_character_id)
|
||
if char is not None:
|
||
return char
|
||
project = shot.project
|
||
script = " ".join(filter(None, [shot.description, shot.synopsis]))
|
||
best: tuple[int, Character] | None = None
|
||
if script.strip() and project is not None:
|
||
for row in list(project.characters or []):
|
||
name = (row.name or "").strip()
|
||
if not name:
|
||
continue
|
||
at = script.find(name)
|
||
if at >= 0 and (best is None or at < best[0]):
|
||
best = (at, row)
|
||
if best is not None:
|
||
return best[1]
|
||
return (shot.characters or [None])[0]
|
||
|
||
|
||
def inferred_speaker(db: Session, shot: Shot) -> Character | None:
|
||
"""说话人是推断出来的(不是用户/脚本明确指定的)时返回该人物,否则 None。"""
|
||
if shot.speaker_character_id:
|
||
return None
|
||
char = speaker_character(db, shot)
|
||
if char is None:
|
||
return None
|
||
if (shot.characters or []) and char.id == int(shot.characters[0].id):
|
||
return None # 与旧口径结论相同,不算新推断
|
||
return char
|
||
|
||
|
||
def speaker_gender(db: Session, shot: Shot, *, speaker_id: int | None = None) -> str:
|
||
"""说话人性别:按 `speaker_character` 的统一口径取人,不再默认出镜第一位。"""
|
||
rows: list[Character] = [char for char in [speaker_character(db, shot, speaker_id=speaker_id)]
|
||
if char is not None]
|
||
rows.extend(shot.characters or [])
|
||
for char in rows:
|
||
gender = character_gender(db, char)
|
||
if gender:
|
||
return gender
|
||
return ""
|
||
|
||
|
||
def speaker_age_band(db: Session, shot: Shot, *, speaker_id: int | None = None) -> str:
|
||
"""说话人年龄档:优先说话角色,其次出镜角色(与 speaker_gender 同口径)。"""
|
||
rows: list[Character] = [char for char in [speaker_character(db, shot, speaker_id=speaker_id)]
|
||
if char is not None]
|
||
rows.extend(shot.characters or [])
|
||
for char in rows:
|
||
band = character_age_band(db, char)
|
||
if band:
|
||
return band
|
||
return ""
|
||
|
||
|
||
def rank_voices(db: Session, project: Project, *, gender: str = "", age_band: str = "",
|
||
speaker_name: str = "", pool: list[VoiceAsset] | None = None) -> list[dict]:
|
||
"""给项目里的声线按「贴不贴合这个说话人」打分,从高到低返回。
|
||
|
||
旧实现只有一条规则:「同性别 + 有参考音频」里挑第一条(按 id 排序),
|
||
于是 25 岁的紫小慧拿到了唯一一条带参考音频的「偏低的中国老年女声」。
|
||
现在按性别 > 年龄档 > 姓名归属 > 参考音频可用性 > 是否已被别人占用打分,
|
||
并把每一项理由带回去,配音失败时能直接告诉用户为什么。
|
||
"""
|
||
rows = pool if pool is not None else db.query(VoiceAsset).filter(
|
||
VoiceAsset.project_id == project.id).all()
|
||
owners: dict[int, str] = {}
|
||
for char in db.query(Character).filter(Character.project_id == project.id).all():
|
||
if char.default_voice_id:
|
||
owners.setdefault(int(char.default_voice_id), char.name)
|
||
scored: list[dict] = []
|
||
for row in rows:
|
||
ready = voice_reference_ready(db, row)
|
||
v_gender = str(row.gender or "").strip().lower()
|
||
v_band = voice_age_band(row)
|
||
score = 0.0
|
||
notes: list[str] = []
|
||
if gender:
|
||
if v_gender == gender:
|
||
score += 100
|
||
elif v_gender in ("", "neutral"):
|
||
score += 15
|
||
notes.append("声线未标注性别")
|
||
else:
|
||
score -= 120
|
||
notes.append("性别不符")
|
||
distance = age_band_distance(age_band, v_band)
|
||
if distance == 0:
|
||
score += 60
|
||
elif distance > 0:
|
||
score += {1: 10, 2: -25}.get(distance, -45)
|
||
notes.append(f"年龄档不符(说话人{AGE_BAND_LABELS.get(age_band, age_band)}"
|
||
f"/声线{AGE_BAND_LABELS.get(v_band, v_band)})")
|
||
name = str(row.name or "")
|
||
if speaker_name and speaker_name in name:
|
||
score += 45
|
||
notes.append("与说话人同名")
|
||
score += 30 if ready else -20
|
||
if not ready:
|
||
notes.append("缺少可用参考音频")
|
||
holder = owners.get(row.id)
|
||
if holder and speaker_name and holder != speaker_name:
|
||
score -= 25
|
||
notes.append(f"已是「{holder}」的默认声线")
|
||
if row.is_default:
|
||
score += 5
|
||
scored.append({"voice": row, "score": round(score, 1), "ready": ready,
|
||
"gender": v_gender, "age_band": v_band, "notes": notes})
|
||
scored.sort(key=lambda item: (-item["score"], item["voice"].id))
|
||
return scored
|
||
|
||
|
||
def resolve_voice(db: Session, project: Project, shot: Shot, *, speaker_id: int | None = None,
|
||
per_line: bool = False) -> tuple[VoiceAsset | None, dict]:
|
||
"""台词声线优先级,并说明「这是精确匹配还是回落来的」。
|
||
|
||
``per_line=True`` 表示按 ``shot.lines`` 里的某一行取声线:此时**不认镜头级
|
||
``voice_asset_id``**——那是单说话人时代留下的字段,多角色对白里沿用它
|
||
会让一整段话从头到尾只有一个嗓子。
|
||
|
||
优先级:镜头显式指定 → 说话角色默认声线 → 按说话人性别匹配 → 出镜角色默认 →
|
||
项目默认 → 项目第一个声线。
|
||
|
||
旧实现在前三档都落空时静默返回项目第一个声线,``generate_voiceover`` 还把它
|
||
固化写回镜头,于是「男性工程师」从第二次配音起永久使用紫小慧的女声,
|
||
而且没有任何地方说明过原因。现在把匹配过程一起返回:回落要告警,
|
||
只有真正有依据的匹配才允许固化到镜头上。
|
||
"""
|
||
gender = speaker_gender(db, shot, speaker_id=speaker_id)
|
||
age_band = speaker_age_band(db, shot, speaker_id=speaker_id)
|
||
speaker = speaker_character(db, shot, speaker_id=speaker_id)
|
||
speaker_name = str(speaker.name or "") if speaker is not None else ""
|
||
info = {"matched": False, "exact": False, "reason": "", "gender": gender,
|
||
"voice": "", "voice_gender": "", "reference_ready": False,
|
||
"gender_mismatch": False, "age_band": age_band, "voice_age_band": "",
|
||
"age_mismatch": False, "alternatives": []}
|
||
voice: VoiceAsset | None = None
|
||
if shot.voice_asset_id and not per_line:
|
||
voice = db.get(VoiceAsset, shot.voice_asset_id)
|
||
if voice is not None:
|
||
info.update(matched=True, exact=True, reason="镜头已指定声线")
|
||
if voice is None and per_line and speaker_id:
|
||
# 逐行:这一行的角色自己的默认声线优先,和「镜头说话人」是两套归属
|
||
char = db.get(Character, int(speaker_id))
|
||
if char is not None and char.default_voice_id:
|
||
candidate = db.get(VoiceAsset, char.default_voice_id)
|
||
if candidate is not None:
|
||
voice = candidate
|
||
info.update(matched=True, exact=True, reason=f"「{char.name}」的默认声线")
|
||
if voice is None and not per_line and shot.speaker_character_id:
|
||
char = db.get(Character, shot.speaker_character_id)
|
||
if char is not None and char.default_voice_id:
|
||
voice = db.get(VoiceAsset, char.default_voice_id)
|
||
if voice is not None:
|
||
info.update(matched=True, exact=True, reason=f"说话人「{char.name}」的默认声线")
|
||
if voice is None and (gender or age_band or speaker_name):
|
||
# 同性别匹配放在「出镜角色默认」之前:对白镜头里坐着一男一女时,
|
||
# 出镜顺序的第一位常常不是说话的人。
|
||
ranked = rank_voices(db, project, gender=gender, age_band=age_band,
|
||
speaker_name=speaker_name)
|
||
info["alternatives"] = [{"voice": item["voice"].name, "score": item["score"],
|
||
"notes": item["notes"]} for item in ranked[:3]]
|
||
# 负分意味着「只有性别相反 / 年龄差两档以上的声线可用」,宁可不配
|
||
# 也不要配出一条明显不对的声线——那会被当成成品缺陷退回。
|
||
best = ranked[0] if ranked and ranked[0]["score"] > 0 else None
|
||
if best is not None:
|
||
voice = best["voice"]
|
||
info.update(matched=True, reference_ready=best["ready"],
|
||
age_mismatch=bool(age_band_distance(age_band, best["age_band"]) >= 2),
|
||
reason=f"按说话人{GENDER_LABELS.get(gender, gender)}"
|
||
f"{AGE_BAND_LABELS.get(age_band, '')}匹配到「{voice.name}」"
|
||
+ (f"({ ';'.join(best['notes']) })" if best["notes"] else ""))
|
||
if voice is None and per_line and not speaker_name:
|
||
info["reason"] = "这一行台词没标说话人,按出镜人物与性别兜底匹配"
|
||
if voice is None:
|
||
for char in shot.characters or []:
|
||
if not char.default_voice_id:
|
||
continue
|
||
candidate = db.get(VoiceAsset, char.default_voice_id)
|
||
if candidate is not None:
|
||
voice = candidate
|
||
info.update(matched=True, exact=True,
|
||
reason=f"出镜人物「{char.name}」的默认声线")
|
||
break
|
||
if voice is None:
|
||
pool = db.query(VoiceAsset).filter(VoiceAsset.project_id == project.id) \
|
||
.order_by(VoiceAsset.id).all()
|
||
defaults = [row for row in pool if row.is_default]
|
||
ready = [row for row in pool if voice_reference_ready(db, row)]
|
||
picked = defaults or ready or pool
|
||
if picked:
|
||
voice = picked[0]
|
||
info["reason"] = ("项目默认声线" if defaults else
|
||
f"项目里没有同性别可用声线,暂用「{voice.name}」")
|
||
if voice is None:
|
||
return None, info
|
||
info["voice"] = voice.name
|
||
info["voice_gender"] = str(voice.gender or "")
|
||
info["voice_age_band"] = voice_age_band(voice)
|
||
info["reference_ready"] = voice_reference_ready(db, voice)
|
||
info["gender_mismatch"] = bool(gender and info["voice_gender"]
|
||
and gender != info["voice_gender"])
|
||
info["age_mismatch"] = info["age_mismatch"] or age_band_distance(
|
||
info.get("age_band") or "", info["voice_age_band"]) >= 2
|
||
return voice, info
|
||
|
||
|
||
def voice_selection_hint(info: dict) -> str:
|
||
"""把匹配过程翻译成一句能直接进事件流/预检的中文说明。"""
|
||
bits: list[str] = []
|
||
if info.get("gender_mismatch"):
|
||
bits.append("说话人是%s,声线「%s」却是%s" % (
|
||
GENDER_LABELS.get(info.get("gender") or "", info.get("gender") or "?"),
|
||
info.get("voice") or "",
|
||
GENDER_LABELS.get(info.get("voice_gender") or "", info.get("voice_gender") or "?")))
|
||
if info.get("age_mismatch"):
|
||
bits.append("说话人是%s,声线「%s」却是%s" % (
|
||
AGE_BAND_LABELS.get(info.get("age_band") or "", info.get("age_band") or "?"),
|
||
info.get("voice") or "",
|
||
AGE_BAND_LABELS.get(info.get("voice_age_band") or "", info.get("voice_age_band") or "?")))
|
||
if info.get("reference_ready") is False:
|
||
bits.append("声线「%s」没有参考音频,音色由模型默认参考决定"
|
||
"(不是这个角色自己的音色)" % (info.get("voice") or ""))
|
||
if not info.get("exact") and info.get("reason"):
|
||
bits.append(str(info["reason"]))
|
||
return ";".join(bits)
|
||
|
||
|
||
def resolve_voice_asset(db: Session, project: Project, shot: Shot) -> VoiceAsset | None:
|
||
"""兼容旧调用:只取声线本体,不关心匹配过程。"""
|
||
return resolve_voice(db, project, shot)[0]
|
||
|
||
|
||
def resolve_tts_spec(db: Session, voice: VoiceAsset | None):
|
||
"""按声线资产选择 TTS 模型;找不到对应 ModelSpec 时退回项目默认配音模型。"""
|
||
candidates = [voice.provider] if voice and voice.provider else []
|
||
candidates += [settings.tts_provider, None]
|
||
for provider in candidates:
|
||
try:
|
||
spec, adapter = model_registry.resolve_voice(db, provider, None)
|
||
if adapter is not None and adapter.is_available(model_registry.runtime_config(db, spec)):
|
||
return spec, adapter
|
||
except Exception:
|
||
continue
|
||
return model_registry.resolve_voice(db, None, None)
|
||
|
||
|
||
def estimate_duration_ms(text: str, *, speed: float = 1.0) -> int:
|
||
text = (text or "").strip()
|
||
if not text:
|
||
return 0
|
||
cjk = sum(1 for ch in text if "\u4e00" <= ch <= "\u9fff")
|
||
latin_words = len(re.findall(r"[A-Za-z0-9']+", text))
|
||
pauses = len(re.findall(r"[,。!?;、,.!?;\n]", text))
|
||
seconds = cjk / CPS_CJK + latin_words / WPS_LATIN + pauses * PAUSE_SECONDS * 0.5
|
||
return int(round(max(0.6, seconds) / max(0.5, min(float(speed or 1.0), 2.0)) * 1000))
|
||
|
||
|
||
def line_voice_speed(db: Session, project: Project, shot: Shot, line: dict) -> float:
|
||
"""某一行的语速:取该行说话人的声线调参(多角色时每人语速可以不同)。"""
|
||
voice, _info = resolve_voice(db, project, shot, speaker_id=line.get("character_id"), per_line=True)
|
||
return float((voice.tuning or {}).get("speed", 1.0) if voice else 1.0)
|
||
|
||
|
||
def shot_voice_duration_ms(db: Session, project: Project, shot: Shot) -> int:
|
||
"""镜头配音时长:优先真实产物时长,其次按文本估算(用于时间线预排)。
|
||
|
||
多行台词要逐行估算再相加,并补上换人之间的静音间隙——
|
||
按整段字数估会短一截,片段被排得太紧,配音尾巴就被切掉。
|
||
"""
|
||
if shot.voiceover_duration_ms and storage.exists(shot.voiceover_path):
|
||
return int(shot.voiceover_duration_ms)
|
||
lines = shot.all_lines()
|
||
if not lines:
|
||
return 0
|
||
if len(lines) == 1:
|
||
text = (lines[0]["text"] or "").strip()
|
||
voice = resolve_voice_asset(db, project, shot)
|
||
speed = float((voice.tuning or {}).get("speed", 1.0) if voice else 1.0)
|
||
return estimate_duration_ms(text, speed=speed)
|
||
total = sum(estimate_duration_ms(str(row.get("text") or ""),
|
||
speed=line_voice_speed(db, project, shot, row))
|
||
for row in lines)
|
||
return int(total + LINE_GAP_MS * (len(lines) - 1))
|
||
|
||
|
||
def _tts_produce(db: Session, project: Project, shot: Shot, text: str, voice: VoiceAsset | None,
|
||
*, out: Path, index: int = 0) -> tuple:
|
||
"""一次 TTS 调用:按声线选模型、合成到 ``out``,返回 (spec, result, produced, duration_ms)。"""
|
||
spec, _adapter = resolve_tts_spec(db, voice)
|
||
tuning = dict(voice.tuning or {}) if voice else {}
|
||
request = VoiceGenRequest(
|
||
text=text, voice_id=voice.name if voice else "",
|
||
provider_voice_id=voice.provider_voice_id if voice else "",
|
||
language=(voice.language if voice else "zh-CN"), speed=float(tuning.get("speed", 1.0)),
|
||
pitch=float(tuning.get("pitch", 1.0)), volume=float(tuning.get("volume", 1.0)),
|
||
sample_rate=int(tuning.get("sample_rate", 24000)), seed=(int(shot.seed or 0) + index),
|
||
params={**(tuning.get("params") or {}), "shot_code": shot.code},
|
||
idempotency_key=f"voice-{shot.id}-{index}", label=f"SHOT-{shot.order_index:02d}-{index + 1:02d}",
|
||
output_path=out)
|
||
result = model_registry.generate_voice(db, request, spec.provider, spec.model)
|
||
produced = Path(result.file_path)
|
||
duration_ms = int(result.duration_ms or ff.duration_ms(produced) or estimate_duration_ms(text))
|
||
return spec, result, produced, duration_ms
|
||
|
||
|
||
def concat_voice_parts(parts: list[Path], out: Path, *, gap_ms: int = LINE_GAP_MS) -> Path:
|
||
"""把逐行配音按顺序拼成一条音轨,行间插一小段静音。
|
||
|
||
一句台词一个嗓子、逐行合成是多人对白的唯一可靠做法:整段丢给 TTS,
|
||
两个人只会共用第一个人的音色。换人处留 220ms 呼吸,否则听起来像同一个人绕口令。
|
||
"""
|
||
parts = [Path(p) for p in parts if p is not None]
|
||
if not parts:
|
||
raise ValueError("没有可拼接的配音片段")
|
||
if len(parts) == 1:
|
||
return parts[0]
|
||
inputs: list[str] = []
|
||
labels: list[str] = []
|
||
for pos, part in enumerate(parts):
|
||
if pos and gap_ms > 0:
|
||
inputs += ["-f", "lavfi", "-t", f"{gap_ms / 1000:.3f}", "-i",
|
||
"anullsrc=channel_layout=stereo:sample_rate=48000"]
|
||
labels.append("gap%02d" % pos)
|
||
inputs += ["-i", str(part)]
|
||
labels.append("seg%02d" % pos)
|
||
chain: list[str] = []
|
||
for idx, label in enumerate(labels):
|
||
# 每段统一 48k/立体声:不同声线的采样率、声道数不一致时 concat 会直接报错
|
||
tail = (f"atrim=0:{gap_ms / 1000:.3f},asetpts=PTS-STARTPTS," if label.startswith("gap") else "")
|
||
chain.append(f"[{idx}:a]{tail}aresample=48000,"
|
||
f"aformat=sample_fmts=s16:channel_layouts=stereo[{label}]")
|
||
concat_in = "".join(f"[{label}]" for label in labels)
|
||
chain.append(f"{concat_in}concat=n={len(labels)}:v=0:a=1[out]")
|
||
cmd = [ff.FFMPEG, "-hide_banner", "-loglevel", "error", "-y", *inputs, "-filter_complex",
|
||
";".join(chain), "-map", "[out]", "-ar", "48000", "-ac", "2", "-c:a", "pcm_s16le", str(out)]
|
||
ff.run(cmd, timeout=300)
|
||
if not out.exists():
|
||
raise ValueError("配音拼接失败(ffmpeg 未产出文件)")
|
||
return out
|
||
|
||
|
||
def generate_voiceover(db: Session, project: Project, shot: Shot, *, text: str | None = None,
|
||
voice: VoiceAsset | None = None, task_id: int | None = None,
|
||
commit: bool = True) -> dict:
|
||
"""调用 TTS 生成配音,登记为镜头资产并回写时长。
|
||
|
||
一镜多人对白:``shot.lines`` 有多行时逐行取各自说话人的声线分别合成再拼接。
|
||
调用方显式给了 ``text`` 或 ``voice`` 时按「整段一个嗓子」处理(手工试听/改写场景)。
|
||
"""
|
||
_lines = shot.all_lines()
|
||
if text is None and voice is None and len(_lines) > 1:
|
||
return generate_voiceover_lines(db, project, shot, _lines, task_id=task_id, commit=commit)
|
||
text = (text if text is not None else (shot.dialogue or "")).strip()
|
||
if not text:
|
||
raise ValueError("镜头没有台词,无法配音")
|
||
match_info: dict = {"matched": bool(voice), "exact": bool(voice),
|
||
"reason": "调用方直接指定声线", "voice": voice.name if voice else "",
|
||
"voice_gender": str(voice.gender or "") if voice else "", "gender": "",
|
||
"reference_ready": voice_reference_ready(db, voice),
|
||
"gender_mismatch": False}
|
||
if voice is None:
|
||
voice, match_info = resolve_voice(db, project, shot)
|
||
guess = inferred_speaker(db, shot)
|
||
if guess is not None and voice is not None:
|
||
# 台词没挂说话人时按画面描述认出来的人,要写回镜头并在事件里说明:
|
||
# 否则用户只看到「俊杰的台词变成了紫小慧的嗓子」,却不知道系统认的是谁。
|
||
shot.speaker_character_id = guess.id
|
||
if all(row.id != guess.id for row in (shot.characters or [])):
|
||
shot.characters = list(shot.characters or []) + [guess]
|
||
match_info["inferred_speaker"] = guess.name
|
||
events.log_event(
|
||
db, "voiceover.speaker_inferred", project_id=project.id, shot_id=shot.id,
|
||
task_id=task_id, level="info",
|
||
message=f"镜头 {shot.order_index:02d} 台词原本没挂说话人,按画面描述认定说话人为"
|
||
f"「{guess.name}」,已用其声线「{voice.name}」配音(分镜里可改)",
|
||
entity_type="shot", entity_id=shot.id,
|
||
data={"speaker": guess.name, "speaker_id": guess.id, "voice": voice.name},
|
||
)
|
||
# 先自愈再合成:用户新加的音色素材必须立刻体现在配音里,否则「换了声音」这件事
|
||
# 只存在于素材列表里,听起来还是旧音色。
|
||
repaired = repair_voice_reference(db, voice, reason=f"镜头 {shot.order_index:02d} 配音前自动对齐音色素材")
|
||
if repaired:
|
||
match_info["reference_ready"] = voice_reference_ready(db, voice)
|
||
events.log_event(db, "voice.reference_refreshed", project_id=project.id, shot_id=shot.id,
|
||
message=repaired, entity_type="voice",
|
||
entity_id=getattr(voice, "id", None), data={"reason": "voiceover"})
|
||
folder = storage.project_dir(project.id, "voiceover")
|
||
stem = f"shot-{shot.order_index:02d}"
|
||
out = folder / f"{stem}_{abs(hash((text, voice.provider_voice_id if voice else '', shot.voice_asset_id))) % 10**10}.wav"
|
||
|
||
spec, result, produced, duration_ms = _tts_produce(db, project, shot, text, voice, out=out)
|
||
rel = storage.to_relative(produced)
|
||
size = produced.stat().st_size if produced.exists() else 0
|
||
info = {"path": rel, "size_bytes": size, "duration_ms": duration_ms,
|
||
"original_name": produced.name, "mime": "audio/wav"}
|
||
storage.register_asset(db, project_id=project.id, owner_type="shot", owner_id=shot.id, kind="voiceover",
|
||
path=rel, name=f"镜头 {shot.order_index:02d} 配音", source="provider", info=info,
|
||
metadata={"provider": spec.provider, "model": spec.model,
|
||
"voice": voice.name if voice else "", "text": text})
|
||
shot.voiceover_path = rel
|
||
shot.voiceover_duration_ms = duration_ms
|
||
# 只有「说话人自己的声线」才固化到镜头:旧实现连静默回落来的声线也写死,
|
||
# 男配音一旦被粘上女声,之后每次重配都还是女声。
|
||
if not shot.voice_asset_id and voice is not None and match_info.get("exact"):
|
||
shot.voice_asset_id = voice.id
|
||
|
||
cost = float(result.cost or costing.estimate_voice_cost(spec, len(text)))
|
||
if cost:
|
||
costing.record_cost(db, project_id=project.id, kind="voiceover", provider=spec.provider,
|
||
model=spec.model, shot_id=shot.id, task_id=task_id, quantity=len(text),
|
||
unit="1k_chars", unit_price=float(getattr(spec, "price_per_1k_chars", 0.0) or 0.0),
|
||
cost=cost, currency=spec.currency, note=f"镜头 {shot.order_index:02d} 配音")
|
||
costing.record_generation(db, project_id=project.id, shot_id=shot.id, task_id=task_id,
|
||
provider=spec.provider, model=spec.model, kind="voiceover", prompt=text,
|
||
params_json={"voice": voice.name if voice else "",
|
||
"duration_ms": duration_ms,
|
||
"voice_match": {k: match_info.get(k) for k in
|
||
("exact", "matched", "reason", "gender",
|
||
"voice_gender", "reference_ready",
|
||
"gender_mismatch", "age_band",
|
||
"voice_age_band", "age_mismatch")},
|
||
"reference_source": (result.metadata or {}).get(
|
||
"reference_source", "")},
|
||
provider_job_id=result.provider_job_id, output_path=rel,
|
||
duration_seconds=round(duration_ms / 1000.0, 3), cost=cost,
|
||
currency=spec.currency,
|
||
metadata_json={"mock": bool((result.metadata or {}).get("mock"))})
|
||
events.log_event(db, "voiceover.generated", project_id=project.id, shot_id=shot.id, task_id=task_id,
|
||
message=f"镜头 {shot.order_index:02d} 配音完成({duration_ms / 1000:.1f}s,"
|
||
f"声线 {voice.name if voice else spec.model})",
|
||
entity_type="shot", entity_id=shot.id,
|
||
data={"path": rel, "duration_ms": duration_ms, "chars": len(text),
|
||
"voice_match": match_info,
|
||
"reference_source": (result.metadata or {}).get("reference_source", "")})
|
||
warning = voice_selection_hint(match_info)
|
||
if warning:
|
||
# 「配音听起来怪」过去只能靠人反复试听发现:性别错了、或者音色其实是模型默认的,
|
||
# 系统一句话都不说。现在把原因写进事件流,用户在总控台就能看到该换哪条声线。
|
||
events.log_event(
|
||
db, "voiceover.voice_fallback", project_id=project.id, shot_id=shot.id, task_id=task_id,
|
||
level="warning",
|
||
message=f"镜头 {shot.order_index:02d} 配音声线待修正:{warning}",
|
||
entity_type="shot", entity_id=shot.id,
|
||
data={"voice": voice.name if voice else "", "match": match_info,
|
||
"reference_source": (result.metadata or {}).get("reference_source", "")},
|
||
)
|
||
if commit:
|
||
db.commit()
|
||
else:
|
||
db.flush()
|
||
return {"path": rel, "url": storage.url_for(rel), "duration_ms": duration_ms,
|
||
"voice": voice.name if voice else spec.model, "provider": spec.provider, "model": spec.model,
|
||
"chars": len(text), "cost": round(cost, 4), "estimated": False}
|
||
|
||
|
||
def generate_voiceover_lines(db: Session, project: Project, shot: Shot, lines: list[dict], *,
|
||
task_id: int | None = None, commit: bool = True) -> dict:
|
||
"""多角色对白:逐行按说话人的声线合成,再拼成镜头的一条配音音轨。"""
|
||
folder = storage.project_dir(project.id, "voiceover")
|
||
stem = f"shot-{shot.order_index:02d}"
|
||
parts: list[Path] = []
|
||
rows: list[dict] = []
|
||
total_cost = 0.0
|
||
provider = model = ""
|
||
for index, line in enumerate(lines):
|
||
text = str(line.get("text") or "").strip()
|
||
if not text:
|
||
continue
|
||
speaker_id = line.get("character_id")
|
||
voice, info = resolve_voice(db, project, shot, speaker_id=speaker_id, per_line=True)
|
||
repaired = repair_voice_reference(
|
||
db, voice, reason=f"镜头 {shot.order_index:02d} 第 {len(rows) + 1} 句配音前自动对齐音色素材")
|
||
if repaired:
|
||
events.log_event(db, "voice.reference_refreshed", project_id=project.id, shot_id=shot.id,
|
||
message=repaired, entity_type="voice",
|
||
entity_id=getattr(voice, "id", None), data={"reason": "voiceover"})
|
||
out = folder / f"{stem}_l{len(rows) + 1:02d}_" \
|
||
f"{abs(hash((text, voice.provider_voice_id if voice else '', speaker_id))) % 10 ** 10}.wav"
|
||
spec, result, produced, duration_ms = _tts_produce(db, project, shot, text, voice,
|
||
out=out, index=len(rows))
|
||
parts.append(produced)
|
||
provider, model = spec.provider, spec.model
|
||
cost = float(result.cost or costing.estimate_voice_cost(spec, len(text)))
|
||
total_cost += cost
|
||
speaker = shot.line_speaker_name(line) or "未标注说话人"
|
||
rows.append({"index": len(rows), "speaker": speaker, "character_id": speaker_id, "text": text,
|
||
"voice": voice.name if voice else spec.model, "voice_id": getattr(voice, "id", None),
|
||
"duration_ms": duration_ms, "cost": cost, "voice_match": info,
|
||
"path": storage.to_relative(produced)})
|
||
if cost:
|
||
costing.record_cost(db, project_id=project.id, kind="voiceover", provider=spec.provider,
|
||
model=spec.model, shot_id=shot.id, task_id=task_id, quantity=len(text),
|
||
unit="1k_chars",
|
||
unit_price=float(getattr(spec, "price_per_1k_chars", 0.0) or 0.0),
|
||
cost=cost, currency=spec.currency,
|
||
note=f"镜头 {shot.order_index:02d} 第 {len(rows)} 句配音({speaker})")
|
||
warning = voice_selection_hint(info)
|
||
if warning:
|
||
events.log_event(
|
||
db, "voiceover.voice_fallback", project_id=project.id, shot_id=shot.id, task_id=task_id,
|
||
level="warning",
|
||
message=f"镜头 {shot.order_index:02d} 第 {len(rows)} 句({speaker})声线待修正:{warning}",
|
||
entity_type="shot", entity_id=shot.id,
|
||
data={"voice": voice.name if voice else "", "speaker": speaker, "match": info})
|
||
if not parts:
|
||
raise ValueError("镜头没有台词,无法配音")
|
||
|
||
joined = folder / f"{stem}_mix_{abs(hash(tuple(str(p) for p in parts))) % 10 ** 10}.wav"
|
||
produced = concat_voice_parts(parts, joined) if len(parts) > 1 else parts[0]
|
||
rel = storage.to_relative(produced)
|
||
duration_ms = ff.duration_ms(produced) or sum(int(row["duration_ms"]) for row in rows)
|
||
info_row = {"path": rel, "size_bytes": produced.stat().st_size if produced.exists() else 0,
|
||
"duration_ms": duration_ms, "original_name": produced.name, "mime": "audio/wav"}
|
||
storage.register_asset(db, project_id=project.id, owner_type="shot", owner_id=shot.id,
|
||
kind="voiceover", path=rel, name=f"镜头 {shot.order_index:02d} 配音(多角色)",
|
||
source="provider", info=info_row,
|
||
metadata={"provider": provider, "model": model, "text": shot.dialogue,
|
||
"lines": [{"speaker": row["speaker"], "voice": row["voice"],
|
||
"text": row["text"], "duration_ms": row["duration_ms"]}
|
||
for row in rows]})
|
||
shot.voiceover_path = rel
|
||
shot.voiceover_duration_ms = duration_ms
|
||
# 多角色镜头不固化 shot.voice_asset_id:那是「整镜一个嗓子」时代的字段,
|
||
# 粘上第一个人的声线会让以后每次重配都变成他一个人说话。
|
||
costing.record_generation(db, project_id=project.id, shot_id=shot.id, task_id=task_id,
|
||
provider=provider, model=model, kind="voiceover", prompt=shot.dialogue,
|
||
params_json={"mode": "per_line", "lines": [
|
||
{"speaker": row["speaker"], "voice": row["voice"],
|
||
"chars": len(row["text"]), "duration_ms": row["duration_ms"],
|
||
"exact": bool(row["voice_match"].get("exact")),
|
||
"reason": row["voice_match"].get("reason", "")} for row in rows],
|
||
"duration_ms": duration_ms, "gap_ms": LINE_GAP_MS},
|
||
output_path=rel, duration_seconds=round(duration_ms / 1000.0, 3),
|
||
cost=total_cost, currency="CNY")
|
||
events.log_event(db, "voiceover.generated", project_id=project.id, shot_id=shot.id, task_id=task_id,
|
||
message=f"镜头 {shot.order_index:02d} 配音完成({duration_ms / 1000:.1f}s,"
|
||
f"{len(rows)} 句:{'、'.join(row['speaker'] for row in rows)})",
|
||
entity_type="shot", entity_id=shot.id,
|
||
data={"path": rel, "duration_ms": duration_ms, "chars": len(shot.dialogue),
|
||
"lines": [{"speaker": row["speaker"], "voice": row["voice"],
|
||
"text": row["text"]} for row in rows]})
|
||
if commit:
|
||
db.commit()
|
||
else:
|
||
db.flush()
|
||
return {"path": rel, "url": storage.url_for(rel), "duration_ms": duration_ms,
|
||
"voice": " / ".join(row["voice"] for row in rows), "provider": provider, "model": model,
|
||
"chars": len(shot.dialogue), "cost": round(total_cost, 4), "estimated": False,
|
||
"lines": [{"speaker": row["speaker"], "voice": row["voice"], "text": row["text"],
|
||
"duration_ms": row["duration_ms"]} for row in rows]}
|
||
|
||
|
||
def plan_voiceovers(db: Session, project: Project, shots: list[Shot] | None = None) -> list[dict]:
|
||
"""为时间线预排提供配音时长计划(含缺失配音的镜头)。"""
|
||
rows = []
|
||
for shot in (shots if shots is not None else list(project.shots or [])):
|
||
lines = shot.all_lines()
|
||
if not lines:
|
||
continue
|
||
voice = resolve_voice_asset(db, project, shot)
|
||
has = storage.exists(shot.voiceover_path)
|
||
rows.append({"shot_id": shot.id, "order_index": shot.order_index,
|
||
"chars": sum(len(str(row["text"])) for row in lines),
|
||
"voice": voice.name if voice else "未绑定声线",
|
||
"speakers": [shot.line_speaker_name(row) or "未标注说话人" for row in lines],
|
||
"lines": [{"speaker": shot.line_speaker_name(row) or "未标注说话人",
|
||
"text": row["text"]} for row in lines],
|
||
"duration_ms": shot_voice_duration_ms(db, project, shot),
|
||
"ready": has, "path": shot.voiceover_path})
|
||
return rows
|
||
|
||
|
||
def split_text(text: str, *, max_chars: int = 22) -> list[str]:
|
||
"""按标点与长度切分字幕句,保证字幕与语音节奏一致。"""
|
||
text = re.sub(r"\s+", " ", (text or "").strip())
|
||
if not text:
|
||
return []
|
||
chunks: list[str] = []
|
||
for sentence in re.split(r"(?<=[。!?!?;;])|(?<=\n)", text):
|
||
sentence = sentence.strip()
|
||
if not sentence:
|
||
continue
|
||
if len(sentence) <= max_chars:
|
||
chunks.append(sentence)
|
||
continue
|
||
buf = ""
|
||
for part in re.split(r"(?<=[,,、::])", sentence):
|
||
if not part:
|
||
continue
|
||
if len(buf) + len(part) <= max_chars:
|
||
buf += part
|
||
else:
|
||
if buf:
|
||
chunks.append(buf)
|
||
while len(part) > max_chars:
|
||
chunks.append(part[:max_chars])
|
||
part = part[max_chars:]
|
||
buf = part
|
||
if buf:
|
||
chunks.append(buf)
|
||
merged: list[str] = []
|
||
for chunk in chunks:
|
||
if merged and len(merged[-1]) < 6:
|
||
merged[-1] += chunk
|
||
else:
|
||
merged.append(chunk)
|
||
return merged
|
||
|
||
|
||
def cue_plan(db: Session, project: Project, shot: Shot, *, window_ms: int, start_ms: int,
|
||
lead_ms: int = 160, voice_ms: int | None = None) -> list[dict]:
|
||
"""把一个镜头的字幕对齐到给定的时间窗(语音时长优先,均分兜底)。
|
||
|
||
``voice_ms=0`` 表示「没有可参照的 TTS 时长」(例如台词直接用模型原声),
|
||
此时字幕按整个片段窗口铺排,而不是按一段并不存在的配音对齐。
|
||
"""
|
||
text = (shot.effective_subtitle or "").strip()
|
||
if not text or window_ms <= 200:
|
||
return []
|
||
# 逐条台词时字幕要标出「这句是谁说的」;单说话人沿用镜头上的口径。
|
||
chunks: list[str] = []
|
||
speakers: list[str] = []
|
||
lines = shot.all_lines()
|
||
if len(lines) > 1:
|
||
for line in lines:
|
||
name = shot.line_speaker_name(line)
|
||
for chunk in split_text(str(line.get("text") or "")):
|
||
chunks.append(chunk)
|
||
speakers.append(name)
|
||
if not chunks:
|
||
chunks = split_text(text)
|
||
default_speaker = ""
|
||
if shot.speaker:
|
||
default_speaker = shot.speaker.name
|
||
elif shot.characters:
|
||
default_speaker = shot.characters[0].name
|
||
speakers = [default_speaker] * len(chunks)
|
||
if not chunks:
|
||
return []
|
||
if voice_ms is None:
|
||
voice_ms = (min(int(shot.voiceover_duration_ms or 0), window_ms - lead_ms)
|
||
if shot.voiceover_path else 0)
|
||
voice_ms = min(int(voice_ms), window_ms - lead_ms)
|
||
usable = max(400, (voice_ms or window_ms - lead_ms))
|
||
start = start_ms + lead_ms
|
||
weights = [max(4, len(c)) for c in chunks]
|
||
total_w = sum(weights)
|
||
cues: list[dict] = []
|
||
cursor = start
|
||
for i, (chunk, weight) in enumerate(zip(chunks, weights)):
|
||
dur = max(700, int(usable * weight / total_w))
|
||
end = min(start_ms + window_ms, cursor + dur)
|
||
if end - cursor < 300:
|
||
break
|
||
cues.append({"start_ms": int(cursor), "end_ms": int(end), "text": chunk,
|
||
"speaker": speakers[i] if i < len(speakers) else "", "shot_id": shot.id})
|
||
cursor = end + 60
|
||
return cues
|