nex_animation/backend/app/services/audio.py

945 lines
50 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""配音(TTS)与台词时间轴:把台词变成可对齐的音频与字幕。
业务流程中的「台词配音」在这里落到统一任务编排(TaskKind.VOICEOVER),
语音时长会回写镜头,供时间线决定片段长度与字幕入出点。
"""
from __future__ import annotations
import re
from pathlib import Path
from sqlalchemy.orm import Session
from app.config import settings
from app.models import Character, Project, Shot, VoiceAsset
from app.providers import registry as model_registry
from app.providers.base import VoiceGenRequest
from app.services import costing, events, storage, timbre
from app.services import ffmpeg as ff
# 多角色对白逐行合成时,换人之间留的静音(毫秒)
LINE_GAP_MS = 220
# 中文约每秒 4.3 字,英文约每秒 2.6 词,句读额外停顿
CPS_CJK = 4.3
WPS_LATIN = 2.6
PAUSE_SECONDS = 0.28
GENDER_LABELS = {"male": "男声", "female": "女声", "neutral": "中性声"}
AUDIO_SUFFIXES = {".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".opus"}
# 性别关键词:中文文本里性别往往只写在名字或设定里(「男性工程师」「25 岁女性」)。
# 双字/多字词整权重 1,单字「男」「女」只算 0.5,避免「男朋友」「女朋友」互相抵消后
# 反而判成无性别。
_GENDER_MARKERS = {
"female": ("女性", "女子", "女士", "小姐", "姑娘", "女孩", "女生", "女主", "母亲", "妈妈",
"姐姐", "妹妹", "阿姨", "奶奶", "女声", "她"),
"male": ("男性", "男子", "先生", "小哥", "大叔", "男主", "父亲", "爸爸", "哥哥", "弟弟",
"叔叔", "爷爷", "男声", "男生", "他"),
}
# 姓名里的性别线索:国产剧本里「俊杰」「紫小慧」这种名字本身携带性别,
# 但 characters.gender 常常是空的(平台人物母版压根没有这一列),
# 只靠「男性/女士」这类关键词就推不出性别,声线只能瞎选。
_NAME_FEMALE_MARKERS = ("慧", "娟", "婷", "燕", "琳", "莉", "娜", "玲", "芳", "秀", "云", "霞",
"梅", "兰", "凤", "姐", "妹", "娘", "婆", "妞", " Girl".strip().lower())
_NAME_MALE_MARKERS = ("杰", "军", "强", "磊", "涛", "鹏", "飞", "刚", "伟", "建", "国", "明", "辉",
"斌", "波", "哥", "弟", "爷", "叔", "伯", "兄")
# 年龄档:只有性别还不够——「25 岁年轻女员工」配到「偏低的中国老年女声」,
# 性别对了,人物依旧不对(这正是上一版成片被吐槽的原因)。
AGE_BANDS = ("child", "young", "middle", "old")
AGE_BAND_LABELS = {"child": "儿童声", "young": "青年声", "middle": "中年声", "old": "老年声"}
_AGE_ORDER = {"child": 0, "young": 1, "middle": 2, "old": 3}
_AGE_MARKERS = {
"child": ("女童", "男童", "儿童", "小孩", "孩子", "萝莉", "正太", "娃娃音", "学龄", "童声"),
"young": ("年轻", "青年", "少女", "少男", "少年", "小伙", "姑娘", "女孩", "女生", "男生",
"大学生", "二十", "25 岁", "25岁", "20 岁", "20岁", "三十", "30 岁", "30岁"),
"middle": ("中年", "四十", "五十", "大叔", "大妈", "阿姨", "姐夫", "嫂子", "母亲", "父亲",
"妈妈", "爸爸", "已婚"),
"old": ("老年", "老者", "老人", "苍老", "年迈", "奶奶", "外婆", "姥姥", "爷爷", "外公",
"姥爷", "老太", "退休金", "拄拐"),
}
def infer_age_band(*texts: str) -> str:
"""从设定/声线描述里推断年龄档;判不出来返回空串。
按「老 → 中 → 青 → 童」顺序取第一个命中的档位:描述文本往往同时出现
「中年男声,语速很慢」这类混合表述,取跨度最大的一端(更老)比取中间值安全,
因为「老年声配年轻角色」是最容易被耳朵听出来的错误。
"""
blob = " ".join(str(text or "") for text in texts)
if not blob.strip():
return ""
for band in ("old", "middle", "young", "child"):
if any(marker in blob for marker in _AGE_MARKERS[band]):
return band
return ""
def voice_age_band(voice: VoiceAsset | None) -> str:
if voice is None:
return ""
return infer_age_band(voice.name, voice.description, voice.tuning.get("prompt")
if isinstance(voice.tuning, dict) else "")
def age_band_distance(a: str, b: str) -> int:
if a not in _AGE_ORDER or b not in _AGE_ORDER:
return -1
return abs(_AGE_ORDER[a] - _AGE_ORDER[b])
def current_voice_audio(db: Session, voice: VoiceAsset) -> str:
"""该声线当前真正可用的音色文件(优先 reference,其次最新的音色样本)。
平台声线母版经常只带一段「试听样本」而没有单独的参考音频;IndexTTS2 需要的
恰好就是一段人声样本,所以两者都能当克隆参考,不能让「没有 reference」= 没有音色。
"""
from app.constants import AssetKind
rows = [row for row in storage.assets_of(db, "voice", voice.id)
if row.kind in (AssetKind.REFERENCE, AssetKind.VOICE_SAMPLE)
and storage.exists(row.path)
and storage.belongs_to_owner(db, "voice", voice.id, row)]
if not rows:
return ""
ranked = [row for row in rows if row.kind == AssetKind.REFERENCE] or rows
return str(sorted(ranked, key=lambda row: (row.id, row.current_version or 1))[-1].path)
def repair_voice_reference(db: Session, voice: VoiceAsset | None, *, reason: str = "") -> str:
"""把声线的克隆参考音频指向它自己当前最新的音色素材,返回变更说明(无变更返回空串)。
两个必须自愈的场景:
1. 用户往声线里补录/新引入了音色素材,但 ``provider_voice_id`` 还停在旧文件上,
于是「声音明明换了,配音听起来一点没变」;
2. ``provider_voice_id`` 指的是已被删除/替换的文件,适配器会静默换成通用音色。
"""
if voice is None or "comfyui" not in str(voice.provider or ""):
return ""
latest = current_voice_audio(db, voice)
if not latest or latest == str(voice.provider_voice_id or "").strip():
return ""
previous = str(voice.provider_voice_id or "").strip()
voice.provider_voice_id = latest
label = "音色样本" if latest.endswith((".wav", ".mp3")) and "reference" not in latest else "参考音频"
return (f"声线「{voice.name}」的克隆{label}已更新为 {latest}"
+ (f"(原 {previous})" if previous else "(此前没有可用参考音频)")
+ (f":{reason}" if reason else ""))
def voice_reference_outdated(db: Session, voice: VoiceAsset | None) -> str:
"""该声线有更新的音色素材、但克隆参考还没跟上时返回那个新文件,否则空串。"""
if voice is None or "comfyui" not in str(voice.provider or ""):
return ""
latest = current_voice_audio(db, voice)
if not latest or latest == str(voice.provider_voice_id or "").strip():
return ""
return latest
def voice_reference_ready(db: Session, voice: VoiceAsset | None) -> bool:
"""声线是否真的能决定音色。
IndexTTS2 这类 zero-shot 声线靠参考音频克隆:``provider_voice_id`` 为空时适配器
会静默回落到模型配置里的通用参考音频,全片听起来都是同一个陌生人。
「项目里有声线记录」不等于「这个角色有自己的音色」,这里连文件是否真实存在一起判定。
"""
if voice is None:
return False
candidate = str(voice.provider_voice_id or "").strip()
if not candidate:
return False
if "comfyui" not in str(voice.provider or ""):
# 云端音色库用的是供应商音色 ID,不是本地参考音频文件
return True
if Path(candidate).suffix.lower() not in AUDIO_SUFFIXES and "/" not in candidate:
return False
return storage.exists(candidate)
def infer_gender(*texts: str) -> str:
"""从人物姓名与设定文本里推断性别;判不出来返回空串。"""
blob = " ".join(str(text or "") for text in texts)
if not blob.strip():
return ""
scores = {"female": 0.0, "male": 0.0}
for gender, markers in _GENDER_MARKERS.items():
scores[gender] += sum(blob.count(marker) for marker in markers)
scores[gender] += 0.5 * blob.count("女" if gender == "female" else "男")
# 姓名只作为弱线索参与(权重 0.4/字),文本里一旦有明确的「男性/女性」表述,
# 关键词得分一定更高,不会被名字翻盘。
name = str(texts[0] or "") if texts else ""
scores["female"] += 0.4 * sum(name.count(m) for m in _NAME_FEMALE_MARKERS)
scores["male"] += 0.4 * sum(name.count(m) for m in _NAME_MALE_MARKERS)
if scores["female"] == scores["male"]:
return ""
return "female" if scores["female"] > scores["male"] else "male"
def character_gender(db: Session, char: Character | None) -> str:
"""人物性别:结构化字段 → 已绑声线的性别 → 姓名与外观设定文本推断。"""
if char is None:
return ""
gender = str(getattr(char, "gender", "") or "").strip().lower()
if gender in ("male", "female", "neutral"):
return gender
if char.default_voice_id:
voice = db.get(VoiceAsset, char.default_voice_id)
voice_gender = str(getattr(voice, "gender", "") or "").strip().lower()
if voice_gender in ("male", "female", "neutral"):
return voice_gender
snapshot = getattr(char, "platform_snapshot_json", None) or {}
if isinstance(snapshot, dict):
snap_gender = str(snapshot.get("gender") or "").strip().lower()
if snap_gender in ("male", "female", "neutral"):
return snap_gender
return infer_gender(char.name, char.appearance, char.persona_prompt,
char.voice_profile, char.personality)
def character_age_band(db: Session, char: Character | None) -> str:
"""人物年龄档。
必须以人物自己的设定为准:绑定声线的描述经常写着「25 岁女性」,看起来很好用,
但它正是我们要审计的对象——把「老年女声」当成角色的年龄依据,错配就永远查不出来
(紫小慧 25 岁配老年女声,第一次就是因为这个自检不出问题)。
人物文本判不出档位时,才退回已绑声线的描述兜底。
"""
if char is None:
return ""
snapshot_from_char = _character_age_band_from_text(char)
if snapshot_from_char:
return snapshot_from_char
if char.default_voice_id:
band = voice_age_band(db.get(VoiceAsset, char.default_voice_id))
if band:
return band
return ""
def _character_age_band_from_text(char: Character) -> str:
snapshot = getattr(char, "platform_snapshot_json", None) or {}
extra = ""
if isinstance(snapshot, dict):
extra = str(snapshot.get("appearance") or "") + " " + str(snapshot.get("role") or "")
return infer_age_band(char.name, char.appearance, char.persona_prompt,
char.voice_profile, char.personality, char.role, extra)
def character_age_band_text(char: Character | None) -> str:
"""只读人物文本判年龄档(不查声线),供自检与用例复用。"""
return "" if char is None else _character_age_band_from_text(char)
def backfill_gender(db: Session, project: Project) -> list[str]:
"""把推断出来的性别写回 characters.gender,让「配音为什么选这条声线」可复核。
只填空字段,绝不覆盖用户手工设置过的值;返回被补齐的人物名,供事件流说明改了什么。
"""
filled: list[str] = []
for char in db.query(Character).filter(Character.project_id == project.id).all():
if str(char.gender or "").strip():
continue
gender = infer_gender(char.name, char.appearance, char.persona_prompt,
char.voice_profile, char.personality)
if gender in ("male", "female"):
char.gender = gender
filled.append(f"{char.name}={gender}")
return filled
def speaker_character(db: Session, shot: Shot, *, speaker_id: int | None = None) -> Character | None:
"""本镜到底是谁在说话:显式说话角色 → 画面描述里最早点名的人 → 出镜名单第一位。
项目 9《自检-异步调度验证》实测:镜头 02 的画面描述是「俊杰坐在服务台后,
面带轻松微笑,微微抬手示意紫小慧不用担心」,台词「别慌,我们公司用的是紫光汇智
的云桌面……」明显是俊杰说的。但俊杰是用户**导入脚本之后**才补进项目的人物,
`speaker_character_id` 一直是空的,旧逻辑直接退回 `shot.characters[0]`(紫小慧),
于是男主的台词配成了女声,第二次配音起这条女声线还被永久粘在镜头上——
这就是用户说的「配音接错人」。
判定口径:画面描述/梗概里**最早被点名**的项目人物就是这一镜的动作主语。
这是可解释的启发式,不是硬猜——推断出来的说话人会写回镜头并留一条事件,
用户在分镜里一眼就能看到、可以直接改。
"""
if speaker_id:
# 逐行配音/逐条字幕:这一行是谁说的由调用方给出,不用再从画面描述里猜。
char = db.get(Character, int(speaker_id))
if char is not None:
return char
if shot.speaker_character_id:
char = db.get(Character, shot.speaker_character_id)
if char is not None:
return char
project = shot.project
script = " ".join(filter(None, [shot.description, shot.synopsis]))
best: tuple[int, Character] | None = None
if script.strip() and project is not None:
for row in list(project.characters or []):
name = (row.name or "").strip()
if not name:
continue
at = script.find(name)
if at >= 0 and (best is None or at < best[0]):
best = (at, row)
if best is not None:
return best[1]
return (shot.characters or [None])[0]
def inferred_speaker(db: Session, shot: Shot) -> Character | None:
"""说话人是推断出来的(不是用户/脚本明确指定的)时返回该人物,否则 None。"""
if shot.speaker_character_id:
return None
char = speaker_character(db, shot)
if char is None:
return None
if (shot.characters or []) and char.id == int(shot.characters[0].id):
return None # 与旧口径结论相同,不算新推断
return char
def speaker_gender(db: Session, shot: Shot, *, speaker_id: int | None = None) -> str:
"""说话人性别:按 `speaker_character` 的统一口径取人,不再默认出镜第一位。"""
rows: list[Character] = [char for char in [speaker_character(db, shot, speaker_id=speaker_id)]
if char is not None]
rows.extend(shot.characters or [])
for char in rows:
gender = character_gender(db, char)
if gender:
return gender
return ""
def speaker_age_band(db: Session, shot: Shot, *, speaker_id: int | None = None) -> str:
"""说话人年龄档:优先说话角色,其次出镜角色(与 speaker_gender 同口径)。"""
rows: list[Character] = [char for char in [speaker_character(db, shot, speaker_id=speaker_id)]
if char is not None]
rows.extend(shot.characters or [])
for char in rows:
band = character_age_band(db, char)
if band:
return band
return ""
def rank_voices(db: Session, project: Project, *, gender: str = "", age_band: str = "",
speaker_name: str = "", pool: list[VoiceAsset] | None = None) -> list[dict]:
"""给项目里的声线按「贴不贴合这个说话人」打分,从高到低返回。
旧实现只有一条规则:「同性别 + 有参考音频」里挑第一条(按 id 排序),
于是 25 岁的紫小慧拿到了唯一一条带参考音频的「偏低的中国老年女声」。
现在按性别 > 年龄档 > 姓名归属 > 参考音频可用性 > 是否已被别人占用打分,
并把每一项理由带回去,配音失败时能直接告诉用户为什么。
"""
rows = pool if pool is not None else db.query(VoiceAsset).filter(
VoiceAsset.project_id == project.id).all()
owners: dict[int, str] = {}
for char in db.query(Character).filter(Character.project_id == project.id).all():
if char.default_voice_id:
owners.setdefault(int(char.default_voice_id), char.name)
scored: list[dict] = []
for row in rows:
ready = voice_reference_ready(db, row)
v_gender = str(row.gender or "").strip().lower()
v_band = voice_age_band(row)
score = 0.0
notes: list[str] = []
if gender:
if v_gender == gender:
score += 100
elif v_gender in ("", "neutral"):
score += 15
notes.append("声线未标注性别")
else:
score -= 120
notes.append("性别不符")
distance = age_band_distance(age_band, v_band)
if distance == 0:
score += 60
elif distance > 0:
score += {1: 10, 2: -25}.get(distance, -45)
notes.append(f"年龄档不符(说话人{AGE_BAND_LABELS.get(age_band, age_band)}"
f"/声线{AGE_BAND_LABELS.get(v_band, v_band)})")
name = str(row.name or "")
if speaker_name and speaker_name in name:
score += 45
notes.append("与说话人同名")
score += 30 if ready else -20
if not ready:
notes.append("缺少可用参考音频")
holder = owners.get(row.id)
if holder and speaker_name and holder != speaker_name:
score -= 25
notes.append(f"已是「{holder}」的默认声线")
if row.is_default:
score += 5
scored.append({"voice": row, "score": round(score, 1), "ready": ready,
"gender": v_gender, "age_band": v_band, "notes": notes})
scored.sort(key=lambda item: (-item["score"], item["voice"].id))
return scored
def resolve_voice(db: Session, project: Project, shot: Shot, *, speaker_id: int | None = None,
per_line: bool = False) -> tuple[VoiceAsset | None, dict]:
"""台词声线优先级,并说明「这是精确匹配还是回落来的」。
``per_line=True`` 表示按 ``shot.lines`` 里的某一行取声线:此时**不认镜头级
``voice_asset_id``**——那是单说话人时代留下的字段,多角色对白里沿用它
会让一整段话从头到尾只有一个嗓子。
优先级:镜头显式指定 → 说话角色默认声线 → 按说话人性别匹配 → 出镜角色默认 →
项目默认 → 项目第一个声线。
旧实现在前三档都落空时静默返回项目第一个声线,``generate_voiceover`` 还把它
固化写回镜头,于是「男性工程师」从第二次配音起永久使用紫小慧的女声,
而且没有任何地方说明过原因。现在把匹配过程一起返回:回落要告警,
只有真正有依据的匹配才允许固化到镜头上。
"""
gender = speaker_gender(db, shot, speaker_id=speaker_id)
age_band = speaker_age_band(db, shot, speaker_id=speaker_id)
speaker = speaker_character(db, shot, speaker_id=speaker_id)
speaker_name = str(speaker.name or "") if speaker is not None else ""
info = {"matched": False, "exact": False, "reason": "", "gender": gender,
"voice": "", "voice_gender": "", "reference_ready": False,
"gender_mismatch": False, "age_band": age_band, "voice_age_band": "",
"age_mismatch": False, "alternatives": []}
voice: VoiceAsset | None = None
if shot.voice_asset_id and not per_line:
voice = db.get(VoiceAsset, shot.voice_asset_id)
if voice is not None:
info.update(matched=True, exact=True, reason="镜头已指定声线")
if voice is None and per_line and speaker_id:
# 逐行:这一行的角色自己的默认声线优先,和「镜头说话人」是两套归属
char = db.get(Character, int(speaker_id))
if char is not None and char.default_voice_id:
candidate = db.get(VoiceAsset, char.default_voice_id)
if candidate is not None:
voice = candidate
info.update(matched=True, exact=True, reason=f"「{char.name}」的默认声线")
if voice is None and not per_line and shot.speaker_character_id:
char = db.get(Character, shot.speaker_character_id)
if char is not None and char.default_voice_id:
voice = db.get(VoiceAsset, char.default_voice_id)
if voice is not None:
info.update(matched=True, exact=True, reason=f"说话人「{char.name}」的默认声线")
if voice is None and (gender or age_band or speaker_name):
# 同性别匹配放在「出镜角色默认」之前:对白镜头里坐着一男一女时,
# 出镜顺序的第一位常常不是说话的人。
ranked = rank_voices(db, project, gender=gender, age_band=age_band,
speaker_name=speaker_name)
info["alternatives"] = [{"voice": item["voice"].name, "score": item["score"],
"notes": item["notes"]} for item in ranked[:3]]
# 负分意味着「只有性别相反 / 年龄差两档以上的声线可用」,宁可不配
# 也不要配出一条明显不对的声线——那会被当成成品缺陷退回。
best = ranked[0] if ranked and ranked[0]["score"] > 0 else None
if best is not None:
voice = best["voice"]
info.update(matched=True, reference_ready=best["ready"],
age_mismatch=bool(age_band_distance(age_band, best["age_band"]) >= 2),
reason=f"按说话人{GENDER_LABELS.get(gender, gender)}"
f"{AGE_BAND_LABELS.get(age_band, '')}匹配到「{voice.name}」"
+ (f"({ ';'.join(best['notes']) })" if best["notes"] else ""))
if voice is None and per_line and not speaker_name:
info["reason"] = "这一行台词没标说话人,按出镜人物与性别兜底匹配"
if voice is None:
for char in shot.characters or []:
if not char.default_voice_id:
continue
candidate = db.get(VoiceAsset, char.default_voice_id)
if candidate is not None:
voice = candidate
info.update(matched=True, exact=True,
reason=f"出镜人物「{char.name}」的默认声线")
break
if voice is None:
pool = db.query(VoiceAsset).filter(VoiceAsset.project_id == project.id) \
.order_by(VoiceAsset.id).all()
defaults = [row for row in pool if row.is_default]
ready = [row for row in pool if voice_reference_ready(db, row)]
picked = defaults or ready or pool
if picked:
voice = picked[0]
info["reason"] = ("项目默认声线" if defaults else
f"项目里没有同性别可用声线,暂用「{voice.name}」")
if voice is None:
return None, info
info["voice"] = voice.name
info["voice_gender"] = str(voice.gender or "")
info["voice_age_band"] = voice_age_band(voice)
info["reference_ready"] = voice_reference_ready(db, voice)
info["gender_mismatch"] = bool(gender and info["voice_gender"]
and gender != info["voice_gender"])
info["age_mismatch"] = info["age_mismatch"] or age_band_distance(
info.get("age_band") or "", info["voice_age_band"]) >= 2
return voice, info
def voice_selection_hint(info: dict) -> str:
"""把匹配过程翻译成一句能直接进事件流/预检的中文说明。"""
bits: list[str] = []
if info.get("gender_mismatch"):
bits.append("说话人是%s,声线「%s」却是%s" % (
GENDER_LABELS.get(info.get("gender") or "", info.get("gender") or "?"),
info.get("voice") or "",
GENDER_LABELS.get(info.get("voice_gender") or "", info.get("voice_gender") or "?")))
if info.get("age_mismatch"):
bits.append("说话人是%s,声线「%s」却是%s" % (
AGE_BAND_LABELS.get(info.get("age_band") or "", info.get("age_band") or "?"),
info.get("voice") or "",
AGE_BAND_LABELS.get(info.get("voice_age_band") or "", info.get("voice_age_band") or "?")))
if info.get("reference_ready") is False:
bits.append("声线「%s」没有参考音频,音色由模型默认参考决定"
"(不是这个角色自己的音色)" % (info.get("voice") or ""))
if not info.get("exact") and info.get("reason"):
bits.append(str(info["reason"]))
return ";".join(bits)
def resolve_voice_asset(db: Session, project: Project, shot: Shot) -> VoiceAsset | None:
"""兼容旧调用:只取声线本体,不关心匹配过程。"""
return resolve_voice(db, project, shot)[0]
def resolve_tts_spec(db: Session, voice: VoiceAsset | None):
"""按声线资产选择 TTS 模型;找不到对应 ModelSpec 时退回库里的默认配音模型。
优先级只有两层:声线自己登记的供应商 → 库里的默认配音模型。
中间原本还夹着一层 `settings.tts_provider`,那是「配音模型写在 .env 里」时代的产物:
模型配置搬进 model_specs 表之后它取不到值、等于代码默认 "mock",于是**没配过供应商的
声线会悄悄用 Mock 配音**,把页面上设好的 comfyui/indextts2 默认位顶掉。
「配音听起来不对」这件事只有听成片才能发现,所以宁可少一层兜底。
"""
candidates = [voice.provider] if voice and voice.provider else []
candidates.append(None)
for provider in candidates:
try:
spec, adapter = model_registry.resolve_voice(db, provider, None)
if adapter is not None and adapter.is_available(model_registry.runtime_config(db, spec)):
return spec, adapter
except Exception:
continue
return model_registry.resolve_voice(db, None, None)
def estimate_duration_ms(text: str, *, speed: float = 1.0) -> int:
text = (text or "").strip()
if not text:
return 0
cjk = sum(1 for ch in text if "\u4e00" <= ch <= "\u9fff")
latin_words = len(re.findall(r"[A-Za-z0-9']+", text))
pauses = len(re.findall(r"[,。!?;、,.!?;\n]", text))
seconds = cjk / CPS_CJK + latin_words / WPS_LATIN + pauses * PAUSE_SECONDS * 0.5
return int(round(max(0.6, seconds) / max(0.5, min(float(speed or 1.0), 2.0)) * 1000))
def line_voice_speed(db: Session, project: Project, shot: Shot, line: dict) -> float:
"""某一行的语速:取该行说话人的声线调参(多角色时每人语速可以不同)。"""
voice, _info = resolve_voice(db, project, shot, speaker_id=line.get("character_id"), per_line=True)
return float((voice.tuning or {}).get("speed", 1.0) if voice else 1.0)
def shot_voice_duration_ms(db: Session, project: Project, shot: Shot) -> int:
"""镜头配音时长:优先真实产物时长,其次按文本估算(用于时间线预排)。
多行台词要逐行估算再相加,并补上换人之间的静音间隙——
按整段字数估会短一截,片段被排得太紧,配音尾巴就被切掉。
"""
if shot.voiceover_duration_ms and storage.exists(shot.voiceover_path):
return int(shot.voiceover_duration_ms)
lines = shot.all_lines()
if not lines:
return 0
if len(lines) == 1:
text = (lines[0]["text"] or "").strip()
voice = resolve_voice_asset(db, project, shot)
speed = float((voice.tuning or {}).get("speed", 1.0) if voice else 1.0)
return estimate_duration_ms(text, speed=speed)
total = sum(estimate_duration_ms(str(row.get("text") or ""),
speed=line_voice_speed(db, project, shot, row))
for row in lines)
return int(total + LINE_GAP_MS * (len(lines) - 1))
def _tts_produce(db: Session, project: Project, shot: Shot, text: str, voice: VoiceAsset | None,
*, out: Path, index: int = 0) -> tuple:
"""一次 TTS 调用:按声线选模型、合成到 ``out``,返回 (spec, result, produced, duration_ms)。"""
spec, _adapter = resolve_tts_spec(db, voice)
tuning = dict(voice.tuning or {}) if voice else {}
request = VoiceGenRequest(
text=text, voice_id=voice.name if voice else "",
provider_voice_id=voice.provider_voice_id if voice else "",
language=(voice.language if voice else "zh-CN"), speed=float(tuning.get("speed", 1.0)),
pitch=float(tuning.get("pitch", 1.0)), volume=float(tuning.get("volume", 1.0)),
sample_rate=int(tuning.get("sample_rate", 24000)), seed=(int(shot.seed or 0) + index),
params={**(tuning.get("params") or {}), "shot_code": shot.code},
# 描述词直出:声线没录过音也能配上音(音色由「音色描述」决定)
instruct_text=(timbre.instruct_text_of(voice)
if voice is not None and timbre.instruct_model(spec) else ""),
idempotency_key=f"voice-{shot.id}-{index}", label=f"SHOT-{shot.order_index:02d}-{index + 1:02d}",
output_path=out)
result = model_registry.generate_voice(db, request, spec.provider, spec.model)
produced = Path(result.file_path)
duration_ms = int(result.duration_ms or ff.duration_ms(produced) or estimate_duration_ms(text))
return spec, result, produced, duration_ms
def concat_voice_parts(parts: list[Path], out: Path, *, gap_ms: int = LINE_GAP_MS) -> Path:
"""把逐行配音按顺序拼成一条音轨,行间插一小段静音。
一句台词一个嗓子、逐行合成是多人对白的唯一可靠做法:整段丢给 TTS,
两个人只会共用第一个人的音色。换人处留 220ms 呼吸,否则听起来像同一个人绕口令。
"""
parts = [Path(p) for p in parts if p is not None]
if not parts:
raise ValueError("没有可拼接的配音片段")
if len(parts) == 1:
return parts[0]
inputs: list[str] = []
labels: list[str] = []
for pos, part in enumerate(parts):
if pos and gap_ms > 0:
inputs += ["-f", "lavfi", "-t", f"{gap_ms / 1000:.3f}", "-i",
"anullsrc=channel_layout=stereo:sample_rate=48000"]
labels.append("gap%02d" % pos)
inputs += ["-i", str(part)]
labels.append("seg%02d" % pos)
chain: list[str] = []
for idx, label in enumerate(labels):
# 每段统一 48k/立体声:不同声线的采样率、声道数不一致时 concat 会直接报错
tail = (f"atrim=0:{gap_ms / 1000:.3f},asetpts=PTS-STARTPTS," if label.startswith("gap") else "")
chain.append(f"[{idx}:a]{tail}aresample=48000,"
f"aformat=sample_fmts=s16:channel_layouts=stereo[{label}]")
concat_in = "".join(f"[{label}]" for label in labels)
chain.append(f"{concat_in}concat=n={len(labels)}:v=0:a=1[out]")
cmd = [ff.FFMPEG, "-hide_banner", "-loglevel", "error", "-y", *inputs, "-filter_complex",
";".join(chain), "-map", "[out]", "-ar", "48000", "-ac", "2", "-c:a", "pcm_s16le", str(out)]
ff.run(cmd, timeout=300)
if not out.exists():
raise ValueError("配音拼接失败(ffmpeg 未产出文件)")
return out
def generate_voiceover(db: Session, project: Project, shot: Shot, *, text: str | None = None,
voice: VoiceAsset | None = None, task_id: int | None = None,
commit: bool = True) -> dict:
"""调用 TTS 生成配音,登记为镜头资产并回写时长。
一镜多人对白:``shot.lines`` 有多行时逐行取各自说话人的声线分别合成再拼接。
调用方显式给了 ``text`` 或 ``voice`` 时按「整段一个嗓子」处理(手工试听/改写场景)。
"""
_lines = shot.all_lines()
if text is None and voice is None and len(_lines) > 1:
return generate_voiceover_lines(db, project, shot, _lines, task_id=task_id, commit=commit)
text = (text if text is not None else (shot.dialogue or "")).strip()
if not text:
raise ValueError("镜头没有台词,无法配音")
match_info: dict = {"matched": bool(voice), "exact": bool(voice),
"reason": "调用方直接指定声线", "voice": voice.name if voice else "",
"voice_gender": str(voice.gender or "") if voice else "", "gender": "",
"reference_ready": voice_reference_ready(db, voice),
"gender_mismatch": False}
if voice is None:
voice, match_info = resolve_voice(db, project, shot)
guess = inferred_speaker(db, shot)
if guess is not None and voice is not None:
# 台词没挂说话人时按画面描述认出来的人,要写回镜头并在事件里说明:
# 否则用户只看到「俊杰的台词变成了紫小慧的嗓子」,却不知道系统认的是谁。
shot.speaker_character_id = guess.id
if all(row.id != guess.id for row in (shot.characters or [])):
shot.characters = list(shot.characters or []) + [guess]
match_info["inferred_speaker"] = guess.name
events.log_event(
db, "voiceover.speaker_inferred", project_id=project.id, shot_id=shot.id,
task_id=task_id, level="info",
message=f"镜头 {shot.order_index:02d} 台词原本没挂说话人,按画面描述认定说话人为"
f"「{guess.name}」,已用其声线「{voice.name}」配音(分镜里可改)",
entity_type="shot", entity_id=shot.id,
data={"speaker": guess.name, "speaker_id": guess.id, "voice": voice.name},
)
# 先自愈再合成:用户新加的音色素材必须立刻体现在配音里,否则「换了声音」这件事
# 只存在于素材列表里,听起来还是旧音色。
repaired = repair_voice_reference(db, voice, reason=f"镜头 {shot.order_index:02d} 配音前自动对齐音色素材")
if repaired:
match_info["reference_ready"] = voice_reference_ready(db, voice)
events.log_event(db, "voice.reference_refreshed", project_id=project.id, shot_id=shot.id,
message=repaired, entity_type="voice",
entity_id=getattr(voice, "id", None), data={"reason": "voiceover"})
folder = storage.project_dir(project.id, "voiceover")
stem = f"shot-{shot.order_index:02d}"
out = folder / f"{stem}_{abs(hash((text, voice.provider_voice_id if voice else '', shot.voice_asset_id))) % 10**10}.wav"
spec, result, produced, duration_ms = _tts_produce(db, project, shot, text, voice, out=out)
rel = storage.to_relative(produced)
size = produced.stat().st_size if produced.exists() else 0
info = {"path": rel, "size_bytes": size, "duration_ms": duration_ms,
"original_name": produced.name, "mime": "audio/wav"}
storage.register_asset(db, project_id=project.id, owner_type="shot", owner_id=shot.id, kind="voiceover",
path=rel, name=f"镜头 {shot.order_index:02d} 配音", source="provider", info=info,
metadata={"provider": spec.provider, "model": spec.model,
"voice": voice.name if voice else "", "text": text})
shot.voiceover_path = rel
shot.voiceover_duration_ms = duration_ms
# 只有「说话人自己的声线」才固化到镜头:旧实现连静默回落来的声线也写死,
# 男配音一旦被粘上女声,之后每次重配都还是女声。
if not shot.voice_asset_id and voice is not None and match_info.get("exact"):
shot.voice_asset_id = voice.id
cost = float(result.cost or costing.estimate_voice_cost(spec, len(text)))
if cost:
costing.record_cost(db, project_id=project.id, kind="voiceover", provider=spec.provider,
model=spec.model, shot_id=shot.id, task_id=task_id, quantity=len(text),
unit="1k_chars", unit_price=float(getattr(spec, "price_per_1k_chars", 0.0) or 0.0),
cost=cost, currency=spec.currency, note=f"镜头 {shot.order_index:02d} 配音")
costing.record_generation(db, project_id=project.id, shot_id=shot.id, task_id=task_id,
provider=spec.provider, model=spec.model, kind="voiceover", prompt=text,
params_json={"voice": voice.name if voice else "",
"duration_ms": duration_ms,
"voice_match": {k: match_info.get(k) for k in
("exact", "matched", "reason", "gender",
"voice_gender", "reference_ready",
"gender_mismatch", "age_band",
"voice_age_band", "age_mismatch")},
"reference_source": (result.metadata or {}).get(
"reference_source", "")},
provider_job_id=result.provider_job_id, output_path=rel,
duration_seconds=round(duration_ms / 1000.0, 3), cost=cost,
currency=spec.currency,
metadata_json={"mock": bool((result.metadata or {}).get("mock"))})
events.log_event(db, "voiceover.generated", project_id=project.id, shot_id=shot.id, task_id=task_id,
message=f"镜头 {shot.order_index:02d} 配音完成({duration_ms / 1000:.1f}s,"
f"声线 {voice.name if voice else spec.model})",
entity_type="shot", entity_id=shot.id,
data={"path": rel, "duration_ms": duration_ms, "chars": len(text),
"voice_match": match_info,
"reference_source": (result.metadata or {}).get("reference_source", "")})
warning = voice_selection_hint(match_info)
if warning:
# 「配音听起来怪」过去只能靠人反复试听发现:性别错了、或者音色其实是模型默认的,
# 系统一句话都不说。现在把原因写进事件流,用户在总控台就能看到该换哪条声线。
events.log_event(
db, "voiceover.voice_fallback", project_id=project.id, shot_id=shot.id, task_id=task_id,
level="warning",
message=f"镜头 {shot.order_index:02d} 配音声线待修正:{warning}",
entity_type="shot", entity_id=shot.id,
data={"voice": voice.name if voice else "", "match": match_info,
"reference_source": (result.metadata or {}).get("reference_source", "")},
)
if commit:
db.commit()
else:
db.flush()
return {"path": rel, "url": storage.url_for(rel), "duration_ms": duration_ms,
"voice": voice.name if voice else spec.model, "provider": spec.provider, "model": spec.model,
"chars": len(text), "cost": round(cost, 4), "estimated": False}
def generate_voiceover_lines(db: Session, project: Project, shot: Shot, lines: list[dict], *,
task_id: int | None = None, commit: bool = True) -> dict:
"""多角色对白:逐行按说话人的声线合成,再拼成镜头的一条配音音轨。"""
folder = storage.project_dir(project.id, "voiceover")
stem = f"shot-{shot.order_index:02d}"
parts: list[Path] = []
rows: list[dict] = []
total_cost = 0.0
provider = model = ""
for index, line in enumerate(lines):
text = str(line.get("text") or "").strip()
if not text:
continue
speaker_id = line.get("character_id")
voice, info = resolve_voice(db, project, shot, speaker_id=speaker_id, per_line=True)
repaired = repair_voice_reference(
db, voice, reason=f"镜头 {shot.order_index:02d} 第 {len(rows) + 1} 句配音前自动对齐音色素材")
if repaired:
events.log_event(db, "voice.reference_refreshed", project_id=project.id, shot_id=shot.id,
message=repaired, entity_type="voice",
entity_id=getattr(voice, "id", None), data={"reason": "voiceover"})
out = folder / f"{stem}_l{len(rows) + 1:02d}_" \
f"{abs(hash((text, voice.provider_voice_id if voice else '', speaker_id))) % 10 ** 10}.wav"
spec, result, produced, duration_ms = _tts_produce(db, project, shot, text, voice,
out=out, index=len(rows))
parts.append(produced)
provider, model = spec.provider, spec.model
cost = float(result.cost or costing.estimate_voice_cost(spec, len(text)))
total_cost += cost
speaker = shot.line_speaker_name(line) or "未标注说话人"
rows.append({"index": len(rows), "speaker": speaker, "character_id": speaker_id, "text": text,
"voice": voice.name if voice else spec.model, "voice_id": getattr(voice, "id", None),
"duration_ms": duration_ms, "cost": cost, "voice_match": info,
"path": storage.to_relative(produced)})
if cost:
costing.record_cost(db, project_id=project.id, kind="voiceover", provider=spec.provider,
model=spec.model, shot_id=shot.id, task_id=task_id, quantity=len(text),
unit="1k_chars",
unit_price=float(getattr(spec, "price_per_1k_chars", 0.0) or 0.0),
cost=cost, currency=spec.currency,
note=f"镜头 {shot.order_index:02d} 第 {len(rows)} 句配音({speaker})")
warning = voice_selection_hint(info)
if warning:
events.log_event(
db, "voiceover.voice_fallback", project_id=project.id, shot_id=shot.id, task_id=task_id,
level="warning",
message=f"镜头 {shot.order_index:02d} 第 {len(rows)} 句({speaker})声线待修正:{warning}",
entity_type="shot", entity_id=shot.id,
data={"voice": voice.name if voice else "", "speaker": speaker, "match": info})
if not parts:
raise ValueError("镜头没有台词,无法配音")
joined = folder / f"{stem}_mix_{abs(hash(tuple(str(p) for p in parts))) % 10 ** 10}.wav"
produced = concat_voice_parts(parts, joined) if len(parts) > 1 else parts[0]
rel = storage.to_relative(produced)
duration_ms = ff.duration_ms(produced) or sum(int(row["duration_ms"]) for row in rows)
info_row = {"path": rel, "size_bytes": produced.stat().st_size if produced.exists() else 0,
"duration_ms": duration_ms, "original_name": produced.name, "mime": "audio/wav"}
storage.register_asset(db, project_id=project.id, owner_type="shot", owner_id=shot.id,
kind="voiceover", path=rel, name=f"镜头 {shot.order_index:02d} 配音(多角色)",
source="provider", info=info_row,
metadata={"provider": provider, "model": model, "text": shot.dialogue,
"lines": [{"speaker": row["speaker"], "voice": row["voice"],
"text": row["text"], "duration_ms": row["duration_ms"]}
for row in rows]})
shot.voiceover_path = rel
shot.voiceover_duration_ms = duration_ms
# 多角色镜头不固化 shot.voice_asset_id:那是「整镜一个嗓子」时代的字段,
# 粘上第一个人的声线会让以后每次重配都变成他一个人说话。
costing.record_generation(db, project_id=project.id, shot_id=shot.id, task_id=task_id,
provider=provider, model=model, kind="voiceover", prompt=shot.dialogue,
params_json={"mode": "per_line", "lines": [
{"speaker": row["speaker"], "voice": row["voice"],
"chars": len(row["text"]), "duration_ms": row["duration_ms"],
"exact": bool(row["voice_match"].get("exact")),
"reason": row["voice_match"].get("reason", "")} for row in rows],
"duration_ms": duration_ms, "gap_ms": LINE_GAP_MS},
output_path=rel, duration_seconds=round(duration_ms / 1000.0, 3),
cost=total_cost, currency="CNY")
events.log_event(db, "voiceover.generated", project_id=project.id, shot_id=shot.id, task_id=task_id,
message=f"镜头 {shot.order_index:02d} 配音完成({duration_ms / 1000:.1f}s,"
f"{len(rows)} 句:{'、'.join(row['speaker'] for row in rows)})",
entity_type="shot", entity_id=shot.id,
data={"path": rel, "duration_ms": duration_ms, "chars": len(shot.dialogue),
"lines": [{"speaker": row["speaker"], "voice": row["voice"],
"text": row["text"]} for row in rows]})
if commit:
db.commit()
else:
db.flush()
return {"path": rel, "url": storage.url_for(rel), "duration_ms": duration_ms,
"voice": " / ".join(row["voice"] for row in rows), "provider": provider, "model": model,
"chars": len(shot.dialogue), "cost": round(total_cost, 4), "estimated": False,
"lines": [{"speaker": row["speaker"], "voice": row["voice"], "text": row["text"],
"duration_ms": row["duration_ms"]} for row in rows]}
def plan_voiceovers(db: Session, project: Project, shots: list[Shot] | None = None) -> list[dict]:
"""为时间线预排提供配音时长计划(含缺失配音的镜头)。"""
rows = []
for shot in (shots if shots is not None else list(project.shots or [])):
lines = shot.all_lines()
if not lines:
continue
voice = resolve_voice_asset(db, project, shot)
has = storage.exists(shot.voiceover_path)
rows.append({"shot_id": shot.id, "order_index": shot.order_index,
"chars": sum(len(str(row["text"])) for row in lines),
"voice": voice.name if voice else "未绑定声线",
"speakers": [shot.line_speaker_name(row) or "未标注说话人" for row in lines],
"lines": [{"speaker": shot.line_speaker_name(row) or "未标注说话人",
"text": row["text"]} for row in lines],
"duration_ms": shot_voice_duration_ms(db, project, shot),
"ready": has, "path": shot.voiceover_path})
return rows
def split_text(text: str, *, max_chars: int = 22) -> list[str]:
"""按标点与长度切分字幕句,保证字幕与语音节奏一致。"""
text = re.sub(r"\s+", " ", (text or "").strip())
if not text:
return []
chunks: list[str] = []
for sentence in re.split(r"(?<=[。!?!?;;])|(?<=\n)", text):
sentence = sentence.strip()
if not sentence:
continue
if len(sentence) <= max_chars:
chunks.append(sentence)
continue
buf = ""
for part in re.split(r"(?<=[,,、::])", sentence):
if not part:
continue
if len(buf) + len(part) <= max_chars:
buf += part
else:
if buf:
chunks.append(buf)
while len(part) > max_chars:
chunks.append(part[:max_chars])
part = part[max_chars:]
buf = part
if buf:
chunks.append(buf)
merged: list[str] = []
for chunk in chunks:
if merged and len(merged[-1]) < 6:
merged[-1] += chunk
else:
merged.append(chunk)
return merged
def cue_plan(db: Session, project: Project, shot: Shot, *, window_ms: int, start_ms: int,
lead_ms: int = 160, voice_ms: int | None = None) -> list[dict]:
"""把一个镜头的字幕对齐到给定的时间窗(语音时长优先,均分兜底)。
``voice_ms=0`` 表示「没有可参照的 TTS 时长」(例如台词直接用模型原声),
此时字幕按整个片段窗口铺排,而不是按一段并不存在的配音对齐。
"""
text = (shot.effective_subtitle or "").strip()
if not text or window_ms <= 200:
return []
# 逐条台词时字幕要标出「这句是谁说的」;单说话人沿用镜头上的口径。
chunks: list[str] = []
speakers: list[str] = []
lines = shot.all_lines()
if len(lines) > 1:
for line in lines:
name = shot.line_speaker_name(line)
for chunk in split_text(str(line.get("text") or "")):
chunks.append(chunk)
speakers.append(name)
if not chunks:
chunks = split_text(text)
default_speaker = ""
if shot.speaker:
default_speaker = shot.speaker.name
elif shot.characters:
default_speaker = shot.characters[0].name
speakers = [default_speaker] * len(chunks)
if not chunks:
return []
if voice_ms is None:
voice_ms = (min(int(shot.voiceover_duration_ms or 0), window_ms - lead_ms)
if shot.voiceover_path else 0)
voice_ms = min(int(voice_ms), window_ms - lead_ms)
usable = max(400, (voice_ms or window_ms - lead_ms))
start = start_ms + lead_ms
weights = [max(4, len(c)) for c in chunks]
total_w = sum(weights)
cues: list[dict] = []
cursor = start
for i, (chunk, weight) in enumerate(zip(chunks, weights)):
dur = max(700, int(usable * weight / total_w))
end = min(start_ms + window_ms, cursor + dur)
if end - cursor < 300:
break
cues.append({"start_ms": int(cursor), "end_ms": int(end), "text": chunk,
"speaker": speakers[i] if i < len(speakers) else "", "shot_id": shot.id})
cursor = end + 60
return cues