nex_animation/backend/tests/test_joint_caption.py

323 lines
18 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""回归用例:喂给联合音视频模型(MiniMax H3)的 caption 里不能出现「会被念出来」的整句。
现场证据(项目 11《告诉你个小秘密第1期》v2 成片逐镜 ASR):
提示词开头那几条陈述整句
<Picture 1>是这一镜的画面。<Picture 2>是俊杰,高大,和蔼。
被模型当成旁白**原样念了出来**(shot-02 前 2.68s、shot-03 前 2.0s 的
「是这一镜的画面和石腰准和碍」就是它),台词本身反而正常。
把绑定改成定语写法「<Picture n>中的俊杰」后重新生成(task 541),
ASR 只剩台词本体、开头完全静音,画面里两个角色的长相/服装仍然与定妆图一致。
所以这里锁两件事:① 默认写法不留任何「<Picture n>是X」整句;
② 台词、身份锚定、镜头语言一样都不能少(不能为了静音把信息删光)。
"""
from __future__ import annotations
import re
from app.models import Character, Project, Scene, Shot
from app.services import prompt as P
ROLES = [{"index": 1, "owner_type": "frame", "owner_id": None, "name": "画面"},
{"index": 2, "owner_type": "human", "owner_id": None, "name": "俊杰"},
{"index": 3, "owner_type": "human", "owner_id": None, "name": "紫小慧"},
{"index": 4, "owner_type": "scene", "owner_id": None, "name": "场景"}]
def _case(db):
project = Project(name="自检-联合caption", aspect_ratio="9:16", resolution="720p", fps=24,
style_guide="办公室轻松风格", default_provider="comfyui", target_duration=20)
db.add(project)
db.flush()
male = Character(project_id=project.id, name="俊杰", gender="male", appearance="高大,和蔼")
female = Character(project_id=project.id, name="紫小慧", gender="female",
appearance="年轻东亚女生,精致淡妆")
scene = Scene(project_id=project.id, name="办公室", prompt="现代简约的办公室,明亮整洁,有电脑工位")
db.add_all([male, female, scene])
db.flush()
shot = Shot(project_id=project.id, order_index=1, code="S01", title="俊杰解答",
description="俊杰坐在工位上,从容自信地微笑着转头看向紫小慧",
duration=8, shot_size="close_up", camera_angle="eye_level",
camera_movement="static", mode="r2v", status="ready",
dialogue="不要着急,恢复系统和数据一键就能完成。", speaker_character_id=male.id,
ambience="办公室环境底噪")
shot.characters = [male, female]
shot.scenes = [scene]
db.add(shot)
db.flush()
roles = [dict(ROLES[0]), dict(ROLES[1], owner_id=male.id),
dict(ROLES[2], owner_id=female.id), dict(ROLES[3], owner_id=scene.id)]
return project, shot, roles
def test_joint_caption_leaves_no_speakable_binding_sentence(db):
"""默认(inline):整段没有任何「<Picture n>是X」陈述句,也不以标签开头。"""
project, shot, roles = _case(db)
text = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles, duration_seconds=8)
assert not re.search(r"<Picture \d+>是", text), f"仍有会被念出来的绑定整句:{text}"
# 每个标签后面都必须是「中的…」定语:定语不是句子,模型不会朗读它
assert re.findall(r"<Picture \d+>(?!中的)", text) == [], f"有标签没走定语写法:{text}"
# 身份锚定一句没少,只是换成不会被朗读的定语
assert "<Picture 2>中的俊杰" in text and "<Picture 3>中的紫小慧" in text
assert "<Picture 1>中的画面" in text and "<Picture 4>中的" in text
# 台词是官方说话段落,不再是「{名}台词:「…」」这种能被朗读的说明句
assert "<d>不要着急,恢复系统和数据一键就能完成。</d>" in text, text
assert text.endswith("</d>"), f"台词必须是最后一段:{text}"
assert "台词" not in text and ":" not in text, text
# 特征短语与动作之间必须留分隔,否则「年轻东亚女生转头看向」会被当成新词
assert ",," not in text, text
assert "转头看向<Picture 3>中的紫小慧,年轻东亚女生" in text, text
# 「和蔼」这类看不见的气质/妆造词不进联合提示词:定妆图已经锁死了人,
# 气质文字模型既演不出来又最容易被当台词念出来(「轻熟纯欲气质」实测被念)。
assert "和蔼" not in text and "精致淡妆" not in text, text
assert "<Picture 2>中的俊杰,高大,坐在工位上" in text, text
# 镜头语言照旧下发;时长/画幅不再写成文字——它们是 length/width/height 结构化入参,
# 写成句子就会被念出来(v2 成片开头那句「时长约 8 秒画幅 9 比 16」)
assert "近景" in text and "固定机位" in text, text
assert "时长" not in text and "画幅" not in text and "秒" not in text, text
# 没有参考音色时不能凭空写 <Audio n>:标签指向不存在的素材会串音
assert "<Audio" not in text, text
def test_joint_caption_only_one_speech_clause(db):
"""除了台词本体,caption 里不得再出现第二处「说话」相关内容。"""
project, shot, roles = _case(db)
text = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles, duration_seconds=8)
speech_words = ("台词", "说", "旁白", "配音", "口型", "语音", "字幕", "背景音", "音色", "BGM")
hits = [w for w in speech_words if w in text]
assert hits == [], f"caption 里混进了声音话题 {hits}:{text}"
def test_joint_caption_binds_each_line_to_its_own_voice(db):
"""``<Audio n>`` 是 H3 里**唯一**合法的「谁用这条嗓子说话」写法。
ComfyUI 的 ``MiniMaxH3ReferenceToVideo`` 没有独立 dialogue 字段,文本编码器把
``<Audio j>:`` 标签和 prompt 裸拼(源码 ``comfy_text_encoders_minimax.py``),
docstring 明确要求提示词用 ``<Audio j>`` 引用 ``ref_audio_j``。序号按节点输入
顺序分配,所以多角色必须逐句绑定,不能整体套第一条。
"""
project, shot, roles = _case(db)
shot.set_lines([{"text": "电脑怎么了?", "character_id": shot.characters[1].id},
{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
db.flush()
text = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles,
duration_seconds=8, voice_names=["俊杰", "紫小慧"])
assert "<Audio 2><d>电脑怎么了?</d>" in text, text # 紫小慧 = 第 2 条参考音色
assert "<Audio 1><d>一键就能恢复。</d>" in text, text # 俊杰 = 第 1 条
assert text.count("<Audio ") == 2 and text.count("<d>") == 2, text
# 两段台词之间不要再补句号:那会被当成句子结束、各配一段气口
assert "</d><Audio 2>" not in text and "</d>。<Audio" not in text, text
# 查不到说话人(没配默认音色)时宁可不加标签,也不能绑错嗓子
blind = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles,
duration_seconds=8, voice_names=["", ""])
assert "<Audio" not in blind and "<d>电脑怎么了?</d>" in blind, blind
# 多人对白只收到一条音色时**不能**给第二个人套 <Audio 1>:
# 标签是硬指令,模型会照着让紫小慧用俊杰的嗓子开口(比不标更糟)。
thin = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles,
duration_seconds=8, voice_names=["俊杰"])
assert "<Audio 1><d>一键就能恢复。</d>" in thin, thin # 俊杰那句仍然绑自己的嗓子
assert "<d>电脑怎么了?</d>" in thin and "<Audio 1><d>电脑怎么了?</d>" not in thin, thin
assert thin.count("<Audio ") == 1, thin
# 只有一个说话人的镜头才允许「这条嗓子就是他本人」的兜底(声线没回指角色时名字是空的)
shot.set_lines([{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
db.flush()
solo = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles,
duration_seconds=8, voice_names=[""])
assert "<Audio 1><d>一键就能恢复。</d>" in solo, solo
def _char(project, name: str):
"""按名字取人物(``project.characters`` 的顺序不保证,别按下标猜)。"""
return next(c for c in project.characters if str(c.name or "").strip() == name)
def _voiced(db, project):
"""给两个角色各配一条**可用的**默认声线(``<Audio n>`` 的唯一依据)。
只有声线记录还不够:真正下发给 H3 的是声线的克隆样本文件,没有文件时
``reference_voice_audios`` 会返回空,``<Audio n>`` 与预检告警都按「没绑」处理。
"""
import io
import wave
from app.constants import AssetKind
from app.models import VoiceAsset
from app.services import storage
by_name = {str(c.name or "").strip(): c for c in project.characters}
male, female = by_name["俊杰"], by_name["紫小慧"]
frames = io.BytesIO()
with wave.open(frames, "wb") as handle:
handle.setnchannels(1)
handle.setsampwidth(2)
handle.setframerate(16000)
handle.writeframes(b"\x20\x00" * 8000)
payload = frames.getvalue()
voices = []
for char, name, gender in ((male, "俊杰-低音", "male"), (female, "紫小慧-清亮", "female")):
voice = VoiceAsset(project_id=project.id, name=name, provider="comfyui",
provider_voice_id=f"v-{char.id}", gender=gender)
db.add(voice)
db.flush()
char.default_voice_id = voice.id
info = storage.save_bytes(payload, project_id=project.id, kind="audio",
stem=f"voice-{voice.id}", suffix=".wav")
storage.register_asset(db, project_id=project.id, owner_type="voice", owner_id=voice.id,
kind=AssetKind.REFERENCE, path=info["path"],
name=f"{name}克隆样本", info=info)
voices.append(voice)
db.flush()
return voices
def test_dialogue_voices_follow_the_order_of_lines(db):
"""参考音色的**顺序**就是 ``<Audio n>`` 的序号,必须由「谁先开口」决定。
旧实现只按 ``shot.speaker_character_id`` 收 1 条,再靠「台词文字里出现角色名」补人:
双人对白镜头(两条 lines、两个 character_id)通常只下发一条参考音频,
第二个人根本没有自己的 ``<Audio n>``,整段话由第一个人的嗓子念完。
"""
project, shot, _ = _case(db)
male_v, female_v = _voiced(db, project)
shot.set_lines([{"text": "电脑怎么了?", "character_id": shot.characters[1].id},
{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
db.flush()
got = P.dialogue_voices(db, project, shot)
assert [v.id for v in got] == [female_v.id, male_v.id], [v.name for v in got]
# 反过来开口:顺序跟着换,标签才不会把台词安到别人的嗓子上
shot.set_lines([{"text": "一键就能恢复。", "character_id": shot.characters[0].id},
{"text": "电脑怎么了?", "character_id": shot.characters[1].id}])
db.flush()
assert [v.id for v in P.dialogue_voices(db, project, shot)] == [male_v.id, female_v.id]
# 同一句话被两个人分着说也只会各收一条(去重按声线 id)
shot.set_lines([{"text": "别慌。", "character_id": shot.characters[0].id},
{"text": "我看看。", "character_id": shot.characters[0].id},
{"text": "好。", "character_id": shot.characters[1].id}])
db.flush()
assert [v.id for v in P.dialogue_voices(db, project, shot)] == [male_v.id, female_v.id]
def test_dialogue_voices_falls_back_only_for_unbound_speakers(db):
"""逐句全都绑好时不再塞镜头级声线;有人没绑才补,免得往参考池里丢无关音色。"""
from app.models import VoiceAsset
from app.services import audio
project, shot, _ = _case(db)
male_v, _female_v = _voiced(db, project)
generic = VoiceAsset(project_id=project.id, name="项目默认声线", provider="comfyui",
provider_voice_id="v-generic", gender="male", is_default=True)
db.add(generic)
db.flush()
shot.voice_asset_id = generic.id # 老数据:镜头上还固化着一条通用音色
shot.set_lines([{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
db.flush()
assert [v.id for v in P.dialogue_voices(db, project, shot)] == [male_v.id]
# 紫小慧没有默认声线 → 补上镜头级解析结果,别让台词完全失去音色参考
shot.set_lines([{"text": "一键就能恢复。", "character_id": shot.characters[0].id},
{"text": "电脑怎么了?", "character_id": shot.characters[1].id}])
project.characters[1].default_voice_id = None
db.flush()
got = [v.id for v in P.dialogue_voices(db, project, shot)]
assert got[0] == male_v.id and audio.resolve_voice_asset(db, project, shot).id in got
def test_joint_unvoiced_speakers_names_every_gap(db):
"""台词多但参考音色少时,必须点名「哪几个人的嗓子没绑上」——这是静默换声的来源。"""
project, shot, _ = _case(db)
male_v, female_v = _voiced(db, project)
shot.set_lines([{"text": "电脑怎么了?", "character_id": shot.characters[1].id},
{"text": "一键就能恢复。", "character_id": shot.characters[0].id}])
db.flush()
assert P.joint_unvoiced_speakers(db, project, shot, voices=[female_v, male_v]) == []
# 只下发了第一条:紫小慧有自己 <Audio 1>,俊杰那句裸奔
assert P.joint_unvoiced_speakers(db, project, shot, voices=[female_v]) == ["俊杰"]
# 一条都没有 / 声线没回指到角色(labels 全是空串)→ 两人都算未绑
assert P.joint_unvoiced_speakers(db, project, shot, voices=[]) == ["紫小慧", "俊杰"]
assert P.joint_unvoiced_speakers(db, project, shot, voices=[male_v]) == ["紫小慧"]
# 没台词的镜头不该报警
shot.set_lines([])
shot.dialogue = ""
db.flush()
assert P.joint_unvoiced_speakers(db, project, shot, voices=[]) == []
def test_preflight_warns_about_unbound_line_speaker(db, monkeypatch):
"""素材检查要为「逐句说话人没声线」出 warning:旧检查只看镜头级 speaker,多说话人漏网。"""
from app.providers.base import Capabilities
from app.services import preflight, timeline
project, shot, _ = _case(db)
_male_v, female_v = _voiced(db, project)
female = _char(project, "紫小慧")
shot.set_lines([{"text": "电脑怎么了?", "character_id": female.id},
{"text": "一键就能恢复。", "character_id": _char(project, "俊杰").id}])
female.default_voice_id = None # 只有紫小慧没绑声线,俊杰那条是齐的
db.flush()
monkeypatch.setattr(timeline, "native_voice_wanted", lambda *a, **k: True)
monkeypatch.setattr(preflight.model_registry, "capabilities_of",
lambda spec: Capabilities(modes=("r2v", "i2v", "t2v"),
resolutions=("720p",), aspect_ratios=("9:16",),
min_duration=2.0, max_duration=15.0,
max_reference_audios=3, joint_av=True))
issues = {i.code: i for i in preflight.check_shot(db, project, shot).issues}
assert "joint_speaker_unbound" in issues, set(issues)
assert "紫小慧" in issues["joint_speaker_unbound"].message, issues["joint_speaker_unbound"]
assert "俊杰" not in issues["joint_speaker_unbound"].message, issues["joint_speaker_unbound"]
assert issues["joint_speaker_unbound"].severity == "warning", "漏绑只是会换嗓子,不该挡住派发"
# 补上默认声线后告警消失
female.default_voice_id = female_v.id
db.flush()
codes = {i.code for i in preflight.check_shot(db, project, shot).issues}
assert "joint_speaker_unbound" not in codes, codes
def test_joint_duration_uses_measured_h3_speed(db):
"""台词时长必须按 H3 实测语速算,且**允许短于**分镜时长。
TTS 口径(``audio.CPS_CJK=4.3`` + 标点停顿 + 不低于 shot.duration)给 35 字台词
留了 9.42s,而 H3 实测 ≈5.0 字/秒 6.8s 就念完——多出来的 2.4s 空档被模型拿来
念提示词(shot 36 前 2.5s 乱语的直接成因)。对齐还要满足 H3 的帧数格点
``n % 17 == 5``@24fps,向下取格点。
"""
from types import SimpleNamespace
project, shot, _ = _case(db)
caps = SimpleNamespace(min_duration=5.0, max_duration=15.0)
voices = [SimpleNamespace(tuning={"speed": 1.0})]
# 「不要着急,恢复系统和数据一键就能完成。」= 19 字 → 19/5 + 0.25 ≈ 4.05s
short = P.joint_speech_duration_seconds(db, project, shot, caps,
voices=voices, base=8.0)
assert short == 5.0, short # 夹到 min_duration
shot.dialogue = "不要着急,我们公司使用的是紫光汇智的云桌面方案,恢复系统和数据一键就能完成。"
db.flush()
got = P.joint_speech_duration_seconds(db, project, shot, caps,
voices=voices, base=9.42)
assert 6.5 <= got <= 7.8, got # 35 字 ≈ 7.25s,明显短于 base
assert got < 9.42, "长于台词的空档会被模型拿来念提示词"
# 超上限时夹到 max,不能让台词被截断
shot.dialogue = "很长" * 60
db.flush()
assert P.joint_speech_duration_seconds(db, project, shot, caps,
voices=voices, base=8.0) == 15.0
def test_joint_ref_style_switch_keeps_legacy_form(db, monkeypatch):
"""留一条回退路:``JOINT_REF_STYLE="prefix"`` 要能还原旧的写法做 A/B。"""
project, shot, roles = _case(db)
monkeypatch.setattr(P, "JOINT_REF_STYLE", "prefix")
legacy = P.compose_prompt(project, shot, db=db, joint_av=True, roles=roles, duration_seconds=8)
assert legacy.startswith("<Picture 1>是这一镜的画面")
assert "<Picture 2>是俊杰" in legacy