nex_animation/backend/tests/test_identity_source_of_tru...

425 lines
23 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""「资产明明加了却没生效」三处根因的回归用例。
对应端到端验收里的三个问题(项目 9《自检-异步调度验证》实测):
1. 定妆照进了模型,出片却还是描述卡上的衣着 —— 模型对文字的服从度高于参考图,
所以人物真的带上了定妆图/三视图时,衣着、发型、影棚摄影类文字必须让位;
2. 新加入的音色素材换了个寂寞 —— 平台声线母版常常只有「试听样本」没有 reference,
只认 reference 会让这条声线被自检当成「没有音色」而跳过,克隆参考还停在
别人的旧文件上(实测 25 岁女主拿到了「偏低的中国老年女声」);
3. 定妆照已经进了参考图,关键帧却还是旧衣着 —— 关键帧生成直接读 shot.prompt 里上一次
组装的缓存字符串,换定妆照之后没有任何东西会让这份缓存过期,于是文字永远盖过参考图
(i2v 的首帧决定全片外观,所以成片里人物一辈子换不了装);
4. 音画不同步 —— 原生音画同步模型的对口型语音被 TTS 整条盖掉,片段还按 TTS
长度定格补帧,于是「张嘴无声 / 闭嘴有声」。现在 voice_source 可控、逐段可见。
"""
from __future__ import annotations
import pytest
from app.models import Character, Project, Shot, VoiceAsset
from app.services import audio, prompt, storage, timeline
from tests.conftest import absolute
from tests.test_delivery import _make_wav
from tests.test_shot_preview import _make_video_with_audio
# ---------------------------------------------------------------- 定妆照优先
@pytest.fixture
def dressed_shot(api, studio, db):
"""一个绑定「有定妆图的人物」的镜头,人物描述卡写满与定妆照冲突的衣着。"""
pid = studio["project_id"]
char = db.get(Character, studio["character_id"])
char.appearance = "25 岁女性,白色胸下截断半截短袖 T 恤,蓝色字母印花,透肤黑丝袜"
sid = api.post(f"/api/projects/{pid}/shots",
json={"title": "面试", "description": "她推门走进会议室", "mode": "i2v",
"duration": 2, "dialogue": "我到了。"})["result"]["id"]
shot = db.get(Shot, sid)
shot.characters = [char]
db.commit()
return {"pid": pid, "sid": sid, "char_id": char.id}
def test_portrait_bound_prompt_lets_the_portrait_win(api, db, dressed_shot):
ctx = dressed_shot
project = db.get(Project, ctx["pid"])
shot = db.get(Shot, ctx["sid"])
assert prompt.portrait_bound_ids(db, shot) == {ctx["char_id"]}, "定妆图必须被识别为本镜头的身份参考"
text = prompt.compose_prompt(project, shot, db=db)
assert "定妆图" in text and "不得按文字另行改写衣着" in text
# 冲突的衣着/印花字样不再进入提示词,年龄这类不冲突信息保留
assert "T 恤" not in text and "黑丝袜" not in text and "字母印花" not in text
assert "25 岁女性" in text
def test_prompt_without_portrait_keeps_the_costume_words(api, db, dressed_shot, make_image_png):
ctx = dressed_shot
project = db.get(Project, ctx["pid"])
shot = db.get(Shot, ctx["sid"])
# 删掉定妆图 → 没有任何身份参考,此时文字描述是唯一依据,必须原样保留
for row in storage.assets_of(db, "human", ctx["char_id"], "portrait"):
db.delete(row)
db.commit()
assert prompt.portrait_bound_ids(db, shot) == set()
text = prompt.compose_prompt(project, shot, db=db)
assert "T 恤" in text and "黑丝袜" in text
def test_shot_detail_previews_the_prompt_that_actually_runs(api, db, dressed_shot):
"""镜头详情里的「组装提示词预览」必须与入模文本一致(曾经不带 db,预览是旧口径)。"""
detail = api.get(f"/api/projects/{dressed_shot['pid']}/shots/{dressed_shot['sid']}")["shot"]
assert "不得按文字另行改写衣着" in detail["assembled_prompt_preview"]
def test_keyframe_prompt_ignores_the_stale_assembled_cache(api, db, dressed_shot):
"""关键帧(i2v 首帧)必须现算提示词:曾经直接读 shot.prompt 里上次组装的缓存,
于是换定妆照之后重画的首帧仍带着描述卡的旧衣着,成片里人物永远换不了装。"""
from app.jobs import handlers
ctx = dressed_shot
project = db.get(Project, ctx["pid"])
shot = db.get(Shot, ctx["sid"])
project.style_guide = "胶片质感,冷色调"
# 模拟「绑定定妆照之前」组装并落库的旧缓存
shot.prompt = "出镜角色(保持造型一致):白色胸下截断半截短袖 T 恤,透肤黑丝袜"
db.commit()
keyframe = handlers._image_asset_prompt(db, project, shot, "shot", "keyframe", "")
assert "定妆图" in keyframe
assert "T 恤" not in keyframe and "黑丝袜" not in keyframe, "旧缓存里的衣着不能盖过定妆照"
assert keyframe.count(project.style_guide) == 1, "项目美术设定只出现一次"
# 人工写的提示词仍然最高优先,不被自动组装覆盖;
# 唯一允许追加的是「谁在镜头前张嘴」这句构图约束——i2v 的出声者是首帧决定的。
hand = handlers._image_asset_prompt(db, project, shot, "shot", "keyframe", "特写,雨夜")
assert hand.startswith("特写,雨夜") and "T 恤" not in hand and "黑丝袜" not in hand
assert "嘴部微张" in hand, "有台词镜头的首帧必须画出正在讲话的人,否则模型会自己找人出声"
# ---------------------------------------------------------------- 音色真正生效
def _voice(db, project_id, *, name, gender, description, sample=None, reference=None,
provider="comfyui", provider_voice_id=""):
row = VoiceAsset(project_id=project_id, name=name, provider=provider, gender=gender,
description=description, provider_voice_id=provider_voice_id)
db.add(row)
db.flush()
for kind, payload in (("reference", reference), ("voice_sample", sample)):
if payload is None:
continue
info = storage.save_bytes(payload, project_id=project_id, kind=kind,
stem=f"voice-{row.id}-{kind}", suffix=".wav")
storage.register_asset(db, project_id=project_id, owner_type="voice", owner_id=row.id,
kind=kind, path=info["path"], name=f"{name}的{kind}",
source="platform", info=info)
db.commit()
return row
def test_voice_sample_alone_can_decide_the_timbre(db, api, studio):
"""平台声线只有试听样本时也必须能用:否则「同名角色」退回陌生人的嗓子。"""
pid = studio["project_id"]
row = _voice(db, pid, name="紫小慧声音", gender="female", description="25 岁青年女声",
sample=_make_wav(1.0, 320.0))
assert audio.voice_reference_ready(db, row) is False # 还没对齐克隆参考
assert audio.current_voice_audio(db, row).endswith(".wav") # 但音色素材是有的
note = audio.repair_voice_reference(db, row, reason="用例")
db.commit()
assert "紫小慧声音" in note and row.provider_voice_id
assert audio.voice_reference_ready(db, row) is True
# 用户换了一段新的音色样本 → 参考音频必须跟着走,不能停在旧文件上
newer = storage.save_bytes(_make_wav(1.2, 420.0), project_id=pid, kind="voice_sample",
stem="voice-new-sample", suffix=".wav")
storage.register_asset(db, project_id=pid, owner_type="voice", owner_id=row.id,
kind="voice_sample", path=newer["path"], name="新音色样本",
source="upload", info=newer)
db.commit()
assert audio.repair_voice_reference(db, row) != ""
assert row.provider_voice_id == newer["path"]
def test_rank_voices_refuses_the_elderly_voice_for_a_25_year_old(db, api, studio):
"""性别相同不代表能用:年龄档必须参与打分,且宁可不配也不给错嗓子。"""
pid = studio["project_id"]
project = db.get(Project, pid)
elder = _voice(db, pid, name="林秀云", gender="female", description="偏低的中国老年女声",
reference=_make_wav(1.0, 180.0))
young = _voice(db, pid, name="紫小慧声音", gender="female", description="25 岁青年女声,清亮",
reference=_make_wav(1.0, 320.0))
char = db.get(Character, studio["character_id"])
char.name, char.gender = "紫小慧", "female"
char.appearance = "25 岁女性,职场新人"
char.default_voice_id = elder.id # 旧逻辑留下的错配
db.commit()
assert audio.character_age_band(db, char) == "young", "年龄档不能被绑错的声线反向定义"
ranked = audio.rank_voices(db, project, gender="female", age_band="young",
speaker_name="紫小慧")
assert ranked[0]["voice"].id == young.id
assert ranked[0]["score"] > 0 and ranked[1]["voice"].id == elder.id
assert any("年龄档不符" in note for note in ranked[1]["notes"])
# 自检要能把这个错配当成缺口报出来(而不是「已有声线,跳过」)
from app.services import agent
gaps = agent._identity_gaps(db, project)
assert any(gap["character"].id == char.id and "年龄" in gap["reason"] for gap in gaps["voice_gaps"])
def test_resolve_voice_marks_age_mismatch(db, api, studio):
pid = studio["project_id"]
project = db.get(Project, pid)
elder = _voice(db, pid, name="林秀云", gender="female", description="偏低的中国老年女声",
reference=_make_wav(1.0, 180.0))
char = db.get(Character, studio["character_id"])
char.gender = "female"
char.appearance = "25 岁女性"
char.default_voice_id = elder.id
sid = api.post(f"/api/projects/{pid}/shots",
json={"title": "开场", "description": "她抬头", "duration": 2,
"dialogue": "你好"})["result"]["id"]
shot = db.get(Shot, sid)
shot.characters = [char]
shot.speaker_character_id = char.id
db.commit()
_voice_row, info = audio.resolve_voice(db, project, shot)
assert info["gender"] == "female" and info["age_band"] == "young"
assert info["voice_age_band"] == "old" and info["age_mismatch"] is True
assert "老年声" in audio.voice_selection_hint(info)
# ---------------------------------------------------------------- 语音来源
@pytest.fixture
def native_output(api, db, studio):
"""一条产物自带原声、且带台词的镜头(模拟 MiniMax H3 音画同出)。"""
pid = studio["project_id"]
sid = api.post(f"/api/projects/{pid}/shots",
json={"title": "雾大", "description": "浓雾渡口", "mode": "i2v",
"duration": 2, "dialogue": "雾大,站稳再下船。"})["result"]["id"]
out = storage.project_dir(pid, "generated") / f"shot-native-{sid}.mp4"
_make_video_with_audio(out, seconds=3.0)
shot = db.get(Shot, sid)
shot.output_path = storage.to_relative(out)
shot.status = "approved"
db.commit()
return {"pid": pid, "sid": sid, "ms": int(round(3.0 * 1000))}
def _attach_voice(db, sid, seconds):
from tests.conftest import MEDIA_ROOT # noqa: F401 (确保媒体根目录已建)
shot = db.get(Shot, sid)
info = storage.save_bytes(_make_wav(seconds, 520.0), project_id=shot.project_id,
kind="voiceover", stem=f"shot-{sid}", suffix=".wav")
shot.voiceover_path = info["path"]
shot.voiceover_duration_ms = int(round(seconds * 1000))
db.commit()
return info["path"]
def test_auto_prefers_model_audio_but_rescues_shots_without_it(api, db, native_output):
"""auto:模型自带原声就用原声;产物没声音才回退配音,绝不留「张嘴无声」。
MiniMax H3 这类原生音画同步模型的对口型语音就是最准的配音,TTS 盖上去
才是错位的来源,所以 auto 的优先级从「配音优先」翻成了「原声优先」。
"""
pid, sid = native_output["pid"], native_output["sid"]
_attach_voice(db, sid, 5.0) # 配音远长于画面:一旦误用就会定格补帧
built = api.post(f"/api/projects/{pid}/timeline",
json={"only_approved": True, "settings": {"voice_source": "auto"}})["result"]
assert built["voice_source"] == "auto" and built["native_voice_shots"] == [sid]
tl = api.get(f"/api/projects/{pid}/timeline")["timeline"]
assert tl["segments"][0]["voice_source"] == "native"
assert [c["kind"] for c in tl["audio_clips"] if c["shot_id"] == sid] == ["voiceover"]
assert tl["audio_clips"][0]["native_audio"] is True
# 原声与画面等长:绝不按 TTS 长度拉长片段(那是「闭嘴还在说」的根因)
assert abs(tl["segments"][0]["duration_ms"] - native_output["ms"]) <= 120
clip = tl["audio_clips"][0]
assert clip["start_ms"] == tl["segments"][0]["start_ms"]
assert abs(clip["duration_ms"] - native_output["ms"]) <= 120
report = api.get(f"/api/projects/{pid}/timeline/report")["report"]
assert report["native_voice_shots"] == [sid]
assert any("原声" in i["message"] for i in report["issues"])
assert api.get(f"/api/projects/{pid}/shots/{sid}")["shot"]["voice_source"] == "native"
# 显式切到 dub → 仍然按平台配音走,配音长于画面时补帧(旧行为没有被改掉)
dubbed = api.patch(f"/api/projects/{pid}/timeline/settings",
json={"settings": {"voice_source": "dub"}})["result"]
assert dubbed["recompute"]["rebuilt"] is True
assert dubbed["timeline"]["segments"][0]["voice_source"] == "dub"
assert dubbed["timeline"]["segments"][0]["duration_ms"] > native_output["ms"]
assert dubbed["timeline"]["audio_clips"][0]["native_audio"] is False
assert dubbed["timeline"]["audio_clips"][0]["start_ms"] == \
dubbed["timeline"]["segments"][0]["start_ms"] + 160
# 产物本身不会出声(换成纯 i2v 之类)→ auto 必须回退配音,不能留哑巴台词
shot = db.get(Shot, sid)
out = storage.to_absolute(shot.output_path)
_make_video_with_audio(out, seconds=3.0, audio=False)
api.post(f"/api/projects/{pid}/timeline",
json={"only_approved": True, "settings": {"voice_source": "auto"}})
tl = api.get(f"/api/projects/{pid}/timeline")["timeline"]
assert tl["segments"][0]["voice_source"] == "dub"
assert [c["kind"] for c in tl["audio_clips"] if c["shot_id"] == sid] == ["voiceover"]
assert tl["audio_clips"][0]["native_audio"] is False
assert api.get(f"/api/projects/{pid}/shots/{sid}")["shot"]["voice_source"] == "dub"
def test_voice_source_native_drops_freeze_padding(api, db, native_output):
pid, sid = native_output["pid"], native_output["sid"]
_attach_voice(db, sid, 5.0) # 配音远长于画面 → dub 模式下必须定格补帧
# 出厂默认已是原声优先,这里显式要 dub,测的才是「补帧」这条旧路径
api.post(f"/api/projects/{pid}/timeline",
json={"only_approved": True, "settings": {"voice_source": "dub"}})
dubbed = api.get(f"/api/projects/{pid}/timeline")["timeline"]
assert dubbed["segments"][0]["duration_ms"] > native_output["ms"]
assert dubbed["segments"][0]["voice_source"] == "dub"
patched = api.patch(f"/api/projects/{pid}/timeline/settings",
json={"settings": {"voice_source": "native"}})["result"]
assert patched["recompute"]["rebuilt"] is True
native = patched["timeline"]
assert abs(native["segments"][0]["duration_ms"] - native_output["ms"]) <= 120
assert native["segments"][0]["voice_source"] == "native"
assert native["settings"]["native_voice_shots"] == [sid]
# 换语音来源会重排时间线,返回与生效的都必须是新版本,不能是旧的静默时间线
assert native["is_current"] is True
assert native["version"] == dubbed["version"] + 1
back = api.patch(f"/api/projects/{pid}/timeline/settings",
json={"settings": {"voice_source": "dub"}})["result"]["timeline"]
assert back["segments"][0]["voice_source"] == "dub"
assert back["audio_clips"][0]["start_ms"] == back["segments"][0]["start_ms"] + 160
def test_native_voice_is_not_mixed_twice(api, db, native_output):
"""keep_shot_audio 与「原声配音」同时开启时,原生总线必须在这些片段静音。"""
from app.services import compose
pid, sid = native_output["pid"], native_output["sid"]
shot = db.get(Shot, sid)
shot.voiceover_path = ""
db.commit()
api.post(f"/api/projects/{pid}/timeline",
json={"only_approved": True, "settings": {"keep_shot_audio": True}})
tl = api.get(f"/api/projects/{pid}/timeline")["timeline"]
timeline_row = db.query(timeline.Timeline).filter(timeline.Timeline.id == tl["id"]).one()
windows = compose.native_voice_windows(timeline_row, list(timeline_row.segments))
assert windows and windows[0][0] == pytest.approx(0.0, abs=0.01)
assert windows[0][1] == pytest.approx(native_output["ms"] / 1000.0, abs=0.2)
# ---------------------------------------------------------- 主键复用与脏素材
def test_rebuilt_voice_does_not_inherit_the_old_timbre(api, db, studio):
"""删掉旧声线后新建的同 id 声线,不能继承上一任的音色文件。
实测:项目 9 的「紫小慧声音」克隆参考指向「林秀云」的老年女声 wav,
因为上一任声线被删时素材留在库里,SQLite 复用主键后新声线直接认了它。
"""
from datetime import timedelta
from app.models import now_utc
pid = studio["project_id"]
elder = _voice(db, pid, name="林秀云", gender="female", description="偏低的中国老年女声",
reference=_make_wav(1.0, 180.0))
stale = storage.assets_of(db, "voice", elder.id)[0]
# 模拟「素材属于上一任声线」:它比现任出生还早
elder.created_at = now_utc() + timedelta(hours=1)
db.commit()
assert not storage.belongs_to_owner(db, "voice", elder.id, stale)
assert audio.current_voice_audio(db, elder) == "", "脏素材不能被当成这条声线的音色"
assert audio.voice_reference_ready(db, elder) is False
# 素材库列表也不该再展示它,否则用户看到的是「有声线、有素材、却配错嗓子」
listed = api.get(f"/api/projects/{pid}/assets",
params={"owner_type": "voice", "owner_id": elder.id})["assets"]
assert [row["id"] for row in listed] == []
detail = api.get(f"/api/projects/{pid}")["result"]["voices"]
assert [row["assets"] for row in detail if row["id"] == elder.id] == [[]]
def test_deleting_a_voice_takes_its_media_with_it(api, db, studio):
"""删除声线要连素材一起清掉,否则脏数据会一直等着下一个同 id 的受害者。"""
pid = studio["project_id"]
victim = _voice(db, pid, name="待删声线", gender="female", description="青年女声",
reference=_make_wav(1.0, 220.0))
paths = [row.path for row in storage.assets_of(db, "voice", victim.id)]
assert paths and all(storage.exists(path) for path in paths)
api.delete(f"/api/projects/{pid}/voices/{victim.id}")
assert storage.assets_of(db, "voice", victim.id) == []
assert not any(storage.exists(path) for path in paths), "媒体文件不能留在盘上"
def test_character_who_speaks_after_appearing_still_needs_a_voice(api, db, studio):
"""只站着不出声的配角,下一个镜头开口时不能逃过声线自检。
实测项目 9:俊杰在镜头 1 同框、镜头 2 才说台词,说话人清单和出镜清单共用
一个去重集合,导致他从来没被当成说话人 → 自检不报缺口 → 整条台词没有他自己的音色。
"""
from app.services import agent
pid = studio["project_id"]
project = db.get(Project, pid)
speaker = db.get(Character, studio["character_id"])
silent = Character(project_id=pid, name="俊杰", appearance="30 岁男性,工程师")
db.add(silent)
db.commit() # 不提交会占着写锁,后面走 API 的另一个会话直接 database is locked
extra = api.post(f"/api/projects/{pid}/shots",
json={"title": "同框", "description": "两人都在画面里", "mode": "i2v",
"duration": 2, "dialogue": ""})["result"]["id"]
first = api.post(f"/api/projects/{pid}/shots",
json={"title": "开口", "description": "他解释方案", "mode": "i2v",
"duration": 2, "dialogue": "重装系统就好,资料都在云上。"})["result"]["id"]
db.get(Shot, extra).characters = [speaker, silent]
second = db.get(Shot, first)
second.characters = [speaker, silent]
second.speaker_character_id = silent.id
db.commit()
db.expire_all() # 镜头是走 API 建的,重新读一次才看得见
ref = agent._referenced_identities(db, db.get(Project, pid))
assert [row.id for row in ref["speakers"]] == [silent.id]
assert silent.id in [row.id for row in ref["characters"]]
gaps = {(gap["character"].id, gap["reason"]) for gap in agent._identity_gaps(db, project)["voice_gaps"]}
assert (silent.id, "说话角色没有绑定声线") in gaps
def test_rebinding_the_speaker_rewrites_the_dialogue_line_in_the_prompt(api, db, studio):
"""分镜里改「说话人 / 台词 / 声线」,提示词必须跟着改。
实测踩过:把小燕子的台词从郭靖名下挪回小燕子,界面立刻对了,提示词里却还写着
「郭靖台词:「那我抢啦!」,说话音色:男声,中年声」——出片继续用错人的嗓子,
而用户在分镜里看不出任何异常。
"""
pid = studio["project_id"]
girl = api.post(f"/api/projects/{pid}/characters",
json={"name": "小燕子", "role": "女主", "gender": "female",
"appearance": "红衣少女,旗头小辫"})["result"]["id"]
girl_voice = _voice(db, pid, name="小燕子声线", gender="female", description="又快又亮的女青年声")
sid = api.post(f"/api/projects/{pid}/shots",
json={"title": "抢地主", "description": "她一拍桌子站起来", "mode": "t2v",
"duration": 2, "dialogue": "那我抢啦!",
"speaker_character_id": studio["character_id"],
"voice_asset_id": studio["voice_id"]})["result"]["id"]
assert "林澈台词" in api.get(f"/api/projects/{pid}/shots/{sid}")["shot"]["prompt"]
api.patch(f"/api/projects/{pid}/shots/{sid}",
json={"speaker_character_id": girl, "voice_asset_id": girl_voice.id})
prompt = api.get(f"/api/projects/{pid}/shots/{sid}")["shot"]["prompt"]
assert "小燕子台词:「那我抢啦!」" in prompt
assert "女青年" in prompt and "林澈台词" not in prompt, "旧说话人和旧音色不能粘在提示词里"
# 只改台词同样要重算
api.patch(f"/api/projects/{pid}/shots/{sid}", json={"dialogue": "皇阿玛来了也拦不住我!"})
prompt = api.get(f"/api/projects/{pid}/shots/{sid}")["shot"]["prompt"]
assert "皇阿玛来了也拦不住我" in prompt and "那我抢啦" not in prompt