nex_animation/backend/tests/test_visual_qc.py

394 lines
20 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""画面质检(多模态看图)与整批复检。
技术指标满分 ≠ 画面没穿帮。这组用例锁住四条底线:
1. 模型报出的分身/鬼影/串装必须让镜头质检不通过(历史上七个镜头全 100 分放行);
2. 看图这条路坏了(模型不支持、调用报错、开关关闭)必须退回纯技术质检,不卡流程;
3. 只影响观感的小毛病(道具文字乱码)扣分提示,但不值得重跑几分钟的出图;
4. 质检规则升级后,已审核通过的镜头能被「整批复检」重新打开。
"""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from app.config import settings
from app.db import SessionLocal
from app.models import ModelSpec, ReviewRecord, Shot
from app.providers import registry as model_registry
from app.providers.base import AssetGenResult, Capabilities, FatalProviderError, TextProviderAdapter
from app.services import quality, visual_qc
CALLS: list[dict] = []
class FakeVisionLLM(TextProviderAdapter):
"""会看图的测试模型:按 config.verdict 回结论,并记下收到的图片清单。"""
@property
def name(self) -> str:
return "fake_vision_llm"
def is_available(self, config: dict | None = None) -> bool:
return True
def capabilities(self, config: dict | None = None) -> Capabilities:
return Capabilities(modes=("chat", "prompt"), vision=True)
def generate(self, request, config: dict | None = None) -> AssetGenResult:
config = dict(config or {})
CALLS.append({"images": list((request.params or {}).get("images") or []),
"prompt": request.prompt,
"temperature": (request.params or {}).get("temperature")})
if config.get("boom"):
raise FatalProviderError("测试模型看图失败", code="test_boom")
return AssetGenResult(text=json.dumps(config.get("verdict") or {}, ensure_ascii=False),
metadata={"usage": {"total_tokens": 120}})
@pytest.fixture
def vision_llm(db):
"""装一个「支持看图」的默认大模型;结论由用例通过 config_json 控制。"""
model_registry.register_llm_adapter("fake_vision_llm", FakeVisionLLM())
CALLS.clear()
def install(verdict: dict | None = None, **config) -> ModelSpec:
db.query(ModelSpec).filter(ModelSpec.media_type == "llm").update({"is_default": False})
db.commit()
row = ModelSpec(provider="fake", model="vision-llm", display_name="测试看图模型",
media_type="llm", adapter="fake_vision_llm", enabled=True, is_default=True,
modes=["chat", "prompt"],
config_json={"vision": True, "verdict": verdict or {}, **config})
db.add(row)
db.commit()
return row
return install
@pytest.fixture
def generated(api, studio, script_text, make_image_png):
"""跑完「导入 → 生成 → 自动质检」,返回项目 id、镜头快照与素材信息。"""
pid = studio["project_id"]
api.post(f"/api/projects/{pid}/import",
data={"script_text": script_text, "default_duration": 2},
files=[("images", ("s01.png", make_image_png(480, 270, "0x226699"), "image/png")),
("images", ("s02.png", make_image_png(480, 270, "0x993355"), "image/png"))])
api.post(f"/api/projects/{pid}/tasks", json={"with_voiceover": True})
api.drain(pid)
shots = {row["order_index"]: row for row in api.get(f"/api/projects/{pid}/shots")["shots"]}
return pid, shots, studio
def verdict_of(issues, frames=(1, 2, 3, 4), people=1, name="林澈"):
return {"characters": [{"name": name, "frames": list(frames), "evidence": "银色短发机甲"}],
"people": people, "issues": issues, "summary": "自检结论"}
CLEAN = {"characters": [{"name": "林澈", "frames": [1, 2, 3, 4], "evidence": "在画"}],
"people": 1, "issues": [], "summary": "画面干净"}
def blend(tech: float, visual: float) -> float:
"""按设置里的权重折算总分(与 quality.check_shot 同一套算法)。"""
weight = settings.qc_visual_weight
return (1.0 - weight) * tech + weight * visual
def shot_status(api, pid, shot_id):
rows = api.get(f"/api/projects/{pid}/shots")["shots"]
return next(row["status"] for row in rows if row["id"] == shot_id)
def recheck(api, pid, shot_id):
"""手动质检一镜并等它跑完,返回落库后的质检报告。"""
body = api.post(f"/api/projects/{pid}/shots/{shot_id}/qc?drain=true")["result"]
return body["score"], body["qc"]
# ---------------------------------------------------------------- 链路与打分
def test_generation_without_vision_model_keeps_legacy_scores(generated):
"""默认模型不支持看图时,分值必须与纯技术质检的历史行为完全一致。"""
pid, shots, _ = generated
qc = shots[1]["qc"]
# 看图没跑 = 分值必须落回纯技术分,与画面质检上线前逐位一致
assert qc["passed"] is True and qc["score"] == qc["tech_score"] > 0
assert qc["probe"]["visual_skipped"] is True and qc["probe"]["visual_score"] is None
assert any(row["label"] == "画面质检·未执行" for row in qc["checks"])
assert qc["checks"] and all(row["weight"] == 0 for row in qc["checks"]
if row["key"].startswith("visual."))
def test_duplicate_person_makes_shot_fail(generated, vision_llm, api):
"""同框两个主角是硬伤:必须不通过,并给出能返工的原因。"""
pid, shots, _ = generated
vision_llm(verdict_of([{"type": "duplicate_person", "severity": "high", "frame": 2,
"subject": "林澈", "detail": "第二帧出现两个相同的林澈"}]))
score, qc = recheck(api, pid, shots[1]["id"])
assert qc["passed"] is False and shot_status(api, pid, shots[1]["id"]) == "generated"
assert any("人物分身" in item for item in qc["issues"])
assert qc["probe"]["visual_score"] == 70.0
assert score == pytest.approx(blend(qc["tech_score"], 70.0))
assert qc["probe"]["visual_frames"] and qc["probe"]["visual_model"] == "fake/vision-llm"
# 送模型的图片:镜头帧在前(帧号=序号)、定妆参考照在后并带标签
images = CALLS[-1]["images"]
assert images[0]["label"] == "镜头帧 第 1 帧"
assert "定妆参考照" in images[-1]["label"] and "林澈" in images[-1]["label"]
assert all(str(row.get("url", "")).startswith("data:image/jpeg;base64,") for row in images)
assert CALLS[-1]["temperature"] == 0 # 判定要稳定,不能自由发挥
assert "机甲" in CALLS[-1]["prompt"] or "银色短发" in CALLS[-1]["prompt"]
def test_clean_picture_still_passes(generated, vision_llm, api):
pid, shots, _ = generated
vision_llm(CLEAN)
score, qc = recheck(api, pid, shots[1]["id"])
assert qc["passed"] is True and score == pytest.approx(blend(qc["tech_score"], 100.0))
review = next(row for row in qc["checks"] if row["key"] == "visual.review")
assert review["passed"] is True and not qc["issues"]
def test_speaker_absent_is_a_hard_issue(generated, vision_llm, api):
pid, shots, _ = generated
vision_llm({"characters": [{"name": "林澈", "frames": [], "evidence": ""}], "people": 0,
"issues": [], "summary": "画里没人"})
score, qc = recheck(api, pid, shots[1]["id"])
assert qc["passed"] is False
assert any("说话人" in item for item in qc["issues"])
def test_vision_failure_does_not_block_pipeline(generated, vision_llm, api):
"""模型报错只记 warning:镜头仍按技术质检通过,不能被看图这一步卡死。"""
pid, shots, _ = generated
vision_llm({}, boom=True)
score, qc = recheck(api, pid, shots[1]["id"])
assert qc["passed"] is True and score == qc["tech_score"]
assert qc["probe"]["visual_score"] is None and qc["probe"]["visual_skipped"] is True
assert any(row["key"] == "visual.attempt" and row["severity"] == "warning" for row in qc["checks"])
def test_switch_off_restores_pure_technical_scores(generated, vision_llm, api, monkeypatch):
pid, shots, _ = generated
vision_llm(verdict_of([{"type": "ghost", "severity": "high", "frame": 3,
"subject": "林澈", "detail": "肩后半个透明人影"}]))
assert recheck(api, pid, shots[1]["id"])[1]["passed"] is False
monkeypatch.setattr(settings, "qc_visual_enabled", False)
score, qc = recheck(api, pid, shots[1]["id"])
assert qc["passed"] is True and score == qc["tech_score"] > 0
assert qc["probe"]["visual_skipped"] is True
def test_low_severity_issue_never_forces_rework(generated, vision_llm, api):
"""道具文字乱码属于「可以更好」:扣分提示,但不值得重跑几分钟的出图。"""
pid, shots, _ = generated
vision_llm(verdict_of([{"type": "garbled_text", "severity": "high", "frame": 2,
"subject": "扑克牌", "detail": "牌面花色是一团乱码"}]))
score, qc = recheck(api, pid, shots[1]["id"])
assert qc["passed"] is True and score == pytest.approx(blend(qc["tech_score"], 90.0))
assert any("文字乱码" in item for item in qc["warnings"]) and not qc["issues"]
def test_review_record_keeps_visual_conclusion(generated, vision_llm, api):
pid, shots, _ = generated
vision_llm(verdict_of([{"type": "camera_break", "severity": "high", "frame": 4,
"subject": "", "detail": "第四帧突然变成单人特写"}]))
recheck(api, pid, shots[1]["id"])
db = SessionLocal()
try:
record = (db.query(ReviewRecord).filter(ReviewRecord.shot_id == shots[1]["id"],
ReviewRecord.kind == "auto")
.order_by(ReviewRecord.id.desc()).first())
assert record is not None and record.verdict == "fail"
assert any("镜头画面瑕疵" in row["label"] for row in record.checks_json["checks"])
finally:
db.close()
# ---------------------------------------------------------------- 整批复检
def test_batch_run_rechecks_approved_shots(generated, vision_llm, api):
"""已审核通过的镜头也要能被整批复检——否则新规则永远管不到历史项目。"""
pid, shots, _ = generated
for row in shots.values():
assert api.post(f"/api/projects/{pid}/shots/{row['id']}/approve",
json={"notes": "自检放行"})["result"]["shot"]["status"] == "approved"
vision_llm(verdict_of([{"type": "costume_crossover", "severity": "high", "frame": 1,
"subject": "林澈", "detail": "主角戴上别人的圆框眼镜"}]))
body = api.post(f"/api/projects/{pid}/qc/run", json={"include_approved": True})["result"]
assert len(body["task_ids"]) == len(shots)
assert all(row["passed"] is False for row in body["shots"])
assert all(row["visual_score"] is not None for row in body["shots"])
assert {row["status"] for row in body["qc"]["shots"]} == {"generated"}
assert body["qc"]["passed"] == 0 and body["qc"]["failed"] == len(shots)
def test_batch_run_rejects_project_without_products(api, studio):
body = api.post(f"/api/projects/{studio['project_id']}/qc/run", json={}, expect=400)
assert "没有可质检的镜头" in body["detail"]
# ---------------------------------------------------------------- 纯函数
def test_sample_points_keeps_first_frame_at_the_very_beginning():
from app.services import ffmpeg as ff
assert ff.sample_points(1) == [0.5]
points = ff.sample_points(4)
assert points[0] == 0.0 # 空镜、说话人缺席都发生在片头
assert points[-1] == pytest.approx(0.85)
assert points == sorted(points)
assert len(ff.sample_points(8)) == 8
def test_extract_frames_produces_real_jpegs(generated, media_root):
"""真抽帧:抽出来的必须是能解码的 jpg,且落在项目 qc 目录里。"""
from app.services import ffmpeg as ff
pid, shots, _ = generated
if not ff.available():
pytest.skip("本机没有 ffmpeg")
from app.services import storage
target = storage.to_absolute(shots[1]["output_path"])
out = ff.extract_frames(target, media_root / "frames", count=3, width=320)
assert len(out) == 3
for frame in out:
assert frame.exists() and frame.read_bytes()[:3] == b"\xff\xd8\xff"
def test_image_payload_labels_reference_photos(tmp_path):
frames = [tmp_path / "f1.jpg", tmp_path / "f2.jpg"]
for row in frames:
row.write_bytes(b"\xff\xd8\xff\xd9")
refs = [{"name": "阿拉蕾", "kind": "portrait", "path": tmp_path / "r.jpg"}]
refs[0]["path"].write_bytes(b"\xff\xd8\xff\xd9")
payload = visual_qc.image_payload(frames, refs)
assert len(payload) == 3
assert payload[0]["label"] == "镜头帧 第 1 帧"
assert "定妆参考照" in payload[-1]["label"] and "阿拉蕾" in payload[-1]["label"]
def test_attach_images_handles_plain_and_labeled_images():
from app.providers.llm import _attach_images
body = {"messages": [{"role": "system", "content": "sys"},
{"role": "user", "content": "看下这几帧"}]}
out = _attach_images(body, ["http://a.jpg", {"url": "http://b.jpg", "label": "定妆参考照"}])
content = out["messages"][1]["content"]
assert out["messages"][0]["content"] == "sys" # 文本提示词位置不动
assert content[0] == {"type": "text", "text": "看下这几帧"}
assert content[1] == {"type": "image_url", "image_url": {"url": "http://a.jpg"}}
assert content[2] == {"type": "text", "text": "定妆参考照"}
body = {"messages": [{"role": "user", "content": [{"type": "text", "text": "x"}]}]}
out = _attach_images(body, [{"url": "u", "label": "L"}])
assert out["messages"][0]["content"][0] == {"type": "text", "text": "x"}
assert out["messages"][0]["content"][2]["image_url"]["url"] == "u"
assert _attach_images({"messages": []}, ["a"]) == {"messages": []}
def test_score_verdict_maps_penalties_and_locks_severity():
shot = Shot(id=1, project_id=1, order_index=1, code="s01", title="测试",
description="三人斗地主", dialogue="郭靖你先出", speaker_character_id=7)
cast = [{"id": 7, "name": "林澈", "gender": "男", "appearance": "银色短发", "speaker": True},
{"id": 8, "name": "小燕子", "gender": "女", "appearance": "红衣", "speaker": False}]
verdict = {"characters": [{"name": "林澈", "frames": [1, 2, 3, 4]}, {"name": "小燕子", "frames": [2]}],
"people": 4,
"issues": [{"type": "extra_people", "severity": "high", "frame": 1, "detail": "多出两人"},
{"type": "extra_people", "severity": "high", "frame": 3, "detail": "又多两人"},
{"type": "other", "severity": "high", "frame": 0, "detail": "背景有点脏"}]}
out = visual_qc.score_verdict(verdict, cast, shot, frames=4, urls=["u1"], model="fake/v")
assert out.completed is True
keys = {row["key"] for row in out.checks}
assert {"visual.review", "visual.extra_people", "visual.other"} <= keys
other = next(row for row in out.checks if row["key"] == "visual.other")
assert other["severity"] == "warning" # 「其它」无论模型判多高都只提示
extra = next(row for row in out.checks if row["key"] == "visual.extra_people")
assert extra["severity"] == "error" and "帧 1" in extra["value"]
assert out.hard_fail is True and 0 < out.score < 100
assert out.issues and out.frames == ["u1"]
assert any("人数超出设定" in text for text in out.issues)
def test_voiceover_shot_downgrades_absent_speaker_to_hint():
shot = Shot(id=2, project_id=1, order_index=2, code="s02", title="空镜",
description="客栈全景,郭靖画外音", dialogue="在下郭靖", speaker_character_id=7)
cast = [{"id": 7, "name": "郭靖", "gender": "男", "appearance": "", "speaker": True}]
out = visual_qc.score_verdict({"characters": [{"name": "郭靖", "frames": []}], "people": 0,
"issues": [], "summary": "空镜"},
cast, shot, frames=4, urls=[], model="m")
assert out.hard_fail is False
assert any(row["key"] == "visual.missing_speaker" and row["severity"] == "warning"
for row in out.checks)
def test_prompt_marks_reference_photos_outside_frame_numbers():
shot = Shot(id=1, project_id=1, order_index=1, code="s01", title="斗地主",
description="三人围坐打牌", dialogue="郭靖你先出", speaker_character_id=1)
cast = [{"id": 1, "name": "郭靖", "gender": "男", "appearance": "红色披风", "speaker": True}]
prompt = visual_qc.build_prompt(shot, cast, 4, [{"name": "郭靖", "kind": "portrait",
"path": Path("p.jpg")}])
assert "帧号就是它们的顺序" in prompt and "不参与帧号计数" in prompt
assert "郭靖" in prompt and "本镜说话人" in prompt
assert "duplicate_person" in prompt
def test_vision_ready_rejects_mock_and_unmarked_models(db):
from app.services import storage # noqa: F401 确保模块已加载
db.query(ModelSpec).filter(ModelSpec.media_type == "llm").delete()
db.commit()
ok, note = visual_qc.vision_ready(db)
assert ok is False and "Mock" in note
def _p9_like_cast_rows():
"""项目 9 的真实形状:镜头只绑了紫小慧,画面描述里却点名要俊杰。"""
from app.models import Character, Project
lead = Character(id=71, project_id=9, name="紫小慧", gender="female",
appearance="白色T恤、透肤黑丝")
extra = Character(id=72, project_id=9, name="俊杰", gender="male",
appearance="深蓝西装、领带")
nobody = Character(id=73, project_id=9, name="展昭", gender="male", appearance="红袍")
project = Project(id=9, name="自检-异步调度验证")
project.characters = [lead, extra, nobody]
shot = Shot(id=91, project_id=9, order_index=1, code="s01", title="求助",
description="紫小慧神色焦急地跑入画面,来到俊杰面前,双手扒着桌面",
dialogue="我的电脑坏了,上面有很重要的资料。", speaker_character_id=71)
shot.characters = [lead]
return project, shot, lead, extra
def test_script_mentioned_character_is_not_an_intruder(db):
"""剧本点名的人必须进设定名单:否则「人数超出设定」会把正常对手戏判成穿帮。"""
project, shot, lead, extra = _p9_like_cast_rows()
cast = visual_qc.expected_cast(db, project, shot)
by_name = {row["name"]: row for row in cast}
assert set(by_name) == {"紫小慧", "俊杰"}, cast # 没提到的展昭不能混进来
assert by_name["紫小慧"]["mentioned"] is False and by_name["紫小慧"]["speaker"] is True
assert by_name["俊杰"]["mentioned"] is True and by_name["俊杰"]["gender"] == "男"
assert by_name["俊杰"]["appearance"] == "深蓝西装、领带"
def test_prompt_tells_the_model_mentioned_people_are_script_required(db):
project, shot, _, _ = _p9_like_cast_rows()
cast = visual_qc.expected_cast(db, project, shot)
prompt = visual_qc.build_prompt(shot, cast, 4, [])
assert "同样是剧本要出场的人" in prompt and "俊杰" in prompt
assert "不得报 extra_people" in prompt
assert "出镜名单里的人物(共 1 位:紫小慧)" in prompt
def test_mentioned_character_absent_from_frames_is_not_a_rework():
"""提到的人可能只在台词里被谈起,没入画不能按「应出场人物缺席」返工。"""
shot = Shot(id=92, project_id=9, order_index=2, code="s02", title="柜台",
description="俊杰坐在服务台后", dialogue="别慌", speaker_character_id=72)
cast = [{"id": 72, "name": "俊杰", "gender": "男", "appearance": "深蓝西装", "speaker": True},
{"id": 71, "name": "紫小慧", "gender": "女", "appearance": "白T", "speaker": False,
"mentioned": True}]
out = visual_qc.score_verdict(
{"characters": [{"name": "俊杰", "frames": [1, 2, 3, 4]}, {"name": "紫小慧", "frames": []}],
"people": 2, "issues": [], "summary": "两人同框符合描述"},
cast, shot, frames=4, urls=[], model="m")
assert not any(row["key"] == "visual.missing_character" for row in out.checks)
assert out.hard_fail is False and out.score == 100