394 lines
20 KiB
Python
394 lines
20 KiB
Python
"""画面质检(多模态看图)与整批复检。
|
||
|
||
技术指标满分 ≠ 画面没穿帮。这组用例锁住四条底线:
|
||
1. 模型报出的分身/鬼影/串装必须让镜头质检不通过(历史上七个镜头全 100 分放行);
|
||
2. 看图这条路坏了(模型不支持、调用报错、开关关闭)必须退回纯技术质检,不卡流程;
|
||
3. 只影响观感的小毛病(道具文字乱码)扣分提示,但不值得重跑几分钟的出图;
|
||
4. 质检规则升级后,已审核通过的镜头能被「整批复检」重新打开。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from app.config import settings
|
||
from app.db import SessionLocal
|
||
from app.models import ModelSpec, ReviewRecord, Shot
|
||
from app.providers import registry as model_registry
|
||
from app.providers.base import AssetGenResult, Capabilities, FatalProviderError, TextProviderAdapter
|
||
from app.services import quality, visual_qc
|
||
|
||
CALLS: list[dict] = []
|
||
|
||
|
||
class FakeVisionLLM(TextProviderAdapter):
|
||
"""会看图的测试模型:按 config.verdict 回结论,并记下收到的图片清单。"""
|
||
|
||
@property
|
||
def name(self) -> str:
|
||
return "fake_vision_llm"
|
||
|
||
def is_available(self, config: dict | None = None) -> bool:
|
||
return True
|
||
|
||
def capabilities(self, config: dict | None = None) -> Capabilities:
|
||
return Capabilities(modes=("chat", "prompt"), vision=True)
|
||
|
||
def generate(self, request, config: dict | None = None) -> AssetGenResult:
|
||
config = dict(config or {})
|
||
CALLS.append({"images": list((request.params or {}).get("images") or []),
|
||
"prompt": request.prompt,
|
||
"temperature": (request.params or {}).get("temperature")})
|
||
if config.get("boom"):
|
||
raise FatalProviderError("测试模型看图失败", code="test_boom")
|
||
return AssetGenResult(text=json.dumps(config.get("verdict") or {}, ensure_ascii=False),
|
||
metadata={"usage": {"total_tokens": 120}})
|
||
|
||
|
||
@pytest.fixture
|
||
def vision_llm(db):
|
||
"""装一个「支持看图」的默认大模型;结论由用例通过 config_json 控制。"""
|
||
model_registry.register_llm_adapter("fake_vision_llm", FakeVisionLLM())
|
||
CALLS.clear()
|
||
|
||
def install(verdict: dict | None = None, **config) -> ModelSpec:
|
||
db.query(ModelSpec).filter(ModelSpec.media_type == "llm").update({"is_default": False})
|
||
db.commit()
|
||
row = ModelSpec(provider="fake", model="vision-llm", display_name="测试看图模型",
|
||
media_type="llm", adapter="fake_vision_llm", enabled=True, is_default=True,
|
||
modes=["chat", "prompt"],
|
||
config_json={"vision": True, "verdict": verdict or {}, **config})
|
||
db.add(row)
|
||
db.commit()
|
||
return row
|
||
|
||
return install
|
||
|
||
|
||
@pytest.fixture
|
||
def generated(api, studio, script_text, make_image_png):
|
||
"""跑完「导入 → 生成 → 自动质检」,返回项目 id、镜头快照与素材信息。"""
|
||
pid = studio["project_id"]
|
||
api.post(f"/api/projects/{pid}/import",
|
||
data={"script_text": script_text, "default_duration": 2},
|
||
files=[("images", ("s01.png", make_image_png(480, 270, "0x226699"), "image/png")),
|
||
("images", ("s02.png", make_image_png(480, 270, "0x993355"), "image/png"))])
|
||
api.post(f"/api/projects/{pid}/tasks", json={"with_voiceover": True})
|
||
api.drain(pid)
|
||
shots = {row["order_index"]: row for row in api.get(f"/api/projects/{pid}/shots")["shots"]}
|
||
return pid, shots, studio
|
||
|
||
|
||
def verdict_of(issues, frames=(1, 2, 3, 4), people=1, name="林澈"):
|
||
return {"characters": [{"name": name, "frames": list(frames), "evidence": "银色短发机甲"}],
|
||
"people": people, "issues": issues, "summary": "自检结论"}
|
||
|
||
|
||
CLEAN = {"characters": [{"name": "林澈", "frames": [1, 2, 3, 4], "evidence": "在画"}],
|
||
"people": 1, "issues": [], "summary": "画面干净"}
|
||
|
||
|
||
def blend(tech: float, visual: float) -> float:
|
||
"""按设置里的权重折算总分(与 quality.check_shot 同一套算法)。"""
|
||
weight = settings.qc_visual_weight
|
||
return (1.0 - weight) * tech + weight * visual
|
||
|
||
|
||
def shot_status(api, pid, shot_id):
|
||
rows = api.get(f"/api/projects/{pid}/shots")["shots"]
|
||
return next(row["status"] for row in rows if row["id"] == shot_id)
|
||
|
||
|
||
def recheck(api, pid, shot_id):
|
||
"""手动质检一镜并等它跑完,返回落库后的质检报告。"""
|
||
body = api.post(f"/api/projects/{pid}/shots/{shot_id}/qc?drain=true")["result"]
|
||
return body["score"], body["qc"]
|
||
|
||
|
||
# ---------------------------------------------------------------- 链路与打分
|
||
def test_generation_without_vision_model_keeps_legacy_scores(generated):
|
||
"""默认模型不支持看图时,分值必须与纯技术质检的历史行为完全一致。"""
|
||
pid, shots, _ = generated
|
||
qc = shots[1]["qc"]
|
||
# 看图没跑 = 分值必须落回纯技术分,与画面质检上线前逐位一致
|
||
assert qc["passed"] is True and qc["score"] == qc["tech_score"] > 0
|
||
assert qc["probe"]["visual_skipped"] is True and qc["probe"]["visual_score"] is None
|
||
assert any(row["label"] == "画面质检·未执行" for row in qc["checks"])
|
||
assert qc["checks"] and all(row["weight"] == 0 for row in qc["checks"]
|
||
if row["key"].startswith("visual."))
|
||
|
||
|
||
def test_duplicate_person_makes_shot_fail(generated, vision_llm, api):
|
||
"""同框两个主角是硬伤:必须不通过,并给出能返工的原因。"""
|
||
pid, shots, _ = generated
|
||
vision_llm(verdict_of([{"type": "duplicate_person", "severity": "high", "frame": 2,
|
||
"subject": "林澈", "detail": "第二帧出现两个相同的林澈"}]))
|
||
score, qc = recheck(api, pid, shots[1]["id"])
|
||
assert qc["passed"] is False and shot_status(api, pid, shots[1]["id"]) == "generated"
|
||
assert any("人物分身" in item for item in qc["issues"])
|
||
assert qc["probe"]["visual_score"] == 70.0
|
||
assert score == pytest.approx(blend(qc["tech_score"], 70.0))
|
||
assert qc["probe"]["visual_frames"] and qc["probe"]["visual_model"] == "fake/vision-llm"
|
||
# 送模型的图片:镜头帧在前(帧号=序号)、定妆参考照在后并带标签
|
||
images = CALLS[-1]["images"]
|
||
assert images[0]["label"] == "镜头帧 第 1 帧"
|
||
assert "定妆参考照" in images[-1]["label"] and "林澈" in images[-1]["label"]
|
||
assert all(str(row.get("url", "")).startswith("data:image/jpeg;base64,") for row in images)
|
||
assert CALLS[-1]["temperature"] == 0 # 判定要稳定,不能自由发挥
|
||
assert "机甲" in CALLS[-1]["prompt"] or "银色短发" in CALLS[-1]["prompt"]
|
||
|
||
|
||
def test_clean_picture_still_passes(generated, vision_llm, api):
|
||
pid, shots, _ = generated
|
||
vision_llm(CLEAN)
|
||
score, qc = recheck(api, pid, shots[1]["id"])
|
||
assert qc["passed"] is True and score == pytest.approx(blend(qc["tech_score"], 100.0))
|
||
review = next(row for row in qc["checks"] if row["key"] == "visual.review")
|
||
assert review["passed"] is True and not qc["issues"]
|
||
|
||
|
||
def test_speaker_absent_is_a_hard_issue(generated, vision_llm, api):
|
||
pid, shots, _ = generated
|
||
vision_llm({"characters": [{"name": "林澈", "frames": [], "evidence": ""}], "people": 0,
|
||
"issues": [], "summary": "画里没人"})
|
||
score, qc = recheck(api, pid, shots[1]["id"])
|
||
assert qc["passed"] is False
|
||
assert any("说话人" in item for item in qc["issues"])
|
||
|
||
|
||
def test_vision_failure_does_not_block_pipeline(generated, vision_llm, api):
|
||
"""模型报错只记 warning:镜头仍按技术质检通过,不能被看图这一步卡死。"""
|
||
pid, shots, _ = generated
|
||
vision_llm({}, boom=True)
|
||
score, qc = recheck(api, pid, shots[1]["id"])
|
||
assert qc["passed"] is True and score == qc["tech_score"]
|
||
assert qc["probe"]["visual_score"] is None and qc["probe"]["visual_skipped"] is True
|
||
assert any(row["key"] == "visual.attempt" and row["severity"] == "warning" for row in qc["checks"])
|
||
|
||
|
||
def test_switch_off_restores_pure_technical_scores(generated, vision_llm, api, monkeypatch):
|
||
pid, shots, _ = generated
|
||
vision_llm(verdict_of([{"type": "ghost", "severity": "high", "frame": 3,
|
||
"subject": "林澈", "detail": "肩后半个透明人影"}]))
|
||
assert recheck(api, pid, shots[1]["id"])[1]["passed"] is False
|
||
monkeypatch.setattr(settings, "qc_visual_enabled", False)
|
||
score, qc = recheck(api, pid, shots[1]["id"])
|
||
assert qc["passed"] is True and score == qc["tech_score"] > 0
|
||
assert qc["probe"]["visual_skipped"] is True
|
||
|
||
|
||
def test_low_severity_issue_never_forces_rework(generated, vision_llm, api):
|
||
"""道具文字乱码属于「可以更好」:扣分提示,但不值得重跑几分钟的出图。"""
|
||
pid, shots, _ = generated
|
||
vision_llm(verdict_of([{"type": "garbled_text", "severity": "high", "frame": 2,
|
||
"subject": "扑克牌", "detail": "牌面花色是一团乱码"}]))
|
||
score, qc = recheck(api, pid, shots[1]["id"])
|
||
assert qc["passed"] is True and score == pytest.approx(blend(qc["tech_score"], 90.0))
|
||
assert any("文字乱码" in item for item in qc["warnings"]) and not qc["issues"]
|
||
|
||
|
||
def test_review_record_keeps_visual_conclusion(generated, vision_llm, api):
|
||
pid, shots, _ = generated
|
||
vision_llm(verdict_of([{"type": "camera_break", "severity": "high", "frame": 4,
|
||
"subject": "", "detail": "第四帧突然变成单人特写"}]))
|
||
recheck(api, pid, shots[1]["id"])
|
||
db = SessionLocal()
|
||
try:
|
||
record = (db.query(ReviewRecord).filter(ReviewRecord.shot_id == shots[1]["id"],
|
||
ReviewRecord.kind == "auto")
|
||
.order_by(ReviewRecord.id.desc()).first())
|
||
assert record is not None and record.verdict == "fail"
|
||
assert any("镜头画面瑕疵" in row["label"] for row in record.checks_json["checks"])
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
# ---------------------------------------------------------------- 整批复检
|
||
def test_batch_run_rechecks_approved_shots(generated, vision_llm, api):
|
||
"""已审核通过的镜头也要能被整批复检——否则新规则永远管不到历史项目。"""
|
||
pid, shots, _ = generated
|
||
for row in shots.values():
|
||
assert api.post(f"/api/projects/{pid}/shots/{row['id']}/approve",
|
||
json={"notes": "自检放行"})["result"]["shot"]["status"] == "approved"
|
||
vision_llm(verdict_of([{"type": "costume_crossover", "severity": "high", "frame": 1,
|
||
"subject": "林澈", "detail": "主角戴上别人的圆框眼镜"}]))
|
||
body = api.post(f"/api/projects/{pid}/qc/run", json={"include_approved": True})["result"]
|
||
assert len(body["task_ids"]) == len(shots)
|
||
assert all(row["passed"] is False for row in body["shots"])
|
||
assert all(row["visual_score"] is not None for row in body["shots"])
|
||
assert {row["status"] for row in body["qc"]["shots"]} == {"generated"}
|
||
assert body["qc"]["passed"] == 0 and body["qc"]["failed"] == len(shots)
|
||
|
||
|
||
def test_batch_run_rejects_project_without_products(api, studio):
|
||
body = api.post(f"/api/projects/{studio['project_id']}/qc/run", json={}, expect=400)
|
||
assert "没有可质检的镜头" in body["detail"]
|
||
|
||
|
||
# ---------------------------------------------------------------- 纯函数
|
||
def test_sample_points_keeps_first_frame_at_the_very_beginning():
|
||
from app.services import ffmpeg as ff
|
||
|
||
assert ff.sample_points(1) == [0.5]
|
||
points = ff.sample_points(4)
|
||
assert points[0] == 0.0 # 空镜、说话人缺席都发生在片头
|
||
assert points[-1] == pytest.approx(0.85)
|
||
assert points == sorted(points)
|
||
assert len(ff.sample_points(8)) == 8
|
||
|
||
|
||
def test_extract_frames_produces_real_jpegs(generated, media_root):
|
||
"""真抽帧:抽出来的必须是能解码的 jpg,且落在项目 qc 目录里。"""
|
||
from app.services import ffmpeg as ff
|
||
|
||
pid, shots, _ = generated
|
||
if not ff.available():
|
||
pytest.skip("本机没有 ffmpeg")
|
||
from app.services import storage
|
||
|
||
target = storage.to_absolute(shots[1]["output_path"])
|
||
out = ff.extract_frames(target, media_root / "frames", count=3, width=320)
|
||
assert len(out) == 3
|
||
for frame in out:
|
||
assert frame.exists() and frame.read_bytes()[:3] == b"\xff\xd8\xff"
|
||
|
||
|
||
def test_image_payload_labels_reference_photos(tmp_path):
|
||
frames = [tmp_path / "f1.jpg", tmp_path / "f2.jpg"]
|
||
for row in frames:
|
||
row.write_bytes(b"\xff\xd8\xff\xd9")
|
||
refs = [{"name": "阿拉蕾", "kind": "portrait", "path": tmp_path / "r.jpg"}]
|
||
refs[0]["path"].write_bytes(b"\xff\xd8\xff\xd9")
|
||
payload = visual_qc.image_payload(frames, refs)
|
||
assert len(payload) == 3
|
||
assert payload[0]["label"] == "镜头帧 第 1 帧"
|
||
assert "定妆参考照" in payload[-1]["label"] and "阿拉蕾" in payload[-1]["label"]
|
||
|
||
|
||
def test_attach_images_handles_plain_and_labeled_images():
|
||
from app.providers.llm import _attach_images
|
||
|
||
body = {"messages": [{"role": "system", "content": "sys"},
|
||
{"role": "user", "content": "看下这几帧"}]}
|
||
out = _attach_images(body, ["http://a.jpg", {"url": "http://b.jpg", "label": "定妆参考照"}])
|
||
content = out["messages"][1]["content"]
|
||
assert out["messages"][0]["content"] == "sys" # 文本提示词位置不动
|
||
assert content[0] == {"type": "text", "text": "看下这几帧"}
|
||
assert content[1] == {"type": "image_url", "image_url": {"url": "http://a.jpg"}}
|
||
assert content[2] == {"type": "text", "text": "定妆参考照"}
|
||
body = {"messages": [{"role": "user", "content": [{"type": "text", "text": "x"}]}]}
|
||
out = _attach_images(body, [{"url": "u", "label": "L"}])
|
||
assert out["messages"][0]["content"][0] == {"type": "text", "text": "x"}
|
||
assert out["messages"][0]["content"][2]["image_url"]["url"] == "u"
|
||
assert _attach_images({"messages": []}, ["a"]) == {"messages": []}
|
||
|
||
|
||
def test_score_verdict_maps_penalties_and_locks_severity():
|
||
shot = Shot(id=1, project_id=1, order_index=1, code="s01", title="测试",
|
||
description="三人斗地主", dialogue="郭靖你先出", speaker_character_id=7)
|
||
cast = [{"id": 7, "name": "林澈", "gender": "男", "appearance": "银色短发", "speaker": True},
|
||
{"id": 8, "name": "小燕子", "gender": "女", "appearance": "红衣", "speaker": False}]
|
||
verdict = {"characters": [{"name": "林澈", "frames": [1, 2, 3, 4]}, {"name": "小燕子", "frames": [2]}],
|
||
"people": 4,
|
||
"issues": [{"type": "extra_people", "severity": "high", "frame": 1, "detail": "多出两人"},
|
||
{"type": "extra_people", "severity": "high", "frame": 3, "detail": "又多两人"},
|
||
{"type": "other", "severity": "high", "frame": 0, "detail": "背景有点脏"}]}
|
||
out = visual_qc.score_verdict(verdict, cast, shot, frames=4, urls=["u1"], model="fake/v")
|
||
assert out.completed is True
|
||
keys = {row["key"] for row in out.checks}
|
||
assert {"visual.review", "visual.extra_people", "visual.other"} <= keys
|
||
other = next(row for row in out.checks if row["key"] == "visual.other")
|
||
assert other["severity"] == "warning" # 「其它」无论模型判多高都只提示
|
||
extra = next(row for row in out.checks if row["key"] == "visual.extra_people")
|
||
assert extra["severity"] == "error" and "帧 1" in extra["value"]
|
||
assert out.hard_fail is True and 0 < out.score < 100
|
||
assert out.issues and out.frames == ["u1"]
|
||
assert any("人数超出设定" in text for text in out.issues)
|
||
|
||
|
||
def test_voiceover_shot_downgrades_absent_speaker_to_hint():
|
||
shot = Shot(id=2, project_id=1, order_index=2, code="s02", title="空镜",
|
||
description="客栈全景,郭靖画外音", dialogue="在下郭靖", speaker_character_id=7)
|
||
cast = [{"id": 7, "name": "郭靖", "gender": "男", "appearance": "", "speaker": True}]
|
||
out = visual_qc.score_verdict({"characters": [{"name": "郭靖", "frames": []}], "people": 0,
|
||
"issues": [], "summary": "空镜"},
|
||
cast, shot, frames=4, urls=[], model="m")
|
||
assert out.hard_fail is False
|
||
assert any(row["key"] == "visual.missing_speaker" and row["severity"] == "warning"
|
||
for row in out.checks)
|
||
|
||
|
||
def test_prompt_marks_reference_photos_outside_frame_numbers():
|
||
shot = Shot(id=1, project_id=1, order_index=1, code="s01", title="斗地主",
|
||
description="三人围坐打牌", dialogue="郭靖你先出", speaker_character_id=1)
|
||
cast = [{"id": 1, "name": "郭靖", "gender": "男", "appearance": "红色披风", "speaker": True}]
|
||
prompt = visual_qc.build_prompt(shot, cast, 4, [{"name": "郭靖", "kind": "portrait",
|
||
"path": Path("p.jpg")}])
|
||
assert "帧号就是它们的顺序" in prompt and "不参与帧号计数" in prompt
|
||
assert "郭靖" in prompt and "本镜说话人" in prompt
|
||
assert "duplicate_person" in prompt
|
||
|
||
|
||
def test_vision_ready_rejects_mock_and_unmarked_models(db):
|
||
from app.services import storage # noqa: F401 确保模块已加载
|
||
|
||
db.query(ModelSpec).filter(ModelSpec.media_type == "llm").delete()
|
||
db.commit()
|
||
ok, note = visual_qc.vision_ready(db)
|
||
assert ok is False and "Mock" in note
|
||
|
||
|
||
def _p9_like_cast_rows():
|
||
"""项目 9 的真实形状:镜头只绑了紫小慧,画面描述里却点名要俊杰。"""
|
||
from app.models import Character, Project
|
||
|
||
lead = Character(id=71, project_id=9, name="紫小慧", gender="female",
|
||
appearance="白色T恤、透肤黑丝")
|
||
extra = Character(id=72, project_id=9, name="俊杰", gender="male",
|
||
appearance="深蓝西装、领带")
|
||
nobody = Character(id=73, project_id=9, name="展昭", gender="male", appearance="红袍")
|
||
project = Project(id=9, name="自检-异步调度验证")
|
||
project.characters = [lead, extra, nobody]
|
||
shot = Shot(id=91, project_id=9, order_index=1, code="s01", title="求助",
|
||
description="紫小慧神色焦急地跑入画面,来到俊杰面前,双手扒着桌面",
|
||
dialogue="我的电脑坏了,上面有很重要的资料。", speaker_character_id=71)
|
||
shot.characters = [lead]
|
||
return project, shot, lead, extra
|
||
|
||
|
||
def test_script_mentioned_character_is_not_an_intruder(db):
|
||
"""剧本点名的人必须进设定名单:否则「人数超出设定」会把正常对手戏判成穿帮。"""
|
||
project, shot, lead, extra = _p9_like_cast_rows()
|
||
cast = visual_qc.expected_cast(db, project, shot)
|
||
by_name = {row["name"]: row for row in cast}
|
||
assert set(by_name) == {"紫小慧", "俊杰"}, cast # 没提到的展昭不能混进来
|
||
assert by_name["紫小慧"]["mentioned"] is False and by_name["紫小慧"]["speaker"] is True
|
||
assert by_name["俊杰"]["mentioned"] is True and by_name["俊杰"]["gender"] == "男"
|
||
assert by_name["俊杰"]["appearance"] == "深蓝西装、领带"
|
||
|
||
|
||
def test_prompt_tells_the_model_mentioned_people_are_script_required(db):
|
||
project, shot, _, _ = _p9_like_cast_rows()
|
||
cast = visual_qc.expected_cast(db, project, shot)
|
||
prompt = visual_qc.build_prompt(shot, cast, 4, [])
|
||
assert "同样是剧本要出场的人" in prompt and "俊杰" in prompt
|
||
assert "不得报 extra_people" in prompt
|
||
assert "出镜名单里的人物(共 1 位:紫小慧)" in prompt
|
||
|
||
|
||
def test_mentioned_character_absent_from_frames_is_not_a_rework():
|
||
"""提到的人可能只在台词里被谈起,没入画不能按「应出场人物缺席」返工。"""
|
||
shot = Shot(id=92, project_id=9, order_index=2, code="s02", title="柜台",
|
||
description="俊杰坐在服务台后", dialogue="别慌", speaker_character_id=72)
|
||
cast = [{"id": 72, "name": "俊杰", "gender": "男", "appearance": "深蓝西装", "speaker": True},
|
||
{"id": 71, "name": "紫小慧", "gender": "女", "appearance": "白T", "speaker": False,
|
||
"mentioned": True}]
|
||
out = visual_qc.score_verdict(
|
||
{"characters": [{"name": "俊杰", "frames": [1, 2, 3, 4]}, {"name": "紫小慧", "frames": []}],
|
||
"people": 2, "issues": [], "summary": "两人同框符合描述"},
|
||
cast, shot, frames=4, urls=[], model="m")
|
||
assert not any(row["key"] == "visual.missing_character" for row in out.checks)
|
||
assert out.hard_fail is False and out.score == 100
|