nex_animation/backend/app/services/visual_qc.py

472 lines
25 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""画面质检(视觉 QC):抽帧 + 会看图的大模型复核。
技术指标(quality.py)只能证明「文件是好的」,证明不了「画面是对的」:
人物分身、半透明鬼影、该出场的人不在画里、阿拉蕾的眼镜跑到小燕子脸上、
扑克牌上的花色是一团乱码——这些观众一眼就能看出的穿帮,在 ffmpeg 眼里
全是满分(历史上七个镜头整批 100 分放行,成片却满是瑕疵)。
本模块把镜头按时间点抽成若干帧,连同人物**当前定妆照**与「这一镜本该出现谁」
的设定一起交给多模态大模型,让它按固定问题词表回报穿帮,再折算成扣分项。
失败策略:视觉维度拿不到结论时一律**不阻断**流程(记 warning、按满分处理),
整体退化成纯技术质检,避免模型抖动或供应商不支持看图把整条流水线卡死。
"""
from __future__ import annotations
import base64
import json
import logging
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from sqlalchemy.orm import Session
from app.config import settings
from app.models import Project, Shot
from app.providers import registry
from app.services import ffmpeg as ff
from app.services import prompt as prompt_service
from app.services import storage
log = logging.getLogger(__name__)
# 固定问题词表:label 给用户看,penalty 是扣分,severity 决定是否必须返工。
# 扣分只作用于「视觉分」,不动技术质检的权重表(那套权重被大量自检用例锚定)。
# lock="warning" 表示无论模型怎么判都只提示不返工:道具文字、手指细节这类
# 属于「可以更好」,为它重跑一次几分钟的出图不划算。
ISSUE_TYPES: dict[str, tuple[str, int, str, str]] = {
"duplicate_person": ("人物分身", 30, "high", ""),
"ghost": ("半透明鬼影", 25, "high", ""),
"extra_people": ("人数超出设定", 25, "high", ""),
"costume_crossover": ("服装道具串人", 20, "high", ""),
"identity_drift": ("人物形象走样", 20, "high", ""),
"missing_character": ("应出场人物缺席", 20, "high", ""),
"camera_break": ("镜头画面瑕疵", 15, "high", ""),
"missing_speaker": ("说话人不在画面", 15, "high", ""),
"deformation": ("五官肢体畸变", 10, "low", ""),
"garbled_text": ("画面文字乱码", 10, "low", "warning"),
"other": ("其它画面问题", 5, "low", "warning"),
}
# 台词写在设定里但人不露脸是正常拍法(画外音),不能算穿帮
VOICEOVER_HINTS = ("画外", "旁白", "OS:", "(OS)", "os:", "背对镜头", "不露脸", "背影")
@dataclass
class VisualOutcome:
"""一镜的视觉结论。checks 交给 quality.py 转成 Check 行。"""
score: float = 100.0
checks: list[dict] = field(default_factory=list)
frames: list[str] = field(default_factory=list) # 判定帧(页面要能给审核人看)
raw: dict = field(default_factory=dict) # 模型原始结论
model: str = ""
# 只有模型真的给出了结论,总分才允许掺入视觉分;跳过/失败时保持纯技术分,
# 免得「看图没跑成」被折算成扣分(或反过来把技术满分稀释成 100 以下)。
completed: bool = False
@property
def issues(self) -> list[str]:
return [f"{c['label']}:{c['detail']}" for c in self.checks
if not c["passed"] and c["severity"] == "error"]
@property
def hard_fail(self) -> bool:
return any(not c["passed"] and c["severity"] == "error" for c in self.checks)
def vision_ready(db: Session) -> tuple[bool, str]:
"""当前默认大模型能不能看图(Mock / 纯文本模型一律 False)。"""
if not settings.qc_visual_enabled:
return False, "画面质检开关已关闭"
if not ff.available():
return False, "本机缺少 ffmpeg,无法抽帧"
try:
spec, _adapter = registry.resolve_llm(db)
except Exception as exc: # 解析失败不应打挂质检
return False, f"大模型解析失败:{exc}"
if spec is None or not getattr(spec, "enabled", True):
return False, "没有启用的大模型"
provider = str(getattr(spec, "provider", "") or "").lower()
adapter = str(getattr(spec, "adapter", "") or "").lower()
model = str(getattr(spec, "model", "") or "").lower()
if provider == "mock" or adapter.startswith("mock") or model.startswith("mock-"):
return False, "当前默认大模型是 Mock,不参与画面质检"
config = dict(getattr(spec, "config_json", None) or {})
if not bool(config.get("vision", adapter in ("http_llm", "http_text"))):
return False, f"模型 {spec.provider}/{spec.model} 未声明支持看图"
return True, f"{spec.provider}/{spec.model}"
def _entry(char, *, speaker: bool, mentioned: bool = False) -> dict:
return {"id": char.id,
"name": char.name,
"gender": {"male": "男", "female": "女"}.get((char.gender or "").lower(), ""),
"appearance": (char.appearance or char.persona_prompt or "").strip()[:400],
"speaker": speaker,
"mentioned": mentioned}
def expected_cast(db: Session, project: Project, shot: Shot) -> list[dict]:
"""这一镜该按谁的设定来检查(含定妆设定),供模型逐人核对。
除了绑定到镜头的角色,**画面描述/台词里点名提到的项目角色**也算:
编导在分镜里写「来到俊杰面前」「俊杰坐在服务台后」,俊杰却是另一个角色、
没有挂到这一镜的出镜名单上——以前质检只按出镜名单数人,把剧本要求的人
判成「人数超出设定:设定仅1人,画面多出一名男子」,于是整镜被判返工,
返工提示词还让模型「只允许出现设定人物」,越改越错。
"""
cast: list[dict] = []
for char in list(shot.characters or []):
cast.append(_entry(char, speaker=int(shot.speaker_character_id or 0) == int(char.id)))
if not cast and shot.speaker_character_id:
speaker = next((c for c in (project.characters or []) if c.id == shot.speaker_character_id), None)
if speaker is not None:
cast.append(_entry(speaker, speaker=True))
known = {row["name"] for row in cast}
script = " ".join(filter(None, [shot.description, shot.synopsis, shot.title, shot.dialogue]))
for char in list(project.characters or []):
name = (char.name or "").strip()
if not name or name in known or name not in script:
continue
cast.append(_entry(char, speaker=int(shot.speaker_character_id or 0) == int(char.id),
mentioned=True))
known.add(name)
return cast
def reference_images(db: Session, cast: list[dict]) -> list[dict]:
"""人物当前定妆照:模型照着它认人,才抓得住「换脸 / 串装」。
取的是资产库里该人物的**当前版本**母版(定妆图优先,缺则退三视图),
与生成阶段喂给视频模型的参照物同源——定妆照没生效时,这里会立刻暴露。
"""
order = ("portrait", "turnaround", "costume", "expression")
out: list[dict] = []
for row in cast:
if len(out) >= max(0, settings.qc_visual_references):
break
for kind in order:
asset = storage.first_asset_row(db, "human", int(row["id"]), kind)
if asset is None:
continue
out.append({"name": row["name"], "kind": kind, "path": storage.to_absolute(asset.path)})
break
return out
def build_prompt(shot: Shot, cast: list[dict], frame_count: int, refs: list[dict]) -> str:
bound = [row for row in cast if not row.get("mentioned")]
extra = [row for row in cast if row.get("mentioned")]
ref_names = {str(row.get("name", "")).strip() for row in refs}
def has_photo(row: dict) -> bool:
return str(row["name"]).strip() in ref_names
def tag(row: dict) -> str:
"""有定妆照的人物:形象准绳是照片,不是那段可能过时的文字设定。"""
return "【已附定妆照,形象以照片为准】" if has_photo(row) else "【无定妆照】"
def setting(row: dict) -> str:
"""人物设定文本。已附定妆照时剥掉与照片打架的衣着/发型字样——
定妆照换过之后那段文字多半已经过时,留着它模型就会拿旧衣服判新片子穿帮。"""
text = str(row.get("appearance") or "").strip()
if not text:
return ""
return prompt_service.non_conflicting_traits(text) if has_photo(row) else text
names = "、".join(c["name"] for c in bound) or "(本镜未绑定人物)"
lines = [
"你是影视后期的画面质检员。随消息附上的图片分两段:"
f"前 {frame_count} 张是同一个镜头按时间顺序抽取的画面帧,"
f"**帧号就是它们的顺序(第 1 帧 = 镜头开头,第 {frame_count} 帧 = 镜头末尾)**;"
+ (f"最后 {len(refs)} 张是人物的定妆参考照,用来认人**并且是人物形象的唯一准绳**,"
"**不是镜头画面,不参与帧号计数**。"
if refs else "本次没有参考照。"),
"请只依据**镜头帧**的内容挑出穿帮与镜头瑕疵,不要评论剧情与美感。",
"",
"【这一镜的拍摄要求】",
f"- 画面描述:{(shot.description or shot.title or '').strip()[:600] or '(未填写)'}",
f"- 出镜名单里的人物(共 {len(bound)} 位:{names}):",
]
for idx, c in enumerate(bound, 1):
setting_text = setting(c)
desc = f":{setting_text}" if setting_text else ""
lines.append(f" {idx}. {c['name']}{('(' + c['gender'] + ')') if c['gender'] else ''}"
f"{'【本镜说话人】' if c['speaker'] else ''}{tag(c)}{desc}")
if extra:
lines.append(f"- 画面描述里还点名提到的人物(共 {len(extra)} 位,**同样是剧本要出场的人**,"
"只是没挂进本镜出镜名单,**不是多余人物**):")
for idx, c in enumerate(extra, 1):
setting_text = setting(c)
desc = f":{setting_text}" if setting_text else ""
lines.append(f" {idx}. {c['name']}{('(' + c['gender'] + ')') if c['gender'] else ''}"
f"{'【本镜说话人】' if c['speaker'] else ''}{tag(c)}{desc}")
dialogue = (shot.dialogue or "").strip()
if dialogue:
speaker = next((c["name"] for c in cast if c["speaker"]), "未指定")
lines.append(f"- 台词(由 {speaker} 说出):{dialogue[:200]}")
lines += [
"",
"【逐项检查,一项都别漏】",
"A 认人:`characters` 里逐个判断每位设定人物出现在**哪几帧**(frames 填帧号数组,没出现就留空数组)。",
"B 分身:同一个人是否在同一帧里同时出现两个及以上(同脸或同套服装的人重复出现)。",
"C 鬼影:是否有半透明、轮廓残缺、运动残留的多余人影或肢体。",
"D 人数:只有「既不在上面任何人物名单、画面描述里也没提到」的真人才算多余人物;"
"名单里的人同框是剧本要求,不得报 extra_people(镜子里的人像、墙上的画像、画中画不计入)。",
"E 串装:甲的服装/配饰跑到了乙身上(例:小燕子戴上了阿拉蕾的圆框眼镜)。",
"F 形象:出现的人与其**定妆参考照**是否明显不同(换了脸、换了发型服装、性别年龄不符);"
"标注【无定妆照】的人物才对照文字设定。",
"G 畸变:五官错位、手指数量不对、肢体断裂穿模、道具上的文字花色是乱码。",
"H 镜头:画中画、异常黑边、画面撕裂、涂抹糊块、同一镜头内机位无故跳变。",
"",
"【判定纪律】",
"- **人物形象(五官/发型/服装/配饰)的准绳是随附的该人物定妆参考照**:设定里那段文字"
"是建档时的旧描述,很可能已经被新的定妆照取代。画面与定妆照一致就是对的,"
"**绝不许**仅因「与文字设定里的衣着/发型不符」而报 identity_drift 或 costume_crossover。",
"- 只报**能在某一帧上指出证据**的问题;「风格略有不同」「氛围不够」这类主观感受不要报。",
"- 背景里合理多出的环境细节(远处的骆驼、屋外的树)不是问题,不要报。",
"- **构图裁切不是瑕疵**:近景、特写、过肩镜头里人物靠镜头的一侧被画框边缘切掉"
"(脸只露大半、手伸出画面、身体没拍全)是正常构图,不得因此报 distorted / extra_people "
"或「面部被裁切」;只有五官、肢体本身画错位、多指、穿模才算畸变。",
"- **一只手必须有主人**:过肩、近景镜头里同时看到两只手是正常的"
"(例如一手搭在键盘上、一手指向屏幕),不得仅因「画面里出现了两只手」就报 deformation;"
"只有当同一人物身上长出第三只手、或画面里有一只手明显连不到任何出镜人物时,"
"才算肢体畸变。\n"
"- 人物缺席只写在 `characters` 里,`issues` 中不要再重复报缺席。\n"
"- 「画面描述里还点名提到的人物」没挂进本镜出镜名单,按名单里的人同样核对五官与服装,"
"但**不得**因为它多出来而报人数超出设定。",
"- severity:观众一眼可辨的穿帮=high;需要凑近看、或只是不够精致=low。",
"- frame 填问题最明显的那一帧帧号(从 1 开始,不超过给定帧数);贯穿整镜的问题填 0。",
"- detail 用不超过 30 个汉字的中文写清「谁、怎么了、在第几帧」。",
"",
"【输出】严格只输出一个 JSON 对象,不要 markdown 代码块、不要任何解释:",
'{"characters":[{"name":"人物名","frames":[1,2],"evidence":"画面依据"}],'
'"people":同框人数,'
'"issues":[{"type":"类型","severity":"high|low","frame":1,"subject":"涉及人物",'
'"detail":"30字以内"}],"summary":"一句话结论"}',
"",
"type 只能取:" + "/".join(ISSUE_TYPES.keys()) + ";画面干净时 issues 为空数组。",
]
return "\n".join(lines)
def _record(db: Session, project: Project, spec, prompt: str, result, *, shot: Shot,
images: int) -> None:
"""把看图开销记进成本中心(失败不影响质检结论)。"""
try:
from app.services import costing
with db.begin_nested(): # 记账失败只回滚这一段,不碰质检主事务
return _record_inner(db, project, spec, prompt, result, shot=shot, images=images)
except Exception as exc:
log.info("画面质检记账失败(忽略):%s", exc)
return None
def _record_inner(db: Session, project: Project, spec, prompt: str, result, *, shot: Shot,
images: int) -> None:
from app.services import costing
usage = dict((result.metadata or {}).get("usage") or {})
tokens = float(usage.get("total_tokens") or 0)
price = float(getattr(spec, "price_per_1k_chars", 0) or 0)
cost = float(result.cost or 0) or (tokens / 1000.0 * price if tokens else 0.0)
currency = getattr(spec, "currency", "CNY") or "CNY"
costing.record_generation(
db, project_id=project.id, provider=spec.provider, model=spec.model,
kind="visual_qc", prompt=prompt[:2000],
params_json={"shot_id": shot.id, "images": images},
output_path="", cost=cost, currency=currency, attempts=1,
metadata_json={"usage": usage, "shot_order": shot.order_index})
if cost:
costing.record_cost(db, project_id=project.id, kind="visual_qc", provider=spec.provider,
model=spec.model, quantity=tokens / 1000.0, unit="1k_tokens",
unit_price=price, cost=cost, currency=currency,
note="画面质检(多模态看图)")
def _data_url(path: Path) -> str:
return "data:image/jpeg;base64," + base64.b64encode(Path(path).read_bytes()).decode()
def image_payload(frames: list[Path], refs: list[dict]) -> list[dict]:
"""送模型的图片清单:**镜头帧在前**(帧号=序号),定妆参考照在后并带标签。"""
payload = [{"url": _data_url(frame), "label": f"镜头帧 第 {index} 帧"}
for index, frame in enumerate(frames, 1)]
for row in refs:
payload.append({"url": _data_url(row["path"]),
"label": f"定妆参考照:{row['name']}(不是镜头画面)"})
return payload
def _loads(text: str) -> dict[str, Any]:
"""容错解析:剥掉代码块与前后废话,取最外层花括号。"""
raw = str(text or "").strip()
raw = re.sub(r"^```(?:json)?|```$", "", raw, flags=re.S).strip()
try:
return json.loads(raw)
except Exception:
pass
start, end = raw.find("{"), raw.rfind("}")
if 0 <= start < end:
return json.loads(raw[start:end + 1])
raise ValueError(f"模型未返回 JSON:{raw[:200]}")
def _check(key: str, label: str, passed: bool, value: str, detail: str, severity: str) -> dict:
return {"key": f"visual.{key}", "label": f"画面质检·{label}", "passed": passed, "weight": 0,
"value": value, "detail": detail, "severity": severity}
def analyse_shot(db: Session, project: Project, shot: Shot, *,
frames: list[Path] | None = None) -> VisualOutcome:
"""跑一镜的画面质检。调用方负责把 checks 并入质检报告。"""
ok, note = vision_ready(db)
if not ok:
return VisualOutcome(checks=[_check("skipped", "未执行", True, "跳过", note, "info")])
rel = shot.output_path
path = storage.to_absolute(rel) if rel else None
if not path or not Path(path).exists():
return VisualOutcome(checks=[_check("skipped", "未执行", True, "跳过", "镜头没有画面产物", "info")])
cast = expected_cast(db, project, shot)
refs = reference_images(db, cast) if cast else []
stem = f"shot-{shot.id}-qc"
folder = storage.project_dir(project.id, "qc")
for stale in folder.glob(f"{stem}*.jpg"):
stale.unlink(missing_ok=True) # 判定帧只保留最近一次
if frames is None:
frames = ff.extract_frames(path, folder, count=max(1, settings.qc_visual_frames),
width=max(160, settings.qc_visual_width), stem=stem,
timeout=max(10, settings.qc_visual_frame_timeout))
if not frames:
return VisualOutcome(checks=[_check("skipped", "未执行", True, "跳过", "抽帧失败,画面质检未完成",
"warning")])
urls = [storage.url_for(storage.to_relative(frame)) for frame in frames]
spec, _adapter = registry.resolve_llm(db)
prompt = build_prompt(shot, cast, len(frames), refs)
try:
from app.providers.base import AssetGenRequest
payload = image_payload(frames, refs)
request = AssetGenRequest(
kind="text", prompt=prompt, seed=None,
params={"model": spec.model, "images": payload, "temperature": 0, "max_tokens": 1500,
"timeout_seconds": float(settings.qc_visual_timeout),
"extra_body": {"enable_thinking": False}},
idempotency_key=f"visual-qc:{shot.id}", label=f"visual-qc:shot-{shot.id}",
)
result = registry.generate_asset(db, request, "llm", spec.provider, spec.model)
verdict = _loads(str(result.text or ""))
_record(db, project, spec, prompt, result, shot=shot, images=len(payload))
except Exception as exc:
return VisualOutcome(
checks=[_check("attempt", "未完成", False, "模型未给出结论",
f"画面质检调用失败(不阻断流程):{type(exc).__name__}: {exc}"[:300],
"warning")],
frames=urls, model=f"{spec.provider}/{spec.model}")
return score_verdict(verdict, cast, shot, frames=len(frames), urls=urls,
model=f"{spec.provider}/{spec.model}")
def score_verdict(verdict: dict, cast: list[dict], shot: Shot, *, frames: int, urls: list[str],
model: str) -> VisualOutcome:
"""把模型结论映射成扣分项 + 逐项 checks。"""
outcome = VisualOutcome(frames=urls, raw=verdict, model=model, completed=True)
voiceover = any(h in (shot.description or "") for h in VOICEOVER_HINTS)
found: dict[str, dict] = {}
def add(itype: str, detail: str, frame: Any, severity: str | None = None) -> None:
label, penalty, default_sev, lock = ISSUE_TYPES.get(itype) or ISSUE_TYPES["other"]
sev = str(severity or "").lower() or default_sev
try:
at = int(frame or 0)
except (TypeError, ValueError):
at = 0
at = at if 0 < at <= frames else 0
row = found.get(itype)
if row is None:
row = found[itype] = {"label": label, "penalty": penalty, "detail": detail,
"frames": [at] if at else [],
"severity": "warning" if lock else
("error" if sev == "high" else "warning")}
else:
row["detail"] = f"{row['detail']};{detail}"[:400]
if at and at not in row["frames"]:
row["frames"].append(at)
if not lock and sev == "high":
row["severity"] = "error"
if len(row["frames"]) > 1:
row["penalty"] = min(45, row["penalty"] + 5) # 多处出现的问题更严重
# 1) 应出场人物:缺席由平台自己判定,比让模型自由发挥更稳
seen = {str(row.get("name", "")).strip(): row
for row in (verdict.get("characters") or []) if isinstance(row, dict)}
for member in cast:
row = seen.get(member["name"])
if row is None:
continue # 模型没答这个人:不臆断
at = [int(n) for n in (row.get("frames") or []) if str(n).strip().isdigit()]
present = bool(at) or bool(row.get("present"))
evidence = str(row.get("evidence") or "").strip()[:120]
if not present and member.get("mentioned"):
# 画面描述提到的人可能只在台词里被提及,没入画不算穿帮
continue
if not present:
if member["speaker"]:
if voiceover:
add("missing_speaker", f"{member['name']}未露脸(本镜为画外音,仅提示)", 0, "low")
continue
add("missing_speaker",
f"所有采样帧中都看不到说话人{member['name']}"
+ (f"(依据:{evidence})" if evidence else ""), 0, "high")
else:
add("missing_character",
f"所有采样帧中都看不到{member['name']}"
+ (f"(依据:{evidence})" if evidence else ""), 0, "high")
continue
# 说话人只露了一帧:多半是镜头没跟上,够不上返工,但要给审核人留痕
if member["speaker"] and at and len(at) <= max(1, frames // 4):
add("missing_speaker", f"说话人{member['name']}仅在 {len(at)}/{frames} 帧中出现", min(at), "low")
# 2) 模型自报的问题
for item in verdict.get("issues") or []:
if not isinstance(item, dict):
continue
itype = str(item.get("type", "")).strip()
if itype not in ISSUE_TYPES:
itype = "other"
subject = str(item.get("subject") or "").strip()
detail = str(item.get("detail") or "").strip()[:200] or ISSUE_TYPES[itype][0]
add(itype, (f"{subject}:" if subject else "") + detail, item.get("frame"), item.get("severity"))
if not cast:
# 没绑定人物时「人数对不对」没有参照,只能作为提示
for key in ("extra_people", "missing_character", "missing_speaker"):
if key in found:
found[key]["severity"] = "warning"
penalty = 0
for itype, row in sorted(found.items(), key=lambda kv: -kv[1]["penalty"]):
penalty += row["penalty"]
where = ("帧 " + "、".join(str(f) for f in row["frames"])) if row["frames"] else "整镜"
outcome.checks.append(_check(itype, row["label"], False, where, row["detail"], row["severity"]))
outcome.score = float(max(0.0, 100.0 - penalty))
people = verdict.get("people")
summary = str(verdict.get("summary") or "").strip()
# 「画面复核」是结论行,不是扣分项:它自己已经计入 penalty,
# 严重度必须跟随真实问题里最重的一条,否则 warning 级小毛病会被它拖成返工。
worst = "error" if any(row["severity"] == "error" for row in found.values()) else "warning"
outcome.checks.insert(0, _check(
"review", "画面复核", not found,
f"{frames} 帧 / 设定 {len(cast)} 人" + (f" / 同框 {people} 人" if people is not None else ""),
summary or ("未发现画面穿帮" if not found
else ";".join(row["detail"] for row in found.values())[:300]),
"info" if not found else worst))
return outcome