nex_animation/backend/app/services/quality.py

420 lines
22 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""自动质量检查(业务流程「质量检查」节点)。
对已下载的镜头产物做客观检测:可读性/时长偏差/分辨率/帧率/码率/解码完整性/
黑屏占比/画面冻结占比/静音占比,形成打分与逐项结论,写入 ReviewRecord(kind=auto)。
"""
from __future__ import annotations
import logging
from dataclasses import asdict, dataclass, field
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.config import settings
from app.constants import ReviewVerdict, ShotStatus, TaskKind
from app.models import GenerationRecord, Project, ReviewRecord, Shot, now_utc
from app.services import events, storage
from app.services import ffmpeg as ff
log = logging.getLogger(__name__)
WEIGHTS = {"integrity": 30, "duration": 20, "resolution": 15, "fps": 8, "bitrate": 10,
"black": 8, "freeze": 9}
PASS_SCORE = 70.0
@dataclass
class Check:
key: str
label: str
passed: bool
weight: int
value: str = ""
detail: str = ""
severity: str = "error" # error / warning
def as_dict(self) -> dict:
return asdict(self)
@dataclass
class QCReport:
shot_id: int
passed: bool
score: float
checks: list[Check] = field(default_factory=list)
probe: dict = field(default_factory=dict)
threshold: float = PASS_SCORE
@property
def issues(self) -> list[str]:
"""必须返工的硬伤(warning 另算,见 warnings)。"""
return [c.label + ":" + c.detail for c in self.checks if not c.passed and c.severity != "warning"]
@property
def failed_checks(self) -> list[str]:
return [c.label + ":" + c.detail for c in self.checks if not c.passed]
def tech_score(self) -> float:
"""纯技术分(画面质检的扣分项权重为 0,不影响这一项)。"""
return float(sum(c.weight for c in self.checks if c.passed))
@property
def warnings(self) -> list[str]:
return [c.label + ":" + c.detail for c in self.checks
if not c.passed and c.severity == "warning"]
def as_dict(self) -> dict:
return {"shot_id": self.shot_id, "passed": self.passed, "score": round(self.score, 1),
"threshold": self.threshold, "probe": self.probe,
"tech_score": self.tech_score(), "visual_score": self.probe.get("visual_score"),
"checks": [c.as_dict() for c in self.checks],
"issues": self.issues, "warnings": self.warnings}
def check_shot(db: Session, project: Project, shot: Shot, *, commit: bool = True) -> QCReport:
path = storage.to_absolute(shot.output_path) if shot.output_path else None
report = QCReport(shot_id=shot.id, passed=False, score=0.0)
if not path or not path.exists():
report.checks.append(Check("integrity", "产物文件", False, WEIGHTS["integrity"], "缺失",
"镜头产物文件不存在,需要重新生成"))
return persist(db, project, shot, report, commit=commit)
if not ff.available():
report.checks.append(Check("integrity", "媒体探测", False, 0, "ffprobe 不可用",
"本机缺少 ffmpeg/ffprobe,无法完成自动质检", "warning"))
report.passed = True
report.score = PASS_SCORE
return persist(db, project, shot, report, commit=commit)
try:
probe = ff.probe(path)
except Exception as exc:
report.checks.append(Check("integrity", "媒体探测", False, WEIGHTS["integrity"], "无法解析",
f"ffprobe 解析失败:{exc}"))
return persist(db, project, shot, report, commit=commit)
report.probe = probe.as_dict()
report.probe.update(size_bytes=path.stat().st_size, url=storage.url_for(shot.output_path))
# 1. 解码完整性
try:
decode = ff.decode_errors(path, timeout=180)
except ff.FFmpegError as exc:
decode = str(exc)
ok = not decode.strip()
report.checks.append(Check("integrity", "解码完整性", ok, WEIGHTS["integrity"],
"无损坏帧" if ok else "存在解码错误",
"逐帧解码通过" if ok else decode.strip()[-400:]))
# 2. 时长偏差
# 基线是「派发时真正下发的秒数」,不是分镜里写的秒数:联合音视频模型会按台词
# 缩短镜头(留空档它就念提示词),拿分镜时长比就变成每条必不合格。
from app.services import prompt as prompt_service
want = prompt_service.joint_target_seconds(db, project, shot)
got = float(probe.duration or 0)
tol = max(settings.qc_duration_tolerance, 0.05)
if want <= 0:
report.checks.append(Check("duration", "时长", False, WEIGHTS["duration"], f"{got:.2f}s",
"镜头未设置目标时长", "warning"))
else:
dev = abs(got - want) / want
report.checks.append(Check("duration", "时长偏差", dev <= tol, WEIGHTS["duration"],
f"实际 {got:.2f}s / 目标 {want:.2f}s(偏差 {dev * 100:.1f}%)",
f"容忍 ±{tol * 100:.0f}%" if dev <= tol else
"时长与脚本不符,建议重新生成或调整时长"))
# 3. 分辨率
from app.constants import canvas_for
tw, th = canvas_for(project.resolution, project.aspect_ratio)
# 基线不能一律用交付画布:声明了 upscale_to 的模型(H3 原生 736x1312 → 1080p 交付)
# 永远达不到交付像素,736/1080=0.68 会被 0.75 的阈值判成不合格,
# 每条正常出片都要被冤枉一次重生成。所以拿模型真正会出的原生画布当基线,
# 交付档位与原生不一致时在结论里写清楚,不静默放水。
bw, bh, via = tw, th, "交付档位"
try:
from app.providers import registry as model_registry
spec, _adapter = model_registry.resolve_video(db, shot.provider or project.default_provider,
shot.model or None)
gen = model_registry.adapter_for_spec(spec).effective_canvas(
model_registry.runtime_config(db, spec), project.aspect_ratio, project.resolution)
if gen and min(int(gen[0]), int(gen[1])) < min(tw, th):
bw, bh, via = int(gen[0]), int(gen[1]), "模型原生画布(交付时放大)"
except Exception: # noqa: BLE001 - 取不到就按交付档位判
pass
res_ok = probe.width >= bw * 0.75 and probe.height >= bh * 0.75
report.checks.append(Check("resolution", "分辨率", res_ok, WEIGHTS["resolution"],
f"{probe.width}x{probe.height}",
f"不低于{via} {bw}x{bh} 的 75%"
+ ("" if (bw, bh) == (tw, th) else f"(交付 {tw}x{th})")
if res_ok else f"低于{via} {bw}x{bh},成片会模糊"))
# 4. 帧率
fps_ok = probe.fps >= max(1, project.fps) * 0.8
report.checks.append(Check("fps", "帧率", fps_ok, WEIGHTS["fps"], f"{probe.fps:.2f} fps",
f"目标 {project.fps} fps" + ("" if fps_ok else ",帧率过低会卡顿"),
"info" if fps_ok else "error"))
# 5. 码率
bitrate_ok = probe.bitrate_kbps >= settings.qc_min_bitrate_kbps
report.checks.append(Check("bitrate", "码率", bitrate_ok, WEIGHTS["bitrate"],
f"{probe.bitrate_kbps:.0f} kbps",
f"低于下限 {settings.qc_min_bitrate_kbps} kbps" if not bitrate_ok else "码率充足",
"warning"))
# 6. 黑屏
black = ff.detect_black_ratio(path, settings.qc_black_threshold, timeout=180)
report.checks.append(Check("black", "黑屏占比", black <= 0.35, WEIGHTS["black"], f"{black * 100:.1f}%",
"画面有效" if black <= 0.35 else "黑屏过长,疑似空镜头或生成失败"))
report.probe["black_ratio"] = round(black, 4)
# 7. 画面冻结
freeze = ff.detect_freeze_ratio(path, timeout=180)
report.checks.append(Check("freeze", "画面冻结占比", freeze <= 0.6, WEIGHTS["freeze"],
f"{freeze * 100:.1f}%",
"画面有运动" if freeze <= 0.6 else "大部分时间画面静止,运镜可能未生效"))
report.probe["freeze_ratio"] = round(freeze, 4)
# 8. 音轨(提示项,不扣分:配音在合成阶段统一混入)
has_audio = bool(probe.has_audio)
report.probe["has_audio"] = has_audio
if not has_audio:
report.probe["audio_note"] = "镜头无音轨,将由配音/配乐轨道补齐"
# 9. 台词听写:画面质检看帧,管不了「开头多念了一句谁的话」
run_speech_qc(db, project, shot, path, report)
# 10. 画面质检:上面几项只能证明「文件是好的」,证明不了「画面是对的」
visual = run_visual_qc(db, project, shot, report)
tech = float(sum(c.weight for c in report.checks if c.passed))
hard_fail = any(not c.passed and c.severity == "error" for c in report.checks)
score = tech
if visual is not None and visual.completed:
# 视觉分只在真正拿到结论时掺入总分;跳过/失败时分值与纯技术质检完全一致
weight = min(max(float(settings.qc_visual_weight), 0.0), 1.0)
score = (1.0 - weight) * tech + weight * visual.score
report.score = score
report.passed = score >= PASS_SCORE and not hard_fail
return persist(db, project, shot, report, commit=commit)
def shot_caption(db: Session, shot: Shot, path=None) -> str:
"""本镜产物**实际下发**的那条提示词。
质检要回答的是「这一版画面是怎么生出来的」,所以必须拿当年真正发给模型的那份,
而不是分镜里现在缓存的 `shots.prompt`——用户改过分镜、或返工过一轮之后,
两者对不上,比对结论就成了无源之水。
优先按产物路径命中记录(同一镜头可能出过多版),命中不了取最新一条,
再退到缓存提示词。
"""
rows = db.scalars(select(GenerationRecord)
.where(GenerationRecord.shot_id == shot.id,
GenerationRecord.kind == TaskKind.GENERATE_SHOT,
GenerationRecord.prompt != "")
.order_by(GenerationRecord.id.desc()).limit(20)).all()
target = str(path or shot.output_path or "")
for row in rows:
produced = str(row.output_path or "")
if target and produced and (target.endswith(produced) or produced.endswith(target)):
return str(row.prompt or "")
for row in rows:
return str(row.prompt or "") # 没有路径可对(老数据)时取最近一次生成
return str(shot.prompt or "")
def caption_without_dialogue(caption: str, lines: list[dict]) -> str:
"""把 caption 里的**台词部分**摘掉,只留画面描述。
比对对象必须是纯画面描述:台词原文本来就写在 `<Audio 1><d>…</d>` 里,
不摘掉的话「念出台词」这件事本身就会被算成「念了提示词」,每镜都误报。
"""
text = str(caption or "")
for line in lines:
piece = str(line.get("text") or "").strip()
if piece:
text = text.replace(piece, "")
return text
def run_speech_qc(db: Session, project: Project, shot: Shot, path, report: QCReport) -> None:
"""把成片音轨听一遍,核对「只念了台词、且从头就念台词」。
为什么必须有这一路:联合音视频模型(MiniMax H3)最常见的穿帮是**听觉**的——
镜头开头先冒出一段提示词里的画面描述(「告诉你个小秘密第 2 期」3/4 镜实测),
而时长、码率、黑屏、冻结、画面质检全部满分放行,直到人肉看片才发现。
权重记 0(不动技术分),但「前面多念」按 error 处理:它一定听得见,
没有「分数够高就可以放过」的道理。缺字只记 warning——ASR 自己也会吞字,
这一项更像提醒人工复核,不足以直接判返工。
**无台词镜头照听**(shot40 的盲区):以前 `if not dialogue: return` 直接跳过,
于是「本不该有人声的镜头把画面描述念了出来」在质检报告里是空白的。
无台词只认逐字命中提示词——环境声、配乐、幻听都不算事故。
"""
if not probe_has_audio(report):
return
from app.services import asr, prompt as prompt_service
if not prompt_service.joint_av_mode(db, project, shot):
return # 走 TTS 配音的镜头,这一版音轨本来就不该有台词
lines = prompt_service.joint_dialogue_lines(shot)
dialogue = "".join(str(x["text"]) for x in lines) or str(shot.dialogue or "").strip()
ok, why = asr.available()
if not ok:
report.checks.append(Check("dialogue", "台词听写", True, 0, "跳过",
f"未做听写复核:{why}", "info"))
return
visual = caption_without_dialogue(shot_caption(db, shot, path), lines)
verdict = asr.audit_dialogue(path, dialogue, prompt=visual)
if verdict is None:
report.checks.append(Check("dialogue", "台词听写", True, 0, "跳过",
"转写失败(whisper 不可用或音轨异常),本镜未经听觉复核",
"warning"))
return
report.probe["speech"] = verdict
hit = str(verdict.get("narration") or "")
heard = str(verdict.get("transcript") or "")[:120]
if verdict["ok"]:
if not dialogue and len(heard) >= asr._LEAK_CHARS:
# 无台词、听到了动静、但念的不是提示词:可能是模型自行加词,
# 也可能是配乐/环境声被 whisper 认成人声(「字幕君」这类幻听极常见)。
# 不定性成提示词事故,但也不装作什么都没听见——记 warning 交人工。
report.checks.append(Check(
"dialogue", "无台词镜头有人声", False, 0, f"听到「{heard[:40]}」",
"本镜没有台词,转写里出现了不属于提示词的内容:可能是模型自行加词,"
f"也可能是配乐或环境声被识别成人声,建议人工听一下。本镜提示词:{visual[:160]}",
"warning"))
return
note = "未念画面描述" if not dialogue else "台词前无杂语"
report.checks.append(Check("dialogue", "台词听写", True, 0, "干净",
f"从 {verdict['speech_start']:.2f}s 起{note}"
f"(转写:{heard})", "info"))
return
if verdict["leaked"]:
evidence = (f"念出的「{hit}」逐字出自本镜提示词的画面描述部分"
if verdict.get("narrated") else
f"台词前这段杂音有 {verdict['before_chars']} 个字")
if dialogue:
report.checks.append(Check(
"dialogue", "台词前多念内容", False, 0,
f"开口前先念了「{verdict['before']}」({verdict['before_chars']} 字)",
f"{evidence}——是画面描述被模型当成台词念了出来。完整转写:{heard}。"
f"台词本身缺失 {verdict['missing']} 字——本镜提示词:{visual[:160]}",
"error"))
else:
report.checks.append(Check(
"dialogue", "无台词镜头念出了画面描述", False, 0,
f"没有台词却念出「{verdict['transcript'][:40]}」",
f"{evidence}。本镜不该有人声(要出声应当是环境声/音效,"
f"不是把提示词念出来)。完整转写:{heard}。本镜提示词:{visual[:160]}",
"error"))
return
else:
report.checks.append(Check(
"dialogue", "台词完整性", False, 0,
f"转写比台词少 {verdict['missing']} 字",
f"可能是念漏、被别的内容顶替,也可能只是识别误差,建议人工听一下。"
f"完整转写:{heard}", "warning"))
def probe_has_audio(report: QCReport) -> bool:
return bool(report.probe.get("has_audio"))
def run_visual_qc(db: Session, project: Project, shot: Shot, report: QCReport):
"""执行画面质检并把结论并进技术质检报告。
这一路是「加分项」而不是「新卡点」:模型没配、不支持看图、抽帧失败、
返回的不是 JSON,一律记一条 warning 然后按纯技术质检出分,
绝不因为看图本身出问题把整条流水线卡死。
"""
from app.services import visual_qc # 延迟导入:避开 provider 注册环
try:
outcome = visual_qc.analyse_shot(db, project, shot)
except Exception as exc: # 兜底:任何异常都不允许打挂质检
log.warning("画面质检异常 shot=%s: %s", shot.id, exc)
report.checks.append(Check("visual_error", "画面质检", True, 0, "未完成",
f"画面质检异常(不阻断流程):{type(exc).__name__}: {exc}"[:300],
"warning"))
return None
for row in outcome.checks:
try:
report.checks.append(Check(**row))
except TypeError: # 模型侧结构变动时宁可少一项也不炸
continue
report.probe["visual_score"] = round(outcome.score, 1) if outcome.completed else None
report.probe["visual_model"] = outcome.model
report.probe["visual_frames"] = list(outcome.frames)
report.probe["visual_summary"] = str((outcome.raw or {}).get("summary") or "")[:400]
report.probe["visual_cast"] = (outcome.raw or {}).get("characters") or []
report.probe["visual_skipped"] = not outcome.completed
return outcome
# **质检规则集版本**。沿用旧结论省的是四分钟出图,代价是规则升级不会自动生效:
# 「台词听写」这一项上线时,上一轮已经判过 passed 的镜头全部被跳过,
# 带着起手式的那版画面就这么进了成片(project 12 shot39 实测)。
# 加了一项检查、改了阈值、换了 ASR 模型,都要在这里 +1。
QC_RULESET = 3
def persist(db: Session, project: Project, shot: Shot, report: QCReport, *, commit: bool = True) -> QCReport:
shot.qc_score = round(report.score, 1)
# 把「这次结论对应哪一版画面」一起落库:产物指纹没变时后面的调度可以直接沿用结论,
# 不必再让看图模型对同一个文件重新摇一次骰子(见 agent.qc_verdict_is_current)。
shot.qc_json = {**report.as_dict(),
"checked_at": now_utc().isoformat(),
"artifact_checksum": shot.output_checksum or "",
"artifact_path": shot.output_path or "",
"qc_ruleset": QC_RULESET}
if report.probe.get("duration"):
shot.output_duration = float(report.probe["duration"])
if report.passed:
if shot.status in (ShotStatus.GENERATED, ShotStatus.RUNNING, ShotStatus.READY):
shot.status = ShotStatus.QC_PASSED
shot.stage = "review"
shot.review_state = ReviewVerdict.PENDING if shot.status != ShotStatus.APPROVED else shot.review_state
else:
shot.status = ShotStatus.FAILED if not report.probe.get("path") else ShotStatus.GENERATED
shot.stage = "quality_check_failed"
shot.review_state = ReviewVerdict.FAIL
shot.error_code = shot.error_code or "qc_failed"
shot.error_message = ";".join(report.issues[:3])[:4000]
db.add(ReviewRecord(project_id=project.id, shot_id=shot.id, kind="auto",
verdict=ReviewVerdict.PASS if report.passed else ReviewVerdict.FAIL,
score=round(report.score, 1), checks_json=report.as_dict(),
notes=";".join(report.issues[:4])[:2000] or "自动质检通过",
reviewer="auto-qc"))
events.log_event(db, "qc.finished", project_id=project.id, shot_id=shot.id,
level="info" if report.passed else "warn",
message=f"镜头 {shot.order_index:02d} 自动质检{'通过' if report.passed else '未通过'}"
f"({report.score:.0f} 分):" + (";".join(report.issues[:2]) or "全部检查项达标"),
entity_type="shot", entity_id=shot.id, data={"score": report.score})
if commit:
db.commit()
else:
db.flush()
return report
def qc_summary(db: Session, project: Project) -> dict:
rows = []
for shot in project.shots or []:
rows.append({"shot_id": shot.id, "order_index": shot.order_index, "title": shot.title,
"status": shot.status, "score": shot.qc_score,
"passed": bool((shot.qc_json or {}).get("passed")),
"issues": (shot.qc_json or {}).get("issues", []),
"warnings": (shot.qc_json or {}).get("warnings", []),
"probe": (shot.qc_json or {}).get("probe", {}),
"review_state": shot.review_state})
checked = [r for r in rows if r["score"] is not None]
return {"shots": rows, "checked": len(checked),
"passed": sum(1 for r in checked if r["passed"]),
"failed": sum(1 for r in checked if not r["passed"]),
"average_score": round(sum(float(r["score"] or 0) for r in checked) / len(checked), 1) if checked else 0}