nex_animation/backend/tests/test_speech_qc.py

213 lines
11 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""回归用例:听觉质检(whisper 听写成片音轨)必须抓得住、也必须不误伤。
现场证据(项目 12《告诉你个小秘密第2期》逐镜 ASR 实测):
| 镜 | 台词 | 转写 | 该怎么判 |
|----|------|------|----------|
| #6 shot39 | 有 | 与台词一致(「汇智」被听成「绘制」) | 干净——同音误判不是事故 |
| #7 shot40 | **无** | 「字幕君」横跨整段 | 干净——幻听,且以前**整项被跳过** |
| #8 shot41 | 有 | 「我是小慧选」+ 台词 | 返工——开口前先念了提示词 |
| #7 shot40 | 无 | 把「近景,过肩镜头,固定机位」念出来 | 返工——只有逐字命中才认得出(2 个字) |
判据的核心不是「杂音有几个字」,而是「**是否逐字出自本镜提示词**」:
字数规则抓改写型泄漏,逐字规则抓短到看不出是短语的泄漏,两条取并集。
"""
from __future__ import annotations
import pytest
from app.models import GenerationRecord, Project, Shot
from app.services import asr, prompt as prompt_service
from app.services import quality
OFFICE = ("<Picture 1>中的画面,<Picture 3>中的充满科技感的IT办公室,有电脑屏幕和服务器机柜。"
"场景明亮。<Picture 2>中的俊杰,高大,在办公室电脑前熟练地敲击键盘,"
"屏幕上显示着云桌面恢复进度条,画面充满科技感。近景,过肩镜头,固定机位")
RELIABLE = ("<Picture 2>中的紫小慧,一名年轻东亚女生,身穿医生白大褂,全景,平视视角,缓慢拉镜。"
"<Audio 1><d>云桌面方案真可靠!</d>")
LINE = "云桌面方案真可靠!"
def _shot(db, *, dialogue=LINE, prompt=OFFICE) -> tuple[Project, Shot]:
project = Project(name="自检-听觉质检", aspect_ratio="9:16", resolution="720p", fps=24,
default_provider="comfyui", target_duration=20)
db.add(project)
db.flush()
shot = Shot(project_id=project.id, order_index=7, code="S07", title="恢复系统",
description="俊杰在电脑前敲击键盘", duration=8, shot_size="close_up",
camera_angle="eye_level", camera_movement="static", mode="r2v",
status="generated", dialogue=dialogue, prompt=prompt,
output_path="12/generated/shot-07-task675.mp4")
db.add(shot)
db.flush()
return project, shot
@pytest.fixture
def listen(monkeypatch):
"""装一个「听写成指定文本」的耳朵:不依赖 whisper,也不依赖真实音轨。"""
def install(text: str):
transcript = asr.Transcript(segments=[asr.Segment(asr.normalise(text), 0.0, 8.0)],
text=asr.normalise(text))
monkeypatch.setattr(asr, "transcribe", lambda media, **kw: transcript)
monkeypatch.setattr(asr, "available", lambda: (True, ""))
monkeypatch.setattr(prompt_service, "joint_av_mode", lambda db, project, shot: True)
return install
def _run(db, project, shot, *, prompt=OFFICE) -> quality.QCReport:
report = quality.QCReport(shot_id=shot.id, passed=False, score=0.0)
report.probe["has_audio"] = True
quality.run_speech_qc(db, project, shot, "/tmp/does-not-matter.mp4", report)
return report
def _by_key(report, key="dialogue"):
return [c for c in report.checks if c.key == key]
def test_short_verbatim_leak_is_caught_even_at_two_chars(db, listen):
"""shot40 形态:只漏出「固定」两字,旧规则按字数放行 → 必须判返工。"""
project, shot = _shot(db)
shot.dialogue = "不要着急,恢复系统和数据一键就能完成。"
listen("固定不要着急,恢复系统和数据一键就能完成。") # 台词前多了 2 个字
report = _run(db, project, shot)
checks = _by_key(report)
assert checks and not checks[0].passed and checks[0].severity == "error"
assert checks[0].label == "台词前多念内容"
assert "固定" in (report.probe["speech"]["narration"])
def test_homophone_misrecognition_stays_clean(db, listen):
"""shot39 形态:台词一字不差地念了,只是 ASR 把「汇智」听成「绘制」→ 不是事故。"""
project, shot = _shot(db)
shot.dialogue = "俊杰,医院的电脑坏了,我们用紫光汇智云桌面一键就恢复好了。"
listen("俊杰,医院的电脑坏了,我们用紫光绘制云桌面一键就恢复好了。")
report = _run(db, project, shot)
checks = _by_key(report)
assert len(checks) == 1 and checks[0].passed and checks[0].value == "干净"
assert report.probe["speech"]["before"] == ""
def test_silent_shot_hallucination_is_not_a_leak(db, listen):
"""shot40 形态:无台词镜 whisper 幻听出「字幕君」→ 不是事故,但也不能装没听见。
记 warning(人工复核)而不是 error:它不逐字出自提示词,判返工就是拿 ASR
的幻听去烧真金白银的卡时;可它也确实是一段听得见的动静,得让用户看得见。
"""
project, shot = _shot(db, dialogue="")
listen("字幕君")
report = _run(db, project, shot)
checks = _by_key(report)
assert len(checks) == 1 and not checks[0].passed and checks[0].severity == "warning"
assert checks[0].label == "无台词镜头有人声"
assert report.probe["speech"]["leaked"] is False
assert report.issues == [] # 不会把镜头打成返工
def test_truly_silent_shot_is_clean(db, listen):
"""无台词、真的没人声 → 干净。"""
project, shot = _shot(db, dialogue="")
listen(" ")
checks = _by_key(_run(db, project, shot))
assert len(checks) == 1 and checks[0].passed and checks[0].value == "干净"
def test_silent_shot_that_reads_the_caption_is_a_leak(db, listen):
"""无台词镜把画面描述念了出来 → 以前整项跳过,现在必须报 error。"""
project, shot = _shot(db, dialogue="", prompt=OFFICE)
listen("近景,过肩镜头,固定机位")
report = _run(db, project, shot)
checks = _by_key(report)
assert len(checks) == 1 and not checks[0].passed and checks[0].severity == "error"
assert checks[0].label == "无台词镜头念出了画面描述"
assert report.issues and "无台词" in report.issues[0] # 会真的卡住返工,不只是打分低
def test_rewritten_leak_still_caught_by_length(db, listen):
"""shot41 形态:「我是小慧选」是 caption 里「<Picture 2>中的紫小慧」的改写,
逐字对不上,靠成词短语字数抓住。"""
project, shot = _shot(db, dialogue=LINE, prompt=RELIABLE)
listen("我是小慧选云桌面方案真可靠!")
report = _run(db, project, shot)
checks = _by_key(report)
assert len(checks) == 1 and not checks[0].passed and checks[0].severity == "error"
verdict = report.probe["speech"]
assert verdict["before"] == "我是小慧选" and verdict["narrated"] is False
assert report.issues and "台词前多念内容" in report.issues[0]
def test_two_char_camera_term_leak_is_caught(db, listen):
"""shot-03-task656 实测:模型开头念了「中景」,而台词第一个字「我」被 ASR 吞了。
台词起点必须落在真正的台词上,否则泄漏被吞成 1 个字,精确规则就够不着。"""
project, shot = _shot(db)
shot.dialogue = "我在后台操作一下,马上就能帮你们恢复系统。"
shot.prompt = "俊杰坐在工位上敲键盘,中景,缓慢推镜"
listen("中景再后台操作一下马上就能帮你们恢复系统")
checks = _by_key(_run(db, project, shot))
assert len(checks) == 1 and not checks[0].passed and checks[0].severity == "error"
assert checks[0].label == "台词前多念内容"
def test_dialogue_text_is_not_counted_as_prompt_leak(db, listen):
"""台词本来就写在 caption 的 <d> 段落里——比对前必须摘掉,否则每镜都误报。"""
project, shot = _shot(db, dialogue=LINE, prompt=RELIABLE)
assert LINE not in quality.caption_without_dialogue(RELIABLE, [{"text": LINE}])
assert "紫小慧" in quality.caption_without_dialogue(RELIABLE, [{"text": LINE}])
def test_caption_prefers_the_record_that_produced_this_file(db):
"""比对用的是**产出当前这一版画面**时下发的提示词,不是分镜里缓存的那份。"""
project, shot = _shot(db, prompt="缓存里的旧提示词")
db.add_all([
GenerationRecord(project_id=project.id, shot_id=shot.id, kind="generate_shot",
prompt="最新一版但产物不是它", output_path="12/generated/shot-07-task999.mp4"),
GenerationRecord(project_id=project.id, shot_id=shot.id, kind="generate_shot",
prompt=OFFICE, output_path="12/generated/shot-07-task675.mp4"),
])
db.flush()
assert quality.shot_caption(db, shot) == OFFICE
def test_caption_falls_back_then_to_cached_prompt(db):
project, shot = _shot(db, prompt="缓存里的提示词")
db.add(GenerationRecord(project_id=project.id, shot_id=shot.id, kind="generate_shot",
prompt="最近一次下发的提示词", output_path="12/generated/other.mp4"))
db.flush()
assert quality.shot_caption(db, shot) == "最近一次下发的提示词"
other = Shot(project_id=project.id, order_index=8, code="S08", title="无记录",
duration=5, status="generated", prompt="只有缓存")
db.add(other)
db.flush()
assert quality.shot_caption(db, other) == "只有缓存"
def test_ruleset_bump_invalidates_previous_pass(db):
"""规则升级必须让旧结论作废——否则 shot40 这类「已经 passed」的镜头会被永久沿用。"""
from app.services import agent
project, shot = _shot(db, dialogue="")
shot.qc_json = {"passed": True, "qc_ruleset": 2, "artifact_checksum": "abc"}
shot.output_checksum = "abc"
assert quality.QC_RULESET == 3
assert agent.qc_verdict_is_current(shot) is False
shot.qc_json = {"passed": True, "qc_ruleset": quality.QC_RULESET, "artifact_checksum": "abc"}
assert agent.qc_verdict_is_current(shot) is True
def test_qc_skipped_without_asr_is_not_a_failure(db, monkeypatch):
"""whisper 不可用时只记一条 info,绝不把镜头判成返工。"""
project, shot = _shot(db)
monkeypatch.setattr(prompt_service, "joint_av_mode", lambda db, project, shot: True)
monkeypatch.setattr(asr, "available", lambda: (False, "未找到 whisper-cli"))
report = _run(db, project, shot)
assert len(_by_key(report)) == 1 and _by_key(report)[0].passed
def test_tts_shot_is_not_listened_to(db, monkeypatch, listen):
"""走 TTS 配音的镜头:这一版音轨本来就不该有台词,听了也只会被环境声绊倒。"""
project, shot = _shot(db)
listen("随便什么声音")
monkeypatch.setattr(prompt_service, "joint_av_mode", lambda db, project, shot: False)
assert _by_key(_run(db, project, shot)) == []