213 lines
11 KiB
Python
213 lines
11 KiB
Python
"""回归用例:听觉质检(whisper 听写成片音轨)必须抓得住、也必须不误伤。
|
||
|
||
现场证据(项目 12《告诉你个小秘密第2期》逐镜 ASR 实测):
|
||
|
||
| 镜 | 台词 | 转写 | 该怎么判 |
|
||
|----|------|------|----------|
|
||
| #6 shot39 | 有 | 与台词一致(「汇智」被听成「绘制」) | 干净——同音误判不是事故 |
|
||
| #7 shot40 | **无** | 「字幕君」横跨整段 | 干净——幻听,且以前**整项被跳过** |
|
||
| #8 shot41 | 有 | 「我是小慧选」+ 台词 | 返工——开口前先念了提示词 |
|
||
| #7 shot40 | 无 | 把「近景,过肩镜头,固定机位」念出来 | 返工——只有逐字命中才认得出(2 个字) |
|
||
|
||
判据的核心不是「杂音有几个字」,而是「**是否逐字出自本镜提示词**」:
|
||
字数规则抓改写型泄漏,逐字规则抓短到看不出是短语的泄漏,两条取并集。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from app.models import GenerationRecord, Project, Shot
|
||
from app.services import asr, prompt as prompt_service
|
||
from app.services import quality
|
||
|
||
OFFICE = ("<Picture 1>中的画面,<Picture 3>中的充满科技感的IT办公室,有电脑屏幕和服务器机柜。"
|
||
"场景明亮。<Picture 2>中的俊杰,高大,在办公室电脑前熟练地敲击键盘,"
|
||
"屏幕上显示着云桌面恢复进度条,画面充满科技感。近景,过肩镜头,固定机位")
|
||
RELIABLE = ("<Picture 2>中的紫小慧,一名年轻东亚女生,身穿医生白大褂,全景,平视视角,缓慢拉镜。"
|
||
"<Audio 1><d>云桌面方案真可靠!</d>")
|
||
LINE = "云桌面方案真可靠!"
|
||
|
||
|
||
def _shot(db, *, dialogue=LINE, prompt=OFFICE) -> tuple[Project, Shot]:
|
||
project = Project(name="自检-听觉质检", aspect_ratio="9:16", resolution="720p", fps=24,
|
||
default_provider="comfyui", target_duration=20)
|
||
db.add(project)
|
||
db.flush()
|
||
shot = Shot(project_id=project.id, order_index=7, code="S07", title="恢复系统",
|
||
description="俊杰在电脑前敲击键盘", duration=8, shot_size="close_up",
|
||
camera_angle="eye_level", camera_movement="static", mode="r2v",
|
||
status="generated", dialogue=dialogue, prompt=prompt,
|
||
output_path="12/generated/shot-07-task675.mp4")
|
||
db.add(shot)
|
||
db.flush()
|
||
return project, shot
|
||
|
||
|
||
@pytest.fixture
|
||
def listen(monkeypatch):
|
||
"""装一个「听写成指定文本」的耳朵:不依赖 whisper,也不依赖真实音轨。"""
|
||
def install(text: str):
|
||
transcript = asr.Transcript(segments=[asr.Segment(asr.normalise(text), 0.0, 8.0)],
|
||
text=asr.normalise(text))
|
||
monkeypatch.setattr(asr, "transcribe", lambda media, **kw: transcript)
|
||
monkeypatch.setattr(asr, "available", lambda: (True, ""))
|
||
monkeypatch.setattr(prompt_service, "joint_av_mode", lambda db, project, shot: True)
|
||
return install
|
||
|
||
|
||
def _run(db, project, shot, *, prompt=OFFICE) -> quality.QCReport:
|
||
report = quality.QCReport(shot_id=shot.id, passed=False, score=0.0)
|
||
report.probe["has_audio"] = True
|
||
quality.run_speech_qc(db, project, shot, "/tmp/does-not-matter.mp4", report)
|
||
return report
|
||
|
||
|
||
def _by_key(report, key="dialogue"):
|
||
return [c for c in report.checks if c.key == key]
|
||
|
||
|
||
def test_short_verbatim_leak_is_caught_even_at_two_chars(db, listen):
|
||
"""shot40 形态:只漏出「固定」两字,旧规则按字数放行 → 必须判返工。"""
|
||
project, shot = _shot(db)
|
||
shot.dialogue = "不要着急,恢复系统和数据一键就能完成。"
|
||
listen("固定不要着急,恢复系统和数据一键就能完成。") # 台词前多了 2 个字
|
||
report = _run(db, project, shot)
|
||
checks = _by_key(report)
|
||
assert checks and not checks[0].passed and checks[0].severity == "error"
|
||
assert checks[0].label == "台词前多念内容"
|
||
assert "固定" in (report.probe["speech"]["narration"])
|
||
|
||
|
||
def test_homophone_misrecognition_stays_clean(db, listen):
|
||
"""shot39 形态:台词一字不差地念了,只是 ASR 把「汇智」听成「绘制」→ 不是事故。"""
|
||
project, shot = _shot(db)
|
||
shot.dialogue = "俊杰,医院的电脑坏了,我们用紫光汇智云桌面一键就恢复好了。"
|
||
listen("俊杰,医院的电脑坏了,我们用紫光绘制云桌面一键就恢复好了。")
|
||
report = _run(db, project, shot)
|
||
checks = _by_key(report)
|
||
assert len(checks) == 1 and checks[0].passed and checks[0].value == "干净"
|
||
assert report.probe["speech"]["before"] == ""
|
||
|
||
|
||
def test_silent_shot_hallucination_is_not_a_leak(db, listen):
|
||
"""shot40 形态:无台词镜 whisper 幻听出「字幕君」→ 不是事故,但也不能装没听见。
|
||
|
||
记 warning(人工复核)而不是 error:它不逐字出自提示词,判返工就是拿 ASR
|
||
的幻听去烧真金白银的卡时;可它也确实是一段听得见的动静,得让用户看得见。
|
||
"""
|
||
project, shot = _shot(db, dialogue="")
|
||
listen("字幕君")
|
||
report = _run(db, project, shot)
|
||
checks = _by_key(report)
|
||
assert len(checks) == 1 and not checks[0].passed and checks[0].severity == "warning"
|
||
assert checks[0].label == "无台词镜头有人声"
|
||
assert report.probe["speech"]["leaked"] is False
|
||
assert report.issues == [] # 不会把镜头打成返工
|
||
|
||
|
||
def test_truly_silent_shot_is_clean(db, listen):
|
||
"""无台词、真的没人声 → 干净。"""
|
||
project, shot = _shot(db, dialogue="")
|
||
listen(" ")
|
||
checks = _by_key(_run(db, project, shot))
|
||
assert len(checks) == 1 and checks[0].passed and checks[0].value == "干净"
|
||
|
||
|
||
def test_silent_shot_that_reads_the_caption_is_a_leak(db, listen):
|
||
"""无台词镜把画面描述念了出来 → 以前整项跳过,现在必须报 error。"""
|
||
project, shot = _shot(db, dialogue="", prompt=OFFICE)
|
||
listen("近景,过肩镜头,固定机位")
|
||
report = _run(db, project, shot)
|
||
checks = _by_key(report)
|
||
assert len(checks) == 1 and not checks[0].passed and checks[0].severity == "error"
|
||
assert checks[0].label == "无台词镜头念出了画面描述"
|
||
assert report.issues and "无台词" in report.issues[0] # 会真的卡住返工,不只是打分低
|
||
|
||
|
||
def test_rewritten_leak_still_caught_by_length(db, listen):
|
||
"""shot41 形态:「我是小慧选」是 caption 里「<Picture 2>中的紫小慧」的改写,
|
||
逐字对不上,靠成词短语字数抓住。"""
|
||
project, shot = _shot(db, dialogue=LINE, prompt=RELIABLE)
|
||
listen("我是小慧选云桌面方案真可靠!")
|
||
report = _run(db, project, shot)
|
||
checks = _by_key(report)
|
||
assert len(checks) == 1 and not checks[0].passed and checks[0].severity == "error"
|
||
verdict = report.probe["speech"]
|
||
assert verdict["before"] == "我是小慧选" and verdict["narrated"] is False
|
||
assert report.issues and "台词前多念内容" in report.issues[0]
|
||
|
||
|
||
def test_two_char_camera_term_leak_is_caught(db, listen):
|
||
"""shot-03-task656 实测:模型开头念了「中景」,而台词第一个字「我」被 ASR 吞了。
|
||
台词起点必须落在真正的台词上,否则泄漏被吞成 1 个字,精确规则就够不着。"""
|
||
project, shot = _shot(db)
|
||
shot.dialogue = "我在后台操作一下,马上就能帮你们恢复系统。"
|
||
shot.prompt = "俊杰坐在工位上敲键盘,中景,缓慢推镜"
|
||
listen("中景再后台操作一下马上就能帮你们恢复系统")
|
||
checks = _by_key(_run(db, project, shot))
|
||
assert len(checks) == 1 and not checks[0].passed and checks[0].severity == "error"
|
||
assert checks[0].label == "台词前多念内容"
|
||
|
||
|
||
def test_dialogue_text_is_not_counted_as_prompt_leak(db, listen):
|
||
"""台词本来就写在 caption 的 <d> 段落里——比对前必须摘掉,否则每镜都误报。"""
|
||
project, shot = _shot(db, dialogue=LINE, prompt=RELIABLE)
|
||
assert LINE not in quality.caption_without_dialogue(RELIABLE, [{"text": LINE}])
|
||
assert "紫小慧" in quality.caption_without_dialogue(RELIABLE, [{"text": LINE}])
|
||
|
||
|
||
def test_caption_prefers_the_record_that_produced_this_file(db):
|
||
"""比对用的是**产出当前这一版画面**时下发的提示词,不是分镜里缓存的那份。"""
|
||
project, shot = _shot(db, prompt="缓存里的旧提示词")
|
||
db.add_all([
|
||
GenerationRecord(project_id=project.id, shot_id=shot.id, kind="generate_shot",
|
||
prompt="最新一版但产物不是它", output_path="12/generated/shot-07-task999.mp4"),
|
||
GenerationRecord(project_id=project.id, shot_id=shot.id, kind="generate_shot",
|
||
prompt=OFFICE, output_path="12/generated/shot-07-task675.mp4"),
|
||
])
|
||
db.flush()
|
||
assert quality.shot_caption(db, shot) == OFFICE
|
||
|
||
|
||
def test_caption_falls_back_then_to_cached_prompt(db):
|
||
project, shot = _shot(db, prompt="缓存里的提示词")
|
||
db.add(GenerationRecord(project_id=project.id, shot_id=shot.id, kind="generate_shot",
|
||
prompt="最近一次下发的提示词", output_path="12/generated/other.mp4"))
|
||
db.flush()
|
||
assert quality.shot_caption(db, shot) == "最近一次下发的提示词"
|
||
other = Shot(project_id=project.id, order_index=8, code="S08", title="无记录",
|
||
duration=5, status="generated", prompt="只有缓存")
|
||
db.add(other)
|
||
db.flush()
|
||
assert quality.shot_caption(db, other) == "只有缓存"
|
||
|
||
|
||
def test_ruleset_bump_invalidates_previous_pass(db):
|
||
"""规则升级必须让旧结论作废——否则 shot40 这类「已经 passed」的镜头会被永久沿用。"""
|
||
from app.services import agent
|
||
|
||
project, shot = _shot(db, dialogue="")
|
||
shot.qc_json = {"passed": True, "qc_ruleset": 2, "artifact_checksum": "abc"}
|
||
shot.output_checksum = "abc"
|
||
assert quality.QC_RULESET == 3
|
||
assert agent.qc_verdict_is_current(shot) is False
|
||
shot.qc_json = {"passed": True, "qc_ruleset": quality.QC_RULESET, "artifact_checksum": "abc"}
|
||
assert agent.qc_verdict_is_current(shot) is True
|
||
|
||
|
||
def test_qc_skipped_without_asr_is_not_a_failure(db, monkeypatch):
|
||
"""whisper 不可用时只记一条 info,绝不把镜头判成返工。"""
|
||
project, shot = _shot(db)
|
||
monkeypatch.setattr(prompt_service, "joint_av_mode", lambda db, project, shot: True)
|
||
monkeypatch.setattr(asr, "available", lambda: (False, "未找到 whisper-cli"))
|
||
report = _run(db, project, shot)
|
||
assert len(_by_key(report)) == 1 and _by_key(report)[0].passed
|
||
|
||
|
||
def test_tts_shot_is_not_listened_to(db, monkeypatch, listen):
|
||
"""走 TTS 配音的镜头:这一版音轨本来就不该有台词,听了也只会被环境声绊倒。"""
|
||
project, shot = _shot(db)
|
||
listen("随便什么声音")
|
||
monkeypatch.setattr(prompt_service, "joint_av_mode", lambda db, project, shot: False)
|
||
assert _by_key(_run(db, project, shot)) == []
|