nex_animation/backend/tests/test_drain_stalled.py

214 lines
10 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""drain_queue 不能被「永远派发不动」的任务拖满超时。
真实场景:质检接口 `POST /shots/{sid}/qc?drain=true` 只想等自己那条质检任务,
但项目里还躺着一条评论区遗留的 queued 任务(它的模型已被停用),
drain 会把 300s 超时全部耗光才返回——本次质检其实 0.3s 就跑完了。
"""
from __future__ import annotations
import time
from app.constants import TaskKind, TaskStatus
from app.models import ModelSpec, Task
from app.services import daemon
def _park_a_blocked_task(db, project_id: int) -> Task:
"""建一条 queued 任务,并把它的模型停用,使其在任何轮次里都派发不出去。"""
task = Task(project_id=project_id, kind=TaskKind.ASSET, provider="mock",
model="mock-blocked", status=TaskStatus.QUEUED)
db.add(task)
db.add(ModelSpec(provider="mock", model="mock-blocked", media_type="image",
enabled=False, max_concurrency=1, requests_per_minute=60))
db.commit()
return task
def test_drain_exits_early_when_only_blocked_tasks_remain(db, api):
project_id = api.project()
parked = _park_a_blocked_task(db, project_id)
started = time.monotonic()
drained = daemon.drain_queue(db, project_id=project_id, timeout=30.0)
elapsed = time.monotonic() - started
assert drained["timed_out"] is False, drained # 不是靠超时脱身
assert drained["stalled"] is True and drained["stalled_detail"], drained
detail = drained["stalled_detail"]
assert detail["open_tasks"] == 1
assert detail["blocked"][0]["provider_model"] == "mock|mock-blocked"
assert "已停用" in detail["blocked"][0]["reason"]
assert elapsed < 10.0, f"卡住的任务让 drain 白等了 {elapsed:.1f}s"
assert db.get(Task, parked.id).status == TaskStatus.QUEUED # 不吞掉、不误判失败
def test_drain_still_waits_for_runnable_work(db, api):
"""额度没满时不得提前收工:这是上一轮的镜像用例。"""
project_id = api.project()
task = Task(project_id=project_id, kind=TaskKind.ASSET, provider="mock",
model="mock-ok", status=TaskStatus.QUEUED)
db.add(task)
db.add(ModelSpec(provider="mock", model="mock-ok", media_type="image",
enabled=True, max_concurrency=1, requests_per_minute=60))
db.commit()
drained = daemon.drain_queue(db, project_id=project_id, timeout=120.0)
assert drained["stalled"] is False, drained
assert drained["remaining"] == 0 and drained["completed_during_drain"] >= 1
def test_watched_task_blocked_by_quota_is_not_abandoned(db, api, monkeypatch):
"""RUN #31 现场复现:关键帧 2 跑完的瞬间并发额度还没释放,
dispatch_due 连着两轮报「派发不动」,drain 直接收工,
LLM 调度因此误判「任务已结束但镜头 3 没有关键帧」而整轮失败。
调用方点名要等的任务还在排队时不得提前收工:等名额释放后照常跑完。
"""
import time as _time
project_id = api.project()
task = Task(project_id=project_id, kind=TaskKind.ASSET, provider="mock",
model="mock-late", status=TaskStatus.QUEUED)
db.add(task)
db.add(ModelSpec(provider="mock", model="mock-late", media_type="image",
enabled=True, max_concurrency=1, requests_per_minute=60))
db.commit()
real = daemon.dispatch.dispatch_due
calls = {"n": 0}
def fake(*args, **kwargs):
calls["n"] += 1
if calls["n"] <= 3: # 模拟前几轮并发额度仍被上一张图占着
return {"dispatched": [], "count": 0,
"blocked": [{"provider_model": "mock|mock-late", "reason": "并发额度用尽"}]}
return real(*args, **kwargs)
monkeypatch.setattr(daemon.dispatch, "dispatch_due", fake)
started = _time.monotonic()
drained = daemon.drain_queue(db, project_id=project_id, timeout=60.0,
watch_task_ids=[task.id])
elapsed = _time.monotonic() - started
assert drained["stalled"] is False, drained
assert drained["timed_out"] is False, drained
assert drained["remaining"] == 0 and drained["watch_open"] == 0, drained
assert calls["n"] > 3, "还没等到派发就收工了"
assert elapsed < 30.0, f"等回自己的任务白等了 {elapsed:.1f}s"
# 执行线程用的是独立会话:绕开本会话的身份映射直接回读数据库
from sqlalchemy import text
final = db.execute(text("select status from tasks where id=:i"),
{"i": task.id}).scalar_one()
assert final in TaskStatus.TERMINAL, final
def test_watched_task_that_truly_cannot_run_gives_up_after_grace(db, api):
"""反过来:模型停用是「等多久都不会动」,点名要等也不能把超时耗满。"""
import time as _time
project_id = api.project()
parked = _park_a_blocked_task(db, project_id)
started = _time.monotonic()
drained = daemon.drain_queue(db, project_id=project_id, timeout=30.0,
watch_task_ids=[parked.id], watch_grace_seconds=0.8)
elapsed = _time.monotonic() - started
assert drained["stalled"] is True and drained["stalled_detail"], drained
assert drained["watch_open"] == 1, drained
assert drained["timed_out"] is False, drained
assert 0.7 <= elapsed < 10.0, f"宽限期没有生效:{elapsed:.1f}s"
assert db.get(Task, parked.id).status == TaskStatus.QUEUED
# ─────────────────────────────────────────────────────────────────────────────
# RUN #39 现场:配音步骤的 2 条任务报「并发已满(1/1)」,180s 宽限期一到
# 就把整条 LLM 调度判死。真实原因不是模型坏了,而是本机 GPU 串行——
# 占着唯一名额的那条出图还在跑(心跳在跳),排队的只是「还得再等几分钟」。
# 于是「这一轮没派发出去」被误读成「没有任务在推进」。
def _slot_occupied_project(api, db, monkeypatch, *, model: str, heartbeat_age: float):
"""造一条「正占着模型唯一并发位」的 running 任务 + 一条排队等名额的任务。
测试环境的 stale_running_seconds=2s(只为快速回收僵尸),这里要按生产口径
把租约放宽,否则用例会先被断点续跑救活,测不到「排队」这一段。
"""
from datetime import timedelta
from app.models import now_utc
monkeypatch.setattr(daemon.dispatch.settings, "stale_running_seconds", 4000)
project_id = api.project()
db.add(ModelSpec(provider="mock", model=model, media_type="tts",
enabled=True, max_concurrency=1, requests_per_minute=60))
blocker = Task(project_id=project_id, kind=TaskKind.VOICEOVER, provider="mock",
model=model, status=TaskStatus.RUNNING, execution_backend="celery",
started_at=now_utc() - timedelta(seconds=60),
heartbeat_at=now_utc() - timedelta(seconds=heartbeat_age))
db.add(blocker)
waiter = Task(project_id=project_id, kind=TaskKind.VOICEOVER, provider="mock",
model=model, status=TaskStatus.QUEUED)
db.add(waiter)
db.commit()
return project_id, blocker, waiter
def test_live_runner_holds_the_slot_so_waiting_is_not_failure(db, api, monkeypatch):
"""占名额的任务还在跳心跳:必须继续等(最多等到总超时),不得判 stalled。"""
project_id, blocker, waiter = _slot_occupied_project(
api, db, monkeypatch, model="mock-busy", heartbeat_age=5)
started = time.monotonic()
drained = daemon.drain_queue(db, project_id=project_id, timeout=2.5,
watch_task_ids=[waiter.id], watch_grace_seconds=0.5)
elapsed = time.monotonic() - started
assert drained["stalled"] is False, drained # 排队 ≠ 失败
assert drained["watch_open"] == 1, drained
assert drained["timed_out"] is True, drained # 交给调用方再等一轮,而不是判死
assert elapsed >= 2.0, f"太早收工:{elapsed:.1f}s"
db.expire_all()
assert db.get(Task, blocker.id).status == TaskStatus.RUNNING
assert db.get(Task, waiter.id).status == TaskStatus.QUEUED
def test_silent_runner_does_not_hide_a_real_stall(db, api, monkeypatch):
"""镜像用例:占名额的那条心跳已经断了一个窗口(但还没到回收租约),
这才是真的「谁也别想推进」,宽限期到点必须认输并带回阻塞原因。"""
project_id, blocker, waiter = _slot_occupied_project(
api, db, monkeypatch, model="mock-quiet",
heartbeat_age=daemon.LIVENESS_SECONDS + 120)
started = time.monotonic()
drained = daemon.drain_queue(db, project_id=project_id, timeout=30.0,
watch_task_ids=[waiter.id], watch_grace_seconds=0.8)
elapsed = time.monotonic() - started
assert drained["stalled"] is True and drained["stalled_detail"], drained
assert drained["watch_open"] == 1, drained
assert elapsed < 12.0, f"没有心跳还干等:{elapsed:.1f}s"
reason = drained["stalled_detail"]["blocked"][0]["reason"]
assert "并发已满" in reason, reason
db.expire_all()
assert db.get(Task, waiter.id).status == TaskStatus.QUEUED # 只是认输,没有误杀任务
def test_serial_queue_of_many_tasks_is_not_a_stall(db, api):
"""总控台一次排 7 个镜头,逐条串行跑十几分钟是常态:
排队期间只要队列在推进,整轮就必须正常跑完,不得报 stalled。"""
project_id = api.project()
db.add(ModelSpec(provider="mock", model="mock-many", media_type="image",
enabled=True, max_concurrency=1, requests_per_minute=60))
tasks = [Task(project_id=project_id, kind=TaskKind.ASSET, provider="mock",
model="mock-many", status=TaskStatus.QUEUED) for _ in range(4)]
db.add_all(tasks)
db.commit()
ids = [row.id for row in tasks]
drained = daemon.drain_queue(db, project_id=project_id, timeout=60.0,
watch_task_ids=ids, watch_grace_seconds=0.5)
assert drained["stalled"] is False, drained
assert drained["timed_out"] is False, drained
assert drained["remaining"] == 0 and drained["watch_open"] == 0, drained
assert drained["completed_during_drain"] >= len(ids), drained