214 lines
10 KiB
Python
214 lines
10 KiB
Python
"""drain_queue 不能被「永远派发不动」的任务拖满超时。
|
||
|
||
真实场景:质检接口 `POST /shots/{sid}/qc?drain=true` 只想等自己那条质检任务,
|
||
但项目里还躺着一条评论区遗留的 queued 任务(它的模型已被停用),
|
||
drain 会把 300s 超时全部耗光才返回——本次质检其实 0.3s 就跑完了。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
|
||
from app.constants import TaskKind, TaskStatus
|
||
from app.models import ModelSpec, Task
|
||
from app.services import daemon
|
||
|
||
|
||
def _park_a_blocked_task(db, project_id: int) -> Task:
|
||
"""建一条 queued 任务,并把它的模型停用,使其在任何轮次里都派发不出去。"""
|
||
task = Task(project_id=project_id, kind=TaskKind.ASSET, provider="mock",
|
||
model="mock-blocked", status=TaskStatus.QUEUED)
|
||
db.add(task)
|
||
db.add(ModelSpec(provider="mock", model="mock-blocked", media_type="image",
|
||
enabled=False, max_concurrency=1, requests_per_minute=60))
|
||
db.commit()
|
||
return task
|
||
|
||
|
||
def test_drain_exits_early_when_only_blocked_tasks_remain(db, api):
|
||
project_id = api.project()
|
||
parked = _park_a_blocked_task(db, project_id)
|
||
|
||
started = time.monotonic()
|
||
drained = daemon.drain_queue(db, project_id=project_id, timeout=30.0)
|
||
elapsed = time.monotonic() - started
|
||
|
||
assert drained["timed_out"] is False, drained # 不是靠超时脱身
|
||
assert drained["stalled"] is True and drained["stalled_detail"], drained
|
||
detail = drained["stalled_detail"]
|
||
assert detail["open_tasks"] == 1
|
||
assert detail["blocked"][0]["provider_model"] == "mock|mock-blocked"
|
||
assert "已停用" in detail["blocked"][0]["reason"]
|
||
assert elapsed < 10.0, f"卡住的任务让 drain 白等了 {elapsed:.1f}s"
|
||
assert db.get(Task, parked.id).status == TaskStatus.QUEUED # 不吞掉、不误判失败
|
||
|
||
|
||
def test_drain_still_waits_for_runnable_work(db, api):
|
||
"""额度没满时不得提前收工:这是上一轮的镜像用例。"""
|
||
project_id = api.project()
|
||
task = Task(project_id=project_id, kind=TaskKind.ASSET, provider="mock",
|
||
model="mock-ok", status=TaskStatus.QUEUED)
|
||
db.add(task)
|
||
db.add(ModelSpec(provider="mock", model="mock-ok", media_type="image",
|
||
enabled=True, max_concurrency=1, requests_per_minute=60))
|
||
db.commit()
|
||
|
||
drained = daemon.drain_queue(db, project_id=project_id, timeout=120.0)
|
||
assert drained["stalled"] is False, drained
|
||
assert drained["remaining"] == 0 and drained["completed_during_drain"] >= 1
|
||
|
||
|
||
def test_watched_task_blocked_by_quota_is_not_abandoned(db, api, monkeypatch):
|
||
"""RUN #31 现场复现:关键帧 2 跑完的瞬间并发额度还没释放,
|
||
dispatch_due 连着两轮报「派发不动」,drain 直接收工,
|
||
LLM 调度因此误判「任务已结束但镜头 3 没有关键帧」而整轮失败。
|
||
|
||
调用方点名要等的任务还在排队时不得提前收工:等名额释放后照常跑完。
|
||
"""
|
||
import time as _time
|
||
|
||
project_id = api.project()
|
||
task = Task(project_id=project_id, kind=TaskKind.ASSET, provider="mock",
|
||
model="mock-late", status=TaskStatus.QUEUED)
|
||
db.add(task)
|
||
db.add(ModelSpec(provider="mock", model="mock-late", media_type="image",
|
||
enabled=True, max_concurrency=1, requests_per_minute=60))
|
||
db.commit()
|
||
|
||
real = daemon.dispatch.dispatch_due
|
||
calls = {"n": 0}
|
||
|
||
def fake(*args, **kwargs):
|
||
calls["n"] += 1
|
||
if calls["n"] <= 3: # 模拟前几轮并发额度仍被上一张图占着
|
||
return {"dispatched": [], "count": 0,
|
||
"blocked": [{"provider_model": "mock|mock-late", "reason": "并发额度用尽"}]}
|
||
return real(*args, **kwargs)
|
||
|
||
monkeypatch.setattr(daemon.dispatch, "dispatch_due", fake)
|
||
started = _time.monotonic()
|
||
drained = daemon.drain_queue(db, project_id=project_id, timeout=60.0,
|
||
watch_task_ids=[task.id])
|
||
elapsed = _time.monotonic() - started
|
||
|
||
assert drained["stalled"] is False, drained
|
||
assert drained["timed_out"] is False, drained
|
||
assert drained["remaining"] == 0 and drained["watch_open"] == 0, drained
|
||
assert calls["n"] > 3, "还没等到派发就收工了"
|
||
assert elapsed < 30.0, f"等回自己的任务白等了 {elapsed:.1f}s"
|
||
# 执行线程用的是独立会话:绕开本会话的身份映射直接回读数据库
|
||
from sqlalchemy import text
|
||
|
||
final = db.execute(text("select status from tasks where id=:i"),
|
||
{"i": task.id}).scalar_one()
|
||
assert final in TaskStatus.TERMINAL, final
|
||
|
||
|
||
def test_watched_task_that_truly_cannot_run_gives_up_after_grace(db, api):
|
||
"""反过来:模型停用是「等多久都不会动」,点名要等也不能把超时耗满。"""
|
||
import time as _time
|
||
|
||
project_id = api.project()
|
||
parked = _park_a_blocked_task(db, project_id)
|
||
|
||
started = _time.monotonic()
|
||
drained = daemon.drain_queue(db, project_id=project_id, timeout=30.0,
|
||
watch_task_ids=[parked.id], watch_grace_seconds=0.8)
|
||
elapsed = _time.monotonic() - started
|
||
|
||
assert drained["stalled"] is True and drained["stalled_detail"], drained
|
||
assert drained["watch_open"] == 1, drained
|
||
assert drained["timed_out"] is False, drained
|
||
assert 0.7 <= elapsed < 10.0, f"宽限期没有生效:{elapsed:.1f}s"
|
||
assert db.get(Task, parked.id).status == TaskStatus.QUEUED
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# RUN #39 现场:配音步骤的 2 条任务报「并发已满(1/1)」,180s 宽限期一到
|
||
# 就把整条 LLM 调度判死。真实原因不是模型坏了,而是本机 GPU 串行——
|
||
# 占着唯一名额的那条出图还在跑(心跳在跳),排队的只是「还得再等几分钟」。
|
||
# 于是「这一轮没派发出去」被误读成「没有任务在推进」。
|
||
|
||
|
||
def _slot_occupied_project(api, db, monkeypatch, *, model: str, heartbeat_age: float):
|
||
"""造一条「正占着模型唯一并发位」的 running 任务 + 一条排队等名额的任务。
|
||
|
||
测试环境的 stale_running_seconds=2s(只为快速回收僵尸),这里要按生产口径
|
||
把租约放宽,否则用例会先被断点续跑救活,测不到「排队」这一段。
|
||
"""
|
||
from datetime import timedelta
|
||
|
||
from app.models import now_utc
|
||
|
||
monkeypatch.setattr(daemon.dispatch.settings, "stale_running_seconds", 4000)
|
||
project_id = api.project()
|
||
db.add(ModelSpec(provider="mock", model=model, media_type="tts",
|
||
enabled=True, max_concurrency=1, requests_per_minute=60))
|
||
blocker = Task(project_id=project_id, kind=TaskKind.VOICEOVER, provider="mock",
|
||
model=model, status=TaskStatus.RUNNING, execution_backend="celery",
|
||
started_at=now_utc() - timedelta(seconds=60),
|
||
heartbeat_at=now_utc() - timedelta(seconds=heartbeat_age))
|
||
db.add(blocker)
|
||
waiter = Task(project_id=project_id, kind=TaskKind.VOICEOVER, provider="mock",
|
||
model=model, status=TaskStatus.QUEUED)
|
||
db.add(waiter)
|
||
db.commit()
|
||
return project_id, blocker, waiter
|
||
|
||
|
||
def test_live_runner_holds_the_slot_so_waiting_is_not_failure(db, api, monkeypatch):
|
||
"""占名额的任务还在跳心跳:必须继续等(最多等到总超时),不得判 stalled。"""
|
||
project_id, blocker, waiter = _slot_occupied_project(
|
||
api, db, monkeypatch, model="mock-busy", heartbeat_age=5)
|
||
started = time.monotonic()
|
||
drained = daemon.drain_queue(db, project_id=project_id, timeout=2.5,
|
||
watch_task_ids=[waiter.id], watch_grace_seconds=0.5)
|
||
elapsed = time.monotonic() - started
|
||
|
||
assert drained["stalled"] is False, drained # 排队 ≠ 失败
|
||
assert drained["watch_open"] == 1, drained
|
||
assert drained["timed_out"] is True, drained # 交给调用方再等一轮,而不是判死
|
||
assert elapsed >= 2.0, f"太早收工:{elapsed:.1f}s"
|
||
db.expire_all()
|
||
assert db.get(Task, blocker.id).status == TaskStatus.RUNNING
|
||
assert db.get(Task, waiter.id).status == TaskStatus.QUEUED
|
||
|
||
|
||
def test_silent_runner_does_not_hide_a_real_stall(db, api, monkeypatch):
|
||
"""镜像用例:占名额的那条心跳已经断了一个窗口(但还没到回收租约),
|
||
这才是真的「谁也别想推进」,宽限期到点必须认输并带回阻塞原因。"""
|
||
project_id, blocker, waiter = _slot_occupied_project(
|
||
api, db, monkeypatch, model="mock-quiet",
|
||
heartbeat_age=daemon.LIVENESS_SECONDS + 120)
|
||
started = time.monotonic()
|
||
drained = daemon.drain_queue(db, project_id=project_id, timeout=30.0,
|
||
watch_task_ids=[waiter.id], watch_grace_seconds=0.8)
|
||
elapsed = time.monotonic() - started
|
||
|
||
assert drained["stalled"] is True and drained["stalled_detail"], drained
|
||
assert drained["watch_open"] == 1, drained
|
||
assert elapsed < 12.0, f"没有心跳还干等:{elapsed:.1f}s"
|
||
reason = drained["stalled_detail"]["blocked"][0]["reason"]
|
||
assert "并发已满" in reason, reason
|
||
db.expire_all()
|
||
assert db.get(Task, waiter.id).status == TaskStatus.QUEUED # 只是认输,没有误杀任务
|
||
|
||
|
||
def test_serial_queue_of_many_tasks_is_not_a_stall(db, api):
|
||
"""总控台一次排 7 个镜头,逐条串行跑十几分钟是常态:
|
||
排队期间只要队列在推进,整轮就必须正常跑完,不得报 stalled。"""
|
||
project_id = api.project()
|
||
db.add(ModelSpec(provider="mock", model="mock-many", media_type="image",
|
||
enabled=True, max_concurrency=1, requests_per_minute=60))
|
||
tasks = [Task(project_id=project_id, kind=TaskKind.ASSET, provider="mock",
|
||
model="mock-many", status=TaskStatus.QUEUED) for _ in range(4)]
|
||
db.add_all(tasks)
|
||
db.commit()
|
||
ids = [row.id for row in tasks]
|
||
|
||
drained = daemon.drain_queue(db, project_id=project_id, timeout=60.0,
|
||
watch_task_ids=ids, watch_grace_seconds=0.5)
|
||
|
||
assert drained["stalled"] is False, drained
|
||
assert drained["timed_out"] is False, drained
|
||
assert drained["remaining"] == 0 and drained["watch_open"] == 0, drained
|
||
assert drained["completed_during_drain"] >= len(ids), drained
|