nex_animation/backend/tests/test_task_activity.py

262 lines
13 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""统一任务表与「平台动态」:所有跑模型的活都必须能被问出进度。
用户的原话(2026-11):「平台应该有个进度,所有跑模型的任务都应该写到一个任务表中。
异步,可以请求进度。然后界面上可以整体展示。」
在那之前平台资产库的 AI 生成是一个同步 HTTP 请求:出图实测四十几秒,反代 60s 读
超时就把连接掐了(504),而**出图线程并没有被取消**——图照样落盘,用户看不见,
只能手动刷新页面去猜它到底成没成。现在接口只做「校验 + 登记 + 派发」,
当场返回任务号,进度与产物全在 ``tasks`` 那一行里,换设备、刷新页面都接着看得到。
这一组用例锁的是三件容易被改回去的事:
1. 平台生成必须落到同一张 ``tasks`` 表,并带上母版归属(``owner_type/owner_id``)
—— 顶栏要按它点亮卡片、活动面板要按它跳回资产;
2. ``GET /api/tasks/activity`` 必须看得见它,且进行中的任务一条都不能被条数截掉;
3. 同一条母版同一类素材在跑的时候再点一次必须挡下来(409),
否则三次点击就是三次 GPU 排队、三份产物、三倍扣费。
"""
from __future__ import annotations
import threading
from app.constants import TaskKind, TaskStatus
from app.providers import registry as model_registry
from app.providers.base import AssetGenResult, ProviderError
from tests.conftest import _make_image_png
def _scene(api, name: str = "星舰舰桥") -> dict:
return api.post("/api/platform/scenes",
json={"name": name, "prompt": "环形舰桥,全息星图", "lighting": "冷蓝"})["result"]
def _png(width: int = 320, height: int = 180) -> bytes:
"""一张能过「产物校验」的图:模型跑完后平台会拒绝小于 128 字节的文件。
本机没有 ffmpeg 时 ``_make_image_png`` 会退回 70 字节的 1x1 图,
那种图在这里会被判成「模型未返回有效图像文件」,用例就变成测 ffmpeg 装没装。
"""
payload = _make_image_png(width, height, "0x2f6d4f")
return payload + b"\x00" * max(0, 256 - len(payload))
def _body(prompt: str = "冷蓝全息舰桥") -> dict:
return {"kind": "scene_image", "provider": "mock", "model": "mock-image-1",
"prompt": prompt, "width": 320, "height": 180}
def _poll(api, task_id: int, done, what: str, timeout: float = 20.0) -> dict:
"""轮询任务行直到 ``done(task)`` 成立——用于观察「正在跑」这一中间态。"""
import time
deadline = time.monotonic() + timeout
task: dict = {}
while time.monotonic() < deadline:
task = api.get(f"/api/tasks/{task_id}")["task"]
if done(task):
return task
time.sleep(0.05)
raise AssertionError(f"任务 #{task_id} 未在 {timeout}s 内{what},当前 "
f"{task.get('status')} / {task.get('message')}")
# ---------------------------------------------------------------- 落进统一任务表
def test_platform_generation_lands_in_the_unified_task_table(api):
"""生成不再是一个「请求-等待-响应」,而是一条可查询、可重试、可取消的任务。"""
scene = _scene(api)
task = api.generate(f"/api/platform/scene/{scene['id']}/generate", _body(), task_only=True)
assert task["kind"] == TaskKind.PLATFORM_ASSET
# 母版本就不属于任何项目:挂 0/-1 会让每一条按项目统计的 SQL 多背一个特例
assert task["project_id"] is None and task["shot_id"] is None
assert task["owner_type"] == "scene" and task["owner_id"] == scene["id"]
assert task["status"] == TaskStatus.SUCCEEDED
assert scene["name"] in task["name"]
assert task["output_url"].startswith("/media/platform/")
# 产物反向指回任务号:素材出问题时能从文件一路追回是哪一次生成
assert task["result"]["media"]["metadata"]["task_id"] == task["id"]
assert task["elapsed_ms"] >= 0 and task["payload"]["asset_kind"] == "scene_image"
def test_activity_shows_finished_platform_work(api):
"""顶栏那一份「平台动态」就是统一任务表的切片,不需要第二个数据源。"""
scene = _scene(api)
task = api.generate(f"/api/platform/scene/{scene['id']}/generate", _body(), task_only=True)
panel = api.get("/api/tasks/activity")["result"]
assert panel["active"] == 0
assert panel["counts"].get(TaskStatus.SUCCEEDED) == 1
assert panel["window_hours"] > 0
row = next(item for item in panel["tasks"] if item["id"] == task["id"])
assert row["owner_type"] == "scene" and row["owner_id"] == scene["id"]
assert row["project_id"] is None
# 面板上必须能说出「这是哪条母版的什么」,只剩一个转圈图标等于没说
assert scene["name"] in row["name"]
assert row["message"] and row["result"]["media"]["id"]
def test_activity_ignores_settled_work_outside_the_window(api):
"""历史只留最近一段时间:三天前跑成功的那条不该一直挂在顶栏角标旁边。"""
from datetime import timedelta
from app.db import SessionLocal
from app.models import Task, now_utc
from app.services import status as status_service
scene = _scene(api)
task = api.generate(f"/api/platform/scene/{scene['id']}/generate", _body(), task_only=True)
with SessionLocal() as session:
row = session.get(Task, task["id"])
# 落定时间推到窗口之外:它已经从「刚刚完成」那一格里掉出去了
row.updated_at = now_utc() - timedelta(hours=status_service.ACTIVITY_WINDOW.total_seconds() / 3600 + 1)
session.commit()
panel = api.get("/api/tasks/activity")["result"]
assert [item["id"] for item in panel["tasks"]] == []
assert panel["counts"] == {}
# ---------------------------------------------------------------- 进行中的可见性
def _blocking_adapter(release: threading.Event, notes: list[str]):
"""替身模型:卡在一个事件上,并在深处报一句「算力排队」——真实链路正是这个样子。
``services/progress`` 靠 thread-local 里的 task_id 写回任务行,
跑在 inline 线程池里的执行体如果没绑定 task_id,这里写进去的就是空气,
界面上也永远看不到「排在别人后面」和「模型正在算」的区别。
"""
from app.services import progress
class Blocking:
name = "blocking"
def __call__(self, db, request, media_type, provider, model):
notes.append(progress.current_task_id() and "bound" or "unbound")
progress.report("排队中:等待算力空闲")
assert release.wait(20), "替身模型没有被放行,用例会挂在这里"
with open(request.output_path, "wb") as handle:
handle.write(_png())
return AssetGenResult(file_path=str(request.output_path), metadata={"model": model})
return Blocking()
def test_a_running_generation_is_visible_and_blocks_the_second_click(api, monkeypatch):
"""跑模型期间:界面看得见它、卡片点不动它,一次点击只该有一份产物。"""
release = threading.Event()
notes: list[str] = []
monkeypatch.setattr(model_registry, "generate_asset", _blocking_adapter(release, notes))
scene = _scene(api)
# 202 而不是 200:接口只做「校验 + 登记 + 派发」,出图这件事交给后台
first = api.post(f"/api/platform/scene/{scene['id']}/generate", json=_body(), expect=202)
assert first["result"]["status"] in (TaskStatus.QUEUED, TaskStatus.RUNNING)
task_id = first["result"]["id"]
# 深处那句排队说明必须能到达任务行:出图四十多秒里,用户看的就只有这一行字
running = _poll(api, task_id, lambda t: t["message"] == "排队中:等待算力空闲",
"报出算力排队说明")
assert running["status"] == TaskStatus.RUNNING
assert notes == ["bound"], "执行体必须把 task_id 绑进线程,否则 progress.report 写不进来"
panel = api.get("/api/tasks/activity")["result"]
assert panel["active"] == 1
assert panel["tasks"][0]["id"] == task_id
# 用户没等完又点了一次:挡下来,而不是悄悄排第二份
again = api.post(f"/api/platform/scene/{scene['id']}/generate",
json=_body("换个提示词再试"), expect=409)
assert "正在生成中" in again["detail"] and scene["name"] in again["detail"]
release.set()
from app.services import execution
assert execution.wait_idle(30)
done = api.get(f"/api/tasks/{task_id}")["task"]
assert done["status"] == TaskStatus.SUCCEEDED
assert api.get("/api/tasks/activity")["result"]["active"] == 0
def test_a_running_platform_task_can_be_cancelled_without_a_project(api, monkeypatch):
"""取消按钮过去靠「按项目取父记录」拿权限,平台任务没有项目,会凭空 404。"""
release = threading.Event()
monkeypatch.setattr(model_registry, "generate_asset", _blocking_adapter(release, []))
scene = _scene(api)
task_id = api.post(f"/api/platform/scene/{scene['id']}/generate", json=_body(),
expect=202)["result"]["id"]
_poll(api, task_id, lambda t: t["status"] == TaskStatus.RUNNING, "开始执行")
cancelled = api.post(f"/api/tasks/{task_id}/cancel", json={"reason": "提示词写错了"})["result"]
assert cancelled["status"] == TaskStatus.CANCELLED
release.set()
from app.services import execution
assert execution.wait_idle(30)
# 出图线程跑完了也不许把「已取消」改写成成功:用户明确说过不要
final = api.get(f"/api/tasks/{task_id}")["task"]
assert final["status"] == TaskStatus.CANCELLED
def test_a_transient_failure_stays_on_the_panel_while_it_waits_to_retry(api, monkeypatch):
"""连不上算力是瞬时故障:平台自己会再试一次,这期间它在面板上仍然是「进行中」。
挡在这里的是「重试中的任务从角标上消失了」这一类回归——用户看到的是
进度条归零、什么都没发生,于是以为失败了,其实排队等下一次派发。
"""
def busy(db, request, media_type, provider, model):
raise ProviderError("算力服务无响应:http://comfyui:9200", code="gpu_unreachable")
monkeypatch.setattr(model_registry, "generate_asset", busy)
scene = _scene(api)
created = api.post(f"/api/platform/scene/{scene['id']}/generate", json=_body(),
expect=202)["result"]
from app.services import execution
assert execution.wait_idle(30)
task = api.get(f"/api/tasks/{created['id']}")["task"]
assert task["status"] == TaskStatus.RETRYING
assert task["error_code"] == "gpu_unreachable"
assert task["max_attempts"] == 2 and task["next_retry_at"] is not None
panel = api.get("/api/tasks/activity")["result"]
assert panel["active"] == 1
row = next(item for item in panel["tasks"] if item["id"] == created["id"])
# 面板上直接把供应商那句原话摊开,用户才知道去模型管理中心查地址,而不是重传素材
assert "算力服务无响应" in row["error_message"]
def test_a_failed_platform_task_can_be_retried_from_the_panel(api, monkeypatch):
"""面板上的「重试」要能立刻再排一次,而不是等下一轮调度扫描(本机 5s)。
这里用 retryable=False 的错误(提示词超限这一类):平台自己不会反复撞墙,
改好之后由人点重试——那正是活动面板上那颗按钮的用途。
"""
fixed = {"ok": False}
def adapter(db, request, media_type, provider, model):
if not fixed["ok"]:
raise ProviderError("提示词超出模型长度上限", code="prompt_too_long", retryable=False)
with open(request.output_path, "wb") as handle:
handle.write(_png())
return AssetGenResult(file_path=str(request.output_path), metadata={"model": model})
monkeypatch.setattr(model_registry, "generate_asset", adapter)
scene = _scene(api)
task_id = api.post(f"/api/platform/scene/{scene['id']}/generate", json=_body(),
expect=202)["result"]["id"]
from app.services import execution
assert execution.wait_idle(30)
failed = api.get(f"/api/tasks/{task_id}")["task"]
assert failed["status"] == TaskStatus.FAILED and failed["attempts"] == 1
fixed["ok"] = True
retried = api.post(f"/api/tasks/{task_id}/retry", json={})["result"]
assert retried["status"] in (TaskStatus.QUEUED, TaskStatus.RUNNING, TaskStatus.SUCCEEDED)
assert execution.wait_idle(30)
done = api.get(f"/api/tasks/{task_id}")["task"]
assert done["status"] == TaskStatus.SUCCEEDED
assert done["result"]["media"]["metadata"]["task_id"] == task_id