262 lines
13 KiB
Python
262 lines
13 KiB
Python
"""统一任务表与「平台动态」:所有跑模型的活都必须能被问出进度。
|
||
|
||
用户的原话(2026-11):「平台应该有个进度,所有跑模型的任务都应该写到一个任务表中。
|
||
异步,可以请求进度。然后界面上可以整体展示。」
|
||
|
||
在那之前平台资产库的 AI 生成是一个同步 HTTP 请求:出图实测四十几秒,反代 60s 读
|
||
超时就把连接掐了(504),而**出图线程并没有被取消**——图照样落盘,用户看不见,
|
||
只能手动刷新页面去猜它到底成没成。现在接口只做「校验 + 登记 + 派发」,
|
||
当场返回任务号,进度与产物全在 ``tasks`` 那一行里,换设备、刷新页面都接着看得到。
|
||
|
||
这一组用例锁的是三件容易被改回去的事:
|
||
1. 平台生成必须落到同一张 ``tasks`` 表,并带上母版归属(``owner_type/owner_id``)
|
||
—— 顶栏要按它点亮卡片、活动面板要按它跳回资产;
|
||
2. ``GET /api/tasks/activity`` 必须看得见它,且进行中的任务一条都不能被条数截掉;
|
||
3. 同一条母版同一类素材在跑的时候再点一次必须挡下来(409),
|
||
否则三次点击就是三次 GPU 排队、三份产物、三倍扣费。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import threading
|
||
|
||
from app.constants import TaskKind, TaskStatus
|
||
from app.providers import registry as model_registry
|
||
from app.providers.base import AssetGenResult, ProviderError
|
||
from tests.conftest import _make_image_png
|
||
|
||
|
||
def _scene(api, name: str = "星舰舰桥") -> dict:
|
||
return api.post("/api/platform/scenes",
|
||
json={"name": name, "prompt": "环形舰桥,全息星图", "lighting": "冷蓝"})["result"]
|
||
|
||
|
||
def _png(width: int = 320, height: int = 180) -> bytes:
|
||
"""一张能过「产物校验」的图:模型跑完后平台会拒绝小于 128 字节的文件。
|
||
|
||
本机没有 ffmpeg 时 ``_make_image_png`` 会退回 70 字节的 1x1 图,
|
||
那种图在这里会被判成「模型未返回有效图像文件」,用例就变成测 ffmpeg 装没装。
|
||
"""
|
||
payload = _make_image_png(width, height, "0x2f6d4f")
|
||
return payload + b"\x00" * max(0, 256 - len(payload))
|
||
|
||
|
||
def _body(prompt: str = "冷蓝全息舰桥") -> dict:
|
||
return {"kind": "scene_image", "provider": "mock", "model": "mock-image-1",
|
||
"prompt": prompt, "width": 320, "height": 180}
|
||
|
||
|
||
def _poll(api, task_id: int, done, what: str, timeout: float = 20.0) -> dict:
|
||
"""轮询任务行直到 ``done(task)`` 成立——用于观察「正在跑」这一中间态。"""
|
||
import time
|
||
|
||
deadline = time.monotonic() + timeout
|
||
task: dict = {}
|
||
while time.monotonic() < deadline:
|
||
task = api.get(f"/api/tasks/{task_id}")["task"]
|
||
if done(task):
|
||
return task
|
||
time.sleep(0.05)
|
||
raise AssertionError(f"任务 #{task_id} 未在 {timeout}s 内{what},当前 "
|
||
f"{task.get('status')} / {task.get('message')}")
|
||
|
||
|
||
# ---------------------------------------------------------------- 落进统一任务表
|
||
def test_platform_generation_lands_in_the_unified_task_table(api):
|
||
"""生成不再是一个「请求-等待-响应」,而是一条可查询、可重试、可取消的任务。"""
|
||
scene = _scene(api)
|
||
task = api.generate(f"/api/platform/scene/{scene['id']}/generate", _body(), task_only=True)
|
||
|
||
assert task["kind"] == TaskKind.PLATFORM_ASSET
|
||
# 母版本就不属于任何项目:挂 0/-1 会让每一条按项目统计的 SQL 多背一个特例
|
||
assert task["project_id"] is None and task["shot_id"] is None
|
||
assert task["owner_type"] == "scene" and task["owner_id"] == scene["id"]
|
||
assert task["status"] == TaskStatus.SUCCEEDED
|
||
assert scene["name"] in task["name"]
|
||
assert task["output_url"].startswith("/media/platform/")
|
||
# 产物反向指回任务号:素材出问题时能从文件一路追回是哪一次生成
|
||
assert task["result"]["media"]["metadata"]["task_id"] == task["id"]
|
||
assert task["elapsed_ms"] >= 0 and task["payload"]["asset_kind"] == "scene_image"
|
||
|
||
|
||
def test_activity_shows_finished_platform_work(api):
|
||
"""顶栏那一份「平台动态」就是统一任务表的切片,不需要第二个数据源。"""
|
||
scene = _scene(api)
|
||
task = api.generate(f"/api/platform/scene/{scene['id']}/generate", _body(), task_only=True)
|
||
|
||
panel = api.get("/api/tasks/activity")["result"]
|
||
assert panel["active"] == 0
|
||
assert panel["counts"].get(TaskStatus.SUCCEEDED) == 1
|
||
assert panel["window_hours"] > 0
|
||
|
||
row = next(item for item in panel["tasks"] if item["id"] == task["id"])
|
||
assert row["owner_type"] == "scene" and row["owner_id"] == scene["id"]
|
||
assert row["project_id"] is None
|
||
# 面板上必须能说出「这是哪条母版的什么」,只剩一个转圈图标等于没说
|
||
assert scene["name"] in row["name"]
|
||
assert row["message"] and row["result"]["media"]["id"]
|
||
|
||
|
||
def test_activity_ignores_settled_work_outside_the_window(api):
|
||
"""历史只留最近一段时间:三天前跑成功的那条不该一直挂在顶栏角标旁边。"""
|
||
from datetime import timedelta
|
||
|
||
from app.db import SessionLocal
|
||
from app.models import Task, now_utc
|
||
from app.services import status as status_service
|
||
|
||
scene = _scene(api)
|
||
task = api.generate(f"/api/platform/scene/{scene['id']}/generate", _body(), task_only=True)
|
||
with SessionLocal() as session:
|
||
row = session.get(Task, task["id"])
|
||
# 落定时间推到窗口之外:它已经从「刚刚完成」那一格里掉出去了
|
||
row.updated_at = now_utc() - timedelta(hours=status_service.ACTIVITY_WINDOW.total_seconds() / 3600 + 1)
|
||
session.commit()
|
||
|
||
panel = api.get("/api/tasks/activity")["result"]
|
||
assert [item["id"] for item in panel["tasks"]] == []
|
||
assert panel["counts"] == {}
|
||
|
||
|
||
# ---------------------------------------------------------------- 进行中的可见性
|
||
def _blocking_adapter(release: threading.Event, notes: list[str]):
|
||
"""替身模型:卡在一个事件上,并在深处报一句「算力排队」——真实链路正是这个样子。
|
||
|
||
``services/progress`` 靠 thread-local 里的 task_id 写回任务行,
|
||
跑在 inline 线程池里的执行体如果没绑定 task_id,这里写进去的就是空气,
|
||
界面上也永远看不到「排在别人后面」和「模型正在算」的区别。
|
||
"""
|
||
from app.services import progress
|
||
|
||
class Blocking:
|
||
name = "blocking"
|
||
|
||
def __call__(self, db, request, media_type, provider, model):
|
||
notes.append(progress.current_task_id() and "bound" or "unbound")
|
||
progress.report("排队中:等待算力空闲")
|
||
assert release.wait(20), "替身模型没有被放行,用例会挂在这里"
|
||
with open(request.output_path, "wb") as handle:
|
||
handle.write(_png())
|
||
return AssetGenResult(file_path=str(request.output_path), metadata={"model": model})
|
||
|
||
return Blocking()
|
||
|
||
|
||
def test_a_running_generation_is_visible_and_blocks_the_second_click(api, monkeypatch):
|
||
"""跑模型期间:界面看得见它、卡片点不动它,一次点击只该有一份产物。"""
|
||
release = threading.Event()
|
||
notes: list[str] = []
|
||
monkeypatch.setattr(model_registry, "generate_asset", _blocking_adapter(release, notes))
|
||
scene = _scene(api)
|
||
|
||
# 202 而不是 200:接口只做「校验 + 登记 + 派发」,出图这件事交给后台
|
||
first = api.post(f"/api/platform/scene/{scene['id']}/generate", json=_body(), expect=202)
|
||
assert first["result"]["status"] in (TaskStatus.QUEUED, TaskStatus.RUNNING)
|
||
task_id = first["result"]["id"]
|
||
# 深处那句排队说明必须能到达任务行:出图四十多秒里,用户看的就只有这一行字
|
||
running = _poll(api, task_id, lambda t: t["message"] == "排队中:等待算力空闲",
|
||
"报出算力排队说明")
|
||
assert running["status"] == TaskStatus.RUNNING
|
||
assert notes == ["bound"], "执行体必须把 task_id 绑进线程,否则 progress.report 写不进来"
|
||
|
||
panel = api.get("/api/tasks/activity")["result"]
|
||
assert panel["active"] == 1
|
||
assert panel["tasks"][0]["id"] == task_id
|
||
|
||
# 用户没等完又点了一次:挡下来,而不是悄悄排第二份
|
||
again = api.post(f"/api/platform/scene/{scene['id']}/generate",
|
||
json=_body("换个提示词再试"), expect=409)
|
||
assert "正在生成中" in again["detail"] and scene["name"] in again["detail"]
|
||
|
||
release.set()
|
||
from app.services import execution
|
||
|
||
assert execution.wait_idle(30)
|
||
done = api.get(f"/api/tasks/{task_id}")["task"]
|
||
assert done["status"] == TaskStatus.SUCCEEDED
|
||
assert api.get("/api/tasks/activity")["result"]["active"] == 0
|
||
|
||
|
||
def test_a_running_platform_task_can_be_cancelled_without_a_project(api, monkeypatch):
|
||
"""取消按钮过去靠「按项目取父记录」拿权限,平台任务没有项目,会凭空 404。"""
|
||
release = threading.Event()
|
||
monkeypatch.setattr(model_registry, "generate_asset", _blocking_adapter(release, []))
|
||
scene = _scene(api)
|
||
task_id = api.post(f"/api/platform/scene/{scene['id']}/generate", json=_body(),
|
||
expect=202)["result"]["id"]
|
||
_poll(api, task_id, lambda t: t["status"] == TaskStatus.RUNNING, "开始执行")
|
||
|
||
cancelled = api.post(f"/api/tasks/{task_id}/cancel", json={"reason": "提示词写错了"})["result"]
|
||
assert cancelled["status"] == TaskStatus.CANCELLED
|
||
|
||
release.set()
|
||
from app.services import execution
|
||
|
||
assert execution.wait_idle(30)
|
||
# 出图线程跑完了也不许把「已取消」改写成成功:用户明确说过不要
|
||
final = api.get(f"/api/tasks/{task_id}")["task"]
|
||
assert final["status"] == TaskStatus.CANCELLED
|
||
|
||
|
||
def test_a_transient_failure_stays_on_the_panel_while_it_waits_to_retry(api, monkeypatch):
|
||
"""连不上算力是瞬时故障:平台自己会再试一次,这期间它在面板上仍然是「进行中」。
|
||
|
||
挡在这里的是「重试中的任务从角标上消失了」这一类回归——用户看到的是
|
||
进度条归零、什么都没发生,于是以为失败了,其实排队等下一次派发。
|
||
"""
|
||
def busy(db, request, media_type, provider, model):
|
||
raise ProviderError("算力服务无响应:http://comfyui:9200", code="gpu_unreachable")
|
||
|
||
monkeypatch.setattr(model_registry, "generate_asset", busy)
|
||
scene = _scene(api)
|
||
created = api.post(f"/api/platform/scene/{scene['id']}/generate", json=_body(),
|
||
expect=202)["result"]
|
||
|
||
from app.services import execution
|
||
|
||
assert execution.wait_idle(30)
|
||
task = api.get(f"/api/tasks/{created['id']}")["task"]
|
||
assert task["status"] == TaskStatus.RETRYING
|
||
assert task["error_code"] == "gpu_unreachable"
|
||
assert task["max_attempts"] == 2 and task["next_retry_at"] is not None
|
||
|
||
panel = api.get("/api/tasks/activity")["result"]
|
||
assert panel["active"] == 1
|
||
row = next(item for item in panel["tasks"] if item["id"] == created["id"])
|
||
# 面板上直接把供应商那句原话摊开,用户才知道去模型管理中心查地址,而不是重传素材
|
||
assert "算力服务无响应" in row["error_message"]
|
||
|
||
|
||
def test_a_failed_platform_task_can_be_retried_from_the_panel(api, monkeypatch):
|
||
"""面板上的「重试」要能立刻再排一次,而不是等下一轮调度扫描(本机 5s)。
|
||
|
||
这里用 retryable=False 的错误(提示词超限这一类):平台自己不会反复撞墙,
|
||
改好之后由人点重试——那正是活动面板上那颗按钮的用途。
|
||
"""
|
||
fixed = {"ok": False}
|
||
|
||
def adapter(db, request, media_type, provider, model):
|
||
if not fixed["ok"]:
|
||
raise ProviderError("提示词超出模型长度上限", code="prompt_too_long", retryable=False)
|
||
with open(request.output_path, "wb") as handle:
|
||
handle.write(_png())
|
||
return AssetGenResult(file_path=str(request.output_path), metadata={"model": model})
|
||
|
||
monkeypatch.setattr(model_registry, "generate_asset", adapter)
|
||
scene = _scene(api)
|
||
task_id = api.post(f"/api/platform/scene/{scene['id']}/generate", json=_body(),
|
||
expect=202)["result"]["id"]
|
||
|
||
from app.services import execution
|
||
|
||
assert execution.wait_idle(30)
|
||
failed = api.get(f"/api/tasks/{task_id}")["task"]
|
||
assert failed["status"] == TaskStatus.FAILED and failed["attempts"] == 1
|
||
|
||
fixed["ok"] = True
|
||
retried = api.post(f"/api/tasks/{task_id}/retry", json={})["result"]
|
||
assert retried["status"] in (TaskStatus.QUEUED, TaskStatus.RUNNING, TaskStatus.SUCCEEDED)
|
||
assert execution.wait_idle(30)
|
||
done = api.get(f"/api/tasks/{task_id}")["task"]
|
||
assert done["status"] == TaskStatus.SUCCEEDED
|
||
assert done["result"]["media"]["metadata"]["task_id"] == task_id
|