307 lines
13 KiB
Python
307 lines
13 KiB
Python
"""内联模式守护线程:进程内跑「周期调度 + 断点续跑」。
|
||
|
||
celery 模式下调度由 celery beat 负责,本模块不启动;
|
||
inline 模式(本机无 Redis/worker)下由 FastAPI 在 startup 时拉起,
|
||
让平台在单机上就能完整跑通图2 的全流程。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import threading
|
||
import time
|
||
|
||
from sqlalchemy import func
|
||
|
||
from app.config import settings
|
||
from app.constants import TaskStatus
|
||
from app.db import SessionLocal
|
||
from app.services import dispatch, execution
|
||
|
||
log = logging.getLogger("nex.daemon")
|
||
|
||
_thread: threading.Thread | None = None
|
||
_stop = threading.Event()
|
||
_state = {"loops": 0, "last_dispatched": 0, "last_error": "", "last_tick": None, "running": False}
|
||
|
||
|
||
def is_enabled() -> bool:
|
||
return execution.backend_name() == "inline"
|
||
|
||
|
||
def is_running() -> bool:
|
||
return bool(_thread and _thread.is_alive())
|
||
|
||
|
||
def snapshot() -> dict:
|
||
return {"enabled": is_enabled(), "running": is_running(), "executor": execution.backend_name(),
|
||
"inline_pending": execution.pending(), **{k: v for k, v in _state.items() if k != "running"}}
|
||
|
||
|
||
def _tick() -> None:
|
||
from app.services import status as status_service
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
dispatch.resume_interrupted(db)
|
||
result = dispatch.dispatch_due(db)
|
||
_state["loops"] += 1
|
||
_state["last_dispatched"] = result.get("count", 0)
|
||
_state["last_error"] = ""
|
||
from datetime import datetime, timezone
|
||
|
||
_state["last_tick"] = datetime.now(timezone.utc).isoformat()
|
||
_last_tick_projects(db)
|
||
except Exception as exc: # 守护线程必须永不崩溃
|
||
_state["last_error"] = str(exc)[:500]
|
||
log.exception("调度守护线程异常")
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def _busy_project_ids(db) -> list[int]:
|
||
"""队列里还有活跃任务的项目——只有它们的状态需要守护线程推一把。"""
|
||
from app.models import Task
|
||
|
||
return sorted({row[0] for row in db.query(Task.project_id)
|
||
.filter(Task.status.in_(TaskStatus.ACTIVE),
|
||
Task.project_id.isnot(None)).distinct().all()})
|
||
|
||
|
||
def _last_tick_projects(db) -> None:
|
||
"""让项目状态视图跟上队列变化(前端轮询即可看到进度)。
|
||
|
||
早期实现是「最近 20 个项目无条件全刷」:status.refresh 会读全部镜头、任务、
|
||
时间线,再对每个产物做文件系统 exists,20 个项目实测 32~45ms,而且每轮都以
|
||
一次写事务收尾——守护线程每 5 秒把整库锁一下,用户浏览资产库时的只读请求
|
||
就排在这条锁后面,表现成「页面莫名其妙转圈」。
|
||
改成只刷真正在跑队列的项目:空闲时两条 SELECT 之后直接返回,连写事务都不发生。
|
||
任务进入终态时的状态回写本来就由 handler(jobs/handlers.py)与 deps.summary
|
||
各自 refresh + commit 负责,不会因此滞后。
|
||
"""
|
||
from app.models import Project
|
||
from app.services import status as status_service
|
||
|
||
ids = _busy_project_ids(db)
|
||
if not ids:
|
||
return
|
||
for project in db.query(Project).filter(Project.id.in_(ids)).all():
|
||
status_service.refresh(db, project, commit=False)
|
||
db.commit()
|
||
|
||
|
||
def _loop() -> None:
|
||
_state["running"] = True
|
||
log.info("内联调度守护线程已启动(间隔 %ss)", settings.dispatch_interval_seconds)
|
||
while not _stop.is_set():
|
||
_tick()
|
||
_stop.wait(max(1, settings.dispatch_interval_seconds))
|
||
_state["running"] = False
|
||
log.info("内联调度守护线程已停止")
|
||
|
||
|
||
def start(*, force: bool = False) -> bool:
|
||
if not (is_enabled() or force) or is_running():
|
||
return False
|
||
global _thread, _stop
|
||
_stop = threading.Event()
|
||
_thread = threading.Thread(target=_loop, name="nex-dispatch-daemon", daemon=True)
|
||
_thread.start()
|
||
return True
|
||
|
||
|
||
def stop(timeout: float = 5.0) -> None:
|
||
global _thread
|
||
if _thread is None:
|
||
return
|
||
_stop.set()
|
||
_thread.join(timeout=timeout)
|
||
_thread = None
|
||
|
||
|
||
def drain_queue(db, *, project_id: int | None = None, timeout: float = 900.0,
|
||
max_rounds: int = 2000, watch_task_ids=None,
|
||
watch_grace_seconds: float = 180.0) -> dict:
|
||
"""同步跑空队列:inline 模式下用于「一键执行到此步」与自动化自检。
|
||
|
||
循环执行:断点续跑 → 派发到期任务 → 等待线程池空闲,直到队列没有待办。
|
||
|
||
watch_task_ids 是「这次调用真正要等的任务」:它们只是暂时排在并发/RPM
|
||
窗口后面时不得提前收工——上一张图一跑完名额就释放,早退会让调用方误判
|
||
「任务已结束但产物缺失」(LLM 调度整轮因此前功尽弃)。真·派发不动
|
||
(模型停用等)时最多再等 watch_grace_seconds 秒后按 stalled 收工。
|
||
"""
|
||
from datetime import datetime, timezone
|
||
|
||
watch = {int(row) for row in (watch_task_ids or []) if row}
|
||
deadline = time.monotonic() + timeout
|
||
rounds = 0
|
||
rounds_limit = max(max_rounds, int(timeout / 2) + 8)
|
||
dispatched_total = 0
|
||
stalled = 0
|
||
stalled_since: float | None = None
|
||
stall_detail: dict = {}
|
||
last_pulse = _progress_pulse(db, project_id)
|
||
finished_before = _terminal_tasks(db, project_id)
|
||
while rounds < rounds_limit and time.monotonic() < deadline:
|
||
rounds += 1
|
||
dispatch.resume_interrupted(db)
|
||
result = dispatch.dispatch_due(db, project_id=project_id)
|
||
pulse = _progress_pulse(db, project_id)
|
||
live = _live_runners(db)
|
||
if result.get("count", 0) or pulse != last_pulse or live:
|
||
# 派发了新任务、队列进度变了、或者还有任务在跳心跳——
|
||
# 任何一种都说明「只是排队,不是卡死」,之前的等待计数作废。
|
||
stalled = 0
|
||
stalled_since = None
|
||
last_pulse = pulse
|
||
dispatched_total += result.get("count", 0)
|
||
execution.wait_idle(timeout=max(0.5, min(60.0, deadline - time.monotonic())))
|
||
_last_tick_projects(db)
|
||
remaining = _open_tasks(db, project_id)
|
||
if not remaining:
|
||
break
|
||
if result.get("count", 0) == 0:
|
||
# 队列里剩下的任务全部因为「模型停用 / 并发额度或 RPM 用尽」派发不动时,
|
||
# 它们等多久都不会自己动起来:确认两轮后直接收工,别把 timeout 耗满,
|
||
# 否则一次只想等自己那条质检结果的调用会被无关任务拖住整段超时。
|
||
blocked = {str(b.get("provider_model")): str(b.get("reason") or "")
|
||
for b in result.get("blocked") or []}
|
||
stuck = _open_task_keys(db, project_id)
|
||
if blocked and stuck and set(stuck) <= set(blocked) and not execution.pending():
|
||
waiting = _watch_open_tasks(db, watch)
|
||
if waiting:
|
||
# 卡住的正是自己要等的任务:先当成「暂时排在名额后面」继续等,
|
||
# 超过宽限期还动不了才认输,并把阻塞原因带回给调用方。
|
||
if live:
|
||
# 占着额度的是「正在跑」的任务:本机 GPU 串行,一条出图要占
|
||
# 几分钟甚至十几分钟,排队的那几条只能等,不能判死。
|
||
stalled_since = None
|
||
if stalled_since is None:
|
||
stalled_since = time.monotonic()
|
||
if time.monotonic() - stalled_since >= watch_grace_seconds:
|
||
stall_detail = {
|
||
"open_tasks": waiting,
|
||
"waited_seconds": round(
|
||
timeout - max(0.0, deadline - time.monotonic()), 1),
|
||
"blocked": [{"provider_model": key, "reason": blocked.get(key, ""),
|
||
"tasks": stuck[key]} for key in sorted(stuck)],
|
||
}
|
||
break
|
||
time.sleep(min(2.0, max(0.2, _seconds_to_next_retry(db, project_id) or 0.5)))
|
||
continue
|
||
stalled += 1
|
||
if stalled >= 2:
|
||
stall_detail = {
|
||
"open_tasks": sum(stuck.values()),
|
||
"waited_seconds": round(timeout - max(0.0, deadline - time.monotonic()), 1),
|
||
"blocked": [{"provider_model": key, "reason": blocked.get(key, ""),
|
||
"tasks": stuck[key]} for key in sorted(stuck)],
|
||
}
|
||
break
|
||
else:
|
||
stalled = 0
|
||
stalled_since = None
|
||
wait = min(2.0, max(0.05, _seconds_to_next_retry(db, project_id) or 0.2))
|
||
time.sleep(wait)
|
||
finished_after = _terminal_tasks(db, project_id)
|
||
return {"rounds": rounds, "dispatched": dispatched_total,
|
||
"completed": finished_after, # 该项目累计已终态任务数
|
||
"completed_during_drain": max(0, finished_after - finished_before),
|
||
"remaining": _open_tasks(db, project_id),
|
||
"watch_open": _watch_open_tasks(db, watch),
|
||
"pending_inline": execution.pending(), "timed_out": time.monotonic() >= deadline,
|
||
"stalled": bool(stall_detail), "stalled_detail": stall_detail or None,
|
||
"finished_at": datetime.now(timezone.utc).isoformat()}
|
||
|
||
|
||
def _watch_open_tasks(db, watch_ids) -> int:
|
||
"""本次调用真正要等的任务里还有几条没结束(没传 watch 时返回 0)。"""
|
||
if not watch_ids:
|
||
return 0
|
||
from app.models import Task
|
||
|
||
return db.query(Task).filter(Task.id.in_(sorted(watch_ids)),
|
||
Task.status.in_(TaskStatus.ACTIVE)).count()
|
||
|
||
|
||
#: 「执行体还活着」的判定窗口。比 stale_running_seconds(回收僵尸任务的租约,
|
||
#: 本机 1800s)紧得多:心跳断了几分钟不足以判定任务死了(不能误杀重跑),
|
||
#: 但也足够说明不该再拿它当「队列还在推进」的证据。
|
||
LIVENESS_SECONDS = 300.0
|
||
|
||
|
||
def _progress_pulse(db, project_id: int | None) -> tuple:
|
||
"""队列「确实往前挪了一点」的指纹:本项目已终态的任务数 + 未结束任务的进度总和。"""
|
||
from app.models import Task
|
||
|
||
scopes = [] if project_id is None else [Task.project_id == project_id]
|
||
terminal = db.query(func.count(Task.id)).filter(
|
||
Task.status.in_(TaskStatus.TERMINAL), *scopes).scalar() or 0
|
||
moving = db.query(func.coalesce(func.sum(Task.progress), 0.0)).filter(
|
||
Task.status.in_(TaskStatus.ACTIVE), *scopes).scalar() or 0.0
|
||
return int(terminal), round(float(moving), 4)
|
||
|
||
|
||
def _live_runners(db) -> int:
|
||
"""全库还有几条心跳在跳的 running 任务。
|
||
|
||
跨项目也计数:它们占的是同一个模型并发位(本机还是同一块 GPU),
|
||
别人的任务在跑 = 我的任务只是排队。
|
||
"""
|
||
from datetime import timedelta
|
||
|
||
from app.models import Task, now_utc
|
||
|
||
cutoff = now_utc() - timedelta(seconds=LIVENESS_SECONDS)
|
||
return db.query(func.count(Task.id)).filter(Task.status == TaskStatus.RUNNING,
|
||
Task.heartbeat_at >= cutoff).scalar() or 0
|
||
|
||
|
||
def _terminal_tasks(db, project_id: int | None) -> int:
|
||
from app.models import Task
|
||
|
||
stmt = db.query(Task).filter(Task.status.in_(TaskStatus.TERMINAL))
|
||
if project_id is not None:
|
||
stmt = stmt.filter(Task.project_id == project_id)
|
||
return stmt.count()
|
||
|
||
|
||
def _open_task_keys(db, project_id: int | None) -> dict[str, int]:
|
||
"""未结束任务按「provider|model」归堆,用来判断是否全部卡在额度上。"""
|
||
from app.models import Task
|
||
|
||
stmt = db.query(Task.provider, Task.model).filter(Task.status.in_(TaskStatus.ACTIVE))
|
||
if project_id is not None:
|
||
stmt = stmt.filter(Task.project_id == project_id)
|
||
out: dict[str, int] = {}
|
||
for provider, model in stmt.all():
|
||
key = f"{provider or ''}|{model or ''}"
|
||
out[key] = out.get(key, 0) + 1
|
||
return out
|
||
|
||
|
||
def _open_tasks(db, project_id: int | None) -> int:
|
||
from app.models import Task
|
||
|
||
stmt = db.query(Task).filter(Task.status.in_(TaskStatus.ACTIVE))
|
||
if project_id is not None:
|
||
stmt = stmt.filter(Task.project_id == project_id)
|
||
return stmt.count()
|
||
|
||
|
||
def _seconds_to_next_retry(db, project_id: int | None) -> float | None:
|
||
from app.models import Task, now_utc
|
||
|
||
stmt = db.query(Task.next_retry_at).filter(Task.status == TaskStatus.RETRYING,
|
||
Task.next_retry_at.isnot(None))
|
||
if project_id is not None:
|
||
stmt = stmt.filter(Task.project_id == project_id)
|
||
rows = [r[0] for r in stmt.all() if r[0] is not None]
|
||
if not rows:
|
||
return None
|
||
from app.services.status import as_utc
|
||
|
||
soonest = min(as_utc(r) for r in rows)
|
||
delta = (soonest - now_utc()).total_seconds()
|
||
return max(0.05, min(delta, 5.0))
|