nex_animation/backend/app/services/dispatch.py

778 lines
42 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""统一任务编排:任务清单生成 → 限流派发 → 失败重试 → 断点续跑。
对应业务流程:检查素材与参数 → 生成任务清单 → 任务队列调度 → 调用视频模型 →
生成成功?否→记录错误并重试;是→下载并保存视频。
"""
from __future__ import annotations
import logging
import os
import socket
import time
from dataclasses import dataclass, field
from datetime import datetime, timedelta
import httpx
from sqlalchemy import or_, select, update
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session
from app.config import settings
from app.constants import ShotStatus, TaskKind, TaskStatus
from app.models import CostRecord, Project, Shot, Task, now_utc
from app.providers.base import ProviderError
from app.providers import registry as model_registry
from app.services import capacity, costing, execution, events, preflight, prompt, status, storage
log = logging.getLogger("nex.dispatch")
RETRYABLE_HINTS = ("timeout", "timed out", "temporarily", "rate limit", "限流", "网络", "connection",
"503", "502", "504", "busy", "重试", "provider_timeout", "download")
FATAL_HINTS = ("invalid", "参数", "审核", "content policy", "unsupported", "not supported", "unauthorized",
"forbidden", "apiKey", "缺少", "不支持", "为空")
# 泄漏到调度层的裸网络异常(httpx / socket / urllib):全部按「服务不可达」处理。
# 这些异常只有英文原文,直接写进任务表会让用户去查素材和参数,
# 而真正该查的是那台跑 ComfyUI 的机器还开不开。
NETWORK_HINTS = ("connecterror", "connect timeout", "connection refused", "connection reset",
"connection aborted", "no route to host", "network is unreachable",
"unreachable host", "failed to establish a new connection", "read timeout",
"readtimeout", "pool timeout", "remoteprotocolerror", "provider_unreachable",
"timed out", "cannot connect to host")
@dataclass
class PlanItem:
shot_id: int
order_index: int
title: str
action: str # enqueue / skip / needs_attention
reason: str
provider: str = ""
model: str = ""
duration: float = 0.0
estimated_cost: float = 0.0
blocking: list[str] = field(default_factory=list)
def as_dict(self) -> dict:
return {"shot_id": self.shot_id, "order_index": self.order_index, "title": self.title,
"action": self.action, "reason": self.reason, "provider": self.provider, "model": self.model,
"duration": round(self.duration, 2), "estimated_cost": round(self.estimated_cost, 4),
"blocking": self.blocking}
# ---------------------------------------------------------------- 任务清单
def plan_task_list(db: Session, project: Project, *, shot_ids: list[int] | None = None,
force: bool = False, allow_unready: bool = False,
include_voiceover: bool = True) -> dict:
"""生成「分镜任务清单」预览:每个镜头要做什么、是否被阻塞、预计成本。"""
shots = list(project.shots or [])
if shot_ids:
wanted = set(shot_ids)
shots = [s for s in shots if s.id in wanted]
readiness = preflight.project_readiness(db, project)
check_index = {c["shot_id"]: c for c in readiness["checks"]}
# 在途任务按**种类**分开看:一条排队中的配音/质检任务不该让「画面生成」被当成
# 已经在做(以前一个 dict 混装所有 kind,后写的覆盖先写的,语义是碰运气的)。
active_rows = db.scalars(select(Task).where(
Task.project_id == project.id, Task.status.in_(TaskStatus.ACTIVE),
Task.shot_id.isnot(None)).order_by(Task.id)).all()
active_by_shot = {t.shot_id: t for t in active_rows if t.kind == TaskKind.GENERATE_SHOT}
active_voice_by_shot = {t.shot_id: t for t in active_rows if t.kind == TaskKind.VOICEOVER}
items: list[PlanItem] = []
total_cost = 0.0
for shot in shots:
spec, _ = model_registry.resolve_video(db, shot.provider or project.default_provider, shot.model or None)
est_cost = costing.estimate_video_cost(spec, shot.duration)
check = check_index.get(shot.id, {"ok": True, "issues": []})
errors = [i["message"] for i in check["issues"] if i["severity"] == "error"]
active = active_by_shot.get(shot.id)
if active is not None and not force:
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip",
f"已有任务 #{active.id} 处于 {active.status}", spec.provider, shot.model or spec.model, shot.duration)
elif shot.status == ShotStatus.APPROVED and not force:
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip",
"镜头已审核合格,跳过(断点续跑)", spec.provider, shot.model or spec.model, shot.duration)
elif not force and shot.output_path and storage.exists(shot.output_path) \
and shot.status not in (ShotStatus.REJECTED, ShotStatus.FAILED):
# 「有产物但还没人工审核通过」不等于「要重跑」:质检/审核是后面步骤的事。
# 以前这种镜头会被重新排队,LLM 总控台每跑一轮就把已经质检合格的画面
# 重新生成一遍——GPU 时间翻倍,还可能把合格画面换成更差的一版,
# 用户看到的就是「指令下了半天,项目还在原地打转」。
# 只有编导明确说「重做/重新生成」(force)才重出。
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip",
f"画面产物已存在({shot.output_path}),镜头状态 {shot.status}:"
"交由质检/审核步骤处理,不重复生成",
spec.provider, shot.model or spec.model, shot.duration)
elif errors and not (allow_unready or force):
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "needs_attention",
"素材或参数未就绪", spec.provider, shot.model or spec.model, shot.duration, blocking=errors)
else:
reason = "准备调用视频模型生成"
if shot.status == ShotStatus.REJECTED:
reason = "人工审核不合格,按新提示词重新生成"
elif shot.status == ShotStatus.FAILED:
reason = "上次生成失败,重新排队"
elif shot.status == ShotStatus.GENERATED:
reason = "已有产物但尚未审核通过,重新生成"
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "enqueue",
reason, shot.provider or project.default_provider, shot.model or spec.model,
shot.duration, est_cost)
total_cost += est_cost
items.append(item)
voice_items = []
if include_voiceover:
# 台词默认交给视频模型直出(H3 原生音画同步 + 角色声线做参考音频)。
# 这条路径上再排一条 TTS 就是白花钱:产物从来不会被时间线读到,
# 反而让人以为「配音还没跑完」。只有显式选了 dub(强制平台配音)才排。
from app.services import timeline as timeline_service
for shot in shots:
if not (shot.dialogue or "").strip():
continue
has_audio = bool(shot.voiceover_path) and storage.exists(shot.voiceover_path)
if timeline_service.native_voice_wanted(db, project, shot):
voice_items.append({"shot_id": shot.id, "order_index": shot.order_index,
"action": "skip",
"reason": "台词走模型原声(H3 原生音画同步,音色用角色声线做参考),无需 TTS 配音",
"chars": len((shot.dialogue or "").strip())})
continue
active_voice = active_voice_by_shot.get(shot.id)
if active_voice is not None and not force:
# 已经有一条配音在队列/执行中:再排一条就是**同一段台词配两遍**,
# 花钱、占并发,还会让后跑的覆盖先跑的产物。
voice_items.append({"shot_id": shot.id, "order_index": shot.order_index,
"action": "skip",
"reason": f"已有配音任务 #{active_voice.id} 处于 {active_voice.status},不重复排队",
"chars": len((shot.dialogue or "").strip())})
else:
voice_items.append({"shot_id": shot.id, "order_index": shot.order_index,
"action": "skip" if (has_audio and not force) else "enqueue",
"reason": "已有配音产物" if has_audio else "台词需要配音",
"chars": len((shot.dialogue or "").strip())})
return {"project_id": project.id, "shots": [i.as_dict() for i in items],
"voiceovers": voice_items,
"queued_count": sum(1 for i in items if i.action == "enqueue"),
"voiceover_count": sum(1 for v in voice_items if v["action"] == "enqueue"),
"skipped_count": sum(1 for i in items if i.action == "skip"),
"blocked_count": sum(1 for i in items if i.action == "needs_attention"),
"estimated_cost": round(total_cost, 4), "currency": settings.cost_currency,
"preflight": {"ready": readiness["ready"], "blocked": readiness["blocked"],
"errors": readiness["errors"], "warnings": readiness["warnings"]}}
# 排队中的任务在真正执行时才现算提示词,所以「同镜头同类别已有排队任务」时
# 再排一条,只会让同一镜头多烧一次 GPU:服务重启/自动续跑/自动返工连着发生时,
# 一次重启就多送一轮出图,用户看到的是「指令跑了半天,任务越排越多」。
REUSABLE_TASK_STATUSES = (TaskStatus.PLANNED, TaskStatus.QUEUED, TaskStatus.RETRYING)
def reusable_task(db: Session, project: Project, shot: Shot, kind: str) -> Task | None:
"""这一镜是否已经有一条还没开始跑的同类任务在队列里(有就复用,别重复排队)。"""
return db.scalars(select(Task).where(
Task.project_id == project.id,
Task.shot_id == shot.id,
Task.kind == kind,
Task.status.in_(REUSABLE_TASK_STATUSES),
).order_by(Task.id)).first()
def create_generation_tasks(db: Session, project: Project, plan: dict | None = None, *,
shot_ids: list[int] | None = None, force: bool = False,
allow_unready: bool = False, priority: int = 5, actor: str = "studio",
with_voiceover: bool = True) -> dict:
"""按任务清单落库并进入队列(不直接执行,由调度器按限流派发)。"""
plan = plan or plan_task_list(db, project, shot_ids=shot_ids, force=force, allow_unready=allow_unready)
created: list[dict] = []
skipped: list[dict] = []
for item in plan["shots"]:
if item["action"] != "enqueue":
skipped.append(item)
continue
shot = db.get(Shot, item["shot_id"])
if shot is None:
continue
task = reusable_task(db, project, shot, TaskKind.GENERATE_SHOT)
reused = task is not None
if task is None:
task = _new_generation_task(db, project, shot, priority=priority, actor=actor,
force=force)
else:
# 复用不等于降级:返工/重做的优先级要压过队列里的老任务
task.priority = min(int(task.priority or priority), priority)
task.queued_at = now_utc()
shot.current_task_id = task.id
created.append({"task_id": task.id, "shot_id": shot.id, "kind": task.kind,
"status": task.status, "reused": reused})
voice_created = []
if with_voiceover:
for item in plan.get("voiceovers", []):
if item["action"] != "enqueue":
continue
shot = db.get(Shot, item["shot_id"])
if shot is None:
continue
task = reusable_task(db, project, shot, TaskKind.VOICEOVER)
reused = task is None
if reused:
task = new_voiceover_task(db, project, shot, priority=max(1, priority - 1))
else:
task.priority = min(int(task.priority or priority), max(1, priority - 1))
task.queued_at = now_utc()
voice_created.append({"task_id": task.id, "shot_id": shot.id, "kind": task.kind,
"reused": not reused})
project.current_stage = "dispatch"
status.refresh(db, project, commit=False)
db.commit()
return {"created": created, "voiceovers": voice_created, "skipped": skipped,
"count": len(created) + len(voice_created)}
def _new_generation_task(db: Session, project: Project, shot: Shot, *, priority: int = 5,
actor: str = "studio", force: bool = False) -> Task:
prompt.sync_prompt(db, project, shot, reason="生成前自动组装提示词", actor=actor, record=True)
if force and shot.output_path and storage.exists(shot.output_path):
# 人工点「重新生成」=明确要求换一个结果。可种子是从「幂等键 + 提示词摘要」
# 派生的,提示词一字未改时种子也不变,供应商直接命中缓存原样返回同一份文件
# (实测 6 秒返回、与上一版 MD5 完全一致),用户等到的还是那一镜。
# 给镜头累加一次 take 轮次并让它进幂等键,重摇才真正重新采样。
params = dict(shot.params_json or {})
params["take"] = int(params.get("take") or 1) + 1
shot.params_json = params
# 解析成**具体**模型再落库:镜头/项目常常只写了 provider(或干脆留空跟默认),
# 若把 model='' 直接写进任务行,限流面板会凭空多出一行没有规格的行
# (拿的是平台默认 2 并发 / 30 RPM),既看不见真实限额,也让 headroom() 取错行,
# 生成记录与成本台账同样失去可比性。
spec, _adapter = model_registry.resolve_video(db, shot.provider or project.default_provider,
shot.model or project.default_model or None)
provider, model = spec.provider, spec.model
key = f"generate:{shot.id}:attempt-cycle:{(shot.attempts or 0) + 1}"
task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.GENERATE_SHOT,
name=f"镜头 {shot.order_index:02d} 视频生成", stage="dispatch", status=TaskStatus.QUEUED,
priority=priority, provider=provider, model=model, mode=shot.mode,
idempotency_key=key, max_attempts=max(1, settings.max_shot_retries + 1),
payload_json={"duration": shot.duration, "resolution": project.resolution,
"aspect_ratio": project.aspect_ratio, "fps": project.fps},
cost=costing.estimate_video_cost(spec, shot.duration),
cost_currency=settings.cost_currency, queued_at=now_utc())
db.add(task)
db.flush()
shot.current_task_id = task.id
shot.status = ShotStatus.QUEUED
shot.stage = "queued"
shot.error_code = ""
shot.error_message = ""
events.log_event(db, "shot.queued", project_id=project.id, shot_id=shot.id, task_id=task.id,
message=f"镜头 {shot.order_index:02d} 已进入任务队列({provider}/{model or '默认模型'})",
entity_type="shot", entity_id=shot.id, data={"provider": provider, "model": model})
return task
def new_voiceover_task(db: Session, project: Project, shot: Shot, *, priority: int = 4) -> Task:
from app.services import audio
# 任务必须记录镜头实际绑定的声线供应商。之前一律按全局默认 TTS 落库,
# 会出现任务显示 mock、产物却由 IndexTTS2 生成的可观测性断层。
voice = audio.resolve_voice_asset(db, project, shot)
spec, _ = audio.resolve_tts_spec(db, voice)
text = (shot.dialogue or "").strip()
task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.VOICEOVER,
name=f"镜头 {shot.order_index:02d} 配音", stage="voiceover", status=TaskStatus.QUEUED,
priority=priority, provider=spec.provider, model=spec.model, mode="voice",
idempotency_key=f"voice:{shot.id}:{abs(hash(text)) % 10**12}",
max_attempts=max(1, settings.max_shot_retries),
payload_json={"text": text, "chars": len(text)},
cost=costing.estimate_voice_cost(spec, len(text)), cost_currency=settings.cost_currency,
queued_at=now_utc())
db.add(task)
db.flush()
events.log_event(db, "voiceover.queued", project_id=project.id, shot_id=shot.id, task_id=task.id,
message=f"镜头 {shot.order_index:02d} 配音任务已入队", entity_type="shot", entity_id=shot.id)
return task
def new_qc_task(db: Session, project: Project, shot: Shot, *, priority: int = 3) -> Task:
task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.QC_SHOT,
name=f"镜头 {shot.order_index:02d} 质量检查", stage="quality_check", status=TaskStatus.QUEUED,
priority=priority, idempotency_key=f"qc:{shot.id}:{shot.output_checksum or 'auto'}",
max_attempts=2, payload_json={"path": shot.output_path}, queued_at=now_utc())
db.add(task)
db.flush()
return task
def new_compose_task(db: Session, project: Project, timeline_id: int, *, preset: str = "web_1080p",
name: str = "", burn_subtitle: bool | None = None,
subtitle_mode: str | None = None, priority: int = 2) -> Task:
task = Task(project_id=project.id, shot_id=None, timeline_id=timeline_id, kind=TaskKind.COMPOSE,
name=name or "整片合成导出", stage="compose", status=TaskStatus.QUEUED, priority=priority,
idempotency_key=f"compose:{timeline_id}:{preset}:{int(datetime.now().timestamp())}",
max_attempts=max(1, settings.max_compose_retries + 1),
payload_json={"preset": preset, "burn_subtitle": burn_subtitle,
"subtitle_mode": subtitle_mode}, queued_at=now_utc())
db.add(task)
db.flush()
events.log_event(db, "compose.queued", project_id=project.id, task_id=task.id,
message=f"合成任务已入队({preset})", entity_type="timeline", entity_id=timeline_id)
return task
# ---------------------------------------------------------------- 队列调度
def dispatch_due(db: Session, *, limit: int | None = None, project_id: int | None = None) -> dict:
"""按限流额度派发任务;返回本次派发与阻塞详情。"""
limit = limit or settings.dispatch_batch_size
now = now_utc()
stmt = (select(Task)
.where(Task.status.in_([TaskStatus.QUEUED, TaskStatus.RETRYING]),
or_(Task.next_retry_at.is_(None), Task.next_retry_at <= now))
.order_by(Task.priority.asc(), Task.id.asc()).limit(limit * 3))
if project_id is not None:
stmt = stmt.where(Task.project_id == project_id)
dispatched: list[dict] = []
blocked: dict[str, str] = {}
slots: dict[str, int] = {}
for task in db.scalars(stmt).all():
if len(dispatched) >= limit:
break
key = f"{task.provider}|{task.model or ''}"
if key not in slots:
available, reason = capacity.headroom(db, task.provider or "", task.model or None)
slots[key] = available
if available <= 0:
blocked[key] = reason
if slots[key] <= 0:
continue
if not claim_task(db, task, now=now):
continue # 已被其它派发者认领,跳过
handle = submit_claimed(db, task, now=now)
if not handle:
continue # 派发失败,已归还队列
slots[key] -= 1
mark_dispatched(db, task, handle, now=now)
dispatched.append({"task_id": task.id, "kind": task.kind, "shot_id": task.shot_id,
"provider": task.provider, "handle": handle})
db.commit()
return {"dispatched": dispatched, "count": len(dispatched), "blocked": [
{"provider_model": k, "reason": v} for k, v in blocked.items()]}
def claim_task(db: Session, task: Task, *, now: datetime | None = None) -> bool:
"""原子认领一条排队中的任务:条件更新只有一个会话能成功。
守护线程、drain、人工 dispatch、平台侧「点一下生成」四条路都会走到这里,
因此同一任务绝不会被跑两次——这是「不重复烧 GPU」的唯一保证,别换成先读后写。
"""
now = now or now_utc()
claimed = db.execute(update(Task)
.where(Task.id == task.id,
Task.status.in_([TaskStatus.QUEUED, TaskStatus.RETRYING]))
# worker_id 先清空:认领到执行体真正起跑之间有个短窗口,
# 期间它还属于「没有 runner」而不是「上一代进程的僵尸」,
# 否则断点续跑可能把刚认领的任务再捞回去跑两遍。
.values(status=TaskStatus.RUNNING, started_at=now, heartbeat_at=now,
attempts=Task.attempts + 1, next_retry_at=None, error_message="",
worker_id="", execution_backend=execution.backend_name())
.execution_options(synchronize_session=False)).rowcount
db.commit()
return bool(claimed)
def submit_claimed(db: Session, task: Task, *, now: datetime | None = None) -> str:
"""把已认领的任务交给执行后端,返回句柄;派发失败时归还队列并稍后重试,返回空串。"""
now = now or now_utc()
try:
return execution.submit(task)
except Exception as exc: # 后端不可用 → 归还队列并稍后重试
log.exception("任务派发失败 task=%s", task.id)
task = db.get(Task, task.id)
task.status = TaskStatus.QUEUED
task.attempts = max(0, (task.attempts or 1) - 1)
task.started_at = None
task.heartbeat_at = None
task.error_message = f"任务派发失败:{exc}"[:4000]
task.next_retry_at = now + timedelta(seconds=30)
events.log_event(db, "task.dispatch_failed", project_id=task.project_id, task_id=task.id,
level="error", message=task.error_message, entity_type="task", entity_id=task.id)
db.commit()
return ""
def mark_dispatched(db: Session, task: Task, handle: str, *, now: datetime | None = None) -> Task:
"""记下执行句柄并留一条事件(不提交:批量派发时由调用方统一收口)。"""
now = now or now_utc()
task = db.get(Task, task.id)
task.celery_task_id = handle
task.execution_backend = execution.backend_name()
task.queued_at = task.queued_at or now
events.log_event(db, "task.dispatched", project_id=task.project_id, task_id=task.id,
shot_id=task.shot_id, message=f"任务已派发到 {task.execution_backend} 执行后端",
entity_type="task", entity_id=task.id,
data={"provider": task.provider, "model": task.model, "handle": handle})
return task
def dispatch_task(db: Session, task: Task) -> dict:
"""立刻派发**这一条**任务:有额度当场起跑,没额度就留在队列里等守护线程。
平台资产生成走这里。用户点完按钮不该干等 dispatch_interval_seconds(默认 5s)
才知道排上了没有;但也不能绕过限流插队——出图、出片、配音打的是同一台
ComfyUI,插队的代价由正在跑的项目买单。所以这里查同样的 headroom、
走同样的原子认领,只是不排队等下一轮扫描。
返回值只用于日志与接口提示,真正的进度一律以任务行为准。
"""
now = now_utc()
available, reason = capacity.headroom(db, task.provider or "", task.model or None)
if available <= 0:
return {"dispatched": False, "queued": True, "task_id": task.id,
"status": task.status, "reason": reason or "算力已满,已进入队列"}
if not claim_task(db, task, now=now):
fresh = db.get(Task, task.id)
return {"dispatched": False, "queued": True, "task_id": task.id,
"status": fresh.status, "reason": "任务已被其它派发者认领"}
handle = submit_claimed(db, task, now=now)
if not handle:
fresh = db.get(Task, task.id)
return {"dispatched": False, "queued": True, "task_id": task.id,
"status": fresh.status, "reason": fresh.error_message or "任务派发失败"}
fresh = mark_dispatched(db, task, handle, now=now)
db.commit()
return {"dispatched": True, "queued": True, "task_id": fresh.id,
"status": fresh.status, "handle": handle, "reason": ""}
# ---------------------------------------------------------------- 状态回写
def _write_through(db: Session, *, commit: bool = False, attempts: int = 4) -> None:
"""把状态回写落库,遇到 SQLite 抢锁先自己退避重试。
心跳/worker 认领这类写入本身不会失败,失败的只是「这一刻抢不到写锁」。
以前一次 database is locked 会把正在跑的整条视频生成打成 runtime_error,
白等一轮四分钟的出图(项目 9 镜头 02 实测)。
"""
for attempt in range(attempts):
try:
db.commit() if commit else db.flush()
return
except OperationalError as exc:
if "locked" not in str(exc).lower() or attempt == attempts - 1:
raise
time.sleep(0.2 * (attempt + 1))
def begin_execution(db: Session, task: Task, *, worker: str = "") -> Task:
"""执行体入口:派发者已认领(running)时只刷新心跳与 worker 标识。
派发者认领时已经把 attempts 记过一次,这里绝不重复累加,
否则「尝试次数」会在并发场景下虚高,导致重试次数提前耗尽。
"""
if task.status == TaskStatus.RUNNING:
task.heartbeat_at = now_utc()
task.started_at = task.started_at or task.heartbeat_at
task.worker_id = worker or task.worker_id
task.attempts = max(1, task.attempts or 0)
task.error_message = ""
_write_through(db)
return task
return mark_running(db, task, worker=worker)
def renew_lease(db: Session, task_id: int) -> bool:
"""执行期心跳续租(独立会话直接提交):只刷新仍在运行的任务,返回是否仍持有执行权。"""
for attempt in range(4):
try:
rows = db.execute(
update(Task).where(Task.id == task_id, Task.status == TaskStatus.RUNNING)
.values(heartbeat_at=now_utc())
.execution_options(synchronize_session=False)).rowcount
db.commit()
return bool(rows)
except OperationalError as exc:
db.rollback()
if "locked" not in str(exc).lower() or attempt == 3:
raise
time.sleep(0.2 * (attempt + 1))
return False
def touch_heartbeat(db: Session, task: Task) -> None:
task.heartbeat_at = now_utc()
db.flush()
def mark_running(db: Session, task: Task, *, worker: str = "") -> Task:
task.status = TaskStatus.RUNNING
task.attempts = (task.attempts or 0) + 1
task.started_at = now_utc()
task.heartbeat_at = task.started_at
task.worker_id = worker or task.worker_id
task.error_message = ""
_write_through(db)
return task
def mark_succeeded(db: Session, task: Task, *, result: dict | None = None, output_path: str | None = None) -> Task:
task.status = TaskStatus.SUCCEEDED
task.progress = 1.0
task.result_json = result or {}
task.output_path = output_path or task.output_path
task.finished_at = now_utc()
task.heartbeat_at = task.finished_at
task.error_code = ""
db.flush()
events.log_event(db, "task.succeeded", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id,
message=f"任务完成:{task.name}", entity_type="task", entity_id=task.id,
data={"output": task.output_path})
return task
def network_message(exc: BaseException, provider: str = "") -> str:
"""裸网络异常的人话版本:说清「查服务、不用改素材」,并交代会自动重试。"""
target = provider or "模型服务"
raw = f"{type(exc).__name__}: {exc}"[:200]
return (f"连不上 {target}({raw})。这是服务或网络不通,不是素材、提示词或参数的问题:"
"请到「模型管理中心」确认服务地址并点「探测」复测,"
"或检查那台机器上的模型进程与端口是否还在;恢复后任务会自动重试。")
def classify_error(exc: BaseException, *, provider: str = "") -> tuple[str, bool, str]:
"""返回 (error_code, retryable, message)。"""
if isinstance(exc, ProviderError):
return exc.code, bool(exc.retryable), exc.message
text = str(exc).lower()
if isinstance(exc, (httpx.RequestError, OSError)) or any(h in text for h in NETWORK_HINTS):
# 网络类一律可重试:服务恢复即可续跑,绝不该把「对机器的连接问题」判成镜头本身失败。
return "provider_unreachable", True, network_message(exc, provider)
if any(h.lower() in text for h in FATAL_HINTS) and not any(h in text for h in RETRYABLE_HINTS):
return "fatal_error", False, str(exc)
if isinstance(exc, (TimeoutError,)) or any(h in text for h in RETRYABLE_HINTS):
return "retryable_error", True, str(exc)
return "runtime_error", True, str(exc)
def schedule_retry_or_fail(db: Session, task: Task, exc: BaseException, *,
shot: Shot | None = None) -> Task:
code, retryable, message = classify_error(exc, provider=task.provider or task.model)
task.error_code = code
task.error_message = message[:4000]
history = list(task.error_history or [])
history.append({"attempt": task.attempts, "code": code, "message": message[:800],
"at": now_utc().isoformat()})
task.error_history = history[-20:]
can_retry = retryable and (task.attempts or 0) < max(1, task.max_attempts)
if can_retry:
backoff = min(settings.retry_backoff_max_seconds,
settings.retry_backoff_base_seconds * (2 ** max(0, (task.attempts or 1) - 1)))
retry_after = getattr(exc, "retry_after", None)
if retry_after:
backoff = max(backoff, min(float(retry_after), float(settings.retry_backoff_max_seconds)))
task.status = TaskStatus.RETRYING
task.next_retry_at = now_utc() + timedelta(seconds=backoff)
task.heartbeat_at = now_utc()
if shot is not None:
shot.status = ShotStatus.RETRYING
shot.stage = "retry"
shot.error_code = code
shot.error_message = task.error_message
events.log_event(db, "task.retry_scheduled", project_id=task.project_id, task_id=task.id,
shot_id=task.shot_id, level="warn",
message=f"记录错误并安排第 {task.attempts + 1} 次尝试({int(backoff)} 秒后,原因:{code})",
entity_type="task", entity_id=task.id,
data={"attempts": task.attempts, "code": code, "backoff": int(backoff)})
else:
task.status = TaskStatus.FAILED
task.finished_at = now_utc()
if shot is not None:
shot.status = ShotStatus.FAILED
shot.stage = "failed"
shot.error_code = code
shot.error_message = task.error_message
events.log_event(db, "task.failed", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id,
level="error",
message=f"任务失败已终止({'不可重试' if not retryable else '达到重试上限'}):{task.error_message[:200]}",
entity_type="task", entity_id=task.id,
data={"code": code, "attempts": task.attempts, "retryable": retryable})
db.flush()
return task
def heartbeat(db: Session, task: Task, *, progress: float | None = None, note: str = "") -> None:
task.heartbeat_at = now_utc()
if progress is not None:
task.progress = max(0.0, min(1.0, float(progress)))
if note:
task.checkpoint_json = {**(task.checkpoint_json or {}), "last_note": note,
"at": task.heartbeat_at.isoformat()}
db.flush()
# ---------------------------------------------------------------- 失败重试 / 断点续跑
def retry_failed(db: Session, project_id: int, *, task_ids: list[int] | None = None,
reset_attempts: bool = True, only_codes: list[str] | None = None) -> dict:
stmt = select(Task).where(Task.project_id == project_id, Task.status.in_([TaskStatus.FAILED]))
if task_ids:
stmt = stmt.where(Task.id.in_(task_ids))
if only_codes:
stmt = stmt.where(Task.error_code.in_(only_codes))
reopened = []
now = now_utc()
for task in db.scalars(stmt).all():
task.status = TaskStatus.QUEUED
if reset_attempts:
task.attempts = 0
task.max_attempts = max(1, settings.max_shot_retries + 1)
task.next_retry_at = None
task.error_code = ""
task.queued_at = now
task.finished_at = None
if task.shot_id:
shot = db.get(Shot, task.shot_id)
if shot:
shot.status = ShotStatus.QUEUED
shot.stage = "queued"
shot.error_message = ""
shot.error_code = ""
if task.kind == TaskKind.GENERATE_SHOT:
shot.attempts = 0
reopened.append(task.id)
events.log_event(db, "task.requeued", project_id=task.project_id, task_id=task.id,
shot_id=task.shot_id, message="人工触发失败重试,任务重新入队",
entity_type="task", entity_id=task.id)
db.commit()
return {"requeued": reopened, "count": len(reopened)}
def cancel_task(db: Session, task: Task, *, reason: str = "人工取消") -> Task:
task.status = TaskStatus.CANCELLED
task.finished_at = now_utc()
task.error_message = reason
if task.shot_id:
shot = db.get(Shot, task.shot_id)
if shot and shot.status in (ShotStatus.QUEUED, ShotStatus.RUNNING, ShotStatus.RETRYING):
shot.status = ShotStatus.READY if shot.status != ShotStatus.RUNNING else ShotStatus.DRAFT
shot.stage = "cancelled"
events.log_event(db, "task.cancelled", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id,
level="warn", message=reason, entity_type="task", entity_id=task.id)
db.commit()
return task
_HOSTNAME = socket.gethostname()
def inline_runner_is_dead(worker_id: str | None, execution_backend: str | None) -> bool:
"""inline 任务的执行线程活在某个进程里:那个进程没了,任务行就是僵尸。
进程重启(uvicorn --reload 改代码、崩溃、部署)只会带走线程,不会改写数据库:
任务行仍停在 running,既占着模型唯一的并发额度,又要等满
stale_running_seconds(本机 1800s)才自愈。表现就是「AI 总控台重新下达指令,
却报并发已满,整个流程原地卡死半小时」。
worker_id 形如 host:pid(见 jobs/handlers.WORKER_ID)。只在**同机且该 pid 已不存在**
时断定死亡;跨机、解析不出、pid 被复用等情况一律交给心跳超时兜底,宁可多等不可误杀
(误杀会把别人正在生成的镜头再跑一遍,成本翻倍)。
"""
if (execution_backend or "").lower() != "inline":
return False
host, _, pid_text = (worker_id or "").rpartition(":")
if not pid_text.isdigit() or host != _HOSTNAME:
return False
pid = int(pid_text)
if pid == os.getpid():
return False
try:
os.kill(pid, 0)
except ProcessLookupError:
return True
except OSError: # 权限不足等情况无法判定,保守放过
return False
return False
def resume_interrupted(db: Session, *, stale_seconds: int | None = None) -> dict:
"""断点续跑:回收中断/孤儿任务,保留尝试次数与检查点,跳过已完成产物。"""
stale = stale_seconds or settings.stale_running_seconds
cutoff = now_utc() - timedelta(seconds=stale)
recovered: list[dict] = []
stale_tasks = list(db.scalars(select(Task).where(Task.status == TaskStatus.RUNNING,
or_(Task.heartbeat_at.is_(None),
Task.heartbeat_at < cutoff))).all())
# 心跳超时只是「疑似中断」;执行进程已经消失是「确认中断」,不必等租约过期。
# 这类僵尸占着模型唯一的并发位,让 AI 总控台重新下达指令时只能干等「并发已满」。
expired = {row.id for row in stale_tasks}
stale_tasks += [row for row in db.scalars(select(Task).where(Task.status == TaskStatus.RUNNING)).all()
if row.id not in expired and inline_runner_is_dead(row.worker_id, row.execution_backend)]
for task in stale_tasks:
task.status = TaskStatus.QUEUED
task.next_retry_at = None
task.queued_at = now_utc()
task.celery_task_id = ""
dead_runner = inline_runner_is_dead(task.worker_id, task.execution_backend)
checkpoint = dict(task.checkpoint_json or {})
checkpoint["recovered_from"] = "dead_runner" if dead_runner else "stale_running"
checkpoint["recovered_at"] = now_utc().isoformat()
task.checkpoint_json = checkpoint
if dead_runner and (task.attempts or 0) > 0:
# attempts 是「开始执行了几次」,而重试额度是给模型侧失败用的。
# 进程被热重载/崩溃带走时,这一次尝试根本没拿到供应商的结论,
# 记在它头上就会出现「什么都没失败过,重试却已经用光」:
# 本机一次开发会话里热重载六次,任务尝试数被刷到 6/3,
# 之后任何一个瞬时抖动都会直接判死。心跳超时那条路径不退还——
# 那种情况可能真的还在别处跑着,宁可少给一次重试也不能无限循环。
task.attempts -= 1
checkpoint["attempt_refunded"] = True
shot = db.get(Shot, task.shot_id) if task.shot_id else None
if shot and task.kind == TaskKind.GENERATE_SHOT:
if shot.output_path and storage.exists(shot.output_path) and shot.status in (
ShotStatus.APPROVED, ShotStatus.GENERATED, ShotStatus.QC_PASSED):
task.status = TaskStatus.SKIPPED
task.result_json = {"reason": "产物已存在,跳过重复生成", "path": shot.output_path}
task.finished_at = now_utc()
else:
shot.status = ShotStatus.QUEUED
shot.stage = "resume"
events.log_event(db, "task.recovered", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id,
level="warn",
message=("执行进程已重启,中断的任务已重新排队(断点续跑)" if dead_runner
else "检测到中断任务,已重新排队(断点续跑)"),
entity_type="task", entity_id=task.id,
data={"attempts": task.attempts, "kind": task.kind,
"refunded": bool(checkpoint.get("attempt_refunded"))})
recovered.append({"task_id": task.id, "kind": task.kind, "status": task.status})
# 镜头状态与任务不一致(例如 API 重启导致 queued 悬挂)→ 归位
dangling = db.scalars(select(Shot).where(Shot.status.in_(ShotStatus.ACTIVE))).all()
for shot in dangling:
live = db.scalars(select(Task).where(Task.shot_id == shot.id,
Task.status.in_(TaskStatus.ACTIVE + [TaskStatus.PLANNED]))).first()
if live is None:
if shot.output_path and storage.exists(shot.output_path):
shot.status = ShotStatus.GENERATED
shot.stage = "resume_generated"
else:
shot.status = ShotStatus.READY
shot.stage = "resume_ready"
events.log_event(db, "shot.recovered", project_id=shot.project_id, shot_id=shot.id, level="warn",
message="镜头状态与队列不一致,已按断点续跑归位", entity_type="shot", entity_id=shot.id)
db.commit()
return {"recovered": recovered, "count": len(recovered)}
def queue_overview(db: Session, *, project_id: int | None = None) -> dict:
from sqlalchemy import func
stmt = select(Task.status, Task.kind, func.count(Task.id)).group_by(Task.status, Task.kind)
if project_id is not None:
stmt = stmt.where(Task.project_id == project_id)
matrix = [{"status": s, "kind": k, "count": int(c)} for s, k, c in db.execute(stmt).all()]
counts: dict[str, int] = {}
for row in matrix:
counts[row["status"]] = counts.get(row["status"], 0) + row["count"]
recent_stmt = select(Task).order_by(Task.id.desc()).limit(200)
if project_id is not None:
recent_stmt = recent_stmt.where(Task.project_id == project_id)
recent = db.scalars(recent_stmt)
return {"counts": counts, "matrix": matrix,
"tasks": [status.task_dict(t) for t in recent],
"capacity": [row.as_dict() for row in capacity.snapshot(db, project_id=project_id)],
"executor": execution.backend_name()}