778 lines
42 KiB
Python
778 lines
42 KiB
Python
"""统一任务编排:任务清单生成 → 限流派发 → 失败重试 → 断点续跑。
|
||
|
||
对应业务流程:检查素材与参数 → 生成任务清单 → 任务队列调度 → 调用视频模型 →
|
||
生成成功?否→记录错误并重试;是→下载并保存视频。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import os
|
||
import socket
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime, timedelta
|
||
|
||
import httpx
|
||
from sqlalchemy import or_, select, update
|
||
from sqlalchemy.exc import OperationalError
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.config import settings
|
||
from app.constants import ShotStatus, TaskKind, TaskStatus
|
||
from app.models import CostRecord, Project, Shot, Task, now_utc
|
||
from app.providers.base import ProviderError
|
||
from app.providers import registry as model_registry
|
||
from app.services import capacity, costing, execution, events, preflight, prompt, status, storage
|
||
|
||
log = logging.getLogger("nex.dispatch")
|
||
|
||
RETRYABLE_HINTS = ("timeout", "timed out", "temporarily", "rate limit", "限流", "网络", "connection",
|
||
"503", "502", "504", "busy", "重试", "provider_timeout", "download")
|
||
FATAL_HINTS = ("invalid", "参数", "审核", "content policy", "unsupported", "not supported", "unauthorized",
|
||
"forbidden", "apiKey", "缺少", "不支持", "为空")
|
||
# 泄漏到调度层的裸网络异常(httpx / socket / urllib):全部按「服务不可达」处理。
|
||
# 这些异常只有英文原文,直接写进任务表会让用户去查素材和参数,
|
||
# 而真正该查的是那台跑 ComfyUI 的机器还开不开。
|
||
NETWORK_HINTS = ("connecterror", "connect timeout", "connection refused", "connection reset",
|
||
"connection aborted", "no route to host", "network is unreachable",
|
||
"unreachable host", "failed to establish a new connection", "read timeout",
|
||
"readtimeout", "pool timeout", "remoteprotocolerror", "provider_unreachable",
|
||
"timed out", "cannot connect to host")
|
||
|
||
|
||
@dataclass
|
||
class PlanItem:
|
||
shot_id: int
|
||
order_index: int
|
||
title: str
|
||
action: str # enqueue / skip / needs_attention
|
||
reason: str
|
||
provider: str = ""
|
||
model: str = ""
|
||
duration: float = 0.0
|
||
estimated_cost: float = 0.0
|
||
blocking: list[str] = field(default_factory=list)
|
||
|
||
def as_dict(self) -> dict:
|
||
return {"shot_id": self.shot_id, "order_index": self.order_index, "title": self.title,
|
||
"action": self.action, "reason": self.reason, "provider": self.provider, "model": self.model,
|
||
"duration": round(self.duration, 2), "estimated_cost": round(self.estimated_cost, 4),
|
||
"blocking": self.blocking}
|
||
|
||
|
||
# ---------------------------------------------------------------- 任务清单
|
||
def plan_task_list(db: Session, project: Project, *, shot_ids: list[int] | None = None,
|
||
force: bool = False, allow_unready: bool = False,
|
||
include_voiceover: bool = True) -> dict:
|
||
"""生成「分镜任务清单」预览:每个镜头要做什么、是否被阻塞、预计成本。"""
|
||
shots = list(project.shots or [])
|
||
if shot_ids:
|
||
wanted = set(shot_ids)
|
||
shots = [s for s in shots if s.id in wanted]
|
||
readiness = preflight.project_readiness(db, project)
|
||
check_index = {c["shot_id"]: c for c in readiness["checks"]}
|
||
# 在途任务按**种类**分开看:一条排队中的配音/质检任务不该让「画面生成」被当成
|
||
# 已经在做(以前一个 dict 混装所有 kind,后写的覆盖先写的,语义是碰运气的)。
|
||
active_rows = db.scalars(select(Task).where(
|
||
Task.project_id == project.id, Task.status.in_(TaskStatus.ACTIVE),
|
||
Task.shot_id.isnot(None)).order_by(Task.id)).all()
|
||
active_by_shot = {t.shot_id: t for t in active_rows if t.kind == TaskKind.GENERATE_SHOT}
|
||
active_voice_by_shot = {t.shot_id: t for t in active_rows if t.kind == TaskKind.VOICEOVER}
|
||
|
||
items: list[PlanItem] = []
|
||
total_cost = 0.0
|
||
for shot in shots:
|
||
spec, _ = model_registry.resolve_video(db, shot.provider or project.default_provider, shot.model or None)
|
||
est_cost = costing.estimate_video_cost(spec, shot.duration)
|
||
check = check_index.get(shot.id, {"ok": True, "issues": []})
|
||
errors = [i["message"] for i in check["issues"] if i["severity"] == "error"]
|
||
active = active_by_shot.get(shot.id)
|
||
if active is not None and not force:
|
||
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip",
|
||
f"已有任务 #{active.id} 处于 {active.status}", spec.provider, shot.model or spec.model, shot.duration)
|
||
elif shot.status == ShotStatus.APPROVED and not force:
|
||
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip",
|
||
"镜头已审核合格,跳过(断点续跑)", spec.provider, shot.model or spec.model, shot.duration)
|
||
elif not force and shot.output_path and storage.exists(shot.output_path) \
|
||
and shot.status not in (ShotStatus.REJECTED, ShotStatus.FAILED):
|
||
# 「有产物但还没人工审核通过」不等于「要重跑」:质检/审核是后面步骤的事。
|
||
# 以前这种镜头会被重新排队,LLM 总控台每跑一轮就把已经质检合格的画面
|
||
# 重新生成一遍——GPU 时间翻倍,还可能把合格画面换成更差的一版,
|
||
# 用户看到的就是「指令下了半天,项目还在原地打转」。
|
||
# 只有编导明确说「重做/重新生成」(force)才重出。
|
||
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip",
|
||
f"画面产物已存在({shot.output_path}),镜头状态 {shot.status}:"
|
||
"交由质检/审核步骤处理,不重复生成",
|
||
spec.provider, shot.model or spec.model, shot.duration)
|
||
elif errors and not (allow_unready or force):
|
||
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "needs_attention",
|
||
"素材或参数未就绪", spec.provider, shot.model or spec.model, shot.duration, blocking=errors)
|
||
else:
|
||
reason = "准备调用视频模型生成"
|
||
if shot.status == ShotStatus.REJECTED:
|
||
reason = "人工审核不合格,按新提示词重新生成"
|
||
elif shot.status == ShotStatus.FAILED:
|
||
reason = "上次生成失败,重新排队"
|
||
elif shot.status == ShotStatus.GENERATED:
|
||
reason = "已有产物但尚未审核通过,重新生成"
|
||
item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "enqueue",
|
||
reason, shot.provider or project.default_provider, shot.model or spec.model,
|
||
shot.duration, est_cost)
|
||
total_cost += est_cost
|
||
items.append(item)
|
||
|
||
voice_items = []
|
||
if include_voiceover:
|
||
# 台词默认交给视频模型直出(H3 原生音画同步 + 角色声线做参考音频)。
|
||
# 这条路径上再排一条 TTS 就是白花钱:产物从来不会被时间线读到,
|
||
# 反而让人以为「配音还没跑完」。只有显式选了 dub(强制平台配音)才排。
|
||
from app.services import timeline as timeline_service
|
||
|
||
for shot in shots:
|
||
if not (shot.dialogue or "").strip():
|
||
continue
|
||
has_audio = bool(shot.voiceover_path) and storage.exists(shot.voiceover_path)
|
||
if timeline_service.native_voice_wanted(db, project, shot):
|
||
voice_items.append({"shot_id": shot.id, "order_index": shot.order_index,
|
||
"action": "skip",
|
||
"reason": "台词走模型原声(H3 原生音画同步,音色用角色声线做参考),无需 TTS 配音",
|
||
"chars": len((shot.dialogue or "").strip())})
|
||
continue
|
||
active_voice = active_voice_by_shot.get(shot.id)
|
||
if active_voice is not None and not force:
|
||
# 已经有一条配音在队列/执行中:再排一条就是**同一段台词配两遍**,
|
||
# 花钱、占并发,还会让后跑的覆盖先跑的产物。
|
||
voice_items.append({"shot_id": shot.id, "order_index": shot.order_index,
|
||
"action": "skip",
|
||
"reason": f"已有配音任务 #{active_voice.id} 处于 {active_voice.status},不重复排队",
|
||
"chars": len((shot.dialogue or "").strip())})
|
||
else:
|
||
voice_items.append({"shot_id": shot.id, "order_index": shot.order_index,
|
||
"action": "skip" if (has_audio and not force) else "enqueue",
|
||
"reason": "已有配音产物" if has_audio else "台词需要配音",
|
||
"chars": len((shot.dialogue or "").strip())})
|
||
return {"project_id": project.id, "shots": [i.as_dict() for i in items],
|
||
"voiceovers": voice_items,
|
||
"queued_count": sum(1 for i in items if i.action == "enqueue"),
|
||
"voiceover_count": sum(1 for v in voice_items if v["action"] == "enqueue"),
|
||
"skipped_count": sum(1 for i in items if i.action == "skip"),
|
||
"blocked_count": sum(1 for i in items if i.action == "needs_attention"),
|
||
"estimated_cost": round(total_cost, 4), "currency": settings.cost_currency,
|
||
"preflight": {"ready": readiness["ready"], "blocked": readiness["blocked"],
|
||
"errors": readiness["errors"], "warnings": readiness["warnings"]}}
|
||
|
||
|
||
# 排队中的任务在真正执行时才现算提示词,所以「同镜头同类别已有排队任务」时
|
||
# 再排一条,只会让同一镜头多烧一次 GPU:服务重启/自动续跑/自动返工连着发生时,
|
||
# 一次重启就多送一轮出图,用户看到的是「指令跑了半天,任务越排越多」。
|
||
REUSABLE_TASK_STATUSES = (TaskStatus.PLANNED, TaskStatus.QUEUED, TaskStatus.RETRYING)
|
||
|
||
|
||
def reusable_task(db: Session, project: Project, shot: Shot, kind: str) -> Task | None:
|
||
"""这一镜是否已经有一条还没开始跑的同类任务在队列里(有就复用,别重复排队)。"""
|
||
return db.scalars(select(Task).where(
|
||
Task.project_id == project.id,
|
||
Task.shot_id == shot.id,
|
||
Task.kind == kind,
|
||
Task.status.in_(REUSABLE_TASK_STATUSES),
|
||
).order_by(Task.id)).first()
|
||
|
||
|
||
def create_generation_tasks(db: Session, project: Project, plan: dict | None = None, *,
|
||
shot_ids: list[int] | None = None, force: bool = False,
|
||
allow_unready: bool = False, priority: int = 5, actor: str = "studio",
|
||
with_voiceover: bool = True) -> dict:
|
||
"""按任务清单落库并进入队列(不直接执行,由调度器按限流派发)。"""
|
||
plan = plan or plan_task_list(db, project, shot_ids=shot_ids, force=force, allow_unready=allow_unready)
|
||
created: list[dict] = []
|
||
skipped: list[dict] = []
|
||
for item in plan["shots"]:
|
||
if item["action"] != "enqueue":
|
||
skipped.append(item)
|
||
continue
|
||
shot = db.get(Shot, item["shot_id"])
|
||
if shot is None:
|
||
continue
|
||
task = reusable_task(db, project, shot, TaskKind.GENERATE_SHOT)
|
||
reused = task is not None
|
||
if task is None:
|
||
task = _new_generation_task(db, project, shot, priority=priority, actor=actor,
|
||
force=force)
|
||
else:
|
||
# 复用不等于降级:返工/重做的优先级要压过队列里的老任务
|
||
task.priority = min(int(task.priority or priority), priority)
|
||
task.queued_at = now_utc()
|
||
shot.current_task_id = task.id
|
||
created.append({"task_id": task.id, "shot_id": shot.id, "kind": task.kind,
|
||
"status": task.status, "reused": reused})
|
||
voice_created = []
|
||
if with_voiceover:
|
||
for item in plan.get("voiceovers", []):
|
||
if item["action"] != "enqueue":
|
||
continue
|
||
shot = db.get(Shot, item["shot_id"])
|
||
if shot is None:
|
||
continue
|
||
task = reusable_task(db, project, shot, TaskKind.VOICEOVER)
|
||
reused = task is None
|
||
if reused:
|
||
task = new_voiceover_task(db, project, shot, priority=max(1, priority - 1))
|
||
else:
|
||
task.priority = min(int(task.priority or priority), max(1, priority - 1))
|
||
task.queued_at = now_utc()
|
||
voice_created.append({"task_id": task.id, "shot_id": shot.id, "kind": task.kind,
|
||
"reused": not reused})
|
||
project.current_stage = "dispatch"
|
||
status.refresh(db, project, commit=False)
|
||
db.commit()
|
||
return {"created": created, "voiceovers": voice_created, "skipped": skipped,
|
||
"count": len(created) + len(voice_created)}
|
||
|
||
|
||
def _new_generation_task(db: Session, project: Project, shot: Shot, *, priority: int = 5,
|
||
actor: str = "studio", force: bool = False) -> Task:
|
||
prompt.sync_prompt(db, project, shot, reason="生成前自动组装提示词", actor=actor, record=True)
|
||
if force and shot.output_path and storage.exists(shot.output_path):
|
||
# 人工点「重新生成」=明确要求换一个结果。可种子是从「幂等键 + 提示词摘要」
|
||
# 派生的,提示词一字未改时种子也不变,供应商直接命中缓存原样返回同一份文件
|
||
# (实测 6 秒返回、与上一版 MD5 完全一致),用户等到的还是那一镜。
|
||
# 给镜头累加一次 take 轮次并让它进幂等键,重摇才真正重新采样。
|
||
params = dict(shot.params_json or {})
|
||
params["take"] = int(params.get("take") or 1) + 1
|
||
shot.params_json = params
|
||
# 解析成**具体**模型再落库:镜头/项目常常只写了 provider(或干脆留空跟默认),
|
||
# 若把 model='' 直接写进任务行,限流面板会凭空多出一行没有规格的行
|
||
# (拿的是平台默认 2 并发 / 30 RPM),既看不见真实限额,也让 headroom() 取错行,
|
||
# 生成记录与成本台账同样失去可比性。
|
||
spec, _adapter = model_registry.resolve_video(db, shot.provider or project.default_provider,
|
||
shot.model or project.default_model or None)
|
||
provider, model = spec.provider, spec.model
|
||
key = f"generate:{shot.id}:attempt-cycle:{(shot.attempts or 0) + 1}"
|
||
task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.GENERATE_SHOT,
|
||
name=f"镜头 {shot.order_index:02d} 视频生成", stage="dispatch", status=TaskStatus.QUEUED,
|
||
priority=priority, provider=provider, model=model, mode=shot.mode,
|
||
idempotency_key=key, max_attempts=max(1, settings.max_shot_retries + 1),
|
||
payload_json={"duration": shot.duration, "resolution": project.resolution,
|
||
"aspect_ratio": project.aspect_ratio, "fps": project.fps},
|
||
cost=costing.estimate_video_cost(spec, shot.duration),
|
||
cost_currency=settings.cost_currency, queued_at=now_utc())
|
||
db.add(task)
|
||
db.flush()
|
||
shot.current_task_id = task.id
|
||
shot.status = ShotStatus.QUEUED
|
||
shot.stage = "queued"
|
||
shot.error_code = ""
|
||
shot.error_message = ""
|
||
events.log_event(db, "shot.queued", project_id=project.id, shot_id=shot.id, task_id=task.id,
|
||
message=f"镜头 {shot.order_index:02d} 已进入任务队列({provider}/{model or '默认模型'})",
|
||
entity_type="shot", entity_id=shot.id, data={"provider": provider, "model": model})
|
||
return task
|
||
|
||
|
||
def new_voiceover_task(db: Session, project: Project, shot: Shot, *, priority: int = 4) -> Task:
|
||
from app.services import audio
|
||
|
||
# 任务必须记录镜头实际绑定的声线供应商。之前一律按全局默认 TTS 落库,
|
||
# 会出现任务显示 mock、产物却由 IndexTTS2 生成的可观测性断层。
|
||
voice = audio.resolve_voice_asset(db, project, shot)
|
||
spec, _ = audio.resolve_tts_spec(db, voice)
|
||
text = (shot.dialogue or "").strip()
|
||
task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.VOICEOVER,
|
||
name=f"镜头 {shot.order_index:02d} 配音", stage="voiceover", status=TaskStatus.QUEUED,
|
||
priority=priority, provider=spec.provider, model=spec.model, mode="voice",
|
||
idempotency_key=f"voice:{shot.id}:{abs(hash(text)) % 10**12}",
|
||
max_attempts=max(1, settings.max_shot_retries),
|
||
payload_json={"text": text, "chars": len(text)},
|
||
cost=costing.estimate_voice_cost(spec, len(text)), cost_currency=settings.cost_currency,
|
||
queued_at=now_utc())
|
||
db.add(task)
|
||
db.flush()
|
||
events.log_event(db, "voiceover.queued", project_id=project.id, shot_id=shot.id, task_id=task.id,
|
||
message=f"镜头 {shot.order_index:02d} 配音任务已入队", entity_type="shot", entity_id=shot.id)
|
||
return task
|
||
|
||
|
||
def new_qc_task(db: Session, project: Project, shot: Shot, *, priority: int = 3) -> Task:
|
||
task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.QC_SHOT,
|
||
name=f"镜头 {shot.order_index:02d} 质量检查", stage="quality_check", status=TaskStatus.QUEUED,
|
||
priority=priority, idempotency_key=f"qc:{shot.id}:{shot.output_checksum or 'auto'}",
|
||
max_attempts=2, payload_json={"path": shot.output_path}, queued_at=now_utc())
|
||
db.add(task)
|
||
db.flush()
|
||
return task
|
||
|
||
|
||
def new_compose_task(db: Session, project: Project, timeline_id: int, *, preset: str = "web_1080p",
|
||
name: str = "", burn_subtitle: bool | None = None,
|
||
subtitle_mode: str | None = None, priority: int = 2) -> Task:
|
||
task = Task(project_id=project.id, shot_id=None, timeline_id=timeline_id, kind=TaskKind.COMPOSE,
|
||
name=name or "整片合成导出", stage="compose", status=TaskStatus.QUEUED, priority=priority,
|
||
idempotency_key=f"compose:{timeline_id}:{preset}:{int(datetime.now().timestamp())}",
|
||
max_attempts=max(1, settings.max_compose_retries + 1),
|
||
payload_json={"preset": preset, "burn_subtitle": burn_subtitle,
|
||
"subtitle_mode": subtitle_mode}, queued_at=now_utc())
|
||
db.add(task)
|
||
db.flush()
|
||
events.log_event(db, "compose.queued", project_id=project.id, task_id=task.id,
|
||
message=f"合成任务已入队({preset})", entity_type="timeline", entity_id=timeline_id)
|
||
return task
|
||
|
||
|
||
# ---------------------------------------------------------------- 队列调度
|
||
def dispatch_due(db: Session, *, limit: int | None = None, project_id: int | None = None) -> dict:
|
||
"""按限流额度派发任务;返回本次派发与阻塞详情。"""
|
||
limit = limit or settings.dispatch_batch_size
|
||
now = now_utc()
|
||
stmt = (select(Task)
|
||
.where(Task.status.in_([TaskStatus.QUEUED, TaskStatus.RETRYING]),
|
||
or_(Task.next_retry_at.is_(None), Task.next_retry_at <= now))
|
||
.order_by(Task.priority.asc(), Task.id.asc()).limit(limit * 3))
|
||
if project_id is not None:
|
||
stmt = stmt.where(Task.project_id == project_id)
|
||
dispatched: list[dict] = []
|
||
blocked: dict[str, str] = {}
|
||
slots: dict[str, int] = {}
|
||
for task in db.scalars(stmt).all():
|
||
if len(dispatched) >= limit:
|
||
break
|
||
key = f"{task.provider}|{task.model or ''}"
|
||
if key not in slots:
|
||
available, reason = capacity.headroom(db, task.provider or "", task.model or None)
|
||
slots[key] = available
|
||
if available <= 0:
|
||
blocked[key] = reason
|
||
if slots[key] <= 0:
|
||
continue
|
||
if not claim_task(db, task, now=now):
|
||
continue # 已被其它派发者认领,跳过
|
||
handle = submit_claimed(db, task, now=now)
|
||
if not handle:
|
||
continue # 派发失败,已归还队列
|
||
slots[key] -= 1
|
||
mark_dispatched(db, task, handle, now=now)
|
||
dispatched.append({"task_id": task.id, "kind": task.kind, "shot_id": task.shot_id,
|
||
"provider": task.provider, "handle": handle})
|
||
db.commit()
|
||
return {"dispatched": dispatched, "count": len(dispatched), "blocked": [
|
||
{"provider_model": k, "reason": v} for k, v in blocked.items()]}
|
||
|
||
|
||
def claim_task(db: Session, task: Task, *, now: datetime | None = None) -> bool:
|
||
"""原子认领一条排队中的任务:条件更新只有一个会话能成功。
|
||
|
||
守护线程、drain、人工 dispatch、平台侧「点一下生成」四条路都会走到这里,
|
||
因此同一任务绝不会被跑两次——这是「不重复烧 GPU」的唯一保证,别换成先读后写。
|
||
"""
|
||
now = now or now_utc()
|
||
claimed = db.execute(update(Task)
|
||
.where(Task.id == task.id,
|
||
Task.status.in_([TaskStatus.QUEUED, TaskStatus.RETRYING]))
|
||
# worker_id 先清空:认领到执行体真正起跑之间有个短窗口,
|
||
# 期间它还属于「没有 runner」而不是「上一代进程的僵尸」,
|
||
# 否则断点续跑可能把刚认领的任务再捞回去跑两遍。
|
||
.values(status=TaskStatus.RUNNING, started_at=now, heartbeat_at=now,
|
||
attempts=Task.attempts + 1, next_retry_at=None, error_message="",
|
||
worker_id="", execution_backend=execution.backend_name())
|
||
.execution_options(synchronize_session=False)).rowcount
|
||
db.commit()
|
||
return bool(claimed)
|
||
|
||
|
||
def submit_claimed(db: Session, task: Task, *, now: datetime | None = None) -> str:
|
||
"""把已认领的任务交给执行后端,返回句柄;派发失败时归还队列并稍后重试,返回空串。"""
|
||
now = now or now_utc()
|
||
try:
|
||
return execution.submit(task)
|
||
except Exception as exc: # 后端不可用 → 归还队列并稍后重试
|
||
log.exception("任务派发失败 task=%s", task.id)
|
||
task = db.get(Task, task.id)
|
||
task.status = TaskStatus.QUEUED
|
||
task.attempts = max(0, (task.attempts or 1) - 1)
|
||
task.started_at = None
|
||
task.heartbeat_at = None
|
||
task.error_message = f"任务派发失败:{exc}"[:4000]
|
||
task.next_retry_at = now + timedelta(seconds=30)
|
||
events.log_event(db, "task.dispatch_failed", project_id=task.project_id, task_id=task.id,
|
||
level="error", message=task.error_message, entity_type="task", entity_id=task.id)
|
||
db.commit()
|
||
return ""
|
||
|
||
|
||
def mark_dispatched(db: Session, task: Task, handle: str, *, now: datetime | None = None) -> Task:
|
||
"""记下执行句柄并留一条事件(不提交:批量派发时由调用方统一收口)。"""
|
||
now = now or now_utc()
|
||
task = db.get(Task, task.id)
|
||
task.celery_task_id = handle
|
||
task.execution_backend = execution.backend_name()
|
||
task.queued_at = task.queued_at or now
|
||
events.log_event(db, "task.dispatched", project_id=task.project_id, task_id=task.id,
|
||
shot_id=task.shot_id, message=f"任务已派发到 {task.execution_backend} 执行后端",
|
||
entity_type="task", entity_id=task.id,
|
||
data={"provider": task.provider, "model": task.model, "handle": handle})
|
||
return task
|
||
|
||
|
||
def dispatch_task(db: Session, task: Task) -> dict:
|
||
"""立刻派发**这一条**任务:有额度当场起跑,没额度就留在队列里等守护线程。
|
||
|
||
平台资产生成走这里。用户点完按钮不该干等 dispatch_interval_seconds(默认 5s)
|
||
才知道排上了没有;但也不能绕过限流插队——出图、出片、配音打的是同一台
|
||
ComfyUI,插队的代价由正在跑的项目买单。所以这里查同样的 headroom、
|
||
走同样的原子认领,只是不排队等下一轮扫描。
|
||
返回值只用于日志与接口提示,真正的进度一律以任务行为准。
|
||
"""
|
||
now = now_utc()
|
||
available, reason = capacity.headroom(db, task.provider or "", task.model or None)
|
||
if available <= 0:
|
||
return {"dispatched": False, "queued": True, "task_id": task.id,
|
||
"status": task.status, "reason": reason or "算力已满,已进入队列"}
|
||
if not claim_task(db, task, now=now):
|
||
fresh = db.get(Task, task.id)
|
||
return {"dispatched": False, "queued": True, "task_id": task.id,
|
||
"status": fresh.status, "reason": "任务已被其它派发者认领"}
|
||
handle = submit_claimed(db, task, now=now)
|
||
if not handle:
|
||
fresh = db.get(Task, task.id)
|
||
return {"dispatched": False, "queued": True, "task_id": task.id,
|
||
"status": fresh.status, "reason": fresh.error_message or "任务派发失败"}
|
||
fresh = mark_dispatched(db, task, handle, now=now)
|
||
db.commit()
|
||
return {"dispatched": True, "queued": True, "task_id": fresh.id,
|
||
"status": fresh.status, "handle": handle, "reason": ""}
|
||
|
||
|
||
# ---------------------------------------------------------------- 状态回写
|
||
def _write_through(db: Session, *, commit: bool = False, attempts: int = 4) -> None:
|
||
"""把状态回写落库,遇到 SQLite 抢锁先自己退避重试。
|
||
|
||
心跳/worker 认领这类写入本身不会失败,失败的只是「这一刻抢不到写锁」。
|
||
以前一次 database is locked 会把正在跑的整条视频生成打成 runtime_error,
|
||
白等一轮四分钟的出图(项目 9 镜头 02 实测)。
|
||
"""
|
||
for attempt in range(attempts):
|
||
try:
|
||
db.commit() if commit else db.flush()
|
||
return
|
||
except OperationalError as exc:
|
||
if "locked" not in str(exc).lower() or attempt == attempts - 1:
|
||
raise
|
||
time.sleep(0.2 * (attempt + 1))
|
||
|
||
|
||
def begin_execution(db: Session, task: Task, *, worker: str = "") -> Task:
|
||
"""执行体入口:派发者已认领(running)时只刷新心跳与 worker 标识。
|
||
|
||
派发者认领时已经把 attempts 记过一次,这里绝不重复累加,
|
||
否则「尝试次数」会在并发场景下虚高,导致重试次数提前耗尽。
|
||
"""
|
||
if task.status == TaskStatus.RUNNING:
|
||
task.heartbeat_at = now_utc()
|
||
task.started_at = task.started_at or task.heartbeat_at
|
||
task.worker_id = worker or task.worker_id
|
||
task.attempts = max(1, task.attempts or 0)
|
||
task.error_message = ""
|
||
_write_through(db)
|
||
return task
|
||
return mark_running(db, task, worker=worker)
|
||
|
||
|
||
def renew_lease(db: Session, task_id: int) -> bool:
|
||
"""执行期心跳续租(独立会话直接提交):只刷新仍在运行的任务,返回是否仍持有执行权。"""
|
||
for attempt in range(4):
|
||
try:
|
||
rows = db.execute(
|
||
update(Task).where(Task.id == task_id, Task.status == TaskStatus.RUNNING)
|
||
.values(heartbeat_at=now_utc())
|
||
.execution_options(synchronize_session=False)).rowcount
|
||
db.commit()
|
||
return bool(rows)
|
||
except OperationalError as exc:
|
||
db.rollback()
|
||
if "locked" not in str(exc).lower() or attempt == 3:
|
||
raise
|
||
time.sleep(0.2 * (attempt + 1))
|
||
return False
|
||
|
||
|
||
def touch_heartbeat(db: Session, task: Task) -> None:
|
||
task.heartbeat_at = now_utc()
|
||
db.flush()
|
||
|
||
|
||
def mark_running(db: Session, task: Task, *, worker: str = "") -> Task:
|
||
task.status = TaskStatus.RUNNING
|
||
task.attempts = (task.attempts or 0) + 1
|
||
task.started_at = now_utc()
|
||
task.heartbeat_at = task.started_at
|
||
task.worker_id = worker or task.worker_id
|
||
task.error_message = ""
|
||
_write_through(db)
|
||
return task
|
||
|
||
|
||
def mark_succeeded(db: Session, task: Task, *, result: dict | None = None, output_path: str | None = None) -> Task:
|
||
task.status = TaskStatus.SUCCEEDED
|
||
task.progress = 1.0
|
||
task.result_json = result or {}
|
||
task.output_path = output_path or task.output_path
|
||
task.finished_at = now_utc()
|
||
task.heartbeat_at = task.finished_at
|
||
task.error_code = ""
|
||
db.flush()
|
||
events.log_event(db, "task.succeeded", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id,
|
||
message=f"任务完成:{task.name}", entity_type="task", entity_id=task.id,
|
||
data={"output": task.output_path})
|
||
return task
|
||
|
||
|
||
def network_message(exc: BaseException, provider: str = "") -> str:
|
||
"""裸网络异常的人话版本:说清「查服务、不用改素材」,并交代会自动重试。"""
|
||
target = provider or "模型服务"
|
||
raw = f"{type(exc).__name__}: {exc}"[:200]
|
||
return (f"连不上 {target}({raw})。这是服务或网络不通,不是素材、提示词或参数的问题:"
|
||
"请到「模型管理中心」确认服务地址并点「探测」复测,"
|
||
"或检查那台机器上的模型进程与端口是否还在;恢复后任务会自动重试。")
|
||
|
||
|
||
def classify_error(exc: BaseException, *, provider: str = "") -> tuple[str, bool, str]:
|
||
"""返回 (error_code, retryable, message)。"""
|
||
if isinstance(exc, ProviderError):
|
||
return exc.code, bool(exc.retryable), exc.message
|
||
text = str(exc).lower()
|
||
if isinstance(exc, (httpx.RequestError, OSError)) or any(h in text for h in NETWORK_HINTS):
|
||
# 网络类一律可重试:服务恢复即可续跑,绝不该把「对机器的连接问题」判成镜头本身失败。
|
||
return "provider_unreachable", True, network_message(exc, provider)
|
||
if any(h.lower() in text for h in FATAL_HINTS) and not any(h in text for h in RETRYABLE_HINTS):
|
||
return "fatal_error", False, str(exc)
|
||
if isinstance(exc, (TimeoutError,)) or any(h in text for h in RETRYABLE_HINTS):
|
||
return "retryable_error", True, str(exc)
|
||
return "runtime_error", True, str(exc)
|
||
|
||
|
||
def schedule_retry_or_fail(db: Session, task: Task, exc: BaseException, *,
|
||
shot: Shot | None = None) -> Task:
|
||
code, retryable, message = classify_error(exc, provider=task.provider or task.model)
|
||
task.error_code = code
|
||
task.error_message = message[:4000]
|
||
history = list(task.error_history or [])
|
||
history.append({"attempt": task.attempts, "code": code, "message": message[:800],
|
||
"at": now_utc().isoformat()})
|
||
task.error_history = history[-20:]
|
||
can_retry = retryable and (task.attempts or 0) < max(1, task.max_attempts)
|
||
if can_retry:
|
||
backoff = min(settings.retry_backoff_max_seconds,
|
||
settings.retry_backoff_base_seconds * (2 ** max(0, (task.attempts or 1) - 1)))
|
||
retry_after = getattr(exc, "retry_after", None)
|
||
if retry_after:
|
||
backoff = max(backoff, min(float(retry_after), float(settings.retry_backoff_max_seconds)))
|
||
task.status = TaskStatus.RETRYING
|
||
task.next_retry_at = now_utc() + timedelta(seconds=backoff)
|
||
task.heartbeat_at = now_utc()
|
||
if shot is not None:
|
||
shot.status = ShotStatus.RETRYING
|
||
shot.stage = "retry"
|
||
shot.error_code = code
|
||
shot.error_message = task.error_message
|
||
events.log_event(db, "task.retry_scheduled", project_id=task.project_id, task_id=task.id,
|
||
shot_id=task.shot_id, level="warn",
|
||
message=f"记录错误并安排第 {task.attempts + 1} 次尝试({int(backoff)} 秒后,原因:{code})",
|
||
entity_type="task", entity_id=task.id,
|
||
data={"attempts": task.attempts, "code": code, "backoff": int(backoff)})
|
||
else:
|
||
task.status = TaskStatus.FAILED
|
||
task.finished_at = now_utc()
|
||
if shot is not None:
|
||
shot.status = ShotStatus.FAILED
|
||
shot.stage = "failed"
|
||
shot.error_code = code
|
||
shot.error_message = task.error_message
|
||
events.log_event(db, "task.failed", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id,
|
||
level="error",
|
||
message=f"任务失败已终止({'不可重试' if not retryable else '达到重试上限'}):{task.error_message[:200]}",
|
||
entity_type="task", entity_id=task.id,
|
||
data={"code": code, "attempts": task.attempts, "retryable": retryable})
|
||
db.flush()
|
||
return task
|
||
|
||
|
||
def heartbeat(db: Session, task: Task, *, progress: float | None = None, note: str = "") -> None:
|
||
task.heartbeat_at = now_utc()
|
||
if progress is not None:
|
||
task.progress = max(0.0, min(1.0, float(progress)))
|
||
if note:
|
||
task.checkpoint_json = {**(task.checkpoint_json or {}), "last_note": note,
|
||
"at": task.heartbeat_at.isoformat()}
|
||
db.flush()
|
||
|
||
|
||
# ---------------------------------------------------------------- 失败重试 / 断点续跑
|
||
def retry_failed(db: Session, project_id: int, *, task_ids: list[int] | None = None,
|
||
reset_attempts: bool = True, only_codes: list[str] | None = None) -> dict:
|
||
stmt = select(Task).where(Task.project_id == project_id, Task.status.in_([TaskStatus.FAILED]))
|
||
if task_ids:
|
||
stmt = stmt.where(Task.id.in_(task_ids))
|
||
if only_codes:
|
||
stmt = stmt.where(Task.error_code.in_(only_codes))
|
||
reopened = []
|
||
now = now_utc()
|
||
for task in db.scalars(stmt).all():
|
||
task.status = TaskStatus.QUEUED
|
||
if reset_attempts:
|
||
task.attempts = 0
|
||
task.max_attempts = max(1, settings.max_shot_retries + 1)
|
||
task.next_retry_at = None
|
||
task.error_code = ""
|
||
task.queued_at = now
|
||
task.finished_at = None
|
||
if task.shot_id:
|
||
shot = db.get(Shot, task.shot_id)
|
||
if shot:
|
||
shot.status = ShotStatus.QUEUED
|
||
shot.stage = "queued"
|
||
shot.error_message = ""
|
||
shot.error_code = ""
|
||
if task.kind == TaskKind.GENERATE_SHOT:
|
||
shot.attempts = 0
|
||
reopened.append(task.id)
|
||
events.log_event(db, "task.requeued", project_id=task.project_id, task_id=task.id,
|
||
shot_id=task.shot_id, message="人工触发失败重试,任务重新入队",
|
||
entity_type="task", entity_id=task.id)
|
||
db.commit()
|
||
return {"requeued": reopened, "count": len(reopened)}
|
||
|
||
|
||
def cancel_task(db: Session, task: Task, *, reason: str = "人工取消") -> Task:
|
||
task.status = TaskStatus.CANCELLED
|
||
task.finished_at = now_utc()
|
||
task.error_message = reason
|
||
if task.shot_id:
|
||
shot = db.get(Shot, task.shot_id)
|
||
if shot and shot.status in (ShotStatus.QUEUED, ShotStatus.RUNNING, ShotStatus.RETRYING):
|
||
shot.status = ShotStatus.READY if shot.status != ShotStatus.RUNNING else ShotStatus.DRAFT
|
||
shot.stage = "cancelled"
|
||
events.log_event(db, "task.cancelled", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id,
|
||
level="warn", message=reason, entity_type="task", entity_id=task.id)
|
||
db.commit()
|
||
return task
|
||
|
||
|
||
_HOSTNAME = socket.gethostname()
|
||
|
||
|
||
def inline_runner_is_dead(worker_id: str | None, execution_backend: str | None) -> bool:
|
||
"""inline 任务的执行线程活在某个进程里:那个进程没了,任务行就是僵尸。
|
||
|
||
进程重启(uvicorn --reload 改代码、崩溃、部署)只会带走线程,不会改写数据库:
|
||
任务行仍停在 running,既占着模型唯一的并发额度,又要等满
|
||
stale_running_seconds(本机 1800s)才自愈。表现就是「AI 总控台重新下达指令,
|
||
却报并发已满,整个流程原地卡死半小时」。
|
||
|
||
worker_id 形如 host:pid(见 jobs/handlers.WORKER_ID)。只在**同机且该 pid 已不存在**
|
||
时断定死亡;跨机、解析不出、pid 被复用等情况一律交给心跳超时兜底,宁可多等不可误杀
|
||
(误杀会把别人正在生成的镜头再跑一遍,成本翻倍)。
|
||
"""
|
||
if (execution_backend or "").lower() != "inline":
|
||
return False
|
||
host, _, pid_text = (worker_id or "").rpartition(":")
|
||
if not pid_text.isdigit() or host != _HOSTNAME:
|
||
return False
|
||
pid = int(pid_text)
|
||
if pid == os.getpid():
|
||
return False
|
||
try:
|
||
os.kill(pid, 0)
|
||
except ProcessLookupError:
|
||
return True
|
||
except OSError: # 权限不足等情况无法判定,保守放过
|
||
return False
|
||
return False
|
||
|
||
|
||
def resume_interrupted(db: Session, *, stale_seconds: int | None = None) -> dict:
|
||
"""断点续跑:回收中断/孤儿任务,保留尝试次数与检查点,跳过已完成产物。"""
|
||
stale = stale_seconds or settings.stale_running_seconds
|
||
cutoff = now_utc() - timedelta(seconds=stale)
|
||
recovered: list[dict] = []
|
||
stale_tasks = list(db.scalars(select(Task).where(Task.status == TaskStatus.RUNNING,
|
||
or_(Task.heartbeat_at.is_(None),
|
||
Task.heartbeat_at < cutoff))).all())
|
||
# 心跳超时只是「疑似中断」;执行进程已经消失是「确认中断」,不必等租约过期。
|
||
# 这类僵尸占着模型唯一的并发位,让 AI 总控台重新下达指令时只能干等「并发已满」。
|
||
expired = {row.id for row in stale_tasks}
|
||
stale_tasks += [row for row in db.scalars(select(Task).where(Task.status == TaskStatus.RUNNING)).all()
|
||
if row.id not in expired and inline_runner_is_dead(row.worker_id, row.execution_backend)]
|
||
for task in stale_tasks:
|
||
task.status = TaskStatus.QUEUED
|
||
task.next_retry_at = None
|
||
task.queued_at = now_utc()
|
||
task.celery_task_id = ""
|
||
dead_runner = inline_runner_is_dead(task.worker_id, task.execution_backend)
|
||
checkpoint = dict(task.checkpoint_json or {})
|
||
checkpoint["recovered_from"] = "dead_runner" if dead_runner else "stale_running"
|
||
checkpoint["recovered_at"] = now_utc().isoformat()
|
||
task.checkpoint_json = checkpoint
|
||
if dead_runner and (task.attempts or 0) > 0:
|
||
# attempts 是「开始执行了几次」,而重试额度是给模型侧失败用的。
|
||
# 进程被热重载/崩溃带走时,这一次尝试根本没拿到供应商的结论,
|
||
# 记在它头上就会出现「什么都没失败过,重试却已经用光」:
|
||
# 本机一次开发会话里热重载六次,任务尝试数被刷到 6/3,
|
||
# 之后任何一个瞬时抖动都会直接判死。心跳超时那条路径不退还——
|
||
# 那种情况可能真的还在别处跑着,宁可少给一次重试也不能无限循环。
|
||
task.attempts -= 1
|
||
checkpoint["attempt_refunded"] = True
|
||
shot = db.get(Shot, task.shot_id) if task.shot_id else None
|
||
if shot and task.kind == TaskKind.GENERATE_SHOT:
|
||
if shot.output_path and storage.exists(shot.output_path) and shot.status in (
|
||
ShotStatus.APPROVED, ShotStatus.GENERATED, ShotStatus.QC_PASSED):
|
||
task.status = TaskStatus.SKIPPED
|
||
task.result_json = {"reason": "产物已存在,跳过重复生成", "path": shot.output_path}
|
||
task.finished_at = now_utc()
|
||
else:
|
||
shot.status = ShotStatus.QUEUED
|
||
shot.stage = "resume"
|
||
events.log_event(db, "task.recovered", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id,
|
||
level="warn",
|
||
message=("执行进程已重启,中断的任务已重新排队(断点续跑)" if dead_runner
|
||
else "检测到中断任务,已重新排队(断点续跑)"),
|
||
entity_type="task", entity_id=task.id,
|
||
data={"attempts": task.attempts, "kind": task.kind,
|
||
"refunded": bool(checkpoint.get("attempt_refunded"))})
|
||
recovered.append({"task_id": task.id, "kind": task.kind, "status": task.status})
|
||
|
||
# 镜头状态与任务不一致(例如 API 重启导致 queued 悬挂)→ 归位
|
||
dangling = db.scalars(select(Shot).where(Shot.status.in_(ShotStatus.ACTIVE))).all()
|
||
for shot in dangling:
|
||
live = db.scalars(select(Task).where(Task.shot_id == shot.id,
|
||
Task.status.in_(TaskStatus.ACTIVE + [TaskStatus.PLANNED]))).first()
|
||
if live is None:
|
||
if shot.output_path and storage.exists(shot.output_path):
|
||
shot.status = ShotStatus.GENERATED
|
||
shot.stage = "resume_generated"
|
||
else:
|
||
shot.status = ShotStatus.READY
|
||
shot.stage = "resume_ready"
|
||
events.log_event(db, "shot.recovered", project_id=shot.project_id, shot_id=shot.id, level="warn",
|
||
message="镜头状态与队列不一致,已按断点续跑归位", entity_type="shot", entity_id=shot.id)
|
||
db.commit()
|
||
return {"recovered": recovered, "count": len(recovered)}
|
||
|
||
|
||
def queue_overview(db: Session, *, project_id: int | None = None) -> dict:
|
||
from sqlalchemy import func
|
||
|
||
stmt = select(Task.status, Task.kind, func.count(Task.id)).group_by(Task.status, Task.kind)
|
||
if project_id is not None:
|
||
stmt = stmt.where(Task.project_id == project_id)
|
||
matrix = [{"status": s, "kind": k, "count": int(c)} for s, k, c in db.execute(stmt).all()]
|
||
counts: dict[str, int] = {}
|
||
for row in matrix:
|
||
counts[row["status"]] = counts.get(row["status"], 0) + row["count"]
|
||
recent_stmt = select(Task).order_by(Task.id.desc()).limit(200)
|
||
if project_id is not None:
|
||
recent_stmt = recent_stmt.where(Task.project_id == project_id)
|
||
recent = db.scalars(recent_stmt)
|
||
return {"counts": counts, "matrix": matrix,
|
||
"tasks": [status.task_dict(t) for t in recent],
|
||
"capacity": [row.as_dict() for row in capacity.snapshot(db, project_id=project_id)],
|
||
"executor": execution.backend_name()}
|