"""统一任务编排:任务清单生成 → 限流派发 → 失败重试 → 断点续跑。 对应业务流程:检查素材与参数 → 生成任务清单 → 任务队列调度 → 调用视频模型 → 生成成功?否→记录错误并重试;是→下载并保存视频。 """ from __future__ import annotations import logging import os import socket import time from dataclasses import dataclass, field from datetime import datetime, timedelta import httpx from sqlalchemy import or_, select, update from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session from app.config import settings from app.constants import ShotStatus, TaskKind, TaskStatus from app.models import CostRecord, Project, Shot, Task, now_utc from app.providers.base import ProviderError from app.providers import registry as model_registry from app.services import capacity, costing, execution, events, preflight, prompt, status, storage log = logging.getLogger("nex.dispatch") RETRYABLE_HINTS = ("timeout", "timed out", "temporarily", "rate limit", "限流", "网络", "connection", "503", "502", "504", "busy", "重试", "provider_timeout", "download") FATAL_HINTS = ("invalid", "参数", "审核", "content policy", "unsupported", "not supported", "unauthorized", "forbidden", "apiKey", "缺少", "不支持", "为空") # 泄漏到调度层的裸网络异常(httpx / socket / urllib):全部按「服务不可达」处理。 # 这些异常只有英文原文,直接写进任务表会让用户去查素材和参数, # 而真正该查的是那台跑 ComfyUI 的机器还开不开。 NETWORK_HINTS = ("connecterror", "connect timeout", "connection refused", "connection reset", "connection aborted", "no route to host", "network is unreachable", "unreachable host", "failed to establish a new connection", "read timeout", "readtimeout", "pool timeout", "remoteprotocolerror", "provider_unreachable", "timed out", "cannot connect to host") @dataclass class PlanItem: shot_id: int order_index: int title: str action: str # enqueue / skip / needs_attention reason: str provider: str = "" model: str = "" duration: float = 0.0 estimated_cost: float = 0.0 blocking: list[str] = field(default_factory=list) def as_dict(self) -> dict: return {"shot_id": self.shot_id, "order_index": self.order_index, "title": self.title, "action": self.action, "reason": self.reason, "provider": self.provider, "model": self.model, "duration": round(self.duration, 2), "estimated_cost": round(self.estimated_cost, 4), "blocking": self.blocking} # ---------------------------------------------------------------- 任务清单 def plan_task_list(db: Session, project: Project, *, shot_ids: list[int] | None = None, force: bool = False, allow_unready: bool = False, include_voiceover: bool = True) -> dict: """生成「分镜任务清单」预览:每个镜头要做什么、是否被阻塞、预计成本。""" shots = list(project.shots or []) if shot_ids: wanted = set(shot_ids) shots = [s for s in shots if s.id in wanted] readiness = preflight.project_readiness(db, project) check_index = {c["shot_id"]: c for c in readiness["checks"]} # 在途任务按**种类**分开看:一条排队中的配音/质检任务不该让「画面生成」被当成 # 已经在做(以前一个 dict 混装所有 kind,后写的覆盖先写的,语义是碰运气的)。 active_rows = db.scalars(select(Task).where( Task.project_id == project.id, Task.status.in_(TaskStatus.ACTIVE), Task.shot_id.isnot(None)).order_by(Task.id)).all() active_by_shot = {t.shot_id: t for t in active_rows if t.kind == TaskKind.GENERATE_SHOT} active_voice_by_shot = {t.shot_id: t for t in active_rows if t.kind == TaskKind.VOICEOVER} items: list[PlanItem] = [] total_cost = 0.0 for shot in shots: spec, _ = model_registry.resolve_video(db, shot.provider or project.default_provider, shot.model or None) est_cost = costing.estimate_video_cost(spec, shot.duration) check = check_index.get(shot.id, {"ok": True, "issues": []}) errors = [i["message"] for i in check["issues"] if i["severity"] == "error"] active = active_by_shot.get(shot.id) if active is not None and not force: item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip", f"已有任务 #{active.id} 处于 {active.status}", spec.provider, shot.model or spec.model, shot.duration) elif shot.status == ShotStatus.APPROVED and not force: item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip", "镜头已审核合格,跳过(断点续跑)", spec.provider, shot.model or spec.model, shot.duration) elif not force and shot.output_path and storage.exists(shot.output_path) \ and shot.status not in (ShotStatus.REJECTED, ShotStatus.FAILED): # 「有产物但还没人工审核通过」不等于「要重跑」:质检/审核是后面步骤的事。 # 以前这种镜头会被重新排队,LLM 总控台每跑一轮就把已经质检合格的画面 # 重新生成一遍——GPU 时间翻倍,还可能把合格画面换成更差的一版, # 用户看到的就是「指令下了半天,项目还在原地打转」。 # 只有编导明确说「重做/重新生成」(force)才重出。 item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "skip", f"画面产物已存在({shot.output_path}),镜头状态 {shot.status}:" "交由质检/审核步骤处理,不重复生成", spec.provider, shot.model or spec.model, shot.duration) elif errors and not (allow_unready or force): item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "needs_attention", "素材或参数未就绪", spec.provider, shot.model or spec.model, shot.duration, blocking=errors) else: reason = "准备调用视频模型生成" if shot.status == ShotStatus.REJECTED: reason = "人工审核不合格,按新提示词重新生成" elif shot.status == ShotStatus.FAILED: reason = "上次生成失败,重新排队" elif shot.status == ShotStatus.GENERATED: reason = "已有产物但尚未审核通过,重新生成" item = PlanItem(shot.id, shot.order_index, shot.title or f"镜头 {shot.order_index}", "enqueue", reason, shot.provider or project.default_provider, shot.model or spec.model, shot.duration, est_cost) total_cost += est_cost items.append(item) voice_items = [] if include_voiceover: # 台词默认交给视频模型直出(H3 原生音画同步 + 角色声线做参考音频)。 # 这条路径上再排一条 TTS 就是白花钱:产物从来不会被时间线读到, # 反而让人以为「配音还没跑完」。只有显式选了 dub(强制平台配音)才排。 from app.services import timeline as timeline_service for shot in shots: if not (shot.dialogue or "").strip(): continue has_audio = bool(shot.voiceover_path) and storage.exists(shot.voiceover_path) if timeline_service.native_voice_wanted(db, project, shot): voice_items.append({"shot_id": shot.id, "order_index": shot.order_index, "action": "skip", "reason": "台词走模型原声(H3 原生音画同步,音色用角色声线做参考),无需 TTS 配音", "chars": len((shot.dialogue or "").strip())}) continue active_voice = active_voice_by_shot.get(shot.id) if active_voice is not None and not force: # 已经有一条配音在队列/执行中:再排一条就是**同一段台词配两遍**, # 花钱、占并发,还会让后跑的覆盖先跑的产物。 voice_items.append({"shot_id": shot.id, "order_index": shot.order_index, "action": "skip", "reason": f"已有配音任务 #{active_voice.id} 处于 {active_voice.status},不重复排队", "chars": len((shot.dialogue or "").strip())}) else: voice_items.append({"shot_id": shot.id, "order_index": shot.order_index, "action": "skip" if (has_audio and not force) else "enqueue", "reason": "已有配音产物" if has_audio else "台词需要配音", "chars": len((shot.dialogue or "").strip())}) return {"project_id": project.id, "shots": [i.as_dict() for i in items], "voiceovers": voice_items, "queued_count": sum(1 for i in items if i.action == "enqueue"), "voiceover_count": sum(1 for v in voice_items if v["action"] == "enqueue"), "skipped_count": sum(1 for i in items if i.action == "skip"), "blocked_count": sum(1 for i in items if i.action == "needs_attention"), "estimated_cost": round(total_cost, 4), "currency": settings.cost_currency, "preflight": {"ready": readiness["ready"], "blocked": readiness["blocked"], "errors": readiness["errors"], "warnings": readiness["warnings"]}} # 排队中的任务在真正执行时才现算提示词,所以「同镜头同类别已有排队任务」时 # 再排一条,只会让同一镜头多烧一次 GPU:服务重启/自动续跑/自动返工连着发生时, # 一次重启就多送一轮出图,用户看到的是「指令跑了半天,任务越排越多」。 REUSABLE_TASK_STATUSES = (TaskStatus.PLANNED, TaskStatus.QUEUED, TaskStatus.RETRYING) def reusable_task(db: Session, project: Project, shot: Shot, kind: str) -> Task | None: """这一镜是否已经有一条还没开始跑的同类任务在队列里(有就复用,别重复排队)。""" return db.scalars(select(Task).where( Task.project_id == project.id, Task.shot_id == shot.id, Task.kind == kind, Task.status.in_(REUSABLE_TASK_STATUSES), ).order_by(Task.id)).first() def create_generation_tasks(db: Session, project: Project, plan: dict | None = None, *, shot_ids: list[int] | None = None, force: bool = False, allow_unready: bool = False, priority: int = 5, actor: str = "studio", with_voiceover: bool = True) -> dict: """按任务清单落库并进入队列(不直接执行,由调度器按限流派发)。""" plan = plan or plan_task_list(db, project, shot_ids=shot_ids, force=force, allow_unready=allow_unready) created: list[dict] = [] skipped: list[dict] = [] for item in plan["shots"]: if item["action"] != "enqueue": skipped.append(item) continue shot = db.get(Shot, item["shot_id"]) if shot is None: continue task = reusable_task(db, project, shot, TaskKind.GENERATE_SHOT) reused = task is not None if task is None: task = _new_generation_task(db, project, shot, priority=priority, actor=actor, force=force) else: # 复用不等于降级:返工/重做的优先级要压过队列里的老任务 task.priority = min(int(task.priority or priority), priority) task.queued_at = now_utc() shot.current_task_id = task.id created.append({"task_id": task.id, "shot_id": shot.id, "kind": task.kind, "status": task.status, "reused": reused}) voice_created = [] if with_voiceover: for item in plan.get("voiceovers", []): if item["action"] != "enqueue": continue shot = db.get(Shot, item["shot_id"]) if shot is None: continue task = reusable_task(db, project, shot, TaskKind.VOICEOVER) reused = task is None if reused: task = new_voiceover_task(db, project, shot, priority=max(1, priority - 1)) else: task.priority = min(int(task.priority or priority), max(1, priority - 1)) task.queued_at = now_utc() voice_created.append({"task_id": task.id, "shot_id": shot.id, "kind": task.kind, "reused": not reused}) project.current_stage = "dispatch" status.refresh(db, project, commit=False) db.commit() return {"created": created, "voiceovers": voice_created, "skipped": skipped, "count": len(created) + len(voice_created)} def _new_generation_task(db: Session, project: Project, shot: Shot, *, priority: int = 5, actor: str = "studio", force: bool = False) -> Task: prompt.sync_prompt(db, project, shot, reason="生成前自动组装提示词", actor=actor, record=True) if force and shot.output_path and storage.exists(shot.output_path): # 人工点「重新生成」=明确要求换一个结果。可种子是从「幂等键 + 提示词摘要」 # 派生的,提示词一字未改时种子也不变,供应商直接命中缓存原样返回同一份文件 # (实测 6 秒返回、与上一版 MD5 完全一致),用户等到的还是那一镜。 # 给镜头累加一次 take 轮次并让它进幂等键,重摇才真正重新采样。 params = dict(shot.params_json or {}) params["take"] = int(params.get("take") or 1) + 1 shot.params_json = params # 解析成**具体**模型再落库:镜头/项目常常只写了 provider(或干脆留空跟默认), # 若把 model='' 直接写进任务行,限流面板会凭空多出一行没有规格的行 # (拿的是平台默认 2 并发 / 30 RPM),既看不见真实限额,也让 headroom() 取错行, # 生成记录与成本台账同样失去可比性。 spec, _adapter = model_registry.resolve_video(db, shot.provider or project.default_provider, shot.model or project.default_model or None) provider, model = spec.provider, spec.model key = f"generate:{shot.id}:attempt-cycle:{(shot.attempts or 0) + 1}" task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.GENERATE_SHOT, name=f"镜头 {shot.order_index:02d} 视频生成", stage="dispatch", status=TaskStatus.QUEUED, priority=priority, provider=provider, model=model, mode=shot.mode, idempotency_key=key, max_attempts=max(1, settings.max_shot_retries + 1), payload_json={"duration": shot.duration, "resolution": project.resolution, "aspect_ratio": project.aspect_ratio, "fps": project.fps}, cost=costing.estimate_video_cost(spec, shot.duration), cost_currency=settings.cost_currency, queued_at=now_utc()) db.add(task) db.flush() shot.current_task_id = task.id shot.status = ShotStatus.QUEUED shot.stage = "queued" shot.error_code = "" shot.error_message = "" events.log_event(db, "shot.queued", project_id=project.id, shot_id=shot.id, task_id=task.id, message=f"镜头 {shot.order_index:02d} 已进入任务队列({provider}/{model or '默认模型'})", entity_type="shot", entity_id=shot.id, data={"provider": provider, "model": model}) return task def new_voiceover_task(db: Session, project: Project, shot: Shot, *, priority: int = 4) -> Task: from app.services import audio # 任务必须记录镜头实际绑定的声线供应商。之前一律按全局默认 TTS 落库, # 会出现任务显示 mock、产物却由 IndexTTS2 生成的可观测性断层。 voice = audio.resolve_voice_asset(db, project, shot) spec, _ = audio.resolve_tts_spec(db, voice) text = (shot.dialogue or "").strip() task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.VOICEOVER, name=f"镜头 {shot.order_index:02d} 配音", stage="voiceover", status=TaskStatus.QUEUED, priority=priority, provider=spec.provider, model=spec.model, mode="voice", idempotency_key=f"voice:{shot.id}:{abs(hash(text)) % 10**12}", max_attempts=max(1, settings.max_shot_retries), payload_json={"text": text, "chars": len(text)}, cost=costing.estimate_voice_cost(spec, len(text)), cost_currency=settings.cost_currency, queued_at=now_utc()) db.add(task) db.flush() events.log_event(db, "voiceover.queued", project_id=project.id, shot_id=shot.id, task_id=task.id, message=f"镜头 {shot.order_index:02d} 配音任务已入队", entity_type="shot", entity_id=shot.id) return task def new_qc_task(db: Session, project: Project, shot: Shot, *, priority: int = 3) -> Task: task = Task(project_id=project.id, shot_id=shot.id, kind=TaskKind.QC_SHOT, name=f"镜头 {shot.order_index:02d} 质量检查", stage="quality_check", status=TaskStatus.QUEUED, priority=priority, idempotency_key=f"qc:{shot.id}:{shot.output_checksum or 'auto'}", max_attempts=2, payload_json={"path": shot.output_path}, queued_at=now_utc()) db.add(task) db.flush() return task def new_compose_task(db: Session, project: Project, timeline_id: int, *, preset: str = "web_1080p", name: str = "", burn_subtitle: bool | None = None, subtitle_mode: str | None = None, priority: int = 2) -> Task: task = Task(project_id=project.id, shot_id=None, timeline_id=timeline_id, kind=TaskKind.COMPOSE, name=name or "整片合成导出", stage="compose", status=TaskStatus.QUEUED, priority=priority, idempotency_key=f"compose:{timeline_id}:{preset}:{int(datetime.now().timestamp())}", max_attempts=max(1, settings.max_compose_retries + 1), payload_json={"preset": preset, "burn_subtitle": burn_subtitle, "subtitle_mode": subtitle_mode}, queued_at=now_utc()) db.add(task) db.flush() events.log_event(db, "compose.queued", project_id=project.id, task_id=task.id, message=f"合成任务已入队({preset})", entity_type="timeline", entity_id=timeline_id) return task # ---------------------------------------------------------------- 队列调度 def dispatch_due(db: Session, *, limit: int | None = None, project_id: int | None = None) -> dict: """按限流额度派发任务;返回本次派发与阻塞详情。""" limit = limit or settings.dispatch_batch_size now = now_utc() stmt = (select(Task) .where(Task.status.in_([TaskStatus.QUEUED, TaskStatus.RETRYING]), or_(Task.next_retry_at.is_(None), Task.next_retry_at <= now)) .order_by(Task.priority.asc(), Task.id.asc()).limit(limit * 3)) if project_id is not None: stmt = stmt.where(Task.project_id == project_id) dispatched: list[dict] = [] blocked: dict[str, str] = {} slots: dict[str, int] = {} for task in db.scalars(stmt).all(): if len(dispatched) >= limit: break key = f"{task.provider}|{task.model or ''}" if key not in slots: available, reason = capacity.headroom(db, task.provider or "", task.model or None) slots[key] = available if available <= 0: blocked[key] = reason if slots[key] <= 0: continue if not claim_task(db, task, now=now): continue # 已被其它派发者认领,跳过 handle = submit_claimed(db, task, now=now) if not handle: continue # 派发失败,已归还队列 slots[key] -= 1 mark_dispatched(db, task, handle, now=now) dispatched.append({"task_id": task.id, "kind": task.kind, "shot_id": task.shot_id, "provider": task.provider, "handle": handle}) db.commit() return {"dispatched": dispatched, "count": len(dispatched), "blocked": [ {"provider_model": k, "reason": v} for k, v in blocked.items()]} def claim_task(db: Session, task: Task, *, now: datetime | None = None) -> bool: """原子认领一条排队中的任务:条件更新只有一个会话能成功。 守护线程、drain、人工 dispatch、平台侧「点一下生成」四条路都会走到这里, 因此同一任务绝不会被跑两次——这是「不重复烧 GPU」的唯一保证,别换成先读后写。 """ now = now or now_utc() claimed = db.execute(update(Task) .where(Task.id == task.id, Task.status.in_([TaskStatus.QUEUED, TaskStatus.RETRYING])) # worker_id 先清空:认领到执行体真正起跑之间有个短窗口, # 期间它还属于「没有 runner」而不是「上一代进程的僵尸」, # 否则断点续跑可能把刚认领的任务再捞回去跑两遍。 .values(status=TaskStatus.RUNNING, started_at=now, heartbeat_at=now, attempts=Task.attempts + 1, next_retry_at=None, error_message="", worker_id="", execution_backend=execution.backend_name()) .execution_options(synchronize_session=False)).rowcount db.commit() return bool(claimed) def submit_claimed(db: Session, task: Task, *, now: datetime | None = None) -> str: """把已认领的任务交给执行后端,返回句柄;派发失败时归还队列并稍后重试,返回空串。""" now = now or now_utc() try: return execution.submit(task) except Exception as exc: # 后端不可用 → 归还队列并稍后重试 log.exception("任务派发失败 task=%s", task.id) task = db.get(Task, task.id) task.status = TaskStatus.QUEUED task.attempts = max(0, (task.attempts or 1) - 1) task.started_at = None task.heartbeat_at = None task.error_message = f"任务派发失败:{exc}"[:4000] task.next_retry_at = now + timedelta(seconds=30) events.log_event(db, "task.dispatch_failed", project_id=task.project_id, task_id=task.id, level="error", message=task.error_message, entity_type="task", entity_id=task.id) db.commit() return "" def mark_dispatched(db: Session, task: Task, handle: str, *, now: datetime | None = None) -> Task: """记下执行句柄并留一条事件(不提交:批量派发时由调用方统一收口)。""" now = now or now_utc() task = db.get(Task, task.id) task.celery_task_id = handle task.execution_backend = execution.backend_name() task.queued_at = task.queued_at or now events.log_event(db, "task.dispatched", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id, message=f"任务已派发到 {task.execution_backend} 执行后端", entity_type="task", entity_id=task.id, data={"provider": task.provider, "model": task.model, "handle": handle}) return task def dispatch_task(db: Session, task: Task) -> dict: """立刻派发**这一条**任务:有额度当场起跑,没额度就留在队列里等守护线程。 平台资产生成走这里。用户点完按钮不该干等 dispatch_interval_seconds(默认 5s) 才知道排上了没有;但也不能绕过限流插队——出图、出片、配音打的是同一台 ComfyUI,插队的代价由正在跑的项目买单。所以这里查同样的 headroom、 走同样的原子认领,只是不排队等下一轮扫描。 返回值只用于日志与接口提示,真正的进度一律以任务行为准。 """ now = now_utc() available, reason = capacity.headroom(db, task.provider or "", task.model or None) if available <= 0: return {"dispatched": False, "queued": True, "task_id": task.id, "status": task.status, "reason": reason or "算力已满,已进入队列"} if not claim_task(db, task, now=now): fresh = db.get(Task, task.id) return {"dispatched": False, "queued": True, "task_id": task.id, "status": fresh.status, "reason": "任务已被其它派发者认领"} handle = submit_claimed(db, task, now=now) if not handle: fresh = db.get(Task, task.id) return {"dispatched": False, "queued": True, "task_id": task.id, "status": fresh.status, "reason": fresh.error_message or "任务派发失败"} fresh = mark_dispatched(db, task, handle, now=now) db.commit() return {"dispatched": True, "queued": True, "task_id": fresh.id, "status": fresh.status, "handle": handle, "reason": ""} # ---------------------------------------------------------------- 状态回写 def _write_through(db: Session, *, commit: bool = False, attempts: int = 4) -> None: """把状态回写落库,遇到 SQLite 抢锁先自己退避重试。 心跳/worker 认领这类写入本身不会失败,失败的只是「这一刻抢不到写锁」。 以前一次 database is locked 会把正在跑的整条视频生成打成 runtime_error, 白等一轮四分钟的出图(项目 9 镜头 02 实测)。 """ for attempt in range(attempts): try: db.commit() if commit else db.flush() return except OperationalError as exc: if "locked" not in str(exc).lower() or attempt == attempts - 1: raise time.sleep(0.2 * (attempt + 1)) def begin_execution(db: Session, task: Task, *, worker: str = "") -> Task: """执行体入口:派发者已认领(running)时只刷新心跳与 worker 标识。 派发者认领时已经把 attempts 记过一次,这里绝不重复累加, 否则「尝试次数」会在并发场景下虚高,导致重试次数提前耗尽。 """ if task.status == TaskStatus.RUNNING: task.heartbeat_at = now_utc() task.started_at = task.started_at or task.heartbeat_at task.worker_id = worker or task.worker_id task.attempts = max(1, task.attempts or 0) task.error_message = "" _write_through(db) return task return mark_running(db, task, worker=worker) def renew_lease(db: Session, task_id: int) -> bool: """执行期心跳续租(独立会话直接提交):只刷新仍在运行的任务,返回是否仍持有执行权。""" for attempt in range(4): try: rows = db.execute( update(Task).where(Task.id == task_id, Task.status == TaskStatus.RUNNING) .values(heartbeat_at=now_utc()) .execution_options(synchronize_session=False)).rowcount db.commit() return bool(rows) except OperationalError as exc: db.rollback() if "locked" not in str(exc).lower() or attempt == 3: raise time.sleep(0.2 * (attempt + 1)) return False def touch_heartbeat(db: Session, task: Task) -> None: task.heartbeat_at = now_utc() db.flush() def mark_running(db: Session, task: Task, *, worker: str = "") -> Task: task.status = TaskStatus.RUNNING task.attempts = (task.attempts or 0) + 1 task.started_at = now_utc() task.heartbeat_at = task.started_at task.worker_id = worker or task.worker_id task.error_message = "" _write_through(db) return task def mark_succeeded(db: Session, task: Task, *, result: dict | None = None, output_path: str | None = None) -> Task: task.status = TaskStatus.SUCCEEDED task.progress = 1.0 task.result_json = result or {} task.output_path = output_path or task.output_path task.finished_at = now_utc() task.heartbeat_at = task.finished_at task.error_code = "" db.flush() events.log_event(db, "task.succeeded", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id, message=f"任务完成:{task.name}", entity_type="task", entity_id=task.id, data={"output": task.output_path}) return task def network_message(exc: BaseException, provider: str = "") -> str: """裸网络异常的人话版本:说清「查服务、不用改素材」,并交代会自动重试。""" target = provider or "模型服务" raw = f"{type(exc).__name__}: {exc}"[:200] return (f"连不上 {target}({raw})。这是服务或网络不通,不是素材、提示词或参数的问题:" "请到「模型管理中心」确认服务地址并点「探测」复测," "或检查那台机器上的模型进程与端口是否还在;恢复后任务会自动重试。") def classify_error(exc: BaseException, *, provider: str = "") -> tuple[str, bool, str]: """返回 (error_code, retryable, message)。""" if isinstance(exc, ProviderError): return exc.code, bool(exc.retryable), exc.message text = str(exc).lower() if isinstance(exc, (httpx.RequestError, OSError)) or any(h in text for h in NETWORK_HINTS): # 网络类一律可重试:服务恢复即可续跑,绝不该把「对机器的连接问题」判成镜头本身失败。 return "provider_unreachable", True, network_message(exc, provider) if any(h.lower() in text for h in FATAL_HINTS) and not any(h in text for h in RETRYABLE_HINTS): return "fatal_error", False, str(exc) if isinstance(exc, (TimeoutError,)) or any(h in text for h in RETRYABLE_HINTS): return "retryable_error", True, str(exc) return "runtime_error", True, str(exc) def schedule_retry_or_fail(db: Session, task: Task, exc: BaseException, *, shot: Shot | None = None) -> Task: code, retryable, message = classify_error(exc, provider=task.provider or task.model) task.error_code = code task.error_message = message[:4000] history = list(task.error_history or []) history.append({"attempt": task.attempts, "code": code, "message": message[:800], "at": now_utc().isoformat()}) task.error_history = history[-20:] can_retry = retryable and (task.attempts or 0) < max(1, task.max_attempts) if can_retry: backoff = min(settings.retry_backoff_max_seconds, settings.retry_backoff_base_seconds * (2 ** max(0, (task.attempts or 1) - 1))) retry_after = getattr(exc, "retry_after", None) if retry_after: backoff = max(backoff, min(float(retry_after), float(settings.retry_backoff_max_seconds))) task.status = TaskStatus.RETRYING task.next_retry_at = now_utc() + timedelta(seconds=backoff) task.heartbeat_at = now_utc() if shot is not None: shot.status = ShotStatus.RETRYING shot.stage = "retry" shot.error_code = code shot.error_message = task.error_message events.log_event(db, "task.retry_scheduled", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id, level="warn", message=f"记录错误并安排第 {task.attempts + 1} 次尝试({int(backoff)} 秒后,原因:{code})", entity_type="task", entity_id=task.id, data={"attempts": task.attempts, "code": code, "backoff": int(backoff)}) else: task.status = TaskStatus.FAILED task.finished_at = now_utc() if shot is not None: shot.status = ShotStatus.FAILED shot.stage = "failed" shot.error_code = code shot.error_message = task.error_message events.log_event(db, "task.failed", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id, level="error", message=f"任务失败已终止({'不可重试' if not retryable else '达到重试上限'}):{task.error_message[:200]}", entity_type="task", entity_id=task.id, data={"code": code, "attempts": task.attempts, "retryable": retryable}) db.flush() return task def heartbeat(db: Session, task: Task, *, progress: float | None = None, note: str = "") -> None: task.heartbeat_at = now_utc() if progress is not None: task.progress = max(0.0, min(1.0, float(progress))) if note: task.checkpoint_json = {**(task.checkpoint_json or {}), "last_note": note, "at": task.heartbeat_at.isoformat()} db.flush() # ---------------------------------------------------------------- 失败重试 / 断点续跑 def retry_failed(db: Session, project_id: int, *, task_ids: list[int] | None = None, reset_attempts: bool = True, only_codes: list[str] | None = None) -> dict: stmt = select(Task).where(Task.project_id == project_id, Task.status.in_([TaskStatus.FAILED])) if task_ids: stmt = stmt.where(Task.id.in_(task_ids)) if only_codes: stmt = stmt.where(Task.error_code.in_(only_codes)) reopened = [] now = now_utc() for task in db.scalars(stmt).all(): task.status = TaskStatus.QUEUED if reset_attempts: task.attempts = 0 task.max_attempts = max(1, settings.max_shot_retries + 1) task.next_retry_at = None task.error_code = "" task.queued_at = now task.finished_at = None if task.shot_id: shot = db.get(Shot, task.shot_id) if shot: shot.status = ShotStatus.QUEUED shot.stage = "queued" shot.error_message = "" shot.error_code = "" if task.kind == TaskKind.GENERATE_SHOT: shot.attempts = 0 reopened.append(task.id) events.log_event(db, "task.requeued", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id, message="人工触发失败重试,任务重新入队", entity_type="task", entity_id=task.id) db.commit() return {"requeued": reopened, "count": len(reopened)} def cancel_task(db: Session, task: Task, *, reason: str = "人工取消") -> Task: task.status = TaskStatus.CANCELLED task.finished_at = now_utc() task.error_message = reason if task.shot_id: shot = db.get(Shot, task.shot_id) if shot and shot.status in (ShotStatus.QUEUED, ShotStatus.RUNNING, ShotStatus.RETRYING): shot.status = ShotStatus.READY if shot.status != ShotStatus.RUNNING else ShotStatus.DRAFT shot.stage = "cancelled" events.log_event(db, "task.cancelled", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id, level="warn", message=reason, entity_type="task", entity_id=task.id) db.commit() return task _HOSTNAME = socket.gethostname() def inline_runner_is_dead(worker_id: str | None, execution_backend: str | None) -> bool: """inline 任务的执行线程活在某个进程里:那个进程没了,任务行就是僵尸。 进程重启(uvicorn --reload 改代码、崩溃、部署)只会带走线程,不会改写数据库: 任务行仍停在 running,既占着模型唯一的并发额度,又要等满 stale_running_seconds(本机 1800s)才自愈。表现就是「AI 总控台重新下达指令, 却报并发已满,整个流程原地卡死半小时」。 worker_id 形如 host:pid(见 jobs/handlers.WORKER_ID)。只在**同机且该 pid 已不存在** 时断定死亡;跨机、解析不出、pid 被复用等情况一律交给心跳超时兜底,宁可多等不可误杀 (误杀会把别人正在生成的镜头再跑一遍,成本翻倍)。 """ if (execution_backend or "").lower() != "inline": return False host, _, pid_text = (worker_id or "").rpartition(":") if not pid_text.isdigit() or host != _HOSTNAME: return False pid = int(pid_text) if pid == os.getpid(): return False try: os.kill(pid, 0) except ProcessLookupError: return True except OSError: # 权限不足等情况无法判定,保守放过 return False return False def resume_interrupted(db: Session, *, stale_seconds: int | None = None) -> dict: """断点续跑:回收中断/孤儿任务,保留尝试次数与检查点,跳过已完成产物。""" stale = stale_seconds or settings.stale_running_seconds cutoff = now_utc() - timedelta(seconds=stale) recovered: list[dict] = [] stale_tasks = list(db.scalars(select(Task).where(Task.status == TaskStatus.RUNNING, or_(Task.heartbeat_at.is_(None), Task.heartbeat_at < cutoff))).all()) # 心跳超时只是「疑似中断」;执行进程已经消失是「确认中断」,不必等租约过期。 # 这类僵尸占着模型唯一的并发位,让 AI 总控台重新下达指令时只能干等「并发已满」。 expired = {row.id for row in stale_tasks} stale_tasks += [row for row in db.scalars(select(Task).where(Task.status == TaskStatus.RUNNING)).all() if row.id not in expired and inline_runner_is_dead(row.worker_id, row.execution_backend)] for task in stale_tasks: task.status = TaskStatus.QUEUED task.next_retry_at = None task.queued_at = now_utc() task.celery_task_id = "" dead_runner = inline_runner_is_dead(task.worker_id, task.execution_backend) checkpoint = dict(task.checkpoint_json or {}) checkpoint["recovered_from"] = "dead_runner" if dead_runner else "stale_running" checkpoint["recovered_at"] = now_utc().isoformat() task.checkpoint_json = checkpoint if dead_runner and (task.attempts or 0) > 0: # attempts 是「开始执行了几次」,而重试额度是给模型侧失败用的。 # 进程被热重载/崩溃带走时,这一次尝试根本没拿到供应商的结论, # 记在它头上就会出现「什么都没失败过,重试却已经用光」: # 本机一次开发会话里热重载六次,任务尝试数被刷到 6/3, # 之后任何一个瞬时抖动都会直接判死。心跳超时那条路径不退还—— # 那种情况可能真的还在别处跑着,宁可少给一次重试也不能无限循环。 task.attempts -= 1 checkpoint["attempt_refunded"] = True shot = db.get(Shot, task.shot_id) if task.shot_id else None if shot and task.kind == TaskKind.GENERATE_SHOT: if shot.output_path and storage.exists(shot.output_path) and shot.status in ( ShotStatus.APPROVED, ShotStatus.GENERATED, ShotStatus.QC_PASSED): task.status = TaskStatus.SKIPPED task.result_json = {"reason": "产物已存在,跳过重复生成", "path": shot.output_path} task.finished_at = now_utc() else: shot.status = ShotStatus.QUEUED shot.stage = "resume" events.log_event(db, "task.recovered", project_id=task.project_id, task_id=task.id, shot_id=task.shot_id, level="warn", message=("执行进程已重启,中断的任务已重新排队(断点续跑)" if dead_runner else "检测到中断任务,已重新排队(断点续跑)"), entity_type="task", entity_id=task.id, data={"attempts": task.attempts, "kind": task.kind, "refunded": bool(checkpoint.get("attempt_refunded"))}) recovered.append({"task_id": task.id, "kind": task.kind, "status": task.status}) # 镜头状态与任务不一致(例如 API 重启导致 queued 悬挂)→ 归位 dangling = db.scalars(select(Shot).where(Shot.status.in_(ShotStatus.ACTIVE))).all() for shot in dangling: live = db.scalars(select(Task).where(Task.shot_id == shot.id, Task.status.in_(TaskStatus.ACTIVE + [TaskStatus.PLANNED]))).first() if live is None: if shot.output_path and storage.exists(shot.output_path): shot.status = ShotStatus.GENERATED shot.stage = "resume_generated" else: shot.status = ShotStatus.READY shot.stage = "resume_ready" events.log_event(db, "shot.recovered", project_id=shot.project_id, shot_id=shot.id, level="warn", message="镜头状态与队列不一致,已按断点续跑归位", entity_type="shot", entity_id=shot.id) db.commit() return {"recovered": recovered, "count": len(recovered)} def queue_overview(db: Session, *, project_id: int | None = None) -> dict: from sqlalchemy import func stmt = select(Task.status, Task.kind, func.count(Task.id)).group_by(Task.status, Task.kind) if project_id is not None: stmt = stmt.where(Task.project_id == project_id) matrix = [{"status": s, "kind": k, "count": int(c)} for s, k, c in db.execute(stmt).all()] counts: dict[str, int] = {} for row in matrix: counts[row["status"]] = counts.get(row["status"], 0) + row["count"] recent_stmt = select(Task).order_by(Task.id.desc()).limit(200) if project_id is not None: recent_stmt = recent_stmt.where(Task.project_id == project_id) recent = db.scalars(recent_stmt) return {"counts": counts, "matrix": matrix, "tasks": [status.task_dict(t) for t in recent], "capacity": [row.as_dict() for row in capacity.snapshot(db, project_id=project_id)], "executor": execution.backend_name()}