nex_animation/backend/app/api/pipeline.py

338 lines
18 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""视频模型调度中心接口(图1 模块③):任务清单、入队、限流派发、重试、
断点续跑、队列观测,以及质检与人工审核(图2 节点 4~10)。"""
from __future__ import annotations
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import select
from sqlalchemy.orm import Session
from app import schemas
from app.api import deps
from app.constants import ShotStatus, TaskKind, TaskStatus
from app.db import get_db
from app.models import Project, ReviewRecord, Shot, Task, now_utc
from app.services import daemon, dispatch, events, quality, status, storage
router = APIRouter(tags=["pipeline"])
# ---------------------------------------------------------------- 任务清单(图2 节点 4)
@router.get("/api/projects/{project_id}/tasklist")
def get_tasklist(project_id: int, force: bool = False, allow_unready: bool = False,
include_voiceover: bool = True, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
return {"plan": dispatch.plan_task_list(db, project, force=force, allow_unready=allow_unready,
include_voiceover=include_voiceover)}
@router.post("/api/projects/{project_id}/tasklist")
def build_tasklist(project_id: int, data: schemas.PlanOptions, db: Session = Depends(get_db)):
"""只生成分镜任务清单(planned),不入队,便于人工确认成本与阻塞项。"""
project = deps.get_project(db, project_id)
plan = dispatch.plan_task_list(db, project, shot_ids=data.shot_ids, force=data.force,
allow_unready=data.allow_unready,
include_voiceover=data.include_voiceover)
deps.summary(db, project)
events.log_event(db, "plan.created", project_id=project.id,
message=f"任务清单:{plan['queued_count']} 个生成 / {plan['voiceover_count']} 个配音 / "
f"{plan['blocked_count']} 个被阻塞,预计成本 {plan['estimated_cost']} {plan['currency']}",
data={"estimated_cost": plan["estimated_cost"]})
return {"plan": plan}
# ---------------------------------------------------------------- 入队(图2 节点 6 前置)
@router.post("/api/projects/{project_id}/tasks")
def create_tasks(project_id: int, data: schemas.TaskCreate, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
result = deps.enqueue_generation(db, project, shot_ids=data.shot_ids, force=data.force,
allow_unready=data.allow_unready, priority=data.priority,
actor=data.actor, with_voiceover=data.with_voiceover)
return {"ok": True, "result": result, "summary": deps.summary(db, project),
"queue": dispatch.queue_overview(db, project_id=project.id)}
# ---------------------------------------------------------------- 派发与队列(图2 节点 6)
@router.get("/api/projects/{project_id}/queue")
def project_queue(project_id: int, db: Session = Depends(get_db)):
deps.get_project(db, project_id)
return dispatch.queue_overview(db, project_id=project_id)
@router.post("/api/projects/{project_id}/dispatch")
def dispatch_now(project_id: int, data: schemas.DispatchOptions, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
dispatched = dispatch.dispatch_due(db, limit=data.limit, project_id=project.id)
drained = None
if data.drain: # inline 模式:同步跑到队列清空
drained = daemon.drain_queue(db, project_id=project.id, timeout=data.timeout)
# 队列任务由独立会话落库:本会话缓存必须失效,否则回读的是执行前的旧值
db.expire_all()
return {"ok": True, "result": {"dispatched": dispatched, "drained": drained},
"summary": deps.summary(db, project), "queue": dispatch.queue_overview(db, project_id=project.id)}
@router.get("/api/tasks")
def list_tasks(project_id: int | None = Query(None), status_filter: str = Query("", alias="status"),
kind: str = Query(""), limit: int = Query(200), db: Session = Depends(get_db)):
stmt = select(Task).order_by(Task.priority, Task.id)
if project_id is not None:
stmt = stmt.where(Task.project_id == project_id)
if status_filter:
stmt = stmt.where(Task.status == status_filter)
if kind:
stmt = stmt.where(Task.kind == kind)
rows = db.scalars(stmt.limit(min(1000, max(1, limit)))).all()
counts: dict[str, int] = {}
for row in rows:
counts[row.status] = counts.get(row.status, 0) + 1
return {"tasks": [status.task_dict(t) for t in rows], "count": len(rows), "counts": counts}
# 注意顺序:这条必须排在 /api/tasks/{task_id} 前面,否则 activity 会被当成 task_id 解析
@router.get("/api/tasks/activity")
def get_task_activity(active_only: bool = Query(False), owner_type: str = Query(""),
limit: int = Query(status.ACTIVITY_LIMIT), db: Session = Depends(get_db)):
"""平台动态:此刻在跑什么、刚刚跑完了什么(项目侧 + 平台侧同源一条接口)。
顶栏每 3 秒问一次,所以它只扫任务表里进行中的那几行 + 最近 6 小时的落定记录,
不碰项目统计。前端据此决定要不要重新拉列表——用户不必再手动刷新页面去猜
「图到底出没了」。
"""
return {"ok": True, "result": status.activity(db, limit=limit, owner_type=owner_type,
active_only=active_only)}
@router.get("/api/tasks/{task_id}")
def get_task(task_id: int, db: Session = Depends(get_db)):
return {"task": status.task_dict(deps.get_task(db, task_id))}
@router.post("/api/tasks/{task_id}/retry")
def retry_task(task_id: int, data: schemas.RetryOptions, db: Session = Depends(get_db)):
task = deps.get_task(db, task_id)
# 平台资产生成不属于任何项目:按项目取父记录会凭空 404,界面就永远没法重试
project = db.get(Project, task.project_id) if task.project_id else None
if task.status not in (TaskStatus.FAILED, TaskStatus.CANCELLED):
raise HTTPException(409, f"当前状态 {task.status} 不能重试")
task.status = TaskStatus.QUEUED
if data.reset_attempts:
task.attempts = 0
task.max_attempts = max(1, task.max_attempts or 3)
task.next_retry_at = None
task.error_code = ""
task.queued_at = now_utc()
task.finished_at = None
if task.shot_id:
shot = db.get(Shot, task.shot_id)
if shot:
shot.status = ShotStatus.QUEUED
shot.stage = "queued"
db.flush()
events.log_event(db, "task.requeued", project_id=project.id if project else None,
task_id=task.id, shot_id=task.shot_id,
message=f"任务 #{task.id} 人工重试入队", entity_type="task", entity_id=task.id)
if project is None:
db.commit()
# 用户是点了「重试」才来的,不该再等一轮调度扫描
dispatch.dispatch_task(db, db.get(Task, task.id))
return {"ok": True, "result": status.task_dict(db.get(Task, task.id))}
return deps.done(db, project, {"task_id": task.id, "status": task.status})
@router.post("/api/tasks/{task_id}/cancel")
def cancel_task(task_id: int, data: schemas.CancelOptions, db: Session = Depends(get_db)):
task = deps.get_task(db, task_id)
project = db.get(Project, task.project_id) if task.project_id else None
if task.status in TaskStatus.TERMINAL:
raise HTTPException(409, f"任务已结束({task.status}),无需取消")
dispatch.cancel_task(db, task, reason=data.reason)
if project is None:
db.commit()
return {"ok": True, "result": status.task_dict(db.get(Task, task.id))}
return deps.done(db, project, {"task_id": task.id, "status": task.status})
@router.post("/api/projects/{project_id}/tasks/retry-failed")
def retry_failed_tasks(project_id: int, data: schemas.RetryOptions, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
result = dispatch.retry_failed(db, project.id, task_ids=data.task_ids,
reset_attempts=data.reset_attempts, only_codes=data.only_codes)
return deps.done(db, project, result)
@router.post("/api/projects/{project_id}/tasks/cancel-active")
def cancel_active(project_id: int, data: schemas.CancelOptions, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
rows = db.scalars(select(Task).where(Task.project_id == project.id,
Task.status.in_(TaskStatus.ACTIVE))).all()
for task in rows:
dispatch.cancel_task(db, task, reason=data.reason)
return deps.done(db, project, {"cancelled": [t.id for t in rows], "count": len(rows)})
# ---------------------------------------------------------------- 断点续跑
@router.post("/api/projects/{project_id}/resume")
def resume_project(project_id: int, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
resumed = dispatch.resume_interrupted(db)
mine = [x for x in resumed.get("recovered", [])
if db.get(Task, x["task_id"]) is not None and db.get(Task, x["task_id"]).project_id == project.id]
dispatched = dispatch.dispatch_due(db, project_id=project.id)
return deps.done(db, project, {"resume": resumed, "recovered_here": mine,
"dispatched": dispatched})
@router.get("/api/projects/{project_id}/tasks/summary")
def project_task_summary(project_id: int, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
return {"plan": dispatch.plan_task_list(db, project), "queue": dispatch.queue_overview(db,
project_id=project.id)}
# ---------------------------------------------------------------- 质检(图2 节点 9)
@router.get("/api/projects/{project_id}/qc")
def qc_overview(project_id: int, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
return {"qc": quality.qc_summary(db, project), "summary": deps.summary(db, project)}
@router.post("/api/projects/{project_id}/shots/{shot_id}/qc")
def qc_shot(project_id: int, shot_id: int, drain: bool = False, db: Session = Depends(get_db)):
"""立即质检:优先走任务队列(可重放),inline 模式支持同步执行。"""
project = deps.get_project(db, project_id)
shot = deps.get_shot(db, project_id, shot_id)
if not shot.output_path:
raise HTTPException(400, "镜头尚未生成产物,无法质检")
task = dispatch.new_qc_task(db, project, shot)
extra = {}
if drain:
drained = daemon.drain_queue(db, project_id=project.id, timeout=300)
# 队列任务由独立会话落库:本会话缓存必须失效,否则回读的是执行前的旧值
db.expire_all()
extra = {"drain": drained}
fresh = db.get(Shot, shot.id)
return deps.done(db, project, {"task_id": task.id, "qc": (fresh.qc_json or {}),
"score": fresh.qc_score, **extra})
@router.post("/api/projects/{project_id}/qc/run")
def qc_run_all(project_id: int, data: schemas.QCRun | None = None, db: Session = Depends(get_db)):
"""整批复检:把项目里所有有产物的镜头重新过一遍质检。
以前质检只在镜头生成后自动跑一次,`quality_check` 步骤又会跳过已审核通过的
镜头,于是「画面质检」这种后来加上的新规则永远管不到历史项目——
成片满是穿帮,界面上却全是 100 分。这里给它一个明确的复查入口。
"""
project = deps.get_project(db, project_id)
data = data or schemas.QCRun()
rows = list(project.shots or [])
if data.shot_ids:
wanted = {int(x) for x in data.shot_ids}
rows = [row for row in rows if row.id in wanted]
rows = [row for row in rows if row.output_path and storage.exists(row.output_path)]
if not data.include_approved:
rows = [row for row in rows if row.status != ShotStatus.APPROVED]
if data.only_failed:
rows = [row for row in rows if not bool((row.qc_json or {}).get("passed"))]
if not rows:
raise HTTPException(400, "没有可质检的镜头产物(请先执行镜头生成)")
task_ids = []
for shot in rows:
active = db.scalars(select(Task).where(
Task.project_id == project.id, Task.shot_id == shot.id,
Task.kind == TaskKind.QC_SHOT, Task.status.in_(TaskStatus.ACTIVE))).first()
task = active or dispatch.new_qc_task(db, project, shot)
task_ids.append(task.id)
deps.summary(db, project)
events.log_event(db, "qc.batch_requested", project_id=project.id, level="info",
message=f"发起整批复检:{len(task_ids)} 个镜头",
data={"shot_ids": [row.id for row in rows], "drain": data.drain})
db.commit()
extra = {}
if data.drain:
extra["drain"] = daemon.drain_queue(db, project_id=project.id, timeout=data.timeout,
watch_task_ids=task_ids)
db.expire_all()
else:
# 页面走这条:先派发再让前端轮询。一镜看图十几秒,整批同步等会把
# 浏览器请求挂成「页面无响应」,用户根本不知道跑到哪一镜了。
extra["dispatched"] = dispatch.dispatch_due(db, project_id=project.id,
limit=max(8, len(task_ids)))
scores = [{"shot_id": row.id, "order_index": row.order_index, "title": row.title,
"score": row.qc_score, "passed": bool((row.qc_json or {}).get("passed")),
"visual_score": (row.qc_json or {}).get("probe", {}).get("visual_score"),
"issues": ((row.qc_json or {}).get("issues") or [])[:3]}
for row in db.scalars(select(Shot).where(Shot.id.in_([r.id for r in rows]))
.order_by(Shot.order_index))]
return deps.done(db, project, {"project_id": project.id, "task_ids": task_ids,
"shots": scores, "qc": quality.qc_summary(db, project), **extra})
# ---------------------------------------------------------------- 人工审核(图2 节点 10)
@router.post("/api/projects/{project_id}/shots/{shot_id}/approve")
def approve_shot(project_id: int, shot_id: int, data: schemas.Approve, db: Session = Depends(get_db)):
project = deps.get_project(db, project_id)
shot = deps.get_shot(db, project_id, shot_id)
if not shot.output_path:
raise HTTPException(400, "镜头还没有产物,无法审核通过")
shot.status = ShotStatus.APPROVED
shot.stage = "review"
shot.review_state = "pass"
shot.review_notes = data.notes
db.add(ReviewRecord(project_id=project.id, shot_id=shot.id, kind="manual", verdict="pass",
score=shot.qc_score, checks_json={"qc": shot.qc_json or {}},
notes=data.notes, reviewer=data.reviewer))
db.flush()
events.log_event(db, "shot.approved", project_id=project.id, shot_id=shot.id,
message=f"人工审核通过:{shot.code}「{shot.title}」", entity_type="shot", entity_id=shot.id,
actor=data.reviewer, data={"notes": data.notes})
return deps.done(db, project, {"shot": status.shot_dict(db, shot)})
@router.post("/api/projects/{project_id}/shots/{shot_id}/reject")
def reject_shot(project_id: int, shot_id: int, data: schemas.Reject, db: Session = Depends(get_db)):
"""不合格 → 可带新提示词直接重新生成(图2 的「调整提示词 → 重新生成」回路)。"""
project = deps.get_project(db, project_id)
shot = deps.get_shot(db, project_id, shot_id)
shot.status = ShotStatus.REJECTED
shot.stage = "review"
shot.review_state = "fail"
shot.review_notes = data.notes
db.add(ReviewRecord(project_id=project.id, shot_id=shot.id, kind="manual", verdict="fail",
score=shot.qc_score, checks_json={"qc": shot.qc_json or {}},
notes=data.notes, reviewer=data.reviewer))
if data.prompt:
shot.params_json = {**(shot.params_json or {}), "prompt_override": data.prompt}
if data.negative_prompt:
shot.params_json = {**(shot.params_json or {}), "negative_override": data.negative_prompt}
db.flush()
events.log_event(db, "shot.rejected", project_id=project.id, shot_id=shot.id, level="warn",
message=f"人工审核不合格:{shot.code}「{shot.title}」{(':' + data.notes) if data.notes else ''}",
entity_type="shot", entity_id=shot.id, actor=data.reviewer,
data={"notes": data.notes, "regenerate": data.regenerate})
result: dict = {"shot": status.shot_dict(db, shot)}
if data.regenerate:
result["requeue"] = deps.enqueue_generation(db, project, shot_ids=[shot.id], force=True,
allow_unready=True, priority=data.priority,
actor=data.reviewer, with_voiceover=False)
return deps.done(db, project, result)
@router.get("/api/projects/{project_id}/reviews")
def list_reviews(project_id: int, kind: str | None = None, shot_id: int | None = None,
db: Session = Depends(get_db)):
"""审核记录:kind=auto 只看机器质检,kind=manual 只看人工审核,缺省返回全部。"""
deps.get_project(db, project_id)
stmt = select(ReviewRecord).where(ReviewRecord.project_id == project_id)
if kind:
stmt = stmt.where(ReviewRecord.kind == kind)
if shot_id:
stmt = stmt.where(ReviewRecord.shot_id == shot_id)
rows = db.scalars(stmt.order_by(ReviewRecord.id.desc()).limit(500)).all()
items = [{"id": r.id, "shot_id": r.shot_id, "task_id": r.task_id, "kind": r.kind,
"verdict": r.verdict, "score": r.score, "notes": r.notes,
"reviewer": r.reviewer, "checks": r.checks_json or {},
"created_at": r.created_at} for r in rows]
return {"reviews": items, "count": len(items)}