nex_animation/backend/app/services/agent.py

2841 lines
138 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""LLM 项目调度器。
LLM 只负责把自然语言目标翻译成受控的 JSON 计划;执行权始终留在平台:
动作有白名单、步骤有顺序、参数由后端推导,模型不能直接执行命令、删除项目
或修改用户资产。计划必须先落库,再由用户确认执行,所有步骤状态可追踪。
"""
from __future__ import annotations
import json
import logging
import re
import threading
import time
from datetime import datetime, timezone
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.constants import (AssetKind, CameraAngle, CameraMovement, ReviewVerdict, ShotSize, ShotStatus,
TaskKind, TaskStatus, Transition, canvas_for)
from app.db import SessionLocal
from app.models import (AgentRun, Asset, ExportRecord, ModelSpec, Project, ReviewRecord, Shot, Task,
Timeline, now_utc)
from app.providers import registry as model_registry
from app.providers.base import AssetGenRequest
from app.services import (costing, daemon, dispatch, events, importer, preflight, prompt as
prompt_service, status, storage, timeline as timeline_service)
log = logging.getLogger("nex.agent")
ACTION_ORDER = (
"resume",
"rewrite_shots",
"generate_identity_assets",
"generate_keyframes",
"generate_shots",
"quality_check",
"approve_passed",
"build_timeline",
"compose",
)
ACTION_LABELS = {
"resume": "恢复中断任务",
"rewrite_shots": "改写分镜脚本(新主题)",
"generate_identity_assets": "补齐人物/场景定妆素材与声线",
"generate_keyframes": "补齐分镜关键帧",
"generate_shots": "生成镜头与配音",
"quality_check": "自动质检",
"approve_passed": "放行质检合格镜头",
"build_timeline": "重建时间线",
"compose": "合成并导出成片",
}
ACTION_DESCRIPTIONS = {
"resume": "只恢复上次中断或僵死的任务,不会重复提交已有产物。",
"rewrite_shots": "按新主题整组改写分镜脚本(画面描述/台词/时长/景别/运镜),"
"沿用项目已有人物、场景与声线;旧镜头产物会作废并从媒体库清理。",
"generate_identity_assets": "为分镜真正引用、但还没有身份素材的资产补缺:人物定妆图与三视图、场景母图(调用图像模型生成并入素材库),以及说话人的可用声线(只补空值或明显性别不符的绑定,不覆盖用户已有的设置)。换主题、新人物或新场景后必须排在 generate_keyframes 之前,否则关键帧与视频只能凭文字描述出图,前后镜头不是同一个人、同一个地方。",
"generate_keyframes": "仅为图生视频/首尾帧模式中缺少关键帧的镜头创建图像任务。",
"generate_shots": "按镜头清单创建视频与缺失配音任务,并同步排空队列。",
"quality_check": "对已有视频产物、尚未完成自动质检的镜头执行质检(含画面质检:抽帧交大模型"
"核对人物分身、鬼影、服装串人、镜头瑕疵);用户要求「重新质检/复查画面/按新标准"
"再过一遍」时给 force=true,连已审核通过的镜头一起复检。",
"approve_passed": "仅当 auto_approve=true 时可用;自动放行自动质检合格的镜头,审核者记录为 agent。",
"build_timeline": "使用已审核通过的镜头重建时间线;当前时间线已覆盖同样镜头时跳过,"
"但 policy.regeneration 命中(编导要求重做)时必须排它,因为片段记录的是旧产物路径。",
"compose": "按当前时间线合成并导出 MP4;同时间线已有成功导出时跳过,"
"但 policy.regeneration 命中时必须排它,否则重做后交付的还是旧成片。",
}
# 编导说「重新生成 / 重做 / 重新配音」时,已交付项目不能因为「产物都在」被判定为无事可做。
# 实测(run 36):LLM 在已交付项目上回了 outcome=complete,界面一片绿、零个任务、
# 盘上一个新文件都没有。重做意图因此由后端确定性识别:既提醒模型,也作为兜底计划的依据。
# 「重新」太常见:重新排序/重新命名/重新审核都不是让模型重跑素材,
# 因此后面紧跟管理类动词时不算重做意图。
REDO_SKIP_AFTER = "排序命整检审总计估说解释调绑上导布置测看汇"
REDO_RE = re.compile(
r"(重新(来一[次遍]|[^\s" + REDO_SKIP_AFTER + r"])|重做|重跑|重生成|重出|重来"
r"|再来一[次遍]|再跑一[次遍]|regenerate|redo|rerun|re-?run)",
re.IGNORECASE)
REDO_SCOPE_PATTERNS = (
("keyframes", re.compile(r"关键帧|首帧|首尾帧|定妆|三视图|立绘|设定图|参考图")),
("video", re.compile(r"画面|镜头|视频|片段|分镜|素材")),
("voice", re.compile(r"配音|声音|音色|语音|台词|口型|旁白|配[^\s]{0,4}音")),
("export", re.compile(r"成片|合成|导出|混剪|剪辑|时间线|字幕")),
)
REDO_SCOPE_LABELS = {"keyframes": "关键帧", "video": "镜头画面", "voice": "配音", "export": "成片"}
def regeneration_scope(message: str) -> dict:
"""编导明确要求重做时返回影响范围;不含重做意图时返回 {}。
范围会向下游传导:换了定妆照/关键帧必然要重出画面,重出画面或配音必然要
重建时间线并重新合成 —— 时间线片段存的是产物路径快照,不重建就还是旧素材。
"""
text = (message or "").strip()
if not text or not REDO_RE.search(text):
return {}
scope = {name: bool(pattern.search(text)) for name, pattern in REDO_SCOPE_PATTERNS}
if not any(scope.values()):
scope = {"keyframes": False, "video": True, "voice": True, "export": True}
if scope["keyframes"] or scope["video"] or scope["voice"]:
scope["export"] = True
if scope["keyframes"]:
scope["video"] = True
return {key: value for key, value in scope.items() if value}
def regeneration_note(scope: dict) -> str:
return "、".join(REDO_SCOPE_LABELS[key] for key in REDO_SCOPE_LABELS if scope.get(key))
class AgentError(RuntimeError):
code = "agent_error"
class AgentConfigurationError(AgentError):
code = "agent_not_configured"
class AgentPlanError(AgentError):
code = "agent_plan_invalid"
class AgentStateError(AgentError):
code = "agent_state_invalid"
class AgentStepError(AgentError):
def __init__(self, message: str, *, code: str = "agent_step_failed") -> None:
super().__init__(message)
self.code = code
class StoryboardDraftShot(BaseModel):
"""LLM 提议的分镜行;字段与导入器的脚本列一一对应,落库仍走 importer。"""
model_config = ConfigDict(extra="ignore")
code: str = Field(default="", max_length=40)
title: str = Field(default="", max_length=200)
description: str = Field(min_length=1, max_length=4000)
duration: float = Field(default=5.0, gt=0, le=600)
shot_size: str = Field(default="", max_length=30)
camera_angle: str = Field(default="", max_length=30)
camera_movement: str = Field(default="", max_length=30)
dialogue: str = Field(default="", max_length=2000)
speaker: str = Field(default="", max_length=120)
characters: list[str] = Field(default_factory=list, max_length=20)
scene: str = Field(default="", max_length=160)
transition: str = Field(default="", max_length=30)
style_note: str = Field(default="", max_length=1000)
class NewSceneDraft(BaseModel):
"""新主题需要的新场景;由 rewrite_shots 一并创建,避免「换主题却卡在没有场景」。"""
model_config = ConfigDict(extra="ignore")
name: str = Field(min_length=1, max_length=160)
description: str = Field(default="", max_length=2000)
prompt: str = Field(default="", max_length=2000)
lighting: str = Field(default="", max_length=120)
time_of_day: str = Field(default="", max_length=60)
palette: str = Field(default="", max_length=160)
class NewCharacterDraft(BaseModel):
model_config = ConfigDict(extra="ignore")
name: str = Field(min_length=1, max_length=120)
role: str = Field(default="配角", max_length=60)
appearance: str = Field(default="", max_length=2000)
persona_prompt: str = Field(default="", max_length=2000)
class PlanStep(BaseModel):
# 不同 OpenAI 兼容模型常会多返回 confidence / status 等辅助字段;
# 安全边界由 action、shot_ids 和执行器决定,允许忽略无害扩展字段。
model_config = ConfigDict(extra="ignore")
action: Literal[
"resume",
"rewrite_shots",
"generate_identity_assets",
"generate_keyframes",
"generate_shots",
"quality_check",
"approve_passed",
"build_timeline",
"compose",
]
reason: str = Field(default="", max_length=1000)
shot_ids: list[int] | None = Field(default=None, max_length=500)
# 仅 quality_check 使用:连「已审核通过」的镜头也重新质检一遍
force: bool = False
# 仅 quality_check 使用:质检不合格时自动把问题写回镜头并重新生成的轮数
rework_rounds: int | None = Field(default=None, ge=0, le=3)
# 仅 rewrite_shots 使用:整组新分镜 + 分镜引用但项目里还没有的场景/人物。
storyboard: list[StoryboardDraftShot] = Field(default_factory=list, max_length=200)
new_scenes: list[NewSceneDraft] = Field(default_factory=list, max_length=40)
new_characters: list[NewCharacterDraft] = Field(default_factory=list, max_length=40)
class PlanPayload(BaseModel):
model_config = ConfigDict(extra="ignore")
intent: str = Field(default="continue", max_length=80)
outcome: Literal["execute", "blocked", "complete"] | None = None
reply: str = Field(min_length=1, max_length=6000)
summary: str = Field(default="", max_length=1200)
steps: list[PlanStep] = Field(default_factory=list, max_length=len(ACTION_ORDER))
blockers: list[str] = Field(default_factory=list, max_length=20)
warnings: list[str] = Field(default_factory=list, max_length=20)
def _is_mock_spec(spec) -> bool:
provider = str(getattr(spec, "provider", "") or "").lower()
model = str(getattr(spec, "model", "") or "").lower()
adapter = str(getattr(spec, "adapter", "") or "").lower()
return provider == "mock" or model.startswith("mock-") or adapter.startswith("mock")
def _resolve_real_llm(db: Session):
spec, adapter = model_registry.resolve_llm(db)
if spec is None or not spec.enabled:
raise AgentConfigurationError("没有启用的大语言模型,请先在模型中心启用真实 LLM")
if _is_mock_spec(spec):
raise AgentConfigurationError("智能调度不能使用 Mock LLM;请在模型中心启用并设为默认的真实 LLM")
config = model_registry.runtime_config(db, spec)
if not adapter.is_available(config):
raise AgentConfigurationError(
f"大语言模型 {spec.provider}/{spec.model} 不可用,请检查 Base URL、API Key 和连接状态"
)
return spec, adapter
def _resolve_real_media(db: Session, media_type: str):
spec, adapter = model_registry.resolve_asset(db, media_type)
if spec is None or not spec.enabled:
raise AgentStepError(f"没有启用的 {media_type} 模型", code="model_not_configured")
if _is_mock_spec(spec):
label = {"image": "图像", "tts": "语音"}.get(media_type, media_type)
raise AgentStepError(
f"没有可用的真实{label}模型;智能调度不会回退到 Mock 模型",
code="mock_model_forbidden",
)
if not adapter.is_available(model_registry.runtime_config(db, spec)):
raise AgentStepError(
f"{media_type} 模型 {spec.provider}/{spec.model} 未配置连接信息",
code="model_unavailable",
)
return spec, adapter
def _jsonable(value):
if isinstance(value, datetime):
return value.isoformat()
return value
def _clip(text: str | None, limit: int = 500) -> str:
value = (text or "").strip()
return value if len(value) <= limit else value[:limit - 1] + "…"
def _model_choices(db: Session) -> dict:
rows = db.scalars(select(ModelSpec)).all()
result: dict[str, list[dict]] = {}
for row in rows:
if not row.enabled or _is_mock_spec(row):
continue
result.setdefault(row.media_type, []).append({
"provider": row.provider,
"model": row.model,
"display_name": row.display_name,
"is_default": bool(row.is_default),
"available": model_registry.is_available(row, db),
})
return result
def _reference_images(db: Session, project: Project, shot: Shot, limit: int) -> tuple[list[int], list[str]]:
"""关键帧阶段实际入模的参考素材:场景母图在前(决定构图与空间),人物母版在后。
与视频阶段共用 ``prompt.collect_reference_details``,保证「素材库里绑了的」
与「模型真正收到的」是同一份清单,不再各写一套优先级。
旧实现每个人物只取一张图(定妆图命中后就跳过三视图),
母版准备的正面像 + 三视图实际只有一张进了模型。
"""
wanted_limit = max(0, limit)
if wanted_limit == 0:
return [], []
details = prompt_service.collect_reference_details(db, shot, scene_first=True)
picked = details[:wanted_limit]
return ([int(row["asset_id"]) for row in picked if row.get("asset_id")],
[str(row["path"]) for row in picked])
def _enum_options() -> dict:
"""把分镜可用枚举交给 LLM,避免它写出「推镜」这类只能靠导入器猜的中文标签。"""
return {
"shot_size": list(ShotSize.LABELS.items()),
"camera_angle": list(CameraAngle.LABELS.items()),
"camera_movement": list(CameraMovement.LABELS.items()),
"transition": list(Transition.LABELS.items()),
}
def _video_model_window(db: Session, project: Project) -> dict:
"""项目默认视频模型的能力区间:rewrite_shots 的时长必须落在其中。"""
spec, _adapter = model_registry.resolve_video(db, project.default_provider, None)
caps = model_registry.capabilities_of(spec)
return {
"provider": spec.provider,
"model": spec.model,
"min_duration": float(caps.min_duration),
"max_duration": float(caps.max_duration),
"aspect_ratios": list(caps.aspect_ratios),
"resolutions": list(caps.resolutions),
}
def build_context(db: Session, project: Project, *, message: str, auto_approve: bool) -> dict:
scope = regeneration_scope(message)
readiness = preflight.project_readiness(db, project)
_gaps = _identity_gaps(db, project)
# 直接告诉模型「谁还没有脸、哪个场景还没有母图、谁的音色没着落」,
# 比让它自己从资产清单里推断可靠得多。
identity_gaps = {
"characters_without_portrait": [row.name for row in _gaps["portraits"]],
"characters_without_turnaround": [row.name for row in _gaps["turnarounds"]],
"scenes_without_image": [row.name for row in _gaps["scene_images"]],
"speakers_without_voice": [
str(gap["character"].name) + ":" + str(gap["reason"])
for gap in _gaps["voice_gaps"]],
}
shots = sorted(project.shots or [], key=lambda row: (row.order_index, row.id))
task_counts = {
state: int(count)
for state, count in db.execute(
select(Task.status, func.count(Task.id)).where(Task.project_id == project.id).group_by(Task.status)
).all()
}
failed_tasks = db.scalars(select(Task).where(
Task.project_id == project.id, Task.status == TaskStatus.FAILED
).order_by(Task.id.desc()).limit(5)).all()
timeline = status.current_timeline(db, project)
exports = db.scalars(select(ExportRecord).where(ExportRecord.project_id == project.id)
.order_by(ExportRecord.id.desc()).limit(3)).all()
shot_rows = []
for shot in shots:
keyframe = storage.first_asset_path(db, "shot", shot.id, AssetKind.KEYFRAME)
shot_rows.append({
"id": shot.id,
"order_index": shot.order_index,
"title": _clip(shot.title, 160),
"description": _clip(shot.description, 320),
"duration": round(float(shot.duration or 0), 2),
"mode": shot.mode,
"status": shot.status,
"stage": shot.stage,
"dialogue": _clip(shot.dialogue, 200),
"characters": [row.name for row in (shot.characters or [])],
"scenes": [row.name for row in (shot.scenes or [])],
"speaker": shot.speaker.name if shot.speaker else "",
"voice": shot.voice.name if shot.voice else "",
"has_keyframe": bool(keyframe),
"has_output": bool(storage.exists(shot.output_path)),
"qc_score": shot.qc_score,
"qc_passed": bool((shot.qc_json or {}).get("passed")),
"review_state": shot.review_state,
})
check_rows = []
for check in readiness.get("checks", []):
if check.get("ok"):
continue
check_rows.append({
"shot_id": check["shot_id"],
"order_index": check["order_index"],
"issues": [
{"code": issue["code"], "severity": issue["severity"], "message": issue["message"]}
for issue in check.get("issues", []) if issue["severity"] == "error"
][:5],
})
return {
"project": {
"id": project.id,
"name": project.name,
"description": _clip(project.description, 500),
"genre": project.genre,
"style_guide": _clip(project.style_guide, 500),
"target_duration": project.target_duration,
"aspect_ratio": project.aspect_ratio,
"resolution": project.resolution,
"fps": project.fps,
"status": project.status,
"current_stage": project.current_stage,
},
"assets": {
"characters": [{"id": row.id, "name": row.name, "role": row.role,
"has_portrait": bool(storage.first_asset_path(
db, "human", row.id, AssetKind.PORTRAIT, AssetKind.TURNAROUND))}
for row in (project.characters or [])],
"scenes": [{"id": row.id, "name": row.name,
"has_image": bool(storage.first_asset_path(db, "scene", row.id, AssetKind.SCENE_IMAGE))}
for row in (project.scenes or [])],
"voices": [{"id": row.id, "name": row.name, "provider": row.provider,
"gender": row.gender,
"has_reference": bool(str(row.provider_voice_id or "").strip()),
"has_sample": bool(storage.first_asset_path(db, "voice", row.id,
AssetKind.VOICE_SAMPLE))}
for row in (project.voices or [])],
"identity_gaps": identity_gaps,
},
"models": _model_choices(db),
"shots": shot_rows,
"preflight": {
"total": readiness["total"],
"ready": readiness["ready"],
"blocked": readiness["blocked"],
"errors": readiness["errors"],
"warnings": readiness["warnings"],
"project_issues": readiness.get("project_issues", []),
"blocked_shots": check_rows,
},
"tasks": {
"counts": task_counts,
"failed": [{"id": row.id, "kind": row.kind, "shot_id": row.shot_id,
"code": row.error_code, "message": _clip(row.error_message, 400)}
for row in failed_tasks],
},
"timeline": ({
"id": timeline.id,
"version": timeline.version,
"segments": len(timeline.segments or []),
"total_duration_ms": timeline.total_duration_ms,
"shot_ids": [seg.shot_id for seg in sorted(timeline.segments or [],
key=lambda row: row.order_index)],
} if timeline else None),
"exports": [{"id": row.id, "timeline_id": row.timeline_id, "status": row.status,
"path": row.path, "duration_ms": row.duration_ms} for row in exports],
"options": _enum_options(),
"video_model": _video_model_window(db, project),
"policy": {
"auto_approve": bool(auto_approve),
"regeneration": ({
"requested": True,
"scope": sorted(scope),
"target": regeneration_note(scope),
"instruction": "编导在本条指令里明确要求重做:禁止返回 outcome=complete 或空 steps,"
"必须从最早受影响的动作一路排到 compose。",
} if scope else None),
"allowed_actions": [
{
"action": action,
"label": ACTION_LABELS[action],
"description": ACTION_DESCRIPTIONS[action],
"allowed": action != "approve_passed" or bool(auto_approve),
}
for action in ACTION_ORDER
],
"forbidden": [
"创建、删除或覆盖项目/人物/场景/声线(唯一例外:"
"generate_identity_assets 只补空缺口,不覆盖已有设置)",
"在 rewrite_shots 之外修改分镜、提示词、资产文件或模型配置",
"调用任意 URL、Shell 或未列出的接口",
"把自动审核伪装成人工审核",
],
},
"conversation": {
"message": message,
},
}
def _system_prompt() -> str:
return """你是 Nex Animation 的项目调度规划器。你不创作媒体,也不直接操作平台。
你的唯一任务是把编导的自然语言目标转换成严格 JSON 执行计划,供后端白名单执行器运行。
必须遵守:
1. 只输出一个 JSON 对象,不要 Markdown、代码围栏、解释或额外文本。
2. 只使用 policy.allowed_actions 中的动作;动作不能重复,执行顺序按依赖关系排列。
3. 不虚构镜头、资产、模型或任务。shot_ids 只能来自上下文,省略或 null 表示全部适用镜头。
4. 改写主题用 rewrite_shots:它整组替换分镜脚本,是剧本/主题变化的唯一入口。
除 rewrite_shots 之外不能改动分镜内容;人物、场景、声线资产也不能改动。
4.1 generate_identity_assets 是唯一能新增素材文件的动作:它只补缺口(缺定妆图的人物、
缺母图的场景、说话人没有可用声线),不覆盖用户已经绑好的设置。
新主题、新人物或新场景的计划必须排它,位置固定在 rewrite_shots 之后、
generate_keyframes 之前。
5. approve_passed 只有在 policy.auto_approve=true 时才可用;否则必须在 blockers 中说明等待人工审核。
6. 缺少分镜、关键帧、真实模型或参考素材时,不要假设成功;给出可执行的计划或明确阻塞。
7. 已经完成且产物有效的步骤不要重复安排。项目已交付时优先返回空 steps 和简短完成说明。
7.1 但 policy.regeneration 非空时例外:编导明确说了「重新生成/重做/重新配音/重新合成」,
即使项目已经交付也必须给出计划,绝不能回 complete 或空 steps(那等于把指令吞掉)。
此时 intent 用 regenerate,并从最早受影响的动作排到 compose。
8. compose 依赖已有或同计划内重建的时间线;build_timeline 只使用已审核合格镜头。
quality_check 判不合格时会自动把质检问题写回镜头提示词并重新生成(默认 1 轮,
可用该步骤的 rework_rounds 调整,0 表示不过就停);反复不过的镜头才需要人工改文案。
9. 若无法安全推进,steps 可以为空,但 blockers 必须给出具体原因。
10. outcome 必须准确:有步骤用 execute;无法推进用 blocked;项目已全部完成用 complete。
complete 时 blockers 必须为空,不能把“已经完成”写成阻塞项。
rewrite_shots 的写法(编导要求换主题、改剧情、增删镜头时使用):
- 该步骤必须带 storyboard 数组,每个元素是一镜头,字段:
description(必填,画面描述)、title、duration(秒,必须在 video_model 的时长区间内)、
shot_size / camera_angle / camera_movement(用 context.options 里的英文枚举值)、
dialogue(台词原文,会被配音与字幕使用)、speaker、characters(人物名数组,只能用
context.assets.characters 里已有的名字)、scene(只能用 context.assets.scenes 里已有的名字)、
transition、style_note。
- 新主题需要项目里还没有的场景或人物时,在同一步骤里给出 new_scenes /
new_characters,其中 scene、characters 必须与分镜里引用的名字逐字一致;
不要因为「项目里还没有这个场景」就判定无法执行。
- 分镜数量由主题决定,不要照抄旧镜头数量;台词要口语化并保留编导原话。
- storyboard 落库后会重建分镜并作废旧产物,因此换主题的计划通常是
rewrite_shots → generate_keyframes → generate_shots → quality_check →(build_timeline → compose)。
- 项目已交付但主题变了,绝不能回「无法重新生成」:直接给 rewrite_shots 开头的完整计划。
JSON schema:
{
"intent": "continue|full_delivery|regenerate|rewrite|diagnose|complete",
"outcome": "execute|blocked|complete",
"reply": "给编导的中文说明,明确将做什么、哪里有风险",
"summary": "一句话计划摘要",
"steps": [{"action": "白名单动作", "reason": "为什么执行", "shot_ids": [1,2],
"storyboard": [{"description": "…", "duration": 5, "characters": ["…"], "scene": "…"}],
"new_scenes": [{"name": "…", "description": "…"}],
"new_characters": [{"name": "…", "appearance": "…"}]}],
"blockers": ["必须人工处理的问题"],
"warnings": ["可继续但需注意的风险"]
}"""
def _extract_json(text: str) -> dict:
value = (text or "").strip()
value = re.sub(r"^```(?:json)?\s*", "", value, flags=re.IGNORECASE)
value = re.sub(r"\s*```$", "", value)
try:
parsed = json.loads(value)
except json.JSONDecodeError:
start, end = value.find("{"), value.rfind("}")
if start < 0 or end <= start:
raise AgentPlanError("LLM 未返回可解析的 JSON 计划")
try:
parsed = json.loads(value[start:end + 1])
except json.JSONDecodeError as exc:
raise AgentPlanError(f"LLM 计划 JSON 解析失败:{exc}") from exc
if not isinstance(parsed, dict):
raise AgentPlanError("LLM 计划必须是 JSON 对象")
return parsed
def _validate_plan(payload: PlanPayload, project: Project, *, auto_approve: bool) -> PlanPayload:
known_shots = {shot.id for shot in (project.shots or [])}
seen: set[str] = set()
for step in payload.steps:
if step.action in seen:
raise AgentPlanError(f"计划重复包含动作 {step.action}")
seen.add(step.action)
if step.action == "approve_passed" and not auto_approve:
raise AgentPlanError("未开启自动审核,计划不得包含 approve_passed")
if step.action == "rewrite_shots":
if not step.storyboard:
raise AgentPlanError("rewrite_shots 必须给出 storyboard 新分镜,否则等于没有改动")
missing = [idx + 1 for idx, row in enumerate(step.storyboard)
if not (row.description or "").strip()]
if missing:
raise AgentPlanError(f"storyboard 第 {missing[:5]} 行缺少画面描述")
elif step.storyboard or step.new_scenes or step.new_characters:
raise AgentPlanError(f"动作 {step.action} 不接受 storyboard/new_scenes/new_characters 字段")
if step.shot_ids:
unknown = sorted(set(step.shot_ids) - known_shots)
if unknown:
raise AgentPlanError(f"动作 {step.action} 引用了不存在的镜头:{unknown}")
step.shot_ids = list(dict.fromkeys(step.shot_ids))
payload.steps.sort(key=lambda step: ACTION_ORDER.index(step.action))
if not payload.steps and not payload.blockers and payload.outcome != "complete" \
and payload.intent != "complete":
raise AgentPlanError("计划既没有执行步骤,也没有说明阻塞原因")
return payload
# 思考型 LLM 的三档思考预算:直接决定「一句话下达后多久能看到计划」。
# 实测(qwen3.7-plus / coding.dashscope):同一份上下文
# 关思考 3.6s(0 思考 token) · budget=1024 ≈ 12s · 默认 12.1s(537) / 大计划 110s+(2469)
THINKING_MODES = {
"off": {"enable_thinking": False},
"low": {"thinking_budget": 1024},
"high": {},
}
def _thinking_extra(mode: str | None) -> dict:
return dict(THINKING_MODES.get(str(mode or "low"), THINKING_MODES["low"]))
def _call_llm(db: Session, spec, prompt: str, *, label: str, project_id: int,
extra_body: dict | None = None) -> tuple[str, dict]:
request = AssetGenRequest(
kind="text",
prompt=prompt,
seed=None,
params={"model": spec.model, "extra_body": dict(extra_body or {})},
idempotency_key=label,
label=label,
)
result = model_registry.generate_asset(db, request, "llm", spec.provider, spec.model)
metadata = dict(result.metadata or {})
price = float(spec.price_per_1k_chars or 0.0)
cost = float(result.cost or 0.0) or (len(prompt) / 1000.0 * price)
if cost:
costing.record_cost(
db, project_id=project_id, kind="agent_plan", provider=spec.provider, model=spec.model,
quantity=len(prompt) / 1000.0, unit="1k_chars", unit_price=price, cost=cost,
currency=spec.currency, note="LLM 项目调度计划",
)
costing.record_generation(
db, project_id=project_id, provider=spec.provider, model=spec.model,
kind="agent_plan", prompt=prompt, params_json={"model": spec.model},
output_path="", cost=cost, currency=spec.currency, attempts=1,
metadata_json={"label": label, "usage": metadata.get("usage") or {},
"provider_metadata": metadata},
)
return str(result.text or "").strip(), metadata
def _parse_plan(text: str, project: Project, *, auto_approve: bool) -> PlanPayload:
try:
payload = PlanPayload.model_validate(_extract_json(text))
except ValidationError as exc:
details = "; ".join(
f"{'.'.join(str(part) for part in error['loc'])}: {error['msg']}"
for error in exc.errors()[:6]
)
raise AgentPlanError(f"LLM 计划字段不合法:{details}") from exc
return _validate_plan(payload, project, auto_approve=auto_approve)
def _plan_fields(db: Session, project: Project, *, message: str, auto_approve: bool,
thinking: str = "low") -> dict:
"""调用 LLM 把指令翻译成白名单计划,返回可直接写进 AgentRun 的字段。
规划与落库分开,同一条指令既能在请求线程里同步跑完(脚本/测试),
也能先在库里占位、再由后台线程补齐(页面用的就是这条路径)。
"""
spec, _adapter = _resolve_real_llm(db)
extra_body = _thinking_extra(thinking)
context = build_context(db, project, message=message, auto_approve=auto_approve)
user_prompt = (
f"{_system_prompt()}\n\n"
f"当前项目上下文(JSON):\n{json.dumps(context, ensure_ascii=False, default=_jsonable)}"
)
started = time.monotonic()
try:
raw, usage = _call_llm(
db, spec, user_prompt,
label=f"agent-plan:{project.id}:{int(now_utc().timestamp())}",
project_id=project.id, extra_body=extra_body,
)
except (AgentPlanError, AgentConfigurationError):
raise
except Exception as exc:
# 思考型模型跑满超时是这里最常见的失败;不给出可操作的下一步,用户只会再点一次
elapsed = time.monotonic() - started
hint = ";本轮等待偏久,可把「深度思考」切到「快速(关思考)」后重试" if elapsed > 30 else ""
raise AgentPlanError(f"大模型未能返回规划结果(已等待 {elapsed:.0f}s):{exc}{hint}") from exc
parse_error = ""
try:
plan = _parse_plan(raw, project, auto_approve=auto_approve)
except AgentPlanError as exc:
parse_error = str(exc)
repair_prompt = (
f"{_system_prompt()}\n\n"
f"上一版回复未通过校验:{parse_error}\n"
f"上一版原文:\n{_clip(raw, 10000)}\n\n"
f"请修正并只返回符合 schema 的 JSON。项目上下文:\n"
f"{json.dumps(context, ensure_ascii=False, default=_jsonable)}"
)
raw, repair_usage = _call_llm(
db, spec, repair_prompt,
label=f"agent-repair:{project.id}:{int(now_utc().timestamp())}",
project_id=project.id, extra_body=extra_body,
)
usage = {"first": usage, "repair": repair_usage,
"first_validation_error": parse_error}
plan = _parse_plan(raw, project, auto_approve=auto_approve)
plan = _with_identity_step(db, project, plan)
scope = regeneration_scope(message)
if scope:
plan = _force_regeneration(db, project, plan, scope, auto_approve=auto_approve)
steps = []
for step in plan.steps:
steps.append({
"action": step.action,
"regenerate": dict(scope),
"label": ACTION_LABELS[step.action],
"reason": step.reason,
"shot_ids": step.shot_ids or [],
"force": bool(step.force),
"storyboard": [row.model_dump() for row in step.storyboard],
"new_scenes": [row.model_dump() for row in step.new_scenes],
"new_characters": [row.model_dump() for row in step.new_characters],
"status": "pending",
"detail": "",
"started_at": None,
"finished_at": None,
"result": {},
})
complete = plan.outcome == "complete" or plan.intent == "complete"
run_status = "planned" if steps else ("succeeded" if complete else "blocked")
blockers = list(plan.blockers)
warnings = list(plan.warnings)
if run_status == "succeeded" and not steps:
warnings = blockers + warnings
blockers = []
usage = dict(usage or {})
usage.update({
"thinking": thinking if thinking in THINKING_MODES else "low",
"elapsed_seconds": round(time.monotonic() - started, 1),
"prompt_chars": len(user_prompt),
})
return {
"status": run_status,
"intent": plan.intent,
"reply": plan.reply,
"summary": plan.summary,
"provider": spec.provider,
"model": spec.model,
"plan_json": {"intent": plan.intent, "outcome": plan.outcome, "summary": plan.summary,
"stage": run_status, "regenerate": dict(scope)},
"context_json": {"auto_approve": auto_approve, "project_stage": project.current_stage},
"steps_json": steps,
"blockers_json": blockers,
"warnings_json": warnings,
"usage_json": usage,
"error_code": "",
"error_message": "",
}
def _force_regeneration(db: Session, project: Project, plan: PlanPayload, scope: dict, *,
auto_approve: bool) -> PlanPayload:
"""编导要求重做,但模型仍回「已完成/空计划」时,后端按影响范围确定性地补出步骤。
重做范围来自指令原文(`regeneration_scope`),不依赖模型记不记得排某一步;
模型自己排出的计划原样保留,只是每步都带上重做标记,让执行器别再走
「已审核合格就跳过」的断点续跑捷径。
"""
if plan.steps:
return plan
steps = []
if scope.get("keyframes"):
steps.append(PlanStep(
action="generate_keyframes",
reason="编导要求用新的定妆图重画关键帧,旧关键帧作为首帧会把新形象挡在门外"))
if scope.get("video") or scope.get("voice"):
steps.append(PlanStep(
action="generate_shots",
reason="按项目资产库里的定妆图与音色重新生成镜头画面与配音"))
if scope.get("video"):
steps.append(PlanStep(action="quality_check", reason="新产物必须重新自动质检"))
if auto_approve:
steps.append(PlanStep(action="approve_passed", reason="质检合格即可放行进入时间线"))
if scope.get("export") and (not scope.get("video") or auto_approve):
steps.append(PlanStep(
action="build_timeline",
reason="时间线片段记录的是旧产物路径,必须用新素材重建才能换掉画面与配音"))
steps.append(PlanStep(action="compose", reason="用重建后的时间线导出新成片"))
if not steps:
return plan
note = regeneration_note(scope)
plan.steps = steps
plan.outcome = "execute"
if plan.intent in ("complete", "continue", ""):
plan.intent = "regenerate"
plan.summary = f"按指令强制重做:{note}"
plan.reply = (
f"已识别到明确要求重做({note})。项目此前已交付,"
f"旧产物不会阻止本轮重新生成,会用资产库里的定妆图与音色重新出片并重新合成。")
plan.warnings = (["模型把本轮判定为「已完成」,已按你的重做指令强制排定步骤"]
+ list(plan.warnings))
if scope.get("video") and not auto_approve:
plan.blockers = (["重新生成的镜头需要人工审核,通过后才能重建时间线并合成成片"]
+ list(plan.blockers))
return plan
def _with_identity_step(db: Session, project: Project, plan: PlanPayload) -> PlanPayload:
"""计划只要涉及出图/出片,就先把身份素材补齐,不依赖模型记得排这一步。
缺定妆图与场景母图正是「素材库里只有占位、视频里的人换了脸」的直接原因,
所以这里由后端确定性地插入动作;模型已经排过就原样放行。
"""
actions = [step.action for step in plan.steps]
if not actions or "generate_identity_assets" in actions:
return plan
if not any(action in actions for action in
("rewrite_shots", "generate_keyframes", "generate_shots")):
return plan
if "rewrite_shots" not in actions and not _has_identity_gaps(db, project, None):
return plan
index = next((k for k, action in enumerate(actions)
if action in ("generate_keyframes", "generate_shots")), len(actions))
plan.steps.insert(index, PlanStep(
action="generate_identity_assets",
reason="分镜引用的人物/场景还缺定妆图、场景母图或可用声线,先补齐再出关键帧,"
"保证前后镜头是同一个人、同一个地方",
))
return plan
def _plan_created_event(db: Session, run: AgentRun) -> None:
events.log_event(
db, "agent.plan_created", project_id=run.project_id,
message=f"LLM 已生成智能调度计划 #{run.id}:{run.summary or run.reply[:120]}",
entity_type="agent_run", entity_id=run.id, actor=f"agent:{run.provider}/{run.model}",
data={"steps": [step.get("action") for step in (run.steps_json or [])],
"auto_approve": bool(run.auto_approve)},
)
def create_plan(db: Session, project: Project, *, message: str, auto_approve: bool,
thinking: str = "low") -> AgentRun:
"""同步规划:请求线程里等模型跑完(脚本与测试用这条)。"""
fields = _plan_fields(db, project, message=message, auto_approve=auto_approve,
thinking=thinking)
run = AgentRun(project_id=project.id, request_text=message, auto_approve=auto_approve, **fields)
db.add(run)
db.flush()
_plan_created_event(db, run)
db.commit()
db.refresh(run)
return run
def create_planning_run(db: Session, project: Project, *, message: str, auto_approve: bool,
thinking: str = "low") -> AgentRun:
"""先落一条 planning 记录,让「正在规划」在页面上看得见、刷新也不丢。
规划是一次动辄几十秒到 110s+ 的同步 LLM 调用。以前记录要等模型返回才写库,
这期间项目主页整块右侧面板停在「还没有调度记录」,页面一刷新、代理一断线,
计划就彻底丢了——用户看到的就是「下达了指令,但项目毫无推进」。
"""
busy = db.scalars(select(AgentRun).where(
AgentRun.project_id == project.id,
AgentRun.status.in_(("planning", "running")),
)).first()
if busy is not None:
raise AgentStateError(
f"项目已有智能调度 #{busy.id} 正在{'规划' if busy.status == 'planning' else '执行'},"
"请等待完成或先取消")
spec, _adapter = _resolve_real_llm(db)
thinking = thinking if thinking in THINKING_MODES else "low"
run = AgentRun(
project_id=project.id,
status="planning",
request_text=message,
provider=spec.provider,
model=spec.model,
auto_approve=auto_approve,
plan_json={"stage": "queued"},
context_json={"auto_approve": auto_approve, "project_stage": project.current_stage},
usage_json={"thinking": thinking},
)
db.add(run)
db.flush()
events.log_event(
db, "agent.plan_requested", project_id=project.id,
message=f"已收到指令,LLM 开始盘点项目(RUN #{run.id})",
entity_type="agent_run", entity_id=run.id, actor=f"agent:{spec.provider}/{spec.model}",
data={"thinking": thinking, "auto_approve": auto_approve, "chars": len(message)},
)
db.commit()
db.refresh(run)
return run
def finish_plan(db: Session, run: AgentRun) -> AgentRun:
"""把 planning 记录补成真正的计划;失败也留在记录里,页面能看到原因。"""
project = db.get(Project, run.project_id)
if project is None:
run.status = "failed"
run.error_code = "project_missing"
run.error_message = "项目不存在,规划中止"
run.finished_at = now_utc()
db.commit()
return run
thinking = str((run.usage_json or {}).get("thinking") or "low")
try:
fields = _plan_fields(db, project, message=run.request_text,
auto_approve=bool(run.auto_approve), thinking=thinking)
except Exception as exc:
db.rollback()
run.status = "failed"
run.error_code = getattr(exc, "code", "agent_plan_failed")
run.error_message = str(exc)[:4000]
run.finished_at = now_utc()
run.plan_json = {**(run.plan_json or {}), "stage": "failed"}
events.log_event(
db, "agent.plan_failed", project_id=run.project_id, level="error",
message=f"LLM 规划失败(RUN #{run.id}):{str(exc)[:300]}",
entity_type="agent_run", entity_id=run.id, actor=f"agent:{run.provider}/{run.model}",
data={"code": run.error_code},
)
db.commit()
db.refresh(run)
return run
for key, value in fields.items():
setattr(run, key, value)
db.flush()
_plan_created_event(db, run)
db.commit()
db.refresh(run)
return run
def auto_execute_plan(db: Session, run: AgentRun, *, timeout_seconds: int) -> None:
"""规划完成后由服务端直接开跑整条白名单,不依赖浏览器再点一次。"""
if run.status != "planned" or not run.steps_json:
return
try:
start_run(db, run, timeout_seconds=timeout_seconds)
except Exception as exc:
db.rollback()
run = db.get(AgentRun, run.id)
if run is not None:
run.status = "failed"
run.error_code = getattr(exc, "code", "agent_state_invalid")
run.error_message = f"计划已生成,但自动执行未能启动:{str(exc)[:1500]}"
run.finished_at = now_utc()
db.commit()
events.log_event(
db, "agent.auto_execute_failed", project_id=run.project_id, level="error",
message=f"计划 #{run.id} 自动生成成功但未能自动开始执行:{str(exc)[:200]}",
entity_type="agent_run", entity_id=run.id,
)
db.commit()
return
events.log_event(
db, "agent.auto_execute", project_id=run.project_id,
message=f"计划 #{run.id} 按「计划生成后立即执行」自动开始调度",
entity_type="agent_run", entity_id=run.id, actor=f"agent:{run.provider}/{run.model}",
)
db.commit()
execute_run(run.id, timeout_seconds=timeout_seconds, prestarted=True)
def resolve_plan(run_id: int, *, auto_execute: bool = False,
timeout_seconds: int = 7200) -> None:
"""后台线程里的完整链路:规划 →(可选)直接执行到底。"""
db = SessionLocal()
try:
run = db.get(AgentRun, run_id)
if run is None or run.status != "planning":
return # 用户已取消或服务已回收这条记录,模型结果直接丢弃
finish_plan(db, run)
if auto_execute:
auto_execute_plan(db, run, timeout_seconds=timeout_seconds)
except Exception as exc: # 规划线程自己崩了也要留痕,否则状态永远停在 planning
db.rollback()
run = db.get(AgentRun, run_id)
if run is not None and run.status == "planning":
run.status = "failed"
run.error_code = getattr(exc, "code", "agent_plan_failed")
run.error_message = str(exc)[:4000]
run.finished_at = now_utc()
db.commit()
finally:
db.close()
def plan_in_background(run_id: int, *, auto_execute: bool = False,
timeout_seconds: int = 7200) -> None:
thread = threading.Thread(
target=resolve_plan,
kwargs={"run_id": run_id, "auto_execute": auto_execute,
"timeout_seconds": timeout_seconds},
name=f"nex-agent-plan-{run_id}",
daemon=True,
)
thread.start()
def _plan_stats(usage: dict | None) -> dict:
"""摊平计划调用的耗时与 token:用户问「LLM 怎么那么久」时要有数据可看。"""
usage = usage if isinstance(usage, dict) else {}
calls = [usage["first"], usage.get("repair") or {}] if isinstance(usage.get("first"), dict) else [usage]
stats = {"completion_tokens": 0, "prompt_tokens": 0, "reasoning_tokens": 0}
for call in calls:
raw = call.get("usage") if isinstance(call.get("usage"), dict) else call
if not isinstance(raw, dict):
continue
details = raw.get("completion_tokens_details") or {}
stats["completion_tokens"] += int(raw.get("completion_tokens") or 0)
stats["prompt_tokens"] += int(raw.get("prompt_tokens") or 0)
stats["reasoning_tokens"] += int((details or {}).get("reasoning_tokens") or 0)
stats = {key: value for key, value in stats.items() if value}
for key in ("thinking", "elapsed_seconds", "prompt_chars"):
if usage.get(key) is not None:
stats[key] = usage[key]
return stats
def run_dict(run: AgentRun) -> dict:
steps = list(run.steps_json or [])
return {
"id": run.id,
"project_id": run.project_id,
"status": run.status,
"intent": run.intent,
"request_text": run.request_text,
"reply": run.reply,
"summary": run.summary,
"provider": run.provider,
"model": run.model,
"auto_approve": run.auto_approve,
"cancel_requested": run.cancel_requested,
"timeout_seconds": run.timeout_seconds,
"steps": steps,
"blockers": list(run.blockers_json or []),
"warnings": list(run.warnings_json or []),
"usage": run.usage_json or {},
"plan_stats": _plan_stats(run.usage_json),
"error_code": run.error_code,
"error_message": run.error_message,
"can_execute": run.status in ("planned", "failed", "waiting_review")
and bool(steps),
# planning = 记录已落库、模型还在思考;页面据此把进度条留在「盘点项目」上
"is_planning": run.status == "planning",
"plan_stage": str((run.plan_json or {}).get("stage") or ""),
"created_at": run.created_at,
"updated_at": run.updated_at,
"started_at": run.started_at,
"finished_at": run.finished_at,
}
def list_runs(db: Session, project_id: int, *, limit: int = 20) -> list[AgentRun]:
return list(db.scalars(
select(AgentRun).where(AgentRun.project_id == project_id)
.order_by(AgentRun.id.desc()).limit(max(1, min(100, limit)))
).all())
def _task_failures(db: Session, task_ids: list[int]) -> list[str]:
if not task_ids:
return []
rows = db.scalars(select(Task).where(
Task.id.in_(task_ids), Task.status.in_((TaskStatus.FAILED, TaskStatus.CANCELLED))
).order_by(Task.id)).all()
return [f"任务 #{row.id} {row.kind}:{row.error_message or row.status}" for row in rows]
def _shot_filter(project: Project, shot_ids: list[int] | None, *, statuses: set[str] | None = None) -> list[Shot]:
wanted = set(shot_ids or [])
rows = []
for shot in sorted(project.shots or [], key=lambda row: (row.order_index, row.id)):
if wanted and shot.id not in wanted:
continue
if statuses is not None and shot.status not in statuses:
continue
rows.append(shot)
return rows
def _drain(db: Session, project: Project, timeout: float, *, limit: int = 64,
watch: list[int] | None = None) -> dict:
"""跑空队列并等回本步骤的产物。
watch = 本步骤自己入队的任务 id:只要它们还没结束,就不允许因为
「模型额度暂时派发不动」而提前收工——早退会让步骤误判「任务已结束但
产物缺失」,整条 LLM 调度因此半途而废。
"""
dispatched = dispatch.dispatch_due(db, limit=limit, project_id=project.id)
drained = daemon.drain_queue(db, project_id=project.id, timeout=timeout,
watch_task_ids=watch or [])
# 队列任务由独立会话落库:本会话缓存必须失效,否则回读的是执行前的旧值
db.expire_all()
if drained.get("timed_out"):
raise AgentStepError(
f"等待任务完成超时(仍有 {drained.get('remaining', 0)} 个任务未结束);可稍后再次确认执行",
code="agent_timeout",
)
if drained.get("stalled") and drained.get("watch_open"):
blocked = (drained.get("stalled_detail") or {}).get("blocked") or []
reasons = ";".join(f"{row.get('provider_model')}:{row.get('reason')}"
for row in blocked[:3]) or "模型额度暂时不可用"
raise AgentStepError(
f"本步骤的 {drained['watch_open']} 个任务迟迟派发不出去({reasons});"
"请在资源管理中恢复对应模型后重新下达指令",
code="agent_stalled",
)
return {"dispatched": dispatched, "drained": drained}
def _keyframe_canvas(project: Project) -> tuple[int, int]:
width, height = canvas_for(project.resolution, project.aspect_ratio)
scale = min(1.0, 1536.0 / max(width, height))
return (
max(256, int(width * scale) // 2 * 2),
max(256, int(height * scale) // 2 * 2),
)
def _compose_preset(project: Project) -> str:
vertical = project.aspect_ratio in ("9:16", "3:4")
if project.resolution in ("360p", "480p", "720p"):
return "vertical_720p" if vertical else "draft_720p"
return "vertical_1080p" if vertical else "web_1080p"
def _step_resume(db: Session, project: Project, step: dict, timeout: float) -> dict:
result = dispatch.resume_interrupted(db)
return {"status": "succeeded", "detail": f"恢复 {result.get('count', 0)} 个中断任务", "result": result}
def _reset_shot_outputs(db: Session, project: Project, shots: list[Shot]) -> int:
"""整组重写分镜后作废旧产物:清镜头产物/配音/质检/审核,并删除媒体库文件。
返回删除的媒体文件数。时间线不在这里重建——后续 build_timeline 会用
合格镜头排新版本,旧时间线自然停用,避免出现「新分镜配旧成片」。
"""
removed = 0
for shot in shots:
for path in (shot.output_path, shot.voiceover_path):
if storage.delete_media_file(path):
removed += 1
# 镜头级素材(关键帧/尾帧/参考图)随镜头一起作废,否则重排后会残留脏资产
for asset in list(storage.assets_of(db, "shot", shot.id)):
for version in list(asset.versions):
if storage.delete_media_file(version.path):
removed += 1
if storage.delete_media_file(asset.path):
removed += 1
db.delete(asset)
shot.output_path = None
shot.output_duration = None
shot.output_checksum = ""
shot.provider_job_id = ""
shot.generated_at = None
shot.voiceover_path = None
shot.voiceover_duration_ms = None
shot.qc_score = None
shot.qc_json = {}
shot.review_state = "pending"
shot.review_notes = ""
shot.error_code = ""
shot.error_message = ""
shot.status = ShotStatus.DRAFT
shot.stage = "rewrite_shots"
db.flush()
return removed
def _ensure_rewrite_assets(db: Session, project: Project, step: dict) -> dict:
"""为改写后的分镜补齐缺失的场景/人物。
换主题几乎一定要新场景(卧室 / 落地窗 / 海面),而 rewrite_shots 之外的动作不能碰资产,
所以这里按「分镜真的引用了才建」的原则创建最小设定,避免计划卡死在没有场景上。
"""
from app.models import Character, Scene
known_scenes = {row.name for row in (project.scenes or [])}
known_chars = {row.name for row in (project.characters or [])}
referenced_scenes = {str(row.get("scene") or "").strip() for row in (step.get("storyboard") or [])}
referenced_chars: set[str] = set()
for row in (step.get("storyboard") or []):
referenced_chars |= {str(name).strip() for name in (row.get("characters") or [])}
if str(row.get("speaker") or "").strip():
referenced_chars.add(str(row["speaker"]).strip())
made: dict[str, list[str]] = {"scenes": [], "characters": []}
for draft in step.get("new_scenes") or []:
name = str(draft.get("name") or "").strip()
if not name or name in known_scenes or name not in referenced_scenes:
continue
db.add(Scene(project_id=project.id, name=name, description=str(draft.get("description") or ""),
prompt=str(draft.get("prompt") or draft.get("description") or ""),
lighting=str(draft.get("lighting") or ""),
time_of_day=str(draft.get("time_of_day") or ""),
palette=str(draft.get("palette") or "")))
known_scenes.add(name)
made["scenes"].append(name)
for draft in step.get("new_characters") or []:
name = str(draft.get("name") or "").strip()
if not name or name in known_chars or name not in referenced_chars:
continue
db.add(Character(project_id=project.id, name=name,
role=str(draft.get("role") or "配角"),
appearance=str(draft.get("appearance") or ""),
persona_prompt=str(draft.get("persona_prompt") or "")))
known_chars.add(name)
made["characters"].append(name)
db.flush()
# 导入器靠 project.scenes / project.characters 关系匹配绑定,
# 不刷新这两个集合,新建的场景在同一个事务里对导入器不可见。
db.expire(project, ["scenes", "characters"])
return made
def _step_rewrite_shots(db: Session, project: Project, step: dict, run: AgentRun) -> dict:
"""按 LLM 提议的分镜整组改写脚本,复用导入器保证解析/绑定/提示词一致。"""
drafts = list(step.get("storyboard") or [])
if not drafts:
raise AgentStepError("计划里的 storyboard 为空,无法改写分镜")
lines: list[dict] = []
for idx, row in enumerate(drafts, start=1):
characters = [str(name).strip() for name in (row.get("characters") or []) if str(name).strip()]
lines.append({
"镜号": str(row.get("code") or f"S{idx:02d}"),
"标题": str(row.get("title") or "").strip(),
"画面描述": str(row.get("description") or "").strip(),
"时长": row.get("duration") or 5.0,
"景别": str(row.get("shot_size") or "").strip(),
"视角": str(row.get("camera_angle") or "").strip(),
"运镜": str(row.get("camera_movement") or "").strip(),
"台词": str(row.get("dialogue") or "").strip(),
"说话人": str(row.get("speaker") or "").strip(),
"出镜角色": "、".join(characters),
"场景": str(row.get("scene") or "").strip(),
"转场": str(row.get("transition") or "").strip(),
"风格": str(row.get("style_note") or "").strip(),
})
script = "\n".join(
"## 镜 %s\n%s" % (line["镜号"], "\n".join(
"- %s: %s" % (key, line[key]) for key in
("标题", "画面描述", "时长", "景别", "视角", "运镜", "台词", "说话人",
"出镜角色", "场景", "转场", "风格") if str(line[key]).strip()))
for line in lines
)
created_assets = _ensure_rewrite_assets(db, project, step)
previous = sorted(project.shots or [], key=lambda row: (row.order_index, row.id))
previous_ids = [shot.id for shot in previous]
removed_files = _reset_shot_outputs(db, project, previous)
# 旧时间线的片段指向已作废的镜头,不能再被 compose 当成「已完成的成片」复用
stale_timelines = [row for row in db.scalars(select(Timeline).where(
Timeline.project_id == project.id, Timeline.is_current.is_(True))).all()]
for stale in stale_timelines:
stale.is_current = False
result = importer.import_storyboard(
db, project, script_text=script, script_name="LLM 改写分镜",
replace=True, keep_existing=False, actor=f"agent:{run.provider}/{run.model}",
)
db.expire_all()
project = db.get(Project, project.id)
readiness = preflight.apply_status(db, project)
status.refresh(db, project, commit=False)
events.log_event(
db, "agent.storyboard_rewritten", project_id=project.id, actor=f"agent:{run.provider}/{run.model}",
message=f"LLM 调度按新主题改写分镜:{len(previous_ids)} → {result['created']} 个镜头,"
f"清理旧产物 {removed_files} 个文件",
entity_type="agent_run", entity_id=run.id,
data={"previous_shot_ids": previous_ids, "created": result["created"],
"removed_files": removed_files},
)
db.commit()
return {
"status": "succeeded",
"detail": f"分镜已改写为 {result['created']} 个镜头(原 {len(previous_ids)} 个镜头的产物已作废)",
"result": {"created": result["created"], "previous_shot_ids": previous_ids,
"created_assets": created_assets, "removed_files": removed_files,
"invalidated_timeline_ids": [row.id for row in stale_timelines],
"ready": readiness["ready"], "blocked": readiness["blocked"]},
}
def _identity_canvas(project: Project, kind: str) -> tuple[int, int]:
"""按素材用途决定画布:定妆图竖幅、三视图横幅、场景母图跟项目画幅。
全部 32 对齐(图像模型按 32 对齐算 latents),并且刻意小于关键帧尺寸:
定妆素材是一次性投入,画得越大整轮调度越慢,而参考图只需要「认得出是同一个人」。
"""
if kind == AssetKind.PORTRAIT:
return 768, 1024
if kind == AssetKind.TURNAROUND:
return 1280, 768
width, height = _keyframe_canvas(project)
scale = min(1.0, 1280.0 / max(width, height))
return (max(256, int(width * scale) // 32 * 32), max(256, int(height * scale) // 32 * 32))
def _referenced_identities(db: Session, project: Project, shot_ids: list[int] | None = None) -> dict:
"""分镜真正引用到的人物(含说话人)与场景;还没有分镜时退回全项目素材。"""
from app.models import Character
shots = _shot_filter(project, shot_ids)
characters: list = []
speakers: list = []
scenes: list = []
seen_chars: set[int] = set()
seen_speakers: set[int] = set()
seen_scenes: set[int] = set()
for shot in shots:
speaker = shot.speaker
# 说话人必须单独去重:俊杰在镜头 1 只是站着不出声、镜头 2 才开口,
# 跟 characters 共用一个集合会让他整个逃过声线自检,配音静默回落成陌生嗓子。
if speaker is not None and str(shot.dialogue or "").strip() and speaker.id not in seen_speakers:
seen_speakers.add(speaker.id)
speakers.append(speaker)
if speaker.id not in seen_chars:
seen_chars.add(speaker.id)
characters.append(speaker)
for char in list(shot.characters or []):
if char.id not in seen_chars:
seen_chars.add(char.id)
characters.append(char)
for scene in list(shot.scenes or []):
if scene.id not in seen_scenes:
seen_scenes.add(scene.id)
scenes.append(scene)
if not characters and not scenes:
characters = [row for row in (project.characters or []) if row.is_active]
if not characters:
characters = list(db.scalars(select(Character).where(
Character.project_id == project.id)).all())
speakers = list(characters)
scenes = list(project.scenes or [])
return {"characters": characters, "speakers": speakers, "scenes": scenes}
def voice_gap_reason(db: Session, char, gender: str, age_band: str, voice) -> str:
"""说话人的声线还差什么;返回空字符串表示这条声线可以放心用。
缺口判定和修复必须共用这份判据,否则「已经绑了一条错嗓子」的角色会被自检当成
「声线齐备」直接放过——25 岁女主配老年女声就是这么漏掉的。
"""
from app.services import audio
if voice is None:
return "说话角色没有绑定声线"
voice_gender = str(getattr(voice, "gender", "") or "")
if gender and voice_gender and gender != voice_gender:
return "声线与说话人性别不符"
if age_band:
# 性别对了但年龄跨两档以上同样是错配音;差一档不动,避免误伤用户的手选
distance = audio.age_band_distance(age_band, audio.voice_age_band(voice))
if distance >= 2:
return "声线年龄与说话人明显不符"
if not audio.voice_reference_ready(db, voice):
return "声线缺少参考音频"
if audio.voice_reference_outdated(db, voice):
return "音色素材换过了,克隆参考还停在旧文件上"
return ""
# 「换一条声线」和「把这条声线修好」是两种处理:后者只需要重新对齐参考音频,不能抢走用户的手选
VOICE_REPAIR_ONLY_REASONS = frozenset({"声线缺少参考音频", "音色素材换过了,克隆参考还停在旧文件上"})
def _identity_gaps(db: Session, project: Project, shot_ids: list[int] | None = None) -> dict:
"""分镜要用、但还没有「身份」的素材。
这是「素材库里只有占位、没有人物图片」的正面回答:分镜引用了角色不等于模型认识这个角色,
没有定妆图/场景母图时关键帧只能凭文字描述出图,于是同一角色在相邻镜头里换了脸。
"""
from app.services import audio
ref = _referenced_identities(db, project, shot_ids)
audio.backfill_gender(db, project)
portraits = [row for row in ref["characters"]
if not storage.first_asset_path(db, "human", row.id, AssetKind.PORTRAIT)]
turnarounds = [row for row in ref["characters"]
if not storage.first_asset_path(db, "human", row.id, AssetKind.TURNAROUND)]
scene_images = [row for row in ref["scenes"]
if not storage.first_asset_path(db, "scene", row.id, AssetKind.SCENE_IMAGE)]
voice_gaps: list[dict] = []
for char in ref["speakers"]:
gender = audio.character_gender(db, char)
age_band = audio.character_age_band(db, char)
voice = None
if char.default_voice_id:
from app.models import VoiceAsset
voice = db.get(VoiceAsset, char.default_voice_id)
reason = voice_gap_reason(db, char, gender, age_band, voice)
if not reason:
continue
voice_gaps.append({
"character": char, "gender": gender, "age_band": age_band, "voice": voice,
"reason": reason,
})
return {**ref, "portraits": portraits, "turnarounds": turnarounds,
"scene_images": scene_images, "voice_gaps": voice_gaps}
def _has_identity_gaps(db: Session, project: Project, shot_ids: list[int] | None = None) -> bool:
gaps = _identity_gaps(db, project, shot_ids)
return bool(gaps["portraits"] or gaps["turnarounds"] or gaps["scene_images"] or gaps["voice_gaps"])
def _platform_voice_audio(db, voice) -> str:
"""平台声线母版上真正可用的音色文件(克隆参考音频或试听样本)。
IndexTTS2 的克隆参考本来就是一段人声样本,所以「没有 reference 素材」不等于
「没有音色」:平台母版常常只带试听样本,把它排除掉会让同名角色退回陌生人的嗓子。
"""
from app.constants import AssetKind as _Kind
from app.models import PlatformMedia
if str(voice.provider_voice_id or "").strip() and storage.exists(voice.provider_voice_id):
return str(voice.provider_voice_id)
rows = db.query(PlatformMedia).filter(
PlatformMedia.owner_type == "voice", PlatformMedia.owner_id == voice.id,
PlatformMedia.kind.in_((_Kind.REFERENCE, _Kind.VOICE_SAMPLE))).all()
files = [row.path for row in rows if storage.exists(row.path)]
return files[-1] if files else ""
def _platform_voice_match(db: Session, char, gender: str, age_band: str = ""):
"""平台声线母版里最贴合这个角色的那一条(必须有真实可用的音色文件)。"""
from app.models import PlatformCharacter, PlatformVoice
rows = list(db.scalars(select(PlatformVoice).where(
PlatformVoice.is_active.is_(True)).order_by(PlatformVoice.id)).all())
usable = [row for row in rows if _platform_voice_audio(db, row)]
named = [row for row in usable if str(row.name or "").startswith(str(char.name or ""))]
if named:
return named[0]
source = db.scalars(select(PlatformCharacter).where(
PlatformCharacter.name == char.name,
PlatformCharacter.default_voice_id.is_not(None))).first()
if source is not None:
linked = db.get(PlatformVoice, source.default_voice_id)
if linked is not None and _platform_voice_audio(db, linked):
return linked
# 同性别里按年龄档就近挑选,避免 25 岁女主拿到「偏低的中国老年女声」
same = [row for row in usable if not gender or str(row.gender or "") == gender]
if age_band:
def _gap(row) -> int:
distance = audio.age_band_distance(age_band, audio.voice_age_band(row))
return 99 if distance < 0 else distance
same.sort(key=lambda row: (_gap(row), row.id))
return same[0] if same else None
def _fix_voice_identities(db: Session, project: Project, gaps: list[dict]) -> dict:
"""让每个说话人都有能决定音色的声线,且只补空、不覆盖用户已经绑好的设置。
三条回落路径,代价从低到高:
1. 声线本身没错、只是克隆参考落后了 → 始终把它对齐到该声线当前最新的
参考/试听素材(用户换了音色却听不出变化、IndexTTS2 静默回落到通用音色,都出在这里);
2. 角色压根没绑声线,或绑的声线性别/年龄档明显不符 → 用 ``audio.rank_voices``
在项目内打分挑一条(性别 > 年龄档 > 姓名归属 > 有无可用音色 > 是否已被他人占用);
3. 项目里没有可用声线 → 从平台声线库引入一条真正带音色文件的(同名优先,
同性别里按年龄档就近),再绑到角色上。只有在明显不匹配时才改写 ``default_voice_id``。
"""
from app.services import audio
if not gaps:
return {"bound": [], "filled": [], "repairs": [], "imported": [], "unresolved": []}
from app.models import VoiceAsset
pool = list(db.scalars(select(VoiceAsset).where(VoiceAsset.project_id == project.id)
.order_by(VoiceAsset.id)).all())
bound: list[str] = []
filled: list[str] = []
repairs: list[str] = []
imported: list[str] = []
unresolved: list[str] = []
for gap in gaps:
char = gap["character"]
voice = gap["voice"]
gender = str(gap.get("gender") or "")
age_band = str(gap.get("age_band") or "") or audio.character_age_band(db, char)
reason = voice_gap_reason(db, char, gender, age_band, voice)
mismatch = bool(reason) and reason not in VOICE_REPAIR_ONLY_REASONS
gap["reason"] = reason or gap.get("reason") or ""
if voice is not None and not mismatch:
# 用户换过音色素材却听不出变化:克隆参考始终对齐到该声线当前最新的素材
change = audio.repair_voice_reference(db, voice, reason="分镜自检自愈")
if change:
filled.append(voice.name)
repairs.append(change)
if audio.voice_reference_ready(db, voice):
continue
# 需要换一个真正匹配的声线:先看项目内,再看平台母版
ranked = [item for item in audio.rank_voices(
db, project, gender=gender, age_band=age_band,
speaker_name=str(char.name or ""), pool=pool)
if item["ready"] and (voice is None or item["voice"].id != voice.id)
and item["score"] > 0]
picked = ranked[0]["voice"] if ranked else None
if picked is None:
platform = _platform_voice_match(db, char, gender, age_band)
if platform is not None:
from app.api.platform import _import_voice
try:
picked = _import_voice(db, project, platform.id)
imported.append(f"{char.name} ← 平台声线「{platform.name}」")
db.flush()
pool.append(picked)
except Exception: # noqa: BLE001 - 引入失败交给预检报错
picked = None
if picked is None:
unresolved.append(f"{char.name}({gap.get('reason') or '声线不可用'})")
continue
if char.default_voice_id != picked.id:
char.default_voice_id = picked.id
bound.append(f"{char.name} → {picked.name}")
result = {"bound": bound, "filled": filled, "repairs": repairs,
"imported": imported, "unresolved": unresolved}
if any(result[key] for key in ("bound", "filled", "repairs", "imported")):
events.log_event(
db, "agent.voice_identity_fixed", project_id=project.id,
message="补齐说话人声线:" + ";".join(
[f"绑定 {item}" for item in bound]
+ [f"补回参考音频 {item}" for item in filled]
+ repairs + imported)[:500],
entity_type="project", actor="agent", data=result,
)
db.flush()
return result
def _queue_identity_image(db: Session, project: Project, spec, *, owner_type: str, owner_id: int,
kind: str, name: str, reference_paths: list[str],
reference_asset_ids: list[int] | None = None) -> Task:
"""排一个资产生成任务,payload 与素材库「AI 生成」按钮走的是同一条通道。"""
width, height = _identity_canvas(project, kind)
task = Task(
project_id=project.id, kind=TaskKind.ASSET, name=f"定妆素材:{name}", stage="asset",
status=TaskStatus.QUEUED, priority=3, provider=spec.provider, model=spec.model,
mode="image", max_attempts=2,
idempotency_key=f"agent-identity:{owner_type}:{owner_id}:{kind}:{int(now_utc().timestamp())}",
payload_json={
"operation": "generate_image", "target_type": owner_type, "target_id": owner_id,
"kind": kind, "name": name, "prompt": "",
"negative_prompt": project.global_negative_prompt or "",
"width": width, "height": height,
"reference_asset_ids": reference_asset_ids or [],
"reference_paths": reference_paths,
},
cost_currency=spec.currency, queued_at=now_utc(),
)
db.add(task)
db.flush()
return task
def _step_generate_identity_assets(db: Session, project: Project, step: dict, timeout: float) -> dict:
"""补缺口里的身份素材:定妆图/三视图/场景母图 + 可用声线。
分两阶段出图,这是人物一致性的关键:先出定妆图和场景母图,再把刚出炉的定妆图
当参考图去出三视图。一次全排下去的话,三视图与定妆图各画各的,
后面按优先级取参考图时等于随机挑一张脸。
"""
gaps = _identity_gaps(db, project, step.get("shot_ids") or None)
voices = _fix_voice_identities(db, project, gaps["voice_gaps"])
if voices.get("imported") or voices.get("bound"):
db.expire_all()
project = db.get(Project, project.id)
pending_images = bool(gaps["portraits"] or gaps["turnarounds"] or gaps["scene_images"])
if not pending_images:
repaired = voices["bound"] + voices["filled"] + voices["repairs"] + voices["imported"]
detail = "人物、场景与声线素材已齐备" if not repaired else \
"素材齐备,并已修正说话人声线:" + ";".join(repaired)[:300]
if not any(voices[key] for key in ("bound", "filled", "repairs", "imported", "unresolved")):
return {"status": "skipped", "detail": detail}
db.commit()
return {"status": "succeeded", "detail": detail, "result": {"voices": voices}}
try:
spec, _adapter = _resolve_real_media(db, "image")
except AgentStepError as exc:
db.commit()
return {"status": "skipped",
"detail": f"缺人物/场景定妆素材,但图像模型不可用({exc});关键帧将退回文字描述",
"result": {"voices": voices,
"missing": [row.name for row in gaps["portraits"] + gaps["turnarounds"]]
+ [row.name for row in gaps["scene_images"]]}}
caps = model_registry.capabilities_of(spec)
max_refs = max(0, int(caps.max_reference_images or 0))
warnings: list[str] = []
tasks: list[Task] = []
# 阶段一:定妆图 + 场景母图(互不依赖,先把「身份」立起来)
for char in gaps["portraits"]:
tasks.append(_queue_identity_image(
db, project, spec, owner_type="human", owner_id=char.id, kind=AssetKind.PORTRAIT,
name=f"{char.name} · AI 定妆图", reference_paths=[]))
for scene in gaps["scene_images"]:
tasks.append(_queue_identity_image(
db, project, spec, owner_type="scene", owner_id=scene.id, kind=AssetKind.SCENE_IMAGE,
name=f"{scene.name} · AI 场景母图", reference_paths=[]))
events.log_event(
db, "agent.identity_queued", project_id=project.id,
message=f"LLM 调度补拍定妆素材:{len(gaps['portraits'])} 张定妆图、"
f"{len(gaps['scene_images'])} 张场景母图({spec.provider}/{spec.model})",
entity_type="project", actor="agent",
data={"characters": [row.name for row in gaps["portraits"]],
"scenes": [row.name for row in gaps["scene_images"]]},
)
db.commit()
first_ids = [task.id for task in tasks]
if first_ids:
_drain(db, project, timeout, limit=max(8, len(first_ids)), watch=first_ids)
failures = _task_failures(db, first_ids)
if failures:
raise AgentStepError("定妆素材生成失败:" + ";".join(failures[:3]))
db.expire_all()
project = db.get(Project, project.id)
# 阶段二:三视图,用刚生成的定妆图做参考,确保多视角是同一个人
gaps = _identity_gaps(db, project, step.get("shot_ids") or None)
tasks = []
for char in gaps["turnarounds"]:
refs: list[str] = []
ref_ids: list[int] = []
if max_refs:
portrait = storage.first_asset_row(db, "human", char.id, AssetKind.PORTRAIT)
if portrait is not None and storage.exists(portrait.path):
refs = [str(storage.to_absolute(portrait.path))][:max_refs]
ref_ids = [portrait.id]
tasks.append(_queue_identity_image(
db, project, spec, owner_type="human", owner_id=char.id, kind=AssetKind.TURNAROUND,
name=f"{char.name} · AI 三视图", reference_paths=refs, reference_asset_ids=ref_ids))
db.commit()
second_ids = [task.id for task in tasks]
if second_ids:
_drain(db, project, timeout, limit=max(8, len(second_ids)), watch=second_ids)
failures = _task_failures(db, second_ids)
if failures:
# 三视图缺失不阻断整轮调度:定妆图已经够关键帧认脸,卡在这里代价更大
warnings.append("三视图生成失败:" + ";".join(failures[:3]))
db.expire_all()
project = db.get(Project, project.id)
still_missing = _identity_gaps(db, project, step.get("shot_ids") or None)
missing_rows = ([f"{row.name}(定妆图)" for row in still_missing["portraits"]]
+ [f"{row.name}(三视图)" for row in still_missing["turnarounds"]]
+ [f"{row.name}(场景母图)" for row in still_missing["scene_images"]])
if missing_rows:
warnings.append("仍未补齐:" + "、".join(missing_rows[:8]))
if still_missing["voice_gaps"]:
warnings.append("声线仍不稳定:" + "、".join(
f"{gap['character'].name}({gap['reason']})" for gap in still_missing["voice_gaps"])[:200])
made = len(first_ids) + len(second_ids)
detail = (f"定妆素材已补齐:{len(first_ids)} 张定妆图/场景母图 + "
f"{len(second_ids)} 张三视图,共 {made} 个图像资产")
repaired = voices["bound"] + voices["filled"] + voices["repairs"] + voices["imported"]
if repaired:
detail += ";声线已修正 " + "、".join(repaired[:4])
if warnings:
detail += ";" + ";".join(warnings)[:300]
return {
"status": "succeeded", "detail": detail,
"result": {"task_ids": first_ids + second_ids, "voices": voices, "warnings": warnings},
}
def _auto_identity_assets_before_keyframes(db: Session, project: Project,
shot_ids: list[int] | None, timeout: float) -> dict:
"""出关键帧前先补身份素材:计划里忘了排也要补,否则一致性只能靠运气。"""
if not _has_identity_gaps(db, project, shot_ids):
return {}
return _step_generate_identity_assets(db, project, {"shot_ids": shot_ids or []}, timeout)
def _shot_needs_keyframe(db: Session, shot: Shot, *, redo: bool = False) -> bool:
"""这个镜头该(重)画关键帧吗。
重做时只重画 AI 生成的关键帧:编导自己上传的首帧属于原始素材,调度器无权覆盖。
重画走同名素材的版本递增,视频阶段仍读同一素材行,因此新首帧一定会生效。
"""
if shot.mode not in ("i2v", "fl"):
return False
asset = storage.first_asset(db, "shot", shot.id, AssetKind.KEYFRAME)
if asset is None:
return True
if not (asset.metadata_json or {}).get("generated"):
return False # 编导自己传的首帧是原始素材,调度器无权覆盖
if redo:
return True
return keyframe_is_stale(db, shot, asset)
def identity_freshness(db: Session, asset_ids: list[int]) -> datetime | None:
"""身份素材(定妆图/三视图/场景母图)最近一次真正换图的时间。
只看 `assets.updated_at` 是不够的:「把某个版本设为当前版本」是版本管理功能,
它只翻 `current_version` 指针、不会碰父行的 updated_at。所以这里取
「父行更新时间」与「所有版本行创建时间」的最大值。
"""
latest: datetime | None = None
for row in db.scalars(select(Asset).where(Asset.id.in_(asset_ids or [0]))):
for moment in [row.updated_at, row.created_at] + [v.created_at for v in (row.versions or [])]:
stamp = storage.as_utc(moment)
if stamp is not None and (latest is None or stamp > latest):
latest = stamp
return latest
def keyframe_is_stale(db: Session, shot: Shot, asset) -> bool:
"""首帧比身份素材旧 = 过期:用户重传的定妆照根本没机会进入画面。
项目 9 实测:三镜的关键帧是 10-27 画的,俊杰的定妆照 10-28 才导入,
`generate_keyframes` 却以「没有缺少关键帧」为由整步跳过,i2v 的第一帧
于是还是旧形象 + 一个凭空气多出来的背景同事;视频返工多少轮都改不掉
首帧里的人,自动质检就永远判不合格。
"""
details = prompt_service.collect_reference_details(db, shot)
refs = [int(row["asset_id"]) for row in details if row.get("asset_id")]
fresh = identity_freshness(db, refs)
made = storage.as_utc(asset.updated_at or asset.created_at)
return bool(fresh and made and fresh > made)
def _step_generate_keyframes(db: Session, project: Project, step: dict, timeout: float) -> dict:
# 先补身份素材再画关键帧:缺定妆图/场景母图时,关键帧只能凭文字描述出图,
# 于是同一角色在相邻镜头里换了脸。计划里忘了排这一步,这里也会自动补。
redo = bool((step.get("regenerate") or {}).get("keyframes"))
auto_identity = _auto_identity_assets_before_keyframes(
db, project, step.get("shot_ids") or None, timeout)
shots = [
shot for shot in _shot_filter(project, step.get("shot_ids"))
if _shot_needs_keyframe(db, shot, redo=redo)
]
if not shots:
return {"status": "skipped",
"detail": "没有需要重画关键帧的图生视频镜头" if redo
else "没有缺少关键帧的图生视频镜头"}
spec, _adapter = _resolve_real_media(db, "image")
caps = model_registry.capabilities_of(spec)
width, height = _keyframe_canvas(project)
tasks: list[Task] = []
for shot in shots:
ref_ids, ref_paths = _reference_images(db, project, shot, caps.max_reference_images)
task = Task(
project_id=project.id,
kind=TaskKind.ASSET,
name=f"镜头 {shot.order_index:02d} AI 关键帧",
stage="asset",
status=TaskStatus.QUEUED,
priority=3,
provider=spec.provider,
model=spec.model,
mode="image",
max_attempts=2,
idempotency_key=f"agent-keyframe:{shot.id}:{int(now_utc().timestamp())}",
payload_json={
"operation": "generate_image",
"target_type": "shot",
"target_id": shot.id,
"kind": AssetKind.KEYFRAME,
"name": f"镜头 {shot.order_index:02d} · AI 关键帧",
"prompt": "",
"negative_prompt": project.global_negative_prompt or "",
"width": width,
"height": height,
"reference_asset_ids": ref_ids,
"reference_paths": ref_paths,
},
cost_currency=spec.currency,
queued_at=now_utc(),
)
db.add(task)
db.flush()
tasks.append(task)
events.log_event(
db, "agent.keyframes_queued", project_id=project.id,
message=f"LLM 调度为 {len(tasks)} 个镜头补充关键帧({spec.provider}/{spec.model})",
entity_type="agent_run", actor="agent",
data={"shot_ids": [shot.id for shot in shots]},
)
db.commit()
outcome = _drain(db, project, timeout, limit=max(8, len(tasks)),
watch=[task.id for task in tasks])
failures = _task_failures(db, [task.id for task in tasks])
if failures:
raise AgentStepError("关键帧生成失败:" + ";".join(failures[:3]))
missing = [
shot for shot in shots
if not storage.first_asset_path(db, "shot", shot.id, AssetKind.KEYFRAME)
]
if missing:
raise AgentStepError(
"关键帧任务已结束,但以下镜头仍没有关键帧:"
+ "、".join(f"#{shot.order_index}" for shot in missing)
)
detail = f"已为 {len(tasks)} 个镜头生成关键帧"
if auto_identity.get("detail"):
detail = "先" + str(auto_identity["detail"]) + ";" + detail
return {
"status": "succeeded",
"detail": detail,
"result": {"task_ids": [task.id for task in tasks], "identity_assets": auto_identity,
**outcome},
}
def _auto_keyframes_before_generate(db: Session, project: Project, shot_ids: list[int] | None,
timeout: float) -> int:
"""图生视频镜头缺关键帧时先补图,避免整步被预检挡住。
换主题后分镜是新建的,一定没有关键帧;此时若计划里没有显式的
generate_keyframes 步骤,_step_generate_shots 会因为「素材未就绪」一个任务都不入队,
表现出来就是「调度完成了但什么都没生成」。缺图像模型时不阻断,交给预检报错。
"""
missing = [
shot for shot in _shot_filter(project, shot_ids)
if _shot_needs_keyframe(db, shot)
]
if not missing:
return 0
try:
_resolve_real_media(db, "image")
except AgentStepError:
return 0
outcome = _step_generate_keyframes(db, project, {"shot_ids": shot_ids}, timeout)
if outcome.get("status") == "skipped":
return 0
db.expire_all()
return len(missing)
def _inflight_generation_tasks(db: Session, project: Project,
shot_ids: list[int] | None) -> list[int]:
"""这批镜头当前仍在排队/执行的画面与配音任务 id。"""
wanted = set(shot_ids or [])
rows = db.scalars(select(Task).where(
Task.project_id == project.id,
Task.kind.in_((TaskKind.GENERATE_SHOT, TaskKind.VOICEOVER)),
Task.status.in_(TaskStatus.ACTIVE),
).order_by(Task.id)).all()
return [row.id for row in rows
if row.shot_id is not None and (not wanted or row.shot_id in wanted)]
def _step_generate_shots(db: Session, project: Project, step: dict, timeout: float) -> dict:
shot_ids = step.get("shot_ids") or None
scope = step.get("regenerate") or {}
redo_video, redo_voice = bool(scope.get("video")), bool(scope.get("voice"))
auto_keyframes = _auto_keyframes_before_generate(db, project, shot_ids, timeout)
plan = dispatch.plan_task_list(db, project, shot_ids=shot_ids, force=redo_video,
include_voiceover=True)
if redo_voice:
# 清单按「已有配音产物就跳过」剪辑配音项;换了音色后必须重录,这里强制翻回排队。
for item in plan.get("voiceovers", []):
if item.get("action") == "skip":
item["action"] = "enqueue"
item["reason"] = "编导要求用资产库里当前音色重新配音"
plan["voiceover_count"] = sum(
1 for item in plan.get("voiceovers", []) if item.get("action") == "enqueue")
if not plan.get("queued_count") and not plan.get("voiceover_count"):
blocked = ["镜头 %s:%s" % (item["order_index"], ";".join(item.get("blocking") or []))
for item in plan.get("shots", []) if item.get("action") == "needs_attention"]
inflight = _inflight_generation_tasks(db, project, shot_ids)
if inflight and not blocked:
# 清单里没有可排队的新任务,**不等于**这批镜头已经做完了:
# 用户自己点过「生成」、或上一轮调度留下的任务还停在队列/正在出图。
# 以前这一步立刻「跳过」收工,于是后面的质检、审核、时间线拿着
# 还没出图的镜头继续往下跑,最后合出一片少镜头的成片,
# 而用户看到的是「指令跑完了,画面却缺镜」。
# 现在:等这批在途任务自己结束(不重复提交、不重复烧 GPU)。
outcome = _drain(db, project, timeout, limit=max(16, len(inflight)), watch=inflight)
failures = _task_failures(db, inflight)
if failures:
raise AgentStepError("在途镜头/配音任务失败:" + ";".join(failures[:4]))
db.expire_all()
return {
"status": "succeeded",
"detail": f"队列中已有 {len(inflight)} 个镜头/配音任务在执行(#{inflight[0]} 起),"
"已等待其完成,未重复提交生成",
"result": {"watched_task_ids": inflight, "already_queued": True, **outcome},
}
detail = "没有待生成的镜头或配音(均已生成并审核合格,断点续跑会跳过)"
if redo_video or redo_voice:
detail = "已按重做指令排队,但清单里仍没有可执行项:" + detail
if blocked:
# 以前这里只说「没有待生成的镜头」,用户完全看不出其实是被预检挡住了
detail = ("镜头素材或参数未就绪,没有创建任何生成任务 → " + " | ".join(blocked[:4])
+ ";请先补齐关键帧/参考素材或调整模型参数")
return {"status": "skipped", "detail": detail, "result": {"plan": plan}}
created = dispatch.create_generation_tasks(
db, project, plan, shot_ids=shot_ids, force=redo_video, priority=4,
actor="agent", with_voiceover=True,
)
task_ids = [row["task_id"] for row in created.get("created", [])] + [
row["task_id"] for row in created.get("voiceovers", [])
]
if not task_ids:
return {"status": "skipped", "detail": "任务清单中没有需要入队的项目", "result": created}
db.commit()
outcome = _drain(db, project, timeout, limit=max(16, len(task_ids)), watch=task_ids)
failures = _task_failures(db, task_ids)
if failures:
raise AgentStepError("镜头/配音生成失败:" + ";".join(failures[:4]))
db.expire_all()
project = db.get(Project, project.id)
failed_shots = [
shot for shot in _shot_filter(project, shot_ids, statuses={ShotStatus.FAILED})
]
if failed_shots:
raise AgentStepError(
"以下镜头生成失败:"
+ "、".join(f"#{shot.order_index} {shot.error_message[:80]}" for shot in failed_shots[:5])
)
detail = (f"已生成 {len(created.get('created', []))} 个镜头、"
f"{len(created.get('voiceovers', []))} 条配音")
if auto_keyframes:
detail = f"先为 {auto_keyframes} 个缺关键帧的镜头补图,再" + detail
return {
"status": "succeeded",
"detail": detail,
"result": {"created": created, "task_ids": task_ids,
"auto_keyframes": auto_keyframes, **outcome},
}
QC_REWORK_MARK = "【返工要求"
# 质检不合格时默认自动返工重生成几轮:图形质检上线后「明显镜头瑕疵」都会被判不合格,
# 若一不合格就把整条计划停掉,LLM 总控台永远跑不到 compose,用户看到的是「项目卡住」。
# 2 轮是实测出来的下限:机位跳变、背景分身这类瑕疵出图有运气成分,只给 1 轮时
# 项目 9 三镜里有两镜「再试一次就好了」却在第 1 轮后交回人工。
QC_REWORK_ROUNDS = 2
def _plan_has_action(run: AgentRun, action: str) -> bool:
"""这条计划里还有没跑完的某个动作吗(决定质检能不能软放行)。"""
return any(row.get("action") == action and row.get("status") not in ("succeeded", "skipped")
for row in (run.steps_json or []))
def qc_issue_brief(shot: Shot, limit: int = 2) -> str:
return ";".join(_qc_hard_issues(shot, limit)) or "画面与人物设定不符"
#: 画面文件本身解不开(连时长都探测不出来):这种素材切进成片只会让导出
#: 直接失败,不属于「有瑕疵但能用」,不能软放行。
UNDECODABLE_MARKS = ("媒体探测", "产物文件", "ffprobe", "无法解析", "解码失败")
def _footage_undecodable(shot: Shot) -> bool:
return any(mark in " ".join(_qc_hard_issues(shot, 8)) for mark in UNDECODABLE_MARKS)
def _qc_hard_issues(shot: Shot, limit: int = 4) -> list[str]:
qc = shot.qc_json or {}
bits = [str(item).strip() for item in (qc.get("issues") or [])]
bits = [item for item in bits if item]
if not bits and (shot.error_message or "").strip():
bits = [shot.error_message.strip()]
return bits[:limit]
def _write_qc_rework_notes(shots: list[Shot], *, round_no: int) -> list[str]:
"""把上一轮质检的硬伤写回镜头,下一轮生成时模型才知道自己上次哪里穿帮。
style_note 会被 prompt.compose_prompt 现算进视频提示词(不读 shot.prompt 缓存),
所以这里只保留一条带标记的返工段落:反复返工不会越堆越长,也不会污染编导原文案。
"""
notes = []
for shot in shots:
ask = ";".join(_qc_hard_issues(shot)) or "画面与人物设定不符"
# 「只允许出现设定人物」这句曾把返工带进死胡同:画面描述里点名的配角
# 没挂进本镜出镜名单,模型被这句话直接删掉,戏就拍不下去了。
note = (f"{QC_REWORK_MARK}第{round_no}轮】上一轮自动质检判定不合格,必须修正:{ask}。"
"画面描述里点名的人物必须保留,除了出镜名单与画面描述里的人之外不得再多出旁人;"
"同一人物绝不能出现第二个分身;"
"每个人物的五官、发型、服装、配饰必须严格以上传的定妆图/三视图为准,"
"定妆照与人物设定卡上的文字描述冲突时一律以定妆照为准,"
"不得为了迎合旧文字改回别的衣着;逐件一致,不得互相借用、不得自行改写衣着;"
"同一镜头内机位不得无交代跳切,画面里的文字要能辨认。")
base = shot.style_note or ""
cut = base.find(QC_REWORK_MARK)
if cut >= 0:
base = base[:cut]
shot.style_note = f"{base.strip()[:600]} {note}".strip()
notes.append(f"#{shot.order_index:02d} {ask[:70]}")
return notes
def _clear_qc_rework_notes(shots: list[Shot]) -> None:
"""质检通过后抹掉返工段落:那些硬伤已经修好了,留着会污染以后每一次重新生成。"""
for shot in shots:
note = shot.style_note or ""
cut = note.find(QC_REWORK_MARK)
if cut < 0:
continue
shot.style_note = note[:cut].strip()
def qc_verdict_is_current(shot: Shot) -> bool:
"""这一版画面已经质检合格过,且产物没被换掉,结论仍然有效。
看图判定是有随机性的:同一支片子先后被判过 100 分和 94 分(项目 9 镜头 03 实测),
而复判一次就把已经合格的镜头推回返工循环——白烧两趟四分钟的出图,
最后往往以「自动返工 2 轮仍未通过」把整条 LLM 调度判失败,
用户看到的就是「指令下了,项目却卡住不动」。
产物指纹没变且上一轮合格时沿用结论,只在画面真的换了版本时才重新看图。
"""
qc = shot.qc_json or {}
if not bool(qc.get("passed")):
return False
from app.services.quality import QC_RULESET
# 画面没换,但**质检规则换了**:旧结论是拿旧尺子量的,不能沿用。
if int(qc.get("qc_ruleset") or 0) != int(QC_RULESET):
return False
if "artifact_checksum" not in qc:
# 老数据没落产物指纹:退化成按画面文件路径比对(重出画面必然换文件名)
probe_path = str((qc.get("probe") or {}).get("path") or "")
return bool(shot.output_path) and probe_path.endswith(str(shot.output_path))
return bool(shot.output_checksum) and str(qc["artifact_checksum"]) == str(shot.output_checksum)
def _qc_pass(db: Session, project: Project, shots: list[Shot], timeout: float, *,
reuse_passed: bool = True) -> tuple[Project, list[int], list[Shot], dict]:
"""给这批镜头排一次自动质检并等它跑完,带回不合格的镜头。
reuse_passed=True 时,产物未变且上一轮已合格的镜头不再重复看图(见上)。
"""
pending: list[Shot] = []
reused: list[Shot] = []
for shot in shots:
(reused if reuse_passed and qc_verdict_is_current(shot) else pending).append(shot)
if reused:
events.log_event(
db, "qc.reused", project_id=project.id,
message="画面未变化,沿用上一轮自动质检合格结论:" + "、".join(
f"#{row.order_index:02d}({row.qc_score:.0f} 分)" for row in reused[:6]),
entity_type="project", entity_id=project.id,
data={"shot_ids": [row.id for row in reused]},
)
db.commit()
task_ids = []
for shot in pending:
active = db.scalars(select(Task).where(
Task.project_id == project.id,
Task.shot_id == shot.id,
Task.kind == TaskKind.QC_SHOT,
Task.status.in_(TaskStatus.ACTIVE),
)).first()
task = active or dispatch.new_qc_task(db, project, shot, priority=3)
task_ids.append(task.id)
db.commit()
if not task_ids:
# 全员沿用旧结论:一个任务都不用排,更不用空等一轮队列
return project, [], [row for row in shots if not bool((row.qc_json or {}).get("passed"))], {
"dispatched": {"dispatched": [], "count": 0, "blocked": []},
"drained": {"timed_out": False, "remaining": 0, "reused_only": True},
"reused_shot_ids": [row.id for row in reused],
}
outcome = _drain(db, project, timeout, limit=max(8, len(task_ids)), watch=task_ids)
failures = _task_failures(db, task_ids)
if failures:
raise AgentStepError("自动质检任务失败:" + ";".join(failures[:3]))
db.expire_all()
project = db.get(Project, project.id)
failed = [row for row in _shot_filter(project, [shot.id for shot in shots])
if not bool((row.qc_json or {}).get("passed"))]
return project, task_ids, failed, outcome
def _step_quality_check(db: Session, project: Project, step: dict, timeout: float,
run: AgentRun | None = None) -> dict:
# run 只有按计划执行时才有;直接调用(自检/人工复查)时为空,
# 这时返工耗尽仍旧硬停,保持原语义。
# 上一步刚刚排空过队列,镜头状态是执行线程改的:这里必须重新读一遍,
# 否则会用执行前的旧状态判断「哪些镜头还要质检」,把已经合格的镜头再返工一轮。
db.expire_all()
project = db.get(Project, project.id)
scoped = _shot_filter(project, step.get("shot_ids"))
missing = [shot for shot in scoped if not storage.exists(shot.output_path)]
if missing:
# 质检这一步的语义是「这一批镜头全过一遍」。以前它只挑有产物的镜头,
# 缺图的镜头被无声跳过,时间线于是只收了「已经合格的那几镜」——
# 成片少镜头,却没有任何一步报错。
raise AgentStepError(
f"{len(missing)} 个镜头还没有画面产物,无法整批质检:"
+ "、".join(f"#{shot.order_index}" for shot in missing[:8])
+ ";请先执行镜头生成(generate_shots)并等待其完成"
"(若画面文件是被外部删掉的,请让编导明确「重做画面」后重跑本步)",
)
target_shots = [shot for shot in scoped if storage.exists(shot.output_path)]
statuses = {ShotStatus.GENERATED, ShotStatus.RUNNING, ShotStatus.READY}
if step.get("force"):
# 质检规则升级后要回头复查:已放行/已过检的镜头也得重新看图
statuses |= {ShotStatus.APPROVED, ShotStatus.QC_PASSED}
shots = [shot for shot in target_shots if shot.status in statuses]
if not shots:
checked = [
shot for shot in target_shots
if bool((shot.qc_json or {}).get("passed"))
]
if target_shots and len(checked) == len(target_shots):
# generate_shots 排空队列时,生成处理器会自动创建并执行 QC 任务。
# 后续显式质检步骤无需重复提交,但应该记录为完成而不是“跳过”。
return {
"status": "succeeded",
"detail": f"{len(checked)} 个镜头已在生成阶段完成自动质检",
"result": {
"shot_ids": [shot.id for shot in checked],
"already_checked": True,
},
}
return {"status": "skipped", "detail": "没有等待自动质检的镜头"}
rounds = step.get("rework_rounds")
rounds = QC_REWORK_ROUNDS if rounds is None else max(0, int(rounds))
checked_ids = [shot.id for shot in shots]
seen_tasks: list[int] = []
rework_log: list[str] = []
round_no = 0
while True:
db.expire_all()
project = db.get(Project, project.id)
current = [shot for shot in _shot_filter(project, checked_ids)
if storage.exists(shot.output_path)]
if not current:
raise AgentStepError("返工重做后这批镜头的画面产物仍不存在,无法继续质检,请重新生成")
project, task_ids, failed, outcome = _qc_pass(db, project, current, timeout,
reuse_passed=not step.get("force"))
seen_tasks += task_ids
if not failed:
_clear_qc_rework_notes(current)
if run is not None and (run.plan_json or {}).get("qc_soft_pass"):
run.plan_json = {key: val for key, val in run.plan_json.items()
if key != "qc_soft_pass"}
db.commit()
detail = f"{len(current)} 个镜头质检通过"
if round_no:
detail = f"自动返工 {round_no} 轮后,{len(current)} 个镜头质检通过"
return {
"status": "succeeded",
"detail": detail,
"result": {"task_ids": seen_tasks,
"shot_ids": [shot.id for shot in current],
"rework_rounds": round_no, "rework": rework_log, **outcome},
}
if round_no >= rounds:
notes = [f"#{shot.order_index:02d} " + ";".join(_qc_hard_issues(shot, 2))
for shot in failed[:4]]
tail = (";已把上一轮的问题写回镜头提示词重做过,仍未通过说明需要人工介入:"
"改文案/提示词后重跑,或把本步骤 rework_rounds 调大" if round_no
else ";平台会自动把质检问题写回镜头提示词并重新生成(默认 1 轮),"
"只想人工处理可把本步骤 rework_rounds 设为 0")
reason = (("自动质检未通过,需要人工调整提示词或重新生成" if not round_no
else f"自动返工 {round_no} 轮后自动质检仍未通过")
+ ":" + " | ".join(notes) + tail)
salvage = [shot for shot in current if shot not in failed]
undecodable = [shot for shot in failed if _footage_undecodable(shot)]
if undecodable:
reason += (";其中 " + "、".join(f"#{row.order_index:02d}" for row in undecodable)
+ " 的画面文件无法解码(不是画面瑕疵),必须先重新出图再质检")
if run is None or undecodable or not _plan_has_action(run, "approve_passed"):
# 人工单点质检(没有 run)、画面根本解不开、或这条计划后面本来就要
# 停下等人工审核:这时判失败是对的,没有可以自动继续的步骤。
# 注意「整批都不合格」不再是硬停理由——那正是项目 9 卡死的地方,
# 用户要的是出片 + 明确留痕,而不是整条计划作废。
raise AgentStepError(reason)
# 少数镜头修不好时,过去整条计划就地判死:用户要的「重建时间线并导出成片」
# 永远轮不到,成片停在上一版(项目 9 镜头03「三只手」实测)。
# 现在按现有画面继续走完出片流程,同时把缺陷镜头留痕交回人工复核。
run.plan_json = {**(run.plan_json or {}), "qc_soft_pass": [
{"shot_id": shot.id, "order_index": shot.order_index,
"issues": _qc_hard_issues(shot)} for shot in failed]}
run.warnings_json = [*(run.warnings_json or []),
"镜头 " + "、".join(f"#{row.order_index:02d}" for row in failed)
+ f" 自动返工 {round_no} 轮后质检仍未通过:" + ";".join(notes[:3])
+ ";已按现有画面继续完成出片,建议单独重做这几个镜头后重新导出"]
events.log_event(
db, "agent.qc_soft_pass", project_id=project.id, level="warn",
message=f"自动质检返工 {round_no} 轮仍有 {len(failed)} 个镜头不合格:"
f"{reason[:400]};平台按现有画面继续执行「放行→时间线→导出」,"
"不合格镜头已标记待人工复核",
entity_type="project", entity_id=project.id,
data={"shot_ids": [shot.id for shot in failed], "round": round_no},
)
db.commit()
return {
"status": "succeeded",
"detail": (f"{len(salvage)} 个镜头质检通过," if salvage else "本步骤镜头全部未通过,")
+ f"{len(failed)} 个镜头自动返工 {round_no} 轮仍未通过"
"(已按现有画面继续出片,标记为待人工复核)",
"result": {"task_ids": seen_tasks,
"shot_ids": [shot.id for shot in current],
"rework_rounds": round_no, "rework": rework_log,
"rework_exhausted": [shot.id for shot in failed],
"need_review": [f"#{row.order_index:02d}" for row in failed]},
}
round_no += 1
notes = _write_qc_rework_notes(failed, round_no=round_no)
events.log_event(
db, "agent.qc_rework", project_id=project.id, level="warn",
message=f"自动质检未通过 → 第 {round_no} 轮自动返工:已把问题写回 "
f"{len(failed)} 个镜头的提示词并重新生成画面 → " + ";".join(notes[:3]),
entity_type="project", entity_id=project.id,
data={"shot_ids": [shot.id for shot in failed], "round": round_no},
)
db.commit()
rework_log.append(f"第{round_no}轮 {len(failed)} 个镜头:" + ";".join(notes[:3]))
# 首帧就是 i2v 的输入图:多余人物、串装、分身一旦画进了关键帧,
# 光重出视频等于同一张脸重摇一万次,返工永远修不掉(项目 9 镜头 03 实测)。
# 所以返工先把首帧重画(关键帧提示词是现算的,会带上刚写回的返工要求),再出视频。
_step_generate_keyframes(
db, project,
{"shot_ids": [shot.id for shot in failed], "regenerate": {"keyframes": True}},
timeout,
)
_step_generate_shots(
db, project,
{"shot_ids": [shot.id for shot in failed], "regenerate": {"video": True}},
timeout,
)
def _step_approve_passed(db: Session, project: Project, step: dict, run: AgentRun) -> dict:
if not run.auto_approve:
raise AgentStepError("该计划未获得自动审核授权", code="auto_approve_required")
shots = _shot_filter(project, step.get("shot_ids"), statuses={ShotStatus.QC_PASSED})
shots = [shot for shot in shots if bool((shot.qc_json or {}).get("passed"))]
# 返工耗尽仍不合格的镜头:按现画面放行,审核记录如实写明它没通过质检
soft_ids = {int(row.get("shot_id") or 0)
for row in ((run.plan_json or {}).get("qc_soft_pass") or [])}
soft = [shot for shot in _shot_filter(project, step.get("shot_ids") or None)
if shot.id in soft_ids and shot not in shots]
if not shots and not soft:
return {"status": "skipped", "detail": "没有质检合格且等待审核的镜头"}
reviewer = f"agent:{run.provider}/{run.model}"
for shot in shots + soft:
flagged = shot in soft
shot.status = ShotStatus.APPROVED
shot.stage = "review"
shot.review_state = ReviewVerdict.PASS
shot.review_notes = (
f"LLM 调度器放行:自动返工后质检仍未通过({qc_issue_brief(shot)}),"
"按现有画面继续出片,需人工复核;不是人工审核" if flagged
else "LLM 调度器基于自动质检通过结果放行;不是人工审核")
db.add(ReviewRecord(
project_id=project.id,
shot_id=shot.id,
kind="agent",
verdict=ReviewVerdict.PASS,
score=shot.qc_score,
checks_json={"qc": shot.qc_json or {}, "agent_run_id": run.id,
"qc_soft_pass": flagged},
notes=shot.review_notes,
reviewer=reviewer,
))
events.log_event(
db, "shot.agent_approved", project_id=project.id, shot_id=shot.id,
level="warn" if flagged else "info",
message=(f"AI 审核放行(质检未通过,待人工复核):{shot.code}「{shot.title}」"
f"——{qc_issue_brief(shot)}" if flagged
else f"AI 审核放行:{shot.code}「{shot.title}」"
f"(质检 {shot.qc_score or 0:.0f} 分,非人工审核)"),
entity_type="shot", entity_id=shot.id, actor=reviewer,
data={"agent_run_id": run.id, "qc_score": shot.qc_score,
"qc_soft_pass": flagged},
)
db.flush()
status.refresh(db, project, commit=False)
db.commit()
detail = f"{len(shots)} 个镜头由 AI 审核放行(已标记 agent,不冒充人工)"
if soft:
detail += (f";另有 {len(soft)} 个镜头质检未通过,按现有画面放行待人工复核:"
+ "、".join(f"#{row.order_index:02d}" for row in soft))
return {
"status": "succeeded",
"detail": detail,
"result": {"shot_ids": [shot.id for shot in shots + soft], "reviewer": reviewer,
"soft_pass_shot_ids": [shot.id for shot in soft]},
}
def _step_build_timeline(db: Session, project: Project, step: dict, run: AgentRun) -> dict:
approved = _shot_filter(project, step.get("shot_ids"), statuses={ShotStatus.APPROVED})
if not approved:
raise AgentStepError("没有已审核合格的镜头,无法建立时间线")
ids = [shot.id for shot in approved]
current = status.current_timeline(db, project)
existing_ids = [seg.shot_id for seg in sorted(current.segments or [],
key=lambda row: row.order_index)] if current else []
if current is not None and existing_ids and set(existing_ids) > set(ids):
# 局部重做(步骤只覆盖 26/27)不能把已经合格的镜头从成片里删掉:
# 照局部范围重建会排出一部少一两镜的「成片」,界面上却写着导出成功。
whole = _shot_filter(project, None, statuses={ShotStatus.APPROVED})
if len(whole) > len(approved):
approved, ids = whole, [shot.id for shot in whole]
redo = bool((step.get("regenerate") or {}).get("video")) \
or bool((step.get("regenerate") or {}).get("voice"))
# 上一版成片里已经存在的镜头,本轮计划未必覆盖(重做会把审核状态打回未放行)。
# 只要画面文件还在就继续留在成片里:否则局部重做后排出的「成片」会少镜头,
# 而界面上照样写着导出成功(项目 9 实测:只重做分镜1/2 就丢了分镜3)。
by_id = {shot.id: shot for shot in (project.shots or [])}
chosen = {shot.id: shot for shot in approved}
carried: list[Shot] = []
lost: list[str] = []
for shot_id in existing_ids:
if not shot_id or shot_id in chosen:
continue
shot = by_id.get(shot_id)
if shot is None or not storage.exists(shot.output_path):
lost.append(f"#{shot.order_index:02d}" if shot is not None else f"镜头{shot_id}")
continue
chosen[shot.id] = shot
carried.append(shot)
ids = [shot.id for shot in sorted(chosen.values(), key=lambda row: (row.order_index, row.id))]
if carried:
names = "、".join(f"#{row.order_index:02d} {row.code}" for row in carried)
run.warnings_json = [*(run.warnings_json or []),
f"镜头 {names} 已在当前成片中,但本轮计划没有放行它;"
"时间线沿用其现有画面以免成片缺镜,如画面有问题请单独重做后重新导出"]
events.log_event(
db, "agent.timeline_keep", project_id=project.id, level="warn",
message=f"重建时间线时保留了 {len(carried)} 个本轮未放行镜头的现有画面:{names}"
"(本轮计划未包含它们的放行,成片缺镜比带瑕疵更严重)",
entity_type="project", entity_id=project.id,
data={"shot_ids": [row.id for row in carried]},
)
db.commit()
if current is not None and existing_ids == ids and not redo: # 重做时片段路径是旧素材快照
return {
"status": "skipped",
"detail": f"当前时间线 v{current.version} 已覆盖全部 {len(ids)} 个合格镜头",
"result": {"timeline_id": current.id, "version": current.version},
}
result = timeline_service.build_timeline(
db, project, name="LLM 调度主时间线", only_approved=not carried, shot_ids=ids,
actor=f"agent:{run.provider}/{run.model}",
)
detail = (f"时间线 v{result['version']} 已排布 {result['segments']} 个片段"
f"({result['total_duration_ms'] / 1000:.1f}s)")
if carried:
detail += (";保留上一版成片中本轮未放行的 "
+ "、".join(f"#{row.order_index:02d}" for row in carried))
if lost:
detail += (";" + "、".join(lost) + " 的画面产物已不存在,无法继续留在成片中")
return {
"status": "succeeded",
"detail": detail,
"result": result,
}
def _step_compose(db: Session, project: Project, step: dict, run: AgentRun, timeout: float) -> dict:
timeline = status.current_timeline(db, project)
if timeline is None:
raise AgentStepError("还没有时间线;请在同一次计划中先执行 build_timeline")
existing = db.scalars(select(ExportRecord).where(
ExportRecord.project_id == project.id,
ExportRecord.timeline_id == timeline.id,
ExportRecord.status == "succeeded",
).order_by(ExportRecord.id.desc())).first()
redo = bool((step.get("regenerate") or {}).get("export"))
if existing is not None and storage.exists(existing.path) and not redo:
return {
"status": "skipped",
"detail": f"当前时间线已有成功导出 #{existing.id},未重复消耗资源",
"result": {"export_id": existing.id, "path": existing.path},
}
report = timeline_service.timeline_report(db, timeline, project=project)
if not report.get("ready"):
errors = [row["message"] for row in report.get("issues", []) if row.get("level") == "error"]
raise AgentStepError("时间线尚未就绪:" + ";".join(errors[:4]))
task = dispatch.new_compose_task(
db, project, timeline.id, preset=_compose_preset(project),
name=f"{project.name} · LLM 调度成片", priority=2,
)
db.commit()
outcome = _drain(db, project, timeout, limit=4, watch=[task.id])
failures = _task_failures(db, [task.id])
if failures:
raise AgentStepError("合成导出失败:" + ";".join(failures))
export = db.scalars(select(ExportRecord).where(ExportRecord.task_id == task.id)
.order_by(ExportRecord.id.desc())).first()
if export is None or export.status != "succeeded" or not export.path:
raise AgentStepError(f"合成任务 #{task.id} 未产出可下载成片")
return {
"status": "succeeded",
"detail": f"成片导出完成:{(export.duration_ms or 0) / 1000:.1f}s,"
f"{export.width}x{export.height}",
"result": {"task_id": task.id, "export_id": export.id, "path": export.path,
"url": storage.url_for(export.path), **outcome},
}
def _execute_step(db: Session, run: AgentRun, project: Project, step: dict, timeout: float) -> dict:
action = step["action"]
if action == "resume":
return _step_resume(db, project, step, timeout)
if action == "rewrite_shots":
return _step_rewrite_shots(db, project, step, run)
if action == "generate_identity_assets":
return _step_generate_identity_assets(db, project, step, timeout)
if action == "generate_keyframes":
return _step_generate_keyframes(db, project, step, timeout)
if action == "generate_shots":
return _step_generate_shots(db, project, step, timeout)
if action == "quality_check":
return _step_quality_check(db, project, step, timeout, run)
if action == "approve_passed":
return _step_approve_passed(db, project, step, run)
if action == "build_timeline":
return _step_build_timeline(db, project, step, run)
if action == "compose":
return _step_compose(db, project, step, run, timeout)
raise AgentStepError(f"不支持的调度动作:{action}")
def _write_step(db: Session, run: AgentRun, index: int, **patch) -> None:
steps = [dict(row) for row in (run.steps_json or [])]
steps[index] = {**steps[index], **patch}
run.steps_json = steps
db.flush()
def _mark_cancelled(db: Session, run: AgentRun, reason: str) -> None:
run.status = "cancelled"
run.error_code = "cancelled"
run.error_message = reason
run.finished_at = now_utc()
db.commit()
project = db.get(Project, run.project_id)
if project is not None:
status.refresh(db, project, commit=False) # 取消之后界面不该继续转圈
db.commit()
def reconcile_project_status(project_id: int) -> bool:
"""独立会话把项目状态推回它真实的阶段(计划收口后必做)。
`status.refresh` 过去只有两处会调:任务 handler 自己(只有出图那条调了),
以及守护线程(只刷「队列里还有活跃任务」的项目)。于是当项目最后一条任务
正好是质检、或整条 LLM 计划以失败收口时,没有任何一方负责回写:
项目永远停在「调用视频模型」,前端一直转圈,用户看到的就是
「指令下了半天,项目卡住不动」(项目 9 自检实测:任务全跑完 6 分钟后
项目仍是 generating/generate)。
"""
db = SessionLocal()
try:
project = db.get(Project, project_id)
if project is None:
return False
status.refresh(db, project, commit=False)
db.commit()
return True
except Exception as exc: # 回写失败不得拖垮主流程
db.rollback()
log.warning("项目状态回写失败 project=%s:%s", project_id, exc)
return False
finally:
db.close()
def _final_status(db: Session, run: AgentRun, project: Project) -> tuple[str, str, str]:
db.expire_all()
project = db.get(Project, project.id)
shots = list(project.shots or [])
waiting = [
shot for shot in shots
if shot.status == ShotStatus.QC_PASSED and bool((shot.qc_json or {}).get("passed"))
]
if waiting:
return (
"waiting_review",
"review_required",
f"{len(waiting)} 个镜头质检合格,等待人工审核;确认后可继续执行剩余计划",
)
if project.status == "delivered" or all(shot.status == ShotStatus.APPROVED for shot in shots):
return "succeeded", "", "计划执行完成"
return "succeeded", "", "本次计划步骤已执行完成"
def start_run(db: Session, run: AgentRun, *, timeout_seconds: int,
reset_resume_budget: bool = True) -> AgentRun:
"""校验计划并把状态翻到 running。
后台线程里的校验失败以前是静默的:线程直接死掉、状态仍停在 planned,
前端却已经收到 200「已启动」,用户看到的就是一次点击后项目毫无推进。
所以状态必须在响应前同步落库,重复点击才会变成明确的 409。
"""
if run.status == "running":
raise AgentStateError("该计划已经在执行")
if run.status == "succeeded":
raise AgentStateError("该计划已经执行完成;如需新任务请重新下达指令")
if run.status not in ("planned", "failed", "waiting_review"):
raise AgentStateError(f"当前计划状态为 {run.status},不能执行")
if not run.steps_json:
raise AgentStateError("该计划没有可执行步骤")
other = db.scalars(select(AgentRun).where(
AgentRun.project_id == run.project_id,
AgentRun.status == "running",
AgentRun.id != run.id,
)).first()
if other is not None:
raise AgentStateError(f"项目已有智能调度 #{other.id} 正在执行")
# 人工重新执行 = 用户重新授权了这条计划:自动续跑预算必须复位。
# 否则一条计划被重启烧掉 max_resume 次以后,用户再点「执行」也再不会自动续跑,
# 下一次服务重启就永远停在 failed(界面上依然是「项目毫无推进」)。
if reset_resume_budget and (run.plan_json or {}).get("resume_attempts"):
run.plan_json = {key: val for key, val in run.plan_json.items()
if key != "resume_attempts"}
run.status = "running"
run.cancel_requested = False
run.timeout_seconds = timeout_seconds
run.error_code = ""
run.error_message = ""
run.started_at = run.started_at or now_utc()
run.finished_at = None
db.commit()
events.log_event(
db, "agent.run_started", project_id=run.project_id,
message=f"开始执行 LLM 调度计划 #{run.id}",
entity_type="agent_run", entity_id=run.id,
actor=f"agent:{run.provider}/{run.model}",
)
db.commit()
return run
def execute_run(run_id: int, *, timeout_seconds: int, prestarted: bool = False) -> None:
"""同步执行一条已确认计划;API 层负责把它放到后台线程。"""
db = SessionLocal()
project_id: int | None = None
try:
run = db.get(AgentRun, run_id)
if run is None:
raise AgentStateError("调度记录不存在")
project_id = run.project_id
project = db.get(Project, run.project_id)
if project is None:
raise AgentStateError("项目不存在")
if not prestarted:
start_run(db, run, timeout_seconds=timeout_seconds)
for index, step in enumerate(list(run.steps_json or [])):
db.refresh(run)
if run.cancel_requested:
_mark_cancelled(db, run, "用户已取消智能调度")
return
if step.get("status") in ("succeeded", "skipped"):
continue
_write_step(
db, run, index, status="running", started_at=now_utc().isoformat(),
finished_at=None, detail="正在执行…", result={},
)
db.commit()
try:
outcome = _execute_step(db, run, project, step, float(timeout_seconds))
if step["action"] == "rewrite_shots":
# 镜头是整组重建的,旧 shot_ids 已不存在:后续步骤改为覆盖全部新镜头
following = [dict(row) for row in (run.steps_json or [])]
for later in following[index + 1:]:
later["shot_ids"] = []
run.steps_json = following
db.flush()
db.expire_all()
project = db.get(Project, project.id)
_write_step(
db, run, index,
status=outcome.get("status", "succeeded"),
detail=outcome.get("detail", ""),
result=outcome.get("result", {}),
finished_at=now_utc().isoformat(),
)
db.commit()
events.log_event(
db, f"agent.step_{outcome.get('status', 'succeeded')}",
project_id=project.id,
message=f"计划 #{run.id} · {ACTION_LABELS[step['action']]}:"
f"{outcome.get('detail', '')}",
entity_type="agent_run", entity_id=run.id,
actor=f"agent:{run.provider}/{run.model}",
data={"action": step["action"], "shot_ids": step.get("shot_ids") or []},
)
db.commit()
except Exception as exc:
db.rollback()
run = db.get(AgentRun, run_id)
if run is not None and (run.cancel_requested or run.status == "cancelled"):
_mark_cancelled(db, run, run.error_message or "用户已取消智能调度")
return
code = getattr(exc, "code", "agent_step_failed")
message = str(exc)
_write_step(
db, run, index, status="failed", detail=message,
finished_at=now_utc().isoformat(), result={},
)
run.status = "failed"
run.error_code = code
run.error_message = message[:4000]
run.finished_at = now_utc()
db.commit()
events.log_event(
db, "agent.run_failed", project_id=run.project_id, level="error",
message=f"LLM 调度计划 #{run.id} 在「{ACTION_LABELS[step['action']]}」步骤停止:{message[:300]}",
entity_type="agent_run", entity_id=run.id,
actor=f"agent:{run.provider}/{run.model}",
data={"action": step["action"], "code": code},
)
db.commit()
return
db.refresh(run)
if run.cancel_requested:
_mark_cancelled(db, run, "用户已取消智能调度")
return
final, code, message = _final_status(db, run, project)
run.status = final
run.error_code = code
run.error_message = "" if final == "succeeded" else message
run.finished_at = now_utc()
db.commit()
events.log_event(
db, "agent.run_finished" if final == "succeeded" else "agent.run_waiting_review",
project_id=run.project_id,
message=f"LLM 调度计划 #{run.id}:{message}",
entity_type="agent_run", entity_id=run.id,
actor=f"agent:{run.provider}/{run.model}",
)
db.commit()
except Exception as exc:
db.rollback()
run = db.get(AgentRun, run_id)
# 含「还没进入步骤循环就炸了」的情况:状态必须留下错误,否则前端无从判断
if run is not None and run.status not in ("succeeded", "failed", "cancelled"):
run.status = "failed"
run.error_code = getattr(exc, "code", "agent_error")
run.error_message = str(exc)[:4000]
run.finished_at = now_utc()
db.commit()
events.log_event(
db, "agent.run_failed", project_id=run.project_id, level="error",
message=f"LLM 调度计划 #{run.id} 启动失败:{str(exc)[:300]}",
entity_type="agent_run", entity_id=run.id,
actor=f"agent:{run.provider}/{run.model}",
data={"code": run.error_code},
)
db.commit()
finally:
db.close()
if project_id is not None:
reconcile_project_status(project_id) # 失败/等待审核时同样要收口
def execute_in_background(run_id: int, *, timeout_seconds: int, prestarted: bool = False) -> None:
thread = threading.Thread(
target=execute_run,
kwargs={"run_id": run_id, "timeout_seconds": timeout_seconds, "prestarted": prestarted},
name=f"nex-agent-{run_id}",
daemon=True,
)
thread.start()
def _has_unfinished_steps(run: AgentRun) -> bool:
"""被中断的计划里还有没跑完的步骤吗(跑完的不会被重复执行,续跑是安全的)。"""
steps = list(run.steps_json or [])
return bool(steps) and any(
step.get("status") not in ("succeeded", "skipped") for step in steps)
def resume_interrupted_run(run_id: int, *, timeout_seconds: int = 0) -> None:
"""服务重启后从断点续跑一条被中断的计划;API/main 负责把它放到独立线程。
任务层早就有「断点续跑」(重启后 queued/running 任务会被重新排队),
但计划层以前只是被翻成 failed 就没人管了:镜头继续出完图,却再也没有
任何一步去质检、放行、排时间线、合成——用户看到的就是「指令跑完,项目毫无推进」。
"""
db = SessionLocal()
try:
run = db.get(AgentRun, run_id)
if run is None or run.status != "failed" or run.error_code != "agent_interrupted":
return
limit = int(timeout_seconds or run.timeout_seconds or 1800)
# 自动续跑不能复位预算,否则预算永远烧不完,反复重启会无限自动续跑
start_run(db, run, timeout_seconds=limit, reset_resume_budget=False)
events.log_event(
db, "agent.run_resumed", project_id=run.project_id,
message=f"执行进程已重启,智能调度 #{run.id} 从断点继续执行剩余步骤",
entity_type="agent_run", entity_id=run.id,
actor=f"agent:{run.provider}/{run.model}",
)
db.commit()
except Exception as exc:
# 续跑没起来(比如项目里已有别的计划在跑):留痕并交回人工,别静默吞掉
db.rollback()
run = db.get(AgentRun, run_id)
if run is not None:
events.log_event(
db, "agent.run_resume_failed", project_id=run.project_id, level="warn",
message=f"智能调度 #{run.id} 自动续跑未能启动:{str(exc)[:200]}"
";可在项目概览点「重试暂停步骤」手动继续",
entity_type="agent_run", entity_id=run.id,
)
db.commit()
return
finally:
db.close()
execute_run(run_id, timeout_seconds=limit, prestarted=True)
def _resume_later(run_id: int, timeout_seconds: int, delay: float) -> None:
"""等一下再续跑:让 daemon 先把中断任务重新排队,避免两头同时抢容量。"""
time.sleep(delay)
resume_interrupted_run(run_id, timeout_seconds=timeout_seconds)
def recover_interrupted_runs(db: Session, *, auto_resume: bool = False,
max_resume: int = 2, resume_delay: float = 5.0) -> int:
"""服务重启后回收进程内 Agent 线程,避免旧记录永久占用项目的执行锁。
auto_resume:inline 后端下直接把计划从断点接着跑(而不是只标成失败等人点按钮)。
反复重启会烧掉 GPU 预算,所以每条计划最多自动续跑 max_resume 次。
"""
rows = db.scalars(select(AgentRun).where(
AgentRun.status.in_(("running", "planning")))).all()
now = now_utc()
pending: list[int] = []
for run in rows:
was_planning = run.status == "planning"
steps = [dict(row) for row in (run.steps_json or [])]
for step in steps:
if step.get("status") == "running":
step["status"] = "failed"
step["detail"] = "服务重启,步骤执行线程已中断"
step["finished_at"] = now.isoformat()
run.steps_json = steps
run.status = "failed"
run.cancel_requested = False
run.error_code = "agent_interrupted"
attempts = int((run.plan_json or {}).get("resume_attempts") or 0)
can_resume = (auto_resume and not was_planning and attempts < max_resume
and _has_unfinished_steps(run))
if can_resume:
run.plan_json = {**(run.plan_json or {}), "resume_attempts": attempts + 1}
run.error_message = (f"服务重启导致智能调度执行中断,已自动从断点继续"
f"(第 {attempts + 1}/{max_resume} 次)")
pending.append(run.id)
else:
run.error_message = (
"服务重启导致规划请求中断,请重新下达指令" if was_planning
else "服务重启导致智能调度执行中断,请重新下达指令,"
"或在项目概览点「重试暂停步骤」从断点继续")
if was_planning:
run.plan_json = {**(run.plan_json or {}), "stage": "interrupted"}
run.finished_at = now
events.log_event(
db, "agent.run_interrupted", project_id=run.project_id, level="warn",
message=f"智能调度 #{run.id} 因服务重启被中断"
+ (",已自动从断点继续" if can_resume else ",已解除项目执行锁"),
entity_type="agent_run", entity_id=run.id,
data={"auto_resumed": can_resume},
)
if rows:
db.commit()
for interrupted_project_id in sorted({row.project_id for row in rows}):
target = db.get(Project, interrupted_project_id)
if target is not None:
status.refresh(db, target, commit=False)
db.commit()
for run_id in pending:
threading.Thread(
target=_resume_later, args=(run_id, 0, resume_delay),
name=f"nex-agent-resume-{run_id}", daemon=True,
).start()
return len(rows)
def cancel_run(db: Session, run: AgentRun, *, reason: str) -> AgentRun:
if run.status in ("succeeded", "cancelled"):
raise AgentStateError(f"计划当前状态为 {run.status},无需取消")
if run.status == "planning":
run.status = "cancelled"
run.cancel_requested = True
run.error_code = "cancelled"
run.error_message = reason
run.finished_at = now_utc()
run.plan_json = {**(run.plan_json or {}), "stage": "cancelled"}
events.log_event(
db, "agent.cancel_requested", project_id=run.project_id, level="warn",
message=f"已取消规划中的智能调度 #{run.id}:{reason}",
entity_type="agent_run", entity_id=run.id,
)
db.commit()
db.refresh(run)
return run
if run.status == "running":
run.cancel_requested = True
run.status = "cancelled"
run.error_code = "cancelled"
run.error_message = reason
run.finished_at = now_utc()
steps = [dict(row) for row in (run.steps_json or [])]
for step in steps:
if step.get("status") == "running":
step["status"] = "cancelled"
step["detail"] = reason
step["finished_at"] = run.finished_at.isoformat()
run.steps_json = steps
active_tasks = db.scalars(select(Task).where(
Task.project_id == run.project_id,
Task.status.in_(TaskStatus.ACTIVE),
)).all()
for task in active_tasks:
task.status = TaskStatus.CANCELLED
task.finished_at = now_utc()
task.next_retry_at = None
task.error_message = reason
shot = db.get(Shot, task.shot_id) if task.shot_id else None
if shot and shot.status in TaskStatus.ACTIVE:
shot.status = ShotStatus.READY
shot.stage = "cancelled"
events.log_event(
db, "task.cancelled", project_id=run.project_id, task_id=task.id,
shot_id=task.shot_id, level="warn",
message=f"随智能调度 #{run.id} 一并取消:{reason}",
entity_type="task", entity_id=task.id,
)
events.log_event(
db, "agent.cancel_requested", project_id=run.project_id, level="warn",
message=f"已取消智能调度 #{run.id} 及 {len(active_tasks)} 个活动任务",
entity_type="agent_run", entity_id=run.id,
)
else:
run.status = "cancelled"
run.cancel_requested = True
run.error_code = "cancelled"
run.error_message = reason
run.finished_at = now_utc()
db.commit()
db.refresh(run)
return run