nex_animation/backend/app/services/platform_generation.py

406 lines
22 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

r"""平台资产库的 AI 生成:登记成任务 → 统一调度执行 → 进度写回任务行。
为什么要走任务表(2026-11 · 504 那一串):
出图实测 46~49s(docs/MiniMaxH3_Spark_性能诊断与优化方案.md:681),反代
``proxy_read_timeout`` 出厂 60s,余量只有十几秒;同卡一排队就直接 504。
而 504 只是**看起来**失败了——出图线程并没有被取消,图照样落盘,
用户看不见结果,只能手动刷新页面去猜它到底成没成。
现在接口只做「校验 + 登记 + 派发」,当场 202 返回任务号,
进度、产物、失败原因全都在 ``tasks`` 那一行里,换设备、刷新页面都接着看得到。
为什么并入 ``tasks`` 而不是另立一张操作表:
「所有要跑模型、要等一会儿的活」本该只有一个地方能问。并表之后
限流派发(capacity.headroom)、失败重试、断点续跑(resume_interrupted)、
心跳与 stale、成本与事件台账,平台侧生成全部自动继承,
不必再维护第二套进行中转圈。代价只是 ``project_id`` 变成可空,
由 ``db._relax_nullability`` 在启动时原地放开。
执行体在 ``jobs/handlers.handle_platform_asset`` 里,与项目侧任务同一条通道;
深处的算力排队说明由 ``services/progress`` 写回(见 providers/comfy_common.gpu_slot)。
"""
from __future__ import annotations
import logging
from pathlib import Path
from fastapi import HTTPException
from sqlalchemy import select
from sqlalchemy.orm import Session
from app import schemas
from app.constants import AssetKind, TaskKind, TaskStatus
from app.models import (PlatformCharacter, PlatformMedia, PlatformScene, PlatformVoice,
Project, Task, now_utc)
from app.providers import registry as model_registry
from app.providers.base import AssetGenRequest, ProviderError, VoiceGenRequest
from app.services import costing, dispatch, events, progress, storage, timbre
log = logging.getLogger("nex.platform_generation")
#: 一条平台生成任务最多能被写成多少次状态说明(message 列宽 400)
MAX_LABEL_CHARS = 200
def landed_message(media: dict) -> str:
"""任务成功落定时要写进 ``tasks.message`` 的那句话。
``progress.report`` 写的是「此刻在做什么」,成功之后没人覆盖它,
于是顶栏「平台动态 · 最近落定」会出现「已完成 + 算力已就绪:模型出图中」
这种自相矛盾的回显(状态胶囊说完了,正文还在说进行中)。
"""
bits = [f"已生成{media.get('kind_label') or '新素材'}"]
if media.get("version"):
bits.append(f"第 {media['version']} 版")
if media.get("duration_ms"):
bits.append(f"{round(media['duration_ms'] / 1000, 1)}s")
elif media.get("size_bytes"):
bits.append(f"{round(media['size_bytes'] / 1024 / 1024, 2)} MB")
return " · ".join(bits)[:MAX_LABEL_CHARS]
# ---------------------------------------------------------------- 母版与素材的公共读法
def normalize_type(owner_type: str) -> str:
return {
"characters": "human", "character": "human", "human": "human", "humans": "human",
"scene": "scene", "scenes": "scene", "voice": "voice", "voices": "voice",
}.get(owner_type, owner_type)
def get_owner(db: Session, owner_type: str, owner_id: int):
owner_type = normalize_type(owner_type)
if owner_type == "human":
row = db.get(PlatformCharacter, owner_id)
elif owner_type == "scene":
row = db.get(PlatformScene, owner_id)
elif owner_type == "voice":
row = db.get(PlatformVoice, owner_id)
else:
raise HTTPException(400, f"不支持的平台资产类型:{owner_type}")
if row is None:
raise HTTPException(404, "平台资产不存在")
return row
def type_label(owner_type: str) -> str:
return {"human": "人物", "scene": "场景", "voice": "声线"}.get(owner_type, owner_type)
def resolve_reference_media(db: Session, owner_type: str, owner_id: int, ids: list[int],
*, kind: str) -> list[PlatformMedia]:
"""校验平台生成参考素材只能来自当前资产,避免跨资产误读本地路径。"""
rows: list[PlatformMedia] = []
seen: set[int] = set()
image_suffixes = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"}
audio_suffixes = {".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".opus"}
for media_id in ids:
if media_id in seen:
continue
seen.add(media_id)
row = db.get(PlatformMedia, media_id)
if (row is None or row.owner_type != owner_type or row.owner_id != owner_id):
raise HTTPException(400, "参考素材不属于当前平台资产")
if not storage.exists(row.path):
raise HTTPException(400, "参考素材文件不存在,请重新上传")
suffix = Path(row.path).suffix.lower()
if kind == "image" and suffix not in image_suffixes:
raise HTTPException(400, "图像参考素材必须是图片文件")
if kind == "audio" and suffix not in audio_suffixes:
raise HTTPException(400, "音色参考素材必须是音频文件")
rows.append(row)
return rows
def build_owner_prompt(project: Project | None, owner_type: str, owner, kind: str, custom: str) -> str:
if custom.strip():
return custom.strip()
bits = []
if owner_type == "human":
bits = [f"{owner.name},电影级角色设定图", owner.appearance, owner.persona_prompt,
"五官、发型、体型、服装与配饰保持一致,正面清晰,材质细节可信"]
elif owner_type == "scene":
bits = [f"{owner.name},电影级场景概念图", owner.prompt or owner.description,
owner.lighting, owner.time_of_day, owner.palette,
"空间结构清晰,透视准确,建筑与陈设保持一致"]
if project:
bits.append(project.style_guide)
return "。".join(str(part).strip() for part in bits if part)
def resolve_spec(db: Session, owner_type: str, data: schemas.PlatformGenerate):
"""按资产类型解析这次生成实际会派给哪个模型(留空即跟随平台默认)。"""
if owner_type == "voice":
spec, _adapter = model_registry.resolve_voice(db, data.provider or None, data.model or None)
else:
spec, _adapter = model_registry.resolve_asset(db, "image",
data.provider or None, data.model or None)
return spec
def resolve_voice_reference(db: Session, owner, spec,
data: schemas.PlatformGenerate) -> tuple[PlatformMedia | None, dict]:
"""把「这次生成会拿到谁的嗓子」算出来:参考音频决定音色,表单里的性别只是元数据。"""
reference_ids = list(data.reference_media_ids)
if data.reference_media_id:
reference_ids.append(data.reference_media_id)
references = resolve_reference_media(db, "voice", owner.id, reference_ids, kind="audio")
reference_media = references[0] if references else None
reference = timbre.resolve_reference(
db, owner,
explicit=str(storage.to_absolute(reference_media.path)) if reference_media else "",
clone=timbre.clone_model(spec), instruct=timbre.instruct_model(spec))
return reference_media, reference
# ---------------------------------------------------------------- 同步校验
def preflight(db: Session, owner_type: str, owner, data: schemas.PlatformGenerate):
"""开工前能当场说清的错误,绝不留到五分钟以后。
异步化之后接口立刻返回,如果连「没填试听文本」都要等轮询才发现,
用户面对的是一个跑完了才知道白跑的进度条。所以这些**便宜且确定**的校验
留在请求里同步做(422/409 原样吐给表单),耗时部分才交给后台。
后台执行时会把同样的校验再跑一遍:排到 GPU 时模型可能已被停用、
参考音频可能已被删掉,以及时报错的是供应商而不是表单。
"""
spec = resolve_spec(db, owner_type, data)
if not spec.enabled:
raise HTTPException(409, f"模型 {spec.provider}/{spec.model} 已停用")
if owner_type != "voice":
return spec
if not (data.sample_text or "").strip():
raise HTTPException(422, "请先填写试听文本:它是模型要朗读的台词,"
"系统不会用「音色描述」代替")
# 嗓子来源必须当场说清:克隆模型缺参考音频时会静默回退到模型默认参考(女声),
# 男声声线就这样被生成成了女声——等四十秒后试听才发现,等于白跑一轮。
_media, reference = resolve_voice_reference(db, owner, spec, data)
if not reference["path"] and reference.get("source") == "none":
raise HTTPException(409, reference["note"])
return spec
# ---------------------------------------------------------------- 登记与执行
def active_task(db: Session, owner_type: str, owner_id: int, asset_kind: str) -> Task | None:
"""这条母版的这一类素材是否已经有生成任务在跑(挡连点:三次点击只该出一张图)。"""
rows = db.scalars(select(Task).where(
Task.owner_type == owner_type, Task.owner_id == owner_id,
Task.kind == TaskKind.PLATFORM_ASSET, Task.status.in_(TaskStatus.ACTIVE)
).order_by(Task.id)).all()
for row in rows:
if str((row.payload_json or {}).get("asset_kind") or "") == asset_kind:
return row
return None
def enqueue(db: Session, *, owner_type: str, owner, spec, data: schemas.PlatformGenerate,
asset_kind: str) -> Task:
"""登记一条平台生成任务并立刻尝试派发;返回任务行(202 的响应体就是它)。"""
running = active_task(db, owner_type, owner.id, asset_kind)
if running is not None:
raise HTTPException(409, f"「{owner.name}」的{AssetKind.LABELS.get(asset_kind, asset_kind)}"
f"正在生成中(任务 #{running.id}),完成前不能重复发起")
label = f"生成「{owner.name}」的{AssetKind.LABELS.get(asset_kind, asset_kind)}"[:MAX_LABEL_CHARS]
cost = (costing.estimate_voice_cost(spec, len((data.sample_text or "").strip()))
if owner_type == "voice" else 0.0)
task = Task(
project_id=None, shot_id=None, kind=TaskKind.PLATFORM_ASSET,
name=label, stage="platform_generate", status=TaskStatus.QUEUED,
priority=5, provider=spec.provider, model=spec.model,
mode="voice" if owner_type == "voice" else data.kind,
owner_type=owner_type, owner_id=owner.id,
# 同一母版同一素材类型的重复点击靠这条键挡,不给它配项目侧那套镜头幂等语义
idempotency_key=f"platform:{owner_type}:{owner.id}:{asset_kind}:{int(now_utc().timestamp())}",
# 只留一次自动重试:连不上算力这类瞬时故障值得再试一把(生成失败没有产物,
# 不存在重复扣费);参数与配置错误在 generate 里标了 retryable=False,当场判死。
max_attempts=2, payload_json=_payload(owner_type, owner, asset_kind, data),
cost=cost, queued_at=now_utc(), message="排队中:等待调度")
db.add(task)
db.flush()
events.log_event(db, "task.queued", project_id=None, task_id=task.id,
message=f"平台{type_label(owner_type)}「{owner.name}」{label}",
entity_type="task", entity_id=task.id,
data={"owner_type": owner_type, "owner_id": owner.id, "asset_kind": asset_kind,
"provider": spec.provider, "model": spec.model})
db.commit()
result = dispatch.dispatch_task(db, task)
log.info("平台生成任务 #%s 已登记:%s(派发 %s)", task.id, label,
"立即开始" if result["dispatched"] else f"排队等待,{result['reason']}")
return db.get(Task, task.id)
def _payload(owner_type: str, owner, asset_kind: str,
data: schemas.PlatformGenerate) -> dict:
"""任务参数:请求体原样收在 ``request`` 里,外层放调度与展示要用的字段。
``owner_name`` 只是给顶栏「平台动态」的「查看」用的:那条深链按名字搜资产库,
而 ``task.name`` 是「生成「俊杰」的定妆图」这样的一句话,拿它当搜索词永远 0 条。
存名字而不是回表查,是为了不给 ``task_dict`` 加一次查询——它全站每条任务都要过。
"""
return {"operation": "generate", "owner_type": owner_type, "owner_id": owner.id,
"owner_name": getattr(owner, "name", ""),
"asset_kind": asset_kind, "request": data.model_dump()}
def request_from(task: Task) -> tuple[str, int, str, schemas.PlatformGenerate]:
"""从任务行还原「当时点的是什么」(执行体只拿得到 task_id)。"""
payload = task.payload_json or {}
raw = dict(payload.get("request") or {})
# 老任务行可能带着外层多写的字段,Strict 模型会当场拒绝——只留请求体认识的键
fields = set(schemas.PlatformGenerate.model_fields)
data = schemas.PlatformGenerate(**{k: v for k, v in raw.items() if k in fields})
return (str(payload.get("owner_type") or ""), int(payload.get("owner_id") or 0),
str(payload.get("asset_kind") or data.kind), data)
def produced_media_metadata(media, reference: dict, check: dict, text: str) -> None:
"""把「这次到底用了谁的嗓子、念了什么、音色对不对」写进素材,界面才说得清楚。"""
metadata = dict(media.metadata_json or {})
metadata.update({
"text": text,
"reference_audio": reference.get("path") or reference.get("attached") or "",
"reference_source": reference.get("source") or "",
"reference_note": reference.get("note") or "使用本声线已绑定的参考音频",
"timbre_check": check,
})
media.metadata_json = metadata
note = str(media.note or "")
tail = check.get("note") or ""
if tail and "音色自检" not in note:
media.note = f"{note}|音色自检:{tail}".strip("|")
def generate(db: Session, task: Task) -> dict:
"""真正跑一次平台资产生成并落库;失败一律抛 ``ProviderError`` 交给任务层归类。"""
owner_type, owner_id, asset_kind, data = request_from(task)
owner = get_owner(db, owner_type, owner_id)
progress.report("准备生成参数")
if owner_type == "voice":
result = _generate_voice(db, owner, data, task)
else:
result = _generate_image(db, owner_type, owner, data, task)
db.commit()
events.log_event(db, "platform.asset.generated", project_id=None, task_id=task.id,
message=f"平台{type_label(owner_type)}「{owner.name}」已生成"
f"{AssetKind.LABELS.get(asset_kind, asset_kind)}"
f"({result['provider']}/{result['model']})",
entity_type=f"platform_{owner_type}", entity_id=owner.id,
data={"media_id": result["media"].get("id"), "task_id": task.id})
db.commit()
return result
def _config_error(message: str, code: str) -> ProviderError:
"""参数/配置类错误:重试一百次也是同一个结果,直接判死并把原因说清。"""
return ProviderError(message, code=code, retryable=False)
def _generate_image(db: Session, owner_type: str, owner,
data: schemas.PlatformGenerate, task: Task) -> dict:
spec = resolve_spec(db, owner_type, data)
if not spec.enabled:
raise _config_error(f"模型 {spec.provider}/{spec.model} 已停用", "model_disabled")
references = resolve_reference_media(db, owner_type, owner.id, data.reference_media_ids,
kind="image")
prompt = build_owner_prompt(None, owner_type, owner, data.kind, data.prompt)
out = storage.platform_dir(owner_type, owner.id, "generated") / f"{data.kind}.png"
request = AssetGenRequest(
kind=data.kind, prompt=prompt, negative_prompt=data.negative_prompt,
width=data.width, height=data.height, seed=data.seed,
reference_images=[str(storage.to_absolute(row.path)) for row in references],
params={"model": spec.model},
idempotency_key=(
f"platform-image:{owner.id}:{data.kind}:"
f"{now_utc().isoformat()}:{data.prompt}:{','.join(map(str, data.reference_media_ids))}"
),
label=f"PLATFORM-{owner_type.upper()}-{owner.id}", output_path=out,
)
progress.report(f"调用 {spec.provider}/{spec.model} 出图")
result = model_registry.generate_asset(db, request, "image", spec.provider, spec.model)
produced = _require_file(result, "图像")
kind = data.kind
info = _store(produced, owner_type=owner_type, owner_id=owner.id, kind=kind,
stem=f"{owner_type}-{owner.id}-{kind}-ai", result=result)
media = storage.register_platform_media(
db, owner_type=owner_type, owner_id=owner.id, kind=kind, path=info["path"],
name=f"{owner.name} · AI {AssetKind.LABELS.get(kind, kind)}",
note=f"平台资产生成:{spec.provider}/{spec.model}", source="provider", info=info,
metadata={"generated": True, "provider": spec.provider, "model": spec.model,
"prompt": request.prompt,
"reference_media_ids": [row.id for row in references],
"task_id": task.id},
)
return {"media": storage.platform_media_dict(media), "kind": kind,
"provider": spec.provider, "model": spec.model}
def _generate_voice(db: Session, owner, data: schemas.PlatformGenerate, task: Task) -> dict:
spec = resolve_spec(db, "voice", data)
if not spec.enabled:
raise _config_error(f"模型 {spec.provider}/{spec.model} 已停用", "model_disabled")
# 试听文本是「模型要念出来的台词」,不是声线设定。旧实现拿 owner.description 兜底,
# 于是用户填的「温和、语速中等,沉稳」被当成台词念了出来,试听样本一听就是设定说明文。
text = (data.sample_text or "").strip()
if not text:
raise _config_error("请先填写试听文本:它是模型要朗读的台词,系统不会用「音色描述」代替",
"missing_sample_text")
reference_media, reference = resolve_voice_reference(db, owner, spec, data)
# 旧实现缺参考时直接回退到模型默认参考(女声),男声声线就这样被生成成了女声。
if not reference["path"] and reference.get("source") == "none":
raise _config_error(reference["note"], "missing_reference_audio")
out = storage.platform_dir("voice", owner.id, "generated") / f"sample-{now_utc().timestamp():.0f}.wav"
# 交给模型的是绝对路径(供应商要直接读文件);声线列里存 media_root 相对路径,
# 换机器 / 进容器时不会因为写死绝对路径而失效。与任务链路 handlers 保持一致。
reference_path = reference["path"] or owner.provider_voice_id
request = VoiceGenRequest(
text=text, voice_id=owner.name, provider_voice_id=reference_path,
language=owner.language, speed=float((owner.tuning or {}).get("speed", 1.0)),
seed=data.seed, params=(owner.tuning or {}).get("params") or {},
# 没有参考音频时靠它出音,有参考音频时用它微调音色(见 comfy_assets.graph)
instruct_text=timbre.instruct_text_of(owner) if timbre.instruct_model(spec) else "",
idempotency_key=f"platform-voice:{owner.id}:{task.id}",
label=f"PLATFORM-VOICE-{owner.id}", output_path=out,
)
progress.report(f"调用 {spec.provider}/{spec.model} 出试听")
result = model_registry.generate_voice(db, request, spec.provider, spec.model)
produced = _require_file(result, "音频")
info = _store(produced, owner_type="voice", owner_id=owner.id, kind="voice_sample",
stem=f"voice-{owner.id}-ai", result=result)
media = storage.register_platform_media(
db, owner_type="voice", owner_id=owner.id, kind="voice_sample", path=info["path"],
name=f"{owner.name} · AI {AssetKind.LABELS.get('voice_sample', 'voice_sample')}",
note=f"平台资产生成:{spec.provider}/{spec.model}", source="provider", info=info,
metadata={"generated": True, "provider": spec.provider, "model": spec.model,
"prompt": request.text,
"reference_media_ids": [reference_media.id] if reference_media else [],
"task_id": task.id},
)
owner.provider = spec.provider
if reference["path"] and timbre.clone_model(spec):
# 把真正参与克隆的那段参考音频绑回声线:否则试听样本是 A 的嗓子,
# 之后项目配音又落回模型默认参考,变成另一种嗓子。
owner.provider_voice_id = str(storage.to_relative(reference["path"]))
# 音色自检:量一遍实际基频,与设定性别比对,结果进素材元数据供界面标注。
check = timbre.check_voice(produced, owner.gender)
produced_media_metadata(media, reference, check, text)
return {"media": storage.platform_media_dict(media), "kind": "voice_sample",
"provider": spec.provider, "model": spec.model}
def _require_file(result, what: str) -> Path:
produced = Path(result.file_path)
if not produced.exists() or produced.stat().st_size < 128:
raise ProviderError(f"模型未返回有效{what}文件", code="missing_output")
return produced
def _store(produced: Path, *, owner_type: str, owner_id: int, kind: str, stem: str,
result) -> dict:
info = storage.save_platform_local_file(
produced, owner_type=owner_type, owner_id=owner_id, kind=kind, stem=stem)
if getattr(result, "duration_ms", 0):
info["duration_ms"] = int(result.duration_ms)
return info
__all__ = ["active_task", "build_owner_prompt", "enqueue", "generate", "get_owner",
"normalize_type", "preflight", "produced_media_metadata", "request_from",
"resolve_reference_media", "resolve_spec", "resolve_voice_reference", "type_label"]