406 lines
22 KiB
Python
406 lines
22 KiB
Python
r"""平台资产库的 AI 生成:登记成任务 → 统一调度执行 → 进度写回任务行。
|
||
|
||
为什么要走任务表(2026-11 · 504 那一串):
|
||
出图实测 46~49s(docs/MiniMaxH3_Spark_性能诊断与优化方案.md:681),反代
|
||
``proxy_read_timeout`` 出厂 60s,余量只有十几秒;同卡一排队就直接 504。
|
||
而 504 只是**看起来**失败了——出图线程并没有被取消,图照样落盘,
|
||
用户看不见结果,只能手动刷新页面去猜它到底成没成。
|
||
现在接口只做「校验 + 登记 + 派发」,当场 202 返回任务号,
|
||
进度、产物、失败原因全都在 ``tasks`` 那一行里,换设备、刷新页面都接着看得到。
|
||
|
||
为什么并入 ``tasks`` 而不是另立一张操作表:
|
||
「所有要跑模型、要等一会儿的活」本该只有一个地方能问。并表之后
|
||
限流派发(capacity.headroom)、失败重试、断点续跑(resume_interrupted)、
|
||
心跳与 stale、成本与事件台账,平台侧生成全部自动继承,
|
||
不必再维护第二套进行中转圈。代价只是 ``project_id`` 变成可空,
|
||
由 ``db._relax_nullability`` 在启动时原地放开。
|
||
|
||
执行体在 ``jobs/handlers.handle_platform_asset`` 里,与项目侧任务同一条通道;
|
||
深处的算力排队说明由 ``services/progress`` 写回(见 providers/comfy_common.gpu_slot)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from pathlib import Path
|
||
|
||
from fastapi import HTTPException
|
||
from sqlalchemy import select
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app import schemas
|
||
from app.constants import AssetKind, TaskKind, TaskStatus
|
||
from app.models import (PlatformCharacter, PlatformMedia, PlatformScene, PlatformVoice,
|
||
Project, Task, now_utc)
|
||
from app.providers import registry as model_registry
|
||
from app.providers.base import AssetGenRequest, ProviderError, VoiceGenRequest
|
||
from app.services import costing, dispatch, events, progress, storage, timbre
|
||
|
||
log = logging.getLogger("nex.platform_generation")
|
||
|
||
#: 一条平台生成任务最多能被写成多少次状态说明(message 列宽 400)
|
||
MAX_LABEL_CHARS = 200
|
||
|
||
|
||
def landed_message(media: dict) -> str:
|
||
"""任务成功落定时要写进 ``tasks.message`` 的那句话。
|
||
|
||
``progress.report`` 写的是「此刻在做什么」,成功之后没人覆盖它,
|
||
于是顶栏「平台动态 · 最近落定」会出现「已完成 + 算力已就绪:模型出图中」
|
||
这种自相矛盾的回显(状态胶囊说完了,正文还在说进行中)。
|
||
"""
|
||
bits = [f"已生成{media.get('kind_label') or '新素材'}"]
|
||
if media.get("version"):
|
||
bits.append(f"第 {media['version']} 版")
|
||
if media.get("duration_ms"):
|
||
bits.append(f"{round(media['duration_ms'] / 1000, 1)}s")
|
||
elif media.get("size_bytes"):
|
||
bits.append(f"{round(media['size_bytes'] / 1024 / 1024, 2)} MB")
|
||
return " · ".join(bits)[:MAX_LABEL_CHARS]
|
||
|
||
|
||
# ---------------------------------------------------------------- 母版与素材的公共读法
|
||
def normalize_type(owner_type: str) -> str:
|
||
return {
|
||
"characters": "human", "character": "human", "human": "human", "humans": "human",
|
||
"scene": "scene", "scenes": "scene", "voice": "voice", "voices": "voice",
|
||
}.get(owner_type, owner_type)
|
||
|
||
|
||
def get_owner(db: Session, owner_type: str, owner_id: int):
|
||
owner_type = normalize_type(owner_type)
|
||
if owner_type == "human":
|
||
row = db.get(PlatformCharacter, owner_id)
|
||
elif owner_type == "scene":
|
||
row = db.get(PlatformScene, owner_id)
|
||
elif owner_type == "voice":
|
||
row = db.get(PlatformVoice, owner_id)
|
||
else:
|
||
raise HTTPException(400, f"不支持的平台资产类型:{owner_type}")
|
||
if row is None:
|
||
raise HTTPException(404, "平台资产不存在")
|
||
return row
|
||
|
||
|
||
def type_label(owner_type: str) -> str:
|
||
return {"human": "人物", "scene": "场景", "voice": "声线"}.get(owner_type, owner_type)
|
||
|
||
|
||
def resolve_reference_media(db: Session, owner_type: str, owner_id: int, ids: list[int],
|
||
*, kind: str) -> list[PlatformMedia]:
|
||
"""校验平台生成参考素材只能来自当前资产,避免跨资产误读本地路径。"""
|
||
rows: list[PlatformMedia] = []
|
||
seen: set[int] = set()
|
||
image_suffixes = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff"}
|
||
audio_suffixes = {".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".opus"}
|
||
for media_id in ids:
|
||
if media_id in seen:
|
||
continue
|
||
seen.add(media_id)
|
||
row = db.get(PlatformMedia, media_id)
|
||
if (row is None or row.owner_type != owner_type or row.owner_id != owner_id):
|
||
raise HTTPException(400, "参考素材不属于当前平台资产")
|
||
if not storage.exists(row.path):
|
||
raise HTTPException(400, "参考素材文件不存在,请重新上传")
|
||
suffix = Path(row.path).suffix.lower()
|
||
if kind == "image" and suffix not in image_suffixes:
|
||
raise HTTPException(400, "图像参考素材必须是图片文件")
|
||
if kind == "audio" and suffix not in audio_suffixes:
|
||
raise HTTPException(400, "音色参考素材必须是音频文件")
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def build_owner_prompt(project: Project | None, owner_type: str, owner, kind: str, custom: str) -> str:
|
||
if custom.strip():
|
||
return custom.strip()
|
||
bits = []
|
||
if owner_type == "human":
|
||
bits = [f"{owner.name},电影级角色设定图", owner.appearance, owner.persona_prompt,
|
||
"五官、发型、体型、服装与配饰保持一致,正面清晰,材质细节可信"]
|
||
elif owner_type == "scene":
|
||
bits = [f"{owner.name},电影级场景概念图", owner.prompt or owner.description,
|
||
owner.lighting, owner.time_of_day, owner.palette,
|
||
"空间结构清晰,透视准确,建筑与陈设保持一致"]
|
||
if project:
|
||
bits.append(project.style_guide)
|
||
return "。".join(str(part).strip() for part in bits if part)
|
||
|
||
|
||
def resolve_spec(db: Session, owner_type: str, data: schemas.PlatformGenerate):
|
||
"""按资产类型解析这次生成实际会派给哪个模型(留空即跟随平台默认)。"""
|
||
if owner_type == "voice":
|
||
spec, _adapter = model_registry.resolve_voice(db, data.provider or None, data.model or None)
|
||
else:
|
||
spec, _adapter = model_registry.resolve_asset(db, "image",
|
||
data.provider or None, data.model or None)
|
||
return spec
|
||
|
||
|
||
def resolve_voice_reference(db: Session, owner, spec,
|
||
data: schemas.PlatformGenerate) -> tuple[PlatformMedia | None, dict]:
|
||
"""把「这次生成会拿到谁的嗓子」算出来:参考音频决定音色,表单里的性别只是元数据。"""
|
||
reference_ids = list(data.reference_media_ids)
|
||
if data.reference_media_id:
|
||
reference_ids.append(data.reference_media_id)
|
||
references = resolve_reference_media(db, "voice", owner.id, reference_ids, kind="audio")
|
||
reference_media = references[0] if references else None
|
||
reference = timbre.resolve_reference(
|
||
db, owner,
|
||
explicit=str(storage.to_absolute(reference_media.path)) if reference_media else "",
|
||
clone=timbre.clone_model(spec), instruct=timbre.instruct_model(spec))
|
||
return reference_media, reference
|
||
|
||
|
||
# ---------------------------------------------------------------- 同步校验
|
||
def preflight(db: Session, owner_type: str, owner, data: schemas.PlatformGenerate):
|
||
"""开工前能当场说清的错误,绝不留到五分钟以后。
|
||
|
||
异步化之后接口立刻返回,如果连「没填试听文本」都要等轮询才发现,
|
||
用户面对的是一个跑完了才知道白跑的进度条。所以这些**便宜且确定**的校验
|
||
留在请求里同步做(422/409 原样吐给表单),耗时部分才交给后台。
|
||
后台执行时会把同样的校验再跑一遍:排到 GPU 时模型可能已被停用、
|
||
参考音频可能已被删掉,以及时报错的是供应商而不是表单。
|
||
"""
|
||
spec = resolve_spec(db, owner_type, data)
|
||
if not spec.enabled:
|
||
raise HTTPException(409, f"模型 {spec.provider}/{spec.model} 已停用")
|
||
if owner_type != "voice":
|
||
return spec
|
||
if not (data.sample_text or "").strip():
|
||
raise HTTPException(422, "请先填写试听文本:它是模型要朗读的台词,"
|
||
"系统不会用「音色描述」代替")
|
||
# 嗓子来源必须当场说清:克隆模型缺参考音频时会静默回退到模型默认参考(女声),
|
||
# 男声声线就这样被生成成了女声——等四十秒后试听才发现,等于白跑一轮。
|
||
_media, reference = resolve_voice_reference(db, owner, spec, data)
|
||
if not reference["path"] and reference.get("source") == "none":
|
||
raise HTTPException(409, reference["note"])
|
||
return spec
|
||
|
||
|
||
# ---------------------------------------------------------------- 登记与执行
|
||
def active_task(db: Session, owner_type: str, owner_id: int, asset_kind: str) -> Task | None:
|
||
"""这条母版的这一类素材是否已经有生成任务在跑(挡连点:三次点击只该出一张图)。"""
|
||
rows = db.scalars(select(Task).where(
|
||
Task.owner_type == owner_type, Task.owner_id == owner_id,
|
||
Task.kind == TaskKind.PLATFORM_ASSET, Task.status.in_(TaskStatus.ACTIVE)
|
||
).order_by(Task.id)).all()
|
||
for row in rows:
|
||
if str((row.payload_json or {}).get("asset_kind") or "") == asset_kind:
|
||
return row
|
||
return None
|
||
|
||
|
||
def enqueue(db: Session, *, owner_type: str, owner, spec, data: schemas.PlatformGenerate,
|
||
asset_kind: str) -> Task:
|
||
"""登记一条平台生成任务并立刻尝试派发;返回任务行(202 的响应体就是它)。"""
|
||
running = active_task(db, owner_type, owner.id, asset_kind)
|
||
if running is not None:
|
||
raise HTTPException(409, f"「{owner.name}」的{AssetKind.LABELS.get(asset_kind, asset_kind)}"
|
||
f"正在生成中(任务 #{running.id}),完成前不能重复发起")
|
||
label = f"生成「{owner.name}」的{AssetKind.LABELS.get(asset_kind, asset_kind)}"[:MAX_LABEL_CHARS]
|
||
cost = (costing.estimate_voice_cost(spec, len((data.sample_text or "").strip()))
|
||
if owner_type == "voice" else 0.0)
|
||
task = Task(
|
||
project_id=None, shot_id=None, kind=TaskKind.PLATFORM_ASSET,
|
||
name=label, stage="platform_generate", status=TaskStatus.QUEUED,
|
||
priority=5, provider=spec.provider, model=spec.model,
|
||
mode="voice" if owner_type == "voice" else data.kind,
|
||
owner_type=owner_type, owner_id=owner.id,
|
||
# 同一母版同一素材类型的重复点击靠这条键挡,不给它配项目侧那套镜头幂等语义
|
||
idempotency_key=f"platform:{owner_type}:{owner.id}:{asset_kind}:{int(now_utc().timestamp())}",
|
||
# 只留一次自动重试:连不上算力这类瞬时故障值得再试一把(生成失败没有产物,
|
||
# 不存在重复扣费);参数与配置错误在 generate 里标了 retryable=False,当场判死。
|
||
max_attempts=2, payload_json=_payload(owner_type, owner, asset_kind, data),
|
||
cost=cost, queued_at=now_utc(), message="排队中:等待调度")
|
||
db.add(task)
|
||
db.flush()
|
||
events.log_event(db, "task.queued", project_id=None, task_id=task.id,
|
||
message=f"平台{type_label(owner_type)}「{owner.name}」{label}",
|
||
entity_type="task", entity_id=task.id,
|
||
data={"owner_type": owner_type, "owner_id": owner.id, "asset_kind": asset_kind,
|
||
"provider": spec.provider, "model": spec.model})
|
||
db.commit()
|
||
result = dispatch.dispatch_task(db, task)
|
||
log.info("平台生成任务 #%s 已登记:%s(派发 %s)", task.id, label,
|
||
"立即开始" if result["dispatched"] else f"排队等待,{result['reason']}")
|
||
return db.get(Task, task.id)
|
||
|
||
|
||
def _payload(owner_type: str, owner, asset_kind: str,
|
||
data: schemas.PlatformGenerate) -> dict:
|
||
"""任务参数:请求体原样收在 ``request`` 里,外层放调度与展示要用的字段。
|
||
|
||
``owner_name`` 只是给顶栏「平台动态」的「查看」用的:那条深链按名字搜资产库,
|
||
而 ``task.name`` 是「生成「俊杰」的定妆图」这样的一句话,拿它当搜索词永远 0 条。
|
||
存名字而不是回表查,是为了不给 ``task_dict`` 加一次查询——它全站每条任务都要过。
|
||
"""
|
||
return {"operation": "generate", "owner_type": owner_type, "owner_id": owner.id,
|
||
"owner_name": getattr(owner, "name", ""),
|
||
"asset_kind": asset_kind, "request": data.model_dump()}
|
||
|
||
|
||
def request_from(task: Task) -> tuple[str, int, str, schemas.PlatformGenerate]:
|
||
"""从任务行还原「当时点的是什么」(执行体只拿得到 task_id)。"""
|
||
payload = task.payload_json or {}
|
||
raw = dict(payload.get("request") or {})
|
||
# 老任务行可能带着外层多写的字段,Strict 模型会当场拒绝——只留请求体认识的键
|
||
fields = set(schemas.PlatformGenerate.model_fields)
|
||
data = schemas.PlatformGenerate(**{k: v for k, v in raw.items() if k in fields})
|
||
return (str(payload.get("owner_type") or ""), int(payload.get("owner_id") or 0),
|
||
str(payload.get("asset_kind") or data.kind), data)
|
||
|
||
|
||
def produced_media_metadata(media, reference: dict, check: dict, text: str) -> None:
|
||
"""把「这次到底用了谁的嗓子、念了什么、音色对不对」写进素材,界面才说得清楚。"""
|
||
metadata = dict(media.metadata_json or {})
|
||
metadata.update({
|
||
"text": text,
|
||
"reference_audio": reference.get("path") or reference.get("attached") or "",
|
||
"reference_source": reference.get("source") or "",
|
||
"reference_note": reference.get("note") or "使用本声线已绑定的参考音频",
|
||
"timbre_check": check,
|
||
})
|
||
media.metadata_json = metadata
|
||
note = str(media.note or "")
|
||
tail = check.get("note") or ""
|
||
if tail and "音色自检" not in note:
|
||
media.note = f"{note}|音色自检:{tail}".strip("|")
|
||
|
||
|
||
def generate(db: Session, task: Task) -> dict:
|
||
"""真正跑一次平台资产生成并落库;失败一律抛 ``ProviderError`` 交给任务层归类。"""
|
||
owner_type, owner_id, asset_kind, data = request_from(task)
|
||
owner = get_owner(db, owner_type, owner_id)
|
||
progress.report("准备生成参数")
|
||
if owner_type == "voice":
|
||
result = _generate_voice(db, owner, data, task)
|
||
else:
|
||
result = _generate_image(db, owner_type, owner, data, task)
|
||
db.commit()
|
||
events.log_event(db, "platform.asset.generated", project_id=None, task_id=task.id,
|
||
message=f"平台{type_label(owner_type)}「{owner.name}」已生成"
|
||
f"{AssetKind.LABELS.get(asset_kind, asset_kind)}"
|
||
f"({result['provider']}/{result['model']})",
|
||
entity_type=f"platform_{owner_type}", entity_id=owner.id,
|
||
data={"media_id": result["media"].get("id"), "task_id": task.id})
|
||
db.commit()
|
||
return result
|
||
|
||
|
||
def _config_error(message: str, code: str) -> ProviderError:
|
||
"""参数/配置类错误:重试一百次也是同一个结果,直接判死并把原因说清。"""
|
||
return ProviderError(message, code=code, retryable=False)
|
||
|
||
|
||
def _generate_image(db: Session, owner_type: str, owner,
|
||
data: schemas.PlatformGenerate, task: Task) -> dict:
|
||
spec = resolve_spec(db, owner_type, data)
|
||
if not spec.enabled:
|
||
raise _config_error(f"模型 {spec.provider}/{spec.model} 已停用", "model_disabled")
|
||
references = resolve_reference_media(db, owner_type, owner.id, data.reference_media_ids,
|
||
kind="image")
|
||
prompt = build_owner_prompt(None, owner_type, owner, data.kind, data.prompt)
|
||
out = storage.platform_dir(owner_type, owner.id, "generated") / f"{data.kind}.png"
|
||
request = AssetGenRequest(
|
||
kind=data.kind, prompt=prompt, negative_prompt=data.negative_prompt,
|
||
width=data.width, height=data.height, seed=data.seed,
|
||
reference_images=[str(storage.to_absolute(row.path)) for row in references],
|
||
params={"model": spec.model},
|
||
idempotency_key=(
|
||
f"platform-image:{owner.id}:{data.kind}:"
|
||
f"{now_utc().isoformat()}:{data.prompt}:{','.join(map(str, data.reference_media_ids))}"
|
||
),
|
||
label=f"PLATFORM-{owner_type.upper()}-{owner.id}", output_path=out,
|
||
)
|
||
progress.report(f"调用 {spec.provider}/{spec.model} 出图")
|
||
result = model_registry.generate_asset(db, request, "image", spec.provider, spec.model)
|
||
produced = _require_file(result, "图像")
|
||
kind = data.kind
|
||
info = _store(produced, owner_type=owner_type, owner_id=owner.id, kind=kind,
|
||
stem=f"{owner_type}-{owner.id}-{kind}-ai", result=result)
|
||
media = storage.register_platform_media(
|
||
db, owner_type=owner_type, owner_id=owner.id, kind=kind, path=info["path"],
|
||
name=f"{owner.name} · AI {AssetKind.LABELS.get(kind, kind)}",
|
||
note=f"平台资产生成:{spec.provider}/{spec.model}", source="provider", info=info,
|
||
metadata={"generated": True, "provider": spec.provider, "model": spec.model,
|
||
"prompt": request.prompt,
|
||
"reference_media_ids": [row.id for row in references],
|
||
"task_id": task.id},
|
||
)
|
||
return {"media": storage.platform_media_dict(media), "kind": kind,
|
||
"provider": spec.provider, "model": spec.model}
|
||
|
||
|
||
def _generate_voice(db: Session, owner, data: schemas.PlatformGenerate, task: Task) -> dict:
|
||
spec = resolve_spec(db, "voice", data)
|
||
if not spec.enabled:
|
||
raise _config_error(f"模型 {spec.provider}/{spec.model} 已停用", "model_disabled")
|
||
# 试听文本是「模型要念出来的台词」,不是声线设定。旧实现拿 owner.description 兜底,
|
||
# 于是用户填的「温和、语速中等,沉稳」被当成台词念了出来,试听样本一听就是设定说明文。
|
||
text = (data.sample_text or "").strip()
|
||
if not text:
|
||
raise _config_error("请先填写试听文本:它是模型要朗读的台词,系统不会用「音色描述」代替",
|
||
"missing_sample_text")
|
||
reference_media, reference = resolve_voice_reference(db, owner, spec, data)
|
||
# 旧实现缺参考时直接回退到模型默认参考(女声),男声声线就这样被生成成了女声。
|
||
if not reference["path"] and reference.get("source") == "none":
|
||
raise _config_error(reference["note"], "missing_reference_audio")
|
||
out = storage.platform_dir("voice", owner.id, "generated") / f"sample-{now_utc().timestamp():.0f}.wav"
|
||
# 交给模型的是绝对路径(供应商要直接读文件);声线列里存 media_root 相对路径,
|
||
# 换机器 / 进容器时不会因为写死绝对路径而失效。与任务链路 handlers 保持一致。
|
||
reference_path = reference["path"] or owner.provider_voice_id
|
||
request = VoiceGenRequest(
|
||
text=text, voice_id=owner.name, provider_voice_id=reference_path,
|
||
language=owner.language, speed=float((owner.tuning or {}).get("speed", 1.0)),
|
||
seed=data.seed, params=(owner.tuning or {}).get("params") or {},
|
||
# 没有参考音频时靠它出音,有参考音频时用它微调音色(见 comfy_assets.graph)
|
||
instruct_text=timbre.instruct_text_of(owner) if timbre.instruct_model(spec) else "",
|
||
idempotency_key=f"platform-voice:{owner.id}:{task.id}",
|
||
label=f"PLATFORM-VOICE-{owner.id}", output_path=out,
|
||
)
|
||
progress.report(f"调用 {spec.provider}/{spec.model} 出试听")
|
||
result = model_registry.generate_voice(db, request, spec.provider, spec.model)
|
||
produced = _require_file(result, "音频")
|
||
info = _store(produced, owner_type="voice", owner_id=owner.id, kind="voice_sample",
|
||
stem=f"voice-{owner.id}-ai", result=result)
|
||
media = storage.register_platform_media(
|
||
db, owner_type="voice", owner_id=owner.id, kind="voice_sample", path=info["path"],
|
||
name=f"{owner.name} · AI {AssetKind.LABELS.get('voice_sample', 'voice_sample')}",
|
||
note=f"平台资产生成:{spec.provider}/{spec.model}", source="provider", info=info,
|
||
metadata={"generated": True, "provider": spec.provider, "model": spec.model,
|
||
"prompt": request.text,
|
||
"reference_media_ids": [reference_media.id] if reference_media else [],
|
||
"task_id": task.id},
|
||
)
|
||
owner.provider = spec.provider
|
||
if reference["path"] and timbre.clone_model(spec):
|
||
# 把真正参与克隆的那段参考音频绑回声线:否则试听样本是 A 的嗓子,
|
||
# 之后项目配音又落回模型默认参考,变成另一种嗓子。
|
||
owner.provider_voice_id = str(storage.to_relative(reference["path"]))
|
||
# 音色自检:量一遍实际基频,与设定性别比对,结果进素材元数据供界面标注。
|
||
check = timbre.check_voice(produced, owner.gender)
|
||
produced_media_metadata(media, reference, check, text)
|
||
return {"media": storage.platform_media_dict(media), "kind": "voice_sample",
|
||
"provider": spec.provider, "model": spec.model}
|
||
|
||
|
||
def _require_file(result, what: str) -> Path:
|
||
produced = Path(result.file_path)
|
||
if not produced.exists() or produced.stat().st_size < 128:
|
||
raise ProviderError(f"模型未返回有效{what}文件", code="missing_output")
|
||
return produced
|
||
|
||
|
||
def _store(produced: Path, *, owner_type: str, owner_id: int, kind: str, stem: str,
|
||
result) -> dict:
|
||
info = storage.save_platform_local_file(
|
||
produced, owner_type=owner_type, owner_id=owner_id, kind=kind, stem=stem)
|
||
if getattr(result, "duration_ms", 0):
|
||
info["duration_ms"] = int(result.duration_ms)
|
||
return info
|
||
|
||
|
||
__all__ = ["active_task", "build_owner_prompt", "enqueue", "generate", "get_owner",
|
||
"normalize_type", "preflight", "produced_media_metadata", "request_from",
|
||
"resolve_reference_media", "resolve_spec", "resolve_voice_reference", "type_label"]
|