nex_animation/backend/app/providers/base.py

365 lines
14 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""视频/语音/图像/文本模型适配层协议。
各家供应商协议差异很大,因此统一收敛到两个抽象:
- VideoProviderAdapter.generate() 文生视频 / 图生视频 / 首尾帧
- VoiceProviderAdapter.synthesize() 台词配音
- ImageProviderAdapter.generate() 角色定妆图 / 场景图 / 关键帧等图像资产
- TextProviderAdapter.generate() 角色设定 / 场景提示词 / 分镜文案
所有适配器必须区分「可重试错误」(限流、超时、5xx)与「致命错误」(参数非法、内容审核不通过),
调度中心据此决定重试或终止。
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
class ProviderError(Exception):
"""供应商错误基类。retryable=True 时调度中心会按退避策略重试。"""
code = "provider_error"
retryable = True
def __init__(self, message: str, *, code: str | None = None, retryable: bool | None = None,
detail: Any = None):
super().__init__(message)
self.message = message
if code:
self.code = code
if retryable is not None:
self.retryable = retryable
self.detail = detail
def as_dict(self) -> dict:
return {"code": self.code, "message": self.message, "retryable": self.retryable,
"detail": _safe(self.detail)}
class RetryableProviderError(ProviderError):
code = "provider_retryable"
retryable = True
class RateLimitedError(ProviderError):
code = "rate_limited"
retryable = True
def __init__(self, message: str = "供应商限流", *, retry_after: float | None = None, **kw):
super().__init__(message, **kw)
self.retry_after = retry_after
class FatalProviderError(ProviderError):
code = "provider_fatal"
retryable = False
class ProviderTimeoutError(RetryableProviderError):
code = "provider_timeout"
def _safe(value: Any) -> Any:
try:
import json
json.dumps(value)
return value
except Exception:
return str(value)
@dataclass
class VideoGenRequest:
prompt: str
duration_seconds: float
aspect_ratio: str = "16:9"
resolution: str = "1080p"
fps: int = 24
mode: str = "i2v" # t2v / i2v / fl
first_frame: str | None = None # 关键帧(本地路径)
last_frame: str | None = None
reference_images: list[str] = field(default_factory=list) # 定妆图/三视图/场景图
# 用户在镜头里显式选择的参考图。与自动带入的人物/场景图分开记录,
# 以便 i2v 适配器在“首帧优先”和“参考图一致性”之间做明确取舍。
explicit_reference_images: list[str] = field(default_factory=list)
# 参考音频(角色声线的音色样本):MiniMax H3 这类原生音画同步模型可以
# 「照着这段嗓子的音色说话」,于是原声人声既对得上口型、又不是模型随机挑的陌生人。
reference_audios: list[str] = field(default_factory=list)
negative_prompt: str = ""
seed: int | None = None
params: dict = field(default_factory=dict)
idempotency_key: str = ""
label: str = "" # 如 shot-12,便于在供应商侧检索
output_path: Path | None = None
@dataclass
class VideoGenResult:
file_path: str
provider_job_id: str = ""
duration_seconds: float = 0.0
cost: float = 0.0
metadata: dict = field(default_factory=dict)
@dataclass
class VoiceGenRequest:
text: str
voice_id: str = ""
provider_voice_id: str = ""
language: str = "zh-CN"
speed: float = 1.0
pitch: float = 1.0
volume: float = 1.0
sample_rate: int = 24000
seed: int | None = None
params: dict = field(default_factory=dict)
idempotency_key: str = ""
label: str = ""
output_path: Path | None = None
#: 音色/情绪描述(声线的「音色描述」)。支持 voice_instruct 的模型在
#: **没有参考音频**时靠它出音;有参考音频时用它做音色偏移与情绪控制。
instruct_text: str = ""
@dataclass
class VoiceGenResult:
file_path: str
duration_ms: int = 0
provider_job_id: str = ""
cost: float = 0.0
metadata: dict = field(default_factory=dict)
@dataclass
class AssetGenRequest:
"""图像或文本资产生成请求。kind 为 image / text。"""
kind: str
prompt: str
negative_prompt: str = ""
width: int = 1024
height: int = 1024
seed: int | None = None
reference_images: list[str] = field(default_factory=list)
params: dict = field(default_factory=dict)
idempotency_key: str = ""
label: str = ""
output_path: Path | None = None
@dataclass
class AssetGenResult:
file_path: str = ""
text: str = ""
provider_job_id: str = ""
cost: float = 0.0
metadata: dict = field(default_factory=dict)
@dataclass(frozen=True)
class DurationGrid:
"""模型只接受特定帧数的「时长网格」。
扩散视频模型按时间块分块,帧数常常必须落在 ``step*k + offset`` 上。
MiniMax H3:24fps、step=17、offset=5 ⇒ 合法档位是 5, 22, 39 … 90, 107, 124, 141 …
相邻两档相差 0.708s。
这个网格必须由平台自己算,不能留给 provider 悄悄对齐:
联合音视频模型的**音轨和画面一样长**,秒数先报大了再对齐,多出来的部分
不会变成静音,而是被模型用「念提示词里的画面描述」填满
(「告诉你个小秘密第 2 期」S03/S04 开头那两句乱语就是这么来的)。
"""
fps: int = 24
step: int = 1
offset: int = 0
min_frames: int = 1
def frames_of(self, seconds: float) -> int:
"""秒 → **不小于**它的第一个合法帧数。"""
step, offset = int(self.step) or 1, int(self.offset)
frames = max(int(self.min_frames) or 1, int(round(float(seconds) * float(self.fps))))
if step > 1:
frames += (offset - frames % step) % step
return frames
def seconds_of(self, frames: int) -> float:
return round(int(frames) / float(self.fps), 4)
def floor_frames(self, floor: float) -> int:
"""下限秒数对齐到网格后的那一档。"""
return self.frames_of(max(0.0, float(floor or 0)))
def snap(self, seconds: float, *, floor: float = 0.0) -> float:
"""把期望时长落到**离它最近**的合法档位(默认不是向上取整)。
为什么取最近:合法档位之间相差 0.708s,一律向上取整最多留下 0.708s 空档,
而 H3 的空档一定会被语音填掉;取最近把偏差压到 ±0.354s。
略微不够时模型自己会提语速念完——实测 15 字台词在 3.75s 档里
压到 5.3 字/秒仍字字清晰,比留 0.5s 空档换来的一句「都市清新风格」划算。
"""
low = self.floor_frames(floor)
want = max(float(seconds), self.seconds_of(low))
up = max(self.frames_of(want), low)
down = up - (int(self.step) or 1)
if down >= low and abs(self.seconds_of(down) - want) < abs(self.seconds_of(up) - want):
return self.seconds_of(down)
return self.seconds_of(up)
@dataclass
class Capabilities:
modes: tuple[str, ...] = ("t2v",)
resolutions: tuple[str, ...] = ("720p", "1080p")
aspect_ratios: tuple[str, ...] = ("16:9",)
min_duration: float = 1.0
max_duration: float = 10.0
max_reference_images: int = 0
# 支持多少个「参考音频」音色槽位(MiniMax H3 r2v = 3)。
# 0 表示该模型不吃音色参考,台词只能走平台 TTS 配音。
max_reference_audios: int = 0
needs_public_media: bool = False # True 表示供应商无法读取本地路径
vision: bool = False # 能不能看图(多模态输入,画面质检依赖它)
# 联合音视频模型(MiniMax H3 这类):一次前向同时出画面和音轨。
# 这类模型把整段提示词既当**画面脚本**又当**声音脚本**——凡是句子里出现
# 「说话/台词/为准/音画不同步」这类**关于声音与下指令**的说法,它就去念给你听
# (项目 11 实测:镜头开头那串听不懂的乱语正是提示词里的返工要求被念了出来)。
# 提示词组装因此要走一条独立的「纯视觉 caption」分支,见 services/prompt.py。
joint_av: bool = False
# 「能生成」和「能交付」是两个不同的分辨率集合。原生画布到 768p 的模型
# (MiniMax H3)照样可以把成片交付成 1080p:模型按原生像素出片,
# 合成端用 lanczos 放大到交付档位。这一档声明的就是后者——
# 选它**不会**多花卡时,只是画质来自放大而不是原生。
upscale_to: tuple[str, ...] = ()
# 时长网格(默认 step=1 ⇒ 任意秒数都合法)。joint_av 模型必须声明真实网格,
# 平台才能把「台词说完要几秒」落到合法帧数上,见 :class:`DurationGrid`。
duration_grid: DurationGrid = field(default_factory=DurationGrid)
# **带台词镜头**可用时长下限。它通常低于 min_duration:min_duration 表达的是
# 「模型的训练舒适区/建议区间」,而联合音视频模型多留一秒就要多念一句废话,
# 于是「宁可低于舒适区」也要贴住台词。0 表示沿用 min_duration。
joint_min_duration: float = 0.0
notes: str = ""
@property
def delivery_resolutions(self) -> tuple[str, ...]:
"""用户能选的交付档位 = 原生档位 ∪ 放大交付档位(保持从低到高的顺序)。"""
order = ["360p", "480p", "720p", "1080p", "2k", "4k"]
allowed = set(self.resolutions) | set(self.upscale_to)
ranked = [r for r in order if r in allowed]
extra = [r for r in dict.fromkeys(list(self.resolutions) + list(self.upscale_to))
if r not in order]
return tuple(ranked + extra)
def as_dict(self) -> dict:
return {"modes": list(self.modes), "resolutions": list(self.resolutions),
"aspect_ratios": list(self.aspect_ratios), "min_duration": self.min_duration,
"max_duration": self.max_duration, "max_reference_images": self.max_reference_images,
"max_reference_audios": self.max_reference_audios,
"needs_public_media": self.needs_public_media, "vision": self.vision,
"joint_av": self.joint_av, "upscale_to": list(self.upscale_to),
"duration_grid": {"fps": self.duration_grid.fps, "step": self.duration_grid.step,
"offset": self.duration_grid.offset,
"min_frames": self.duration_grid.min_frames},
"joint_min_duration": self.joint_min_duration,
"delivery_resolutions": list(self.delivery_resolutions),
"notes": self.notes}
class VideoProviderAdapter(ABC):
media_type = "video"
@property
@abstractmethod
def name(self) -> str: ...
def capabilities(self, config: dict | None = None) -> Capabilities:
return Capabilities()
def is_available(self, config: dict | None = None) -> bool:
"""是否可用:默认恒真;HTTP 类适配器会检查地址与密钥。config 为模型注册表配置。"""
return True
def prepare_media(self, local_path: str, config: dict | None = None) -> str:
"""把本地素材转换为供应商可访问的地址(对象存储/签名 URL)。默认原样返回。"""
return local_path
def effective_canvas(self, config: dict | None = None,
aspect_ratio: str = "16:9",
resolution: str | None = None) -> tuple[int, int] | None:
"""该适配器在此配置下真正交付的画面尺寸(宽, 高)。
resolution 传项目档位时返回「按档位抬升后」的真实画布;
不传则只看配置本身,用于预检判断配置是否低于档位。
返回 None 表示画面尺寸完全由供应商按声明档位决定(云端 API 常见)。
本地 ComfyUI 一类「按像素数算画布」的适配器必须实现它:
档位声明写着 720p,配置里的 megapixels 却只够 480p,
出片会静默降档,只在画面上变糊,接口与预检都不报错。
"""
return None
@abstractmethod
def generate(self, request: VideoGenRequest, config: dict | None = None) -> VideoGenResult: ...
class VoiceProviderAdapter(ABC):
media_type = "tts"
@property
@abstractmethod
def name(self) -> str: ...
def is_available(self, config: dict | None = None) -> bool:
return True
def capabilities(self, config: dict | None = None) -> Capabilities:
return Capabilities(modes=("voice",), resolutions=(), aspect_ratios=(),
min_duration=0, max_duration=0, max_reference_images=1,
notes="台词配音与音色样本生成")
@abstractmethod
def synthesize(self, request: VoiceGenRequest, config: dict | None = None) -> VoiceGenResult: ...
class ImageProviderAdapter(ABC):
media_type = "image"
@property
@abstractmethod
def name(self) -> str: ...
def capabilities(self, config: dict | None = None) -> Capabilities:
return Capabilities(modes=("t2i", "edit"), resolutions=("512p", "1024p"),
aspect_ratios=("1:1", "16:9", "9:16", "4:3"), min_duration=0,
max_duration=0, max_reference_images=4,
notes="图像资产生成:定妆图、三视图、服装、表情与场景图")
def is_available(self, config: dict | None = None) -> bool:
return True
@abstractmethod
def generate(self, request: AssetGenRequest, config: dict | None = None) -> AssetGenResult: ...
class TextProviderAdapter(ABC):
media_type = "text"
@property
@abstractmethod
def name(self) -> str: ...
def capabilities(self, config: dict | None = None) -> Capabilities:
return Capabilities(modes=("chat", "prompt"), resolutions=(), aspect_ratios=(),
min_duration=0, max_duration=0, max_reference_images=0,
notes="文本资产生成:角色设定、场景提示词、分镜文案")
def is_available(self, config: dict | None = None) -> bool:
return True
@abstractmethod
def generate(self, request: AssetGenRequest, config: dict | None = None) -> AssetGenResult: ...