362 lines
14 KiB
Python
362 lines
14 KiB
Python
"""视频/语音/图像/文本模型适配层协议。
|
||
|
||
各家供应商协议差异很大,因此统一收敛到两个抽象:
|
||
- VideoProviderAdapter.generate() 文生视频 / 图生视频 / 首尾帧
|
||
- VoiceProviderAdapter.synthesize() 台词配音
|
||
- ImageProviderAdapter.generate() 角色定妆图 / 场景图 / 关键帧等图像资产
|
||
- TextProviderAdapter.generate() 角色设定 / 场景提示词 / 分镜文案
|
||
|
||
所有适配器必须区分「可重试错误」(限流、超时、5xx)与「致命错误」(参数非法、内容审核不通过),
|
||
调度中心据此决定重试或终止。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from abc import ABC, abstractmethod
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
class ProviderError(Exception):
|
||
"""供应商错误基类。retryable=True 时调度中心会按退避策略重试。"""
|
||
|
||
code = "provider_error"
|
||
retryable = True
|
||
|
||
def __init__(self, message: str, *, code: str | None = None, retryable: bool | None = None,
|
||
detail: Any = None):
|
||
super().__init__(message)
|
||
self.message = message
|
||
if code:
|
||
self.code = code
|
||
if retryable is not None:
|
||
self.retryable = retryable
|
||
self.detail = detail
|
||
|
||
def as_dict(self) -> dict:
|
||
return {"code": self.code, "message": self.message, "retryable": self.retryable,
|
||
"detail": _safe(self.detail)}
|
||
|
||
|
||
class RetryableProviderError(ProviderError):
|
||
code = "provider_retryable"
|
||
retryable = True
|
||
|
||
|
||
class RateLimitedError(ProviderError):
|
||
code = "rate_limited"
|
||
retryable = True
|
||
|
||
def __init__(self, message: str = "供应商限流", *, retry_after: float | None = None, **kw):
|
||
super().__init__(message, **kw)
|
||
self.retry_after = retry_after
|
||
|
||
|
||
class FatalProviderError(ProviderError):
|
||
code = "provider_fatal"
|
||
retryable = False
|
||
|
||
|
||
class ProviderTimeoutError(RetryableProviderError):
|
||
code = "provider_timeout"
|
||
|
||
|
||
def _safe(value: Any) -> Any:
|
||
try:
|
||
import json
|
||
|
||
json.dumps(value)
|
||
return value
|
||
except Exception:
|
||
return str(value)
|
||
|
||
|
||
@dataclass
|
||
class VideoGenRequest:
|
||
prompt: str
|
||
duration_seconds: float
|
||
aspect_ratio: str = "16:9"
|
||
resolution: str = "1080p"
|
||
fps: int = 24
|
||
mode: str = "i2v" # t2v / i2v / fl
|
||
first_frame: str | None = None # 关键帧(本地路径)
|
||
last_frame: str | None = None
|
||
reference_images: list[str] = field(default_factory=list) # 定妆图/三视图/场景图
|
||
# 用户在镜头里显式选择的参考图。与自动带入的人物/场景图分开记录,
|
||
# 以便 i2v 适配器在“首帧优先”和“参考图一致性”之间做明确取舍。
|
||
explicit_reference_images: list[str] = field(default_factory=list)
|
||
# 参考音频(角色声线的音色样本):MiniMax H3 这类原生音画同步模型可以
|
||
# 「照着这段嗓子的音色说话」,于是原声人声既对得上口型、又不是模型随机挑的陌生人。
|
||
reference_audios: list[str] = field(default_factory=list)
|
||
negative_prompt: str = ""
|
||
seed: int | None = None
|
||
params: dict = field(default_factory=dict)
|
||
idempotency_key: str = ""
|
||
label: str = "" # 如 shot-12,便于在供应商侧检索
|
||
output_path: Path | None = None
|
||
|
||
|
||
@dataclass
|
||
class VideoGenResult:
|
||
file_path: str
|
||
provider_job_id: str = ""
|
||
duration_seconds: float = 0.0
|
||
cost: float = 0.0
|
||
metadata: dict = field(default_factory=dict)
|
||
|
||
|
||
@dataclass
|
||
class VoiceGenRequest:
|
||
text: str
|
||
voice_id: str = ""
|
||
provider_voice_id: str = ""
|
||
language: str = "zh-CN"
|
||
speed: float = 1.0
|
||
pitch: float = 1.0
|
||
volume: float = 1.0
|
||
sample_rate: int = 24000
|
||
seed: int | None = None
|
||
params: dict = field(default_factory=dict)
|
||
idempotency_key: str = ""
|
||
label: str = ""
|
||
output_path: Path | None = None
|
||
|
||
|
||
@dataclass
|
||
class VoiceGenResult:
|
||
file_path: str
|
||
duration_ms: int = 0
|
||
provider_job_id: str = ""
|
||
cost: float = 0.0
|
||
metadata: dict = field(default_factory=dict)
|
||
|
||
|
||
@dataclass
|
||
class AssetGenRequest:
|
||
"""图像或文本资产生成请求。kind 为 image / text。"""
|
||
|
||
kind: str
|
||
prompt: str
|
||
negative_prompt: str = ""
|
||
width: int = 1024
|
||
height: int = 1024
|
||
seed: int | None = None
|
||
reference_images: list[str] = field(default_factory=list)
|
||
params: dict = field(default_factory=dict)
|
||
idempotency_key: str = ""
|
||
label: str = ""
|
||
output_path: Path | None = None
|
||
|
||
|
||
@dataclass
|
||
class AssetGenResult:
|
||
file_path: str = ""
|
||
text: str = ""
|
||
provider_job_id: str = ""
|
||
cost: float = 0.0
|
||
metadata: dict = field(default_factory=dict)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class DurationGrid:
|
||
"""模型只接受特定帧数的「时长网格」。
|
||
|
||
扩散视频模型按时间块分块,帧数常常必须落在 ``step*k + offset`` 上。
|
||
MiniMax H3:24fps、step=17、offset=5 ⇒ 合法档位是 5, 22, 39 … 90, 107, 124, 141 …
|
||
相邻两档相差 0.708s。
|
||
|
||
这个网格必须由平台自己算,不能留给 provider 悄悄对齐:
|
||
联合音视频模型的**音轨和画面一样长**,秒数先报大了再对齐,多出来的部分
|
||
不会变成静音,而是被模型用「念提示词里的画面描述」填满
|
||
(「告诉你个小秘密第 2 期」S03/S04 开头那两句乱语就是这么来的)。
|
||
"""
|
||
|
||
fps: int = 24
|
||
step: int = 1
|
||
offset: int = 0
|
||
min_frames: int = 1
|
||
|
||
def frames_of(self, seconds: float) -> int:
|
||
"""秒 → **不小于**它的第一个合法帧数。"""
|
||
step, offset = int(self.step) or 1, int(self.offset)
|
||
frames = max(int(self.min_frames) or 1, int(round(float(seconds) * float(self.fps))))
|
||
if step > 1:
|
||
frames += (offset - frames % step) % step
|
||
return frames
|
||
|
||
def seconds_of(self, frames: int) -> float:
|
||
return round(int(frames) / float(self.fps), 4)
|
||
|
||
def floor_frames(self, floor: float) -> int:
|
||
"""下限秒数对齐到网格后的那一档。"""
|
||
return self.frames_of(max(0.0, float(floor or 0)))
|
||
|
||
def snap(self, seconds: float, *, floor: float = 0.0) -> float:
|
||
"""把期望时长落到**离它最近**的合法档位(默认不是向上取整)。
|
||
|
||
为什么取最近:合法档位之间相差 0.708s,一律向上取整最多留下 0.708s 空档,
|
||
而 H3 的空档一定会被语音填掉;取最近把偏差压到 ±0.354s。
|
||
略微不够时模型自己会提语速念完——实测 15 字台词在 3.75s 档里
|
||
压到 5.3 字/秒仍字字清晰,比留 0.5s 空档换来的一句「都市清新风格」划算。
|
||
"""
|
||
low = self.floor_frames(floor)
|
||
want = max(float(seconds), self.seconds_of(low))
|
||
up = max(self.frames_of(want), low)
|
||
down = up - (int(self.step) or 1)
|
||
if down >= low and abs(self.seconds_of(down) - want) < abs(self.seconds_of(up) - want):
|
||
return self.seconds_of(down)
|
||
return self.seconds_of(up)
|
||
|
||
|
||
@dataclass
|
||
class Capabilities:
|
||
modes: tuple[str, ...] = ("t2v",)
|
||
resolutions: tuple[str, ...] = ("720p", "1080p")
|
||
aspect_ratios: tuple[str, ...] = ("16:9",)
|
||
min_duration: float = 1.0
|
||
max_duration: float = 10.0
|
||
max_reference_images: int = 0
|
||
# 支持多少个「参考音频」音色槽位(MiniMax H3 r2v = 3)。
|
||
# 0 表示该模型不吃音色参考,台词只能走平台 TTS 配音。
|
||
max_reference_audios: int = 0
|
||
needs_public_media: bool = False # True 表示供应商无法读取本地路径
|
||
vision: bool = False # 能不能看图(多模态输入,画面质检依赖它)
|
||
# 联合音视频模型(MiniMax H3 这类):一次前向同时出画面和音轨。
|
||
# 这类模型把整段提示词既当**画面脚本**又当**声音脚本**——凡是句子里出现
|
||
# 「说话/台词/为准/音画不同步」这类**关于声音与下指令**的说法,它就去念给你听
|
||
# (项目 11 实测:镜头开头那串听不懂的乱语正是提示词里的返工要求被念了出来)。
|
||
# 提示词组装因此要走一条独立的「纯视觉 caption」分支,见 services/prompt.py。
|
||
joint_av: bool = False
|
||
# 「能生成」和「能交付」是两个不同的分辨率集合。原生画布到 768p 的模型
|
||
# (MiniMax H3)照样可以把成片交付成 1080p:模型按原生像素出片,
|
||
# 合成端用 lanczos 放大到交付档位。这一档声明的就是后者——
|
||
# 选它**不会**多花卡时,只是画质来自放大而不是原生。
|
||
upscale_to: tuple[str, ...] = ()
|
||
# 时长网格(默认 step=1 ⇒ 任意秒数都合法)。joint_av 模型必须声明真实网格,
|
||
# 平台才能把「台词说完要几秒」落到合法帧数上,见 :class:`DurationGrid`。
|
||
duration_grid: DurationGrid = field(default_factory=DurationGrid)
|
||
# **带台词镜头**可用时长下限。它通常低于 min_duration:min_duration 表达的是
|
||
# 「模型的训练舒适区/建议区间」,而联合音视频模型多留一秒就要多念一句废话,
|
||
# 于是「宁可低于舒适区」也要贴住台词。0 表示沿用 min_duration。
|
||
joint_min_duration: float = 0.0
|
||
notes: str = ""
|
||
|
||
@property
|
||
def delivery_resolutions(self) -> tuple[str, ...]:
|
||
"""用户能选的交付档位 = 原生档位 ∪ 放大交付档位(保持从低到高的顺序)。"""
|
||
order = ["360p", "480p", "720p", "1080p", "2k", "4k"]
|
||
allowed = set(self.resolutions) | set(self.upscale_to)
|
||
ranked = [r for r in order if r in allowed]
|
||
extra = [r for r in dict.fromkeys(list(self.resolutions) + list(self.upscale_to))
|
||
if r not in order]
|
||
return tuple(ranked + extra)
|
||
|
||
def as_dict(self) -> dict:
|
||
return {"modes": list(self.modes), "resolutions": list(self.resolutions),
|
||
"aspect_ratios": list(self.aspect_ratios), "min_duration": self.min_duration,
|
||
"max_duration": self.max_duration, "max_reference_images": self.max_reference_images,
|
||
"max_reference_audios": self.max_reference_audios,
|
||
"needs_public_media": self.needs_public_media, "vision": self.vision,
|
||
"joint_av": self.joint_av, "upscale_to": list(self.upscale_to),
|
||
"duration_grid": {"fps": self.duration_grid.fps, "step": self.duration_grid.step,
|
||
"offset": self.duration_grid.offset,
|
||
"min_frames": self.duration_grid.min_frames},
|
||
"joint_min_duration": self.joint_min_duration,
|
||
"delivery_resolutions": list(self.delivery_resolutions),
|
||
"notes": self.notes}
|
||
|
||
|
||
class VideoProviderAdapter(ABC):
|
||
media_type = "video"
|
||
|
||
@property
|
||
@abstractmethod
|
||
def name(self) -> str: ...
|
||
|
||
def capabilities(self, config: dict | None = None) -> Capabilities:
|
||
return Capabilities()
|
||
|
||
def is_available(self, config: dict | None = None) -> bool:
|
||
"""是否可用:默认恒真;HTTP 类适配器会检查地址与密钥。config 为模型注册表配置。"""
|
||
return True
|
||
|
||
def prepare_media(self, local_path: str, config: dict | None = None) -> str:
|
||
"""把本地素材转换为供应商可访问的地址(对象存储/签名 URL)。默认原样返回。"""
|
||
return local_path
|
||
|
||
def effective_canvas(self, config: dict | None = None,
|
||
aspect_ratio: str = "16:9",
|
||
resolution: str | None = None) -> tuple[int, int] | None:
|
||
"""该适配器在此配置下真正交付的画面尺寸(宽, 高)。
|
||
|
||
resolution 传项目档位时返回「按档位抬升后」的真实画布;
|
||
不传则只看配置本身,用于预检判断配置是否低于档位。
|
||
|
||
返回 None 表示画面尺寸完全由供应商按声明档位决定(云端 API 常见)。
|
||
本地 ComfyUI 一类「按像素数算画布」的适配器必须实现它:
|
||
档位声明写着 720p,配置里的 megapixels 却只够 480p,
|
||
出片会静默降档,只在画面上变糊,接口与预检都不报错。
|
||
"""
|
||
return None
|
||
|
||
@abstractmethod
|
||
def generate(self, request: VideoGenRequest, config: dict | None = None) -> VideoGenResult: ...
|
||
|
||
|
||
class VoiceProviderAdapter(ABC):
|
||
media_type = "tts"
|
||
|
||
@property
|
||
@abstractmethod
|
||
def name(self) -> str: ...
|
||
|
||
def is_available(self, config: dict | None = None) -> bool:
|
||
return True
|
||
|
||
def capabilities(self, config: dict | None = None) -> Capabilities:
|
||
return Capabilities(modes=("voice",), resolutions=(), aspect_ratios=(),
|
||
min_duration=0, max_duration=0, max_reference_images=1,
|
||
notes="台词配音与音色样本生成")
|
||
|
||
@abstractmethod
|
||
def synthesize(self, request: VoiceGenRequest, config: dict | None = None) -> VoiceGenResult: ...
|
||
|
||
|
||
class ImageProviderAdapter(ABC):
|
||
media_type = "image"
|
||
|
||
@property
|
||
@abstractmethod
|
||
def name(self) -> str: ...
|
||
|
||
def capabilities(self, config: dict | None = None) -> Capabilities:
|
||
return Capabilities(modes=("t2i", "edit"), resolutions=("512p", "1024p"),
|
||
aspect_ratios=("1:1", "16:9", "9:16", "4:3"), min_duration=0,
|
||
max_duration=0, max_reference_images=4,
|
||
notes="图像资产生成:定妆图、三视图、服装、表情与场景图")
|
||
|
||
def is_available(self, config: dict | None = None) -> bool:
|
||
return True
|
||
|
||
@abstractmethod
|
||
def generate(self, request: AssetGenRequest, config: dict | None = None) -> AssetGenResult: ...
|
||
|
||
|
||
class TextProviderAdapter(ABC):
|
||
media_type = "text"
|
||
|
||
@property
|
||
@abstractmethod
|
||
def name(self) -> str: ...
|
||
|
||
def capabilities(self, config: dict | None = None) -> Capabilities:
|
||
return Capabilities(modes=("chat", "prompt"), resolutions=(), aspect_ratios=(),
|
||
min_duration=0, max_duration=0, max_reference_images=0,
|
||
notes="文本资产生成:角色设定、场景提示词、分镜文案")
|
||
|
||
def is_available(self, config: dict | None = None) -> bool:
|
||
return True
|
||
|
||
@abstractmethod
|
||
def generate(self, request: AssetGenRequest, config: dict | None = None) -> AssetGenResult: ...
|