nex_animation/backend/app/services/storage.py

556 lines
25 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""素材仓库:上传校验、相对路径存储、资产登记与版本化。"""
from __future__ import annotations
import hashlib
import mimetypes
import shutil
import uuid
from datetime import timezone
from pathlib import Path
from fastapi import HTTPException, UploadFile
from sqlalchemy import func
from app.config import settings
from app.constants import AssetKind
from app.models import Asset, AssetVersion, PlatformMedia, PlatformMediaVersion, Shot, now_utc
from app.services import ffmpeg as ff
ALLOWED_KINDS = set(AssetKind.LABELS)
def media_root() -> Path:
root = Path(settings.media_root).resolve()
root.mkdir(parents=True, exist_ok=True)
return root
def to_relative(path: str | Path) -> str:
"""统一保存为相对 media_root 的路径,便于迁移与容器挂载。"""
p = Path(path)
if not p.is_absolute():
return str(p)
try:
return str(p.resolve().relative_to(media_root()))
except ValueError:
return str(p)
def to_absolute(path: str | Path) -> Path:
p = Path(path)
return p if p.is_absolute() else (media_root() / p)
def exists(path: str | Path | None) -> bool:
return bool(path) and to_absolute(path).exists()
def delete_media_file(path: str | Path | None) -> bool:
"""删除媒体库内的单个文件;拒绝越界路径,数据库记录由调用方负责。"""
if not path:
return False
target = to_absolute(path)
try:
target.resolve().relative_to(media_root())
except ValueError:
return False
if not target.is_file():
return False
target.unlink()
from app.services.thumbs import drop as _drop_thumbs # 局部导入:thumbs 依赖 storage
_drop_thumbs(target)
return True
def url_for(path: str | Path | None) -> str:
if not path:
return ""
return f"{settings.media_url_prefix}/{to_relative(path).lstrip('/')}"
def media_mime(parent) -> str:
"""版本行缺少 mime 时的兜底:先取父行的,再按扩展名猜。"""
import mimetypes
return getattr(parent, "mime", "") or mimetypes.guess_type(str(getattr(parent, "path", "")))[0] or ""
def version_dict(version, current_version=None) -> dict:
"""版本行的对外结构;`exists` 供前端判断该版本能否设为当前版本。"""
return {"version_no": version.version_no, "path": version.path, "url": url_for(version.path),
"source": version.source, "note": version.note, "checksum": version.checksum,
"size_bytes": version.size_bytes, "mime": getattr(version, "mime", "") or "",
"width": getattr(version, "width", None), "height": getattr(version, "height", None),
"duration_ms": getattr(version, "duration_ms", None),
"exists": exists(version.path),
"current": current_version is not None and int(version.version_no) == int(current_version),
"created_at": version.created_at.isoformat() if version.created_at else None}
def project_dir(project_id: int, *parts: str) -> Path:
folder = media_root() / str(project_id) / Path(*parts) if parts else media_root() / str(project_id)
folder.mkdir(parents=True, exist_ok=True)
return folder
def platform_dir(owner_type: str, owner_id: int, *parts: str) -> Path:
folder = media_root() / "platform" / f"{owner_type}-{owner_id}"
if parts:
folder = folder.joinpath(*parts)
folder.mkdir(parents=True, exist_ok=True)
return folder
def _group_for_kind(kind: str) -> str:
if kind in AssetKind.IMAGE_KINDS:
return "image"
if kind in AssetKind.AUDIO_KINDS:
return "audio"
if kind in AssetKind.VIDEO_KINDS:
return "video"
return "doc"
async def save_upload(file: UploadFile, *, project_id: int, kind: str, stem: str = "file",
media_group: str = "") -> dict:
suffix = Path(file.filename or "upload.bin").suffix.lower()
allowed = settings.allowed_extensions(media_group or _group_for_kind(kind))
if suffix not in allowed:
raise HTTPException(400, f"{AssetKind.LABELS.get(kind, kind)} 不支持 {suffix or '无扩展名'} 文件;"
f"允许:{', '.join(allowed)}")
folder = project_dir(project_id, kind)
safe_stem = "".join(c for c in (stem or "file") if c.isalnum() or c in "-_")[:60] or "file"
dest = folder / f"{safe_stem}_{uuid.uuid4().hex[:10]}{suffix}"
size = 0
digest = hashlib.sha256()
with dest.open("wb") as out:
while True:
chunk = await file.read(1 << 20)
if not chunk:
break
size += len(chunk)
if size > 2 * 1024 * 1024 * 1024:
out.close()
dest.unlink(missing_ok=True)
raise HTTPException(413, "单个文件不得超过 2GB")
digest.update(chunk)
out.write(chunk)
if size == 0:
dest.unlink(missing_ok=True)
raise HTTPException(400, "上传文件为空")
info = {"path": to_relative(dest), "size_bytes": size, "checksum": digest.hexdigest()[:32],
"mime": file.content_type or "", "original_name": file.filename or dest.name}
if ff.available():
try:
probe = ff.probe(dest)
info.update(width=probe.width or None, height=probe.height or None,
duration_ms=int(probe.duration * 1000) if probe.duration else None)
except Exception:
pass
return info
async def save_platform_upload(file: UploadFile, *, owner_type: str, owner_id: int, kind: str,
stem: str = "asset", media_group: str = "") -> dict:
suffix = Path(file.filename or "upload.bin").suffix.lower()
allowed = settings.allowed_extensions(media_group or _group_for_kind(kind))
if suffix not in allowed:
raise HTTPException(400, f"{AssetKind.LABELS.get(kind, kind)} 不支持 {suffix or '无扩展名'} 文件;"
f"允许:{', '.join(allowed)}")
safe = "".join(c for c in (stem or "asset") if c.isalnum() or c in "-_")[:60] or "asset"
dest = platform_dir(owner_type, owner_id, kind) / f"{safe}_{uuid.uuid4().hex[:10]}{suffix}"
size = 0
digest = hashlib.sha256()
with dest.open("wb") as out:
while True:
chunk = await file.read(1 << 20)
if not chunk:
break
size += len(chunk)
if size > 2 * 1024 * 1024 * 1024:
out.close()
dest.unlink(missing_ok=True)
raise HTTPException(413, "单个文件不得超过 2GB")
digest.update(chunk)
out.write(chunk)
if size == 0:
dest.unlink(missing_ok=True)
raise HTTPException(400, "上传文件为空")
info = {"path": to_relative(dest), "size_bytes": size, "checksum": digest.hexdigest()[:32],
"mime": file.content_type or "", "original_name": file.filename or dest.name}
if ff.available():
try:
probe = ff.probe(dest)
info.update(width=probe.width or None, height=probe.height or None,
duration_ms=int(probe.duration * 1000) if probe.duration else None)
except Exception:
pass
return info
def save_local_file(src: str | Path, *, project_id: int, kind: str, stem: str) -> dict:
src = Path(src)
folder = project_dir(project_id, kind)
dest = folder / f"{stem[:60]}_{uuid.uuid4().hex[:8]}{src.suffix.lower()}"
shutil.copy2(src, dest)
info = {"path": to_relative(dest), "size_bytes": dest.stat().st_size, "original_name": src.name}
if ff.available():
try:
probe = ff.probe(dest)
info.update(width=probe.width or None, height=probe.height or None,
duration_ms=int(probe.duration * 1000) if probe.duration else None)
except Exception:
pass
return info
def save_platform_local_file(src: str | Path, *, owner_type: str, owner_id: int, kind: str,
stem: str) -> dict:
src = Path(src)
dest = platform_dir(owner_type, owner_id, kind) / f"{stem[:60]}_{uuid.uuid4().hex[:8]}{src.suffix.lower()}"
shutil.copy2(src, dest)
info = {"path": to_relative(dest), "size_bytes": dest.stat().st_size, "original_name": src.name}
if ff.available():
try:
probe = ff.probe(dest)
info.update(width=probe.width or None, height=probe.height or None,
duration_ms=int(probe.duration * 1000) if probe.duration else None)
except Exception:
pass
return info
def sniff_suffix(payload: bytes, *, kind: str = "") -> str:
"""从字节头判断常见素材格式,用于压缩包/内联图片导入。"""
head = payload[:16]
if head.startswith(b"\x89PNG"):
return ".png"
if head.startswith(b"\xff\xd8\xff"):
return ".jpg"
if head.startswith(b"GIF8"):
return ".gif"
if head[:4] == b"RIFF" and payload[8:12] == b"WEBP":
return ".webp"
if head.startswith(b"BM"):
return ".bmp"
if head.startswith(b"II*\x00") or head.startswith(b"MM\x00*"):
return ".tiff"
if payload[4:8] == b"ftyp":
return ".mp4"
if head.startswith(b"OggS"):
return ".ogg"
if head.startswith(b"fLaC"):
return ".flac"
if head.startswith(b"ID3") or head[:2] in (b"\xff\xfb", b"\xff\xf3", b"\xff\xf2"):
return ".mp3"
if head.startswith(b"RIFF") and payload[8:12] == b"WAVE":
return ".wav"
return ".png" if kind in AssetKind.IMAGE_KINDS else (".wav" if kind in AssetKind.AUDIO_KINDS else ".bin")
def save_bytes(payload: bytes, *, project_id: int, kind: str, stem: str = "file",
suffix: str = "", original_name: str = "") -> dict:
"""保存内存中的字节(脚本内联图片、zip 解包结果等)到素材仓库。"""
if not payload:
raise HTTPException(400, "文件内容为空")
if len(payload) > 2 * 1024 * 1024 * 1024:
raise HTTPException(413, "单个文件不得超过 2GB")
suffix = (suffix or "").lower() or sniff_suffix(payload, kind=kind)
folder = project_dir(project_id, kind)
safe = "".join(c for c in (stem or "file") if c.isalnum() or c in "-_")[:60] or "file"
dest = folder / f"{safe}_{uuid.uuid4().hex[:10]}{suffix}"
dest.write_bytes(payload)
info = {"path": to_relative(dest), "size_bytes": len(payload),
"checksum": hashlib.sha256(payload).hexdigest()[:32],
"original_name": original_name or dest.name}
if ff.available():
try:
probe = ff.probe(dest)
info.update(width=probe.width or None, height=probe.height or None,
duration_ms=int(probe.duration * 1000) if probe.duration else None,
mime=probe.mime if hasattr(probe, "mime") else None)
except Exception:
pass
return info
def _max_version_no(db, version_model, owner_column, owner_id: int) -> int:
"""版本子表里该父行已占用的最大版本号(无记录时返回 0)。
`assets.id` / `platform_media.id` 都是 `INTEGER PRIMARY KEY` 且没有 AUTOINCREMENT,
删掉最大行后 SQLite 会**复用**同一个 id;再叠加历史上母版批量删除没走 ORM 级联、
在 `*_versions` 里留下的孤儿行,登记时无条件写 `version_no=1`(或
`current_version + 1`)都可能撞上 `(父 id, version_no)` 唯一约束而抛
IntegrityError(如 `uq_platform_media_version`)。所以版本号一律按子表现状推导,
遇到主键复用或脏历史也能自愈,而不是让接口 500。
"""
current = db.query(func.max(version_model.version_no)).filter(owner_column == owner_id).scalar()
return int(current or 0)
def register_asset(db, *, project_id: int, owner_type: str, owner_id: int, kind: str, path: str,
name: str = "", note: str = "", source: str = "upload", info: dict | None = None,
metadata: dict | None = None, replace: bool = True) -> Asset:
"""登记资产;同名同类型资产再次上传时递增版本并保留历史。"""
info = info or {}
stmt_kind = kind
existing = None
if replace:
existing = db.query(Asset).filter(Asset.project_id == project_id, Asset.owner_type == owner_type,
Asset.owner_id == owner_id, Asset.kind == stmt_kind,
Asset.name == (name or "")).order_by(Asset.id.desc()).first()
if existing:
# 取「子表实际最大值」与「父行 current_version」的较大者再 +1:正常数据两者相等,
# 行为与旧实现一致;子表落后/超前(孤儿行、主键复用)时也不会写重复版本号。
next_version = max(_max_version_no(db, AssetVersion, AssetVersion.asset_id, existing.id),
int(existing.current_version or 0)) + 1
db.add(AssetVersion(asset_id=existing.id, version_no=next_version, path=path,
checksum=info.get("checksum", ""), source=source, note=note or "",
size_bytes=info.get("size_bytes", 0), mime=info.get("mime", "") or "",
width=info.get("width"), height=info.get("height"),
duration_ms=info.get("duration_ms"), created_at=now_utc()))
existing.path = path
existing.checksum = info.get("checksum", existing.checksum)
existing.size_bytes = info.get("size_bytes", existing.size_bytes)
existing.width = info.get("width") or existing.width
existing.height = info.get("height") or existing.height
existing.duration_ms = info.get("duration_ms") or existing.duration_ms
existing.current_version = next_version
existing.note = note or existing.note
if metadata:
existing.metadata_json = {**(existing.metadata_json or {}), **metadata}
db.flush()
# expire_on_commit=False 会让已加载过的 versions 关系一直缓存,
# 不显式过期的话上传响应会回吐「新增版本之前」的历史列表。
db.expire(existing, ["versions"])
return existing
asset = Asset(project_id=project_id, owner_type=owner_type, owner_id=owner_id, kind=kind,
name=name or AssetKind.LABELS.get(kind, kind), note=note or "", path=path,
checksum=info.get("checksum", ""), mime=info.get("mime", ""),
size_bytes=info.get("size_bytes", 0), width=info.get("width"), height=info.get("height"),
duration_ms=info.get("duration_ms"), current_version=1,
metadata_json={"original_name": info.get("original_name", ""), **(metadata or {})})
db.add(asset)
db.flush()
# 新资产也可能是「被复用的旧主键」:子表里残留的孤儿版本行同样要避开。
asset.current_version = _max_version_no(db, AssetVersion, AssetVersion.asset_id, asset.id) + 1
db.add(AssetVersion(asset_id=asset.id, version_no=asset.current_version, path=path,
checksum=asset.checksum, source=source,
note=note or "", size_bytes=asset.size_bytes, mime=info.get("mime", "") or media_mime(asset),
width=info.get("width"), height=info.get("height"),
duration_ms=info.get("duration_ms"), created_at=now_utc()))
db.flush()
db.expire(asset, ["versions"])
return asset
def assets_of(db, owner_type: str, owner_id: int, kind: str | None = None) -> list[Asset]:
stmt = db.query(Asset).filter(Asset.owner_type == owner_type, Asset.owner_id == owner_id)
if kind:
stmt = stmt.filter(Asset.kind == kind)
return stmt.order_by(Asset.id).all()
def as_utc(value):
"""SQLite 读回来的 datetime 是 naive UTC,内存里的对象却是 aware —— 统一后再比较。
(直接 `a >= b` 会抛 TypeError: can't compare offset-naive and offset-aware datetimes。)
"""
if value is None:
return None
return value if value.tzinfo else value.replace(tzinfo=timezone.utc)
_OWNER_MODELS: dict = {}
def _owner_models() -> dict:
"""宿主类型 → 模型类。延迟导入,避免和 app.models 形成循环依赖。"""
global _OWNER_MODELS
if not _OWNER_MODELS:
from app.models import Character, Scene, Shot, VoiceAsset
_OWNER_MODELS.update({"shot": Shot, "human": Character, "scene": Scene, "voice": VoiceAsset})
return _OWNER_MODELS
def belongs_to_owner(db, owner_type: str, owner_id: int, asset: Asset) -> bool:
"""资产是否真的属于当前宿主。
SQLite 的主键会被复用:删掉镜头后新建的镜头可能拿到同一个 id,于是上一轮
的关键帧/配音看起来「还在」,预检与提示词都按旧画面走,成片却是新剧情。
比宿主镜头还老的镜头级资产一律视为脏数据。
"""
model = _owner_models().get(owner_type)
if model is None:
return True
owner = db.get(model, owner_id)
stamp = asset_born(asset)
if owner is None or stamp is None or not getattr(owner, "created_at", None):
return True
return stamp >= as_utc(owner.created_at)
def asset_born(asset: Asset):
"""素材最后一次被写入的时间:重新导入会复用同一行、只推进 updated_at。
只用 created_at 会把「刚刚替新宿主重写过的素材」误判成上一任留下的脏数据。
"""
stamps = [as_utc(value) for value in (asset.created_at, getattr(asset, "updated_at", None)) if value]
return max(stamps) if stamps else None
def purge_owner_assets(db, owner_type: str, owner_id: int) -> int:
"""清掉某个对象名下的全部素材记录与媒体文件,返回删除的媒体文件数。
人物/场景/声线的素材都按 ``owner_type + owner_id`` 挂载,删掉宿主却留着素材的话,
主键一旦被新对象复用,新对象就会静默继承上一任的定妆图、场景母图和音色文件
(项目 9「紫小慧」拿到「林秀云」的嗓子就是这么来的)。
"""
removed = 0
for asset in list(assets_of(db, owner_type, owner_id)):
for version in list(asset.versions):
if version.path and delete_media_file(version.path):
removed += 1
if asset.path and delete_media_file(asset.path):
removed += 1
db.delete(asset)
db.flush()
return removed
def purge_shot_assets(db, shot) -> int:
"""删除镜头时清理它名下的素材与产物文件,返回删除的媒体文件数。
镜头级资产(关键帧/尾帧/参考图/配音)和成片文件都跟着镜头走,
否则主键被复用后新镜头会继承上一轮的脏产物。
"""
removed = 0
for path in (shot.output_path, shot.voiceover_path):
if path and delete_media_file(path):
removed += 1
return removed + purge_owner_assets(db, "shot", shot.id)
def first_asset_row(db, owner_type: str, owner_id: int, kind: str, *,
newest_first: bool = True) -> Asset | None:
"""取某宿主某一种类的可用资产行(当前版本文件真实存在)。
默认取最新一条:重复生成/重新上传的资产可能并存多行,「人物现在长什么样」
由最新的母版决定,而不是最早那次试验产物。
"""
stmt = db.query(Asset).filter(Asset.owner_type == owner_type, Asset.owner_id == owner_id,
Asset.kind == kind)
stmt = stmt.order_by(Asset.id.desc() if newest_first else Asset.id.asc())
for asset in stmt.all():
if exists(asset.path) and belongs_to_owner(db, owner_type, owner_id, asset):
return asset
return None
def first_asset(db, owner_type: str, owner_id: int, *kinds: str) -> Asset | None:
"""宿主当前真正在用的素材行(调用方需要看 metadata 时用这个,不只是路径)。"""
for kind in kinds:
stmt = (db.query(Asset).filter(Asset.owner_type == owner_type, Asset.owner_id == owner_id,
Asset.kind == kind).order_by(Asset.id))
for asset in stmt.all():
if exists(asset.path) and belongs_to_owner(db, owner_type, owner_id, asset):
return asset
return None
def first_asset_path(db, owner_type: str, owner_id: int, *kinds: str) -> str | None:
asset = first_asset(db, owner_type, owner_id, *kinds)
return to_relative(asset.path) if asset is not None else None
def asset_dict(asset: Asset) -> dict:
return {"id": asset.id, "kind": asset.kind, "kind_label": AssetKind.LABELS.get(asset.kind, asset.kind),
"name": asset.name, "note": asset.note, "path": asset.path, "url": url_for(asset.path),
"version": asset.current_version, "checksum": asset.checksum, "size_bytes": asset.size_bytes,
"width": asset.width, "height": asset.height, "duration_ms": asset.duration_ms,
"owner_type": asset.owner_type, "owner_id": asset.owner_id, "scope": "project",
"exists": exists(asset.path),
"metadata": asset.metadata_json or {},
"versions": [version_dict(v, asset.current_version) for v in asset.versions]}
def register_platform_media(db, *, owner_type: str, owner_id: int, kind: str, path: str,
name: str = "", note: str = "", source: str = "upload",
info: dict | None = None, metadata: dict | None = None,
replace: bool = True) -> PlatformMedia:
"""登记平台媒体;同 owner + kind + name 再次写入时递增版本。"""
info = info or {}
media = None
if replace:
media = db.query(PlatformMedia).filter(
PlatformMedia.owner_type == owner_type, PlatformMedia.owner_id == owner_id,
PlatformMedia.kind == kind, PlatformMedia.name == (name or ""),
).order_by(PlatformMedia.id.desc()).first()
if media:
next_version = max(_max_version_no(db, PlatformMediaVersion, PlatformMediaVersion.media_id, media.id),
int(media.current_version or 0)) + 1
db.add(PlatformMediaVersion(
media_id=media.id, version_no=next_version, path=path,
checksum=info.get("checksum", ""), source=source, note=note or "",
size_bytes=info.get("size_bytes", 0), mime=info.get("mime", "") or "",
width=info.get("width"), height=info.get("height"),
duration_ms=info.get("duration_ms"), created_at=now_utc(),
))
media.path = path
media.checksum = info.get("checksum", media.checksum)
media.size_bytes = info.get("size_bytes", media.size_bytes)
media.width = info.get("width") or media.width
media.height = info.get("height") or media.height
media.duration_ms = info.get("duration_ms") or media.duration_ms
media.current_version = next_version
media.note = note or media.note
if metadata:
media.metadata_json = {**(media.metadata_json or {}), **metadata}
db.flush()
db.expire(media, ["versions"])
return media
media = PlatformMedia(
owner_type=owner_type, owner_id=owner_id, kind=kind,
name=name or AssetKind.LABELS.get(kind, kind), note=note or "", path=path,
checksum=info.get("checksum", ""), mime=info.get("mime", ""),
size_bytes=info.get("size_bytes", 0), width=info.get("width"),
height=info.get("height"), duration_ms=info.get("duration_ms"),
current_version=1,
metadata_json={"original_name": info.get("original_name", ""), **(metadata or {})},
)
db.add(media)
db.flush()
# 同上:母版删除泄漏过版本行,而 SQLite 会复用被删掉的最大主键,
# 所以首版号也不能硬编码 1(历史上这里就是 UNIQUE 约束 500 的现场)。
media.current_version = _max_version_no(db, PlatformMediaVersion, PlatformMediaVersion.media_id,
media.id) + 1
db.add(PlatformMediaVersion(
media_id=media.id, version_no=media.current_version, path=path, checksum=media.checksum,
source=source, note=note or "", size_bytes=media.size_bytes,
mime=info.get("mime", "") or media.mime, width=info.get("width"), height=info.get("height"),
duration_ms=info.get("duration_ms"), created_at=now_utc(),
))
db.flush()
db.expire(media, ["versions"])
return media
def platform_media_dict(media: PlatformMedia) -> dict:
return {
"id": media.id, "kind": media.kind,
"kind_label": AssetKind.LABELS.get(media.kind, media.kind),
"name": media.name, "note": media.note, "path": media.path, "url": url_for(media.path),
"version": media.current_version, "checksum": media.checksum,
"size_bytes": media.size_bytes, "width": media.width, "height": media.height,
"duration_ms": media.duration_ms, "metadata": media.metadata_json or {},
"scope": "platform", "exists": exists(media.path),
"versions": [version_dict(version, media.current_version) for version in media.versions],
}
def cleanup_project_files(project_id: int) -> None:
folder = media_root() / str(project_id)
if folder.exists():
shutil.rmtree(folder, ignore_errors=True)