nex_animation/backend/app/services/thumbs.py

275 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""列表页缩略图:惰性生成 + 磁盘缓存。
【为什么要这一层】
资产库 / 分镜 / 审片页此前把**原图**当缩略图渲染:一张定妆 PNG 就有 1.5~2MB(960×1280),
人物页首屏 12 张就是 19MB,三个人物/场景/声线页合计 26MB,浏览器要先下载再解码 12 张
1.2MP 位图,页面才会「转圈很久」。缩略成 384px webp 后单张约 20KB,首屏 <1MB。
【设计约束】
* 惰性:只在第一次被请求时生成,绝不做全量预构建(媒体库几百 GB 也扛得住);
* 缓存键含源文件 mtime+size,AI 重新生成同名覆盖后缩略图自动失效;
* 缩略图**永远不是正确性依赖**:没有 ffmpeg、文件损坏、越界路径,一律回落原图或 404,
绝不因为出图失败把列表页打成 500;
* 缓存放进 media_root 里的 `.thumbs/`,跟着媒体卷走(容器重建不丢),
且不会出现在任何业务列表里(后端不递归扫盘,静态目录以 . 开头也不会被 URL 猜中)。
"""
from __future__ import annotations
import hashlib
import logging
import os
import subprocess
import threading
import time
from pathlib import Path
from app.config import settings
from app.services import ffmpeg as ff
from app.services.storage import media_root, to_relative, url_for
logger = logging.getLogger("nex.thumbs")
CACHE_DIR = ".thumbs"
# 只有这些扩展名值得出缩略图;svg(矢量,ffmpeg 不认)与音频/文档直接排除在外
IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff", ".gif", ".heic"}
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".mkv", ".m4v", ".avi"}
# 量化档位:前端各尺寸向上取整到这些值,保证同一个 URL 能被全平台复用
WIDTHS = (64, 128, 192, 256, 384, 512, 768, 1024, 1536)
DEFAULT_WIDTH = 384
# 生成失败后短期内不再重试(坏文件反复起 ffmpeg 会把接口拖死)
FAIL_TTL = 120.0
TIMEOUT = 25
MEDIA_TYPES = {".webp": "image/webp", ".jpg": "image/jpeg", ".png": "image/png"}
_OUTPUTS = (".webp", ".jpg")
_locks: dict[str, threading.Lock] = {}
_locks_guard = threading.Lock()
_failed: dict[str, float] = {}
# 上次成功的输出编码器(webp / jpg):避免每张图片都先拿不可用的编码器白跑一次
_preferred: str | None = None
def normalize_width(width: int | str | None) -> int:
"""把任意请求宽度向上取整到最近的档位(提高缓存命中率、限制变体数量)。"""
try:
want = int(width)
except (TypeError, ValueError):
return DEFAULT_WIDTH
if want <= 0:
return DEFAULT_WIDTH
for step in WIDTHS:
if want <= step:
return step
return WIDTHS[-1]
def is_thumbable(path: str | Path | None) -> bool:
if not path:
return False
suffix = Path(str(path).split("?")[0]).suffix.lower()
return suffix in IMAGE_EXTS or suffix in VIDEO_EXTS
def resolve(path: str | Path | None) -> Path | None:
"""把「绝对路径 / 相对 media_root 的路径 / `/media/...` 网址」统一解析成媒体库内的真实文件。
越界(`../` 出逃、指向库外绝对路径)与不存在都返回 None:
这条接口是浏览器可以随便拼参数的入口,必须自己守住目录边界。
"""
if not path:
return None
root = media_root()
raw = str(path).strip()
prefix = settings.media_url_prefix.strip("/")
if prefix:
for form in (f"/{prefix}/", f"{prefix}/"):
if raw.startswith(form):
raw = raw[len(form):] # 前端有时传的是整条 /media/... 网址
break
attempts: list[Path] = []
absolute = Path(raw)
if absolute.is_absolute():
attempts.append(absolute)
attempts.append(root / raw.lstrip("/"))
for probe in attempts:
try:
fixed = probe.resolve()
fixed.relative_to(root) # 抛出 ValueError = 逃出媒体库
except (OSError, ValueError):
continue
if fixed.is_file():
return fixed
return None
def cache_url(path: str | Path | None, width: int | str | None = DEFAULT_WIDTH) -> str:
"""给素材字典/前端用的缩略图地址;不值得缩略或文件缺失时返回空串。"""
target = resolve(path)
if target is None or not is_thumbable(target):
return ""
rel = to_relative(target).lstrip("/")
return f"/api/media/thumb/{normalize_width(width)}/{rel}"
def _key(target: Path, width: int) -> str:
stat = target.stat()
rel = to_relative(target).replace("\\", "/")
digest = hashlib.sha1(rel.encode("utf-8")).hexdigest()[:24]
return f"{digest}-{int(stat.st_mtime)}-{stat.st_size}-w{width}"
def _cache_dir() -> Path:
folder = media_root() / CACHE_DIR
folder.mkdir(parents=True, exist_ok=True)
return folder
def _existing(stem: str) -> Path | None:
folder = _cache_dir()
for suffix in _OUTPUTS:
hit = folder / f"{stem}{suffix}"
if hit.is_file() and hit.stat().st_size > 0:
return hit
return None
def _lock_for(stem: str) -> threading.Lock:
with _locks_guard:
lock = _locks.get(stem)
if lock is None:
lock = _locks[stem] = threading.Lock()
return lock
def _recently_failed(stem: str) -> bool:
expiry = _failed.get(stem, 0.0)
if expiry and expiry > time.time():
return True
_failed.pop(stem, None)
return False
def _mark_failed(stem: str) -> None:
if len(_failed) > 512: # 防止坏文件把内存表撑爆
_failed.clear()
_failed[stem] = time.time() + FAIL_TTL
def _scale_filter(width: int) -> str:
# decrease = 只缩不放;force_divisible_by 保证编码器不抱怨奇数尺寸
return f"scale={width}:{width}:force_original_aspect_ratio=decrease:force_divisible_by=2"
def _ffmpeg_plans(target: Path, width: int) -> list[tuple[str, list[str], list[str]]]:
"""返回候选方案:(输出后缀, -i 之前的参数, -i 之后的参数)。
优先用上次在这台机器上真的成功的编码器(`_preferred`),省得每张图都先拿
缺失的编码器白跑一次;视频先跳 0.3 秒取帧(首帧常常是全黑),
极短视频没有帧时再去掉 -ss 重取首帧。
"""
scale = ["-frames:v", "1", "-vf", _scale_filter(width), "-an"]
codecs: list[tuple[str, list[str]]] = []
if ff.can_encode("libwebp"):
codecs.append((".webp", ["-c:v", "libwebp", "-quality", "82"]))
codecs.append((".jpg", ["-c:v", "mjpeg", "-q:v", "6"]))
codecs.sort(key=lambda item: item[0] != _preferred)
heads: list[list[str]] = (["-ss", "0.3"], []) if target.suffix.lower() in VIDEO_EXTS else [([])]
return [(suffix, head, [*scale, *codec]) for head in heads for suffix, codec in codecs]
def _run(cmd: list[str]) -> tuple[bool, str]:
"""跑一条命令,返回 (是否成功, 失败原因)。
后端是常驻服务(容器里 stdin 未必是 tty),ffmpeg 一律带 `-nostdin` 并把
子进程 stdin 接到 DEVNULL:否则它会去继承父进程的 stdin,服务化之后
可能立刻拿到 EOF/EBADF 而中断转码。
"""
try:
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=TIMEOUT,
stdin=subprocess.DEVNULL)
except subprocess.TimeoutExpired:
return False, f"超时(>{TIMEOUT}s)"
except OSError as exc:
return False, f"无法执行 {cmd[0]}:{exc}"
if proc.returncode != 0:
return False, (proc.stderr or "").strip()[-400:] or f"返回码 {proc.returncode}"
return True, ""
def _generate(target: Path, stem: str, width: int) -> Path | None:
folder = _cache_dir()
global _preferred
last_error = ""
if ff._usable(ff.FFMPEG):
for suffix, pre, post in _ffmpeg_plans(target, width):
tmp = folder / f"{stem}.tmp-{os.getpid()}-{threading.get_ident()}{suffix}"
cmd = [ff.FFMPEG, "-hide_banner", "-nostdin", "-loglevel", "error", "-y",
*pre, "-i", str(target), *post, str(tmp)]
try:
ok, why = _run(cmd)
if ok and tmp.is_file() and tmp.stat().st_size > 0:
made = folder / f"{stem}{suffix}"
os.replace(tmp, made) # 原子落盘,避免半张图
_preferred = suffix # 记住这台机器真能用的编码器
return made
last_error = why or "没有产出文件"
finally:
tmp.unlink(missing_ok=True)
# macOS 上没有 ffmpeg 时的兜底(Docker 镜像自带 ffmpeg,走不到这里)
if target.suffix.lower() in IMAGE_EXTS and Path("/usr/bin/sips").is_file():
tmp = folder / f"{stem}.tmp-sips.jpg"
cmd = ["/usr/bin/sips", "-Z", str(width), "-s", "format", "jpeg", "-s", "formatOptions", "74",
str(target), "--out", str(tmp)]
try:
ok, why = _run(cmd)
last_error = why
if ok and tmp.is_file() and tmp.stat().st_size > 0:
os.replace(tmp, folder / f"{stem}.jpg")
return folder / f"{stem}.jpg"
finally:
tmp.unlink(missing_ok=True)
logger.warning("缩略图生成失败 %s(宽 %d):%s", to_relative(target), width, last_error or "无可用编码器")
return None
def build(path: str | Path | None, width: int | str | None = DEFAULT_WIDTH) -> Path | None:
"""取(必要时生成)缩略图文件;不可缩略 / 生成失败返回 None,由调用方回落原图。"""
target = resolve(path)
if target is None or not is_thumbable(target):
return None
size = normalize_width(width)
stem = _key(target, size)
hit = _existing(stem)
if hit:
return hit
if _recently_failed(stem):
return None
with _lock_for(stem): # 同一文件并发请求只生成一次
hit = _existing(stem)
if hit:
return hit
made = _generate(target, stem, size)
if made is None:
_mark_failed(stem)
return made
def original_url(path: str | Path | None) -> str:
return url_for(to_relative(path) if path else "")
def drop(path: str | Path | None) -> int:
"""源文件被删除时清掉它的全部缩略图变体(同一 digest 前缀),返回删除个数。"""
if not path:
return 0
rel = to_relative(path).replace("\\", "/")
digest = hashlib.sha1(rel.encode("utf-8")).hexdigest()[:24]
folder = media_root() / CACHE_DIR
removed = 0
for entry in folder.glob(f"{digest}-*") if folder.is_dir() else []:
if entry.is_file() and entry.suffix in MEDIA_TYPES:
entry.unlink(missing_ok=True)
removed += 1
return removed