275 lines
11 KiB
Python
275 lines
11 KiB
Python
"""列表页缩略图:惰性生成 + 磁盘缓存。
|
||
|
||
【为什么要这一层】
|
||
资产库 / 分镜 / 审片页此前把**原图**当缩略图渲染:一张定妆 PNG 就有 1.5~2MB(960×1280),
|
||
人物页首屏 12 张就是 19MB,三个人物/场景/声线页合计 26MB,浏览器要先下载再解码 12 张
|
||
1.2MP 位图,页面才会「转圈很久」。缩略成 384px webp 后单张约 20KB,首屏 <1MB。
|
||
|
||
【设计约束】
|
||
* 惰性:只在第一次被请求时生成,绝不做全量预构建(媒体库几百 GB 也扛得住);
|
||
* 缓存键含源文件 mtime+size,AI 重新生成同名覆盖后缩略图自动失效;
|
||
* 缩略图**永远不是正确性依赖**:没有 ffmpeg、文件损坏、越界路径,一律回落原图或 404,
|
||
绝不因为出图失败把列表页打成 500;
|
||
* 缓存放进 media_root 里的 `.thumbs/`,跟着媒体卷走(容器重建不丢),
|
||
且不会出现在任何业务列表里(后端不递归扫盘,静态目录以 . 开头也不会被 URL 猜中)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import logging
|
||
import os
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
from pathlib import Path
|
||
|
||
from app.config import settings
|
||
from app.services import ffmpeg as ff
|
||
from app.services.storage import media_root, to_relative, url_for
|
||
|
||
logger = logging.getLogger("nex.thumbs")
|
||
|
||
CACHE_DIR = ".thumbs"
|
||
# 只有这些扩展名值得出缩略图;svg(矢量,ffmpeg 不认)与音频/文档直接排除在外
|
||
IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp", ".tif", ".tiff", ".gif", ".heic"}
|
||
VIDEO_EXTS = {".mp4", ".mov", ".webm", ".mkv", ".m4v", ".avi"}
|
||
# 量化档位:前端各尺寸向上取整到这些值,保证同一个 URL 能被全平台复用
|
||
WIDTHS = (64, 128, 192, 256, 384, 512, 768, 1024, 1536)
|
||
DEFAULT_WIDTH = 384
|
||
# 生成失败后短期内不再重试(坏文件反复起 ffmpeg 会把接口拖死)
|
||
FAIL_TTL = 120.0
|
||
TIMEOUT = 25
|
||
|
||
MEDIA_TYPES = {".webp": "image/webp", ".jpg": "image/jpeg", ".png": "image/png"}
|
||
_OUTPUTS = (".webp", ".jpg")
|
||
|
||
_locks: dict[str, threading.Lock] = {}
|
||
_locks_guard = threading.Lock()
|
||
_failed: dict[str, float] = {}
|
||
# 上次成功的输出编码器(webp / jpg):避免每张图片都先拿不可用的编码器白跑一次
|
||
_preferred: str | None = None
|
||
|
||
|
||
def normalize_width(width: int | str | None) -> int:
|
||
"""把任意请求宽度向上取整到最近的档位(提高缓存命中率、限制变体数量)。"""
|
||
try:
|
||
want = int(width)
|
||
except (TypeError, ValueError):
|
||
return DEFAULT_WIDTH
|
||
if want <= 0:
|
||
return DEFAULT_WIDTH
|
||
for step in WIDTHS:
|
||
if want <= step:
|
||
return step
|
||
return WIDTHS[-1]
|
||
|
||
|
||
def is_thumbable(path: str | Path | None) -> bool:
|
||
if not path:
|
||
return False
|
||
suffix = Path(str(path).split("?")[0]).suffix.lower()
|
||
return suffix in IMAGE_EXTS or suffix in VIDEO_EXTS
|
||
|
||
|
||
def resolve(path: str | Path | None) -> Path | None:
|
||
"""把「绝对路径 / 相对 media_root 的路径 / `/media/...` 网址」统一解析成媒体库内的真实文件。
|
||
|
||
越界(`../` 出逃、指向库外绝对路径)与不存在都返回 None:
|
||
这条接口是浏览器可以随便拼参数的入口,必须自己守住目录边界。
|
||
"""
|
||
if not path:
|
||
return None
|
||
root = media_root()
|
||
raw = str(path).strip()
|
||
prefix = settings.media_url_prefix.strip("/")
|
||
if prefix:
|
||
for form in (f"/{prefix}/", f"{prefix}/"):
|
||
if raw.startswith(form):
|
||
raw = raw[len(form):] # 前端有时传的是整条 /media/... 网址
|
||
break
|
||
attempts: list[Path] = []
|
||
absolute = Path(raw)
|
||
if absolute.is_absolute():
|
||
attempts.append(absolute)
|
||
attempts.append(root / raw.lstrip("/"))
|
||
for probe in attempts:
|
||
try:
|
||
fixed = probe.resolve()
|
||
fixed.relative_to(root) # 抛出 ValueError = 逃出媒体库
|
||
except (OSError, ValueError):
|
||
continue
|
||
if fixed.is_file():
|
||
return fixed
|
||
return None
|
||
|
||
|
||
def cache_url(path: str | Path | None, width: int | str | None = DEFAULT_WIDTH) -> str:
|
||
"""给素材字典/前端用的缩略图地址;不值得缩略或文件缺失时返回空串。"""
|
||
target = resolve(path)
|
||
if target is None or not is_thumbable(target):
|
||
return ""
|
||
rel = to_relative(target).lstrip("/")
|
||
return f"/api/media/thumb/{normalize_width(width)}/{rel}"
|
||
|
||
|
||
def _key(target: Path, width: int) -> str:
|
||
stat = target.stat()
|
||
rel = to_relative(target).replace("\\", "/")
|
||
digest = hashlib.sha1(rel.encode("utf-8")).hexdigest()[:24]
|
||
return f"{digest}-{int(stat.st_mtime)}-{stat.st_size}-w{width}"
|
||
|
||
|
||
def _cache_dir() -> Path:
|
||
folder = media_root() / CACHE_DIR
|
||
folder.mkdir(parents=True, exist_ok=True)
|
||
return folder
|
||
|
||
|
||
def _existing(stem: str) -> Path | None:
|
||
folder = _cache_dir()
|
||
for suffix in _OUTPUTS:
|
||
hit = folder / f"{stem}{suffix}"
|
||
if hit.is_file() and hit.stat().st_size > 0:
|
||
return hit
|
||
return None
|
||
|
||
|
||
def _lock_for(stem: str) -> threading.Lock:
|
||
with _locks_guard:
|
||
lock = _locks.get(stem)
|
||
if lock is None:
|
||
lock = _locks[stem] = threading.Lock()
|
||
return lock
|
||
|
||
|
||
def _recently_failed(stem: str) -> bool:
|
||
expiry = _failed.get(stem, 0.0)
|
||
if expiry and expiry > time.time():
|
||
return True
|
||
_failed.pop(stem, None)
|
||
return False
|
||
|
||
|
||
def _mark_failed(stem: str) -> None:
|
||
if len(_failed) > 512: # 防止坏文件把内存表撑爆
|
||
_failed.clear()
|
||
_failed[stem] = time.time() + FAIL_TTL
|
||
|
||
|
||
def _scale_filter(width: int) -> str:
|
||
# decrease = 只缩不放;force_divisible_by 保证编码器不抱怨奇数尺寸
|
||
return f"scale={width}:{width}:force_original_aspect_ratio=decrease:force_divisible_by=2"
|
||
|
||
|
||
def _ffmpeg_plans(target: Path, width: int) -> list[tuple[str, list[str], list[str]]]:
|
||
"""返回候选方案:(输出后缀, -i 之前的参数, -i 之后的参数)。
|
||
|
||
优先用上次在这台机器上真的成功的编码器(`_preferred`),省得每张图都先拿
|
||
缺失的编码器白跑一次;视频先跳 0.3 秒取帧(首帧常常是全黑),
|
||
极短视频没有帧时再去掉 -ss 重取首帧。
|
||
"""
|
||
scale = ["-frames:v", "1", "-vf", _scale_filter(width), "-an"]
|
||
codecs: list[tuple[str, list[str]]] = []
|
||
if ff.can_encode("libwebp"):
|
||
codecs.append((".webp", ["-c:v", "libwebp", "-quality", "82"]))
|
||
codecs.append((".jpg", ["-c:v", "mjpeg", "-q:v", "6"]))
|
||
codecs.sort(key=lambda item: item[0] != _preferred)
|
||
heads: list[list[str]] = (["-ss", "0.3"], []) if target.suffix.lower() in VIDEO_EXTS else [([])]
|
||
return [(suffix, head, [*scale, *codec]) for head in heads for suffix, codec in codecs]
|
||
|
||
|
||
def _run(cmd: list[str]) -> tuple[bool, str]:
|
||
"""跑一条命令,返回 (是否成功, 失败原因)。
|
||
|
||
后端是常驻服务(容器里 stdin 未必是 tty),ffmpeg 一律带 `-nostdin` 并把
|
||
子进程 stdin 接到 DEVNULL:否则它会去继承父进程的 stdin,服务化之后
|
||
可能立刻拿到 EOF/EBADF 而中断转码。
|
||
"""
|
||
try:
|
||
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=TIMEOUT,
|
||
stdin=subprocess.DEVNULL)
|
||
except subprocess.TimeoutExpired:
|
||
return False, f"超时(>{TIMEOUT}s)"
|
||
except OSError as exc:
|
||
return False, f"无法执行 {cmd[0]}:{exc}"
|
||
if proc.returncode != 0:
|
||
return False, (proc.stderr or "").strip()[-400:] or f"返回码 {proc.returncode}"
|
||
return True, ""
|
||
|
||
|
||
def _generate(target: Path, stem: str, width: int) -> Path | None:
|
||
folder = _cache_dir()
|
||
global _preferred
|
||
last_error = ""
|
||
if ff._usable(ff.FFMPEG):
|
||
for suffix, pre, post in _ffmpeg_plans(target, width):
|
||
tmp = folder / f"{stem}.tmp-{os.getpid()}-{threading.get_ident()}{suffix}"
|
||
cmd = [ff.FFMPEG, "-hide_banner", "-nostdin", "-loglevel", "error", "-y",
|
||
*pre, "-i", str(target), *post, str(tmp)]
|
||
try:
|
||
ok, why = _run(cmd)
|
||
if ok and tmp.is_file() and tmp.stat().st_size > 0:
|
||
made = folder / f"{stem}{suffix}"
|
||
os.replace(tmp, made) # 原子落盘,避免半张图
|
||
_preferred = suffix # 记住这台机器真能用的编码器
|
||
return made
|
||
last_error = why or "没有产出文件"
|
||
finally:
|
||
tmp.unlink(missing_ok=True)
|
||
# macOS 上没有 ffmpeg 时的兜底(Docker 镜像自带 ffmpeg,走不到这里)
|
||
if target.suffix.lower() in IMAGE_EXTS and Path("/usr/bin/sips").is_file():
|
||
tmp = folder / f"{stem}.tmp-sips.jpg"
|
||
cmd = ["/usr/bin/sips", "-Z", str(width), "-s", "format", "jpeg", "-s", "formatOptions", "74",
|
||
str(target), "--out", str(tmp)]
|
||
try:
|
||
ok, why = _run(cmd)
|
||
last_error = why
|
||
if ok and tmp.is_file() and tmp.stat().st_size > 0:
|
||
os.replace(tmp, folder / f"{stem}.jpg")
|
||
return folder / f"{stem}.jpg"
|
||
finally:
|
||
tmp.unlink(missing_ok=True)
|
||
logger.warning("缩略图生成失败 %s(宽 %d):%s", to_relative(target), width, last_error or "无可用编码器")
|
||
return None
|
||
|
||
|
||
def build(path: str | Path | None, width: int | str | None = DEFAULT_WIDTH) -> Path | None:
|
||
"""取(必要时生成)缩略图文件;不可缩略 / 生成失败返回 None,由调用方回落原图。"""
|
||
target = resolve(path)
|
||
if target is None or not is_thumbable(target):
|
||
return None
|
||
size = normalize_width(width)
|
||
stem = _key(target, size)
|
||
hit = _existing(stem)
|
||
if hit:
|
||
return hit
|
||
if _recently_failed(stem):
|
||
return None
|
||
with _lock_for(stem): # 同一文件并发请求只生成一次
|
||
hit = _existing(stem)
|
||
if hit:
|
||
return hit
|
||
made = _generate(target, stem, size)
|
||
if made is None:
|
||
_mark_failed(stem)
|
||
return made
|
||
|
||
|
||
def original_url(path: str | Path | None) -> str:
|
||
return url_for(to_relative(path) if path else "")
|
||
|
||
|
||
def drop(path: str | Path | None) -> int:
|
||
"""源文件被删除时清掉它的全部缩略图变体(同一 digest 前缀),返回删除个数。"""
|
||
if not path:
|
||
return 0
|
||
rel = to_relative(path).replace("\\", "/")
|
||
digest = hashlib.sha1(rel.encode("utf-8")).hexdigest()[:24]
|
||
folder = media_root() / CACHE_DIR
|
||
removed = 0
|
||
for entry in folder.glob(f"{digest}-*") if folder.is_dir() else []:
|
||
if entry.is_file() and entry.suffix in MEDIA_TYPES:
|
||
entry.unlink(missing_ok=True)
|
||
removed += 1
|
||
return removed
|