147 lines
6.8 KiB
Python
147 lines
6.8 KiB
Python
#!/usr/bin/env python3
|
||
"""清点存储后端:库里的引用和桶里的对象对不上账的那些,一条条列给人看。
|
||
|
||
::
|
||
|
||
python3 scripts/audit_storage.py # 只清点,不动东西
|
||
python3 scripts/audit_storage.py --prefix portal # 只看资讯这一块(封面 + 附件,桶里那截上传路径自动补上)
|
||
python3 scripts/audit_storage.py --retry # 顺手补删"当时删不掉"欠下的
|
||
python3 scripts/audit_storage.py --delete # 清掉确认没人认领的孤儿对象
|
||
python3 scripts/audit_storage.py --fix-legacy # 把裸文件名 / 绝对路径改成 local:// 标准写法
|
||
python3 scripts/audit_storage.py --to-minio # 把该进 MinIO 却还在磁盘上的文件搬进去
|
||
python3 scripts/audit_storage.py --to-minio --keep-source # 搬完不删磁盘上那份
|
||
python3 scripts/audit_storage.py --min-age-minutes 0 --json
|
||
|
||
两类对不上账:
|
||
|
||
* **孤儿对象**:桶里有、库里没有任何记录引用。来路无非三种——早期清库脚本默认只删
|
||
记录不删文件、MinIO 在删除那一刻连不上、先传后存又被放弃。
|
||
* **悬空引用**:库里写着位置、后端上已经没这个文件(有人在控制台手动删过、换过桶)。
|
||
这种点开来就是报错,界面上只能提示"文件已丢失"。
|
||
|
||
``--delete`` 有两道闸:只删 ``--min-age-minutes``(默认一天)以前落盘的,给"先传后存"
|
||
留出编辑时间;只删列举出来、且全库无人引用的。即便如此也别拿它当定时任务跑——先看
|
||
一眼清点结果,比什么都稳。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
|
||
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[0].parent))
|
||
|
||
from app.db import SessionLocal # noqa: E402
|
||
from app.service import storage_gc # noqa: E402
|
||
|
||
|
||
def human(size: int) -> str:
|
||
value = float(size)
|
||
for unit in ("B", "KB", "MB", "GB"):
|
||
if value < 1024 or unit == "GB":
|
||
return f"{value:.0f}{unit}" if unit == "B" else f"{value:.1f}{unit}"
|
||
value /= 1024
|
||
return f"{value:.1f}GB"
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="清点 MinIO / 本地磁盘上的孤儿文件")
|
||
parser.add_argument(
|
||
"--prefix",
|
||
default="",
|
||
help="只清点这个前缀下的对象(如 portal、attachment);桶里那截上传路径会自动补上",
|
||
)
|
||
parser.add_argument("--delete", action="store_true", help="删掉确认无人认领的孤儿对象")
|
||
parser.add_argument(
|
||
"--min-age-minutes",
|
||
type=int,
|
||
default=storage_gc.AUDIT_MIN_AGE_MINUTES,
|
||
help=f"只动这么多分钟以前的对象(默认 {storage_gc.AUDIT_MIN_AGE_MINUTES})",
|
||
)
|
||
parser.add_argument("--retry", action="store_true", help="顺便补删待补删账上欠着的对象")
|
||
parser.add_argument(
|
||
"--fix-legacy", action="store_true", help="把老写法(裸文件名 / 绝对路径)改写成后端标准位置串"
|
||
)
|
||
parser.add_argument(
|
||
"--to-minio", action="store_true", help="把按用途该进 MinIO、目前还在本地磁盘上的文件搬进去"
|
||
)
|
||
parser.add_argument("--keep-source", action="store_true", help="配合 --to-minio:搬完保留本地那一份")
|
||
parser.add_argument("--limit", type=int, default=20, help="每个后端列几条样例")
|
||
parser.add_argument("--json", action="store_true", help="输出 JSON,给脚本看")
|
||
args = parser.parse_args()
|
||
|
||
with SessionLocal() as db:
|
||
legacy = storage_gc.normalize_legacy_locations(db) if args.fix_legacy else None
|
||
if args.to_minio:
|
||
moved = storage_gc.relocate_to_backend(db, delete_source=not args.keep_source)
|
||
else:
|
||
moved = None
|
||
retried = storage_gc.retry_pending(db) if args.retry else None
|
||
report = storage_gc.audit(
|
||
db,
|
||
prefix=args.prefix,
|
||
delete=args.delete,
|
||
min_age_minutes=args.min_age_minutes,
|
||
sample=args.limit,
|
||
)
|
||
if legacy is not None:
|
||
report["legacy"] = {"changed": len(legacy), "items": legacy[:50]}
|
||
if moved is not None:
|
||
report["relocated"] = {"changed": len(moved), "items": moved[:50]}
|
||
if retried is not None:
|
||
report["retry"] = retried
|
||
|
||
if args.json:
|
||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||
return 0
|
||
|
||
print(
|
||
f"库里登记的引用:{report['references']} 条(后端位置 {report['locations']} 个);"
|
||
f"待补删账上欠着 {report['pending']} 个"
|
||
)
|
||
if legacy is not None:
|
||
print(f"老写法改写:{len(legacy)} 条")
|
||
for item in legacy[:10]:
|
||
print(f" - {item['table']}.{item['column']} {item['id']} -> {item['to']}")
|
||
if len(legacy) > 10:
|
||
print(f" …另有 {len(legacy) - 10} 条")
|
||
if moved is not None:
|
||
print(f"搬进 MinIO:{len(moved)} 条(本地那份{'保留' if args.keep_source else '已删'})")
|
||
for item in moved[:10]:
|
||
print(f" - {item['table']}.{item['column']} {item['id']}: {item['from']} -> {item['to']}")
|
||
if len(moved) > 10:
|
||
print(f" …另有 {len(moved) - 10} 条")
|
||
for entry in report["backends"]:
|
||
where = entry.get("location") or entry.get("root") or entry.get("kind")
|
||
if "error" in entry:
|
||
print(f"\n[{entry['kind']}] {where} 清点失败:{entry['error']}")
|
||
continue
|
||
print(
|
||
f"\n[{entry['kind']}] {where}:对象 {entry['listed']} 个,被引用 {entry['referenced']} 个,"
|
||
f"孤儿 {entry['orphans']} 个({human(entry['orphan_bytes'])})"
|
||
)
|
||
if entry["protected_by_age"]:
|
||
print(f" 其中 {entry['protected_by_age']} 个太新({args.min_age_minutes} 分钟内),按年龄闸留着")
|
||
if args.delete:
|
||
print(f" 本轮删掉 {entry['deleted']} 个")
|
||
for item in entry["samples"]:
|
||
print(f" - {item['key']} {human(item['size'])} {item['last_modified']}")
|
||
if entry["orphans"] > len(entry["samples"]):
|
||
print(f" …另有 {entry['orphans'] - len(entry['samples'])} 个,加 --limit 看更多")
|
||
if report["dangling_total"]:
|
||
print(f"\n悬空引用 {report['dangling_total']} 条(库里有记录、后端上没这个文件):")
|
||
for item in report["dangling"]:
|
||
print(f" - {item['label']} {item['uri']}")
|
||
else:
|
||
print("\n悬空引用:0 条")
|
||
if retried is not None:
|
||
print(f"\n补删:查了 {retried['checked']} 条,删成 {retried['deleted']} 条,仍失败 {retried['failed']} 条")
|
||
if not args.delete and any(entry.get("orphans") for entry in report["backends"]):
|
||
print("\n确认过确实没人要了,再加 --delete 清掉。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|