unis-oa/backend/scripts/audit_storage.py

147 lines
6.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#!/usr/bin/env python3
"""清点存储后端:库里的引用和桶里的对象对不上账的那些,一条条列给人看。
::
python3 scripts/audit_storage.py # 只清点,不动东西
python3 scripts/audit_storage.py --prefix portal # 只看资讯这一块(封面 + 附件,桶里那截上传路径自动补上)
python3 scripts/audit_storage.py --retry # 顺手补删"当时删不掉"欠下的
python3 scripts/audit_storage.py --delete # 清掉确认没人认领的孤儿对象
python3 scripts/audit_storage.py --fix-legacy # 把裸文件名 / 绝对路径改成 local:// 标准写法
python3 scripts/audit_storage.py --to-minio # 把该进 MinIO 却还在磁盘上的文件搬进去
python3 scripts/audit_storage.py --to-minio --keep-source # 搬完不删磁盘上那份
python3 scripts/audit_storage.py --min-age-minutes 0 --json
两类对不上账:
* **孤儿对象**:桶里有、库里没有任何记录引用。来路无非三种——早期清库脚本默认只删
记录不删文件、MinIO 在删除那一刻连不上、先传后存又被放弃。
* **悬空引用**:库里写着位置、后端上已经没这个文件(有人在控制台手动删过、换过桶)。
这种点开来就是报错,界面上只能提示"文件已丢失"。
``--delete`` 有两道闸:只删 ``--min-age-minutes``(默认一天)以前落盘的,给"先传后存"
留出编辑时间;只删列举出来、且全库无人引用的。即便如此也别拿它当定时任务跑——先看
一眼清点结果,比什么都稳。
"""
from __future__ import annotations
import argparse
import json
import pathlib
import sys
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[0].parent))
from app.db import SessionLocal # noqa: E402
from app.service import storage_gc # noqa: E402
def human(size: int) -> str:
value = float(size)
for unit in ("B", "KB", "MB", "GB"):
if value < 1024 or unit == "GB":
return f"{value:.0f}{unit}" if unit == "B" else f"{value:.1f}{unit}"
value /= 1024
return f"{value:.1f}GB"
def main() -> int:
parser = argparse.ArgumentParser(description="清点 MinIO / 本地磁盘上的孤儿文件")
parser.add_argument(
"--prefix",
default="",
help="只清点这个前缀下的对象(如 portal、attachment);桶里那截上传路径会自动补上",
)
parser.add_argument("--delete", action="store_true", help="删掉确认无人认领的孤儿对象")
parser.add_argument(
"--min-age-minutes",
type=int,
default=storage_gc.AUDIT_MIN_AGE_MINUTES,
help=f"只动这么多分钟以前的对象(默认 {storage_gc.AUDIT_MIN_AGE_MINUTES})",
)
parser.add_argument("--retry", action="store_true", help="顺便补删待补删账上欠着的对象")
parser.add_argument(
"--fix-legacy", action="store_true", help="把老写法(裸文件名 / 绝对路径)改写成后端标准位置串"
)
parser.add_argument(
"--to-minio", action="store_true", help="把按用途该进 MinIO、目前还在本地磁盘上的文件搬进去"
)
parser.add_argument("--keep-source", action="store_true", help="配合 --to-minio:搬完保留本地那一份")
parser.add_argument("--limit", type=int, default=20, help="每个后端列几条样例")
parser.add_argument("--json", action="store_true", help="输出 JSON,给脚本看")
args = parser.parse_args()
with SessionLocal() as db:
legacy = storage_gc.normalize_legacy_locations(db) if args.fix_legacy else None
if args.to_minio:
moved = storage_gc.relocate_to_backend(db, delete_source=not args.keep_source)
else:
moved = None
retried = storage_gc.retry_pending(db) if args.retry else None
report = storage_gc.audit(
db,
prefix=args.prefix,
delete=args.delete,
min_age_minutes=args.min_age_minutes,
sample=args.limit,
)
if legacy is not None:
report["legacy"] = {"changed": len(legacy), "items": legacy[:50]}
if moved is not None:
report["relocated"] = {"changed": len(moved), "items": moved[:50]}
if retried is not None:
report["retry"] = retried
if args.json:
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0
print(
f"库里登记的引用:{report['references']} 条(后端位置 {report['locations']} 个);"
f"待补删账上欠着 {report['pending']} 个"
)
if legacy is not None:
print(f"老写法改写:{len(legacy)} 条")
for item in legacy[:10]:
print(f" - {item['table']}.{item['column']} {item['id']} -> {item['to']}")
if len(legacy) > 10:
print(f" …另有 {len(legacy) - 10} 条")
if moved is not None:
print(f"搬进 MinIO:{len(moved)} 条(本地那份{'保留' if args.keep_source else '已删'})")
for item in moved[:10]:
print(f" - {item['table']}.{item['column']} {item['id']}: {item['from']} -> {item['to']}")
if len(moved) > 10:
print(f" …另有 {len(moved) - 10} 条")
for entry in report["backends"]:
where = entry.get("location") or entry.get("root") or entry.get("kind")
if "error" in entry:
print(f"\n[{entry['kind']}] {where} 清点失败:{entry['error']}")
continue
print(
f"\n[{entry['kind']}] {where}:对象 {entry['listed']} 个,被引用 {entry['referenced']} 个,"
f"孤儿 {entry['orphans']} 个({human(entry['orphan_bytes'])})"
)
if entry["protected_by_age"]:
print(f" 其中 {entry['protected_by_age']} 个太新({args.min_age_minutes} 分钟内),按年龄闸留着")
if args.delete:
print(f" 本轮删掉 {entry['deleted']} 个")
for item in entry["samples"]:
print(f" - {item['key']} {human(item['size'])} {item['last_modified']}")
if entry["orphans"] > len(entry["samples"]):
print(f" …另有 {entry['orphans'] - len(entry['samples'])} 个,加 --limit 看更多")
if report["dangling_total"]:
print(f"\n悬空引用 {report['dangling_total']} 条(库里有记录、后端上没这个文件):")
for item in report["dangling"]:
print(f" - {item['label']} {item['uri']}")
else:
print("\n悬空引用:0 条")
if retried is not None:
print(f"\n补删:查了 {retried['checked']} 条,删成 {retried['deleted']} 条,仍失败 {retried['failed']} 条")
if not args.delete and any(entry.get("orphans") for entry in report["backends"]):
print("\n确认过确实没人要了,再加 --delete 清掉。")
return 0
if __name__ == "__main__":
sys.exit(main())