376 lines
18 KiB
Python
Executable File
376 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""部署数据集的导出与安装:表结构 + 模型配置 + 管理员账号。
|
||
|
||
服务器部署**不搬开发库**。14MB 的 nex_animation.db 里绝大多数是开发过程产物
|
||
(项目、镜头、任务、事件日志、演示账号),新环境真正需要的是三样东西:
|
||
|
||
| 内容 | 来源 |
|
||
|---------------------|----------------------------------------|
|
||
| 表结构 + 列迁移 | `app.db.init_db()`(应用启动自带) |
|
||
| 模型配置 | `model_specs` + `model_secret_keys` |
|
||
| 管理员账号 | `users` 里 role=admin 的那几条 |
|
||
|
||
模型配置必须带走,否则新环境只有 `seed_specs()` 播的 mock,得在模型中心
|
||
从零填一遍 ComfyUI / Qwen / IndexTTS 的端点与工作流;而 API Key 是
|
||
`app/services/secrets.py` 用库内主密钥加密的,所以 **`model_secret_keys` 那一行
|
||
必须和密文一起走**——换机器只带 model_specs,密文会解成空串,
|
||
表现为「模型中心显示已配置,派发却报缺 Key」。
|
||
|
||
# 开发机:导出
|
||
.venv/bin/python scripts/seed.py export
|
||
.venv/bin/python scripts/seed.py export --with-platform-library # 顺带平台资源库
|
||
|
||
# 服务器:安装(空库;先建表再灌数据,幂等,可反复执行)
|
||
.venv/bin/python scripts/seed.py install
|
||
|
||
安装顺序刻意是「主密钥 → 模型配置 → 账号」:主密钥先到位,后面导入的密文才可解。
|
||
`model_secret_keys` 已有值且与导出值不同时默认**中止**(不是覆盖),
|
||
因为覆盖会让这台机器上原有模型配置的 Key 全部解不开——真要做这种手术请加
|
||
--force-master-key,并准备好在模型中心重填所有 Key。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import datetime as dt
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "backend"))
|
||
|
||
from sqlalchemy import inspect, select, text, types as sa_types # noqa: E402
|
||
|
||
from app.db import Base, SessionLocal, engine, init_db # noqa: E402
|
||
from app.models import ModelSecretKey, ModelSpec, User # noqa: E402
|
||
|
||
SEED_DIR = ROOT / "scripts" / "seed"
|
||
|
||
# 平台资源库(可选):严格说不属于「部署必需」,但换机时重建 37 条素材很费手工,
|
||
# 所以做成显式开关;媒体文件在 media/platform/ 下,不在数据库里,需另行拷贝。
|
||
PLATFORM_TABLES = ("platform_characters", "platform_scenes", "platform_voices",
|
||
"platform_media", "platform_media_versions")
|
||
|
||
# users 里这些列是「在哪台机器上登过」的流水,跨机没有意义,还会误导审计。
|
||
USER_DROP = ("last_login_at", "last_login_ip", "login_count")
|
||
|
||
|
||
# ------------------------------------------------------------------ 行 ⇄ dict
|
||
def row_to_dict(obj) -> dict:
|
||
"""ORM 行转 JSON 可序列化 dict(时间转 ISO,JSON 列原样,二进制转 base64)。"""
|
||
out: dict = {}
|
||
for col in inspect(obj.__table__).columns:
|
||
val = getattr(obj, col.key)
|
||
if isinstance(val, dt.datetime):
|
||
val = val.isoformat(sep=" ", timespec="microseconds")
|
||
elif isinstance(val, (bytes, bytearray)):
|
||
val = bytes(val).hex()
|
||
elif isinstance(val, dt.date):
|
||
val = val.isoformat()
|
||
out[col.key] = val
|
||
return out
|
||
|
||
|
||
def write_json(path: Path, payload) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(f" 写出 {path.relative_to(ROOT)}")
|
||
|
||
|
||
def read_json(src: Path, name: str):
|
||
p = src / name
|
||
if not p.is_file():
|
||
sys.exit(f"缺少 {p};先在开发机跑 .venv/bin/python scripts/seed.py export")
|
||
return json.loads(p.read_text(encoding="utf-8"))
|
||
|
||
|
||
def _apply(row, data: dict, *, skip: tuple[str, ...] = ()) -> None:
|
||
mapper = inspect(row.__table__)
|
||
for key, val in data.items():
|
||
if key in skip or key not in mapper.c:
|
||
continue
|
||
col = mapper.c[key]
|
||
if isinstance(val, str):
|
||
# JSON 导出把时间写成了 ISO 字符串,回填时必须还原成 datetime:
|
||
# SQLite 驱动遇到字符串直接抛 «only accepts Python datetime and date
|
||
# objects as input»,Postgres 则可能按会话时区误读。
|
||
# 本库的时间列一律是 DateTime(timezone=True),写进去的是带 UTC 的值,
|
||
# 读回来却是 naive(见 app/models/base.py:as_utc),所以这里补回 UTC。
|
||
if isinstance(col.type, sa_types.DateTime):
|
||
parsed = dt.datetime.fromisoformat(val)
|
||
val = parsed.replace(tzinfo=dt.timezone.utc) if parsed.tzinfo is None else parsed
|
||
elif isinstance(col.type, sa_types.Date):
|
||
val = dt.date.fromisoformat(val)
|
||
setattr(row, key, val)
|
||
|
||
|
||
# ================================================================== export
|
||
def do_export(args) -> None:
|
||
out = Path(args.out)
|
||
db = SessionLocal()
|
||
try:
|
||
specs = [row_to_dict(s) for s in db.scalars(select(ModelSpec).order_by(ModelSpec.id)).all()]
|
||
keys = [row_to_dict(k) for k in db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()]
|
||
admins = [row_to_dict(u) for u in db.scalars(
|
||
select(User).where(User.role == "admin").order_by(User.id)).all()]
|
||
if not specs:
|
||
sys.exit("model_specs 是空的:这台机器还没配过模型,导出去也只是 mock。")
|
||
if not admins:
|
||
sys.exit("users 里没有 admin 角色账号。")
|
||
|
||
# 明文密钥自检:导出文件里只允许出现密文。任何一条看起来像
|
||
# "sk-xxxx" 的明文都说明有人绕过 secrets.py 直接写了库。
|
||
leaks = [f"{s['provider']}/{s['model']}" for s in specs
|
||
if s.get("api_key_encrypted", "").startswith("sk-")]
|
||
if leaks:
|
||
sys.exit(f"疑似明文 API Key 直接入库:{leaks};请先在模型中心重填一次。")
|
||
|
||
write_json(out / "model_specs.json", {
|
||
"_meta": {"table": "model_specs", "count": len(specs),
|
||
"defaults": sorted(f"{s['media_type']}={s['provider']}/{s['model']}"
|
||
for s in specs if s["is_default"]),
|
||
"exported_at": dt.datetime.now().isoformat(timespec="seconds")},
|
||
"rows": specs})
|
||
write_json(out / "model_secret_keys.json", {
|
||
"_meta": {"table": "model_secret_keys", "count": len(keys),
|
||
"note": "密钥加密主密钥;缺了它 model_specs.api_key_encrypted 解不开"},
|
||
"rows": keys})
|
||
write_json(out / "users.json", {
|
||
"_meta": {"table": "users", "count": len(admins),
|
||
"note": "只导管理员;口令是 PBKDF2 哈希,不含明文"},
|
||
"rows": [{k: v for k, v in u.items() if k not in USER_DROP} for u in admins]})
|
||
|
||
if args.with_platform_library:
|
||
lib = {}
|
||
for name in PLATFORM_TABLES:
|
||
if name not in Base.metadata.tables:
|
||
print(f" ! 跳过 {name}(模型里没有这张表)")
|
||
continue
|
||
rows = [dict(r._mapping) for r in db.execute(text(f"SELECT * FROM {name}")).all()]
|
||
lib[name] = rows
|
||
print(f" 平台资源库 {name}: {len(rows)} 行")
|
||
write_json(out / "platform_library.json", {
|
||
"_meta": {"tables": list(lib), "note": "媒体文件在 media/platform/,需另行拷贝"},
|
||
"rows": lib})
|
||
|
||
# 解密自检:能解出内容才说明「密文 + 主密钥」这一对是完整的。
|
||
from app.services import secrets as secret_svc
|
||
checked = [f"{s['provider']}/{s['model']}" for s in specs if s.get("api_key_encrypted")]
|
||
ok = [f"{s['provider']}/{s['model']}" for s in specs
|
||
if s.get("api_key_encrypted")
|
||
and secret_svc.decrypt_value(db, s["api_key_encrypted"])]
|
||
if len(ok) != len(checked):
|
||
sys.exit(f"主密钥与密文不匹配:{set(checked) - set(ok)} 解不开,导出会导致换机后 Key 丢失。")
|
||
print(f"\n导出完成 → {out}")
|
||
print(f" 模型配置 {len(specs)} 条(默认:{', '.join(s['media_type'] for s in specs if s['is_default'])})"
|
||
f" · 带 Key {len(ok)} 条且解密通过 · 管理员 {len(admins)} 个")
|
||
print(" ⚠️ scripts/seed/ 含加密主密钥,已在 .gitignore 内;部署包里必须带上它。")
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
# ================================================================== install
|
||
def _keys_in_use(db) -> int:
|
||
"""本机已有几条模型配置真的用上了当前主密钥(密文能被它解开)。"""
|
||
from app.services import secrets as secret_svc
|
||
|
||
used = 0
|
||
for s in db.scalars(select(ModelSpec)).all():
|
||
if s.api_key_encrypted and secret_svc.decrypt_value(db, s.api_key_encrypted):
|
||
used += 1
|
||
return used
|
||
|
||
|
||
def _install_master_key(db, rows, *, dry: bool, force: bool) -> None:
|
||
have = db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()
|
||
if not rows:
|
||
print(" · 导出包里没有主密钥(开发机可能用 MODEL_SECRET_KEY 环境变量):跳过")
|
||
return
|
||
if not have:
|
||
for data in rows:
|
||
row = ModelSecretKey()
|
||
_apply(row, data, skip=("id",))
|
||
db.add(row)
|
||
print(f" ✓ 主密钥已导入({len(rows)} 条)")
|
||
return
|
||
if have[0].key_value == rows[0]["key_value"]:
|
||
print(" · 主密钥已一致,跳过")
|
||
return
|
||
msg = (f"目标库已有主密钥且与导出包不同(现有 {have[0].key_value[:8]}… vs "
|
||
f"导入 {rows[0]['key_value'][:8]}…)")
|
||
in_use = _keys_in_use(db)
|
||
if in_use == 0 and not force:
|
||
# 「先起了服务、后导数据」是最常见的顺序:应用首启会自己生成一把主密钥,
|
||
# 但此刻还没有任何模型配置真的依赖它(mock 没有 Key),换掉零损失。
|
||
# 拦在这里只会让人以为工具坏了,所以直接换,并把话讲明白。
|
||
for data in rows:
|
||
_apply(have[0], data, skip=("id",))
|
||
print(f" · {msg}\n · 本机没有任何已解密成功的 API Key 依赖旧密钥 → 直接换成部署包的主密钥")
|
||
return
|
||
if not force:
|
||
sys.exit(msg + f";本机有 {in_use} 条模型配置的 Key 靠它解密,"
|
||
"覆盖后全部要重填。确认就加 --force-master-key。")
|
||
for data in rows:
|
||
_apply(have[0], data, skip=("id",))
|
||
print(f" ! {msg} → 已按 --force-master-key 覆盖,请复查模型中心的 Key")
|
||
|
||
|
||
def _install_specs(db, rows, *, dry: bool, only_missing: bool = False) -> None:
|
||
added = updated = skipped = 0
|
||
for data in rows:
|
||
key = {"provider": data["provider"], "model": data.get("model", ""),
|
||
"media_type": data["media_type"]}
|
||
row = db.scalars(select(ModelSpec).filter_by(**key)).first()
|
||
if row is not None and only_missing:
|
||
skipped += 1
|
||
continue
|
||
if row is None:
|
||
row = ModelSpec()
|
||
_apply(row, data, skip=("id", "created_at", "updated_at"))
|
||
db.add(row)
|
||
added += 1
|
||
else:
|
||
_apply(row, data, skip=("id", "created_at"))
|
||
updated += 1
|
||
# 默认位以导出包为准:同能力下不在包里的旧默认必须让位,
|
||
# 否则 seed_specs / resolve_* 会因为排序偶然选中开发机没在用的那个模型。
|
||
db.flush()
|
||
for media_type in {d["media_type"] for d in rows}:
|
||
want = next((d for d in rows if d["media_type"] == media_type and d["is_default"]), None)
|
||
if want is None:
|
||
continue
|
||
others = db.query(ModelSpec).filter(
|
||
ModelSpec.media_type == media_type,
|
||
ModelSpec.is_default.is_(True),
|
||
~((ModelSpec.provider == want["provider"]) & (ModelSpec.model == want.get("model", "")))
|
||
).update({"is_default": False})
|
||
if others:
|
||
print(f" · {media_type}: 取消 {others} 条旧默认,改为 "
|
||
f"{want['provider']}/{want['model']}")
|
||
if not dry:
|
||
db.commit()
|
||
note = ",--only-missing 跳过 " + str(skipped) if skipped else ""
|
||
verb = ("按部署包覆盖" if updated and not only_missing else "写入")
|
||
print(f" ✓ 模型配置:新增 {added},{verb} {updated}{note}"
|
||
+ ("" if not updated or only_missing else "(更新=用部署包的值覆盖服务器上的同名模型)"))
|
||
|
||
|
||
def _install_users(db, rows, *, dry: bool) -> None:
|
||
for data in rows:
|
||
row = db.scalars(select(User).where(User.username == data["username"])).first()
|
||
if row is None:
|
||
row = User()
|
||
_apply(row, data, skip=("id", "created_at", "updated_at"))
|
||
row.must_change_password = False
|
||
db.add(row)
|
||
print(f" ✓ 管理员 {row.username}(role={row.role})已创建,"
|
||
f"口令沿用导出时的哈希——登录后请立即修改")
|
||
else:
|
||
# 已存在的账号不重置口令:这台机器可能早改过密码,
|
||
# 覆盖成开发机的口令是个反直觉的倒退。
|
||
_apply(row, data, skip=("id", "created_at", "updated_at", "password_hash",
|
||
"must_change_password"))
|
||
diff = (data.get("password_hash") or "") != row.password_hash
|
||
print(f" · 管理员 {row.username} 已存在:只同步角色/显示名,不改口令"
|
||
+ ("(注意:它的口令与部署包不同,登录请用这台机器的口令)" if diff else ""))
|
||
if not dry:
|
||
db.commit()
|
||
|
||
|
||
def _install_platform(db, tables: dict, *, dry: bool) -> None:
|
||
for name in PLATFORM_TABLES:
|
||
rows = tables.get(name) or []
|
||
if not rows:
|
||
continue
|
||
table = Base.metadata.tables.get(name)
|
||
if table is None:
|
||
print(f" ! 跳过 {name}(当前代码没有这张表)")
|
||
continue
|
||
cols = [c.key for c in inspect(table).columns if c.key != "id"]
|
||
for data in rows:
|
||
pk = data.get("id")
|
||
if db.execute(text(f"SELECT 1 FROM {name} WHERE id=:i"), {"i": pk}).first():
|
||
continue
|
||
col_sql = ", ".join(cols)
|
||
val_sql = ", ".join(f":{c}" for c in cols)
|
||
db.execute(text(f"INSERT INTO {name} (id, {col_sql}) VALUES (:id, {val_sql})"),
|
||
{**{c: data.get(c) for c in cols}, "id": pk})
|
||
print(f" ✓ 平台资源库 {name}: 导入后共 "
|
||
f"{db.execute(text(f'SELECT COUNT(*) FROM {name}')).scalar()} 行")
|
||
if not dry:
|
||
db.commit()
|
||
|
||
|
||
def do_install(args) -> None:
|
||
src = Path(args.src)
|
||
specs = read_json(src, "model_specs.json")["rows"]
|
||
keys = read_json(src, "model_secret_keys.json")["rows"]
|
||
users = read_json(src, "users.json")["rows"]
|
||
lib = None
|
||
if args.with_platform_library:
|
||
lib = read_json(src, "platform_library.json")["rows"]
|
||
|
||
if args.schema:
|
||
init_db() # 建表 + 列迁移,幂等
|
||
print(f" ✓ 表结构就绪:{engine.url.render_as_string(hide_password=True)}")
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
print("安装部署数据集(幂等,可反复执行)")
|
||
_install_master_key(db, keys, dry=args.dry_run, force=args.force_master_key)
|
||
_install_specs(db, specs, dry=args.dry_run, only_missing=args.only_missing)
|
||
_install_users(db, users, dry=args.dry_run)
|
||
if lib is not None:
|
||
_install_platform(db, lib, dry=args.dry_run)
|
||
if args.dry_run:
|
||
db.rollback()
|
||
print("\n--dry-run:未写入任何数据")
|
||
return
|
||
|
||
# 装完就验一次解密:这一步不过,说明主密钥和密文不是一对,
|
||
# 与其等业务报「缺 Key」,不如现在停下来。
|
||
from app.services import secrets as secret_svc
|
||
bad = [f"{s.provider}/{s.model}" for s in db.scalars(select(ModelSpec)).all()
|
||
if s.api_key_encrypted and not secret_svc.decrypt_value(db, s.api_key_encrypted)]
|
||
defaults = {s.media_type: f"{s.provider}/{s.model}"
|
||
for s in db.scalars(select(ModelSpec).where(ModelSpec.is_default.is_(True))).all()}
|
||
admins = db.scalars(select(User.username).where(User.role == "admin")).all()
|
||
print("\n校验")
|
||
print(f" 默认模型:{', '.join(f'{k}={v}' for k, v in sorted(defaults.items())) or '(无)'}")
|
||
print(f" 管理员:{', '.join(admins) or '(无)'}")
|
||
if bad:
|
||
sys.exit(f" ✗ 这些模型的 API Key 解不开(主密钥不匹配):{bad}")
|
||
print(" ✓ 所有 API Key 解密通过")
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
# ================================================================== CLI
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0],
|
||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||
|
||
e = sub.add_parser("export", help="导出模型配置 + 主密钥 + 管理员到 JSON")
|
||
e.add_argument("--out", default=str(SEED_DIR), help="输出目录(默认 scripts/seed)")
|
||
e.add_argument("--with-platform-library", action="store_true", help="顺带导出平台资源库")
|
||
e.set_defaults(func=do_export)
|
||
|
||
i = sub.add_parser("install", help="把 JSON 装进目标库(服务器首次部署用)")
|
||
i.add_argument("--src", default=str(SEED_DIR), help="JSON 目录(默认 scripts/seed)")
|
||
i.add_argument("--no-schema", dest="schema", action="store_false", help="不先建表")
|
||
i.add_argument("--dry-run", action="store_true", help="只报告将要发生什么")
|
||
i.add_argument("--force-master-key", action="store_true", help="允许覆盖已有主密钥")
|
||
i.add_argument("--with-platform-library", action="store_true", help="顺带导入平台资源库")
|
||
i.add_argument("--only-missing", action="store_true",
|
||
help="模型配置只补缺,不覆盖服务器上已有的同名模型")
|
||
i.set_defaults(func=do_install)
|
||
|
||
args = ap.parse_args()
|
||
args.func(args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|