nex_animation/scripts/seed.py

376 lines
18 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""部署数据集的导出与安装:表结构 + 模型配置 + 管理员账号。
服务器部署**不搬开发库**。14MB 的 nex_animation.db 里绝大多数是开发过程产物
(项目、镜头、任务、事件日志、演示账号),新环境真正需要的是三样东西:
| 内容 | 来源 |
|---------------------|----------------------------------------|
| 表结构 + 列迁移 | `app.db.init_db()`(应用启动自带) |
| 模型配置 | `model_specs` + `model_secret_keys` |
| 管理员账号 | `users` 里 role=admin 的那几条 |
模型配置必须带走,否则新环境只有 `seed_specs()` 播的 mock,得在模型中心
从零填一遍 ComfyUI / Qwen / IndexTTS 的端点与工作流;而 API Key 是
`app/services/secrets.py` 用库内主密钥加密的,所以 **`model_secret_keys` 那一行
必须和密文一起走**——换机器只带 model_specs,密文会解成空串,
表现为「模型中心显示已配置,派发却报缺 Key」。
# 开发机:导出
.venv/bin/python scripts/seed.py export
.venv/bin/python scripts/seed.py export --with-platform-library # 顺带平台资源库
# 服务器:安装(空库;先建表再灌数据,幂等,可反复执行)
.venv/bin/python scripts/seed.py install
安装顺序刻意是「主密钥 → 模型配置 → 账号」:主密钥先到位,后面导入的密文才可解。
`model_secret_keys` 已有值且与导出值不同时默认**中止**(不是覆盖),
因为覆盖会让这台机器上原有模型配置的 Key 全部解不开——真要做这种手术请加
--force-master-key,并准备好在模型中心重填所有 Key。
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
from sqlalchemy import inspect, select, text, types as sa_types # noqa: E402
from app.db import Base, SessionLocal, engine, init_db # noqa: E402
from app.models import ModelSecretKey, ModelSpec, User # noqa: E402
SEED_DIR = ROOT / "scripts" / "seed"
# 平台资源库(可选):严格说不属于「部署必需」,但换机时重建 37 条素材很费手工,
# 所以做成显式开关;媒体文件在 media/platform/ 下,不在数据库里,需另行拷贝。
PLATFORM_TABLES = ("platform_characters", "platform_scenes", "platform_voices",
"platform_media", "platform_media_versions")
# users 里这些列是「在哪台机器上登过」的流水,跨机没有意义,还会误导审计。
USER_DROP = ("last_login_at", "last_login_ip", "login_count")
# ------------------------------------------------------------------ 行 ⇄ dict
def row_to_dict(obj) -> dict:
"""ORM 行转 JSON 可序列化 dict(时间转 ISO,JSON 列原样,二进制转 base64)。"""
out: dict = {}
for col in inspect(obj.__table__).columns:
val = getattr(obj, col.key)
if isinstance(val, dt.datetime):
val = val.isoformat(sep=" ", timespec="microseconds")
elif isinstance(val, (bytes, bytearray)):
val = bytes(val).hex()
elif isinstance(val, dt.date):
val = val.isoformat()
out[col.key] = val
return out
def write_json(path: Path, payload) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(f" 写出 {path.relative_to(ROOT)}")
def read_json(src: Path, name: str):
p = src / name
if not p.is_file():
sys.exit(f"缺少 {p};先在开发机跑 .venv/bin/python scripts/seed.py export")
return json.loads(p.read_text(encoding="utf-8"))
def _apply(row, data: dict, *, skip: tuple[str, ...] = ()) -> None:
mapper = inspect(row.__table__)
for key, val in data.items():
if key in skip or key not in mapper.c:
continue
col = mapper.c[key]
if isinstance(val, str):
# JSON 导出把时间写成了 ISO 字符串,回填时必须还原成 datetime:
# SQLite 驱动遇到字符串直接抛 «only accepts Python datetime and date
# objects as input»,Postgres 则可能按会话时区误读。
# 本库的时间列一律是 DateTime(timezone=True),写进去的是带 UTC 的值,
# 读回来却是 naive(见 app/models/base.py:as_utc),所以这里补回 UTC。
if isinstance(col.type, sa_types.DateTime):
parsed = dt.datetime.fromisoformat(val)
val = parsed.replace(tzinfo=dt.timezone.utc) if parsed.tzinfo is None else parsed
elif isinstance(col.type, sa_types.Date):
val = dt.date.fromisoformat(val)
setattr(row, key, val)
# ================================================================== export
def do_export(args) -> None:
out = Path(args.out)
db = SessionLocal()
try:
specs = [row_to_dict(s) for s in db.scalars(select(ModelSpec).order_by(ModelSpec.id)).all()]
keys = [row_to_dict(k) for k in db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()]
admins = [row_to_dict(u) for u in db.scalars(
select(User).where(User.role == "admin").order_by(User.id)).all()]
if not specs:
sys.exit("model_specs 是空的:这台机器还没配过模型,导出去也只是 mock。")
if not admins:
sys.exit("users 里没有 admin 角色账号。")
# 明文密钥自检:导出文件里只允许出现密文。任何一条看起来像
# "sk-xxxx" 的明文都说明有人绕过 secrets.py 直接写了库。
leaks = [f"{s['provider']}/{s['model']}" for s in specs
if s.get("api_key_encrypted", "").startswith("sk-")]
if leaks:
sys.exit(f"疑似明文 API Key 直接入库:{leaks};请先在模型中心重填一次。")
write_json(out / "model_specs.json", {
"_meta": {"table": "model_specs", "count": len(specs),
"defaults": sorted(f"{s['media_type']}={s['provider']}/{s['model']}"
for s in specs if s["is_default"]),
"exported_at": dt.datetime.now().isoformat(timespec="seconds")},
"rows": specs})
write_json(out / "model_secret_keys.json", {
"_meta": {"table": "model_secret_keys", "count": len(keys),
"note": "密钥加密主密钥;缺了它 model_specs.api_key_encrypted 解不开"},
"rows": keys})
write_json(out / "users.json", {
"_meta": {"table": "users", "count": len(admins),
"note": "只导管理员;口令是 PBKDF2 哈希,不含明文"},
"rows": [{k: v for k, v in u.items() if k not in USER_DROP} for u in admins]})
if args.with_platform_library:
lib = {}
for name in PLATFORM_TABLES:
if name not in Base.metadata.tables:
print(f" ! 跳过 {name}(模型里没有这张表)")
continue
rows = [dict(r._mapping) for r in db.execute(text(f"SELECT * FROM {name}")).all()]
lib[name] = rows
print(f" 平台资源库 {name}: {len(rows)} 行")
write_json(out / "platform_library.json", {
"_meta": {"tables": list(lib), "note": "媒体文件在 media/platform/,需另行拷贝"},
"rows": lib})
# 解密自检:能解出内容才说明「密文 + 主密钥」这一对是完整的。
from app.services import secrets as secret_svc
checked = [f"{s['provider']}/{s['model']}" for s in specs if s.get("api_key_encrypted")]
ok = [f"{s['provider']}/{s['model']}" for s in specs
if s.get("api_key_encrypted")
and secret_svc.decrypt_value(db, s["api_key_encrypted"])]
if len(ok) != len(checked):
sys.exit(f"主密钥与密文不匹配:{set(checked) - set(ok)} 解不开,导出会导致换机后 Key 丢失。")
print(f"\n导出完成 → {out}")
print(f" 模型配置 {len(specs)} 条(默认:{', '.join(s['media_type'] for s in specs if s['is_default'])})"
f" · 带 Key {len(ok)} 条且解密通过 · 管理员 {len(admins)} 个")
print(" ⚠️ scripts/seed/ 含加密主密钥,已在 .gitignore 内;部署包里必须带上它。")
finally:
db.close()
# ================================================================== install
def _keys_in_use(db) -> int:
"""本机已有几条模型配置真的用上了当前主密钥(密文能被它解开)。"""
from app.services import secrets as secret_svc
used = 0
for s in db.scalars(select(ModelSpec)).all():
if s.api_key_encrypted and secret_svc.decrypt_value(db, s.api_key_encrypted):
used += 1
return used
def _install_master_key(db, rows, *, dry: bool, force: bool) -> None:
have = db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()
if not rows:
print(" · 导出包里没有主密钥(开发机可能用 MODEL_SECRET_KEY 环境变量):跳过")
return
if not have:
for data in rows:
row = ModelSecretKey()
_apply(row, data, skip=("id",))
db.add(row)
print(f" ✓ 主密钥已导入({len(rows)} 条)")
return
if have[0].key_value == rows[0]["key_value"]:
print(" · 主密钥已一致,跳过")
return
msg = (f"目标库已有主密钥且与导出包不同(现有 {have[0].key_value[:8]}… vs "
f"导入 {rows[0]['key_value'][:8]}…)")
in_use = _keys_in_use(db)
if in_use == 0 and not force:
# 「先起了服务、后导数据」是最常见的顺序:应用首启会自己生成一把主密钥,
# 但此刻还没有任何模型配置真的依赖它(mock 没有 Key),换掉零损失。
# 拦在这里只会让人以为工具坏了,所以直接换,并把话讲明白。
for data in rows:
_apply(have[0], data, skip=("id",))
print(f" · {msg}\n · 本机没有任何已解密成功的 API Key 依赖旧密钥 → 直接换成部署包的主密钥")
return
if not force:
sys.exit(msg + f";本机有 {in_use} 条模型配置的 Key 靠它解密,"
"覆盖后全部要重填。确认就加 --force-master-key。")
for data in rows:
_apply(have[0], data, skip=("id",))
print(f" ! {msg} → 已按 --force-master-key 覆盖,请复查模型中心的 Key")
def _install_specs(db, rows, *, dry: bool, only_missing: bool = False) -> None:
added = updated = skipped = 0
for data in rows:
key = {"provider": data["provider"], "model": data.get("model", ""),
"media_type": data["media_type"]}
row = db.scalars(select(ModelSpec).filter_by(**key)).first()
if row is not None and only_missing:
skipped += 1
continue
if row is None:
row = ModelSpec()
_apply(row, data, skip=("id", "created_at", "updated_at"))
db.add(row)
added += 1
else:
_apply(row, data, skip=("id", "created_at"))
updated += 1
# 默认位以导出包为准:同能力下不在包里的旧默认必须让位,
# 否则 seed_specs / resolve_* 会因为排序偶然选中开发机没在用的那个模型。
db.flush()
for media_type in {d["media_type"] for d in rows}:
want = next((d for d in rows if d["media_type"] == media_type and d["is_default"]), None)
if want is None:
continue
others = db.query(ModelSpec).filter(
ModelSpec.media_type == media_type,
ModelSpec.is_default.is_(True),
~((ModelSpec.provider == want["provider"]) & (ModelSpec.model == want.get("model", "")))
).update({"is_default": False})
if others:
print(f" · {media_type}: 取消 {others} 条旧默认,改为 "
f"{want['provider']}/{want['model']}")
if not dry:
db.commit()
note = ",--only-missing 跳过 " + str(skipped) if skipped else ""
verb = ("按部署包覆盖" if updated and not only_missing else "写入")
print(f" ✓ 模型配置:新增 {added},{verb} {updated}{note}"
+ ("" if not updated or only_missing else "(更新=用部署包的值覆盖服务器上的同名模型)"))
def _install_users(db, rows, *, dry: bool) -> None:
for data in rows:
row = db.scalars(select(User).where(User.username == data["username"])).first()
if row is None:
row = User()
_apply(row, data, skip=("id", "created_at", "updated_at"))
row.must_change_password = False
db.add(row)
print(f" ✓ 管理员 {row.username}(role={row.role})已创建,"
f"口令沿用导出时的哈希——登录后请立即修改")
else:
# 已存在的账号不重置口令:这台机器可能早改过密码,
# 覆盖成开发机的口令是个反直觉的倒退。
_apply(row, data, skip=("id", "created_at", "updated_at", "password_hash",
"must_change_password"))
diff = (data.get("password_hash") or "") != row.password_hash
print(f" · 管理员 {row.username} 已存在:只同步角色/显示名,不改口令"
+ ("(注意:它的口令与部署包不同,登录请用这台机器的口令)" if diff else ""))
if not dry:
db.commit()
def _install_platform(db, tables: dict, *, dry: bool) -> None:
for name in PLATFORM_TABLES:
rows = tables.get(name) or []
if not rows:
continue
table = Base.metadata.tables.get(name)
if table is None:
print(f" ! 跳过 {name}(当前代码没有这张表)")
continue
cols = [c.key for c in inspect(table).columns if c.key != "id"]
for data in rows:
pk = data.get("id")
if db.execute(text(f"SELECT 1 FROM {name} WHERE id=:i"), {"i": pk}).first():
continue
col_sql = ", ".join(cols)
val_sql = ", ".join(f":{c}" for c in cols)
db.execute(text(f"INSERT INTO {name} (id, {col_sql}) VALUES (:id, {val_sql})"),
{**{c: data.get(c) for c in cols}, "id": pk})
print(f" ✓ 平台资源库 {name}: 导入后共 "
f"{db.execute(text(f'SELECT COUNT(*) FROM {name}')).scalar()} 行")
if not dry:
db.commit()
def do_install(args) -> None:
src = Path(args.src)
specs = read_json(src, "model_specs.json")["rows"]
keys = read_json(src, "model_secret_keys.json")["rows"]
users = read_json(src, "users.json")["rows"]
lib = None
if args.with_platform_library:
lib = read_json(src, "platform_library.json")["rows"]
if args.schema:
init_db() # 建表 + 列迁移,幂等
print(f" ✓ 表结构就绪:{engine.url.render_as_string(hide_password=True)}")
db = SessionLocal()
try:
print("安装部署数据集(幂等,可反复执行)")
_install_master_key(db, keys, dry=args.dry_run, force=args.force_master_key)
_install_specs(db, specs, dry=args.dry_run, only_missing=args.only_missing)
_install_users(db, users, dry=args.dry_run)
if lib is not None:
_install_platform(db, lib, dry=args.dry_run)
if args.dry_run:
db.rollback()
print("\n--dry-run:未写入任何数据")
return
# 装完就验一次解密:这一步不过,说明主密钥和密文不是一对,
# 与其等业务报「缺 Key」,不如现在停下来。
from app.services import secrets as secret_svc
bad = [f"{s.provider}/{s.model}" for s in db.scalars(select(ModelSpec)).all()
if s.api_key_encrypted and not secret_svc.decrypt_value(db, s.api_key_encrypted)]
defaults = {s.media_type: f"{s.provider}/{s.model}"
for s in db.scalars(select(ModelSpec).where(ModelSpec.is_default.is_(True))).all()}
admins = db.scalars(select(User.username).where(User.role == "admin")).all()
print("\n校验")
print(f" 默认模型:{', '.join(f'{k}={v}' for k, v in sorted(defaults.items())) or '(无)'}")
print(f" 管理员:{', '.join(admins) or '(无)'}")
if bad:
sys.exit(f" ✗ 这些模型的 API Key 解不开(主密钥不匹配):{bad}")
print(" ✓ 所有 API Key 解密通过")
finally:
db.close()
# ================================================================== CLI
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0],
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="cmd", required=True)
e = sub.add_parser("export", help="导出模型配置 + 主密钥 + 管理员到 JSON")
e.add_argument("--out", default=str(SEED_DIR), help="输出目录(默认 scripts/seed)")
e.add_argument("--with-platform-library", action="store_true", help="顺带导出平台资源库")
e.set_defaults(func=do_export)
i = sub.add_parser("install", help="把 JSON 装进目标库(服务器首次部署用)")
i.add_argument("--src", default=str(SEED_DIR), help="JSON 目录(默认 scripts/seed)")
i.add_argument("--no-schema", dest="schema", action="store_false", help="不先建表")
i.add_argument("--dry-run", action="store_true", help="只报告将要发生什么")
i.add_argument("--force-master-key", action="store_true", help="允许覆盖已有主密钥")
i.add_argument("--with-platform-library", action="store_true", help="顺带导入平台资源库")
i.add_argument("--only-missing", action="store_true",
help="模型配置只补缺,不覆盖服务器上已有的同名模型")
i.set_defaults(func=do_install)
args = ap.parse_args()
args.func(args)
if __name__ == "__main__":
main()