nex_animation/scripts/seed.py

844 lines
44 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""部署数据集的体检、导出与安装:表结构 + 模型配置 + 管理员账号。
服务器部署**不搬开发库**。14MB 的 nex_animation.db 里绝大多数是开发过程产物
(项目、镜头、任务、事件日志、演示账号),新环境真正需要的是三样东西:
| 内容 | 来源 |
|---------------------|----------------------------------------|
| 表结构 + 列迁移 | `app.db.init_db()`(应用启动自带) |
| 模型配置 | `model_specs` + `model_secret_keys` |
| 管理员账号 | `users` 里 role=admin 的那几条 |
模型配置必须带走,否则新环境只有 `seed_specs()` 播的 mock,得在模型管理中心
从零填一遍 ComfyUI / Qwen / IndexTTS 的端点与工作流;而 API Key 是
`app/services/secrets.py` 用库内主密钥加密的,所以 **`model_secret_keys` 那一行
必须和密文一起走**——换机器只带 model_specs,密文会解成空串,
表现为「模型管理中心显示已配置,派发却报缺 Key」。
# 开发机:体检 →(必要时校准)→ 导出
.venv/bin/python scripts/seed.py doctor # 只量不改,退出码非 0 = 有别部署
.venv/bin/python scripts/seed.py doctor --fix # 把库补齐到当前代码(只补缺)
.venv/bin/python scripts/seed.py export --fix # 校准 + 体检 + 导出(部署前就这一条)
.venv/bin/python scripts/seed.py export --with-platform-library # 顺带平台资源库
# 服务器:安装(空库;先量包再建表再灌数据,幂等,可反复执行)
.venv/bin/python scripts/seed.py install
.venv/bin/python scripts/seed.py install --only-missing # 只补库里没有的,本机已调的不动
.venv/bin/python scripts/seed.py doctor --src scripts/seed # 只量部署包(deploy.sh 用这条)
.venv/bin/python scripts/seed.py boundary # 数据边界一览(server.sh status 用这条)
**体检(doctor)是这三条命令共用的同一把尺子**:列对得上表结构、适配器在代码里注册过、
`modes` 不比代码声明的能力少、每类能力恰好一个默认、密文能被包里的主密钥解开、管理员登得进来。
导出前量库、安装前量包,两边用的是同一个 `audit_dataset()`——
否则「导出时没说、装上才发现」的老路会一直走下去(详见下面体检小节的注释)。
安装顺序刻意是「主密钥 → 模型配置 → 账号」:主密钥先到位,后面导入的密文才可解。
`model_secret_keys` 已有值且与导出值不同时默认**中止**(不是覆盖),
因为覆盖会让这台机器上原有模型配置的 Key 全部解不开——真要做这种手术请加
--force-master-key,并准备好在模型管理中心重填所有 Key。
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import sys
from pathlib import Path
from types import SimpleNamespace
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
from sqlalchemy import inspect, select, text, types as sa_types # noqa: E402
from sqlalchemy.orm import Session # noqa: E402
from app.db import Base, SessionLocal, engine, init_db # noqa: E402
from app.models import ModelSecretKey, ModelSpec, User # noqa: E402
SEED_DIR = ROOT / "scripts" / "seed"
# 平台资源库(可选):严格说不属于「部署必需」,但换机时重建 37 条素材很费手工,
# 所以做成显式开关;媒体文件在 media/platform/ 下,不在数据库里,需另行拷贝。
PLATFORM_TABLES = ("platform_characters", "platform_scenes", "platform_voices",
"platform_media", "platform_media_versions")
# users 里这些列是「在哪台机器上登过」的流水,跨机没有意义,还会误导审计。
USER_DROP = ("last_login_at", "last_login_ip", "login_count")
# ------------------------------------------------------------------ 行 ⇄ dict
def row_to_dict(obj) -> dict:
"""ORM 行转 JSON 可序列化 dict(时间转 ISO,JSON 列原样,二进制转 base64)。"""
out: dict = {}
for col in inspect(obj.__table__).columns:
val = getattr(obj, col.key)
if isinstance(val, dt.datetime):
val = val.isoformat(sep=" ", timespec="microseconds")
elif isinstance(val, (bytes, bytearray)):
val = bytes(val).hex()
elif isinstance(val, dt.date):
val = val.isoformat()
out[col.key] = val
return out
def write_json(path: Path, payload) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(f" 写出 {path.relative_to(ROOT)}")
def read_json(src: Path, name: str):
p = src / name
if not p.is_file():
sys.exit(f"缺少 {p};先在开发机跑 .venv/bin/python scripts/seed.py export")
return json.loads(p.read_text(encoding="utf-8"))
def _apply(row, data: dict, *, skip: tuple[str, ...] = ()) -> None:
mapper = inspect(row.__table__)
for key, val in data.items():
if key in skip or key not in mapper.c:
continue
col = mapper.c[key]
if isinstance(val, str):
# JSON 导出把时间写成了 ISO 字符串,回填时必须还原成 datetime:
# SQLite 驱动遇到字符串直接抛 «only accepts Python datetime and date
# objects as input»,Postgres 则可能按会话时区误读。
# 本库的时间列一律是 DateTime(timezone=True),写进去的是带 UTC 的值,
# 读回来却是 naive(见 app/models/base.py:as_utc),所以这里补回 UTC。
if isinstance(col.type, sa_types.DateTime):
parsed = dt.datetime.fromisoformat(val)
val = parsed.replace(tzinfo=dt.timezone.utc) if parsed.tzinfo is None else parsed
elif isinstance(col.type, sa_types.Date):
val = dt.date.fromisoformat(val)
setattr(row, key, val)
# ================================================================== 体检(doctor)
# 为什么必须有这一节:部署数据集是「快照」,代码是「活水」。快照一旦落后于代码,
# 服务器上的表现不是报错而是**静默降级**,比崩掉更难查:
#
# * 导出时 modes 里还没有 `voice_instruct` → 装上之后「音色描述直出」整个功能
# 在模型管理中心和声线页都看不见(`services/timbre.py:supports_instruct()` 读的是
# `model_specs.modes`,不是代码里的 Capabilities),排查的人会去查 ComfyUI;
# * 导出之后代码给工作流加了新节点键(`generate_full_node` / `instruct_node`)
# → 配置里少键,走到描述词分支才炸;
# * 导出之后 ORM 加了非空列 → `install` 在服务器上抛 IntegrityError,
# 而这句话离真正的根因(数据集是旧的)隔了 40 行 traceback;
# * 开发机改了模型、删了模型,忘了重新 export → 包里还是上周那套端点。
#
# 所以体检做三件事:**列对得上、能力对得上、密钥对得上**,并且把「代码比数据新」
# 当成错误而不是提示——它唯一的修复动作在开发机上(`doctor --fix` + 重新 export),
# 在服务器上加 --allow-drift 硬闯才是合理的逃生门。
CAPABILITY_MEDIA_TYPES = ("video", "image", "tts", "llm")
SEED_TABLES = ("model_specs", "model_secret_keys", "users")
# 这几张表非空 = 目标库已经在跑业务,不该拿它当首装库
BUSINESS_TABLES = ("projects", "shots", "tasks", "assets", "generation_records", "event_logs")
class Report:
"""体检结论:error 会挡住 export/install,warn 只打印。"""
def __init__(self) -> None:
self.items: list[tuple[str, str, str]] = []
def error(self, code: str, msg: str) -> None:
self.items.append(("error", code, msg))
def warn(self, code: str, msg: str) -> None:
self.items.append(("warn", code, msg))
@property
def errors(self) -> list[tuple[str, str, str]]:
return [i for i in self.items if i[0] == "error"]
@property
def warns(self) -> list[tuple[str, str, str]]:
return [i for i in self.items if i[0] == "warn"]
def show(self) -> None:
for level, code, msg in self.items:
mark = "XX" if level == "error" else " !"
color = "\033[31m" if level == "error" else "\033[33m"
print(f" {color}{mark}\033[0m [{code}] {msg}")
if not self.items:
print(" \033[32mok\033[0m 没有告警")
def _adapter_of(media_type: str, adapter: str, provider: str):
"""按 media_type/adapter 取适配器;取不到返回 None(调用方记错误)。"""
from app.providers import registry as model_registry
from app.providers.base import FatalProviderError
try:
return model_registry.adapter_for_spec(
SimpleNamespace(media_type=media_type, adapter=adapter, provider=provider, config_json={}))
except FatalProviderError:
return None
def _code_defaults_for(adapter_name: str) -> dict:
"""代码里维护的本地工作流默认配置(用于「只补缺失键」,不覆盖既有值)。"""
from app.providers import registry as model_registry
builders = {"comfyui": model_registry.comfyui_config,
"comfyui_image": model_registry.comfyui_image_config,
"comfyui_tts": model_registry.comfyui_tts_config}
build = builders.get((adapter_name or "").lower())
return build() if build else {}
def _rows_decrypt(keys: list[dict], specs: list[dict]) -> list[str]:
"""用「包里的主密钥」试解「包里的密文」,返回解不开的那几条。
临时摘掉 MODEL_SECRET_KEY:不然这里量到的是本机环境变量的密钥,
服务器上没有这个变量时反而会漏报(用 env 的那条检查单独告警)。
"""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from app.services import secrets as secret_svc
ciphered = [(f"{s.get('provider')}/{s.get('model')}", s.get("api_key_encrypted") or "")
for s in specs if s.get("api_key_encrypted")]
if not ciphered:
return []
if not keys or not (keys[0].get("key_value") or ""):
return [name for name, _ in ciphered]
engine = create_engine("sqlite://", future=True)
Base.metadata.create_all(engine, tables=[ModelSecretKey.__table__])
session = sessionmaker(bind=engine, class_=Session, future=True)()
row = ModelSecretKey()
_apply(row, keys[0], skip=("id",))
session.add(row)
session.commit()
saved = os.environ.pop("MODEL_SECRET_KEY", None)
try:
bad = [name for name, cipher in ciphered
if not secret_svc.decrypt_value(session, cipher)]
finally:
if saved is not None:
os.environ["MODEL_SECRET_KEY"] = saved
session.close()
engine.dispose()
return bad
def audit_specs(rep: Report, specs: list[dict], keys: list[dict], *, origin: str) -> None:
"""模型配置:列 ⇄ 表结构、适配器、能力位、默认位、端点、密钥。"""
table = Base.metadata.tables["model_specs"]
known_cols = {c.key for c in inspect(table).columns}
required_cols = {c.key for c in inspect(table).columns
if not c.nullable and c.default is None and c.server_default is None
and c.key != "id"}
def label(row: dict) -> str:
return f"{row.get('provider')}/{row.get('model')}({row.get('media_type')})"
# ---- 列:数据集与当前表结构的差集
unknown = sorted({k for r in specs for k in r} - known_cols - {"_meta"})
if unknown:
rep.warn("列", f"{origin} 里有当前表结构没有的列,导入时会被静默丢弃:{unknown}")
for row in specs:
missing = sorted(required_cols - set(row))
if missing:
rep.error("列", f"{label(row)} 缺非空列 {missing}:数据集比表结构旧,"
"在开发机跑 scripts/seed.py doctor --fix 再 export")
break
# ---- 适配器 + 能力位
for row in specs:
adapter = _adapter_of(row.get("media_type") or "", row.get("adapter") or "",
row.get("provider") or "")
if adapter is None:
rep.error("适配器", f"{label(row)} 的 adapter={row.get('adapter')!r} 代码里没注册,"
"派发时直接 FatalProviderError")
continue
try:
caps = adapter.capabilities(row.get("config_json") or {})
except Exception as exc: # noqa: BLE001
rep.warn("适配器", f"{label(row)} 读适配器能力失败:{type(exc).__name__}: {exc}")
continue
drift = sorted(set(caps.modes) - set(row.get("modes") or []))
if drift:
rep.error("能力位", f"{label(row)} 的 modes 缺 {drift}:代码已支持、数据集还是旧的。"
"装上去这些能力在模型管理中心/派发热路径里都不存在"
"(修复:开发机 doctor --fix + 重新 export;确认要收窄就加 --allow-drift)")
# ---- 默认位:每类能力必须恰好一个默认,且它自己必须是启用的
for media_type in sorted({r.get("media_type") for r in specs}):
group = [r for r in specs if r.get("media_type") == media_type]
wanted = [r for r in group if r.get("is_default")]
if len(wanted) != 1:
rep.error("默认位", f"{media_type} 有 {len(wanted)} 个默认模型"
f"({'、'.join(label(r) for r in wanted) or '无'}):"
"0 个会让这类能力退回 mock,>1 个则派发按排序偶然选中")
continue
if not wanted[0].get("enabled", True):
rep.error("默认位", f"{media_type} 的默认模型 {label(wanted[0])} 是停用状态")
missing_caps = [m for m in CAPABILITY_MEDIA_TYPES
if not any(r.get("media_type") == m and not (r.get("provider") or "").startswith("mock")
for r in specs)]
if missing_caps:
rep.warn("默认位", f"这几类能力只有 mock 可用:{missing_caps}(部署后跑不出真片)")
# ---- 端点:真实模型没端点 = 装上去必然连不上
for row in specs:
if (row.get("provider") or "").startswith("mock"):
continue
cfg = row.get("config_json") or {}
base = str(cfg.get("base_url") or "").strip()
if not base:
rep.error("端点", f"{label(row)} 的 config_json.base_url 是空的:服务器上无处可连")
elif base.startswith(("http://localhost", "http://127.0.0.1")):
rep.warn("端点", f"{label(row)} 指向 localhost:服务器与模型不同机时连不上")
if row.get("media_type") == "llm" and not (row.get("api_key_encrypted") or ""):
rep.warn("端点", f"{label(row)} 没有 API Key(若该端点免鉴权可忽略)")
# ---- 密钥:主密钥与密文必须成对
if os.environ.get("MODEL_SECRET_KEY"):
rep.warn("密钥", "本机设了 MODEL_SECRET_KEY:密文是它加密的,包里的 model_secret_keys "
"在服务器上不起作用——部署时必须把同一个变量写进服务器 .env")
ciphered = [f"{r.get('provider')}/{r.get('model')}" for r in specs if r.get("api_key_encrypted")]
leaks = [f"{r.get('provider')}/{r.get('model')}" for r in specs
if str(r.get("api_key_encrypted") or "").startswith("sk-")]
if leaks:
rep.error("密钥", f"疑似明文 API Key 直接入库:{leaks};在模型管理中心重填一次再导出")
bad = _rows_decrypt(keys, specs)
if bad:
rep.error("密钥", f"这些模型配置的 Key 用包内主密钥解不开:{bad}"
"(表现是模型管理中心显示已配置、派发报缺 Key)")
def audit_users(rep: Report, users: list[dict], *, origin: str = "数据集") -> None:
"""管理员:能登进来、没有跨机无意义的登录流水列。"""
from app.services import auth as auth_service
table = Base.metadata.tables["users"]
known_cols = {c.key for c in inspect(table).columns}
unknown = sorted({k for u in users for k in u} - known_cols)
if unknown:
rep.warn("账号", f"{origin} 里有表结构没有的列,导入时会被丢弃:{unknown}")
# 「有没有登录流水列」只对导出文件成立:直接从库里读出来的行当然带这些列,
# 是 export 负责在写文件前剔掉它们(USER_DROP)。
if origin != "开发库":
stale = [k for u in users for k in USER_DROP if k in u]
if stale:
rep.warn("账号", f"{origin} 带着机器相关的登录流水列 {sorted(set(stale))}:"
"跨机无意义,建议重新 export")
admins = [u for u in users if u.get("role") == "admin"]
if not admins:
rep.error("账号", "部署包里没有 role=admin 的账号:服务器首启会另建一个 admin,"
"口令与这台机器不同")
for u in users:
if not (u.get("password_hash") or ""):
rep.error("账号", f"账号 {u.get('username')} 没有口令哈希,登不进去")
continue
try:
auth_service.verify_password("probe-only", u["password_hash"])
except Exception as exc: # noqa: BLE001
rep.error("账号", f"账号 {u.get('username')} 的 password_hash 解析失败:"
f"{type(exc).__name__}: {exc}")
if not u.get("is_active", True):
rep.warn("账号", f"账号 {u.get('username')} 在导出机上是停用状态")
def audit_dataset(specs: list[dict], keys: list[dict], users: list[dict], *, origin: str,
rep: Report | None = None) -> Report:
"""量一份数据集。`rep` 传进来时可以与其他检查(如目标库现状)汇到同一份结论里。"""
rep = rep or Report()
if not specs:
rep.error("数据集", f"{origin}:model_specs 一条都没有,装上只有 mock")
audit_specs(rep, specs, keys, origin=origin)
audit_users(rep, users, origin=origin)
return rep
def _table_counts(bind, names: tuple[str, ...]) -> dict[str, int]:
from sqlalchemy import text as sa_text
existing = set(inspect(bind).get_table_names())
out: dict[str, int] = {}
with bind.begin() as conn:
for name in names:
if name in existing:
out[name] = conn.execute(sa_text(f"SELECT COUNT(*) FROM {name}")).scalar() or 0
return out
def audit_orphan_tables(rep: Report, bind) -> None:
"""库里躺着代码不认识的表 = 历史手术留下的备份表,它不该跟着部署走。
只报不删:工程规范 §6 说得很清楚,这种表要人确认后手工 DROP,
脚本偷偷删表等于把「备份」这个唯一的退路删掉。
"""
orphan = sorted(set(inspect(bind).get_table_names()) - set(Base.metadata.tables))
if orphan:
rep.warn("边界", f"库里有代码不认识的表 {orphan}:历史遗留备份表,"
"确认后手工 DROP(它不影响部署包,但会让 status 的数据边界说不清)")
def audit_target_db(rep: Report, db, *, only_missing: bool = False) -> None:
"""install 的目标库现状:别把部署数据灌进一个正在跑业务的库。
「库里有业务数据」在两档模式下不是一回事,所以报法也不同:
· 覆盖式(不带 --only-missing / 容器 AUTO_SEED=force)真的会用包里的值改掉同名模型,
指错库就是事故 → 保持黄色告警;
· --only-missing 一行都不动已有数据。而容器每次重启都会走到这里,
跑了几天的服务器上必然有项目、有事件日志——每次都黄一条,
等于训练运维无视黄条,所以降级成一行普通提示(只补空缺这件事后面还会再打印一遍)。
孤儿表那条与模式无关,照旧告警。
"""
counts = {k: v for k, v in _table_counts(db.bind, BUSINESS_TABLES).items() if v}
if counts:
if only_missing:
print(f" · 目标库已有业务数据 {counts}:--only-missing 只补库里没有的行,已有配置不动")
else:
rep.warn("边界", f"目标库里已有业务数据 {counts}:部署数据集只该进干净库,"
"确认 DATABASE_URL 没指到开发库(覆盖式导入会改掉这些模型配置)")
audit_orphan_tables(rep, db.bind)
# ------------------------------------------------------------------ 校准(--fix)
def calibrate_specs(db, *, apply: bool) -> list[str]:
"""把开发库的模型配置补齐到当前代码:只加不减,从不覆盖已有值。
只做两件事,因为这两件事代码才是权威:
1. `modes` 补上适配器已声明、行里却缺的能力位;
2. `config_json` 补上代码默认配置里新增、行里却缺的键(工作流节点等)。
其余字段(端点、限流、单价、notes)都是人工在页面上调的,代码一律不猜。
"""
changes: list[str] = []
for spec in db.scalars(select(ModelSpec).order_by(ModelSpec.id)).all():
adapter = _adapter_of(spec.media_type, spec.adapter, spec.provider)
if adapter is None:
continue
caps = adapter.capabilities(spec.config_json or {})
missing_modes = sorted(set(caps.modes) - set(spec.modes or []))
defaults = _code_defaults_for(spec.adapter)
cfg = dict(spec.config_json or {})
missing_cfg = sorted(k for k in defaults if k not in cfg)
if not missing_modes and not missing_cfg:
continue
if apply:
if missing_modes:
spec.modes = list(spec.modes or []) + missing_modes
for key in missing_cfg:
cfg[key] = defaults[key]
spec.config_json = cfg
changes.append(f"{spec.provider}/{spec.model}({spec.media_type}):"
+ (f"补 modes {missing_modes}" if missing_modes else "")
+ ("、" if missing_modes and missing_cfg else "")
+ (f"补 config_json 键 {missing_cfg}" if missing_cfg else ""))
if apply and changes:
db.commit()
return changes
# ================================================================== export
def do_export(args) -> None:
out = Path(args.out)
db = SessionLocal()
try:
if args.fix:
changes = calibrate_specs(db, apply=True)
if changes:
print("校准开发库(只补缺,不覆盖人工调过的值)")
for line in changes:
print(f" · {line}")
# 不在这里量「库里有没有业务数据」:开发库当然有,那条检查是给 install
# 挡「把部署数据灌进正在跑的库」用的(见 do_install)。
specs = [row_to_dict(s) for s in db.scalars(select(ModelSpec).order_by(ModelSpec.id)).all()]
keys = [row_to_dict(k) for k in db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()]
admins = [row_to_dict(u) for u in db.scalars(
select(User).where(User.role == "admin").order_by(User.id)).all()]
if not specs:
sys.exit("model_specs 是空的:这台机器还没配过模型,导出去也只是 mock。")
if not admins:
sys.exit("users 里没有 admin 角色账号。")
# 导出与安装用同一把尺子量:量库一次、装前量包一次,不会出现
# 「导出时没说、装到服务器才发现」的错配。明文 Key 自检也在体检里
# (secrets.py 只该写密文,出现 sk- 开头的明文就是有人绕过了封装)。
rep = audit_dataset(specs, keys, admins, origin="开发库")
print("体检(部署数据集)")
rep.show()
print_dataset_summary(rep, specs, keys, admins)
if rep.errors and not args.allow_drift:
sys.exit("导出中止:上面是 error。修完再导;确认要带着这些问题部署,加 --allow-drift。")
write_json(out / "model_specs.json", {
"_meta": {"table": "model_specs", "count": len(specs),
"defaults": sorted(f"{s['media_type']}={s['provider']}/{s['model']}"
for s in specs if s["is_default"]),
"exported_at": dt.datetime.now().isoformat(timespec="seconds")},
"rows": specs})
write_json(out / "model_secret_keys.json", {
"_meta": {"table": "model_secret_keys", "count": len(keys),
"note": "密钥加密主密钥;缺了它 model_specs.api_key_encrypted 解不开"},
"rows": keys})
write_json(out / "users.json", {
"_meta": {"table": "users", "count": len(admins),
"note": "只导管理员;口令是 PBKDF2 哈希,不含明文"},
"rows": [{k: v for k, v in u.items() if k not in USER_DROP} for u in admins]})
if args.with_platform_library:
lib = {}
for name in PLATFORM_TABLES:
if name not in Base.metadata.tables:
print(f" ! 跳过 {name}(模型里没有这张表)")
continue
rows = [dict(r._mapping) for r in db.execute(text(f"SELECT * FROM {name}")).all()]
lib[name] = rows
print(f" 平台资源库 {name}: {len(rows)} 行")
write_json(out / "platform_library.json", {
"_meta": {"tables": list(lib), "note": "媒体文件在 media/platform/,需另行拷贝"},
"rows": lib})
# 解密自检:能解出内容才说明「密文 + 主密钥」这一对是完整的。
from app.services import secrets as secret_svc
checked = [f"{s['provider']}/{s['model']}" for s in specs if s.get("api_key_encrypted")]
ok = [f"{s['provider']}/{s['model']}" for s in specs
if s.get("api_key_encrypted")
and secret_svc.decrypt_value(db, s["api_key_encrypted"])]
if len(ok) != len(checked):
sys.exit(f"主密钥与密文不匹配:{set(checked) - set(ok)} 解不开,导出会导致换机后 Key 丢失。")
print(f"\n导出完成 → {out}")
print(f" 模型配置 {len(specs)} 条(默认:{', '.join(s['media_type'] for s in specs if s['is_default'])})"
f" · 带 Key {len(ok)} 条且解密通过 · 管理员 {len(admins)} 个")
print(" ⚠️ scripts/seed/ 含加密主密钥,已在 .gitignore 内;部署包里必须带上它。")
finally:
db.close()
# ================================================================== install
def _keys_in_use(db) -> int:
"""本机已有几条模型配置真的用上了当前主密钥(密文能被它解开)。"""
from app.services import secrets as secret_svc
used = 0
for s in db.scalars(select(ModelSpec)).all():
if s.api_key_encrypted and secret_svc.decrypt_value(db, s.api_key_encrypted):
used += 1
return used
def _install_master_key(db, rows, *, dry: bool, force: bool) -> None:
have = db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()
if not rows:
print(" · 导出包里没有主密钥(开发机可能用 MODEL_SECRET_KEY 环境变量):跳过")
return
if not have:
for data in rows:
row = ModelSecretKey()
_apply(row, data, skip=("id",))
db.add(row)
print(f" ✓ 主密钥已导入({len(rows)} 条)")
return
if have[0].key_value == rows[0]["key_value"]:
print(" · 主密钥已一致,跳过")
return
msg = (f"目标库已有主密钥且与导出包不同(现有 {have[0].key_value[:8]}… vs "
f"导入 {rows[0]['key_value'][:8]}…)")
in_use = _keys_in_use(db)
if in_use == 0 and not force:
# 「先起了服务、后导数据」是最常见的顺序:应用首启会自己生成一把主密钥,
# 但此刻还没有任何模型配置真的依赖它(mock 没有 Key),换掉零损失。
# 拦在这里只会让人以为工具坏了,所以直接换,并把话讲明白。
for data in rows:
_apply(have[0], data, skip=("id",))
print(f" · {msg}\n · 本机没有任何已解密成功的 API Key 依赖旧密钥 → 直接换成部署包的主密钥")
return
if not force:
sys.exit(msg + f";本机有 {in_use} 条模型配置的 Key 靠它解密,"
"覆盖后全部要重填。确认就加 --force-master-key。")
for data in rows:
_apply(have[0], data, skip=("id",))
print(f" ! {msg} → 已按 --force-master-key 覆盖,请复查模型管理中心的 Key")
def _install_specs(db, rows, *, dry: bool, only_missing: bool = False) -> None:
added = updated = skipped = 0
added_keys: set[tuple[str, str, str]] = set()
for data in rows:
key = {"provider": data["provider"], "model": data.get("model", ""),
"media_type": data["media_type"]}
row = db.scalars(select(ModelSpec).filter_by(**key)).first()
if row is not None and only_missing:
skipped += 1
continue
if row is None:
row = ModelSpec()
_apply(row, data, skip=("id", "created_at", "updated_at"))
db.add(row)
added += 1
added_keys.add((row.provider, row.model, row.media_type))
else:
_apply(row, data, skip=("id", "created_at"))
updated += 1
# 默认位以导出包为准:同能力下不在包里的旧默认必须让位,
# 否则 seed_specs / resolve_* 会因为排序偶然选中开发机没在用的那个模型。
# 例外是 --only-missing:这个开关的承诺是「这台机器已有的东西一概不动」,
# 所以只有当包里的默认模型是这次**新插进来**的那一条时才设默认。
# 早先无例外时翻过车:一次「新增 0、跳过 8」的导入什么都没写,却把服务器上
# 自选的 tts 默认悄悄改回开发机那套,日志上还是一片祥和。
db.flush()
kept: list[str] = []
for media_type in {d["media_type"] for d in rows}:
want = next((d for d in rows if d["media_type"] == media_type and d["is_default"]), None)
if want is None:
continue
if only_missing and (want["provider"], want.get("model", ""), media_type) not in added_keys:
kept.append(f"{media_type}={want['provider']}/{want['model']}")
continue
others = db.query(ModelSpec).filter(
ModelSpec.media_type == media_type,
ModelSpec.is_default.is_(True),
~((ModelSpec.provider == want["provider"]) & (ModelSpec.model == want.get("model", "")))
).update({"is_default": False})
if others:
print(f" · {media_type}: 取消 {others} 条旧默认,改为 "
f"{want['provider']}/{want['model']}")
if kept:
print(f" · --only-missing:保留本机默认位不改(包里的是 {'、'.join(sorted(kept))})")
if not dry:
db.commit()
note = ",--only-missing 跳过 " + str(skipped) if skipped else ""
verb = ("按部署包覆盖" if updated and not only_missing else "写入")
print(f" ✓ 模型配置:新增 {added},{verb} {updated}{note}"
+ ("" if not updated or only_missing else "(更新=用部署包的值覆盖服务器上的同名模型)"))
def _install_users(db, rows, *, dry: bool) -> None:
for data in rows:
row = db.scalars(select(User).where(User.username == data["username"])).first()
if row is None:
row = User()
_apply(row, data, skip=("id", "created_at", "updated_at"))
row.must_change_password = False
db.add(row)
print(f" ✓ 管理员 {row.username}(role={row.role})已创建,"
f"口令沿用导出时的哈希——登录后请立即修改")
else:
# 已存在的账号不重置口令:这台机器可能早改过密码,
# 覆盖成开发机的口令是个反直觉的倒退。
_apply(row, data, skip=("id", "created_at", "updated_at", "password_hash",
"must_change_password"))
diff = (data.get("password_hash") or "") != row.password_hash
print(f" · 管理员 {row.username} 已存在:只同步角色/显示名,不改口令"
+ ("(注意:它的口令与部署包不同,登录请用这台机器的口令)" if diff else ""))
if not dry:
db.commit()
def _install_platform(db, tables: dict, *, dry: bool) -> None:
for name in PLATFORM_TABLES:
rows = tables.get(name) or []
if not rows:
continue
table = Base.metadata.tables.get(name)
if table is None:
print(f" ! 跳过 {name}(当前代码没有这张表)")
continue
cols = [c.key for c in inspect(table).columns if c.key != "id"]
for data in rows:
pk = data.get("id")
if db.execute(text(f"SELECT 1 FROM {name} WHERE id=:i"), {"i": pk}).first():
continue
col_sql = ", ".join(cols)
val_sql = ", ".join(f":{c}" for c in cols)
db.execute(text(f"INSERT INTO {name} (id, {col_sql}) VALUES (:id, {val_sql})"),
{**{c: data.get(c) for c in cols}, "id": pk})
print(f" ✓ 平台资源库 {name}: 导入后共 "
f"{db.execute(text(f'SELECT COUNT(*) FROM {name}')).scalar()} 行")
if not dry:
db.commit()
def do_install(args) -> None:
src = Path(args.src)
specs = read_json(src, "model_specs.json")["rows"]
keys = read_json(src, "model_secret_keys.json")["rows"]
users = read_json(src, "users.json")["rows"]
lib = None
if args.with_platform_library:
lib = read_json(src, "platform_library.json")["rows"]
# 先量包再动库:数据集比代码旧时,装上去的后果是「静默少功能」,而 install 一旦
# 跑过就改掉了表结构与默认位、回不到原样,所以宁可在这里停住不动。
print(f"体检部署数据集({src})")
rep = audit_dataset(specs, keys, users, origin=f"部署包 {src.name}")
rep.show()
print_dataset_summary(rep, specs, keys, users)
if rep.errors and not args.allow_drift:
sys.exit("导入中止:部署包与当前代码对不上(上面 error 条目)。"
"在开发机 doctor --fix + 重新 export 再打包;确认无碍加 --allow-drift。")
if args.schema:
init_db() # 建表 + 列迁移,幂等
print(f" ✓ 表结构就绪:{engine.url.render_as_string(hide_password=True)}")
db = SessionLocal()
try:
if not args.dry_run:
target = Report()
audit_target_db(target, db, only_missing=args.only_missing)
if target.items:
print("体检目标库")
target.show()
print("安装部署数据集(幂等,可反复执行)")
_install_master_key(db, keys, dry=args.dry_run, force=args.force_master_key)
_install_specs(db, specs, dry=args.dry_run, only_missing=args.only_missing)
_install_users(db, users, dry=args.dry_run)
if lib is not None:
_install_platform(db, lib, dry=args.dry_run)
if args.dry_run:
db.rollback()
print("\n--dry-run:未写入任何数据")
return
# 装完就验一次解密:这一步不过,说明主密钥和密文不是一对,
# 与其等业务报「缺 Key」,不如现在停下来。
from app.services import secrets as secret_svc
bad = [f"{s.provider}/{s.model}" for s in db.scalars(select(ModelSpec)).all()
if s.api_key_encrypted and not secret_svc.decrypt_value(db, s.api_key_encrypted)]
defaults = {s.media_type: f"{s.provider}/{s.model}"
for s in db.scalars(select(ModelSpec).where(ModelSpec.is_default.is_(True))).all()}
admins = db.scalars(select(User.username).where(User.role == "admin")).all()
print("\n校验")
print(f" 默认模型:{', '.join(f'{k}={v}' for k, v in sorted(defaults.items())) or '(无)'}")
print(f" 管理员:{', '.join(admins) or '(无)'}")
if bad:
sys.exit(f" ✗ 这些模型的 API Key 解不开(主密钥不匹配):{bad}")
print(" ✓ 所有 API Key 解密通过")
finally:
db.close()
def print_dataset_summary(rep: Report, specs: list[dict], keys: list[dict],
users: list[dict]) -> None:
"""把数据集摊开给人看:装到服务器上的到底是什么,一行一个结论。"""
defaults = sorted(f"{s['media_type']}={s['provider']}/{s['model']}"
for s in specs if s.get("is_default"))
real = sorted(f"{s['media_type']}/{s['provider']}/{s['model']}"
for s in specs if not (s["provider"] or "").startswith("mock"))
with_key = sorted(f"{s['provider']}/{s['model']}"
for s in specs if s.get("api_key_encrypted"))
print(f" 数据集:模型配置 {len(specs)} 条(真实模型 {'、'.join(real) or '无'})")
print(f" 默认位:{'、'.join(defaults) or '(无)'}")
print(f" 凭据:带 API Key {len(with_key)} 条"
+ ("(" + "、".join(with_key) + ")" if with_key else "")
+ f" · 主密钥 {len(keys)} 条 · 管理员 "
+ ("、".join(u["username"] for u in users) or "(无)"))
def do_doctor(args) -> None:
"""体检:默认量当前库(导出前自查),--src 时量部署包(deploy.sh 用这条)。
退出码非 0 = 有 error。`deploy.sh` 把它当打包闸门:在开发机上就把「数据比代码旧」
拦下来,比到了服务器上 install 才炸便宜得多——那时人已经不在终端前了。
"""
if args.src:
src = Path(args.src)
specs = read_json(src, "model_specs.json")["rows"]
keys = read_json(src, "model_secret_keys.json")["rows"]
users = read_json(src, "users.json")["rows"]
rep = audit_dataset(specs, keys, users, origin=f"部署包 {src}")
else:
db = SessionLocal()
try:
if args.fix:
changes = calibrate_specs(db, apply=True)
print("校准模型配置(只补缺,不覆盖人工调过的值)")
for line in changes or ["没有需要补的:数据与当前代码一致"]:
print(f" · {line}")
specs = [row_to_dict(s) for s in db.scalars(select(ModelSpec).order_by(ModelSpec.id)).all()]
keys = [row_to_dict(k) for k in db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()]
users = [{k: v for k, v in row_to_dict(u).items() if k not in USER_DROP}
for u in db.scalars(select(User).where(User.role == "admin")
.order_by(User.id)).all()]
rep = Report()
# 开发库里当然有业务数据,所以这里只查遗留表,不查「是不是干净库」
audit_orphan_tables(rep, engine)
audit_dataset(specs, keys, users, origin=f"开发库 {engine.url.database}", rep=rep)
finally:
db.close()
print_dataset_summary(rep, specs, keys, users)
print("体检结论")
rep.show()
if rep.errors and not args.allow_drift:
sys.exit(f"体检未过:{len(rep.errors)} 条 error、{len(rep.warns)} 条 warn")
print(f" → 判定通过(error 0 · warn {len(rep.warns)})")
# ================================================================== boundary
def do_boundary(args) -> None:
"""数据边界一览:这台机器上「部署包带来的」和「本机业务跑出来的」各有多少行。
给 `server.sh status` 用。写在 seed.py 而不是 shell 里内嵌 SQL,是为了和体检共用
同一批常量(SEED_TABLES / BUSINESS_TABLES):边界定义哪天变了,两边不会各说一套——
「部署只带配置、不带业务数据」这条约定最怕的就是两个地方各数各的。
"""
cfg = _table_counts(engine, SEED_TABLES)
if not cfg:
print(" 表结构还没建:先 ./scripts/server.sh install")
return
biz = _table_counts(engine, BUSINESS_TABLES)
print(" 配置(随部署包):" + "、".join(f"{k} {v}" for k, v in sorted(cfg.items())))
print(" 业务(本机产生):" + ("、".join(f"{k} {v}" for k, v in sorted(biz.items())) or "(表还没建)"))
if not any(biz.values()):
print(" → 业务表全空:干净起点,符合部署边界")
else:
print(f" → 本机已产生 {sum(biz.values())} 行业务数据。它不在部署包里,"
"换机器属于迁移,要单独备份 media/ 与库文件")
orphan = sorted(set(inspect(engine).get_table_names()) - set(Base.metadata.tables))
if orphan:
print(f" ! 代码不认识的表:{'、'.join(orphan)}(确认后手工 DROP,见工程规范 §6)")
# ================================================================== CLI
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0],
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = ap.add_subparsers(dest="cmd", required=True)
e = sub.add_parser("export", help="导出模型配置 + 主密钥 + 管理员到 JSON")
e.add_argument("--out", default=str(SEED_DIR), help="输出目录(默认 scripts/seed)")
e.add_argument("--with-platform-library", action="store_true", help="顺带导出平台资源库")
e.add_argument("--fix", action="store_true",
help="导出前先把库里的模型配置补齐到当前代码(补 modes、补工作流节点键)")
e.add_argument("--allow-drift", action="store_true", help="体检有 error 也照样导出")
e.set_defaults(func=do_export)
i = sub.add_parser("install", help="把 JSON 装进目标库(服务器首次部署用)")
i.add_argument("--src", default=str(SEED_DIR), help="JSON 目录(默认 scripts/seed)")
i.add_argument("--no-schema", dest="schema", action="store_false", help="不先建表")
i.add_argument("--dry-run", action="store_true", help="只报告将要发生什么")
i.add_argument("--force-master-key", action="store_true", help="允许覆盖已有主密钥")
i.add_argument("--with-platform-library", action="store_true", help="顺带导入平台资源库")
i.add_argument("--only-missing", action="store_true",
help="模型配置只补缺,不覆盖服务器上已有的同名模型")
i.add_argument("--allow-drift", action="store_true", help="体检有 error 也照样导入")
i.set_defaults(func=do_install)
d = sub.add_parser("doctor", help="体检:量开发库(默认)或量部署包(--src),不改数据")
d.add_argument("--src", default="", help="改量部署包:JSON 目录(如 scripts/seed)")
d.add_argument("--fix", action="store_true", help="量库时把模型配置补齐到当前代码(只补缺)")
d.add_argument("--allow-drift", action="store_true", help="有 error 也返回 0(只打印不拦)")
d.set_defaults(func=do_doctor)
b = sub.add_parser("boundary", help="数据边界一览:配置表与业务表各多少行(server.sh status 用)")
b.set_defaults(func=do_boundary)
args = ap.parse_args()
args.func(args)
if __name__ == "__main__":
main()