844 lines
44 KiB
Python
Executable File
844 lines
44 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""部署数据集的体检、导出与安装:表结构 + 模型配置 + 管理员账号。
|
||
|
||
服务器部署**不搬开发库**。14MB 的 nex_animation.db 里绝大多数是开发过程产物
|
||
(项目、镜头、任务、事件日志、演示账号),新环境真正需要的是三样东西:
|
||
|
||
| 内容 | 来源 |
|
||
|---------------------|----------------------------------------|
|
||
| 表结构 + 列迁移 | `app.db.init_db()`(应用启动自带) |
|
||
| 模型配置 | `model_specs` + `model_secret_keys` |
|
||
| 管理员账号 | `users` 里 role=admin 的那几条 |
|
||
|
||
模型配置必须带走,否则新环境只有 `seed_specs()` 播的 mock,得在模型管理中心
|
||
从零填一遍 ComfyUI / Qwen / IndexTTS 的端点与工作流;而 API Key 是
|
||
`app/services/secrets.py` 用库内主密钥加密的,所以 **`model_secret_keys` 那一行
|
||
必须和密文一起走**——换机器只带 model_specs,密文会解成空串,
|
||
表现为「模型管理中心显示已配置,派发却报缺 Key」。
|
||
|
||
# 开发机:体检 →(必要时校准)→ 导出
|
||
.venv/bin/python scripts/seed.py doctor # 只量不改,退出码非 0 = 有别部署
|
||
.venv/bin/python scripts/seed.py doctor --fix # 把库补齐到当前代码(只补缺)
|
||
.venv/bin/python scripts/seed.py export --fix # 校准 + 体检 + 导出(部署前就这一条)
|
||
.venv/bin/python scripts/seed.py export --with-platform-library # 顺带平台资源库
|
||
|
||
# 服务器:安装(空库;先量包再建表再灌数据,幂等,可反复执行)
|
||
.venv/bin/python scripts/seed.py install
|
||
.venv/bin/python scripts/seed.py install --only-missing # 只补库里没有的,本机已调的不动
|
||
.venv/bin/python scripts/seed.py doctor --src scripts/seed # 只量部署包(deploy.sh 用这条)
|
||
.venv/bin/python scripts/seed.py boundary # 数据边界一览(server.sh status 用这条)
|
||
|
||
**体检(doctor)是这三条命令共用的同一把尺子**:列对得上表结构、适配器在代码里注册过、
|
||
`modes` 不比代码声明的能力少、每类能力恰好一个默认、密文能被包里的主密钥解开、管理员登得进来。
|
||
导出前量库、安装前量包,两边用的是同一个 `audit_dataset()`——
|
||
否则「导出时没说、装上才发现」的老路会一直走下去(详见下面体检小节的注释)。
|
||
|
||
安装顺序刻意是「主密钥 → 模型配置 → 账号」:主密钥先到位,后面导入的密文才可解。
|
||
`model_secret_keys` 已有值且与导出值不同时默认**中止**(不是覆盖),
|
||
因为覆盖会让这台机器上原有模型配置的 Key 全部解不开——真要做这种手术请加
|
||
--force-master-key,并准备好在模型管理中心重填所有 Key。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import datetime as dt
|
||
import json
|
||
import os
|
||
import sys
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "backend"))
|
||
|
||
from sqlalchemy import inspect, select, text, types as sa_types # noqa: E402
|
||
from sqlalchemy.orm import Session # noqa: E402
|
||
|
||
from app.db import Base, SessionLocal, engine, init_db # noqa: E402
|
||
from app.models import ModelSecretKey, ModelSpec, User # noqa: E402
|
||
|
||
SEED_DIR = ROOT / "scripts" / "seed"
|
||
|
||
# 平台资源库(可选):严格说不属于「部署必需」,但换机时重建 37 条素材很费手工,
|
||
# 所以做成显式开关;媒体文件在 media/platform/ 下,不在数据库里,需另行拷贝。
|
||
PLATFORM_TABLES = ("platform_characters", "platform_scenes", "platform_voices",
|
||
"platform_media", "platform_media_versions")
|
||
|
||
# users 里这些列是「在哪台机器上登过」的流水,跨机没有意义,还会误导审计。
|
||
USER_DROP = ("last_login_at", "last_login_ip", "login_count")
|
||
|
||
|
||
# ------------------------------------------------------------------ 行 ⇄ dict
|
||
def row_to_dict(obj) -> dict:
|
||
"""ORM 行转 JSON 可序列化 dict(时间转 ISO,JSON 列原样,二进制转 base64)。"""
|
||
out: dict = {}
|
||
for col in inspect(obj.__table__).columns:
|
||
val = getattr(obj, col.key)
|
||
if isinstance(val, dt.datetime):
|
||
val = val.isoformat(sep=" ", timespec="microseconds")
|
||
elif isinstance(val, (bytes, bytearray)):
|
||
val = bytes(val).hex()
|
||
elif isinstance(val, dt.date):
|
||
val = val.isoformat()
|
||
out[col.key] = val
|
||
return out
|
||
|
||
|
||
def write_json(path: Path, payload) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(f" 写出 {path.relative_to(ROOT)}")
|
||
|
||
|
||
def read_json(src: Path, name: str):
|
||
p = src / name
|
||
if not p.is_file():
|
||
sys.exit(f"缺少 {p};先在开发机跑 .venv/bin/python scripts/seed.py export")
|
||
return json.loads(p.read_text(encoding="utf-8"))
|
||
|
||
|
||
def _apply(row, data: dict, *, skip: tuple[str, ...] = ()) -> None:
|
||
mapper = inspect(row.__table__)
|
||
for key, val in data.items():
|
||
if key in skip or key not in mapper.c:
|
||
continue
|
||
col = mapper.c[key]
|
||
if isinstance(val, str):
|
||
# JSON 导出把时间写成了 ISO 字符串,回填时必须还原成 datetime:
|
||
# SQLite 驱动遇到字符串直接抛 «only accepts Python datetime and date
|
||
# objects as input»,Postgres 则可能按会话时区误读。
|
||
# 本库的时间列一律是 DateTime(timezone=True),写进去的是带 UTC 的值,
|
||
# 读回来却是 naive(见 app/models/base.py:as_utc),所以这里补回 UTC。
|
||
if isinstance(col.type, sa_types.DateTime):
|
||
parsed = dt.datetime.fromisoformat(val)
|
||
val = parsed.replace(tzinfo=dt.timezone.utc) if parsed.tzinfo is None else parsed
|
||
elif isinstance(col.type, sa_types.Date):
|
||
val = dt.date.fromisoformat(val)
|
||
setattr(row, key, val)
|
||
|
||
|
||
|
||
# ================================================================== 体检(doctor)
|
||
# 为什么必须有这一节:部署数据集是「快照」,代码是「活水」。快照一旦落后于代码,
|
||
# 服务器上的表现不是报错而是**静默降级**,比崩掉更难查:
|
||
#
|
||
# * 导出时 modes 里还没有 `voice_instruct` → 装上之后「音色描述直出」整个功能
|
||
# 在模型管理中心和声线页都看不见(`services/timbre.py:supports_instruct()` 读的是
|
||
# `model_specs.modes`,不是代码里的 Capabilities),排查的人会去查 ComfyUI;
|
||
# * 导出之后代码给工作流加了新节点键(`generate_full_node` / `instruct_node`)
|
||
# → 配置里少键,走到描述词分支才炸;
|
||
# * 导出之后 ORM 加了非空列 → `install` 在服务器上抛 IntegrityError,
|
||
# 而这句话离真正的根因(数据集是旧的)隔了 40 行 traceback;
|
||
# * 开发机改了模型、删了模型,忘了重新 export → 包里还是上周那套端点。
|
||
#
|
||
# 所以体检做三件事:**列对得上、能力对得上、密钥对得上**,并且把「代码比数据新」
|
||
# 当成错误而不是提示——它唯一的修复动作在开发机上(`doctor --fix` + 重新 export),
|
||
# 在服务器上加 --allow-drift 硬闯才是合理的逃生门。
|
||
CAPABILITY_MEDIA_TYPES = ("video", "image", "tts", "llm")
|
||
SEED_TABLES = ("model_specs", "model_secret_keys", "users")
|
||
# 这几张表非空 = 目标库已经在跑业务,不该拿它当首装库
|
||
BUSINESS_TABLES = ("projects", "shots", "tasks", "assets", "generation_records", "event_logs")
|
||
|
||
|
||
class Report:
|
||
"""体检结论:error 会挡住 export/install,warn 只打印。"""
|
||
|
||
def __init__(self) -> None:
|
||
self.items: list[tuple[str, str, str]] = []
|
||
|
||
def error(self, code: str, msg: str) -> None:
|
||
self.items.append(("error", code, msg))
|
||
|
||
def warn(self, code: str, msg: str) -> None:
|
||
self.items.append(("warn", code, msg))
|
||
|
||
@property
|
||
def errors(self) -> list[tuple[str, str, str]]:
|
||
return [i for i in self.items if i[0] == "error"]
|
||
|
||
@property
|
||
def warns(self) -> list[tuple[str, str, str]]:
|
||
return [i for i in self.items if i[0] == "warn"]
|
||
|
||
def show(self) -> None:
|
||
for level, code, msg in self.items:
|
||
mark = "XX" if level == "error" else " !"
|
||
color = "\033[31m" if level == "error" else "\033[33m"
|
||
print(f" {color}{mark}\033[0m [{code}] {msg}")
|
||
if not self.items:
|
||
print(" \033[32mok\033[0m 没有告警")
|
||
|
||
|
||
def _adapter_of(media_type: str, adapter: str, provider: str):
|
||
"""按 media_type/adapter 取适配器;取不到返回 None(调用方记错误)。"""
|
||
from app.providers import registry as model_registry
|
||
from app.providers.base import FatalProviderError
|
||
|
||
try:
|
||
return model_registry.adapter_for_spec(
|
||
SimpleNamespace(media_type=media_type, adapter=adapter, provider=provider, config_json={}))
|
||
except FatalProviderError:
|
||
return None
|
||
|
||
|
||
def _code_defaults_for(adapter_name: str) -> dict:
|
||
"""代码里维护的本地工作流默认配置(用于「只补缺失键」,不覆盖既有值)。"""
|
||
from app.providers import registry as model_registry
|
||
|
||
builders = {"comfyui": model_registry.comfyui_config,
|
||
"comfyui_image": model_registry.comfyui_image_config,
|
||
"comfyui_tts": model_registry.comfyui_tts_config}
|
||
build = builders.get((adapter_name or "").lower())
|
||
return build() if build else {}
|
||
|
||
|
||
def _rows_decrypt(keys: list[dict], specs: list[dict]) -> list[str]:
|
||
"""用「包里的主密钥」试解「包里的密文」,返回解不开的那几条。
|
||
|
||
临时摘掉 MODEL_SECRET_KEY:不然这里量到的是本机环境变量的密钥,
|
||
服务器上没有这个变量时反而会漏报(用 env 的那条检查单独告警)。
|
||
"""
|
||
from sqlalchemy import create_engine
|
||
from sqlalchemy.orm import sessionmaker
|
||
|
||
from app.services import secrets as secret_svc
|
||
|
||
ciphered = [(f"{s.get('provider')}/{s.get('model')}", s.get("api_key_encrypted") or "")
|
||
for s in specs if s.get("api_key_encrypted")]
|
||
if not ciphered:
|
||
return []
|
||
if not keys or not (keys[0].get("key_value") or ""):
|
||
return [name for name, _ in ciphered]
|
||
engine = create_engine("sqlite://", future=True)
|
||
Base.metadata.create_all(engine, tables=[ModelSecretKey.__table__])
|
||
session = sessionmaker(bind=engine, class_=Session, future=True)()
|
||
row = ModelSecretKey()
|
||
_apply(row, keys[0], skip=("id",))
|
||
session.add(row)
|
||
session.commit()
|
||
saved = os.environ.pop("MODEL_SECRET_KEY", None)
|
||
try:
|
||
bad = [name for name, cipher in ciphered
|
||
if not secret_svc.decrypt_value(session, cipher)]
|
||
finally:
|
||
if saved is not None:
|
||
os.environ["MODEL_SECRET_KEY"] = saved
|
||
session.close()
|
||
engine.dispose()
|
||
return bad
|
||
|
||
|
||
def audit_specs(rep: Report, specs: list[dict], keys: list[dict], *, origin: str) -> None:
|
||
"""模型配置:列 ⇄ 表结构、适配器、能力位、默认位、端点、密钥。"""
|
||
table = Base.metadata.tables["model_specs"]
|
||
known_cols = {c.key for c in inspect(table).columns}
|
||
required_cols = {c.key for c in inspect(table).columns
|
||
if not c.nullable and c.default is None and c.server_default is None
|
||
and c.key != "id"}
|
||
|
||
def label(row: dict) -> str:
|
||
return f"{row.get('provider')}/{row.get('model')}({row.get('media_type')})"
|
||
|
||
# ---- 列:数据集与当前表结构的差集
|
||
unknown = sorted({k for r in specs for k in r} - known_cols - {"_meta"})
|
||
if unknown:
|
||
rep.warn("列", f"{origin} 里有当前表结构没有的列,导入时会被静默丢弃:{unknown}")
|
||
for row in specs:
|
||
missing = sorted(required_cols - set(row))
|
||
if missing:
|
||
rep.error("列", f"{label(row)} 缺非空列 {missing}:数据集比表结构旧,"
|
||
"在开发机跑 scripts/seed.py doctor --fix 再 export")
|
||
break
|
||
|
||
# ---- 适配器 + 能力位
|
||
for row in specs:
|
||
adapter = _adapter_of(row.get("media_type") or "", row.get("adapter") or "",
|
||
row.get("provider") or "")
|
||
if adapter is None:
|
||
rep.error("适配器", f"{label(row)} 的 adapter={row.get('adapter')!r} 代码里没注册,"
|
||
"派发时直接 FatalProviderError")
|
||
continue
|
||
try:
|
||
caps = adapter.capabilities(row.get("config_json") or {})
|
||
except Exception as exc: # noqa: BLE001
|
||
rep.warn("适配器", f"{label(row)} 读适配器能力失败:{type(exc).__name__}: {exc}")
|
||
continue
|
||
drift = sorted(set(caps.modes) - set(row.get("modes") or []))
|
||
if drift:
|
||
rep.error("能力位", f"{label(row)} 的 modes 缺 {drift}:代码已支持、数据集还是旧的。"
|
||
"装上去这些能力在模型管理中心/派发热路径里都不存在"
|
||
"(修复:开发机 doctor --fix + 重新 export;确认要收窄就加 --allow-drift)")
|
||
|
||
# ---- 默认位:每类能力必须恰好一个默认,且它自己必须是启用的
|
||
for media_type in sorted({r.get("media_type") for r in specs}):
|
||
group = [r for r in specs if r.get("media_type") == media_type]
|
||
wanted = [r for r in group if r.get("is_default")]
|
||
if len(wanted) != 1:
|
||
rep.error("默认位", f"{media_type} 有 {len(wanted)} 个默认模型"
|
||
f"({'、'.join(label(r) for r in wanted) or '无'}):"
|
||
"0 个会让这类能力退回 mock,>1 个则派发按排序偶然选中")
|
||
continue
|
||
if not wanted[0].get("enabled", True):
|
||
rep.error("默认位", f"{media_type} 的默认模型 {label(wanted[0])} 是停用状态")
|
||
missing_caps = [m for m in CAPABILITY_MEDIA_TYPES
|
||
if not any(r.get("media_type") == m and not (r.get("provider") or "").startswith("mock")
|
||
for r in specs)]
|
||
if missing_caps:
|
||
rep.warn("默认位", f"这几类能力只有 mock 可用:{missing_caps}(部署后跑不出真片)")
|
||
|
||
# ---- 端点:真实模型没端点 = 装上去必然连不上
|
||
for row in specs:
|
||
if (row.get("provider") or "").startswith("mock"):
|
||
continue
|
||
cfg = row.get("config_json") or {}
|
||
base = str(cfg.get("base_url") or "").strip()
|
||
if not base:
|
||
rep.error("端点", f"{label(row)} 的 config_json.base_url 是空的:服务器上无处可连")
|
||
elif base.startswith(("http://localhost", "http://127.0.0.1")):
|
||
rep.warn("端点", f"{label(row)} 指向 localhost:服务器与模型不同机时连不上")
|
||
if row.get("media_type") == "llm" and not (row.get("api_key_encrypted") or ""):
|
||
rep.warn("端点", f"{label(row)} 没有 API Key(若该端点免鉴权可忽略)")
|
||
|
||
# ---- 密钥:主密钥与密文必须成对
|
||
if os.environ.get("MODEL_SECRET_KEY"):
|
||
rep.warn("密钥", "本机设了 MODEL_SECRET_KEY:密文是它加密的,包里的 model_secret_keys "
|
||
"在服务器上不起作用——部署时必须把同一个变量写进服务器 .env")
|
||
ciphered = [f"{r.get('provider')}/{r.get('model')}" for r in specs if r.get("api_key_encrypted")]
|
||
leaks = [f"{r.get('provider')}/{r.get('model')}" for r in specs
|
||
if str(r.get("api_key_encrypted") or "").startswith("sk-")]
|
||
if leaks:
|
||
rep.error("密钥", f"疑似明文 API Key 直接入库:{leaks};在模型管理中心重填一次再导出")
|
||
bad = _rows_decrypt(keys, specs)
|
||
if bad:
|
||
rep.error("密钥", f"这些模型配置的 Key 用包内主密钥解不开:{bad}"
|
||
"(表现是模型管理中心显示已配置、派发报缺 Key)")
|
||
|
||
|
||
def audit_users(rep: Report, users: list[dict], *, origin: str = "数据集") -> None:
|
||
"""管理员:能登进来、没有跨机无意义的登录流水列。"""
|
||
from app.services import auth as auth_service
|
||
|
||
table = Base.metadata.tables["users"]
|
||
known_cols = {c.key for c in inspect(table).columns}
|
||
unknown = sorted({k for u in users for k in u} - known_cols)
|
||
if unknown:
|
||
rep.warn("账号", f"{origin} 里有表结构没有的列,导入时会被丢弃:{unknown}")
|
||
# 「有没有登录流水列」只对导出文件成立:直接从库里读出来的行当然带这些列,
|
||
# 是 export 负责在写文件前剔掉它们(USER_DROP)。
|
||
if origin != "开发库":
|
||
stale = [k for u in users for k in USER_DROP if k in u]
|
||
if stale:
|
||
rep.warn("账号", f"{origin} 带着机器相关的登录流水列 {sorted(set(stale))}:"
|
||
"跨机无意义,建议重新 export")
|
||
admins = [u for u in users if u.get("role") == "admin"]
|
||
if not admins:
|
||
rep.error("账号", "部署包里没有 role=admin 的账号:服务器首启会另建一个 admin,"
|
||
"口令与这台机器不同")
|
||
for u in users:
|
||
if not (u.get("password_hash") or ""):
|
||
rep.error("账号", f"账号 {u.get('username')} 没有口令哈希,登不进去")
|
||
continue
|
||
try:
|
||
auth_service.verify_password("probe-only", u["password_hash"])
|
||
except Exception as exc: # noqa: BLE001
|
||
rep.error("账号", f"账号 {u.get('username')} 的 password_hash 解析失败:"
|
||
f"{type(exc).__name__}: {exc}")
|
||
if not u.get("is_active", True):
|
||
rep.warn("账号", f"账号 {u.get('username')} 在导出机上是停用状态")
|
||
|
||
|
||
def audit_dataset(specs: list[dict], keys: list[dict], users: list[dict], *, origin: str,
|
||
rep: Report | None = None) -> Report:
|
||
"""量一份数据集。`rep` 传进来时可以与其他检查(如目标库现状)汇到同一份结论里。"""
|
||
rep = rep or Report()
|
||
if not specs:
|
||
rep.error("数据集", f"{origin}:model_specs 一条都没有,装上只有 mock")
|
||
audit_specs(rep, specs, keys, origin=origin)
|
||
audit_users(rep, users, origin=origin)
|
||
return rep
|
||
|
||
|
||
def _table_counts(bind, names: tuple[str, ...]) -> dict[str, int]:
|
||
from sqlalchemy import text as sa_text
|
||
|
||
existing = set(inspect(bind).get_table_names())
|
||
out: dict[str, int] = {}
|
||
with bind.begin() as conn:
|
||
for name in names:
|
||
if name in existing:
|
||
out[name] = conn.execute(sa_text(f"SELECT COUNT(*) FROM {name}")).scalar() or 0
|
||
return out
|
||
|
||
|
||
def audit_orphan_tables(rep: Report, bind) -> None:
|
||
"""库里躺着代码不认识的表 = 历史手术留下的备份表,它不该跟着部署走。
|
||
|
||
只报不删:工程规范 §6 说得很清楚,这种表要人确认后手工 DROP,
|
||
脚本偷偷删表等于把「备份」这个唯一的退路删掉。
|
||
"""
|
||
orphan = sorted(set(inspect(bind).get_table_names()) - set(Base.metadata.tables))
|
||
if orphan:
|
||
rep.warn("边界", f"库里有代码不认识的表 {orphan}:历史遗留备份表,"
|
||
"确认后手工 DROP(它不影响部署包,但会让 status 的数据边界说不清)")
|
||
|
||
|
||
def audit_target_db(rep: Report, db, *, only_missing: bool = False) -> None:
|
||
"""install 的目标库现状:别把部署数据灌进一个正在跑业务的库。
|
||
|
||
「库里有业务数据」在两档模式下不是一回事,所以报法也不同:
|
||
· 覆盖式(不带 --only-missing / 容器 AUTO_SEED=force)真的会用包里的值改掉同名模型,
|
||
指错库就是事故 → 保持黄色告警;
|
||
· --only-missing 一行都不动已有数据。而容器每次重启都会走到这里,
|
||
跑了几天的服务器上必然有项目、有事件日志——每次都黄一条,
|
||
等于训练运维无视黄条,所以降级成一行普通提示(只补空缺这件事后面还会再打印一遍)。
|
||
孤儿表那条与模式无关,照旧告警。
|
||
"""
|
||
counts = {k: v for k, v in _table_counts(db.bind, BUSINESS_TABLES).items() if v}
|
||
if counts:
|
||
if only_missing:
|
||
print(f" · 目标库已有业务数据 {counts}:--only-missing 只补库里没有的行,已有配置不动")
|
||
else:
|
||
rep.warn("边界", f"目标库里已有业务数据 {counts}:部署数据集只该进干净库,"
|
||
"确认 DATABASE_URL 没指到开发库(覆盖式导入会改掉这些模型配置)")
|
||
audit_orphan_tables(rep, db.bind)
|
||
|
||
|
||
# ------------------------------------------------------------------ 校准(--fix)
|
||
def calibrate_specs(db, *, apply: bool) -> list[str]:
|
||
"""把开发库的模型配置补齐到当前代码:只加不减,从不覆盖已有值。
|
||
|
||
只做两件事,因为这两件事代码才是权威:
|
||
1. `modes` 补上适配器已声明、行里却缺的能力位;
|
||
2. `config_json` 补上代码默认配置里新增、行里却缺的键(工作流节点等)。
|
||
其余字段(端点、限流、单价、notes)都是人工在页面上调的,代码一律不猜。
|
||
"""
|
||
changes: list[str] = []
|
||
for spec in db.scalars(select(ModelSpec).order_by(ModelSpec.id)).all():
|
||
adapter = _adapter_of(spec.media_type, spec.adapter, spec.provider)
|
||
if adapter is None:
|
||
continue
|
||
caps = adapter.capabilities(spec.config_json or {})
|
||
missing_modes = sorted(set(caps.modes) - set(spec.modes or []))
|
||
defaults = _code_defaults_for(spec.adapter)
|
||
cfg = dict(spec.config_json or {})
|
||
missing_cfg = sorted(k for k in defaults if k not in cfg)
|
||
if not missing_modes and not missing_cfg:
|
||
continue
|
||
if apply:
|
||
if missing_modes:
|
||
spec.modes = list(spec.modes or []) + missing_modes
|
||
for key in missing_cfg:
|
||
cfg[key] = defaults[key]
|
||
spec.config_json = cfg
|
||
changes.append(f"{spec.provider}/{spec.model}({spec.media_type}):"
|
||
+ (f"补 modes {missing_modes}" if missing_modes else "")
|
||
+ ("、" if missing_modes and missing_cfg else "")
|
||
+ (f"补 config_json 键 {missing_cfg}" if missing_cfg else ""))
|
||
if apply and changes:
|
||
db.commit()
|
||
return changes
|
||
|
||
|
||
# ================================================================== export
|
||
def do_export(args) -> None:
|
||
out = Path(args.out)
|
||
db = SessionLocal()
|
||
try:
|
||
if args.fix:
|
||
changes = calibrate_specs(db, apply=True)
|
||
if changes:
|
||
print("校准开发库(只补缺,不覆盖人工调过的值)")
|
||
for line in changes:
|
||
print(f" · {line}")
|
||
# 不在这里量「库里有没有业务数据」:开发库当然有,那条检查是给 install
|
||
# 挡「把部署数据灌进正在跑的库」用的(见 do_install)。
|
||
specs = [row_to_dict(s) for s in db.scalars(select(ModelSpec).order_by(ModelSpec.id)).all()]
|
||
keys = [row_to_dict(k) for k in db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()]
|
||
admins = [row_to_dict(u) for u in db.scalars(
|
||
select(User).where(User.role == "admin").order_by(User.id)).all()]
|
||
if not specs:
|
||
sys.exit("model_specs 是空的:这台机器还没配过模型,导出去也只是 mock。")
|
||
if not admins:
|
||
sys.exit("users 里没有 admin 角色账号。")
|
||
|
||
# 导出与安装用同一把尺子量:量库一次、装前量包一次,不会出现
|
||
# 「导出时没说、装到服务器才发现」的错配。明文 Key 自检也在体检里
|
||
# (secrets.py 只该写密文,出现 sk- 开头的明文就是有人绕过了封装)。
|
||
rep = audit_dataset(specs, keys, admins, origin="开发库")
|
||
print("体检(部署数据集)")
|
||
rep.show()
|
||
print_dataset_summary(rep, specs, keys, admins)
|
||
if rep.errors and not args.allow_drift:
|
||
sys.exit("导出中止:上面是 error。修完再导;确认要带着这些问题部署,加 --allow-drift。")
|
||
|
||
write_json(out / "model_specs.json", {
|
||
"_meta": {"table": "model_specs", "count": len(specs),
|
||
"defaults": sorted(f"{s['media_type']}={s['provider']}/{s['model']}"
|
||
for s in specs if s["is_default"]),
|
||
"exported_at": dt.datetime.now().isoformat(timespec="seconds")},
|
||
"rows": specs})
|
||
write_json(out / "model_secret_keys.json", {
|
||
"_meta": {"table": "model_secret_keys", "count": len(keys),
|
||
"note": "密钥加密主密钥;缺了它 model_specs.api_key_encrypted 解不开"},
|
||
"rows": keys})
|
||
write_json(out / "users.json", {
|
||
"_meta": {"table": "users", "count": len(admins),
|
||
"note": "只导管理员;口令是 PBKDF2 哈希,不含明文"},
|
||
"rows": [{k: v for k, v in u.items() if k not in USER_DROP} for u in admins]})
|
||
|
||
if args.with_platform_library:
|
||
lib = {}
|
||
for name in PLATFORM_TABLES:
|
||
if name not in Base.metadata.tables:
|
||
print(f" ! 跳过 {name}(模型里没有这张表)")
|
||
continue
|
||
rows = [dict(r._mapping) for r in db.execute(text(f"SELECT * FROM {name}")).all()]
|
||
lib[name] = rows
|
||
print(f" 平台资源库 {name}: {len(rows)} 行")
|
||
write_json(out / "platform_library.json", {
|
||
"_meta": {"tables": list(lib), "note": "媒体文件在 media/platform/,需另行拷贝"},
|
||
"rows": lib})
|
||
|
||
# 解密自检:能解出内容才说明「密文 + 主密钥」这一对是完整的。
|
||
from app.services import secrets as secret_svc
|
||
checked = [f"{s['provider']}/{s['model']}" for s in specs if s.get("api_key_encrypted")]
|
||
ok = [f"{s['provider']}/{s['model']}" for s in specs
|
||
if s.get("api_key_encrypted")
|
||
and secret_svc.decrypt_value(db, s["api_key_encrypted"])]
|
||
if len(ok) != len(checked):
|
||
sys.exit(f"主密钥与密文不匹配:{set(checked) - set(ok)} 解不开,导出会导致换机后 Key 丢失。")
|
||
print(f"\n导出完成 → {out}")
|
||
print(f" 模型配置 {len(specs)} 条(默认:{', '.join(s['media_type'] for s in specs if s['is_default'])})"
|
||
f" · 带 Key {len(ok)} 条且解密通过 · 管理员 {len(admins)} 个")
|
||
print(" ⚠️ scripts/seed/ 含加密主密钥,已在 .gitignore 内;部署包里必须带上它。")
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
# ================================================================== install
|
||
def _keys_in_use(db) -> int:
|
||
"""本机已有几条模型配置真的用上了当前主密钥(密文能被它解开)。"""
|
||
from app.services import secrets as secret_svc
|
||
|
||
used = 0
|
||
for s in db.scalars(select(ModelSpec)).all():
|
||
if s.api_key_encrypted and secret_svc.decrypt_value(db, s.api_key_encrypted):
|
||
used += 1
|
||
return used
|
||
|
||
|
||
def _install_master_key(db, rows, *, dry: bool, force: bool) -> None:
|
||
have = db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()
|
||
if not rows:
|
||
print(" · 导出包里没有主密钥(开发机可能用 MODEL_SECRET_KEY 环境变量):跳过")
|
||
return
|
||
if not have:
|
||
for data in rows:
|
||
row = ModelSecretKey()
|
||
_apply(row, data, skip=("id",))
|
||
db.add(row)
|
||
print(f" ✓ 主密钥已导入({len(rows)} 条)")
|
||
return
|
||
if have[0].key_value == rows[0]["key_value"]:
|
||
print(" · 主密钥已一致,跳过")
|
||
return
|
||
msg = (f"目标库已有主密钥且与导出包不同(现有 {have[0].key_value[:8]}… vs "
|
||
f"导入 {rows[0]['key_value'][:8]}…)")
|
||
in_use = _keys_in_use(db)
|
||
if in_use == 0 and not force:
|
||
# 「先起了服务、后导数据」是最常见的顺序:应用首启会自己生成一把主密钥,
|
||
# 但此刻还没有任何模型配置真的依赖它(mock 没有 Key),换掉零损失。
|
||
# 拦在这里只会让人以为工具坏了,所以直接换,并把话讲明白。
|
||
for data in rows:
|
||
_apply(have[0], data, skip=("id",))
|
||
print(f" · {msg}\n · 本机没有任何已解密成功的 API Key 依赖旧密钥 → 直接换成部署包的主密钥")
|
||
return
|
||
if not force:
|
||
sys.exit(msg + f";本机有 {in_use} 条模型配置的 Key 靠它解密,"
|
||
"覆盖后全部要重填。确认就加 --force-master-key。")
|
||
for data in rows:
|
||
_apply(have[0], data, skip=("id",))
|
||
print(f" ! {msg} → 已按 --force-master-key 覆盖,请复查模型管理中心的 Key")
|
||
|
||
|
||
def _install_specs(db, rows, *, dry: bool, only_missing: bool = False) -> None:
|
||
added = updated = skipped = 0
|
||
added_keys: set[tuple[str, str, str]] = set()
|
||
for data in rows:
|
||
key = {"provider": data["provider"], "model": data.get("model", ""),
|
||
"media_type": data["media_type"]}
|
||
row = db.scalars(select(ModelSpec).filter_by(**key)).first()
|
||
if row is not None and only_missing:
|
||
skipped += 1
|
||
continue
|
||
if row is None:
|
||
row = ModelSpec()
|
||
_apply(row, data, skip=("id", "created_at", "updated_at"))
|
||
db.add(row)
|
||
added += 1
|
||
added_keys.add((row.provider, row.model, row.media_type))
|
||
else:
|
||
_apply(row, data, skip=("id", "created_at"))
|
||
updated += 1
|
||
# 默认位以导出包为准:同能力下不在包里的旧默认必须让位,
|
||
# 否则 seed_specs / resolve_* 会因为排序偶然选中开发机没在用的那个模型。
|
||
# 例外是 --only-missing:这个开关的承诺是「这台机器已有的东西一概不动」,
|
||
# 所以只有当包里的默认模型是这次**新插进来**的那一条时才设默认。
|
||
# 早先无例外时翻过车:一次「新增 0、跳过 8」的导入什么都没写,却把服务器上
|
||
# 自选的 tts 默认悄悄改回开发机那套,日志上还是一片祥和。
|
||
db.flush()
|
||
kept: list[str] = []
|
||
for media_type in {d["media_type"] for d in rows}:
|
||
want = next((d for d in rows if d["media_type"] == media_type and d["is_default"]), None)
|
||
if want is None:
|
||
continue
|
||
if only_missing and (want["provider"], want.get("model", ""), media_type) not in added_keys:
|
||
kept.append(f"{media_type}={want['provider']}/{want['model']}")
|
||
continue
|
||
others = db.query(ModelSpec).filter(
|
||
ModelSpec.media_type == media_type,
|
||
ModelSpec.is_default.is_(True),
|
||
~((ModelSpec.provider == want["provider"]) & (ModelSpec.model == want.get("model", "")))
|
||
).update({"is_default": False})
|
||
if others:
|
||
print(f" · {media_type}: 取消 {others} 条旧默认,改为 "
|
||
f"{want['provider']}/{want['model']}")
|
||
if kept:
|
||
print(f" · --only-missing:保留本机默认位不改(包里的是 {'、'.join(sorted(kept))})")
|
||
if not dry:
|
||
db.commit()
|
||
note = ",--only-missing 跳过 " + str(skipped) if skipped else ""
|
||
verb = ("按部署包覆盖" if updated and not only_missing else "写入")
|
||
print(f" ✓ 模型配置:新增 {added},{verb} {updated}{note}"
|
||
+ ("" if not updated or only_missing else "(更新=用部署包的值覆盖服务器上的同名模型)"))
|
||
|
||
|
||
def _install_users(db, rows, *, dry: bool) -> None:
|
||
for data in rows:
|
||
row = db.scalars(select(User).where(User.username == data["username"])).first()
|
||
if row is None:
|
||
row = User()
|
||
_apply(row, data, skip=("id", "created_at", "updated_at"))
|
||
row.must_change_password = False
|
||
db.add(row)
|
||
print(f" ✓ 管理员 {row.username}(role={row.role})已创建,"
|
||
f"口令沿用导出时的哈希——登录后请立即修改")
|
||
else:
|
||
# 已存在的账号不重置口令:这台机器可能早改过密码,
|
||
# 覆盖成开发机的口令是个反直觉的倒退。
|
||
_apply(row, data, skip=("id", "created_at", "updated_at", "password_hash",
|
||
"must_change_password"))
|
||
diff = (data.get("password_hash") or "") != row.password_hash
|
||
print(f" · 管理员 {row.username} 已存在:只同步角色/显示名,不改口令"
|
||
+ ("(注意:它的口令与部署包不同,登录请用这台机器的口令)" if diff else ""))
|
||
if not dry:
|
||
db.commit()
|
||
|
||
|
||
def _install_platform(db, tables: dict, *, dry: bool) -> None:
|
||
for name in PLATFORM_TABLES:
|
||
rows = tables.get(name) or []
|
||
if not rows:
|
||
continue
|
||
table = Base.metadata.tables.get(name)
|
||
if table is None:
|
||
print(f" ! 跳过 {name}(当前代码没有这张表)")
|
||
continue
|
||
cols = [c.key for c in inspect(table).columns if c.key != "id"]
|
||
for data in rows:
|
||
pk = data.get("id")
|
||
if db.execute(text(f"SELECT 1 FROM {name} WHERE id=:i"), {"i": pk}).first():
|
||
continue
|
||
col_sql = ", ".join(cols)
|
||
val_sql = ", ".join(f":{c}" for c in cols)
|
||
db.execute(text(f"INSERT INTO {name} (id, {col_sql}) VALUES (:id, {val_sql})"),
|
||
{**{c: data.get(c) for c in cols}, "id": pk})
|
||
print(f" ✓ 平台资源库 {name}: 导入后共 "
|
||
f"{db.execute(text(f'SELECT COUNT(*) FROM {name}')).scalar()} 行")
|
||
if not dry:
|
||
db.commit()
|
||
|
||
|
||
def do_install(args) -> None:
|
||
src = Path(args.src)
|
||
specs = read_json(src, "model_specs.json")["rows"]
|
||
keys = read_json(src, "model_secret_keys.json")["rows"]
|
||
users = read_json(src, "users.json")["rows"]
|
||
lib = None
|
||
if args.with_platform_library:
|
||
lib = read_json(src, "platform_library.json")["rows"]
|
||
|
||
# 先量包再动库:数据集比代码旧时,装上去的后果是「静默少功能」,而 install 一旦
|
||
# 跑过就改掉了表结构与默认位、回不到原样,所以宁可在这里停住不动。
|
||
print(f"体检部署数据集({src})")
|
||
rep = audit_dataset(specs, keys, users, origin=f"部署包 {src.name}")
|
||
rep.show()
|
||
print_dataset_summary(rep, specs, keys, users)
|
||
if rep.errors and not args.allow_drift:
|
||
sys.exit("导入中止:部署包与当前代码对不上(上面 error 条目)。"
|
||
"在开发机 doctor --fix + 重新 export 再打包;确认无碍加 --allow-drift。")
|
||
|
||
if args.schema:
|
||
init_db() # 建表 + 列迁移,幂等
|
||
print(f" ✓ 表结构就绪:{engine.url.render_as_string(hide_password=True)}")
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
if not args.dry_run:
|
||
target = Report()
|
||
audit_target_db(target, db, only_missing=args.only_missing)
|
||
if target.items:
|
||
print("体检目标库")
|
||
target.show()
|
||
print("安装部署数据集(幂等,可反复执行)")
|
||
_install_master_key(db, keys, dry=args.dry_run, force=args.force_master_key)
|
||
_install_specs(db, specs, dry=args.dry_run, only_missing=args.only_missing)
|
||
_install_users(db, users, dry=args.dry_run)
|
||
if lib is not None:
|
||
_install_platform(db, lib, dry=args.dry_run)
|
||
if args.dry_run:
|
||
db.rollback()
|
||
print("\n--dry-run:未写入任何数据")
|
||
return
|
||
|
||
# 装完就验一次解密:这一步不过,说明主密钥和密文不是一对,
|
||
# 与其等业务报「缺 Key」,不如现在停下来。
|
||
from app.services import secrets as secret_svc
|
||
bad = [f"{s.provider}/{s.model}" for s in db.scalars(select(ModelSpec)).all()
|
||
if s.api_key_encrypted and not secret_svc.decrypt_value(db, s.api_key_encrypted)]
|
||
defaults = {s.media_type: f"{s.provider}/{s.model}"
|
||
for s in db.scalars(select(ModelSpec).where(ModelSpec.is_default.is_(True))).all()}
|
||
admins = db.scalars(select(User.username).where(User.role == "admin")).all()
|
||
print("\n校验")
|
||
print(f" 默认模型:{', '.join(f'{k}={v}' for k, v in sorted(defaults.items())) or '(无)'}")
|
||
print(f" 管理员:{', '.join(admins) or '(无)'}")
|
||
if bad:
|
||
sys.exit(f" ✗ 这些模型的 API Key 解不开(主密钥不匹配):{bad}")
|
||
print(" ✓ 所有 API Key 解密通过")
|
||
finally:
|
||
db.close()
|
||
|
||
|
||
def print_dataset_summary(rep: Report, specs: list[dict], keys: list[dict],
|
||
users: list[dict]) -> None:
|
||
"""把数据集摊开给人看:装到服务器上的到底是什么,一行一个结论。"""
|
||
defaults = sorted(f"{s['media_type']}={s['provider']}/{s['model']}"
|
||
for s in specs if s.get("is_default"))
|
||
real = sorted(f"{s['media_type']}/{s['provider']}/{s['model']}"
|
||
for s in specs if not (s["provider"] or "").startswith("mock"))
|
||
with_key = sorted(f"{s['provider']}/{s['model']}"
|
||
for s in specs if s.get("api_key_encrypted"))
|
||
print(f" 数据集:模型配置 {len(specs)} 条(真实模型 {'、'.join(real) or '无'})")
|
||
print(f" 默认位:{'、'.join(defaults) or '(无)'}")
|
||
print(f" 凭据:带 API Key {len(with_key)} 条"
|
||
+ ("(" + "、".join(with_key) + ")" if with_key else "")
|
||
+ f" · 主密钥 {len(keys)} 条 · 管理员 "
|
||
+ ("、".join(u["username"] for u in users) or "(无)"))
|
||
|
||
|
||
def do_doctor(args) -> None:
|
||
"""体检:默认量当前库(导出前自查),--src 时量部署包(deploy.sh 用这条)。
|
||
|
||
退出码非 0 = 有 error。`deploy.sh` 把它当打包闸门:在开发机上就把「数据比代码旧」
|
||
拦下来,比到了服务器上 install 才炸便宜得多——那时人已经不在终端前了。
|
||
"""
|
||
if args.src:
|
||
src = Path(args.src)
|
||
specs = read_json(src, "model_specs.json")["rows"]
|
||
keys = read_json(src, "model_secret_keys.json")["rows"]
|
||
users = read_json(src, "users.json")["rows"]
|
||
rep = audit_dataset(specs, keys, users, origin=f"部署包 {src}")
|
||
else:
|
||
db = SessionLocal()
|
||
try:
|
||
if args.fix:
|
||
changes = calibrate_specs(db, apply=True)
|
||
print("校准模型配置(只补缺,不覆盖人工调过的值)")
|
||
for line in changes or ["没有需要补的:数据与当前代码一致"]:
|
||
print(f" · {line}")
|
||
specs = [row_to_dict(s) for s in db.scalars(select(ModelSpec).order_by(ModelSpec.id)).all()]
|
||
keys = [row_to_dict(k) for k in db.scalars(select(ModelSecretKey).order_by(ModelSecretKey.id)).all()]
|
||
users = [{k: v for k, v in row_to_dict(u).items() if k not in USER_DROP}
|
||
for u in db.scalars(select(User).where(User.role == "admin")
|
||
.order_by(User.id)).all()]
|
||
rep = Report()
|
||
# 开发库里当然有业务数据,所以这里只查遗留表,不查「是不是干净库」
|
||
audit_orphan_tables(rep, engine)
|
||
audit_dataset(specs, keys, users, origin=f"开发库 {engine.url.database}", rep=rep)
|
||
finally:
|
||
db.close()
|
||
print_dataset_summary(rep, specs, keys, users)
|
||
print("体检结论")
|
||
rep.show()
|
||
if rep.errors and not args.allow_drift:
|
||
sys.exit(f"体检未过:{len(rep.errors)} 条 error、{len(rep.warns)} 条 warn")
|
||
print(f" → 判定通过(error 0 · warn {len(rep.warns)})")
|
||
|
||
|
||
# ================================================================== boundary
|
||
def do_boundary(args) -> None:
|
||
"""数据边界一览:这台机器上「部署包带来的」和「本机业务跑出来的」各有多少行。
|
||
|
||
给 `server.sh status` 用。写在 seed.py 而不是 shell 里内嵌 SQL,是为了和体检共用
|
||
同一批常量(SEED_TABLES / BUSINESS_TABLES):边界定义哪天变了,两边不会各说一套——
|
||
「部署只带配置、不带业务数据」这条约定最怕的就是两个地方各数各的。
|
||
"""
|
||
cfg = _table_counts(engine, SEED_TABLES)
|
||
if not cfg:
|
||
print(" 表结构还没建:先 ./scripts/server.sh install")
|
||
return
|
||
biz = _table_counts(engine, BUSINESS_TABLES)
|
||
print(" 配置(随部署包):" + "、".join(f"{k} {v}" for k, v in sorted(cfg.items())))
|
||
print(" 业务(本机产生):" + ("、".join(f"{k} {v}" for k, v in sorted(biz.items())) or "(表还没建)"))
|
||
if not any(biz.values()):
|
||
print(" → 业务表全空:干净起点,符合部署边界")
|
||
else:
|
||
print(f" → 本机已产生 {sum(biz.values())} 行业务数据。它不在部署包里,"
|
||
"换机器属于迁移,要单独备份 media/ 与库文件")
|
||
orphan = sorted(set(inspect(engine).get_table_names()) - set(Base.metadata.tables))
|
||
if orphan:
|
||
print(f" ! 代码不认识的表:{'、'.join(orphan)}(确认后手工 DROP,见工程规范 §6)")
|
||
|
||
|
||
# ================================================================== CLI
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0],
|
||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||
|
||
e = sub.add_parser("export", help="导出模型配置 + 主密钥 + 管理员到 JSON")
|
||
e.add_argument("--out", default=str(SEED_DIR), help="输出目录(默认 scripts/seed)")
|
||
e.add_argument("--with-platform-library", action="store_true", help="顺带导出平台资源库")
|
||
e.add_argument("--fix", action="store_true",
|
||
help="导出前先把库里的模型配置补齐到当前代码(补 modes、补工作流节点键)")
|
||
e.add_argument("--allow-drift", action="store_true", help="体检有 error 也照样导出")
|
||
e.set_defaults(func=do_export)
|
||
|
||
i = sub.add_parser("install", help="把 JSON 装进目标库(服务器首次部署用)")
|
||
i.add_argument("--src", default=str(SEED_DIR), help="JSON 目录(默认 scripts/seed)")
|
||
i.add_argument("--no-schema", dest="schema", action="store_false", help="不先建表")
|
||
i.add_argument("--dry-run", action="store_true", help="只报告将要发生什么")
|
||
i.add_argument("--force-master-key", action="store_true", help="允许覆盖已有主密钥")
|
||
i.add_argument("--with-platform-library", action="store_true", help="顺带导入平台资源库")
|
||
i.add_argument("--only-missing", action="store_true",
|
||
help="模型配置只补缺,不覆盖服务器上已有的同名模型")
|
||
i.add_argument("--allow-drift", action="store_true", help="体检有 error 也照样导入")
|
||
i.set_defaults(func=do_install)
|
||
|
||
d = sub.add_parser("doctor", help="体检:量开发库(默认)或量部署包(--src),不改数据")
|
||
d.add_argument("--src", default="", help="改量部署包:JSON 目录(如 scripts/seed)")
|
||
d.add_argument("--fix", action="store_true", help="量库时把模型配置补齐到当前代码(只补缺)")
|
||
d.add_argument("--allow-drift", action="store_true", help="有 error 也返回 0(只打印不拦)")
|
||
d.set_defaults(func=do_doctor)
|
||
|
||
b = sub.add_parser("boundary", help="数据边界一览:配置表与业务表各多少行(server.sh status 用)")
|
||
b.set_defaults(func=do_boundary)
|
||
|
||
args = ap.parse_args()
|
||
args.func(args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|