diff --git a/deploy_artifacts/.gitattributes b/deploy_artifacts/.gitattributes new file mode 100644 index 000000000..9ac5a78e4 --- /dev/null +++ b/deploy_artifacts/.gitattributes @@ -0,0 +1,4 @@ +*.sh text eol=lf +*.py text eol=lf +*.json text eol=lf +*.template text eol=lf diff --git a/deploy_artifacts/.gitignore b/deploy_artifacts/.gitignore new file mode 100644 index 000000000..b8d5cfd3e --- /dev/null +++ b/deploy_artifacts/.gitignore @@ -0,0 +1,6 @@ +*.tar +*.tar.gz +*.env +.env +releases/ +__pycache__/ diff --git a/deploy_artifacts/Dockerfile.kb b/deploy_artifacts/Dockerfile.kb new file mode 100644 index 000000000..970d9e75e --- /dev/null +++ b/deploy_artifacts/Dockerfile.kb @@ -0,0 +1,36 @@ +FROM node:24-alpine AS web-build +WORKDIR /build/ui +COPY ui/package.json ui/pnpm-lock.yaml ui/pnpm-workspace.yaml ./ +RUN npm install -g pnpm@11.23.0 && pnpm install --frozen-lockfile +COPY ui/ ./ +# Never reuse dist or Vite's relative development base for a release. +RUN rm -rf dist && npm run build-only -- --base /admin/ && npm run build-only-chat -- --base /chat/ && \ + test -f dist/admin/index.html && test -f dist/chat/index.html && \ + grep -q '/admin/assets/' dist/admin/index.html + +FROM ghcr.io/1panel-dev/maxkb-base:python3.11-pg17.6 AS backend-build +WORKDIR /opt/maxkb-app +COPY . . +RUN apt-get update && apt-get install -y --no-install-recommends gcc g++ gettext libexpat1-dev libffi-dev && \ + gcc -shared -fPIC -o ${MAXKB_SANDBOX_HOME}/sandbox.so installer/sandbox.c -ldl && \ + pip install uv --break-system-packages && python -m uv pip install -r pyproject.toml && \ + python apps/manage.py compilemessages && \ + python -m uv pip install --target=/opt/maxkb-app/sandbox/python-packages requests pymysql psycopg2-binary && \ + rm -rf ui installer deploy_artifacts && rm -rf /var/lib/apt/lists/* + +FROM ghcr.io/1panel-dev/maxkb-base:python3.11-pg17.6 +ARG RELEASE_VERSION +ARG GITHUB_COMMIT +LABEL io.unis.component=kb io.unis.layout=refact org.opencontainers.image.revision=${GITHUB_COMMIT} +ENV MAXKB_VERSION="${RELEASE_VERSION} (${GITHUB_COMMIT})" \ + AI_HUB_ASSET_DIR=/opt/maxkb/data/ai-hub-assets +WORKDIR /opt/maxkb-app +COPY --from=backend-build /opt/maxkb-app /opt/maxkb-app +COPY --from=backend-build /opt/py3 /opt/py3 +# main.py collect_static also uses ui/dist. Ship both locations in the image. +COPY --from=web-build /build/ui/dist /opt/maxkb-app/ui/dist +COPY --from=web-build /build/ui/dist /opt/maxkb-app/apps/static +EXPOSE 8080 +VOLUME /opt/maxkb +ENTRYPOINT ["bash", "-c"] +CMD ["/usr/bin/start-all.sh"] diff --git a/deploy_artifacts/Dockerfile.kb.dockerignore b/deploy_artifacts/Dockerfile.kb.dockerignore new file mode 100644 index 000000000..b23ce67a5 --- /dev/null +++ b/deploy_artifacts/Dockerfile.kb.dockerignore @@ -0,0 +1,34 @@ +**/.git +**/.idea +**/.venv +**/venv +**/__pycache__ +**/*.pyc +**/*.log +**/*.tar +**/*.tar.gz +**/*.zip +**/.env +**/.env.* +!ui/env/.env +!ui/env/.env.chat +**/node_modules +ui/dist* +ui/.summary-build +apps/static +data +logs +tmp +model +conf +config.yml +config.yaml +*.db +*.sqlite* +jieba.cache +mcp-engine-* +deploy_artifacts/releases +deploy_artifacts/tests +.codex +.agents +.ssh* diff --git a/deploy_artifacts/README_UPGRADE.md b/deploy_artifacts/README_UPGRADE.md new file mode 100644 index 000000000..1c919bea9 --- /dev/null +++ b/deploy_artifacts/README_UPGRADE.md @@ -0,0 +1,147 @@ +# v3.2 / develop → v3.2-refact / develop-refact + +本目录是重构版离线升级入口。旧部署材料已归档到 MaxKB `v3.2` 分支的 `deploy_artifacts/`。 + +升级保留原账号、知识库、文档、向量、应用、聊天记录和配置,不重新导入文档、不批量重建向量。数据库迁移在旧数据的**独立副本**上执行,原目录、原容器和原镜像均保留。需要维护窗口;这不是零停机切换。升级后回退到旧数据时间点,不会自动合并新版产生的数据。 + +## 1. 部署结构与兼容范围 + +| 项目 | 原版 | 重构版 | +|---|---|---| +| 应用 | MaxKB `v3.2` | MaxKB `v3.2-refact` | +| 解析 | markitdown `develop`,包含旧工作台 | `develop-refact`,固定 MinerU 2.7.4 源码,仅解析 API | +| 导图界面 | `mindmap-web` 独立页面 | MaxKB `/admin/mindmap`,需要登录和授权 | +| 旧入口 | 8989 `/admin/`、`/chat/`、`/mindmap/`;9898 | 保留原发布 IP/端口,导图入口跳转到新工作台 | +| 数据库/Redis | `airag` 容器内置 PostgreSQL 17 / Redis | 同一基础镜像系列,沿用原环境和持久化数据副本 | +| 异步任务 | MaxKB 自带消费者 | 自带消费者 + 同版本独立 Hub worker;Hub worker 与 airag 共用网络命名空间及数据目录 | +| NVIDIA | CUDA 镜像 | `unis-mineru:版本-cuda`,Docker GPU device request | +| 天数 MR100 | CoreX 镜像 | `unis-mineru:版本-corex`,CoreX 4.4.0 / 厂商 Torch / vLLM 0.11.2,保留厂商二进制版本 | + +脚本自动支持原四容器 `airag`、`mineru-api`、`mindmap-web`、`unis-gateway`,且 `/opt/maxkb`、`/models` 是 bind mount、PostgreSQL 17 与 Redis 内置的部署。外置数据库、命名卷数据库、额外挂载或定制配置模式会停止自动升级,按文末流程处理。 + +大模型服务仍通过平台已有模型接口使用;此包不安装或替换 Ollama/vLLM 大模型服务。GPU 配置针对 MinerU 解析容器,MaxKB 的嵌入模型与原包保持一致。CUDA 驱动须与选定 vLLM 基础镜像兼容;MR100 需要可用的 CoreX 驱动。两类卡均须在目标机器实测,不能把 Dockerfile 存在视为硬件验收通过。 + +## 2. 构建离线包 + +在 Linux 联网构建机准备两个相邻仓库 `maxkb/`、`markitdown/`,切到对应重构分支。先将本次功能所需源码及迁移纳入 Git;构建脚本拒绝脏工作区,避免提交分支与镜像内容不一致。不要将临时输出、凭据、虚拟环境或模型提交到仓库。 + +```bash +cd maxkb +VERSION=refact-20261106-1 GPU_FLAVOR=cuda bash deploy_artifacts/build-release.sh +# MR100 单独构建,不混用 CUDA 包: +VERSION=refact-20261106-1 GPU_FLAVOR=corex bash deploy_artifacts/build-release.sh +``` + +构建脚本重新构建 `/admin/` 和 `/chat/`,忽略本机 dist,将产物同时写入镜像的 `ui/dist` 与 `apps/static`。后端保持 PostgreSQL 17.6 基础镜像,不跨主版本升级数据库。解析镜像保留源码的六级标题改造,固定基础镜像中的 Torch、Torchvision、vLLM,依赖冲突会使构建失败。入口脚本使用 Bash,支持 `pipefail`。 + +每类包位于 `deploy_artifacts/releases/版本-卡型/`,包含 `images.tar`、`upgrade.py`、本说明、源码提交号及 SHA256。镜像使用版本标签,升级时再固定为本机 image ID;不覆盖旧 `latest`。模型权重单独复用,不进镜像包。包不携带目标服务器的凭据。 + +```bash +cd /path/to/offline-package +sha256sum -c SHA256SUMS.txt +sudo docker load -i images.tar +``` + +## 3. 检查旧部署 + +部署主机需要 Python 3.10+、Docker Compose v2(支持 `up --wait`)、GNU tar、足够磁盘空间及 root/Docker 权限。推荐统一通过 `sudo python3 upgrade.py` 运行。脚本不改源码测试服务,也不占用 8080/3000/8100;公开端口继承旧网关和独立导图端口。 + +```bash +sudo python3 upgrade.py inspect +``` + +`inspect` 不启动容器、不连数据库、不输出环境凭据。核对目录、镜像 ID、四个容器状态。若原容器在运行,先在维护窗口停止这四个容器,等异步任务完成;不要停止源码测试服务或共享数据库/Redis。 + +```bash +sudo docker stop --time 330 unis-gateway mindmap-web mineru-api airag +``` + +2026-11-06 对 52.118 的核查:四个旧业务容器已停止;原数据 `/home/tlw/unis/maxkb`,模型 `/home/tlw/unis/mineru-models`;源码测试服务继续占用 8080/3000/8100。**`ares-pg` 仍在运行,并直接挂载原数据下的 `data/postgresql/pgdata`**,不能启动旧 airag 的内置 PostgreSQL,也不能将该目录直接 tar 当作一致性冷备。该机是 NVIDIA RTX 3070 Ti,不是 MR100。执行前以 `inspect` 为准;共享挂载会使默认冷备模式停止。 + +## 4. 备份并准备副本 + +新数据目录必须不存在,且与原数据、模型、发布备份目录互不包含。准备父目录,不递归修改任何业务目录权限。预留至少原数据大小两倍加余量的空间,用于冷备归档和运行副本。 + +```bash +sudo mkdir -p /home/tlw/unis/releases +sudo python3 upgrade.py prepare \ + --release-dir /home/tlw/unis/releases/refact-20261106-1-cuda \ + --data-dir /home/tlw/unis/maxkb-refact-20261106-1 \ + --kb-image unis-kb:refact-20261106-1 \ + --parser-image unis-mineru:refact-20261106-1-cuda \ + --gateway-image unis-gateway:refact-20261106-1 \ + --gpu cuda \ + --online-db-container ares-pg +``` + +MR100 将 `--gpu` 改为 `corex`,解析镜像及发布目录后缀改为 `corex`。不要仅修改镜像标签而保留 NVIDIA 的运行配置。 + +上例为 52.118 的共享数据库情况,显式指定 `--online-db-container ares-pg`:只允许该 PostgreSQL 17 容器恰好挂载旧 PGDATA,使用 `pg_basebackup`(含 WAL)生成在线一致性物理备份,非数据库文件另存,随后恢复到独立目录。原容器和源码数据库继续运行,新 Docker 版从备份时点分离;之后两边新增的数据不会同步。备份不修改原数据库角色/认证设置,缺少复制权限会失败退出。 + +原生产部署的 PGDATA 若没有其它运行容器/进程使用,省略 `--online-db-container`,走停机冷备。正式切换有其它写入方时,须在维护窗口统一停止写入,再完成最终备份;在线复制不是双向数据同步或无损实时切换。 + +`prepare` 要求四个旧业务容器停止,检查其它数据使用方,校验 PG_VERSION=17。生成 `data.tar`(在线模式另有 `database.tar`)、SHA256、原容器配置、固定镜像 ID 的 `compose.json`,并提取到新数据目录;不会启动或迁移数据库。发现未明确指定的共享挂载会退出。 + +发布目录为 0700,状态/Compose 文件为 0600,包含原容器私有环境用于保留数据库密码等配置。不要上传 Git、在终端打印文件,或将发布目录当作普通安装包分发。新数据目录用于长期运行,升级后不要删除。模型按只读方式复用,MinerU 输出及导图图片均持久化到新数据目录。 + +## 5. 执行切换 + +```bash +sudo python3 upgrade.py apply \ + --release-dir /home/tlw/unis/releases/refact-20261106-1-cuda +``` + +执行顺序:校验备份/Compose → 检查原容器身份及端口 → 用新解析镜像检查 Torch/vLLM 和 GPU 可见性(不加载模型)→ 将旧容器改名保留 → 启动新容器及隔离网络 → 仅在新副本上执行 Django 迁移 → 等待应用/解析 HTTP 就绪 → 检查迁移全部应用。 + +应用镜像内原 `start-all.sh`/`main.py start` 会执行全部前向迁移,不使用 `--fake`,也不把目标迁移固定在旧的 `0007_ai_hub`。本次版本需包含 knowledge 0004/0005/**0006**、users 0002、models_provider 0002、system_manage 0007/0008,Django 自动处理依赖。没有新增数据库迁移文件。 + +新 Hub worker 使用同一应用 image ID,只消费 Hub 数据库任务;不启动额外 Celery/Beat,不执行任务恢复或清理命令。原 MaxKB 自带消费者继续使用副本中的 Redis。模型目录不重新下载,不运行 `down -v`、全局 prune 或 `chmod -R`。 + +失败时停止新服务、保留副本和原容器;不会自动启动用户原先停掉的旧程序。诊断应在服务器本地查看 `docker logs`,其中可能包含私有配置,不要直接公开。 + +## 6. 一次性配置与验收 + +旧导图的 `MINDMAP_LLM_*` 是独立服务环境,新版从平台模型管理读取;脚本保存旧容器配置,但**不猜测模型供应商、不把旧明文 Key 写入新模型表、不覆盖已有平台模型或授权**。升级后管理员需要完成: + +1. 在“解析服务”登记 `http://mineru-api:8000`,版本 2.7.4,协议 legacy,先用 pipeline;在工作台默认设置中选择该资源。 +2. 如平台已有对应大模型,直接选择;否则将旧导图模型通过平台模型管理导入,再设置工作台默认模型。旧环境中的 Key 仅在服务器本地核对输入。 +3. 将解析服务和模型授权给原使用人群;不扩大原知识库/应用权限。旧知识库、应用、模型原有记录由迁移保留。 + +旧浏览器本地的未保存编辑内容和导出文件不会自动转为新版“我的导图”;升级前下载需要保留的内容,升级后可重新导入 Markdown。不能将此类浏览器数据描述为数据库自动迁移。 + +完成以下真实检查后再开放使用: + +- 8989 `/admin/login`、`/admin/register` 及 `/chat/` 可打开;加载的 JS/CSS 来自新构建,没有 404。直接 8080 的源码测试页面不代表 Docker 版验收。 +- 旧账号可登录;管理员及普通用户授权正确;原知识库/文档/应用/聊天记录可见;原问答实际向量检索可用。 +- `/mindmap/` 和 9898 进入新版工作台;真实 PDF(含图片/六级标题)解析、总结、保存、重开和导出成功;Hub worker 确实完成任务。 +- 模型文件、导图图片和 MinerU 输出在重建容器后仍存在;源码测试服务和共享 Redis 未受影响。 +- MR100 另在对应机器完成同一组验收;52.118 的 NVIDIA 结果不替代 MR100 验收。 + +“HTTP healthy / 迁移完成”仅是启动检查,不等于以上业务验收。 + +## 7. 回退 + +```bash +# 保持与本次升级前相同的停机状态,只恢复旧容器名称: +sudo python3 upgrade.py rollback \ + --release-dir /home/tlw/unis/releases/refact-20261106-1-cuda + +# 如确需恢复旧版运行,在 rollback 同一次调用增加 --start-old。 +``` + +回退仅停止本发布项目的新容器并改名保留,恢复原容器名称;不会删除容器、数据、镜像、网络或模型。原目录没有被本脚本迁移,无需对新库做反向迁移。共享 PGDATA 仍被使用时,`--start-old` 会拒绝启动,以免两个 PostgreSQL 同时打开同一目录;恢复旧程序运行也必须使用另一个一致性数据副本。**新版开放写入后,回退旧库不包含新增数据**,需先停止写入并决定是否导出/补录;保留新副本用于恢复。 + +## 8. 非标准部署 / 源码测试服务 + +52.118 的源码测试目录分支名仍显示旧分支,但已含大量增量修改。不要在远程直接 `git checkout`、`reset --hard`、覆盖配置或切换数据库;该环境和旧 Docker 数据各自核验,各自发布。 + +外部 DB/Redis 或其它挂载方式:先确认数据库所有写入方、队列消费者及源码服务是否共用它们。在隔离数据库恢复 `pg_dump -Fc` 备份并复制持久化文件;为新版分配独立 Redis 实例/逻辑库及 Celery 队列,Web/Hub worker 指向同一新库。明确保留原 SECRET_KEY、加密密钥和模型配置。使用候选镜像运行 `python apps/manage.py migrate --plan` 审核,再执行完整 `migrate --noinput`;更新 `apps/static/admin` 和 chat 构建后启动同版本 Web/worker,完成上节验收。禁止在共享原库直接试迁移,也不调用本机 `main.py dev web`。 + +## 9. 脚本验证 + +```bash +python3 -m unittest discover -s deploy_artifacts/tests -v +bash -n deploy_artifacts/build-release.sh +``` + +测试覆盖 Compose 凭据原样传递、GPU 隔离配置、内部数据库/worker 网络、原数据保护和失败回退身份检查。它们不启动 Docker 或访问共享服务。实际镜像构建、数据库演练和两类 GPU 业务验收仍需分别记录。 diff --git a/deploy_artifacts/VALIDATION_20261106.md b/deploy_artifacts/VALIDATION_20261106.md new file mode 100644 index 000000000..2981ee435 --- /dev/null +++ b/deploy_artifacts/VALIDATION_20261106.md @@ -0,0 +1,57 @@ +# 升级准备与验证记录 · 2026-11-06 + +## 已完成 + +- 原工作区 `deploy_artifacts/` 归档到 MaxKB `v3.2`,本地提交 `e34d9011e`;未加入镜像、模型或私有配置。 +- 补齐重构版已引用但未跟踪的功能源码、knowledge 0006 迁移及回归脚本;整理前端 pnpm 11.23.0 锁文件。临时构建、缓存不纳入发布。 +- 新离线构建入口重新编译 admin/chat,将静态文件写入实际运行路径;CUDA/CoreX 解析镜像分开标记,入口使用 Bash,保留厂商二进制依赖约束。 +- 升级使用数据副本和独立网络,保留旧容器/镜像,提供冷备及共享 PostgreSQL 在线一致性备份两种模式。 + +## 真实环境发现 + +52.118 的源码目录仍显示 `v3.2` / `develop`,但有增量功能修改,不能按分支名认定实际内容。源码服务未切换分支或覆盖文件。 + +原 `airag`、`mineru-api`、`mindmap-web`、`unis-gateway` 均已停止;**运行中的 `ares-pg` 使用 `/home/tlw/unis/maxkb/data/postgresql/pgdata`**。因此不能启动原 airag 的内置 PostgreSQL,默认冷备升级会拒绝继续。在线模式从 `ares-pg` 取得 `pg_basebackup`,恢复到独立目录,原数据库保持运行。 + +该机为 NVIDIA RTX 3070 Ti 8 GB;没有 MR100 实机验收条件。 + +## 验证结果 + +| 检查 | 结果及边界 | +|---|---| +| 升级脚本测试 | 本机 Windows 15 项、52.118 Linux 15 项均通过;使用模拟 Docker 调用,不是完整发布测试 | +| Compose 配置 | CUDA / CoreX 均由 52.118 的 Docker Compose v5.5.1 校验通过,未启动对应解析业务镜像 | +| 网关 | 使用已有 Nginx 镜像对新模板执行 `nginx -t` 通过,未替换在线网关 | +| 前端 | 本机 admin/chat 的 Vite 生产构建通过;存在既有 eval 和大 chunk 警告;未做完整 TypeScript 检查或新 Docker 页面验收 | +| 知识版本 | 本机 SQLite 回归 35 项,2 项跳过;未访问共享数据库/Redis | +| 组件/总结/工作台/上下文 | Linux 候选源码中 SQLite 与模拟接口回归 120 项通过,未调用真实模型或外部 MCP | +| Python | 应用源码和升级脚本编译检查通过 | +| Shell | 两仓打包脚本 Bash 语法检查通过 | + +## 在线备份与副本迁移检查 + +有效演练目录:`/home/tlw/unis/upgrade-rehearsal-20261106-144635`。该目录中的 `database.tar` 来自在线 `pg_basebackup`(含 WAL),非数据库文件另存。早期直接 tar 的副本不作为一致性备份或验收依据。 + +在候选源码、现有测试机 Python 依赖和独立临时 PostgreSQL 中执行了完整 `migrate --plan`、`migrate --noinput`、`migrate --check`。未启动 Web、Redis 或任务消费者。检查前后: + +| 表 | 检查前 | 检查后 | +|---|---:|---:| +| user | 13 | 13 | +| knowledge | 8 | 8 | +| document | 8 | 8 | +| paragraph | 738 | 738 | +| embedding | 1021 | 1021 | +| application | 9 | 9 | +| model | 7 | 7 | + +**该共享库已经应用当前全部迁移,本轮新增执行的迁移数为 0。** 这证明此备份可独立启动、当前候选可连接并检查已有结构,记录数保留;不能据此声称纯旧版 `v3.2` 库首次前向迁移已经验证。临时数据库已停止,四个原业务容器保持停止,`ares-pg` 和源码测试服务继续运行。 + +## 发布前仍需完成 + +1. 在构建机实际构建两个 GPU 镜像包,检查依赖解析与模型路径;本轮未构建或导入候选业务镜像。 +2. 使用尚未应用重构迁移的独立旧版数据库备份执行首次升级,并进行登录、权限、问答、文件、向量和聊天记录验收。 +3. 完成导图解析资源与平台模型的映射及授权,处理旧浏览器未保存内容。此部分不是旧 `MINDMAP_LLM_*` 环境变量自动迁移。 +4. 在候选 Docker 8989/9898 入口检查 admin/chat、登录/注册及导图完整流程;NVIDIA、MR100 分别做真实模型/PDF 验收。 +5. 在开放写入前明确最终数据时间点与回退边界;共享源码环境与新 Docker 数据副本后续不会自动同步。 + +升级命令及限制见 [README_UPGRADE.md](README_UPGRADE.md)。本记录不等于上线完成报告。 diff --git a/deploy_artifacts/build-release.sh b/deploy_artifacts/build-release.sh new file mode 100644 index 000000000..092f583fc --- /dev/null +++ b/deploy_artifacts/build-release.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=$(cd "$(dirname "$0")/.." && pwd) +MINERU_ROOT=${MINERU_ROOT:-$(cd "$ROOT/../markitdown" && pwd)} +VERSION=${VERSION:?Set an immutable VERSION, e.g. refact-20261106-1} +GPU_FLAVOR=${GPU_FLAVOR:?Set GPU_FLAVOR=cuda or corex} +[[ "$VERSION" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]*$ ]] || exit 2 +case "$GPU_FLAVOR" in cuda|corex) ;; *) echo 'GPU_FLAVOR must be cuda or corex' >&2; exit 2 ;; esac +for item in "$ROOT:v3.2-refact" "$MINERU_ROOT:develop-refact"; do + repo=${item%:*}; branch=${item##*:} + test "$(git -C "$repo" branch --show-current)" = "$branch" || { echo "Wrong source branch: $branch required" >&2; exit 2; } + test -z "$(git -C "$repo" status --porcelain --untracked-files=normal)" || { + echo "Source tree is dirty. Commit release sources and exclude local outputs before building: $repo" >&2; exit 2; + } +done +OUT=${ARTIFACT_DIR:-$ROOT/deploy_artifacts/releases/$VERSION-$GPU_FLAVOR} +test ! -e "$OUT" || { echo 'Artifact directory already exists' >&2; exit 2; } +mkdir -p "$OUT" +docker build -f "$ROOT/deploy_artifacts/Dockerfile.kb" --build-arg RELEASE_VERSION="$VERSION" \ + --build-arg GITHUB_COMMIT="$(git -C "$ROOT" rev-parse HEAD)" -t "unis-kb:$VERSION" "$ROOT" +docker build -f "$ROOT/deploy_artifacts/gateway/Dockerfile" -t "unis-gateway:$VERSION" "$ROOT/deploy_artifacts/gateway" +API_FLAVOR="$GPU_FLAVOR" VERSION="$VERSION" bash "$MINERU_ROOT/scripts/build-images.sh" +docker save -o "$OUT/images.tar" "unis-kb:$VERSION" "unis-gateway:$VERSION" "unis-mineru:$VERSION-$GPU_FLAVOR" +cp "$ROOT/deploy_artifacts/upgrade.py" "$ROOT/deploy_artifacts/README_UPGRADE.md" "$OUT/" +printf 'MAXKB_COMMIT=%s\nMINERU_COMMIT=%s\nVERSION=%s\nGPU_FLAVOR=%s\n' \ + "$(git -C "$ROOT" rev-parse HEAD)" "$(git -C "$MINERU_ROOT" rev-parse HEAD)" "$VERSION" "$GPU_FLAVOR" > "$OUT/SOURCES.txt" +(cd "$OUT" && sha256sum images.tar upgrade.py README_UPGRADE.md SOURCES.txt > SHA256SUMS.txt) +echo "Offline release created: $OUT" diff --git a/deploy_artifacts/gateway/Dockerfile b/deploy_artifacts/gateway/Dockerfile new file mode 100644 index 000000000..d416085a8 --- /dev/null +++ b/deploy_artifacts/gateway/Dockerfile @@ -0,0 +1,5 @@ +FROM nginx:alpine +LABEL io.unis.component=gateway io.unis.layout=refact +ENV KB_UPSTREAM=http://airag:8080 +COPY nginx.conf.template /etc/nginx/templates/default.conf.template +EXPOSE 8989 9898 diff --git a/deploy_artifacts/gateway/nginx.conf.template b/deploy_artifacts/gateway/nginx.conf.template new file mode 100644 index 000000000..4b44b9354 --- /dev/null +++ b/deploy_artifacts/gateway/nginx.conf.template @@ -0,0 +1,35 @@ +server { + listen 8989; + client_max_body_size 100M; + location = /mindmap { return 302 /admin/mindmap; } + location ^~ /mindmap/ { return 302 /admin/mindmap; } + location / { + proxy_pass ${KB_UPSTREAM}; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + } +} +server { + listen 9898; + client_max_body_size 100M; + location = / { return 302 /admin/mindmap; } + location = /mindmap { return 302 /admin/mindmap; } + location ^~ /mindmap/ { return 302 /admin/mindmap; } + location / { + proxy_pass ${KB_UPSTREAM}; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + proxy_read_timeout 600s; + proxy_send_timeout 600s; + } +} diff --git a/deploy_artifacts/tests/test_upgrade.py b/deploy_artifacts/tests/test_upgrade.py new file mode 100644 index 000000000..2416bfe98 --- /dev/null +++ b/deploy_artifacts/tests/test_upgrade.py @@ -0,0 +1,193 @@ +import copy +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +SPEC = importlib.util.spec_from_file_location('upgrade', Path(__file__).resolve().parents[1] / 'upgrade.py') +upgrade = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(upgrade) + + +def containers(): + result = {} + for name in upgrade.LEGACY: + result[name] = {'Id': 'old-' + name, 'Image': 'sha256:old', 'Name': '/' + name, + 'State': {'Status': 'exited', 'Running': False}, + 'Config': {'Env': [], 'Labels': {}}, 'Mounts': [], + 'HostConfig': {'PortBindings': {}}} + result['airag']['Config']['Env'] = [ + 'MAXKB_DB_HOST=127.0.0.1', 'MAXKB_DB_PASSWORD=literal$secret=${VALUE}', + 'MAXKB_REDIS_HOST=127.0.0.1', 'MAXKB_VERSION=old', 'PATH=/old/python', + 'PGDATA=/opt/maxkb/data/postgresql/pgdata', 'POSTGRES_PASSWORD=literal$pg', + ] + result['mineru-api']['Config']['Env'] = ['CUDA_VISIBLE_DEVICES=1', 'MINERU_GPU_MEMORY_UTILIZATION=0.3', 'PYTHONPATH=/old/vendor', 'MINDMAP_LLM_API_KEY=legacy-private'] + for name, port in (('unis-gateway', 8989), ('mindmap-web', 9898)): + result[name]['HostConfig']['PortBindings'][str(port) + '/tcp'] = [{'HostIp': '127.0.0.1', 'HostPort': str(port + 100)}] + return result + + +class UpgradeTests(unittest.TestCase): + def config(self, flavor='cuda'): + return upgrade.make_compose(containers(), {'kb': 'sha256:kb', 'parser': 'sha256:parser', 'gateway': 'sha256:gateway'}, Path('/srv/unis/clone'), Path('/srv/unis/models'), flavor, 'release-test') + + def test_cuda_and_corex_have_separate_device_configuration(self): + cuda = self.config()['services']['mineru-api'] + corex = self.config('corex')['services']['mineru-api'] + self.assertEqual(cuda['deploy']['resources']['reservations']['devices'][0]['driver'], 'nvidia') + self.assertNotIn('privileged', cuda) + self.assertNotIn('runtime', corex) + self.assertNotIn('deploy', corex) + self.assertTrue(corex['privileged']) + self.assertIn('corex-4.4.0', corex['environment']['PYTHONPATH']) + self.assertEqual(cuda['environment']['CUDA_VISIBLE_DEVICES'], '1') + self.assertEqual(cuda['environment']['MINERU_GPU_MEMORY_UTILIZATION'], '0.3') + + def test_worker_shares_local_db_and_durable_assets(self): + config = self.config() + kb = config['services']['airag'] + worker = config['services']['hub-worker'] + self.assertEqual(worker['network_mode'], 'service:airag') + self.assertEqual(worker['environment'], kb['environment']) + self.assertEqual(worker['volumes'], kb['volumes']) + self.assertEqual(worker['image'], kb['image']) + self.assertEqual(worker['environment']['AI_HUB_ASSET_DIR'], '/opt/maxkb/data/ai-hub-assets') + self.assertNotIn('--recover', worker['command']) + self.assertNotIn('MAXKB_VERSION', kb['environment']) + self.assertNotIn('PATH', kb['environment']) + + def test_models_readonly_and_original_data_never_used_by_candidate(self): + config = self.config() + for service in config['services'].values(): + for volume in service.get('volumes', []): + self.assertNotEqual(volume['source'], '/srv/unis/original') + if volume['target'] == '/models': + self.assertTrue(volume['read_only']) + self.assertFalse(volume['bind']['create_host_path']) + self.assertEqual(config['networks']['default']['name'], 'release-test') + self.assertNotIn('mindmap-web', config['services']) + + def test_credentials_are_escaped_for_compose_and_not_sent_to_parser(self): + config = self.config() + escaped = upgrade.escape_compose(config) + value = escaped['services']['airag']['environment']['MAXKB_DB_PASSWORD'] + self.assertEqual(value, 'literal$$secret=$${VALUE}') + self.assertNotIn('MINDMAP_LLM_API_KEY', config['services']['mineru-api']['environment']) + self.assertEqual(config['services']['airag']['environment']['MAXKB_DB_PASSWORD'], 'literal$secret=${VALUE}') + + def test_existing_public_bindings_preserved(self): + ports = self.config()['services']['gateway']['ports'] + self.assertEqual(ports[0]['published'], '9089') + self.assertEqual(ports[1]['published'], '9998') + self.assertTrue(all(p['host_ip'] == '127.0.0.1' for p in ports)) + + def test_running_or_paused_legacy_container_refused(self): + for status in ('running', 'paused', 'restarting', 'dead'): + with self.subTest(status=status): + rows = containers() + rows['airag']['State']['Status'] = status + with self.assertRaises(ValueError): + upgrade.ensure_stopped(rows) + + def test_prepare_rejects_running_container_before_creating_files(self): + rows = containers() + rows['airag']['State']['Status'] = 'running' + with patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), Path('/srv/unis/models'))), patch.object(upgrade, 'run') as runner: + with self.assertRaises(ValueError): + upgrade.prepare(None) + runner.assert_not_called() + + def test_rollback_refuses_unrelated_container(self): + state = {'phase': 'failed', 'project': 'this-release', 'old': containers(), 'renamed': list(upgrade.LEGACY)} + stranger = {'Config': {'Labels': {'com.docker.compose.project': 'some-other-release'}}, 'Id': 'unrelated'} + with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inspect', return_value=stranger), patch.object(upgrade, 'run', return_value='unis-hub-worker\n') as runner: + with self.assertRaises(ValueError): + upgrade.rollback(Path('/srv/unis/release')) + self.assertEqual(runner.call_count, 1) + + def test_rollback_keeps_new_data_and_restores_original_stopped_state(self): + rows = containers() + state = {'phase': 'failed', 'project': 'this-release', 'old': rows, 'renamed': list(upgrade.LEGACY)} + def read(name): + if name.startswith('old-'): + row = copy.deepcopy(rows[name[4:]]) + row['Name'] += '-before-this-release' + return row + return {'Config': {'Labels': {'com.docker.compose.project': 'this-release'}}, 'Id': 'new-' + name} + with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inspect', side_effect=read), patch.object(upgrade, 'save'), patch.object(upgrade, 'run', return_value='\n'.join(upgrade.NEW)) as runner: + upgrade.rollback(Path('/srv/unis/release')) + commands = [call.args for call in runner.call_args_list] + self.assertFalse(any('start' in command or 'rm' in command for command in commands)) + stops = [command[-1] for command in commands if command[:2] == ('docker', 'stop')] + self.assertEqual(stops[-1], 'airag') + self.assertEqual(state['phase'], 'rolled-back') + + def test_backup_and_compose_tampering_refused(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'compose.json').write_text('{}') + (root / 'data.tar').write_bytes(b'cold-backup') + state = {'compose_sha256': upgrade.digest(root / 'compose.json'), 'backup_sha256': upgrade.digest(root / 'data.tar')} + (root / 'state.json').write_text(json.dumps(state)) + self.assertEqual(upgrade.load_release(root), state) + (root / 'data.tar').write_bytes(b'changed') + with self.assertRaisesRegex(ValueError, 'Backup checksum'): + upgrade.load_release(root) + (root / 'compose.json').write_text('{"services":{}}') + with self.assertRaisesRegex(ValueError, 'Compose configuration'): + upgrade.load_release(root) + + def test_wrong_migration_phase_refused(self): + with patch.object(upgrade, 'load_release', return_value={'phase': 'started-awaiting-acceptance'}), patch.object(upgrade, 'inventory') as inv: + with self.assertRaises(ValueError): + upgrade.apply(Path('/srv/unis/release')) + inv.assert_not_called() + + def test_data_changed_since_preparation_is_rejected_before_gpu_or_rename(self): + rows = containers() + state = {'phase': 'prepared', 'data': '/srv/unis/clone', 'old': rows, + 'original_signature': 'before', 'clone_signature': 'before'} + with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), None)), patch.object(upgrade, 'ensure_exclusive'), patch.object(upgrade, 'safe_path', return_value=Path('/srv/unis/clone')), patch.object(upgrade, 'tree_signature', return_value='changed'), patch.object(upgrade, 'compose') as compose, patch.object(upgrade, 'run') as runner: + with self.assertRaisesRegex(ValueError, 'data changed'): + upgrade.apply(Path('/srv/unis/release')) + runner.assert_not_called() + compose.assert_not_called() + + def test_migration_failure_freezes_clone_and_retains_original_containers(self): + rows = containers() + state = {'phase': 'prepared', 'project': 'this-release', 'data': '/srv/unis/clone', 'old': rows, + 'original_signature': 'same', 'clone_signature': 'same', 'renamed': []} + with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), None)), patch.object(upgrade, 'ensure_exclusive'), patch.object(upgrade, 'safe_path', return_value=Path('/srv/unis/clone')), patch.object(upgrade, 'tree_signature', return_value='same'), patch.object(upgrade.socket, 'socket'), patch.object(upgrade, 'save'), patch.object(upgrade, 'compose', side_effect=['gpu-ok', RuntimeError('migration failed'), 'stopped']) as compose, patch.object(upgrade, 'run') as runner: + with self.assertRaisesRegex(RuntimeError, 'migration failed'): + upgrade.apply(Path('/srv/unis/release')) + self.assertEqual(state['phase'], 'failed') + self.assertEqual(state['renamed'], list(upgrade.LEGACY)) + self.assertTrue(all(call.args[:2] == ('docker', 'rename') for call in runner.call_args_list)) + self.assertEqual(compose.call_args.args[1], 'stop') + + + def test_online_backup_container_must_mount_exact_legacy_pgdata(self): + row = {'Id': 'shared-pg', 'State': {'Running': True}, 'Config': {'Env': ['PG_MAJOR=17', 'PGDATA=/var/lib/postgresql/data']}, + 'Mounts': [{'Type': 'bind', 'Source': '/srv/unis/old/data/postgresql/pgdata', 'Destination': '/var/lib/postgresql/data'}]} + with patch.object(upgrade, 'inspect', return_value=row), patch.object(upgrade, 'safe_path', side_effect=Path): + value = upgrade.online_database('shared-pg', Path('/srv/unis/old'), containers()) + self.assertEqual(value['id'], 'shared-pg') + row['Mounts'][0]['Source'] = '/srv/unis/another-db' + with self.assertRaisesRegex(ValueError, 'exactly'): + upgrade.online_database('shared-pg', Path('/srv/unis/old'), containers()) + + def test_online_backup_does_not_put_password_in_command_arguments(self): + row = {'Config': {'Env': ['POSTGRES_USER=postgres', 'POSTGRES_PASSWORD=private$replication']}} + with tempfile.TemporaryDirectory() as directory, patch.object(upgrade, 'inspect', return_value=row), patch.object(upgrade.subprocess, 'run', return_value=Mock(returncode=0)) as runner: + upgrade.backup_online({'id': 'shared-pg'}, Path(directory) / 'database.tar', containers()) + args = runner.call_args.args[0] + self.assertNotIn('private$replication', ' '.join(args)) + self.assertEqual(runner.call_args.kwargs['env']['PGPASSWORD'], 'private$replication') + self.assertIn('pg_basebackup', args) + self.assertIn('fetch', args) + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy_artifacts/upgrade.py b/deploy_artifacts/upgrade.py new file mode 100644 index 000000000..486c7b630 --- /dev/null +++ b/deploy_artifacts/upgrade.py @@ -0,0 +1,458 @@ +"""Offline, cold-clone upgrade of the legacy four-container Unis deployment. + +Python 3.10+, Linux, Docker Compose v2. inspect is read-only. prepare never starts +containers. apply only migrates the cloned data; rollback preserves that clone. +Release state contains private container configuration: keep the directory 0700. +""" +import argparse +from contextlib import contextmanager +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import shutil +import socket +import subprocess +import sys + +LEGACY = ('airag', 'mineru-api', 'mindmap-web', 'unis-gateway') +NEW = ('unis-hub-worker', 'unis-gateway', 'mineru-api', 'airag') + + +def run(*args): + value = subprocess.run(args, capture_output=True, text=True, timeout=1800) + if value.returncode: + # Do not echo Docker's full configuration or application diagnostics. + raise RuntimeError('Command failed: ' + args[0] + ' ' + (args[1] if len(args) > 1 else '')) + return value.stdout + + +def inspect(name): + return json.loads(run('docker', 'inspect', name))[0] + + +def environment(row): + return dict(item.split('=', 1) for item in row['Config'].get('Env', []) if '=' in item) + + +def mount(row, destination): + matches = [m for m in row['Mounts'] if m['Destination'] == destination] + if len(matches) != 1 or matches[0]['Type'] != 'bind': + raise ValueError('A single bind mount is required: ' + destination) + return safe_path(matches[0]['Source']) + + +def safe_path(value): + path = Path(value) + if not path.is_absolute() or len(path.parts) < 4 or '..' in path.parts: + raise ValueError('Use a dedicated absolute data/release path') + if any(p.is_symlink() for p in (path, *path.parents)): + raise ValueError('Symlink paths are not supported') + return path + + +def overlaps(a, b): + return a == b or a in b.parents or b in a.parents + + +def inventory(): + rows = {name: inspect(name) for name in LEGACY} + kb = rows['airag'] + env = environment(kb) + if env.get('MAXKB_CONFIG_TYPE') != 'ENV': + raise ValueError('Automatic upgrade supports the legacy ENV deployment only') + if env.get('MAXKB_DB_HOST') != '127.0.0.1' or env.get('MAXKB_REDIS_HOST') != '127.0.0.1': + raise ValueError('External DB/Redis requires the manual upgrade procedure') + data = mount(kb, '/opt/maxkb') + models = mount(rows['mineru-api'], '/models') + pgdata = Path(env.get('PGDATA', '')) + if not pgdata.is_relative_to('/opt/maxkb'): + raise ValueError('PostgreSQL must reside inside the /opt/maxkb bind mount') + # A child volume could mask the real database. The legacy unused anonymous + # /var/lib/postgresql/data volume is harmless because PGDATA is elsewhere. + for m in kb['Mounts']: + destination = Path(m['Destination']) + if destination != Path('/opt/maxkb') and overlaps(destination, pgdata): + raise ValueError('PostgreSQL has an additional mount; manual backup is required') + if destination.is_relative_to('/opt/maxkb') and destination != Path('/opt/maxkb'): + raise ValueError('Nested persistent mounts require manual backup') + if destination.is_relative_to('/opt/maxkb-app'): + raise ValueError('Application code mounts must be reviewed manually') + version = data / pgdata.relative_to('/opt/maxkb') / 'PG_VERSION' + if not version.is_file() or version.read_text().strip() != '17': + raise ValueError('Expected the legacy PostgreSQL 17 data; do not initialize a new database') + if not (models / 'pipeline').is_dir() or not (models / 'vlm').is_dir(): + raise ValueError('Existing pipeline/vlm model directories are required') + return rows, data, models + + +def ensure_stopped(rows): + if any(row['State']['Status'] not in ('exited', 'created') for row in rows.values()): + raise ValueError('Stop all four legacy containers during the maintenance window first') + + +def ensure_exclusive(data, rows, online=None): + ids = run('docker', 'ps', '-q').split() + for container in ids: + row = inspect(container) + for m in row['Mounts']: + if overlaps(Path(m['Source']), data): + if online and row['Id'] == online['id'] and Path(m['Source']) == Path(online['pgdata']): + continue + raise ValueError('A running container shares the legacy data directory') + # Detect a native PostgreSQL using the same directory as well. + for proc in Path('/proc').iterdir(): + if not proc.name.isdigit(): + continue + try: + args = (proc / 'cmdline').read_bytes().split(b'\0') + except (OSError, PermissionError): + continue + if args and b'postgres' in args[0] and any(str(data).encode() in arg for arg in args[1:]): + raise ValueError('A native PostgreSQL process uses the legacy data directory') + + +def online_database(name, original, rows): + if not name: + return None + row = inspect(name) + if not row['State']['Running']: + raise ValueError('The online database container must be running') + env = environment(row) + pgdata = env.get('PGDATA', '/var/lib/postgresql/data') + expected = original / Path(environment(rows['airag'])['PGDATA']).relative_to('/opt/maxkb') + actual = mount(row, pgdata) + if actual != expected or env.get('PG_MAJOR') != '17': + raise ValueError('Online backup container must mount exactly the legacy PostgreSQL 17 directory') + return {'name': name, 'id': row['Id'], 'pgdata': str(expected)} + + +def backup_online(online, archive, rows): + row = inspect(online['id']) + env = environment(row) + client_env = os.environ.copy() + client_env['PGPASSWORD'] = env.get('POSTGRES_PASSWORD') or environment(rows['airag']).get('MAXKB_DB_PASSWORD', '') + user = env.get('POSTGRES_USER') or environment(rows['airag']).get('MAXKB_DB_USER', '') + # Password is inherited by docker exec through the client environment, never + # placed in argv, a script or a credential file. PostgreSQL supplies a consistent + # physical base backup including WAL while the source service stays online. + with archive.open('xb') as stream: + result = subprocess.run(['docker', 'exec', '-e', 'PGPASSWORD', online['id'], 'pg_basebackup', + '-U', user, '-w', '-D', '-', '-Ft', '-X', 'fetch', '--checkpoint=fast'], + env=client_env, stdout=stream, stderr=subprocess.PIPE, timeout=1800) + if result.returncode: + raise RuntimeError('Online pg_basebackup failed; no source roles or permissions were changed') + + +def published(row, port): + values = row['HostConfig'].get('PortBindings', {}).get(str(port) + '/tcp') or [] + if len(values) != 1: + raise ValueError('Exactly one legacy public port binding is required') + value = values[0] + return {'target': port, 'published': str(value['HostPort']), 'host_ip': value.get('HostIp') or '0.0.0.0', 'protocol': 'tcp'} + + +def make_compose(rows, images, data, models, flavor, project): + old = environment(rows['airag']) + env = {k: v for k, v in old.items() if k.startswith(('MAXKB_', 'POSTGRES_', 'REDIS_')) or k in ('PGDATA', 'PIP_TARGET')} + env.pop('MAXKB_VERSION', None) + env.update(MAXKB_MINDMAP_URL='/admin/mindmap', AI_HUB_ASSET_DIR='/opt/maxkb/data/ai-hub-assets') + volumes = [{'type': 'bind', 'source': str(data), 'target': '/opt/maxkb', 'bind': {'create_host_path': False}}] + health = {'test': ['CMD', 'curl', '-fsS', 'http://127.0.0.1:8080/admin/'], 'interval': '5s', 'timeout': '5s', 'retries': 60, 'start_period': '30s'} + parser_env = {k: v for k, v in environment(rows['mineru-api']).items() if k.startswith('MINERU_')} + parser_env.update(MINERU_MODEL_DIR='/models', MINERU_MODEL_SOURCE='local', MINERU_PIPELINE_MODEL_DIR='/models/pipeline', MINERU_VLM_MODEL_DIR='/models/vlm', MINERU_API_MAX_CONCURRENT_REQUESTS='1') + parser_env.pop('MINERU_TOOLS_CONFIG_JSON', None) + parser_env['CUDA_VISIBLE_DEVICES'] = environment(rows['mineru-api']).get('CUDA_VISIBLE_DEVICES', '0') + parser = {'image': images['parser'], 'container_name': 'mineru-api', 'restart': 'unless-stopped', 'ipc': 'host', + 'environment': parser_env, 'volumes': [ + {'type': 'bind', 'source': str(models), 'target': '/models', 'read_only': True, 'bind': {'create_host_path': False}}, + {'type': 'bind', 'source': str(data / 'mineru-output'), 'target': '/app/output', 'bind': {'create_host_path': False}}], + 'healthcheck': {'test': ['CMD', 'python3', '-c', 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8000/openapi.json", timeout=3)'], 'interval': '5s', 'timeout': '5s', 'retries': 60}} + if flavor == 'cuda': + parser['deploy'] = {'resources': {'reservations': {'devices': [{'driver': 'nvidia', 'count': 'all', 'capabilities': ['gpu']}]}}} + else: + # Preserve the established CoreX device exposure; never select NVIDIA runtime. + parser['privileged'] = True + parser['environment']['PYTHONPATH'] = '/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages' + services = { + 'airag': {'image': images['kb'], 'container_name': 'airag', 'restart': 'unless-stopped', 'environment': env, 'volumes': volumes, 'healthcheck': health}, + 'mineru-api': parser, + 'hub-worker': {'image': images['kb'], 'container_name': 'unis-hub-worker', 'restart': 'unless-stopped', + 'entrypoint': ['python', 'scripts/ai_hub_ops.py'], 'command': ['worker', '--concurrency', '1'], + 'network_mode': 'service:airag', 'environment': env, 'volumes': volumes, + 'depends_on': {'airag': {'condition': 'service_healthy'}}, 'stop_grace_period': '330s'}, + 'gateway': {'image': images['gateway'], 'container_name': 'unis-gateway', 'restart': 'unless-stopped', + 'environment': {'KB_UPSTREAM': 'http://airag:8080'}, + 'ports': [published(rows['unis-gateway'], 8989), published(rows['mindmap-web'], 9898)], + 'depends_on': {'airag': {'condition': 'service_healthy'}, 'mineru-api': {'condition': 'service_healthy'}}}, + } + return {'name': project, 'services': services, 'networks': {'default': {'name': project}}} + + +def escape_compose(value): + # Compose interpolates dollar signs even in a JSON file. Preserve legacy secrets literally. + if isinstance(value, str): + return value.replace('$', '$$') + if isinstance(value, list): + return [escape_compose(v) for v in value] + if isinstance(value, dict): + return {k: escape_compose(v) for k, v in value.items()} + return value + + +def save(path, value): + temp = path.with_suffix(path.suffix + '.tmp') + with temp.open('w', encoding='utf-8') as stream: + json.dump(value, stream, ensure_ascii=False, indent=2) + stream.write('\n') + temp.chmod(0o600) + temp.replace(path) + + +def digest(path): + h = hashlib.sha256() + with path.open('rb') as stream: + for block in iter(lambda: stream.read(1024 * 1024), b''): + h.update(block) + return h.hexdigest() + + +def tree_signature(root, exclude=None): + """Detect writes/restarts between the cold snapshot and cutover, without reading DB contents.""" + h = hashlib.sha256() + for path in sorted([root, *root.rglob('*')]): + if exclude and (path == exclude or exclude in path.parents): + continue + info = path.lstat() + value = (str(path.relative_to(root)), info.st_mode, info.st_size, info.st_mtime_ns, + info.st_uid, info.st_gid, os.readlink(path) if path.is_symlink() else '') + h.update(json.dumps(value).encode()) + return h.hexdigest() + + +def compose(release, *args): + return run('docker', 'compose', '-f', str(release / 'compose.json'), *args) + + +def prepare(args): + rows, original, models = inventory() + ensure_stopped(rows) + online = online_database(args.online_db_container, original, rows) + ensure_exclusive(original, rows, online) + release, data = safe_path(args.release_dir), safe_path(args.data_dir) + paths = (original, models, release, data) + if any(overlaps(a, b) for i, a in enumerate(paths) for b in paths[i + 1:]): + raise ValueError('Original data, models, release and cloned data directories must be separate') + if release.exists() or data.exists(): + raise ValueError('Release and cloned data directories must be new') + images = {} + for key, name in (('kb', args.kb_image), ('parser', args.parser_image), ('gateway', args.gateway_image)): + row = json.loads(run('docker', 'image', 'inspect', name))[0] + labels = row['Config'].get('Labels') or {} + if labels.get('io.unis.layout') != 'refact' or labels.get('io.unis.component') != key: + raise ValueError('Use candidate images built with build-release.sh') + if key == 'parser' and labels.get('io.unis.gpu') != args.gpu: + raise ValueError('Parser image GPU flavor does not match --gpu') + images[key] = row['Id'] + run('docker', 'compose', 'version') + # Require space for both an independent archive and the live clone. + size = int(run('du', '-sk', str(original)).split()[0]) * 1024 + for target in (release.parent, data.parent): + if not target.is_dir() or shutil.disk_usage(target).free < size * 2 + 1024**3: + raise ValueError('Create parent directories and reserve space for archive + data clone + 1 GiB') + release.mkdir(mode=0o700) + project = 'unis-refact-' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') + state = {'phase': 'preparing', 'project': project, 'original_data': str(original), 'data': str(data), + 'flavor': args.gpu, 'old': rows, 'renamed': [], 'new_ids': {}, 'images': images, 'online_db': online} + exclude = Path(online['pgdata']) if online else None + state['original_signature'] = tree_signature(original, exclude) + save(release / 'state.json', state) + excludes = ['--exclude=./' + str(exclude.relative_to(original))] if exclude else [] + run('tar', '--numeric-owner', '--acls', '--xattrs', *excludes, '-cpf', str(release / 'data.tar'), '-C', str(original), '.') + if online: + backup_online(online, release / 'database.tar', rows) + state['database_sha256'] = digest(release / 'database.tar') + ensure_stopped({name: inspect(name) for name in LEGACY}) + if tree_signature(original, exclude) != state['original_signature']: + raise ValueError('Original data changed during backup; discard this incomplete candidate') + state['backup_sha256'] = digest(release / 'data.tar') + data.mkdir(mode=0o700) + run('tar', '--same-owner', '--acls', '--xattrs', '-xpf', str(release / 'data.tar'), '-C', str(data)) + if online: + pgclone = data / exclude.relative_to(original) + pgclone.mkdir(parents=True, mode=0o700) + run('tar', '--same-owner', '-xpf', str(release / 'database.tar'), '-C', str(pgclone)) + info = exclude.stat() + os.chown(pgclone, info.st_uid, info.st_gid) + pgclone.chmod(0o700) + (data / 'mineru-output').mkdir(exist_ok=True) + state['clone_signature'] = tree_signature(data) + config = make_compose(rows, images, data, models, args.gpu, project) + save(release / 'compose.json', escape_compose(config)) + state['compose_sha256'] = digest(release / 'compose.json') + compose(release, 'config', '--quiet') + state['phase'] = 'prepared' + save(release / 'state.json', state) + print('Prepared: original data untouched; cloned data and private rollback material are ready.') + + +def load_release(release): + state = json.loads((release / 'state.json').read_text(encoding='utf-8')) + if digest(release / 'compose.json') != state['compose_sha256']: + raise ValueError('Compose configuration changed after preparation') + if digest(release / 'data.tar') != state['backup_sha256']: + raise ValueError('Backup checksum mismatch') + if state.get('online_db') and digest(release / 'database.tar') != state['database_sha256']: + raise ValueError('Online database backup checksum mismatch') + return state + + +def apply(release): + state = load_release(release) + if state['phase'] != 'prepared': + raise ValueError('Apply requires a prepared release; inspect failure state before retrying') + current, original, _ = inventory() + ensure_stopped(current) + online = state.get('online_db') + if online: + actual = online_database(online['name'], original, current) + if actual != online: + raise ValueError('Online database container changed after preparation') + ensure_exclusive(original, current, online) + clone = safe_path(state['data']) + ensure_exclusive(clone, current) + exclude = Path(online['pgdata']) if online else None + if tree_signature(original, exclude) != state['original_signature'] or tree_signature(clone) != state['clone_signature']: + raise ValueError('Original or cloned data changed after preparation; prepare a fresh release') + for name in LEGACY: + if current[name]['Id'] != state['old'][name]['Id']: + raise ValueError('Legacy containers changed after preparation') + if any(current[name]['State'].get(key) != state['old'][name]['State'].get(key) for key in ('StartedAt', 'FinishedAt')): + raise ValueError('Legacy containers ran after preparation; create a fresh data clone') + for row, port in ((current['unis-gateway'], 8989), (current['mindmap-web'], 9898)): + binding = published(row, port) + family = socket.AF_INET6 if ':' in binding['host_ip'] else socket.AF_INET + with socket.socket(family) as sock: + sock.bind((binding['host_ip'], int(binding['published']))) + # No model is loaded in this smoke test. A driver/runtime mismatch fails before switchover. + compose(release, 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'python3', 'mineru-api', '-c', + 'import torch, vllm; from mineru.version import __version__; assert __version__ == "2.7.4"; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))') + state['phase'] = 'switching' + save(release / 'state.json', state) + try: + for name in LEGACY: + backup_name = name + '-before-' + state['project'] + # Save the intention before rename, so interrupted operations can be recovered. + state['renamed'].append(name) + save(release / 'state.json', state) + run('docker', 'rename', name, backup_name) + # start-all.sh runs Django migrate only in the new cloned PostgreSQL. + state['phase'] = 'migrating-clone' + save(release / 'state.json', state) + compose(release, 'up', '-d', '--pull', 'never', '--wait', '--wait-timeout', '360') + for name in NEW: + row = inspect(name) + if row['Config'].get('Labels', {}).get('com.docker.compose.project') != state['project']: + raise ValueError('Unexpected container identity') + state['new_ids'][name] = row['Id'] + run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check') + state['phase'] = 'started-awaiting-acceptance' + save(release / 'state.json', state) + print('New containers are healthy. Complete the logged-in acceptance checklist before opening traffic.') + except Exception: + state['phase'] = 'failed' + save(release / 'state.json', state) + # Freeze failure data; retain all old containers and original data. + try: + compose(release, 'stop', '--timeout', '330') + except Exception: + pass + raise + + +def rollback(release, start_old=False): + state = load_release(release) + if state['phase'] not in ('switching', 'migrating-clone', 'failed', 'started-awaiting-acceptance'): + raise ValueError('No switched release to roll back') + if start_old: + ensure_exclusive(safe_path(state['original_data']), state['old']) + # Only act on containers belonging to this exact generated project. + names = run('docker', 'ps', '-a', '--format', '{{.Names}}').splitlines() + for name in NEW: + if name not in names: + continue + row = inspect(name) + if row['Config'].get('Labels', {}).get('com.docker.compose.project') != state['project']: + if row['Id'] == state['old'].get(name, {}).get('Id'): + continue + raise ValueError('Refusing to stop an unrelated container') + run('docker', 'stop', '--time', '330', name) + run('docker', 'rename', name, name + '-failed-' + state['project']) + for name in state['renamed']: + old_id = state['old'][name]['Id'] + row = inspect(old_id) + if row['State']['Running']: + raise ValueError('An original container was started outside this release') + if row['Name'].lstrip('/') != name: + run('docker', 'rename', old_id, name) + if start_old: + run('docker', 'start', *LEGACY) + state['phase'] = 'rolled-back' + save(release / 'state.json', state) + print('Original names restored. Original data unchanged; failed clone retained. New-version writes are not merged back.') + + +@contextmanager +def lock(): + import fcntl + flags = os.O_CREAT | os.O_WRONLY | os.O_NOFOLLOW + fd = os.open('/tmp/unis-refact-upgrade.lock', flags, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + yield + finally: + os.close(fd) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + actions = parser.add_subparsers(dest='action', required=True) + actions.add_parser('inspect') + p = actions.add_parser('prepare') + for name in ('release-dir', 'data-dir', 'kb-image', 'parser-image', 'gateway-image'): + p.add_argument('--' + name, required=True) + p.add_argument('--gpu', choices=['cuda', 'corex'], required=True) + p.add_argument('--online-db-container', help='Explicit PostgreSQL 17 container sharing the legacy PGDATA; use a consistent online base backup') + for name in ('apply', 'rollback'): + p = actions.add_parser(name) + p.add_argument('--release-dir', required=True) + if name == 'rollback': + p.add_argument('--start-old', action='store_true', help='Start old containers using the pre-upgrade data; new-version writes are not included') + args = parser.parse_args() + if os.name != 'posix' or not Path('/proc').is_dir(): + raise ValueError('Run on the Linux deployment host') + os.umask(0o077) + with lock(): + if args.action == 'inspect': + rows, data, models = inventory() + cold, reason = True, '' + try: + ensure_exclusive(data, rows) + except ValueError as exc: + cold, reason = False, str(exc) + print(json.dumps({'containers': {k: {'status': v['State']['Status'], 'image_id': v['Image']} for k, v in rows.items()}, 'data': str(data), 'models': str(models), 'cold_backup_eligible': cold, 'reason': reason}, indent=2)) + elif args.action == 'prepare': + prepare(args) + elif args.action == 'apply': + apply(safe_path(args.release_dir)) + else: + rollback(safe_path(args.release_dir), args.start_old) + + +if __name__ == '__main__': + try: + main() + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError, KeyError, json.JSONDecodeError) as exc: + print('Upgrade stopped: ' + str(exc) if isinstance(exc, (ValueError, RuntimeError)) else 'Upgrade stopped: ' + type(exc).__name__, file=sys.stderr) + sys.exit(1)