From 4aea21b6a102ca2a98d058f235ea0a1ac12fc495 Mon Sep 17 00:00:00 2001 From: panyy Date: Fri, 6 Nov 2026 15:27:47 +0800 Subject: [PATCH] fix(deploy): verify bundled PostgreSQL and Redis after upgrade --- deploy_artifacts/Dockerfile.kb | 1 + deploy_artifacts/README_UPGRADE.md | 6 +++ deploy_artifacts/VALIDATION_20261106.md | 17 +++++- deploy_artifacts/check_internal_services.py | 57 ++++++++++++++++++++ deploy_artifacts/tests/test_upgrade.py | 60 +++++++++++++++++++++ deploy_artifacts/upgrade.py | 21 ++++++-- 6 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 deploy_artifacts/check_internal_services.py diff --git a/deploy_artifacts/Dockerfile.kb b/deploy_artifacts/Dockerfile.kb index 970d9e75e..f3a617714 100644 --- a/deploy_artifacts/Dockerfile.kb +++ b/deploy_artifacts/Dockerfile.kb @@ -30,6 +30,7 @@ COPY --from=backend-build /opt/py3 /opt/py3 # main.py collect_static also uses ui/dist. Ship both locations in the image. COPY --from=web-build /build/ui/dist /opt/maxkb-app/ui/dist COPY --from=web-build /build/ui/dist /opt/maxkb-app/apps/static +COPY deploy_artifacts/check_internal_services.py /usr/local/bin/unis-check-internal-services.py EXPOSE 8080 VOLUME /opt/maxkb ENTRYPOINT ["bash", "-c"] diff --git a/deploy_artifacts/README_UPGRADE.md b/deploy_artifacts/README_UPGRADE.md index 1c919bea9..206f45687 100644 --- a/deploy_artifacts/README_UPGRADE.md +++ b/deploy_artifacts/README_UPGRADE.md @@ -19,6 +19,10 @@ 脚本自动支持原四容器 `airag`、`mineru-api`、`mindmap-web`、`unis-gateway`,且 `/opt/maxkb`、`/models` 是 bind mount、PostgreSQL 17 与 Redis 内置的部署。外置数据库、命名卷数据库、额外挂载或定制配置模式会停止自动升级,按文末流程处理。 +升级后的运行拓扑仍是 **MaxKB 内置 PG/Redis**:`airag` 的 `/usr/bin/start-all.sh` 启动容器内部 `127.0.0.1:5432` 和 `127.0.0.1:6379`,持久化目录分别为 `/opt/maxkb/data/postgresql/pgdata`、`/opt/maxkb/data/redis`。Compose 不创建独立 PG/Redis 服务,也不把 `ares-pg`、`ares-redis` 当作新版运行依赖。`--online-db-container ares-pg` 只用于开发环境的在线备份来源。Hub worker 共用 airag 的网络命名空间,因此它的 localhost 同样指向这两个内置服务。 + +启动后脚本还会用 Django 的实际数据库连接和实际缓存客户端检查服务地址、服务端数据目录及 Redis PING,结果写入私有发布目录的 `internal-services.json`。连接到外部服务或持久化路径错误会使此次发布验收失败。可在容器内单独复核:`docker exec airag python /usr/local/bin/unis-check-internal-services.py`,该检查不修改业务记录、不输出密码。 + 大模型服务仍通过平台已有模型接口使用;此包不安装或替换 Ollama/vLLM 大模型服务。GPU 配置针对 MinerU 解析容器,MaxKB 的嵌入模型与原包保持一致。CUDA 驱动须与选定 vLLM 基础镜像兼容;MR100 需要可用的 CoreX 驱动。两类卡均须在目标机器实测,不能把 Dockerfile 存在视为硬件验收通过。 ## 2. 构建离线包 @@ -99,6 +103,8 @@ sudo python3 upgrade.py apply \ 失败时停止新服务、保留副本和原容器;不会自动启动用户原先停掉的旧程序。诊断应在服务器本地查看 `docker logs`,其中可能包含私有配置,不要直接公开。 +2026-11-06 追加核查发现原天数包 `unis-mindmap-api-corex-latest.tar` 仍保留,包内源码为 2.7.1。它可作为旧解析栈回退材料,但当前脚本仍按 2.7.4 候选镜像校验,不能直接将该旧包换入本流程:旧包接口为 `/api/file_parse`,还需处理版本目录、连通性检查及启动校验的差异。是否采用该发布基线待明确;此处未执行 MinerU 源码回退。 + ## 6. 一次性配置与验收 旧导图的 `MINDMAP_LLM_*` 是独立服务环境,新版从平台模型管理读取;脚本保存旧容器配置,但**不猜测模型供应商、不把旧明文 Key 写入新模型表、不覆盖已有平台模型或授权**。升级后管理员需要完成: diff --git a/deploy_artifacts/VALIDATION_20261106.md b/deploy_artifacts/VALIDATION_20261106.md index 2981ee435..7b38698cf 100644 --- a/deploy_artifacts/VALIDATION_20261106.md +++ b/deploy_artifacts/VALIDATION_20261106.md @@ -19,7 +19,7 @@ | 检查 | 结果及边界 | |---|---| -| 升级脚本测试 | 本机 Windows 15 项、52.118 Linux 15 项均通过;使用模拟 Docker 调用,不是完整发布测试 | +| 升级脚本测试 | 追加内置服务验证后,本机 Windows 19 项、52.118 Linux 19 项均通过;模拟 Docker 调用不等于完整发布测试 | | Compose 配置 | CUDA / CoreX 均由 52.118 的 Docker Compose v5.5.1 校验通过,未启动对应解析业务镜像 | | 网关 | 使用已有 Nginx 镜像对新模板执行 `nginx -t` 通过,未替换在线网关 | | 前端 | 本机 admin/chat 的 Vite 生产构建通过;存在既有 eval 和大 chunk 警告;未做完整 TypeScript 检查或新 Docker 页面验收 | @@ -46,6 +46,21 @@ **该共享库已经应用当前全部迁移,本轮新增执行的迁移数为 0。** 这证明此备份可独立启动、当前候选可连接并检查已有结构,记录数保留;不能据此声称纯旧版 `v3.2` 库首次前向迁移已经验证。临时数据库已停止,四个原业务容器保持停止,`ares-pg` 和源码测试服务继续运行。 +## 内置 PG/Redis 追加验证 + +用户确认 `ares-pg` 是开发阶段单独挂载旧 PGDATA 的安排;正式 Docker 拓扑仍使用 MaxKB 内置 PostgreSQL 和 Redis。 + +验证目录:`/home/tlw/unis/internal-services-check-20261106-151833`。从上节的一致性备份恢复到另一个新目录,以原 MaxKB 应用镜像启动自带 `start-postgres.sh`、`start-redis.sh`,容器 `network=none`、无端口映射,不启动 Web/Celery/Hub worker。使用 Django 实际连接配置执行只读查询及缓存 PING: + +| 服务 | 实际地址 | 实际版本 | 服务端数据目录 | +|---|---|---|---| +| 内置 PostgreSQL | 127.0.0.1:5432 | 17.6 | /opt/maxkb/data/postgresql/pgdata | +| 内置 Redis | 127.0.0.1:6379 | 8.0.2 | /opt/maxkb/data/redis | + +两者均通过。因容器禁用外部网络,此结果不依赖开发用 `ares-pg` / `ares-redis`。检查后临时容器停止,`ares-pg` 的容器 ID、启动时间及运行状态未变,原 airag 仍停止。 + +此测试覆盖原应用镜像自带的 PG/Redis 启动链和新检查程序;尚未构建候选完整业务镜像。候选 Compose 已明确使用内置地址、标准端口和 `start-all.sh`,且发布流程新增实际连接检查,不能把本次底层检查写成新版整包已验收。 + ## 发布前仍需完成 1. 在构建机实际构建两个 GPU 镜像包,检查依赖解析与模型路径;本轮未构建或导入候选业务镜像。 diff --git a/deploy_artifacts/check_internal_services.py b/deploy_artifacts/check_internal_services.py new file mode 100644 index 000000000..02a571234 --- /dev/null +++ b/deploy_artifacts/check_internal_services.py @@ -0,0 +1,57 @@ +"""Verify effective MaxKB DB/cache connections without modifying business data.""" +import json +import os +from pathlib import Path +import sys + + +def verify(db, cache, pgdata): + config = db.settings_dict + redis_config = cache.connection_pool.connection_kwargs + if config.get('HOST') != '127.0.0.1' or int(config.get('PORT') or 5432) != 5432: + raise ValueError('MaxKB is not connected to its internal PostgreSQL') + if redis_config.get('host') != '127.0.0.1' or int(redis_config.get('port', 6379)) != 6379: + raise ValueError('MaxKB is not connected to its internal Redis') + with db.cursor() as cursor: + cursor.execute("SELECT current_setting('data_directory'), host(inet_server_addr()), inet_server_port(), current_setting('server_version')") + actual_dir, address, port, version = cursor.fetchone() + if actual_dir != pgdata or address != '127.0.0.1' or port != 5432: + raise ValueError('PostgreSQL server path or listener is outside the internal deployment') + if not str(version).startswith('17.'): + raise ValueError('Expected the internal PostgreSQL 17 runtime') + if not cache.ping(): + raise ValueError('Internal Redis PING failed') + config_dir = cache.config_get('dir') + directory = config_dir.get('dir', config_dir.get(b'dir')) + if isinstance(directory, bytes): + directory = directory.decode() + if directory != '/opt/maxkb/data/redis': + raise ValueError('Redis data directory is not on the persistent MaxKB mount') + return {'postgresql': {'host': address, 'port': port, 'version': version, 'data_directory': actual_dir}, + 'redis': {'host': '127.0.0.1', 'port': 6379, 'version': cache.info('server')['redis_version'], + 'data_directory': directory, 'ping': True}} + + +def main(): + root = Path('/opt/maxkb-app') + os.chdir(root) + sys.path.insert(0, str(root / 'apps')) + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings') + os.environ['SERVER_NAME'] = 'web' + import django + django.setup() + from django.db import connection + from django_redis import get_redis_connection + pgdata = os.environ.get('PGDATA', '') + if not Path(pgdata).is_relative_to('/opt/maxkb/data/postgresql'): + raise ValueError('PGDATA is outside the persistent MaxKB mount') + print(json.dumps(verify(connection, get_redis_connection('default'), pgdata))) + + +if __name__ == '__main__': + try: + main() + except Exception: + # Connection exceptions can include credentials/DSNs; do not echo them. + print('Internal service verification failed; check DB/cache configuration privately.', file=sys.stderr) + sys.exit(1) diff --git a/deploy_artifacts/tests/test_upgrade.py b/deploy_artifacts/tests/test_upgrade.py index 2416bfe98..64b606096 100644 --- a/deploy_artifacts/tests/test_upgrade.py +++ b/deploy_artifacts/tests/test_upgrade.py @@ -3,12 +3,16 @@ import importlib.util import json from pathlib import Path import tempfile +from types import SimpleNamespace import unittest from unittest.mock import Mock, patch SPEC = importlib.util.spec_from_file_location('upgrade', Path(__file__).resolve().parents[1] / 'upgrade.py') upgrade = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(upgrade) +CHECK_SPEC = importlib.util.spec_from_file_location('internal_check', Path(__file__).resolve().parents[1] / 'check_internal_services.py') +internal_check = importlib.util.module_from_spec(CHECK_SPEC) +CHECK_SPEC.loader.exec_module(internal_check) def containers(): @@ -57,6 +61,62 @@ class UpgradeTests(unittest.TestCase): self.assertNotIn('--recover', worker['command']) self.assertNotIn('MAXKB_VERSION', kb['environment']) self.assertNotIn('PATH', kb['environment']) + self.assertEqual(kb['command'], ['/usr/bin/start-all.sh']) + self.assertEqual(kb['environment']['MAXKB_DB_PORT'], '5432') + self.assertEqual(kb['environment']['MAXKB_REDIS_PORT'], '6379') + self.assertNotIn('postgres', config['services']) + self.assertNotIn('redis', config['services']) + + def test_external_and_sentinel_settings_cannot_enter_internal_deployment(self): + for field, value in (('MAXKB_DB_HOST', 'ares-pg'), ('MAXKB_REDIS_HOST', 'ares-redis'), + ('MAXKB_DB_PORT', '5433'), ('MAXKB_REDIS_PORT', '6380'), + ('MAXKB_REDIS_SENTINEL_SENTINELS', 'sentinel:26379')): + rows = containers() + env = upgrade.environment(rows['airag']) + env[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + upgrade.require_internal_services(env) + + def service_clients(self): + db = Mock() + db.settings_dict = {'HOST': '127.0.0.1', 'PORT': '5432'} + cursor = db.cursor.return_value.__enter__ = Mock(return_value=Mock()) + db.cursor.return_value.__exit__ = Mock(return_value=False) + cursor.return_value.fetchone.return_value = ('/opt/maxkb/data/postgresql/pgdata', '127.0.0.1', 5432, '17.6') + cache = Mock() + cache.connection_pool = SimpleNamespace(connection_kwargs={'host': '127.0.0.1', 'port': 6379}) + cache.ping.return_value = True + cache.config_get.return_value = {'dir': '/opt/maxkb/data/redis'} + cache.info.return_value = {'redis_version': '7.0.15'} + return db, cache + + def test_runtime_check_uses_actual_db_and_cache_configuration(self): + db, cache = self.service_clients() + report = internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata') + self.assertEqual(report['postgresql']['host'], '127.0.0.1') + self.assertTrue(report['redis']['ping']) + query = db.cursor.return_value.__enter__.return_value.execute.call_args.args[0] + self.assertIn('data_directory', query) + self.assertNotIn('UPDATE', query) + + def test_runtime_check_rejects_effective_external_connection(self): + db, cache = self.service_clients() + db.settings_dict['HOST'] = 'ares-pg' + with self.assertRaisesRegex(ValueError, 'internal PostgreSQL'): + internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata') + db.settings_dict['HOST'] = '127.0.0.1' + cache.connection_pool.connection_kwargs['host'] = 'ares-redis' + with self.assertRaisesRegex(ValueError, 'internal Redis'): + internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata') + + def test_runtime_check_rejects_wrong_server_data_directory(self): + db, cache = self.service_clients() + cache.config_get.return_value = {'dir': '/tmp/redis'} + with self.assertRaisesRegex(ValueError, 'persistent MaxKB mount'): + internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata') + cache.config_get.return_value = {'dir': '/opt/maxkb/data/redis'} + with self.assertRaisesRegex(ValueError, 'server path'): + internal_check.verify(db, cache, '/var/lib/postgresql/data') def test_models_readonly_and_original_data_never_used_by_candidate(self): config = self.config() diff --git a/deploy_artifacts/upgrade.py b/deploy_artifacts/upgrade.py index 486c7b630..9bb210eba 100644 --- a/deploy_artifacts/upgrade.py +++ b/deploy_artifacts/upgrade.py @@ -1,4 +1,4 @@ -"""Offline, cold-clone upgrade of the legacy four-container Unis deployment. +"""Offline, consistent-clone upgrade of the legacy four-container Unis deployment. Python 3.10+, Linux, Docker Compose v2. inspect is read-only. prepare never starts containers. apply only migrates the cloned data; rollback preserves that clone. @@ -56,14 +56,22 @@ def overlaps(a, b): return a == b or a in b.parents or b in a.parents +def require_internal_services(env): + if env.get('MAXKB_DB_HOST') != '127.0.0.1' or env.get('MAXKB_REDIS_HOST') != '127.0.0.1': + raise ValueError('External DB/Redis requires the manual upgrade procedure') + if str(env.get('MAXKB_DB_PORT', '5432')) != '5432' or str(env.get('MAXKB_REDIS_PORT', '6379')) != '6379': + raise ValueError('Expected the internal PostgreSQL/Redis ports 5432/6379') + if any(value for key, value in env.items() if key.startswith('MAXKB_REDIS_SENTINEL_')): + raise ValueError('Redis Sentinel requires the manual upgrade procedure') + + def inventory(): rows = {name: inspect(name) for name in LEGACY} kb = rows['airag'] env = environment(kb) if env.get('MAXKB_CONFIG_TYPE') != 'ENV': raise ValueError('Automatic upgrade supports the legacy ENV deployment only') - if env.get('MAXKB_DB_HOST') != '127.0.0.1' or env.get('MAXKB_REDIS_HOST') != '127.0.0.1': - raise ValueError('External DB/Redis requires the manual upgrade procedure') + require_internal_services(env) data = mount(kb, '/opt/maxkb') models = mount(rows['mineru-api'], '/models') pgdata = Path(env.get('PGDATA', '')) @@ -155,9 +163,11 @@ def published(row, port): def make_compose(rows, images, data, models, flavor, project): old = environment(rows['airag']) + require_internal_services(old) env = {k: v for k, v in old.items() if k.startswith(('MAXKB_', 'POSTGRES_', 'REDIS_')) or k in ('PGDATA', 'PIP_TARGET')} env.pop('MAXKB_VERSION', None) env.update(MAXKB_MINDMAP_URL='/admin/mindmap', AI_HUB_ASSET_DIR='/opt/maxkb/data/ai-hub-assets') + env.update(MAXKB_DB_HOST='127.0.0.1', MAXKB_DB_PORT='5432', MAXKB_REDIS_HOST='127.0.0.1', MAXKB_REDIS_PORT='6379') volumes = [{'type': 'bind', 'source': str(data), 'target': '/opt/maxkb', 'bind': {'create_host_path': False}}] health = {'test': ['CMD', 'curl', '-fsS', 'http://127.0.0.1:8080/admin/'], 'interval': '5s', 'timeout': '5s', 'retries': 60, 'start_period': '30s'} parser_env = {k: v for k, v in environment(rows['mineru-api']).items() if k.startswith('MINERU_')} @@ -176,7 +186,8 @@ def make_compose(rows, images, data, models, flavor, project): parser['privileged'] = True parser['environment']['PYTHONPATH'] = '/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages' services = { - 'airag': {'image': images['kb'], 'container_name': 'airag', 'restart': 'unless-stopped', 'environment': env, 'volumes': volumes, 'healthcheck': health}, + 'airag': {'image': images['kb'], 'container_name': 'airag', 'restart': 'unless-stopped', + 'command': ['/usr/bin/start-all.sh'], 'environment': env, 'volumes': volumes, 'healthcheck': health}, 'mineru-api': parser, 'hub-worker': {'image': images['kb'], 'container_name': 'unis-hub-worker', 'restart': 'unless-stopped', 'entrypoint': ['python', 'scripts/ai_hub_ops.py'], 'command': ['worker', '--concurrency', '1'], @@ -357,6 +368,8 @@ def apply(release): raise ValueError('Unexpected container identity') state['new_ids'][name] = row['Id'] run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check') + checks = run('docker', 'exec', 'airag', 'python', '/usr/local/bin/unis-check-internal-services.py') + save(release / 'internal-services.json', json.loads(checks.strip().splitlines()[-1])) state['phase'] = 'started-awaiting-acceptance' save(release / 'state.json', state) print('New containers are healthy. Complete the logged-in acceptance checklist before opening traffic.')