UnisKB/deploy_artifacts/upgrade.py

477 lines
24 KiB
Python

"""Offline, consistent-clone upgrade of the legacy four-container Unis deployment.
Python 3.10+, Linux, Docker Compose v2. inspect is read-only. prepare never starts
containers. apply only migrates the cloned data; rollback preserves that clone.
Release state contains private container configuration: keep the directory 0700.
"""
import argparse
from contextlib import contextmanager
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import shutil
import socket
import subprocess
import sys
LEGACY = ('airag', 'mineru-api', 'mindmap-web', 'unis-gateway')
NEW = ('unis-hub-worker', 'unis-gateway', 'mineru-api', 'airag')
def run(*args):
value = subprocess.run(args, capture_output=True, text=True, timeout=1800)
if value.returncode:
# Do not echo Docker's full configuration or application diagnostics.
raise RuntimeError('Command failed: ' + args[0] + ' ' + (args[1] if len(args) > 1 else ''))
return value.stdout
def inspect(name):
return json.loads(run('docker', 'inspect', name))[0]
def environment(row):
return dict(item.split('=', 1) for item in row['Config'].get('Env', []) if '=' in item)
def mount(row, destination):
matches = [m for m in row['Mounts'] if m['Destination'] == destination]
if len(matches) != 1 or matches[0]['Type'] != 'bind':
raise ValueError('A single bind mount is required: ' + destination)
return safe_path(matches[0]['Source'])
def safe_path(value):
path = Path(value)
if not path.is_absolute() or len(path.parts) < 4 or '..' in path.parts:
raise ValueError('Use a dedicated absolute data/release path')
if any(p.is_symlink() for p in (path, *path.parents)):
raise ValueError('Symlink paths are not supported')
return path
def overlaps(a, b):
return a == b or a in b.parents or b in a.parents
def require_internal_services(env):
if env.get('MAXKB_DB_HOST') != '127.0.0.1' or env.get('MAXKB_REDIS_HOST') != '127.0.0.1':
raise ValueError('External DB/Redis requires the manual upgrade procedure')
if str(env.get('MAXKB_DB_PORT', '5432')) != '5432' or str(env.get('MAXKB_REDIS_PORT', '6379')) != '6379':
raise ValueError('Expected the internal PostgreSQL/Redis ports 5432/6379')
if any(value for key, value in env.items() if key.startswith('MAXKB_REDIS_SENTINEL_')):
raise ValueError('Redis Sentinel requires the manual upgrade procedure')
def inventory():
rows = {name: inspect(name) for name in LEGACY}
kb = rows['airag']
env = environment(kb)
if env.get('MAXKB_CONFIG_TYPE') != 'ENV':
raise ValueError('Automatic upgrade supports the legacy ENV deployment only')
require_internal_services(env)
data = mount(kb, '/opt/maxkb')
models = mount(rows['mineru-api'], '/models')
pgdata = Path(env.get('PGDATA', ''))
if not pgdata.is_relative_to('/opt/maxkb'):
raise ValueError('PostgreSQL must reside inside the /opt/maxkb bind mount')
# A child volume could mask the real database. The legacy unused anonymous
# /var/lib/postgresql/data volume is harmless because PGDATA is elsewhere.
for m in kb['Mounts']:
destination = Path(m['Destination'])
if destination != Path('/opt/maxkb') and overlaps(destination, pgdata):
raise ValueError('PostgreSQL has an additional mount; manual backup is required')
if destination.is_relative_to('/opt/maxkb') and destination != Path('/opt/maxkb'):
raise ValueError('Nested persistent mounts require manual backup')
if destination.is_relative_to('/opt/maxkb-app'):
raise ValueError('Application code mounts must be reviewed manually')
version = data / pgdata.relative_to('/opt/maxkb') / 'PG_VERSION'
if not version.is_file() or version.read_text().strip() != '17':
raise ValueError('Expected the legacy PostgreSQL 17 data; do not initialize a new database')
if not (models / 'pipeline').is_dir() or not (models / 'vlm').is_dir():
raise ValueError('Existing pipeline/vlm model directories are required')
return rows, data, models
def ensure_stopped(rows):
if any(row['State']['Status'] not in ('exited', 'created') for row in rows.values()):
raise ValueError('Stop all four legacy containers during the maintenance window first')
def ensure_exclusive(data, rows, online=None):
ids = run('docker', 'ps', '-q').split()
for container in ids:
row = inspect(container)
for m in row['Mounts']:
if overlaps(Path(m['Source']), data):
if online and row['Id'] == online['id'] and Path(m['Source']) == Path(online['pgdata']):
continue
raise ValueError('A running container shares the legacy data directory')
# Detect a native PostgreSQL using the same directory as well.
for proc in Path('/proc').iterdir():
if not proc.name.isdigit():
continue
try:
args = (proc / 'cmdline').read_bytes().split(b'\0')
except (OSError, PermissionError):
continue
if args and b'postgres' in args[0] and any(str(data).encode() in arg for arg in args[1:]):
raise ValueError('A native PostgreSQL process uses the legacy data directory')
def online_database(name, original, rows):
if not name:
return None
row = inspect(name)
if not row['State']['Running']:
raise ValueError('The online database container must be running')
env = environment(row)
pgdata = env.get('PGDATA', '/var/lib/postgresql/data')
expected = original / Path(environment(rows['airag'])['PGDATA']).relative_to('/opt/maxkb')
actual = mount(row, pgdata)
if actual != expected or env.get('PG_MAJOR') != '17':
raise ValueError('Online backup container must mount exactly the legacy PostgreSQL 17 directory')
return {'name': name, 'id': row['Id'], 'pgdata': str(expected)}
def backup_online(online, archive, rows):
row = inspect(online['id'])
env = environment(row)
client_env = os.environ.copy()
client_env['PGPASSWORD'] = env.get('POSTGRES_PASSWORD') or environment(rows['airag']).get('MAXKB_DB_PASSWORD', '')
user = env.get('POSTGRES_USER') or environment(rows['airag']).get('MAXKB_DB_USER', '')
# Password is inherited by docker exec through the client environment, never
# placed in argv, a script or a credential file. PostgreSQL supplies a consistent
# physical base backup including WAL while the source service stays online.
with archive.open('xb') as stream:
result = subprocess.run(['docker', 'exec', '-e', 'PGPASSWORD', online['id'], 'pg_basebackup',
'-U', user, '-w', '-D', '-', '-Ft', '-X', 'fetch', '--checkpoint=fast'],
env=client_env, stdout=stream, stderr=subprocess.PIPE, timeout=1800)
if result.returncode:
raise RuntimeError('Online pg_basebackup failed; no source roles or permissions were changed')
def published(row, port):
values = row['HostConfig'].get('PortBindings', {}).get(str(port) + '/tcp') or []
if len(values) != 1:
raise ValueError('Exactly one legacy public port binding is required')
value = values[0]
return {'target': port, 'published': str(value['HostPort']), 'host_ip': value.get('HostIp') or '0.0.0.0', 'protocol': 'tcp'}
def make_compose(rows, images, data, models, flavor, project):
old = environment(rows['airag'])
require_internal_services(old)
env = {k: v for k, v in old.items() if k.startswith(('MAXKB_', 'POSTGRES_', 'REDIS_')) or k in ('PGDATA', 'PIP_TARGET')}
env.pop('MAXKB_VERSION', None)
env.update(MAXKB_MINDMAP_URL='/admin/mindmap', AI_HUB_ASSET_DIR='/opt/maxkb/data/ai-hub-assets')
env.update(MAXKB_DB_HOST='127.0.0.1', MAXKB_DB_PORT='5432', MAXKB_REDIS_HOST='127.0.0.1', MAXKB_REDIS_PORT='6379')
volumes = [{'type': 'bind', 'source': str(data), 'target': '/opt/maxkb', 'bind': {'create_host_path': False}}]
health = {'test': ['CMD', 'curl', '-fsS', 'http://127.0.0.1:8080/admin/'], 'interval': '5s', 'timeout': '5s', 'retries': 60, 'start_period': '30s'}
parser_env = {k: v for k, v in environment(rows['mineru-api']).items() if k.startswith('MINERU_')}
parser_env.update(MINERU_MODEL_DIR='/models', MINERU_MODEL_SOURCE='local', MINERU_PIPELINE_MODEL_DIR='/models/pipeline', MINERU_VLM_MODEL_DIR='/models/vlm', MINERU_API_MAX_CONCURRENT_REQUESTS='1')
parser_env.pop('MINERU_TOOLS_CONFIG_JSON', None)
parser_env['CUDA_VISIBLE_DEVICES'] = environment(rows['mineru-api']).get('CUDA_VISIBLE_DEVICES', '0')
parser = {'image': images['parser'], 'container_name': 'mineru-api', 'restart': 'unless-stopped', 'ipc': 'host',
'environment': parser_env, 'volumes': [
{'type': 'bind', 'source': str(models), 'target': '/models', 'read_only': True, 'bind': {'create_host_path': False}},
{'type': 'bind', 'source': str(data / 'mineru-output'), 'target': '/app/output', 'bind': {'create_host_path': False}}],
'healthcheck': {'test': ['CMD', 'python3', '-c', 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=3)'], 'interval': '5s', 'timeout': '5s', 'retries': 60}}
if flavor == 'cuda':
parser['runtime'] = 'nvidia'
parser['deploy'] = {'resources': {'reservations': {'devices': [{'driver': 'nvidia', 'count': 'all', 'capabilities': ['gpu']}]}}}
else:
# Preserve the established CoreX device exposure; never select NVIDIA runtime.
parser['privileged'] = True
parser['environment']['PYTHONPATH'] = '/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages'
services = {
'airag': {'image': images['kb'], 'container_name': 'airag', 'restart': 'unless-stopped',
'command': ['/usr/bin/start-all.sh'], 'environment': env, 'volumes': volumes, 'healthcheck': health},
'mineru-api': parser,
'hub-worker': {'image': images['kb'], 'container_name': 'unis-hub-worker', 'restart': 'unless-stopped',
'entrypoint': ['python', 'scripts/ai_hub_ops.py'], 'command': ['worker', '--concurrency', '1'],
'network_mode': 'service:airag', 'environment': env, 'volumes': volumes,
'depends_on': {'airag': {'condition': 'service_healthy'}}, 'stop_grace_period': '330s'},
'gateway': {'image': images['gateway'], 'container_name': 'unis-gateway', 'restart': 'unless-stopped',
'environment': {'KB_UPSTREAM': 'http://airag:8080'},
'ports': [published(rows['unis-gateway'], 8989), published(rows['mindmap-web'], 9898)],
'depends_on': {'airag': {'condition': 'service_healthy'}, 'mineru-api': {'condition': 'service_healthy'}}},
}
return {'name': project, 'services': services, 'networks': {'default': {'name': project}}}
def escape_compose(value):
# Compose interpolates dollar signs even in a JSON file. Preserve legacy secrets literally.
if isinstance(value, str):
return value.replace('$', '$$')
if isinstance(value, list):
return [escape_compose(v) for v in value]
if isinstance(value, dict):
return {k: escape_compose(v) for k, v in value.items()}
return value
def save(path, value):
temp = path.with_suffix(path.suffix + '.tmp')
with temp.open('w', encoding='utf-8') as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
stream.write('\n')
temp.chmod(0o600)
temp.replace(path)
def digest(path):
h = hashlib.sha256()
with path.open('rb') as stream:
for block in iter(lambda: stream.read(1024 * 1024), b''):
h.update(block)
return h.hexdigest()
def tree_signature(root, exclude=None):
"""Detect writes/restarts between the cold snapshot and cutover, without reading DB contents."""
h = hashlib.sha256()
for path in sorted([root, *root.rglob('*')]):
if exclude and (path == exclude or exclude in path.parents):
continue
info = path.lstat()
value = (str(path.relative_to(root)), info.st_mode, info.st_size, info.st_mtime_ns,
info.st_uid, info.st_gid, os.readlink(path) if path.is_symlink() else '')
h.update(json.dumps(value).encode())
return h.hexdigest()
def compose(release, *args):
return run('docker', 'compose', '-f', str(release / 'compose.json'), *args)
def prepare(args):
rows, original, models = inventory()
ensure_stopped(rows)
online = online_database(args.online_db_container, original, rows)
ensure_exclusive(original, rows, online)
release, data = safe_path(args.release_dir), safe_path(args.data_dir)
paths = (original, models, release, data)
if any(overlaps(a, b) for i, a in enumerate(paths) for b in paths[i + 1:]):
raise ValueError('Original data, models, release and cloned data directories must be separate')
if release.exists() or data.exists():
raise ValueError('Release and cloned data directories must be new')
images = {}
for key, name in (('kb', args.kb_image), ('parser', args.parser_image), ('gateway', args.gateway_image)):
row = json.loads(run('docker', 'image', 'inspect', name))[0]
labels = row['Config'].get('Labels') or {}
if labels.get('io.unis.layout') != 'refact' or labels.get('io.unis.component') != key:
raise ValueError('Use candidate images built with build-release.sh')
if key == 'parser' and labels.get('io.unis.gpu') != args.gpu:
raise ValueError('Parser image GPU flavor does not match --gpu')
images[key] = row['Id']
run('docker', 'compose', 'version')
# Require space for both an independent archive and the live clone.
size = int(run('du', '-sk', str(original)).split()[0]) * 1024
for target in (release.parent, data.parent):
if not target.is_dir() or shutil.disk_usage(target).free < size * 2 + 1024**3:
raise ValueError('Create parent directories and reserve space for archive + data clone + 1 GiB')
release.mkdir(mode=0o700)
project = 'unis-refact-' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')
state = {'phase': 'preparing', 'project': project, 'original_data': str(original), 'data': str(data),
'flavor': args.gpu, 'old': rows, 'renamed': [], 'new_ids': {}, 'images': images, 'online_db': online}
exclude = Path(online['pgdata']) if online else None
state['original_signature'] = tree_signature(original, exclude)
save(release / 'state.json', state)
excludes = ['--exclude=./' + str(exclude.relative_to(original))] if exclude else []
run('tar', '--numeric-owner', '--acls', '--xattrs', *excludes, '-cpf', str(release / 'data.tar'), '-C', str(original), '.')
if online:
backup_online(online, release / 'database.tar', rows)
state['database_sha256'] = digest(release / 'database.tar')
ensure_stopped({name: inspect(name) for name in LEGACY})
if tree_signature(original, exclude) != state['original_signature']:
raise ValueError('Original data changed during backup; discard this incomplete candidate')
state['backup_sha256'] = digest(release / 'data.tar')
data.mkdir(mode=0o700)
run('tar', '--same-owner', '--acls', '--xattrs', '-xpf', str(release / 'data.tar'), '-C', str(data))
if online:
pgclone = data / exclude.relative_to(original)
pgclone.mkdir(parents=True, mode=0o700)
run('tar', '--same-owner', '-xpf', str(release / 'database.tar'), '-C', str(pgclone))
info = exclude.stat()
os.chown(pgclone, info.st_uid, info.st_gid)
pgclone.chmod(0o700)
(data / 'mineru-output').mkdir(exist_ok=True)
state['clone_signature'] = tree_signature(data)
config = make_compose(rows, images, data, models, args.gpu, project)
save(release / 'compose.json', escape_compose(config))
state['compose_sha256'] = digest(release / 'compose.json')
compose(release, 'config', '--quiet')
state['phase'] = 'prepared'
save(release / 'state.json', state)
print('Prepared: original data untouched; cloned data and private rollback material are ready.')
def load_release(release):
state = json.loads((release / 'state.json').read_text(encoding='utf-8'))
if digest(release / 'compose.json') != state['compose_sha256']:
raise ValueError('Compose configuration changed after preparation')
if digest(release / 'data.tar') != state['backup_sha256']:
raise ValueError('Backup checksum mismatch')
if state.get('online_db') and digest(release / 'database.tar') != state['database_sha256']:
raise ValueError('Online database backup checksum mismatch')
return state
def apply(release):
state = load_release(release)
if state['phase'] != 'prepared':
raise ValueError('Apply requires a prepared release; inspect failure state before retrying')
current, original, _ = inventory()
ensure_stopped(current)
online = state.get('online_db')
if online:
actual = online_database(online['name'], original, current)
if actual != online:
raise ValueError('Online database container changed after preparation')
ensure_exclusive(original, current, online)
clone = safe_path(state['data'])
ensure_exclusive(clone, current)
exclude = Path(online['pgdata']) if online else None
if tree_signature(original, exclude) != state['original_signature'] or tree_signature(clone) != state['clone_signature']:
raise ValueError('Original or cloned data changed after preparation; prepare a fresh release')
for name in LEGACY:
if current[name]['Id'] != state['old'][name]['Id']:
raise ValueError('Legacy containers changed after preparation')
if any(current[name]['State'].get(key) != state['old'][name]['State'].get(key) for key in ('StartedAt', 'FinishedAt')):
raise ValueError('Legacy containers ran after preparation; create a fresh data clone')
for row, port in ((current['unis-gateway'], 8989), (current['mindmap-web'], 9898)):
binding = published(row, port)
family = socket.AF_INET6 if ':' in binding['host_ip'] else socket.AF_INET
with socket.socket(family) as sock:
sock.bind((binding['host_ip'], int(binding['published'])))
# No model is loaded in this smoke test. A driver/runtime mismatch fails before switchover.
compose(release, 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'python3', 'mineru-api', '-c',
'import torch, vllm; from mineru.version import __version__; assert __version__ == "2.7.6"; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))')
state['phase'] = 'switching'
save(release / 'state.json', state)
try:
for name in LEGACY:
backup_name = name + '-before-' + state['project']
# Save the intention before rename, so interrupted operations can be recovered.
state['renamed'].append(name)
save(release / 'state.json', state)
run('docker', 'rename', name, backup_name)
# start-all.sh runs Django migrate only in the new cloned PostgreSQL.
state['phase'] = 'migrating-clone'
save(release / 'state.json', state)
compose(release, 'up', '-d', '--pull', 'never', '--wait', '--wait-timeout', '360')
for name in NEW:
row = inspect(name)
if row['Config'].get('Labels', {}).get('com.docker.compose.project') != state['project']:
raise ValueError('Unexpected container identity')
state['new_ids'][name] = row['Id']
run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check')
# Migration 0008 creates the tables, but the one-time importer is a
# separate command. It imports legacy ADMIN users only on first creation
# of the unified admin role and never restores later demotions.
run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'unify_platform_roles')
checks = run('docker', 'exec', 'airag', 'python', '/usr/local/bin/unis-check-internal-services.py')
save(release / 'internal-services.json', json.loads(checks.strip().splitlines()[-1]))
state['phase'] = 'started-awaiting-acceptance'
save(release / 'state.json', state)
print('New containers are healthy. Complete the logged-in acceptance checklist before opening traffic.')
except Exception:
state['phase'] = 'failed'
save(release / 'state.json', state)
# Freeze failure data; retain all old containers and original data.
try:
compose(release, 'stop', '--timeout', '330')
except Exception:
pass
raise
def rollback(release, start_old=False):
state = load_release(release)
if state['phase'] not in ('switching', 'migrating-clone', 'failed', 'started-awaiting-acceptance'):
raise ValueError('No switched release to roll back')
if start_old:
ensure_exclusive(safe_path(state['original_data']), state['old'])
# Only act on containers belonging to this exact generated project.
names = run('docker', 'ps', '-a', '--format', '{{.Names}}').splitlines()
for name in NEW:
if name not in names:
continue
row = inspect(name)
if row['Config'].get('Labels', {}).get('com.docker.compose.project') != state['project']:
if row['Id'] == state['old'].get(name, {}).get('Id'):
continue
raise ValueError('Refusing to stop an unrelated container')
run('docker', 'stop', '--time', '330', name)
run('docker', 'rename', name, name + '-failed-' + state['project'])
for name in state['renamed']:
old_id = state['old'][name]['Id']
row = inspect(old_id)
if row['State']['Running']:
raise ValueError('An original container was started outside this release')
if row['Name'].lstrip('/') != name:
run('docker', 'rename', old_id, name)
if start_old:
run('docker', 'start', *LEGACY)
state['phase'] = 'rolled-back'
save(release / 'state.json', state)
print('Original names restored. Original data unchanged; failed clone retained. New-version writes are not merged back.')
@contextmanager
def lock():
import fcntl
flags = os.O_CREAT | os.O_WRONLY | os.O_NOFOLLOW
fd = os.open('/tmp/unis-refact-upgrade.lock', flags, 0o600)
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
yield
finally:
os.close(fd)
def main():
parser = argparse.ArgumentParser(description=__doc__)
actions = parser.add_subparsers(dest='action', required=True)
actions.add_parser('inspect')
p = actions.add_parser('prepare')
for name in ('release-dir', 'data-dir', 'kb-image', 'parser-image', 'gateway-image'):
p.add_argument('--' + name, required=True)
p.add_argument('--gpu', choices=['cuda', 'corex'], required=True)
p.add_argument('--online-db-container', help='Explicit PostgreSQL 17 container sharing the legacy PGDATA; use a consistent online base backup')
for name in ('apply', 'rollback'):
p = actions.add_parser(name)
p.add_argument('--release-dir', required=True)
if name == 'rollback':
p.add_argument('--start-old', action='store_true', help='Start old containers using the pre-upgrade data; new-version writes are not included')
args = parser.parse_args()
if os.name != 'posix' or not Path('/proc').is_dir():
raise ValueError('Run on the Linux deployment host')
os.umask(0o077)
with lock():
if args.action == 'inspect':
rows, data, models = inventory()
cold, reason = True, ''
try:
ensure_exclusive(data, rows)
except ValueError as exc:
cold, reason = False, str(exc)
print(json.dumps({'containers': {k: {'status': v['State']['Status'], 'image_id': v['Image']} for k, v in rows.items()}, 'data': str(data), 'models': str(models), 'cold_backup_eligible': cold, 'reason': reason}, indent=2))
elif args.action == 'prepare':
prepare(args)
elif args.action == 'apply':
apply(safe_path(args.release_dir))
else:
rollback(safe_path(args.release_dir), args.start_old)
if __name__ == '__main__':
try:
main()
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError, KeyError, json.JSONDecodeError) as exc:
print('Upgrade stopped: ' + str(exc) if isinstance(exc, (ValueError, RuntimeError)) else 'Upgrade stopped: ' + type(exc).__name__, file=sys.stderr)
sys.exit(1)