"""Offline, consistent-clone upgrade of the legacy four-container Unis deployment. Python 3.10+, Linux, Docker Compose v2. inspect is read-only. prepare never starts containers. apply only migrates the cloned data; rollback preserves that clone. Release state contains private container configuration: keep the directory 0700. """ import argparse from contextlib import contextmanager from datetime import datetime, timezone import hashlib import json import os from pathlib import Path import shutil import socket import subprocess import sys LEGACY = ('airag', 'mineru-api', 'mindmap-web', 'unis-gateway') NEW = ('unis-hub-worker', 'unis-gateway', 'mineru-api', 'airag') def run(*args): value = subprocess.run(args, capture_output=True, text=True, timeout=1800) if value.returncode: # Do not echo Docker's full configuration or application diagnostics. raise RuntimeError('Command failed: ' + args[0] + ' ' + (args[1] if len(args) > 1 else '')) return value.stdout def inspect(name): return json.loads(run('docker', 'inspect', name))[0] def environment(row): return dict(item.split('=', 1) for item in row['Config'].get('Env', []) if '=' in item) def mount(row, destination): matches = [m for m in row['Mounts'] if m['Destination'] == destination] if len(matches) != 1 or matches[0]['Type'] != 'bind': raise ValueError('A single bind mount is required: ' + destination) return safe_path(matches[0]['Source']) def safe_path(value): path = Path(value) if not path.is_absolute() or len(path.parts) < 4 or '..' in path.parts: raise ValueError('Use a dedicated absolute data/release path') if any(p.is_symlink() for p in (path, *path.parents)): raise ValueError('Symlink paths are not supported') return path def overlaps(a, b): return a == b or a in b.parents or b in a.parents def require_internal_services(env): if env.get('MAXKB_DB_HOST') != '127.0.0.1' or env.get('MAXKB_REDIS_HOST') != '127.0.0.1': raise ValueError('External DB/Redis requires the manual upgrade procedure') if str(env.get('MAXKB_DB_PORT', '5432')) != '5432' or str(env.get('MAXKB_REDIS_PORT', '6379')) != '6379': raise ValueError('Expected the internal PostgreSQL/Redis ports 5432/6379') if any(value for key, value in env.items() if key.startswith('MAXKB_REDIS_SENTINEL_')): raise ValueError('Redis Sentinel requires the manual upgrade procedure') def inventory(): rows = {name: inspect(name) for name in LEGACY} kb = rows['airag'] env = environment(kb) if env.get('MAXKB_CONFIG_TYPE') != 'ENV': raise ValueError('Automatic upgrade supports the legacy ENV deployment only') require_internal_services(env) data = mount(kb, '/opt/maxkb') models = mount(rows['mineru-api'], '/models') pgdata = Path(env.get('PGDATA', '')) if not pgdata.is_relative_to('/opt/maxkb'): raise ValueError('PostgreSQL must reside inside the /opt/maxkb bind mount') # A child volume could mask the real database. The legacy unused anonymous # /var/lib/postgresql/data volume is harmless because PGDATA is elsewhere. for m in kb['Mounts']: destination = Path(m['Destination']) if destination != Path('/opt/maxkb') and overlaps(destination, pgdata): raise ValueError('PostgreSQL has an additional mount; manual backup is required') if destination.is_relative_to('/opt/maxkb') and destination != Path('/opt/maxkb'): raise ValueError('Nested persistent mounts require manual backup') if destination.is_relative_to('/opt/maxkb-app'): raise ValueError('Application code mounts must be reviewed manually') version = data / pgdata.relative_to('/opt/maxkb') / 'PG_VERSION' if not version.is_file() or version.read_text().strip() != '17': raise ValueError('Expected the legacy PostgreSQL 17 data; do not initialize a new database') if not (models / 'pipeline').is_dir() or not (models / 'vlm').is_dir(): raise ValueError('Existing pipeline/vlm model directories are required') return rows, data, models def ensure_stopped(rows): if any(row['State']['Status'] not in ('exited', 'created') for row in rows.values()): raise ValueError('Stop all four legacy containers during the maintenance window first') def ensure_exclusive(data, rows, online=None): ids = run('docker', 'ps', '-q').split() for container in ids: row = inspect(container) for m in row['Mounts']: if overlaps(Path(m['Source']), data): if online and row['Id'] == online['id'] and Path(m['Source']) == Path(online['pgdata']): continue raise ValueError('A running container shares the legacy data directory') # Detect a native PostgreSQL using the same directory as well. for proc in Path('/proc').iterdir(): if not proc.name.isdigit(): continue try: args = (proc / 'cmdline').read_bytes().split(b'\0') except (OSError, PermissionError): continue if args and b'postgres' in args[0] and any(str(data).encode() in arg for arg in args[1:]): raise ValueError('A native PostgreSQL process uses the legacy data directory') def online_database(name, original, rows): if not name: return None row = inspect(name) if not row['State']['Running']: raise ValueError('The online database container must be running') env = environment(row) pgdata = env.get('PGDATA', '/var/lib/postgresql/data') expected = original / Path(environment(rows['airag'])['PGDATA']).relative_to('/opt/maxkb') actual = mount(row, pgdata) if actual != expected or env.get('PG_MAJOR') != '17': raise ValueError('Online backup container must mount exactly the legacy PostgreSQL 17 directory') return {'name': name, 'id': row['Id'], 'pgdata': str(expected)} def backup_online(online, archive, rows): row = inspect(online['id']) env = environment(row) client_env = os.environ.copy() client_env['PGPASSWORD'] = env.get('POSTGRES_PASSWORD') or environment(rows['airag']).get('MAXKB_DB_PASSWORD', '') user = env.get('POSTGRES_USER') or environment(rows['airag']).get('MAXKB_DB_USER', '') # Password is inherited by docker exec through the client environment, never # placed in argv, a script or a credential file. PostgreSQL supplies a consistent # physical base backup including WAL while the source service stays online. with archive.open('xb') as stream: result = subprocess.run(['docker', 'exec', '-e', 'PGPASSWORD', online['id'], 'pg_basebackup', '-U', user, '-w', '-D', '-', '-Ft', '-X', 'fetch', '--checkpoint=fast'], env=client_env, stdout=stream, stderr=subprocess.PIPE, timeout=1800) if result.returncode: raise RuntimeError('Online pg_basebackup failed; no source roles or permissions were changed') def published(row, port): values = row['HostConfig'].get('PortBindings', {}).get(str(port) + '/tcp') or [] if len(values) != 1: raise ValueError('Exactly one legacy public port binding is required') value = values[0] return {'target': port, 'published': str(value['HostPort']), 'host_ip': value.get('HostIp') or '0.0.0.0', 'protocol': 'tcp'} def make_compose(rows, images, data, models, flavor, project): old = environment(rows['airag']) require_internal_services(old) env = {k: v for k, v in old.items() if k.startswith(('MAXKB_', 'POSTGRES_', 'REDIS_')) or k in ('PGDATA', 'PIP_TARGET')} env.pop('MAXKB_VERSION', None) env.update(MAXKB_MINDMAP_URL='/admin/mindmap', AI_HUB_ASSET_DIR='/opt/maxkb/data/ai-hub-assets') env.update(MAXKB_DB_HOST='127.0.0.1', MAXKB_DB_PORT='5432', MAXKB_REDIS_HOST='127.0.0.1', MAXKB_REDIS_PORT='6379') volumes = [{'type': 'bind', 'source': str(data), 'target': '/opt/maxkb', 'bind': {'create_host_path': False}}] health = {'test': ['CMD', 'curl', '-fsS', 'http://127.0.0.1:8080/admin/'], 'interval': '5s', 'timeout': '5s', 'retries': 60, 'start_period': '30s'} parser_env = {k: v for k, v in environment(rows['mineru-api']).items() if k.startswith('MINERU_')} parser_env.update(MINERU_MODEL_DIR='/models', MINERU_MODEL_SOURCE='local', MINERU_PIPELINE_MODEL_DIR='/models/pipeline', MINERU_VLM_MODEL_DIR='/models/vlm', MINERU_API_MAX_CONCURRENT_REQUESTS='1') parser_env.pop('MINERU_TOOLS_CONFIG_JSON', None) parser_env['CUDA_VISIBLE_DEVICES'] = environment(rows['mineru-api']).get('CUDA_VISIBLE_DEVICES', '0') parser = {'image': images['parser'], 'container_name': 'mineru-api', 'restart': 'unless-stopped', 'ipc': 'host', 'environment': parser_env, 'volumes': [ {'type': 'bind', 'source': str(models), 'target': '/models', 'read_only': True, 'bind': {'create_host_path': False}}, {'type': 'bind', 'source': str(data / 'mineru-output'), 'target': '/app/output', 'bind': {'create_host_path': False}}], 'healthcheck': {'test': ['CMD', 'python3', '-c', 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=3)'], 'interval': '5s', 'timeout': '5s', 'retries': 60}} if flavor == 'cuda': parser['runtime'] = 'nvidia' parser['deploy'] = {'resources': {'reservations': {'devices': [{'driver': 'nvidia', 'count': 'all', 'capabilities': ['gpu']}]}}} else: # Preserve the established CoreX device exposure; never select NVIDIA runtime. parser['privileged'] = True parser['environment']['PYTHONPATH'] = '/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages' services = { 'airag': {'image': images['kb'], 'container_name': 'airag', 'restart': 'unless-stopped', 'command': ['/usr/bin/start-all.sh'], 'environment': env, 'volumes': volumes, 'healthcheck': health}, 'mineru-api': parser, 'hub-worker': {'image': images['kb'], 'container_name': 'unis-hub-worker', 'restart': 'unless-stopped', 'entrypoint': ['python', 'scripts/ai_hub_ops.py'], 'command': ['worker', '--concurrency', '1'], 'network_mode': 'service:airag', 'environment': env, 'volumes': volumes, 'depends_on': {'airag': {'condition': 'service_healthy'}}, 'stop_grace_period': '330s'}, 'gateway': {'image': images['gateway'], 'container_name': 'unis-gateway', 'restart': 'unless-stopped', 'environment': {'KB_UPSTREAM': 'http://airag:8080'}, 'ports': [published(rows['unis-gateway'], 8989), published(rows['mindmap-web'], 9898)], 'depends_on': {'airag': {'condition': 'service_healthy'}, 'mineru-api': {'condition': 'service_healthy'}}}, } return {'name': project, 'services': services, 'networks': {'default': {'name': project}}} def escape_compose(value): # Compose interpolates dollar signs even in a JSON file. Preserve legacy secrets literally. if isinstance(value, str): return value.replace('$', '$$') if isinstance(value, list): return [escape_compose(v) for v in value] if isinstance(value, dict): return {k: escape_compose(v) for k, v in value.items()} return value def save(path, value): temp = path.with_suffix(path.suffix + '.tmp') with temp.open('w', encoding='utf-8') as stream: json.dump(value, stream, ensure_ascii=False, indent=2) stream.write('\n') temp.chmod(0o600) temp.replace(path) def digest(path): h = hashlib.sha256() with path.open('rb') as stream: for block in iter(lambda: stream.read(1024 * 1024), b''): h.update(block) return h.hexdigest() def tree_signature(root, exclude=None): """Detect writes/restarts between the cold snapshot and cutover, without reading DB contents.""" h = hashlib.sha256() for path in sorted([root, *root.rglob('*')]): if exclude and (path == exclude or exclude in path.parents): continue info = path.lstat() value = (str(path.relative_to(root)), info.st_mode, info.st_size, info.st_mtime_ns, info.st_uid, info.st_gid, os.readlink(path) if path.is_symlink() else '') h.update(json.dumps(value).encode()) return h.hexdigest() def compose(release, *args): return run('docker', 'compose', '-f', str(release / 'compose.json'), *args) def prepare(args): rows, original, models = inventory() ensure_stopped(rows) online = online_database(args.online_db_container, original, rows) ensure_exclusive(original, rows, online) release, data = safe_path(args.release_dir), safe_path(args.data_dir) paths = (original, models, release, data) if any(overlaps(a, b) for i, a in enumerate(paths) for b in paths[i + 1:]): raise ValueError('Original data, models, release and cloned data directories must be separate') if release.exists() or data.exists(): raise ValueError('Release and cloned data directories must be new') images = {} for key, name in (('kb', args.kb_image), ('parser', args.parser_image), ('gateway', args.gateway_image)): row = json.loads(run('docker', 'image', 'inspect', name))[0] labels = row['Config'].get('Labels') or {} if labels.get('io.unis.layout') != 'refact' or labels.get('io.unis.component') != key: raise ValueError('Use candidate images built with build-release.sh') if key == 'parser' and labels.get('io.unis.gpu') != args.gpu: raise ValueError('Parser image GPU flavor does not match --gpu') images[key] = row['Id'] run('docker', 'compose', 'version') # Require space for both an independent archive and the live clone. size = int(run('du', '-sk', str(original)).split()[0]) * 1024 for target in (release.parent, data.parent): if not target.is_dir() or shutil.disk_usage(target).free < size * 2 + 1024**3: raise ValueError('Create parent directories and reserve space for archive + data clone + 1 GiB') release.mkdir(mode=0o700) project = 'unis-refact-' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') state = {'phase': 'preparing', 'project': project, 'original_data': str(original), 'data': str(data), 'flavor': args.gpu, 'old': rows, 'renamed': [], 'new_ids': {}, 'images': images, 'online_db': online} exclude = Path(online['pgdata']) if online else None state['original_signature'] = tree_signature(original, exclude) save(release / 'state.json', state) excludes = ['--exclude=./' + str(exclude.relative_to(original))] if exclude else [] run('tar', '--numeric-owner', '--acls', '--xattrs', *excludes, '-cpf', str(release / 'data.tar'), '-C', str(original), '.') if online: backup_online(online, release / 'database.tar', rows) state['database_sha256'] = digest(release / 'database.tar') ensure_stopped({name: inspect(name) for name in LEGACY}) if tree_signature(original, exclude) != state['original_signature']: raise ValueError('Original data changed during backup; discard this incomplete candidate') state['backup_sha256'] = digest(release / 'data.tar') data.mkdir(mode=0o700) run('tar', '--same-owner', '--acls', '--xattrs', '-xpf', str(release / 'data.tar'), '-C', str(data)) if online: pgclone = data / exclude.relative_to(original) pgclone.mkdir(parents=True, mode=0o700) run('tar', '--same-owner', '-xpf', str(release / 'database.tar'), '-C', str(pgclone)) info = exclude.stat() os.chown(pgclone, info.st_uid, info.st_gid) pgclone.chmod(0o700) (data / 'mineru-output').mkdir(exist_ok=True) state['clone_signature'] = tree_signature(data) config = make_compose(rows, images, data, models, args.gpu, project) save(release / 'compose.json', escape_compose(config)) state['compose_sha256'] = digest(release / 'compose.json') compose(release, 'config', '--quiet') state['phase'] = 'prepared' save(release / 'state.json', state) print('Prepared: original data untouched; cloned data and private rollback material are ready.') def load_release(release): state = json.loads((release / 'state.json').read_text(encoding='utf-8')) if digest(release / 'compose.json') != state['compose_sha256']: raise ValueError('Compose configuration changed after preparation') if digest(release / 'data.tar') != state['backup_sha256']: raise ValueError('Backup checksum mismatch') if state.get('online_db') and digest(release / 'database.tar') != state['database_sha256']: raise ValueError('Online database backup checksum mismatch') return state def apply(release): state = load_release(release) if state['phase'] != 'prepared': raise ValueError('Apply requires a prepared release; inspect failure state before retrying') current, original, _ = inventory() ensure_stopped(current) online = state.get('online_db') if online: actual = online_database(online['name'], original, current) if actual != online: raise ValueError('Online database container changed after preparation') ensure_exclusive(original, current, online) clone = safe_path(state['data']) ensure_exclusive(clone, current) exclude = Path(online['pgdata']) if online else None if tree_signature(original, exclude) != state['original_signature'] or tree_signature(clone) != state['clone_signature']: raise ValueError('Original or cloned data changed after preparation; prepare a fresh release') for name in LEGACY: if current[name]['Id'] != state['old'][name]['Id']: raise ValueError('Legacy containers changed after preparation') if any(current[name]['State'].get(key) != state['old'][name]['State'].get(key) for key in ('StartedAt', 'FinishedAt')): raise ValueError('Legacy containers ran after preparation; create a fresh data clone') for row, port in ((current['unis-gateway'], 8989), (current['mindmap-web'], 9898)): binding = published(row, port) family = socket.AF_INET6 if ':' in binding['host_ip'] else socket.AF_INET with socket.socket(family) as sock: sock.bind((binding['host_ip'], int(binding['published']))) # No model is loaded in this smoke test. A driver/runtime mismatch fails before switchover. compose(release, 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'python3', 'mineru-api', '-c', 'import torch, vllm; from mineru.version import __version__; assert __version__ == "2.7.6"; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))') state['phase'] = 'switching' save(release / 'state.json', state) try: for name in LEGACY: backup_name = name + '-before-' + state['project'] # Save the intention before rename, so interrupted operations can be recovered. state['renamed'].append(name) save(release / 'state.json', state) run('docker', 'rename', name, backup_name) # start-all.sh runs Django migrate only in the new cloned PostgreSQL. state['phase'] = 'migrating-clone' save(release / 'state.json', state) compose(release, 'up', '-d', '--pull', 'never', '--wait', '--wait-timeout', '360') for name in NEW: row = inspect(name) if row['Config'].get('Labels', {}).get('com.docker.compose.project') != state['project']: raise ValueError('Unexpected container identity') state['new_ids'][name] = row['Id'] run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check') # Migration 0008 creates the tables, but the one-time importer is a # separate command. It imports legacy ADMIN users only on first creation # of the unified admin role and never restores later demotions. run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'unify_platform_roles') checks = run('docker', 'exec', 'airag', 'python', '/usr/local/bin/unis-check-internal-services.py') save(release / 'internal-services.json', json.loads(checks.strip().splitlines()[-1])) state['phase'] = 'started-awaiting-acceptance' save(release / 'state.json', state) print('New containers are healthy. Complete the logged-in acceptance checklist before opening traffic.') except Exception: state['phase'] = 'failed' save(release / 'state.json', state) # Freeze failure data; retain all old containers and original data. try: compose(release, 'stop', '--timeout', '330') except Exception: pass raise def rollback(release, start_old=False): state = load_release(release) if state['phase'] not in ('switching', 'migrating-clone', 'failed', 'started-awaiting-acceptance'): raise ValueError('No switched release to roll back') if start_old: ensure_exclusive(safe_path(state['original_data']), state['old']) # Only act on containers belonging to this exact generated project. names = run('docker', 'ps', '-a', '--format', '{{.Names}}').splitlines() for name in NEW: if name not in names: continue row = inspect(name) if row['Config'].get('Labels', {}).get('com.docker.compose.project') != state['project']: if row['Id'] == state['old'].get(name, {}).get('Id'): continue raise ValueError('Refusing to stop an unrelated container') run('docker', 'stop', '--time', '330', name) run('docker', 'rename', name, name + '-failed-' + state['project']) for name in state['renamed']: old_id = state['old'][name]['Id'] row = inspect(old_id) if row['State']['Running']: raise ValueError('An original container was started outside this release') if row['Name'].lstrip('/') != name: run('docker', 'rename', old_id, name) if start_old: run('docker', 'start', *LEGACY) state['phase'] = 'rolled-back' save(release / 'state.json', state) print('Original names restored. Original data unchanged; failed clone retained. New-version writes are not merged back.') @contextmanager def lock(): import fcntl flags = os.O_CREAT | os.O_WRONLY | os.O_NOFOLLOW fd = os.open('/tmp/unis-refact-upgrade.lock', flags, 0o600) try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) yield finally: os.close(fd) def main(): parser = argparse.ArgumentParser(description=__doc__) actions = parser.add_subparsers(dest='action', required=True) actions.add_parser('inspect') p = actions.add_parser('prepare') for name in ('release-dir', 'data-dir', 'kb-image', 'parser-image', 'gateway-image'): p.add_argument('--' + name, required=True) p.add_argument('--gpu', choices=['cuda', 'corex'], required=True) p.add_argument('--online-db-container', help='Explicit PostgreSQL 17 container sharing the legacy PGDATA; use a consistent online base backup') for name in ('apply', 'rollback'): p = actions.add_parser(name) p.add_argument('--release-dir', required=True) if name == 'rollback': p.add_argument('--start-old', action='store_true', help='Start old containers using the pre-upgrade data; new-version writes are not included') args = parser.parse_args() if os.name != 'posix' or not Path('/proc').is_dir(): raise ValueError('Run on the Linux deployment host') os.umask(0o077) with lock(): if args.action == 'inspect': rows, data, models = inventory() cold, reason = True, '' try: ensure_exclusive(data, rows) except ValueError as exc: cold, reason = False, str(exc) print(json.dumps({'containers': {k: {'status': v['State']['Status'], 'image_id': v['Image']} for k, v in rows.items()}, 'data': str(data), 'models': str(models), 'cold_backup_eligible': cold, 'reason': reason}, indent=2)) elif args.action == 'prepare': prepare(args) elif args.action == 'apply': apply(safe_path(args.release_dir)) else: rollback(safe_path(args.release_dir), args.start_old) if __name__ == '__main__': try: main() except (OSError, ValueError, RuntimeError, subprocess.SubprocessError, KeyError, json.JSONDecodeError) as exc: print('Upgrade stopped: ' + str(exc) if isinstance(exc, (ValueError, RuntimeError)) else 'Upgrade stopped: ' + type(exc).__name__, file=sys.stderr) sys.exit(1)