477 lines
24 KiB
Python
477 lines
24 KiB
Python
"""Offline, consistent-clone upgrade of the legacy four-container Unis deployment.
|
|
|
|
Python 3.10+, Linux, Docker Compose v2. inspect is read-only. prepare never starts
|
|
containers. apply only migrates the cloned data; rollback preserves that clone.
|
|
Release state contains private container configuration: keep the directory 0700.
|
|
"""
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
|
|
LEGACY = ('airag', 'mineru-api', 'mindmap-web', 'unis-gateway')
|
|
NEW = ('unis-hub-worker', 'unis-gateway', 'mineru-api', 'airag')
|
|
|
|
|
|
def run(*args):
|
|
value = subprocess.run(args, capture_output=True, text=True, timeout=1800)
|
|
if value.returncode:
|
|
# Do not echo Docker's full configuration or application diagnostics.
|
|
raise RuntimeError('Command failed: ' + args[0] + ' ' + (args[1] if len(args) > 1 else ''))
|
|
return value.stdout
|
|
|
|
|
|
def inspect(name):
|
|
return json.loads(run('docker', 'inspect', name))[0]
|
|
|
|
|
|
def environment(row):
|
|
return dict(item.split('=', 1) for item in row['Config'].get('Env', []) if '=' in item)
|
|
|
|
|
|
def mount(row, destination):
|
|
matches = [m for m in row['Mounts'] if m['Destination'] == destination]
|
|
if len(matches) != 1 or matches[0]['Type'] != 'bind':
|
|
raise ValueError('A single bind mount is required: ' + destination)
|
|
return safe_path(matches[0]['Source'])
|
|
|
|
|
|
def safe_path(value):
|
|
path = Path(value)
|
|
if not path.is_absolute() or len(path.parts) < 4 or '..' in path.parts:
|
|
raise ValueError('Use a dedicated absolute data/release path')
|
|
if any(p.is_symlink() for p in (path, *path.parents)):
|
|
raise ValueError('Symlink paths are not supported')
|
|
return path
|
|
|
|
|
|
def overlaps(a, b):
|
|
return a == b or a in b.parents or b in a.parents
|
|
|
|
|
|
def require_internal_services(env):
|
|
if env.get('MAXKB_DB_HOST') != '127.0.0.1' or env.get('MAXKB_REDIS_HOST') != '127.0.0.1':
|
|
raise ValueError('External DB/Redis requires the manual upgrade procedure')
|
|
if str(env.get('MAXKB_DB_PORT', '5432')) != '5432' or str(env.get('MAXKB_REDIS_PORT', '6379')) != '6379':
|
|
raise ValueError('Expected the internal PostgreSQL/Redis ports 5432/6379')
|
|
if any(value for key, value in env.items() if key.startswith('MAXKB_REDIS_SENTINEL_')):
|
|
raise ValueError('Redis Sentinel requires the manual upgrade procedure')
|
|
|
|
|
|
def inventory():
|
|
rows = {name: inspect(name) for name in LEGACY}
|
|
kb = rows['airag']
|
|
env = environment(kb)
|
|
if env.get('MAXKB_CONFIG_TYPE') != 'ENV':
|
|
raise ValueError('Automatic upgrade supports the legacy ENV deployment only')
|
|
require_internal_services(env)
|
|
data = mount(kb, '/opt/maxkb')
|
|
models = mount(rows['mineru-api'], '/models')
|
|
pgdata = Path(env.get('PGDATA', ''))
|
|
if not pgdata.is_relative_to('/opt/maxkb'):
|
|
raise ValueError('PostgreSQL must reside inside the /opt/maxkb bind mount')
|
|
# A child volume could mask the real database. The legacy unused anonymous
|
|
# /var/lib/postgresql/data volume is harmless because PGDATA is elsewhere.
|
|
for m in kb['Mounts']:
|
|
destination = Path(m['Destination'])
|
|
if destination != Path('/opt/maxkb') and overlaps(destination, pgdata):
|
|
raise ValueError('PostgreSQL has an additional mount; manual backup is required')
|
|
if destination.is_relative_to('/opt/maxkb') and destination != Path('/opt/maxkb'):
|
|
raise ValueError('Nested persistent mounts require manual backup')
|
|
if destination.is_relative_to('/opt/maxkb-app'):
|
|
raise ValueError('Application code mounts must be reviewed manually')
|
|
version = data / pgdata.relative_to('/opt/maxkb') / 'PG_VERSION'
|
|
if not version.is_file() or version.read_text().strip() != '17':
|
|
raise ValueError('Expected the legacy PostgreSQL 17 data; do not initialize a new database')
|
|
if not (models / 'pipeline').is_dir() or not (models / 'vlm').is_dir():
|
|
raise ValueError('Existing pipeline/vlm model directories are required')
|
|
return rows, data, models
|
|
|
|
|
|
def ensure_stopped(rows):
|
|
if any(row['State']['Status'] not in ('exited', 'created') for row in rows.values()):
|
|
raise ValueError('Stop all four legacy containers during the maintenance window first')
|
|
|
|
|
|
def ensure_exclusive(data, rows, online=None):
|
|
ids = run('docker', 'ps', '-q').split()
|
|
for container in ids:
|
|
row = inspect(container)
|
|
for m in row['Mounts']:
|
|
if overlaps(Path(m['Source']), data):
|
|
if online and row['Id'] == online['id'] and Path(m['Source']) == Path(online['pgdata']):
|
|
continue
|
|
raise ValueError('A running container shares the legacy data directory')
|
|
# Detect a native PostgreSQL using the same directory as well.
|
|
for proc in Path('/proc').iterdir():
|
|
if not proc.name.isdigit():
|
|
continue
|
|
try:
|
|
args = (proc / 'cmdline').read_bytes().split(b'\0')
|
|
except (OSError, PermissionError):
|
|
continue
|
|
if args and b'postgres' in args[0] and any(str(data).encode() in arg for arg in args[1:]):
|
|
raise ValueError('A native PostgreSQL process uses the legacy data directory')
|
|
|
|
|
|
def online_database(name, original, rows):
|
|
if not name:
|
|
return None
|
|
row = inspect(name)
|
|
if not row['State']['Running']:
|
|
raise ValueError('The online database container must be running')
|
|
env = environment(row)
|
|
pgdata = env.get('PGDATA', '/var/lib/postgresql/data')
|
|
expected = original / Path(environment(rows['airag'])['PGDATA']).relative_to('/opt/maxkb')
|
|
actual = mount(row, pgdata)
|
|
if actual != expected or env.get('PG_MAJOR') != '17':
|
|
raise ValueError('Online backup container must mount exactly the legacy PostgreSQL 17 directory')
|
|
return {'name': name, 'id': row['Id'], 'pgdata': str(expected)}
|
|
|
|
|
|
def backup_online(online, archive, rows):
|
|
row = inspect(online['id'])
|
|
env = environment(row)
|
|
client_env = os.environ.copy()
|
|
client_env['PGPASSWORD'] = env.get('POSTGRES_PASSWORD') or environment(rows['airag']).get('MAXKB_DB_PASSWORD', '')
|
|
user = env.get('POSTGRES_USER') or environment(rows['airag']).get('MAXKB_DB_USER', '')
|
|
# Password is inherited by docker exec through the client environment, never
|
|
# placed in argv, a script or a credential file. PostgreSQL supplies a consistent
|
|
# physical base backup including WAL while the source service stays online.
|
|
with archive.open('xb') as stream:
|
|
result = subprocess.run(['docker', 'exec', '-e', 'PGPASSWORD', online['id'], 'pg_basebackup',
|
|
'-U', user, '-w', '-D', '-', '-Ft', '-X', 'fetch', '--checkpoint=fast'],
|
|
env=client_env, stdout=stream, stderr=subprocess.PIPE, timeout=1800)
|
|
if result.returncode:
|
|
raise RuntimeError('Online pg_basebackup failed; no source roles or permissions were changed')
|
|
|
|
|
|
def published(row, port):
|
|
values = row['HostConfig'].get('PortBindings', {}).get(str(port) + '/tcp') or []
|
|
if len(values) != 1:
|
|
raise ValueError('Exactly one legacy public port binding is required')
|
|
value = values[0]
|
|
return {'target': port, 'published': str(value['HostPort']), 'host_ip': value.get('HostIp') or '0.0.0.0', 'protocol': 'tcp'}
|
|
|
|
|
|
def make_compose(rows, images, data, models, flavor, project):
|
|
old = environment(rows['airag'])
|
|
require_internal_services(old)
|
|
env = {k: v for k, v in old.items() if k.startswith(('MAXKB_', 'POSTGRES_', 'REDIS_')) or k in ('PGDATA', 'PIP_TARGET')}
|
|
env.pop('MAXKB_VERSION', None)
|
|
env.update(MAXKB_MINDMAP_URL='/admin/mindmap', AI_HUB_ASSET_DIR='/opt/maxkb/data/ai-hub-assets')
|
|
env.update(MAXKB_DB_HOST='127.0.0.1', MAXKB_DB_PORT='5432', MAXKB_REDIS_HOST='127.0.0.1', MAXKB_REDIS_PORT='6379')
|
|
volumes = [{'type': 'bind', 'source': str(data), 'target': '/opt/maxkb', 'bind': {'create_host_path': False}}]
|
|
health = {'test': ['CMD', 'curl', '-fsS', 'http://127.0.0.1:8080/admin/'], 'interval': '5s', 'timeout': '5s', 'retries': 60, 'start_period': '30s'}
|
|
parser_env = {k: v for k, v in environment(rows['mineru-api']).items() if k.startswith('MINERU_')}
|
|
parser_env.update(MINERU_MODEL_DIR='/models', MINERU_MODEL_SOURCE='local', MINERU_PIPELINE_MODEL_DIR='/models/pipeline', MINERU_VLM_MODEL_DIR='/models/vlm', MINERU_API_MAX_CONCURRENT_REQUESTS='1')
|
|
parser_env.pop('MINERU_TOOLS_CONFIG_JSON', None)
|
|
parser_env['CUDA_VISIBLE_DEVICES'] = environment(rows['mineru-api']).get('CUDA_VISIBLE_DEVICES', '0')
|
|
parser = {'image': images['parser'], 'container_name': 'mineru-api', 'restart': 'unless-stopped', 'ipc': 'host',
|
|
'environment': parser_env, 'volumes': [
|
|
{'type': 'bind', 'source': str(models), 'target': '/models', 'read_only': True, 'bind': {'create_host_path': False}},
|
|
{'type': 'bind', 'source': str(data / 'mineru-output'), 'target': '/app/output', 'bind': {'create_host_path': False}}],
|
|
'healthcheck': {'test': ['CMD', 'python3', '-c', 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8000/health", timeout=3)'], 'interval': '5s', 'timeout': '5s', 'retries': 60}}
|
|
if flavor == 'cuda':
|
|
parser['runtime'] = 'nvidia'
|
|
parser['deploy'] = {'resources': {'reservations': {'devices': [{'driver': 'nvidia', 'count': 'all', 'capabilities': ['gpu']}]}}}
|
|
else:
|
|
# Preserve the established CoreX device exposure; never select NVIDIA runtime.
|
|
parser['privileged'] = True
|
|
parser['environment']['PYTHONPATH'] = '/app:/usr/local/corex-4.4.0/lib64/python3/dist-packages'
|
|
services = {
|
|
'airag': {'image': images['kb'], 'container_name': 'airag', 'restart': 'unless-stopped',
|
|
'command': ['/usr/bin/start-all.sh'], 'environment': env, 'volumes': volumes, 'healthcheck': health},
|
|
'mineru-api': parser,
|
|
'hub-worker': {'image': images['kb'], 'container_name': 'unis-hub-worker', 'restart': 'unless-stopped',
|
|
'entrypoint': ['python', 'scripts/ai_hub_ops.py'], 'command': ['worker', '--concurrency', '1'],
|
|
'network_mode': 'service:airag', 'environment': env, 'volumes': volumes,
|
|
'depends_on': {'airag': {'condition': 'service_healthy'}}, 'stop_grace_period': '330s'},
|
|
'gateway': {'image': images['gateway'], 'container_name': 'unis-gateway', 'restart': 'unless-stopped',
|
|
'environment': {'KB_UPSTREAM': 'http://airag:8080'},
|
|
'ports': [published(rows['unis-gateway'], 8989), published(rows['mindmap-web'], 9898)],
|
|
'depends_on': {'airag': {'condition': 'service_healthy'}, 'mineru-api': {'condition': 'service_healthy'}}},
|
|
}
|
|
return {'name': project, 'services': services, 'networks': {'default': {'name': project}}}
|
|
|
|
|
|
def escape_compose(value):
|
|
# Compose interpolates dollar signs even in a JSON file. Preserve legacy secrets literally.
|
|
if isinstance(value, str):
|
|
return value.replace('$', '$$')
|
|
if isinstance(value, list):
|
|
return [escape_compose(v) for v in value]
|
|
if isinstance(value, dict):
|
|
return {k: escape_compose(v) for k, v in value.items()}
|
|
return value
|
|
|
|
|
|
def save(path, value):
|
|
temp = path.with_suffix(path.suffix + '.tmp')
|
|
with temp.open('w', encoding='utf-8') as stream:
|
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
|
stream.write('\n')
|
|
temp.chmod(0o600)
|
|
temp.replace(path)
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with path.open('rb') as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b''):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def tree_signature(root, exclude=None):
|
|
"""Detect writes/restarts between the cold snapshot and cutover, without reading DB contents."""
|
|
h = hashlib.sha256()
|
|
for path in sorted([root, *root.rglob('*')]):
|
|
if exclude and (path == exclude or exclude in path.parents):
|
|
continue
|
|
info = path.lstat()
|
|
value = (str(path.relative_to(root)), info.st_mode, info.st_size, info.st_mtime_ns,
|
|
info.st_uid, info.st_gid, os.readlink(path) if path.is_symlink() else '')
|
|
h.update(json.dumps(value).encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def compose(release, *args):
|
|
return run('docker', 'compose', '-f', str(release / 'compose.json'), *args)
|
|
|
|
|
|
def prepare(args):
|
|
rows, original, models = inventory()
|
|
ensure_stopped(rows)
|
|
online = online_database(args.online_db_container, original, rows)
|
|
ensure_exclusive(original, rows, online)
|
|
release, data = safe_path(args.release_dir), safe_path(args.data_dir)
|
|
paths = (original, models, release, data)
|
|
if any(overlaps(a, b) for i, a in enumerate(paths) for b in paths[i + 1:]):
|
|
raise ValueError('Original data, models, release and cloned data directories must be separate')
|
|
if release.exists() or data.exists():
|
|
raise ValueError('Release and cloned data directories must be new')
|
|
images = {}
|
|
for key, name in (('kb', args.kb_image), ('parser', args.parser_image), ('gateway', args.gateway_image)):
|
|
row = json.loads(run('docker', 'image', 'inspect', name))[0]
|
|
labels = row['Config'].get('Labels') or {}
|
|
if labels.get('io.unis.layout') != 'refact' or labels.get('io.unis.component') != key:
|
|
raise ValueError('Use candidate images built with build-release.sh')
|
|
if key == 'parser' and labels.get('io.unis.gpu') != args.gpu:
|
|
raise ValueError('Parser image GPU flavor does not match --gpu')
|
|
images[key] = row['Id']
|
|
run('docker', 'compose', 'version')
|
|
# Require space for both an independent archive and the live clone.
|
|
size = int(run('du', '-sk', str(original)).split()[0]) * 1024
|
|
for target in (release.parent, data.parent):
|
|
if not target.is_dir() or shutil.disk_usage(target).free < size * 2 + 1024**3:
|
|
raise ValueError('Create parent directories and reserve space for archive + data clone + 1 GiB')
|
|
release.mkdir(mode=0o700)
|
|
project = 'unis-refact-' + datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')
|
|
state = {'phase': 'preparing', 'project': project, 'original_data': str(original), 'data': str(data),
|
|
'flavor': args.gpu, 'old': rows, 'renamed': [], 'new_ids': {}, 'images': images, 'online_db': online}
|
|
exclude = Path(online['pgdata']) if online else None
|
|
state['original_signature'] = tree_signature(original, exclude)
|
|
save(release / 'state.json', state)
|
|
excludes = ['--exclude=./' + str(exclude.relative_to(original))] if exclude else []
|
|
run('tar', '--numeric-owner', '--acls', '--xattrs', *excludes, '-cpf', str(release / 'data.tar'), '-C', str(original), '.')
|
|
if online:
|
|
backup_online(online, release / 'database.tar', rows)
|
|
state['database_sha256'] = digest(release / 'database.tar')
|
|
ensure_stopped({name: inspect(name) for name in LEGACY})
|
|
if tree_signature(original, exclude) != state['original_signature']:
|
|
raise ValueError('Original data changed during backup; discard this incomplete candidate')
|
|
state['backup_sha256'] = digest(release / 'data.tar')
|
|
data.mkdir(mode=0o700)
|
|
run('tar', '--same-owner', '--acls', '--xattrs', '-xpf', str(release / 'data.tar'), '-C', str(data))
|
|
if online:
|
|
pgclone = data / exclude.relative_to(original)
|
|
pgclone.mkdir(parents=True, mode=0o700)
|
|
run('tar', '--same-owner', '-xpf', str(release / 'database.tar'), '-C', str(pgclone))
|
|
info = exclude.stat()
|
|
os.chown(pgclone, info.st_uid, info.st_gid)
|
|
pgclone.chmod(0o700)
|
|
(data / 'mineru-output').mkdir(exist_ok=True)
|
|
state['clone_signature'] = tree_signature(data)
|
|
config = make_compose(rows, images, data, models, args.gpu, project)
|
|
save(release / 'compose.json', escape_compose(config))
|
|
state['compose_sha256'] = digest(release / 'compose.json')
|
|
compose(release, 'config', '--quiet')
|
|
state['phase'] = 'prepared'
|
|
save(release / 'state.json', state)
|
|
print('Prepared: original data untouched; cloned data and private rollback material are ready.')
|
|
|
|
|
|
def load_release(release):
|
|
state = json.loads((release / 'state.json').read_text(encoding='utf-8'))
|
|
if digest(release / 'compose.json') != state['compose_sha256']:
|
|
raise ValueError('Compose configuration changed after preparation')
|
|
if digest(release / 'data.tar') != state['backup_sha256']:
|
|
raise ValueError('Backup checksum mismatch')
|
|
if state.get('online_db') and digest(release / 'database.tar') != state['database_sha256']:
|
|
raise ValueError('Online database backup checksum mismatch')
|
|
return state
|
|
|
|
|
|
def apply(release):
|
|
state = load_release(release)
|
|
if state['phase'] != 'prepared':
|
|
raise ValueError('Apply requires a prepared release; inspect failure state before retrying')
|
|
current, original, _ = inventory()
|
|
ensure_stopped(current)
|
|
online = state.get('online_db')
|
|
if online:
|
|
actual = online_database(online['name'], original, current)
|
|
if actual != online:
|
|
raise ValueError('Online database container changed after preparation')
|
|
ensure_exclusive(original, current, online)
|
|
clone = safe_path(state['data'])
|
|
ensure_exclusive(clone, current)
|
|
exclude = Path(online['pgdata']) if online else None
|
|
if tree_signature(original, exclude) != state['original_signature'] or tree_signature(clone) != state['clone_signature']:
|
|
raise ValueError('Original or cloned data changed after preparation; prepare a fresh release')
|
|
for name in LEGACY:
|
|
if current[name]['Id'] != state['old'][name]['Id']:
|
|
raise ValueError('Legacy containers changed after preparation')
|
|
if any(current[name]['State'].get(key) != state['old'][name]['State'].get(key) for key in ('StartedAt', 'FinishedAt')):
|
|
raise ValueError('Legacy containers ran after preparation; create a fresh data clone')
|
|
for row, port in ((current['unis-gateway'], 8989), (current['mindmap-web'], 9898)):
|
|
binding = published(row, port)
|
|
family = socket.AF_INET6 if ':' in binding['host_ip'] else socket.AF_INET
|
|
with socket.socket(family) as sock:
|
|
sock.bind((binding['host_ip'], int(binding['published'])))
|
|
# No model is loaded in this smoke test. A driver/runtime mismatch fails before switchover.
|
|
compose(release, 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'python3', 'mineru-api', '-c',
|
|
'import torch, vllm; from mineru.version import __version__; assert __version__ == "2.7.6"; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))')
|
|
state['phase'] = 'switching'
|
|
save(release / 'state.json', state)
|
|
try:
|
|
for name in LEGACY:
|
|
backup_name = name + '-before-' + state['project']
|
|
# Save the intention before rename, so interrupted operations can be recovered.
|
|
state['renamed'].append(name)
|
|
save(release / 'state.json', state)
|
|
run('docker', 'rename', name, backup_name)
|
|
# start-all.sh runs Django migrate only in the new cloned PostgreSQL.
|
|
state['phase'] = 'migrating-clone'
|
|
save(release / 'state.json', state)
|
|
compose(release, 'up', '-d', '--pull', 'never', '--wait', '--wait-timeout', '360')
|
|
for name in NEW:
|
|
row = inspect(name)
|
|
if row['Config'].get('Labels', {}).get('com.docker.compose.project') != state['project']:
|
|
raise ValueError('Unexpected container identity')
|
|
state['new_ids'][name] = row['Id']
|
|
run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check')
|
|
# Migration 0008 creates the tables, but the one-time importer is a
|
|
# separate command. It imports legacy ADMIN users only on first creation
|
|
# of the unified admin role and never restores later demotions.
|
|
run('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'unify_platform_roles')
|
|
checks = run('docker', 'exec', 'airag', 'python', '/usr/local/bin/unis-check-internal-services.py')
|
|
save(release / 'internal-services.json', json.loads(checks.strip().splitlines()[-1]))
|
|
state['phase'] = 'started-awaiting-acceptance'
|
|
save(release / 'state.json', state)
|
|
print('New containers are healthy. Complete the logged-in acceptance checklist before opening traffic.')
|
|
except Exception:
|
|
state['phase'] = 'failed'
|
|
save(release / 'state.json', state)
|
|
# Freeze failure data; retain all old containers and original data.
|
|
try:
|
|
compose(release, 'stop', '--timeout', '330')
|
|
except Exception:
|
|
pass
|
|
raise
|
|
|
|
|
|
def rollback(release, start_old=False):
|
|
state = load_release(release)
|
|
if state['phase'] not in ('switching', 'migrating-clone', 'failed', 'started-awaiting-acceptance'):
|
|
raise ValueError('No switched release to roll back')
|
|
if start_old:
|
|
ensure_exclusive(safe_path(state['original_data']), state['old'])
|
|
# Only act on containers belonging to this exact generated project.
|
|
names = run('docker', 'ps', '-a', '--format', '{{.Names}}').splitlines()
|
|
for name in NEW:
|
|
if name not in names:
|
|
continue
|
|
row = inspect(name)
|
|
if row['Config'].get('Labels', {}).get('com.docker.compose.project') != state['project']:
|
|
if row['Id'] == state['old'].get(name, {}).get('Id'):
|
|
continue
|
|
raise ValueError('Refusing to stop an unrelated container')
|
|
run('docker', 'stop', '--time', '330', name)
|
|
run('docker', 'rename', name, name + '-failed-' + state['project'])
|
|
for name in state['renamed']:
|
|
old_id = state['old'][name]['Id']
|
|
row = inspect(old_id)
|
|
if row['State']['Running']:
|
|
raise ValueError('An original container was started outside this release')
|
|
if row['Name'].lstrip('/') != name:
|
|
run('docker', 'rename', old_id, name)
|
|
if start_old:
|
|
run('docker', 'start', *LEGACY)
|
|
state['phase'] = 'rolled-back'
|
|
save(release / 'state.json', state)
|
|
print('Original names restored. Original data unchanged; failed clone retained. New-version writes are not merged back.')
|
|
|
|
|
|
@contextmanager
|
|
def lock():
|
|
import fcntl
|
|
flags = os.O_CREAT | os.O_WRONLY | os.O_NOFOLLOW
|
|
fd = os.open('/tmp/unis-refact-upgrade.lock', flags, 0o600)
|
|
try:
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
yield
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
actions = parser.add_subparsers(dest='action', required=True)
|
|
actions.add_parser('inspect')
|
|
p = actions.add_parser('prepare')
|
|
for name in ('release-dir', 'data-dir', 'kb-image', 'parser-image', 'gateway-image'):
|
|
p.add_argument('--' + name, required=True)
|
|
p.add_argument('--gpu', choices=['cuda', 'corex'], required=True)
|
|
p.add_argument('--online-db-container', help='Explicit PostgreSQL 17 container sharing the legacy PGDATA; use a consistent online base backup')
|
|
for name in ('apply', 'rollback'):
|
|
p = actions.add_parser(name)
|
|
p.add_argument('--release-dir', required=True)
|
|
if name == 'rollback':
|
|
p.add_argument('--start-old', action='store_true', help='Start old containers using the pre-upgrade data; new-version writes are not included')
|
|
args = parser.parse_args()
|
|
if os.name != 'posix' or not Path('/proc').is_dir():
|
|
raise ValueError('Run on the Linux deployment host')
|
|
os.umask(0o077)
|
|
with lock():
|
|
if args.action == 'inspect':
|
|
rows, data, models = inventory()
|
|
cold, reason = True, ''
|
|
try:
|
|
ensure_exclusive(data, rows)
|
|
except ValueError as exc:
|
|
cold, reason = False, str(exc)
|
|
print(json.dumps({'containers': {k: {'status': v['State']['Status'], 'image_id': v['Image']} for k, v in rows.items()}, 'data': str(data), 'models': str(models), 'cold_backup_eligible': cold, 'reason': reason}, indent=2))
|
|
elif args.action == 'prepare':
|
|
prepare(args)
|
|
elif args.action == 'apply':
|
|
apply(safe_path(args.release_dir))
|
|
else:
|
|
rollback(safe_path(args.release_dir), args.start_old)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError, KeyError, json.JSONDecodeError) as exc:
|
|
print('Upgrade stopped: ' + str(exc) if isinstance(exc, (ValueError, RuntimeError)) else 'Upgrade stopped: ' + type(exc).__name__, file=sys.stderr)
|
|
sys.exit(1)
|