UnisKB/scripts/release_review.py

309 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

"""Prepare LOCAL release/rollback material using read-only SSH inspection.
There is deliberately no apply, upload, migration, or service-control command.
Run from the workspace with the existing project Python/SSH libraries.
"""
import argparse
from datetime import datetime, timedelta, timezone
import difflib
import errno
import hashlib
import importlib.util
import json
from pathlib import Path, PurePosixPath
import re
import shlex
import stat
import sys
import zipfile
WORKSPACE = Path(__file__).resolve().parents[2]
PROJECT = WORKSPACE / 'maxkb'
REMOTE_ROOT = '/home/tlw/unis/UnisKB'
HOST = '10.100.52.118'
MAX_FILE = 32 * 1024 * 1024
SOURCES = [
'apps/system_manage/document_export.py', 'apps/system_manage/test_document_export.py',
'apps/system_manage/views/ai_hub.py', 'apps/system_manage/urls.py',
'ui/src/api/ai-hub.ts', 'ui/src/views/mindmap/workbench/api/hub.ts',
'ui/src/views/mindmap/workbench/utils/graphModel.ts',
'ui/src/views/mindmap/workbench/utils/graphLayout.ts',
'ui/src/views/mindmap/workbench/utils/exportFilename.ts',
'ui/src/views/mindmap/workbench/utils/imageAssets.ts',
'ui/src/views/mindmap/workbench/components/MindMapRenderer.vue',
'ui/src/views/mindmap/workbench/components/MarkdownRenderer.vue',
'ui/src/views/mindmap/workbench/views/DocumentProcessor.vue',
'apps/system_manage/access_control.py',
'apps/system_manage/views/access_control_policy.py',
'ui/src/api/system-settings/access-control-policy.ts',
'ui/src/views/system-setting/policy/index.vue',
]
ENTRIES = ['apps/static/admin/index.html', 'ui/dist/admin/index.html']
STATIC_ROOTS = ['apps/static/admin', 'ui/dist/admin']
ASSET_PATTERN = re.compile(r'''(?:src|href)=["']/admin/(assets/[A-Za-z0-9_.-]+)["']''')
# Fixed code only: metadata, hashes, and recognized process entries. No importing
# the application, reading /proc/*/environ, writing files, or opening a DB/cache.
REMOTE_PROBE = r'''
import hashlib,json,os,stat,sys
from pathlib import Path,PurePosixPath
root=Path('/home/tlw/unis/UnisKB')
payload=json.load(sys.stdin)
if len(payload['paths'])>2000:raise ValueError('Too many paths')
files={}
for relative in payload['paths']:
p=PurePosixPath(relative)
if p.is_absolute() or '..' in p.parts or '\\' in relative:raise ValueError('Invalid path')
allowed=relative.startswith(('apps/system_manage/','ui/src/','apps/static/admin/','ui/dist/admin/'))
if not allowed:raise ValueError('Outside release scope')
path=root/relative
try:
for parent in [path,*path.parents]:
if parent==root:break
if parent.is_symlink():raise ValueError('Symlink target refused')
info=path.stat()
if not stat.S_ISREG(info.st_mode) or info.st_size>33554432:raise ValueError('Unexpected file')
h=hashlib.sha256()
with path.open('rb') as stream:
for chunk in iter(lambda:stream.read(1048576),b''):h.update(chunk)
files[relative]={'exists':True,'sha256':h.hexdigest(),'size':info.st_size}
except FileNotFoundError:files[relative]={'exists':False}
processes=[]
for proc in Path('/proc').iterdir():
if not proc.name.isdigit():continue
try:
if (proc/'cwd').resolve()!=root:continue
args=[a for a in (proc/'cmdline').read_bytes().decode().split('\0') if a]
known=next((entry for entry in ('scripts/dev_local.py','scripts/dev_local_model.py','ai_hub_worker') if entry in args),None)
if known:processes.append({'pid':int(proc.name),'entry':known,'executable':args[0]})
except (OSError,UnicodeError):pass
print(json.dumps({'files':files,'processes':processes,'inspection':'read_only'}))
'''
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def safe_relative(value):
path = PurePosixPath(value)
if path.is_absolute() or '..' in path.parts or '\\' in value or ':' in value or not path.parts:
raise ValueError('Unsafe relative path')
return path.as_posix()
def compare(path, before, after):
state = 'added' if before is None else 'same' if before == after else 'modified'
delta = list(difflib.unified_diff(
(before or b'').decode('utf-8-sig').splitlines(),
after.decode('utf-8-sig').splitlines(),
fromfile='remote/' + path, tofile='local/' + path, lineterm=''))
return {'path': path, 'status': state,
'remote_sha256': sha(before) if before is not None else None,
'local_sha256': sha(after), 'size': len(after),
'added_lines': sum(line.startswith('+') and not line.startswith('+++') for line in delta),
'removed_lines': sum(line.startswith('-') and not line.startswith('---') for line in delta),
'line_endings_only': state == 'modified' and not delta}, delta
def read_remote(sftp, relative):
"""All SFTP opens are rb; missing files are distinct from denied reads."""
relative = safe_relative(relative)
for index in range(1, len(PurePosixPath(relative).parts) + 1):
partial = '/'.join(PurePosixPath(relative).parts[:index])
try:
info = sftp.lstat(REMOTE_ROOT + '/' + partial)
except OSError as exc:
if exc.errno == errno.ENOENT:
return None
raise RuntimeError('Remote metadata read failed: ' + relative) from None
if stat.S_ISLNK(info.st_mode):
raise ValueError('Remote symlink refused: ' + relative)
if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_FILE:
raise ValueError('Remote file outside size/type limit: ' + relative)
with sftp.open(REMOTE_ROOT + '/' + relative, 'rb') as stream:
raw = stream.read(MAX_FILE + 1)
if len(raw) > MAX_FILE:
raise ValueError('Remote file too large: ' + relative)
return raw
def local_file(root, relative):
relative = safe_relative(relative)
file = root / relative
if file.is_symlink() or not file.resolve().is_relative_to(root.resolve()):
raise ValueError('Local symlink or escaped path refused')
if not file.is_file() or file.stat().st_size > MAX_FILE:
raise ValueError('Local file missing or too large: ' + relative)
return file.read_bytes()
def connect():
# Reuse the established secret reader and pinned host keys. Never copy secret
# values into this script, a manifest, command-line arguments, or its outputs.
sys.path.insert(0, str(WORKSPACE / '.codex_tmp/ssh-libs'))
spec = importlib.util.spec_from_file_location('release_connection', WORKSPACE / '.codex_tmp/local-dev-runner.py')
runner = importlib.util.module_from_spec(spec)
spec.loader.exec_module(runner)
if runner.ssh_settings()[0] != HOST:
raise ValueError('Existing connection target does not match the approved host')
return runner.connect()
def probe(client, paths):
stdin, stdout, stderr = client.exec_command('python3 -c ' + shlex.quote(REMOTE_PROBE), timeout=60)
stdin.write(json.dumps({'paths': sorted(set(paths))}))
stdin.channel.shutdown_write()
raw = stdout.read()
error = stderr.read() # Consume without echoing arbitrary remote diagnostics.
if stdout.channel.recv_exit_status() or error:
raise RuntimeError('Remote read-only metadata inspection failed')
return json.loads(raw)
def save_json(path, value):
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
def prepare(output, build, client):
"""Only local output is written; remote source/entry drift aborts the snapshot."""
output.mkdir(parents=True, exist_ok=False)
save_json(output / 'inspection-status.json', {'status': 'incomplete', 'remote_mutation': False})
sftp = client.open_sftp()
before_files, after_files, source_rows, source_diff, entry_rows, entry_diff = {}, {}, [], [], [], []
rollback_remove = []
try:
for relative in SOURCES:
after = local_file(PROJECT, relative)
before = read_remote(sftp, relative)
row, delta = compare(relative, before, after)
source_rows.append(row); source_diff.extend(delta)
if row['status'] != 'same':
after_files['source/' + relative] = after
if before is not None: before_files[relative] = before
else: rollback_remove.append(relative)
html = local_file(build, 'admin.html')
if not ASSET_PATTERN.search(html.decode('utf-8')):
raise ValueError('Build entry does not use /admin/assets/')
for relative in ENTRIES:
before = read_remote(sftp, relative)
row, delta = compare(relative, before, html)
entry_rows.append(row); entry_diff.extend(delta)
if before is not None:
before_files[relative] = before
# Preserve the old entry's primary assets as additional local
# rollback evidence. Transitive old assets must remain on host.
prefix = str(PurePosixPath(relative).parent)
for asset in ASSET_PATTERN.findall(before.decode('utf-8')):
name = prefix + '/' + safe_relative(asset)
if name not in before_files:
content = read_remote(sftp, name)
if content is None: raise ValueError('Existing entry asset missing: ' + name)
before_files[name] = content
else: rollback_remove.append(relative)
build_rows = []
for file in sorted(build.rglob('*')):
if not file.is_file(): continue
relative = file.relative_to(build).as_posix()
content = local_file(build, relative)
build_rows.append({'path': relative, 'sha256': sha(content), 'size': len(content)})
remote_paths = list(SOURCES) + ENTRIES + list(before_files)
remote_paths.extend(prefix + '/' + row['path'] for prefix in STATIC_ROOTS for row in build_rows if row['path'] != 'admin.html')
inventory = probe(client, remote_paths)
for row in source_rows + entry_rows:
metadata = inventory['files'][row['path']]
if metadata.get('sha256') != row['remote_sha256']:
raise RuntimeError('Remote file changed during inspection: ' + row['path'])
for name, content in before_files.items():
if inventory['files'][name].get('sha256') != sha(content):
raise RuntimeError('Rollback source changed during inspection: ' + name)
static_rows = []
for prefix in STATIC_ROOTS:
counts = {'same': 0, 'added': 0, 'modified': 0}
for row in build_rows:
if row['path'] == 'admin.html': continue
name = prefix + '/' + row['path']; remote = inventory['files'][name]
status = 'added' if not remote['exists'] else 'same' if remote['sha256'] == row['sha256'] else 'modified'
counts[status] += 1
static_rows.append({'target': name, 'status': status, 'local_sha256': row['sha256'], 'remote_sha256': remote.get('sha256')})
print(json.dumps({'static_root': prefix, **counts}), flush=True)
finally:
sftp.close()
# Check build provenance without treating an old test log as fresh acceptance.
provenance = WORKSPACE / '.codex_tmp/mindmap-speed/manifest.json'
known = {row['path']: row['sha256'] for row in json.loads(provenance.read_text(encoding='utf-8'))} if provenance.is_file() else {}
mismatches = [row['path'] for row in source_rows if row['path'] in known and row['local_sha256'] != known[row['path']]]
newer_ui = [row['path'] for row in source_rows if row['path'].startswith('ui/') and (PROJECT / row['path']).stat().st_mtime > (build / 'admin.html').stat().st_mtime + 1]
# Capture the candidate now; later working-tree edits must not silently enter it.
with zipfile.ZipFile(output / 'publish-candidate.zip', 'w', zipfile.ZIP_DEFLATED) as zipped:
for name, content in after_files.items(): zipped.writestr(name, content)
for row in build_rows:
content = local_file(build, row['path'])
if sha(content) != row['sha256']: raise RuntimeError('Build changed while packaging')
zipped.writestr('static/' + row['path'], content)
with zipfile.ZipFile(output / 'rollback-before.zip', 'w', zipfile.ZIP_DEFLATED) as zipped:
for name, content in before_files.items(): zipped.writestr(name, content)
snapshots = [{'path': name, 'sha256': sha(content), 'size': len(content)} for name, content in sorted(before_files.items())]
rollback = {'schema': 1, 'host': HOST, 'remote_root': REMOTE_ROOT,
'archive': 'rollback-before.zip', 'archive_sha256': sha((output / 'rollback-before.zip').read_bytes()),
'files': snapshots, 'new_paths_to_remove_only_after_future_approval': rollback_remove,
'preserve_all_existing_static_assets': True, 'restore_database': False,
'remote_execution_available': False}
plan = {'schema': 1, 'inspection_mode': 'read_only', 'remote_mutation': False,
'host': HOST, 'remote_root': REMOTE_ROOT,
'inspected_at': datetime.now(timezone(timedelta(hours=8))).isoformat(),
'sources': source_rows, 'entries': entry_rows, 'static_assets': static_rows,
'build': {'directory': str(build), 'files': build_rows, 'provenance_mismatches': mismatches, 'newer_ui_sources': newer_ui},
'processes': inventory['processes'], 'database_migration_planned': False,
'candidate_archive_sha256': sha((output / 'publish-candidate.zip').read_bytes()),
'pending_requirements': ['导图操作栏单行精简尚未实现', '访问记录分页及五类筛选尚未实现'],
'release_ready': False,
'future_order_only': ['复核并完成待办、重新构建和验收', '重新校验远程 SHA256 防止覆盖他人改动', '备份并同步本轮变更源文件', '同步新静态资源并保留所有旧资源', '按后端改动决定是否重启现有 Web', '最后替换 apps/static/admin 和 ui/dist/admin 两处入口', '真实页面回归后决定保留或回滚']}
save_json(output / 'release-plan.json', plan)
save_json(output / 'rollback-plan.json', rollback)
(output / 'source.diff').write_text('\n'.join(source_diff) + '\n', encoding='utf-8')
(output / 'static-entry.diff').write_text('\n'.join(entry_diff) + '\n', encoding='utf-8')
counts = {state: sum(row['status'] == state for row in source_rows) for state in ('added', 'modified', 'same')}
lines = ['# 本地发布准备与远程只读核对', '',
'本轮仅检查远程文件并在本地生成材料,没有上传、远程写入、迁移、服务重启或部署。', '',
f"源文件共 {len(source_rows)} 个:新增 {counts['added']},修改 {counts['modified']},相同 {counts['same']}。", '',
'## 源码差异', '', '| 文件 | 状态 | 新增行 | 删除行 |', '|---|---|---:|---:|']
lines += [f"| {row['path']} | {row['status']} | {row['added_lines']} | {row['removed_lines']} |" for row in source_rows]
lines += ['', '## 发布边界', '', '当前材料不是可直接发布的完成版本。单行操作栏及访问记录分页筛选尚未实现;必须补齐后重新构建、核对和验收。',
'构建来源哈希不匹配:' + (', '.join(mismatches) or '无(仅覆盖旧构建清单记载的文件)'),
'晚于构建入口的 UI 源码:' + (', '.join(newer_ui) or '无'), '',
'## 回滚材料', '',
'rollback-before.zip 保存本轮涉及的旧源文件、两处旧入口及入口直接引用的旧 JS/CSS。',
'归档不是整站或数据库备份;JS 的间接依赖不全部收录,因此将来发布必须保留全部现有静态资源。',
'新增源文件没有旧内容,其路径仅列入回滚计划,本脚本不会删除远程文件。',
'release_restore_local.py 只能验证归档或解包到一个新的本地目录,不能连接服务器、覆盖现有目录或执行回滚。', '',
'## 核对依据', '', 'release-plan.json:逐文件 SHA256、两处静态入口、构建资源与现有进程入口。',
'source.diff / static-entry.diff:源码与入口逐行差异。',
'publish-candidate.zip:核对时刻的本地候选源文件和构建,尚未发布。',
'rollback-plan.json:旧内容 SHA256、回滚条件与新增路径清单。']
(output / 'README.md').write_text('\n'.join(lines) + '\n', encoding='utf-8')
save_json(output / 'inspection-status.json', {'status': 'complete', 'remote_mutation': False, 'source_counts': counts})
print(json.dumps({'output': str(output), 'source_counts': counts, 'remote_mutation': False, 'release_ready': False}), flush=True)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True, help='New LOCAL output directory within the workspace')
parser.add_argument('--build', type=Path, default=PROJECT / 'ui/dist-mindmap-speed')
args = parser.parse_args()
output = args.output.resolve(); build = args.build.resolve()
if not output.is_relative_to(WORKSPACE) or not build.is_relative_to(PROJECT / 'ui'):
parser.error('Output/build must remain inside the workspace/project UI')
if output.exists(): parser.error('Output directory already exists; use a new snapshot directory')
client = connect()
try: prepare(output, build, client)
finally: client.close()
if __name__ == '__main__':
try: main()
except Exception as exc:
# Do not echo dependency/connection exceptions that might include secrets.
print('Release review failed (' + type(exc).__name__ + '); no remote mutation was requested.', file=sys.stderr)
sys.exit(1)