#!/usr/bin/env python3 """Replay a signed install/rollback test through a real Ultimate Odycer WebAdmin.""" from __future__ import annotations import argparse import hashlib import json import os import posixpath import re import shlex import shutil import socket import subprocess import tarfile import tempfile import time import urllib.error import urllib.request from datetime import datetime, timezone from pathlib import Path from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey from playwright.sync_api import sync_playwright VERSION_TOKEN = re.compile(r'^[0-9A-Za-z][0-9A-Za-z._+-]{0,63}$') ABSOLUTE_REMOTE = re.compile(r'^/[A-Za-z0-9._/-]+$') def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--apply', action='store_true', help='perform SSH writes and restart services') parser.add_argument('--host', required=True, help='SSH destination, for example root@203.0.113.10') parser.add_argument('--ssh-port', type=int, default=22) parser.add_argument('--ssh-key', type=Path, required=True) parser.add_argument('--known-hosts', type=Path) parser.add_argument('--archive', type=Path, required=True) parser.add_argument('--manifest', type=Path, required=True) parser.add_argument('--signature', type=Path, required=True) parser.add_argument('--public-key', type=Path, required=True, help='raw Ed25519 public key in hexadecimal') parser.add_argument('--current-version', required=True) parser.add_argument('--candidate-version', required=True) parser.add_argument('--remote-root', required=True, help='activation root containing live') parser.add_argument('--channel-dir', required=True) parser.add_argument('--state-file', required=True) parser.add_argument('--services-config', required=True) parser.add_argument('--server-library-path', default='') parser.add_argument('--webadmin-port', type=int, default=28082) parser.add_argument('--local-port', type=int, default=0) parser.add_argument('--username-env', default='UO_WEBADMIN_TEST_USER') parser.add_argument('--password-env', default='UO_WEBADMIN_TEST_PASSWORD') parser.add_argument('--browser') parser.add_argument('--output', type=Path, default=Path('pilot-webadmin-update-proof.json')) return parser.parse_args() def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open('rb') as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b''): digest.update(chunk) return digest.hexdigest() def run(command: list[str], *, timeout: int = 180) -> str: completed = subprocess.run( command, check=False, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, ) if completed.returncode != 0: raise RuntimeError((completed.stdout + completed.stderr)[-5000:]) return completed.stdout def free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.bind(('127.0.0.1', 0)) return sock.getsockname()[1] def wait_port(port: int, timeout: float = 30) -> None: deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: with socket.create_connection(('127.0.0.1', port), timeout=0.5): return except OSError: time.sleep(0.25) raise RuntimeError(f'local tunnel port {port} did not open') def request_json(base: str, path: str, *, token: str | None = None, payload: dict | None = None): headers = {'Accept': 'application/json'} data = None if token: headers['Authorization'] = f'Bearer {token}' if payload is not None: headers['Content-Type'] = 'application/json' data = json.dumps(payload).encode('utf-8') request = urllib.request.Request(base + path, headers=headers, data=data) opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) try: with opener.open(request, timeout=10) as response: return response.status, json.loads(response.read() or b'{}') except urllib.error.HTTPError as error: return error.code, json.loads(error.read() or b'{}') def wait_for(base: str, token: str, version: str, activation: str, timeout: float = 120) -> dict: deadline = time.monotonic() + timeout last = None while time.monotonic() < deadline: try: status, body = request_json(base, '/api/update/status', token=token) last = body if status == 200 and body.get('version') == version: current = (body.get('activation') or {}).get('status') if current == activation: return body if current in ('rollback_failed', 'rollback_unhealthy'): raise RuntimeError(f'rollback requires intervention: {body}') except (OSError, urllib.error.URLError, TimeoutError, json.JSONDecodeError): pass time.sleep(0.5) raise RuntimeError(f'WebAdmin did not reach {version}/{activation}: {last}') def archive_root(archive: Path) -> str: with tarfile.open(archive, 'r:gz') as bundle: roots = {Path(member.name).parts[0] for member in bundle.getmembers() if member.name} if len(roots) != 1: raise SystemExit('archive must contain exactly one top-level directory') root = roots.pop() if not VERSION_TOKEN.fullmatch(root): raise SystemExit(f'unsafe archive root: {root!r}') return root def validate_signed_candidate(args: argparse.Namespace) -> dict: for value in (args.current_version, args.candidate_version): if not VERSION_TOKEN.fullmatch(value): raise SystemExit(f'invalid version token: {value!r}') for value in (args.remote_root, args.channel_dir, args.state_file, args.services_config): if not ABSOLUTE_REMOTE.fullmatch(value): raise SystemExit(f'unsafe absolute remote path: {value!r}') if args.current_version == args.candidate_version: raise SystemExit('candidate version must differ from current version') manifest_bytes = args.manifest.read_bytes() signature = args.signature.read_bytes() public = bytes.fromhex(args.public_key.read_text(encoding='utf-8').strip()) Ed25519PublicKey.from_public_bytes(public).verify(signature, manifest_bytes) manifest = json.loads(manifest_bytes) if manifest.get('version') != args.candidate_version: raise SystemExit('signed manifest version does not match --candidate-version') linux = next((item for item in manifest.get('artifacts', []) if item.get('platform') == 'linux-x86_64'), None) if linux is None: raise SystemExit('signed manifest has no linux-x86_64 artifact') if linux.get('sha256') != sha256(args.archive) or linux.get('bytes') != args.archive.stat().st_size: raise SystemExit('candidate archive does not match the signed manifest') return manifest def transport_options(args: argparse.Namespace) -> list[str]: options = [ '-i', str(args.ssh_key), '-o', 'IdentitiesOnly=yes', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'ConnectTimeout=10', ] if args.known_hosts: options.extend(['-o', f'UserKnownHostsFile={args.known_hosts}']) return options def ssh_options(args: argparse.Namespace) -> list[str]: return [*transport_options(args), '-p', str(args.ssh_port)] def ssh(args: argparse.Namespace, command: str, timeout: int = 180) -> str: return run(['ssh', *ssh_options(args), args.host, command], timeout=timeout) def scp(args: argparse.Namespace, sources: list[Path], destination: str) -> None: run(['scp', '-P', str(args.ssh_port), *transport_options(args), *[str(path) for path in sources], destination]) def main() -> None: args = parse_args() validate_signed_candidate(args) root = archive_root(args.archive) plan = { 'host': args.host, 'current_version': args.current_version, 'candidate_version': args.candidate_version, 'archive_sha256': sha256(args.archive), 'manifest_sha256': sha256(args.manifest), 'signature_sha256': sha256(args.signature), 'target': posixpath.join(args.remote_root, args.candidate_version), 'channel_dir': args.channel_dir, 'temporary_admin_created': False, 'credentials_source': [args.username_env, args.password_env], } print(json.dumps({**plan, 'apply': bool(args.apply)}, indent=2, ensure_ascii=False)) if not args.apply: return username = os.environ.get(args.username_env) password = os.environ.get(args.password_env) if not username or not password: raise SystemExit(f'set credentials through {args.username_env} and {args.password_env}') target = posixpath.join(args.remote_root, args.candidate_version) previous = posixpath.join(args.remote_root, 'live.previous') failed = posixpath.join(args.remote_root, f'live.failed-rollback-{args.candidate_version}') output = args.output.resolve() output.parent.mkdir(parents=True, exist_ok=True) screenshots = { 'before': output.with_name(output.stem + '-before.png'), 'installed': output.with_name(output.stem + '-installed.png'), 'rolled_back': output.with_name(output.stem + '-rolled-back.png'), } local_port = args.local_port or free_port() base = f'http://127.0.0.1:{local_port}' proof: dict[str, object] = { 'ok': False, 'checked_at_utc': datetime.now(timezone.utc).isoformat(timespec='seconds'), **plan, 'screenshots': {name: str(path) for name, path in screenshots.items()}, } tunnel = None tokens: list[str] = [] with tempfile.TemporaryDirectory(prefix='uo-webadmin-update-test-') as temporary: temp = Path(temporary) try: for path in (target, previous, failed): if subprocess.run( ['ssh', *ssh_options(args), args.host, f'test -e {shlex.quote(path)}'], capture_output=True, timeout=30, check=False, ).returncode == 0: raise RuntimeError(f'refusing to overwrite existing remote state: {path}') # Download the production channel before replacing it. run(['scp', '-P', str(args.ssh_port), *transport_options(args), args.host + ':' + posixpath.join(args.channel_dir, 'manifest.json'), str(temp / 'production-manifest.json')]) run(['scp', '-P', str(args.ssh_port), *transport_options(args), args.host + ':' + posixpath.join(args.channel_dir, 'manifest.sig'), str(temp / 'production-manifest.sig')]) remote_archive = '/tmp/' + args.archive.name remote_manifest = '/tmp/uo-test-manifest.json' remote_signature = '/tmp/uo-test-manifest.sig' scp(args, [args.archive], args.host + ':' + remote_archive) local_manifest = temp / 'uo-test-manifest.json' local_signature = temp / 'uo-test-manifest.sig' shutil.copyfile(args.manifest, local_manifest) shutil.copyfile(args.signature, local_signature) scp(args, [local_manifest, local_signature], args.host + ':/tmp/') ssh(args, f'''set -euo pipefail mkdir -p {shlex.quote(posixpath.join(args.remote_root, '.pilot-webadmin-update-stage'))} tar -xzf {shlex.quote(remote_archive)} -C {shlex.quote(posixpath.join(args.remote_root, '.pilot-webadmin-update-stage'))} test -d {shlex.quote(posixpath.join(args.remote_root, '.pilot-webadmin-update-stage', root))} mv {shlex.quote(posixpath.join(args.remote_root, '.pilot-webadmin-update-stage', root))} {shlex.quote(target)} rmdir {shlex.quote(posixpath.join(args.remote_root, '.pilot-webadmin-update-stage'))} printf '{sha256(args.archive)}\n' > {shlex.quote(posixpath.join(target, 'archive.sha256'))} cp {shlex.quote(remote_manifest)} {shlex.quote(posixpath.join(args.channel_dir, 'manifest.json'))} cp {shlex.quote(remote_signature)} {shlex.quote(posixpath.join(args.channel_dir, 'manifest.sig'))} ''') tunnel = subprocess.Popen( ['ssh', *ssh_options(args), '-N', '-o', 'ExitOnForwardFailure=yes', '-L', f'127.0.0.1:{local_port}:127.0.0.1:{args.webadmin_port}', args.host], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0), ) wait_port(local_port) def login() -> str: status, body = request_json(base, '/api/auth/login', payload={'username': username, 'password': password}) if status != 200 or not body.get('token'): raise RuntimeError(f'WebAdmin login failed: HTTP {status}') tokens.append(body['token']) return body['token'] token = login() status_code, before = request_json(base, '/api/update/status', token=token) if status_code != 200 or before.get('channel_version') != args.candidate_version or before.get('update_available') is not True: raise RuntimeError(f'candidate update is not visible: {before}') if before.get('apply_supported') is not True: raise RuntimeError(f'activation is not configured: {before}') page_errors: list[str] = [] with sync_playwright() as playwright: browser = playwright.chromium.launch(executable_path=args.browser, headless=True) context = browser.new_context(viewport={'width': 1440, 'height': 1000}, locale='fr-FR') def browser_login(page) -> None: page.goto(base + '/', wait_until='networkidle', timeout=30000) page.locator('input[autocomplete="username"]:visible').first.fill(username) page.locator('input[type="password"]:visible').first.fill(password) page.get_by_role('button').filter(has_text=re.compile(r'connect|connexion', re.IGNORECASE)).first.click() page.wait_for_url(re.compile(r'.*/dashboard(?:\?.*)?$'), timeout=15000) page = context.new_page() page.on('pageerror', lambda error: page_errors.append(error.message)) browser_login(page) page.goto(base + '/system/updates', wait_until='networkidle', timeout=30000) page.get_by_role('button', name=re.compile(r'Installer et redémarrer', re.IGNORECASE)).wait_for(timeout=15000) page.screenshot(path=str(screenshots['before']), full_page=True) page.get_by_role('button', name=re.compile(r'Installer et redémarrer', re.IGNORECASE)).click() page.get_by_role('dialog').wait_for(timeout=10000) page.get_by_role('button', name=re.compile(r'Confirmer l’installation', re.IGNORECASE)).click() installed = wait_for(base, token, args.candidate_version, 'activated') if installed.get('rollback_supported') is not True: raise RuntimeError(f'rollback button is unavailable after install: {installed}') page = context.new_page() page.on('pageerror', lambda error: page_errors.append(error.message)) browser_login(page) page.goto(base + '/system/updates', wait_until='networkidle', timeout=30000) page.get_by_role('button', name=re.compile(r'Revenir à la version précédente', re.IGNORECASE)).wait_for(timeout=15000) page.get_by_text('Activation terminée et contrôles de santé réussis').wait_for(timeout=10000) page.screenshot(path=str(screenshots['installed']), full_page=True) page.get_by_role('button', name=re.compile(r'Revenir à la version précédente', re.IGNORECASE)).click() page.get_by_role('dialog').wait_for(timeout=10000) page.get_by_role('button', name=re.compile(r'Confirmer le rollback', re.IGNORECASE)).click() rolled_back = wait_for(base, token, args.current_version, 'rolled_back') scp(args, [temp / 'production-manifest.json', temp / 'production-manifest.sig'], args.host + ':/tmp/') ssh(args, f'''set -euo pipefail cp /tmp/production-manifest.json {shlex.quote(posixpath.join(args.channel_dir, 'manifest.json'))} cp /tmp/production-manifest.sig {shlex.quote(posixpath.join(args.channel_dir, 'manifest.sig'))} ''') status_code, final_status = request_json(base, '/api/update/status', token=token) if ( status_code != 200 or final_status.get('version') != args.current_version or final_status.get('channel_version') != args.current_version or final_status.get('update_available') is not False ): raise RuntimeError(f'production channel was not restored cleanly: {final_status}') page = context.new_page() page.on('pageerror', lambda error: page_errors.append(error.message)) browser_login(page) page.goto(base + '/system/updates', wait_until='networkidle', timeout=30000) page.get_by_text('Version précédente restaurée', exact=True).wait_for(timeout=15000) page.screenshot(path=str(screenshots['rolled_back']), full_page=True) browser.close() if page_errors: raise RuntimeError(f'WebAdmin emitted page errors: {page_errors}') proof.update({ 'ok': True, 'before': before, 'installed': installed, 'rolled_back': rolled_back, 'final_status': final_status, 'page_errors': page_errors, }) except Exception as error: proof['error'] = f'{type(error).__name__}: {error}' raise finally: for token in tokens: try: request_json(base, '/api/auth/logout', token=token, payload={}) except (OSError, RuntimeError, subprocess.SubprocessError): pass if tunnel: tunnel.terminate() try: tunnel.wait(timeout=10) except subprocess.TimeoutExpired: tunnel.kill() tunnel.wait(timeout=5) try: if (temp / 'production-manifest.json').is_file(): scp(args, [temp / 'production-manifest.json', temp / 'production-manifest.sig'], args.host + ':/tmp/') ssh(args, f'''set -euo pipefail cp /tmp/production-manifest.json {shlex.quote(posixpath.join(args.channel_dir, 'manifest.json'))} cp /tmp/production-manifest.sig {shlex.quote(posixpath.join(args.channel_dir, 'manifest.sig'))} ''') except (OSError, RuntimeError, subprocess.SubprocessError) as error: proof['channel_restore_error'] = str(error) try: env_prefix = '' if args.server_library_path: env_prefix = 'env LD_LIBRARY_PATH=' + shlex.quote(args.server_library_path) + ' ' current = ssh(args, env_prefix + shlex.quote(posixpath.join(args.remote_root, 'live', 'bin', 'webadmin-server')) + ' --version', timeout=30).strip() if current != args.current_version: ssh(args, f'''cd {shlex.quote(posixpath.join(args.remote_root, "live"))} python3 update/update_activate.py rollback --root {shlex.quote(args.remote_root)} --services-config {shlex.quote(args.services_config)} --state-file {shlex.quote(args.state_file)} --failed-label emergency-tutorial-test ''', timeout=180) except (OSError, RuntimeError, subprocess.SubprocessError) as error: proof['emergency_restore_error'] = str(error) output.write_text(json.dumps(proof, indent=2, ensure_ascii=False) + chr(10), encoding='utf-8') print(json.dumps(proof, indent=2, ensure_ascii=False)) if __name__ == '__main__': main()