Skip to content

Let install.py resume a failed first install #716

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SaladDay merged 1 commit into from
Oct 15, 2026
Merged

Let install.py resume a failed first install #716

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions deploy/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ An existing directory runs the release named by `OAC_REVISION` in its `compose.y

`install.py --upgrade` upgrades a source-bundle installation the same way, under the same lock. It publishes the bundle's node payload beside earlier releases, renders the new `compose.yaml` and rewrites only its own `compose.override.yaml` entries, the four service images and Web's `/node-payload` bind, keeping every other entry. After the same two checks it moves the payload's `active.json`, renames `oac`, `compose.override.yaml` and `compose.yaml` into place and starts the services. On its own release it resumes like `oac install`; another release is refused without `--upgrade`, and `--public-url` is required only for a new directory.

`install.py` prepares a new installation in a `.staging-*` directory inside the installation directory: the node payload, `.env`, `oac` and both Compose files. After `docker compose config` accepts them, it moves them into place, `compose.yaml` last. An earlier failure removes the stage, and the directory too when the run created it, so the same command can be rerun; the next run also removes a stage left by a killed run. A directory with other contents but no `compose.yaml` is refused.

The host `oac` command also implements `apply`, `core-key` and `rotate-core-key`. `apply` runs init, which finishes a first start that stopped before it and refuses a data volume committed to another release, then checks Core configuration before starting services and recreating those that changed. Reading the key executes `oac-web core-key` in Web. Rotation runs in the initialization container, where Linux file ownership is identical on every host, then restarts Core and Web. Start, stop, logs and removal use `docker compose`. The initialization image verifies and copies bundled node metadata without network access. Its receipt authenticates immutable metadata and credentials; the operator-rotatable Core key and its digest remain mutable. No service receives a Docker socket. Initialization emits structured step logs without credential values; view them with `docker compose logs --timestamps init`.

Web serves the console and forwards `/v1` and `/api/v1` to Core, so it is the only published service. HTTPS is terminated by the operator's reverse proxy or hosting platform, which routes to `web:8080`; `OAC_PUBLIC_URL` records that origin.
Expand Down
108 changes: 68 additions & 40 deletions deploy/install-source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,17 @@
from urllib.parse import urlsplit
import uuid

# The bundle ships the node installer's distribution modules beside this file.
BUNDLE = Path(__file__).resolve().parent
sys.path.insert(0, str(BUNDLE))
from distribution import DistributionError, ensure_docker_image, load_manifest
import node_payload

def run(*args, **kwargs):
return subprocess.run(args, check=True, text=True, **kwargs)

def run(failure, *args, **kwargs):
"""Run a command whose output the user sees; a nonzero exit raises failure."""
if subprocess.run(args, **kwargs).returncode:
raise ValueError(failure)


def compose_revision(text):
Expand All @@ -24,11 +32,6 @@ def compose_revision(text):


def install(bundle, destination, host, port, public_url, upgrade):
# Use the distribution's own node download and publication implementation.
sys.path.insert(0, str(bundle))
from distribution import ensure_docker_image, load_manifest
import node_payload

manifest = load_manifest(offline_root=bundle)
revision = manifest['source_commit']
for name in manifest['artifacts'].values():
Expand All @@ -37,6 +40,7 @@ def install(bundle, destination, host, port, public_url, upgrade):
lock = Path(str(destination) + '.lock')
lock.mkdir(parents=True, exist_ok=True, mode=0o700)
descriptor = os.open(lock, os.O_RDONLY | os.O_DIRECTORY)
created = committed = False
try:
# The same lock as oac apply and the other mutating oac commands.
try:
Expand All @@ -46,7 +50,8 @@ def install(bundle, destination, host, port, public_url, upgrade):
# A staging directory left by a killed run is never published.
for leftover in destination.glob('.staging-*'):
shutil.rmtree(leftover)
existing = destination.exists() and any(destination.iterdir())
created = not destination.exists()
existing = not created and any(destination.iterdir())
if existing and not all((destination / name).is_file() for name in ('compose.yaml', '.env')):
raise ValueError(f'Incomplete installation: preserve {destination} and choose another directory')
previous = compose_revision((destination / 'compose.yaml').read_text()) if existing else ''
Expand All @@ -60,18 +65,20 @@ def install(bundle, destination, host, port, public_url, upgrade):
images = {name: ensure_docker_image(manifest, name, lambda name=name: bundle / 'images' / (name + '.tar'))
for name in ('core', 'web', 'ingress', 'database')}
destination.mkdir(parents=True, exist_ok=True, mode=0o700)
payload = destination / 'node-payload'
node_payload.publish(payload, bundle)
# These files contain only public distribution bytes and must be readable by Web.
for root, dirs, files in os.walk(payload):
os.chmod(root, 0o755)
for name in files:
os.chmod(Path(root) / name, 0o644)
if not existing:
(destination / '.env').write_text(f"COMPOSE_PROJECT_NAME=oac-{uuid.uuid4().hex[:10]}\nOAC_HOST='{host}'\nOAC_WEB_PORT='{port}'\nOAC_PUBLIC_URL='{public_url}'\n")
os.chmod(destination / '.env', 0o600)
with tempfile.TemporaryDirectory(prefix='.staging-', dir=destination) as stage:
stage = Path(stage)
# A new installation is assembled in the stage and moved in only after its check.
root = destination if existing else stage
payload = root / 'node-payload'
node_payload.publish(payload, bundle)
# These files contain only public distribution bytes and must be readable by Web.
for directory, dirs, files in os.walk(payload):
os.chmod(directory, 0o755)
for name in files:
os.chmod(Path(directory) / name, 0o644)
if not existing:
(stage / '.env').write_text(f"COMPOSE_PROJECT_NAME=oac-{uuid.uuid4().hex[:10]}\nOAC_HOST='{host}'\nOAC_WEB_PORT='{port}'\nOAC_PUBLIC_URL='{public_url}'\n")
os.chmod(stage / '.env', 0o600)
template = (bundle / 'compose.yaml.template').read_text().replace('__OAC_REVISION__', revision).replace('__OAC_INIT_IMAGE__', images['ingress'])
for name in ('core', 'web'):
template = template.replace('__OAC_' + name.upper() + '_IMAGE__', images[name])
Expand All @@ -87,67 +94,88 @@ def install(bundle, destination, host, port, public_url, upgrade):
services.setdefault(name, {})['image'] = images[image]
web = services['web']
web['volumes'] = [volume for volume in web.get('volumes', []) if not (isinstance(volume, dict) and volume.get('target') == '/node-payload')]
web['volumes'].append({'type': 'bind', 'source': str(payload), 'target': '/node-payload', 'read_only': True})
web['volumes'].append({'type': 'bind', 'source': str(destination / 'node-payload'), 'target': '/node-payload', 'read_only': True})
(stage / 'compose.override.yaml').write_text(json.dumps(override, indent=2) + '\n')
os.chmod(stage / 'compose.override.yaml', 0o600)
shutil.copyfile(bundle / 'oac', stage / 'oac')
os.chmod(stage / 'oac', 0o700)
compose = ('docker', 'compose', '-f', str(stage / 'compose.yaml'), '-f', str(stage / 'compose.override.yaml'), '--project-directory', str(destination))
compose = ('docker', 'compose', '-f', str(stage / 'compose.yaml'), '-f', str(stage / 'compose.override.yaml'), '--project-directory', str(root))
if existing:
# Init upgrades the data volume or refuses; until then the running release is untouched.
# On the same release init comes first: it finishes a first start that stopped before it,
# so check-config finds the secrets.
checks = [('run', '--rm', '--no-deps', '--entrypoint', '/usr/local/bin/oac-core', 'core', 'check-config'),
('run', '--rm', '--no-deps', 'init')]
checks = [('Configuration check', ('run', '--rm', '--no-deps', '--entrypoint', '/usr/local/bin/oac-core', 'core', 'check-config')),
('Data volume check', ('run', '--rm', '--no-deps', 'init'))]
if previous == revision:
checks.reverse()
try:
for check in checks:
run(*compose, *check, cwd=destination)
except subprocess.CalledProcessError:
raise ValueError('Upgrade stopped; the running services are unchanged') from None
for step, check in checks:
failure = (f'{step} failed; no service was changed. See the output above.' if previous == revision
else f'Upgrade stopped; the running services are unchanged. {step} failed; see the output above.')
run(failure, *compose, *check, cwd=destination)
else:
run(*compose, 'config', '--quiet', cwd=destination)
run('Compose configuration check failed; see the output above. Nothing was installed.', *compose, 'config', '--quiet', cwd=destination)
node_payload.activate(payload, revision)
for name in ('oac', 'compose.override.yaml', 'compose.yaml'):
# compose.yaml goes last: it marks the directory as an installation.
for name in ('oac', 'compose.override.yaml', 'compose.yaml') if existing else ('node-payload', '.env', 'oac', 'compose.override.yaml', 'compose.yaml'):
os.replace(stage / name, destination / name)
committed = True
# Installations from earlier releases kept a checksum that no longer describes compose.yaml.
(destination / 'compose-sha256sums.txt').unlink(missing_ok=True)
print(f'Configuration ready in {destination}. Starting services…', flush=True)
run('docker', 'compose', 'up', '-d', '--wait', '--wait-timeout', '180', '--pull', 'never', '--remove-orphans', cwd=destination)
run('Starting services failed; see the output above. Settings and data are kept: fix the cause and rerun the same command.',
'docker', 'compose', 'up', '-d', '--wait', '--wait-timeout', '180', '--pull', 'never', '--remove-orphans', cwd=destination)
if not existing:
print(f'Console: {public_url}')
if previous and previous != revision:
print(f"Upgraded from {previous[:12]} to {revision[:12]}. Settings → Sandbox and scheduling offers this release's Runtime; sandboxes keep their current Runtime until you apply it.")
print('Core key:', flush=True)
run('docker', 'compose', 'exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key', cwd=destination)
run(f'Reading the Core key failed; see the output above. Show it with {destination / "oac"} core-key --show.',
'docker', 'compose', 'exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key', cwd=destination)
finally:
# A new installation that failed before compose.yaml was in place leaves nothing behind.
if created and not committed:
shutil.rmtree(destination, ignore_errors=True)
os.close(descriptor)


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--bundle-dir', type=Path, default=Path(__file__).resolve().parent)
parser.add_argument('--install-dir', type=Path, default=Path.home() / '.oac' / 'local')
parser.add_argument('--host', default='127.0.0.1')
parser.add_argument('--web-port', type=int, default=8080)
parser.add_argument('--public-url', help='HTTP(S) origin reachable by browsers and nodes; required for a new installation')
parser.add_argument('--upgrade', action='store_true', help='upgrade an existing installation to this release')
args = parser.parse_args()
if not args.install_dir.is_absolute() or args.install_dir == Path('/') or not args.bundle_dir.is_absolute():
parser.error('bundle and installation directories must be absolute; installation cannot be /')
ipaddress.ip_address(args.host)
if not args.install_dir.is_absolute() or args.install_dir == Path('/'):
parser.error('installation directory must be absolute and cannot be /')
try:
ipaddress.ip_address(args.host)
except ValueError:
parser.error('host must be an IP address')
if not 1 <= args.web_port <= 65535:
parser.error('web port must be between 1 and 65535')
if args.public_url is not None:
url = urlsplit(args.public_url)
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.path not in ('', '/') or url.query or url.fragment or any(c in args.public_url for c in "\r\n'\\$"):
parser.error('public URL must be an HTTP(S) origin without credentials')
install(args.bundle_dir.resolve(), args.install_dir.resolve(), args.host, args.web_port, args.public_url, args.upgrade)
destination = args.install_dir.resolve()
try:
install(BUNDLE, destination, args.host, args.web_port, args.public_url, args.upgrade)
except KeyboardInterrupt:
sys.exit('Interrupted; rerun the same command to continue.')
except PermissionError as error:
# Typically left by an earlier run under sudo. A path that could not be created names its parent.
path = Path(error.filename or destination)
if destination in path.parents:
path = destination
elif not path.exists():
path = path.parent
sys.exit(f'{path} is not writable by this account; if an earlier run used sudo, return it with: sudo chown -R "$(id -un)" {path}')
except OSError as error:
sys.exit(f'{error.strerror}: {error.filename}' if error.strerror and error.filename else str(error))
except (ValueError, DistributionError, node_payload.PayloadError) as error:
sys.exit(str(error))


if __name__ == '__main__':
try:
main()
except (OSError, ValueError, subprocess.CalledProcessError) as error:
sys.exit(str(error))
main()
46 changes: 41 additions & 5 deletions deploy/test_install_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class SourceInstallTest(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.root = Path(self.temp.name).resolve()
self.bundle, self.destination = self.root / 'bundle', self.root / 'installation'
self.bundle.mkdir()
self.revision = 'a' * 40
Expand All @@ -40,13 +40,20 @@ def write_sums(self):
(self.bundle / 'SHA256SUMS').write_text(hashlib.sha256((self.bundle / 'manifest.json').read_bytes()).hexdigest() + ' manifest.json\n')

def install(self, upgrade=False, image='sha256:' + '1' * 64, failure=None):
def run(*args, **kwargs):
def run(message, *args, **kwargs):
self.calls.append((args, kwargs))
if failure and failure in args:
raise subprocess.CalledProcessError(1, args)
with patch('distribution.ensure_docker_image', return_value=image), patch.object(installer, 'run', side_effect=run):
raise ValueError(message)
with patch.object(installer, 'ensure_docker_image', return_value=image), patch.object(installer, 'run', side_effect=run):
installer.install(self.bundle, self.destination, '127.0.0.1', 19873, None if upgrade else 'http://127.0.0.1:19873', upgrade)

def main(self):
"""Run the command line and return its exit message."""
argv = ['install.py', '--install-dir', str(self.destination), '--public-url', 'http://127.0.0.1:19873']
with patch.object(sys, 'argv', argv), patch.object(installer, 'BUNDLE', self.bundle), self.assertRaises(SystemExit) as exit:
installer.main()
return exit.exception.code

def commands(self):
"""Compose subcommands, without the staged file arguments."""
return [args[args.index('--project-directory') + 2:] if '--project-directory' in args else args[2:] for args, _ in self.calls]
Expand Down Expand Up @@ -82,12 +89,23 @@ def test_existing_installation_is_not_replaced(self):
self.assertEqual((self.destination / 'keep').read_text(), 'existing')
self.assertEqual(self.calls, [])

def test_failed_first_install_leaves_nothing_and_reruns(self):
with self.assertRaisesRegex(ValueError, '^Compose configuration check failed; see the output above. Nothing was installed.$'):
self.install(failure='config')
self.assertFalse(self.destination.exists())
# A killed run leaves only its stage, which the rerun removes.
(self.destination / '.staging-killed/node-payload').mkdir(parents=True)
self.install()
self.assertEqual(sorted(path.name for path in self.destination.iterdir()), ['.env', 'compose.override.yaml', 'compose.yaml', 'node-payload', 'oac'])

def test_same_release_reruns_init_before_the_configuration_check(self):
# A first start that stopped before init left no secrets for check-config.
self.install()
self.calls = []
self.install()
self.assertEqual([command[-1] for command in self.commands()], ['init', 'check-config', '--remove-orphans', 'core-key'])
with self.assertRaisesRegex(ValueError, '^Data volume check failed; no service was changed. See the output above.$'):
self.install(failure='init')

def test_upgrade_checks_the_new_release_before_replacing_it_and_keeps_operator_entries(self):
self.install()
Expand All @@ -106,7 +124,7 @@ def test_upgrade_checks_the_new_release_before_replacing_it_and_keeps_operator_e
with self.assertRaisesRegex(ValueError, 'rerun with --upgrade'):
self.install(image='sha256:' + '2' * 64)
self.assertEqual(self.snapshot(), before)
with self.assertRaisesRegex(ValueError, 'Upgrade stopped; the running services are unchanged'):
with self.assertRaisesRegex(ValueError, '^Upgrade stopped; the running services are unchanged. Data volume check failed; see the output above.$'):
self.install(upgrade=True, image='sha256:' + '2' * 64, failure='init')
self.assertEqual(self.snapshot(), before)

Expand All @@ -133,6 +151,24 @@ def test_thin_bundle_rejected_before_creating_installation(self):
self.install()
self.assertFalse(self.destination.exists())

def test_failures_exit_with_one_line_and_the_next_step(self):
(self.bundle / 'SHA256SUMS').write_text('0' * 64 + ' manifest.json\n')
self.assertEqual(self.main(), 'Distribution manifest checksum mismatch')
self.write_sums()

def interrupt(*args, **kwargs):
raise KeyboardInterrupt
with patch.object(installer, 'ensure_docker_image', return_value='sha256:' + '1' * 64), patch.object(installer, 'run', side_effect=interrupt):
self.assertEqual(self.main(), 'Interrupted; rerun the same command to continue.')
self.assertFalse(self.destination.exists())

if os.geteuid():
# Root-owned after an earlier sudo run; root itself is never denied.
lock = Path(str(self.destination) + '.lock')
lock.chmod(0)
self.addCleanup(lock.chmod, 0o700)
self.assertEqual(self.main(), f'{lock} is not writable by this account; if an earlier run used sudo, return it with: sudo chown -R "$(id -un)" {lock}')


if __name__ == '__main__':
unittest.main()
Loading
Loading