diff --git a/.github/workflows/catalogue-lint.yml b/.github/workflows/catalogue-lint.yml new file mode 100644 index 00000000..d74dadae --- /dev/null +++ b/.github/workflows/catalogue-lint.yml @@ -0,0 +1,65 @@ +name: catalogue-lint + +# Gates changes to the app-store catalogue (catalogue/catalogue.json): +# +# * Stateful-app release freeze. Nodes upgrade installed apps hourly with the +# pilotctl they already run, and a pilotctl without the app-state fix +# deletes an app's saved state (wallet EVM key + data.db, smol secrets, +# per-app identities) when it applies an update. Any update to an app listed +# in catalogue/stateful-apps.json, or whose bundle manifest grants fs.write +# or key.sign, fails until it is approved: an approved_bumps entry in +# catalogue/stateful-apps.json, or the PR label +# `catalogue:stateful-bump-approved` (re-runs on label changes). +# * Bundle checks for every added/changed entry: sha pins, manifest id and +# version, binary pin, and that each binary runs on the platform it is +# published for (a legacy single bundle must not ship a native binary). +# +# See catalogue/README.md ("Stateful apps: release freeze"). + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + paths: + - 'catalogue/**' + - '.github/workflows/catalogue-lint.yml' + +permissions: + contents: read + +jobs: + lint: + name: catalogue lint + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + GOWORK: 'off' + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + - name: Unit tests (catalogue/lint) + working-directory: catalogue/lint + run: go test -count=1 ./... + + - name: Lint catalogue changes against the PR base + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + STATEFUL_BUMP_APPROVED: ${{ contains(github.event.pull_request.labels.*.name, 'catalogue:stateful-bump-approved') }} + run: | + set -euo pipefail + merge_base="$(git merge-base "$BASE_SHA" HEAD)" + base_json="$RUNNER_TEMP/base-catalogue.json" + # A base without a catalogue (first introduction) lints every entry as new. + git show "$merge_base:catalogue/catalogue.json" > "$base_json" 2>/dev/null || : > "$base_json" + args=(--base "$base_json" --head "$GITHUB_WORKSPACE/catalogue/catalogue.json" --policy "$GITHUB_WORKSPACE/catalogue/stateful-apps.json") + if [ "$STATEFUL_BUMP_APPROVED" = "true" ]; then + echo "::notice::PR carries catalogue:stateful-bump-approved; stateful-app updates are reported as warnings" + args+=(--allow-stateful-bumps) + fi + cd catalogue/lint + go run . "${args[@]}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 83b6b90b..fa082935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,33 @@ Detailed per-release notes are on the or an installer re-run — bypassed the disabled flag and re-injected skills a user had turned off. The opt-out is now a hard gate on every write path; only the read-only `pilotctl skills` status still previews. (skillinject) +- **App-store installs and upgrades keep an app's saved state.** + `pilotctl appstore install --force` and `appstore upgrade` (which the updater + runs hourly as `upgrade --all`) deleted everything an app kept in its own + directory, such as the wallet's EVM key (`identity-evm.json`) and `data.db`, + smol's `secrets.json` and per-app identities. Every file that is not part of + the bundle is now carried into the new install, and the replaced install is + kept as a backup in `app-backups//` beside the install root + (`$PILOT_APPSTORE_BACKUP_ROOT` overrides). The newest 3 routine backups of + each kind are kept; a backup that may be the only copy of state is never + pruned. `uninstall` lists the backups that remain. + - `install ` on an installed app is now a no-op (exit 0) that points to + `upgrade`. Before, it failed with `conflict`. `conflict` now means + `--version` or a local bundle names another version without `--force`. + - New `--reset-state` (implies `--force`) reinstalls without the old state, + with a warning. The backup is still kept. + - Installs of one app are serialized. A second one waits up to 5 minutes, + then fails with `timeout`. + - `upgrade --all` tries every app and exits 1 at the end with + `upgrade_failed`, naming the apps that failed. + - Install JSON adds `already_installed`, `hint`, `preserved_state`, + `state_reset`, `state_not_carried`, `backup_dir` and `backup_warning`. + Uninstall JSON adds `backups`. + - The check that refuses a bundle whose binary cannot run on this host now + also covers universal Mach-O and PE images. The refusal names the app, + version and host platform, and says nothing was installed. + - The catalogue lint blocks releases of stateful apps until nodes run a + pilotctl with this fix. ## [1.12.8] - 2026-07-16 diff --git a/catalogue/README.md b/catalogue/README.md index e369bea7..c2a7baf2 100644 --- a/catalogue/README.md +++ b/catalogue/README.md @@ -203,6 +203,114 @@ verifies the signature against the embedded catalogue public key before trusting any entry. An unsigned, missing-signature, or tampered catalogue is refused (fail-closed). +### A published update reaches every node within the hour + +Nodes with auto-update on run `pilotctl appstore upgrade --all` every hour. +Anything that changes what a node would install triggers it: a new `version`, +or a new bundle sha under the same version (a republish, which newer pilotctl +detects via the `.bundle-sha256` it records). Each node runs the upgrade with +**its own installed pilotctl**, so the upgrade behaves the way the oldest +pilotctl in the fleet does. + +## Stateful apps: release freeze (CI lint) + +Apps keep their state inside their install dir (`$APP` = `~/.pilot/apps//`): +the wallet's `identity-evm.json` (its EVM private key) and `data.db`, smol's +`secrets.json`, each metered app's `identity.json`, the `cap-state.jsonl` +spend-cap ledger and `supervisor.log`. A pilotctl **without** the app-state +fix (it landed with the "appstore: keep app state across install --force and +upgrade" change) replaces that dir on every `install --force` and every +`upgrade` and deletes it, keys included. A catalogue update for a stateful app +therefore wipes that app's state on every node still running an older +pilotctl, within the hour, with no prompt. + +So every PR that touches `catalogue/` runs the **catalogue-lint** job +(`.github/workflows/catalogue-lint.yml`, code in `catalogue/lint/`). It +compares the PR's catalogue with its base and **fails** when an update (new +version or same-version republish) targets a stateful app: + +- an app listed in `catalogue/stateful-apps.json` (`stateful_apps`: wallet, + smol, agentphone, bowmark, orthogonal), or +- any app whose old or new bundle manifest grants `fs.write` or `key.sign` + (it writes files into `$APP`, or signs with its own identity key). A bundle + that cannot be downloaded to check counts as stateful. + +**Hold the release** until the fleet runs the fixed pilotctl. To ship one +anyway (the fleet has caught up, or the release is urgent and the risk is +accepted), approve that exact version, one of two ways: + +1. **Approval file (preferred, stays in history):** add an entry to + `approved_bumps` in `catalogue/stateful-apps.json` in the same PR: + ```json + {"id": "io.pilot.wallet", "version": "0.3.4", + "reason": "fleet runs the fixed pilotctl (registry version query, 2026-10-01)", + "approved_by": ""} + ``` + All four fields are required; an approval only covers that id + version. +2. **PR label:** a maintainer applies `catalogue:stateful-bump-approved`. The + job re-runs on label changes and reports the update as a warning. + +Remove the freeze (empty `stateful_apps`, or delete the check) only once the +registry's node-version distribution shows the fleet on a pilotctl with the +fix. + +The same job also checks, for every **added or changed** entry, each bundle it +publishes: the download matches `bundle_sha256`, the manifest's `id` and +`app_version` match the entry (a mismatched version makes every node reinstall +the app every hour), the binary matches the manifest's pin, and the binary runs +on the platform it is published under. An entry **without** a `bundles` map is +installed by every platform, so it must not ship a native (ELF, Mach-O, PE) +binary at all; publish per-platform `bundles` instead. Scripts and portable +adapters are fine in a single bundle. pilotctl enforces the same at install +time: a binary built for another platform is refused with `platform_mismatch` +and nothing is installed. + +Run it locally: + +```bash +git show origin/main:catalogue/catalogue.json > /tmp/base.json +(cd catalogue/lint && GOWORK=off go run . --base /tmp/base.json --head ../catalogue.json) +``` + +### What install and upgrade do with app state (fixed pilotctl) + +- `pilotctl appstore install ` on an installed app changes nothing and + points at `pilotctl appstore upgrade `. With `--version X` for a version + other than the installed one (or a local bundle of another version) it fails + with `conflict` unless `--force` is given, and with `version_unavailable` + when the catalogue does not offer X. +- `install --force` and `upgrade` carry everything in `$APP` that the new + bundle does not ship into the new install, except control files + (`manifest.json`, `install.json`, `install.sh`, `.sideloaded`, `.suspended`, + `.resume`, `.bundle-sha256`, next-steps caches) and sockets. Files are + hard-linked, so writes the still-running app makes to them in place are + kept; read-only dirs carry like any other; an entry this user cannot link + or read (say, a root-owned file) is moved across instead. After the swap + the old dir is checked again: a file the app replaced (write + rename) or + created there during the install is taken into the new install, unless the + new install's copy changed since (the newer write wins). A write the old + process makes after that through a path relative to its working directory + (not through `$APP`) still lands in the backup, until the supervisor + restarts it on the new version (within ~30s). The old dir stays at + `.previous` until the new one verifies. +- Installs, upgrades and uninstalls of one app take a lock + (`/..lock`), so the hourly `upgrade --all` and an agent's + `install` never interleave. +- The replaced dir is kept as a backup in `app-backups//` beside the + install root (`~/.pilot/app-backups`, or `$PILOT_APPSTORE_BACKUP_ROOT`, + which may be on another filesystem: it is then copied). If that location is + unusable, the backup goes to the default location, then to + `/.app-backups//`, and pilotctl warns. Each backup has a + `.pilot-backup.json` saying what kind it is. Routine backups are rotated + (the newest 3 upgrades and the newest 3 same-version reinstalls per app); + a backup that holds the only copy of state (`--reset-state`, state that + could not be carried, a crash leftover) is never removed automatically. + `uninstall` leaves backups and lists every one of them. +- `install --reset-state` (implies `--force`) is the explicit way to start an + app empty. It warns loudly and still keeps the backup. +- `upgrade --all` goes on to the next app when one fails, and exits 1 at the + end naming the apps that were not upgraded. + ## Catalogue signing key The catalogue is signed with a dedicated ed25519 key, separate from any diff --git a/catalogue/lint/go.mod b/catalogue/lint/go.mod new file mode 100644 index 00000000..41e043f0 --- /dev/null +++ b/catalogue/lint/go.mod @@ -0,0 +1,3 @@ +module github.com/pilot-protocol/pilotprotocol/catalogue/lint + +go 1.25 diff --git a/catalogue/lint/main.go b/catalogue/lint/main.go new file mode 100644 index 00000000..3c9d7473 --- /dev/null +++ b/catalogue/lint/main.go @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command catalogue-lint gates changes to catalogue/catalogue.json in CI. It +// compares the catalogue at the PR's base with the PR's head and fails on: +// +// 1. Stateful-app release freeze (overridable). Every node upgrades its +// installed apps hourly (`pilotctl appstore upgrade --all`) with the +// pilotctl it already has. Until a node runs a pilotctl that carries the +// app-state fix, that upgrade DELETES the app's saved state: the wallet's +// EVM key and payment DB, smol's secrets, per-app identities. So any update +// of a stateful app (a new version, or a same-version republish, which +// newer pilotctl also upgrades to) is refused unless it is approved in +// catalogue/stateful-apps.json or the PR carries the approval label. An app +// is stateful when it is listed in stateful-apps.json or when its bundle +// manifest (old or new) grants fs.write or key.sign. +// +// 2. Bundle checks (not overridable) for every added or changed entry: each +// published bundle downloads, matches its pinned sha256, carries a manifest +// whose id and app_version match the entry and whose binary matches its +// pinned sha256, and its binary can run on the platform it is published +// for. A legacy single-bundle entry (no `bundles` map) is installed by every +// platform, so it must not ship a native binary at all. +// +// Usage: +// +// catalogue-lint --base base.json --head catalogue/catalogue.json \ +// [--policy catalogue/stateful-apps.json] [--allow-stateful-bumps] [--offline] +package main + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "debug/elf" + "debug/macho" + "debug/pe" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "strings" + "time" +) + +// Platforms pilotctl resolves a `bundles` entry for (see resolveBundle in +// cmd/pilotctl/appstore_catalogue.go) and the release matrix builds. +var knownPlatforms = []string{"darwin/amd64", "darwin/arm64", "linux/amd64", "linux/arm64"} + +const ( + approvalLabel = "catalogue:stateful-bump-approved" + maxBundleBytes = 128 << 20 // same cap pilotctl applies to a download + maxEntryBytes = 64 << 20 // same per-file cap pilotctl applies to an extract +) + +type catalogue struct { + Version int `json:"version"` + Apps []entry `json:"apps"` +} + +type entry struct { + ID string `json:"id"` + Version string `json:"version"` + BundleURL string `json:"bundle_url"` + BundleSHA string `json:"bundle_sha256"` + Bundles map[string]variant `json:"bundles,omitempty"` + RenamedTo string `json:"renamed_to,omitempty"` +} + +type variant struct { + BundleURL string `json:"bundle_url"` + BundleSHA string `json:"bundle_sha256"` +} + +type policy struct { + StatefulApps []string `json:"stateful_apps"` + ApprovedBumps []approval `json:"approved_bumps"` +} + +type approval struct { + ID string `json:"id"` + Version string `json:"version"` + Reason string `json:"reason"` + ApprovedBy string `json:"approved_by"` +} + +type manifest struct { + ID string `json:"id"` + AppVersion string `json:"app_version"` + Binary struct { + Path string `json:"path"` + SHA256 string `json:"sha256"` + } `json:"binary"` + Grants []struct { + Cap string `json:"cap"` + Target string `json:"target"` + } `json:"grants"` +} + +// finding is one lint result. Warnings never fail the run. +type finding struct { + Warning bool + AppID string + Title string + Msg string +} + +type linter struct { + policy policy + allowBump bool + offline bool + fetch func(rawURL string) (io.ReadCloser, error) + cache map[string]*bundleInfo +} + +type bundleInfo struct { + manifest *manifest + binary binaryFormat + binarySHA string + fetchError error +} + +func main() { + base := flag.String("base", "", "catalogue.json at the PR base (omit or empty file: every entry is new)") + head := flag.String("head", "catalogue/catalogue.json", "catalogue.json at the PR head") + policyPath := flag.String("policy", "", "stateful-apps.json (default: beside --head)") + allow := flag.Bool("allow-stateful-bumps", false, "the PR carries the "+approvalLabel+" label: report stateful updates as warnings") + offline := flag.Bool("offline", false, "skip bundle downloads (stateful detection uses the policy list only; bundle checks are skipped)") + flag.Parse() + + if *policyPath == "" { + *policyPath = filepath.Join(filepath.Dir(*head), "stateful-apps.json") + } + l, err := newLinter(*policyPath, *allow, *offline) + if err != nil { + fmt.Fprintf(os.Stderr, "catalogue-lint: %v\n", err) + os.Exit(2) + } + baseCat, err := loadCatalogue(*base, true) + if err != nil { + fmt.Fprintf(os.Stderr, "catalogue-lint: base: %v\n", err) + os.Exit(2) + } + headCat, err := loadCatalogue(*head, false) + if err != nil { + fmt.Fprintf(os.Stderr, "catalogue-lint: head: %v\n", err) + os.Exit(2) + } + findings := l.lint(baseCat, headCat) + os.Exit(report(os.Stdout, findings, os.Getenv("GITHUB_ACTIONS") == "true")) +} + +func newLinter(policyPath string, allow, offline bool) (*linter, error) { + raw, err := os.ReadFile(policyPath) // #nosec G304 -- CI tool reading the repo's own policy file + if err != nil { + return nil, fmt.Errorf("read policy: %w", err) + } + var p policy + if err := json.Unmarshal(raw, &p); err != nil { + return nil, fmt.Errorf("parse policy %s: %w", policyPath, err) + } + return &linter{policy: p, allowBump: allow, offline: offline, fetch: openURL, cache: map[string]*bundleInfo{}}, nil +} + +func loadCatalogue(p string, optional bool) (*catalogue, error) { + if p == "" && optional { + return &catalogue{}, nil + } + raw, err := os.ReadFile(p) // #nosec G304 -- CI tool reading a catalogue it was pointed at + if err != nil { + return nil, err + } + if optional && len(strings.TrimSpace(string(raw))) == 0 { + return &catalogue{}, nil + } + var c catalogue + if err := json.Unmarshal(raw, &c); err != nil { + return nil, fmt.Errorf("parse %s: %w", p, err) + } + return &c, nil +} + +// lint runs every check and returns the findings sorted by app id. +func (l *linter) lint(base, head *catalogue) []finding { + var out []finding + for _, a := range l.policy.ApprovedBumps { + if a.ID == "" || a.Version == "" || strings.TrimSpace(a.Reason) == "" || strings.TrimSpace(a.ApprovedBy) == "" { + out = append(out, finding{AppID: a.ID, Title: "incomplete stateful-bump approval", + Msg: fmt.Sprintf("approved_bumps entry %+v needs id, version, reason and approved_by", a)}) + } + } + baseByID := map[string]entry{} + for _, e := range base.Apps { + baseByID[e.ID] = e + } + seen := map[string]bool{} + for _, e := range head.Apps { + if seen[e.ID] { + out = append(out, finding{AppID: e.ID, Title: "duplicate catalogue id", Msg: fmt.Sprintf("%s appears more than once in the catalogue", e.ID)}) + } + seen[e.ID] = true + old, existed := baseByID[e.ID] + if existed && reflect.DeepEqual(old, e) { + continue // untouched entry + } + if tombstone(e) { + continue // not installable; nothing to check + } + out = append(out, l.checkBundles(e)...) + if existed { + if reason := updateTrigger(old, e); reason != "" { + out = append(out, l.checkStatefulUpdate(old, e, reason)...) + } + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].AppID < out[j].AppID }) + return out +} + +func tombstone(e entry) bool { return e.BundleURL == "" && len(e.Bundles) == 0 } + +// resolved mirrors pilotctl's resolveBundle for one platform. +func resolved(e entry, plat string) variant { + if len(e.Bundles) == 0 { + return variant{e.BundleURL, e.BundleSHA} + } + return e.Bundles[plat] +} + +// updateTrigger says why nodes that have base installed will reinstall head, +// or "" when they will not. A new version triggers every pilotctl; a changed +// bundle sha under the same version triggers pilotctl's same-version +// republish detection. A platform that had no bundle has no installs to touch. +func updateTrigger(base, head entry) string { + if base.Version != head.Version { + return fmt.Sprintf("version %s → %s", base.Version, head.Version) + } + var plats []string + for _, p := range knownPlatforms { + b, h := resolved(base, p), resolved(head, p) + if b.BundleSHA != "" && h.BundleSHA != "" && b.BundleSHA != h.BundleSHA { + plats = append(plats, p) + } + } + if len(plats) > 0 { + return fmt.Sprintf("same-version republish of %s (new bundle for %s)", head.Version, strings.Join(plats, ", ")) + } + return "" +} + +func (l *linter) checkStatefulUpdate(base, head entry, trigger string) []finding { + why := l.statefulReason(base, head) + if why == "" { + return nil + } + for _, a := range l.policy.ApprovedBumps { + if a.ID == head.ID && a.Version == head.Version && strings.TrimSpace(a.Reason) != "" && strings.TrimSpace(a.ApprovedBy) != "" { + return []finding{{Warning: true, AppID: head.ID, Title: "approved stateful-app update", + Msg: fmt.Sprintf("%s: %s is approved in stateful-apps.json by %s (%s)", head.ID, trigger, a.ApprovedBy, a.Reason)}} + } + } + msg := fmt.Sprintf("%s: %s updates a stateful app (%s). Every node applies it within the hour via `pilotctl appstore upgrade --all`, "+ + "and a node whose pilotctl predates the app-state fix DELETES the app's saved state (keys, data.db, secrets) while doing so. "+ + "Hold this release until the fleet runs the fixed pilotctl, or approve it: add {\"id\":%q,\"version\":%q,\"reason\":...,\"approved_by\":...} "+ + "to approved_bumps in catalogue/stateful-apps.json, or apply the PR label %q. See catalogue/README.md.", + head.ID, trigger, why, head.ID, head.Version, approvalLabel) + if l.allowBump { + return []finding{{Warning: true, AppID: head.ID, Title: "stateful-app update (approved by label)", Msg: msg}} + } + return []finding{{AppID: head.ID, Title: "stateful-app update needs approval", Msg: msg}} +} + +// statefulReason returns why an app counts as stateful, or "". +func (l *linter) statefulReason(base, head entry) string { + for _, id := range l.policy.StatefulApps { + if id == head.ID { + return "listed in catalogue/stateful-apps.json" + } + } + if l.offline { + return "" + } + for _, e := range []entry{base, head} { + v, plat, ok := anyBundle(e) + if !ok { + continue + } + info := l.inspect(v) + if info.fetchError != nil { + return fmt.Sprintf("its %s bundle for %s could not be inspected (%v), so it is treated as stateful", e.Version, plat, info.fetchError) + } + if g := persistentGrant(info.manifest); g != "" { + return fmt.Sprintf("its %s manifest grants %s", e.Version, g) + } + } + return "" +} + +// persistentGrant names the first grant through which an app keeps state in +// $APP across restarts: fs.write (files it writes) or key.sign (its identity key). +func persistentGrant(m *manifest) string { + for _, g := range m.Grants { + if g.Cap == "fs.write" || g.Cap == "key.sign" { + return g.Cap + " " + g.Target + } + } + return "" +} + +func anyBundle(e entry) (variant, string, bool) { + if len(e.Bundles) == 0 { + return variant{e.BundleURL, e.BundleSHA}, "every platform", e.BundleURL != "" + } + keys := make([]string, 0, len(e.Bundles)) + for k := range e.Bundles { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if e.Bundles[k].BundleURL != "" { + return e.Bundles[k], k, true + } + } + return variant{}, "", false +} + +// checkBundles downloads each bundle of an added or changed entry and checks +// it against its pin, the entry, and the platform it is published for. +func (l *linter) checkBundles(e entry) []finding { + var out []finding + fail := func(title, format string, args ...any) { + out = append(out, finding{AppID: e.ID, Title: title, Msg: e.ID + " " + e.Version + ": " + fmt.Sprintf(format, args...)}) + } + type target struct { + plat string // "" = legacy single bundle, installed on every platform + v variant + } + var targets []target + if len(e.Bundles) == 0 { + targets = append(targets, target{"", variant{e.BundleURL, e.BundleSHA}}) + } else { + keys := make([]string, 0, len(e.Bundles)) + for k := range e.Bundles { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if !contains(knownPlatforms, k) { + fail("unknown bundle platform", "bundles key %q is not a platform pilotctl selects (%s)", k, strings.Join(knownPlatforms, ", ")) + continue + } + targets = append(targets, target{k, e.Bundles[k]}) + } + } + for _, t := range targets { + where := t.plat + if where == "" { + where = "bundle_url" + } + if !isSHA256(t.v.BundleSHA) { + fail("bad bundle pin", "%s: bundle_sha256 %q is not a sha256", where, t.v.BundleSHA) + continue + } + if l.offline { + continue + } + info := l.inspect(t.v) + if info.fetchError != nil { + fail("bundle does not verify", "%s: %v", where, info.fetchError) + continue + } + m := info.manifest + if m.ID != e.ID { + fail("bundle does not match entry", "%s: manifest id is %q", where, m.ID) + } + if m.AppVersion != e.Version { + fail("bundle does not match entry", "%s: manifest app_version is %q; nodes would see the app as outdated forever and reinstall it every hour", where, m.AppVersion) + } + if info.binarySHA != m.Binary.SHA256 { + fail("bundle does not match entry", "%s: binary %s has sha256 %s but the manifest pins %s; pilotctl refuses to install it", where, m.Binary.Path, info.binarySHA, m.Binary.SHA256) + } + bf := info.binary + switch { + case !bf.Native: + // Scripts and adapters are portable; the OS decides. + case t.plat == "": + fail("single-platform binary in a legacy bundle", + "bundle_url ships a native binary (%s: %s) with no per-platform `bundles` map, so every platform installs it and it only runs on %s. Publish a `bundles` map (catalogue/README.md)", + m.Binary.Path, bf.Desc, strings.Join(bf.Platforms(), ", ")) + case !bf.RunsOn(t.plat): + fail("bundle binary is for another platform", "%s: binary %s is %s", t.plat, m.Binary.Path, bf.Desc) + } + } + return out +} + +// inspect downloads a bundle (once per sha), verifies its pin and reads its +// manifest and binary. +func (l *linter) inspect(v variant) *bundleInfo { + key := v.BundleURL + "#" + v.BundleSHA + if info, ok := l.cache[key]; ok { + return info + } + info := &bundleInfo{} + info.manifest, info.binary, info.binarySHA, info.fetchError = l.readBundle(v) + l.cache[key] = info + return info +} + +func (l *linter) readBundle(v variant) (*manifest, binaryFormat, string, error) { + var none binaryFormat + body, err := l.fetch(v.BundleURL) + if err != nil { + return nil, none, "", fmt.Errorf("fetch %s: %w", v.BundleURL, err) + } + defer body.Close() + tmp, err := os.MkdirTemp("", "catalogue-lint-*") + if err != nil { + return nil, none, "", err + } + defer os.RemoveAll(tmp) + tarPath := filepath.Join(tmp, "bundle.tar.gz") + f, err := os.Create(tarPath) // #nosec G304 -- fixed name in our own temp dir + if err != nil { + return nil, none, "", err + } + h := sha256.New() + n, err := io.Copy(io.MultiWriter(f, h), io.LimitReader(body, maxBundleBytes+1)) + _ = f.Close() + if err != nil { + return nil, none, "", fmt.Errorf("download %s: %w", v.BundleURL, err) + } + if n > maxBundleBytes { + return nil, none, "", fmt.Errorf("%s is larger than pilotctl's %d-byte download cap", v.BundleURL, maxBundleBytes) + } + if got := hex.EncodeToString(h.Sum(nil)); got != v.BundleSHA { + return nil, none, "", fmt.Errorf("%s has sha256 %s, the catalogue pins %s", v.BundleURL, got, v.BundleSHA) + } + files, err := extract(tarPath, tmp) + if err != nil { + return nil, none, "", fmt.Errorf("unpack %s: %w", v.BundleURL, err) + } + mfPath, ok := files["manifest.json"] + if !ok { + return nil, none, "", errors.New("bundle has no top-level manifest.json") + } + raw, err := os.ReadFile(mfPath) // #nosec G304 -- a file we extracted into our temp dir + if err != nil { + return nil, none, "", err + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + return nil, none, "", fmt.Errorf("parse manifest.json: %w", err) + } + binPath, ok := files[path.Clean(m.Binary.Path)] + if !ok { + return nil, none, "", fmt.Errorf("manifest binary %q is not in the bundle", m.Binary.Path) + } + bin, err := os.ReadFile(binPath) // #nosec G304 -- a file we extracted into our temp dir + if err != nil { + return nil, none, "", err + } + sum := sha256.Sum256(bin) + bf, err := detectBinary(binPath) + if err != nil { + return nil, none, "", err + } + return &m, bf, hex.EncodeToString(sum[:]), nil +} + +// extract writes each regular file of the gzipped tar at tarPath into dir under +// a generated name (never the archive's own path, so a hostile entry name +// cannot escape dir) and returns archive path → extracted file. +func extract(tarPath, dir string) (map[string]string, error) { + f, err := os.Open(tarPath) // #nosec G304 -- our own temp file + if err != nil { + return nil, err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + defer gz.Close() + tr := tar.NewReader(gz) + files := map[string]string{} + for i := 0; ; i++ { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return files, nil + } + if err != nil { + return nil, err + } + if hdr.Typeflag != tar.TypeReg { + continue + } + out := filepath.Join(dir, fmt.Sprintf("entry-%d", i)) + w, err := os.Create(out) // #nosec G304 -- generated name in our own temp dir + if err != nil { + return nil, err + } + n, err := io.Copy(w, io.LimitReader(tr, maxEntryBytes+1)) + _ = w.Close() + if err != nil { + return nil, err + } + if n > maxEntryBytes { + return nil, fmt.Errorf("entry %q exceeds pilotctl's %d-byte extract cap", hdr.Name, maxEntryBytes) + } + files[path.Clean(strings.TrimPrefix(hdr.Name, "./"))] = out + } +} + +// binaryFormat describes an app binary for platform matching. +type binaryFormat struct { + Native bool + OS string + Archs []string + Desc string +} + +func (b binaryFormat) RunsOn(plat string) bool { + for _, p := range b.Platforms() { + if p == plat { + return true + } + } + return false +} + +func (b binaryFormat) Platforms() []string { + var out []string + for _, a := range b.Archs { + out = append(out, b.OS+"/"+a) + } + return out +} + +// detectBinary classifies an executable: native ELF, Mach-O (thin or +// universal) and PE images report their platform; anything else (scripts, +// portable adapters) is not native. +func detectBinary(p string) (binaryFormat, error) { + if f, err := elf.Open(p); err == nil { + defer f.Close() + arch := elfArch(f.Machine) + return binaryFormat{Native: true, OS: "linux", Archs: []string{arch}, Desc: "ELF " + arch}, nil + } + if f, err := macho.OpenFat(p); err == nil { + defer f.Close() + var archs []string + for _, a := range f.Arches { + archs = append(archs, machoArch(a.Cpu)) + } + sort.Strings(archs) + return binaryFormat{Native: true, OS: "darwin", Archs: archs, Desc: "universal Mach-O " + strings.Join(archs, "+")}, nil + } + if f, err := macho.Open(p); err == nil { + defer f.Close() + arch := machoArch(f.Cpu) + return binaryFormat{Native: true, OS: "darwin", Archs: []string{arch}, Desc: "Mach-O " + arch}, nil + } + if f, err := pe.Open(p); err == nil { + defer f.Close() + arch := peArch(f.Machine) + return binaryFormat{Native: true, OS: "windows", Archs: []string{arch}, Desc: "PE " + arch}, nil + } + return binaryFormat{Desc: "portable (script or adapter)"}, nil +} + +func elfArch(m elf.Machine) string { + switch m { + case elf.EM_X86_64: + return "amd64" + case elf.EM_AARCH64: + return "arm64" + case elf.EM_386: + return "386" + case elf.EM_ARM: + return "arm" + } + return m.String() +} + +func machoArch(c macho.Cpu) string { + switch c { + case macho.CpuAmd64: + return "amd64" + case macho.CpuArm64: + return "arm64" + case macho.Cpu386: + return "386" + case macho.CpuArm: + return "arm" + } + return c.String() +} + +func peArch(m uint16) string { + switch m { + case pe.IMAGE_FILE_MACHINE_AMD64: + return "amd64" + case pe.IMAGE_FILE_MACHINE_ARM64: + return "arm64" + case pe.IMAGE_FILE_MACHINE_I386: + return "386" + } + return fmt.Sprintf("machine-%#x", m) +} + +func openURL(raw string) (io.ReadCloser, error) { + u, err := url.Parse(raw) + if err != nil { + return nil, err + } + switch u.Scheme { + case "file": + return os.Open(u.Path) + case "https", "http": + c := &http.Client{Timeout: 5 * time.Minute} + resp, err := c.Get(raw) // #nosec G107 -- CI tool fetching the bundles the catalogue under review pins + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() + return nil, fmt.Errorf("http %d", resp.StatusCode) + } + return resp.Body, nil + } + return nil, fmt.Errorf("unsupported url scheme %q", u.Scheme) +} + +func isSHA256(s string) bool { + if len(s) != 64 { + return false + } + _, err := hex.DecodeString(s) + return err == nil && strings.ToLower(s) == s +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// report prints findings (as GitHub annotations under Actions) and returns the +// exit code: 1 when any finding is an error. +func report(w io.Writer, findings []finding, github bool) int { + errs := 0 + for _, f := range findings { + level := "warning" + if !f.Warning { + level = "error" + errs++ + } + if github { + fmt.Fprintf(w, "::%s file=catalogue/catalogue.json,title=%s::%s\n", level, f.Title, strings.ReplaceAll(f.Msg, "\n", " ")) + } else { + fmt.Fprintf(w, "%s: [%s] %s\n", strings.ToUpper(level), f.Title, f.Msg) + } + } + if errs > 0 { + fmt.Fprintf(w, "catalogue-lint: %d error(s)\n", errs) + return 1 + } + fmt.Fprintf(w, "catalogue-lint: ok (%d warning(s))\n", len(findings)) + return 0 +} diff --git a/catalogue/lint/main_test.go b/catalogue/lint/main_test.go new file mode 100644 index 00000000..0ff8f491 --- /dev/null +++ b/catalogue/lint/main_test.go @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "debug/macho" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +var ( + scriptBin = []byte("#!/bin/sh\necho hi\n") + elfAmd64 = elfHeader(62) + elfArm64 = elfHeader(183) + machArm64 = machHeader(macho.CpuArm64) +) + +func elfHeader(machine uint16) []byte { + h := make([]byte, 64) + copy(h, "\x7fELF") + h[4], h[5], h[6] = 2, 1, 1 // 64-bit, little endian, EV_CURRENT + binary.LittleEndian.PutUint16(h[16:], 2) + binary.LittleEndian.PutUint16(h[18:], machine) + binary.LittleEndian.PutUint32(h[20:], 1) + binary.LittleEndian.PutUint16(h[52:], 64) + binary.LittleEndian.PutUint16(h[54:], 56) + binary.LittleEndian.PutUint16(h[58:], 64) + return h +} + +func machHeader(cpu macho.Cpu) []byte { + h := make([]byte, 32) + binary.LittleEndian.PutUint32(h[0:], macho.Magic64) + binary.LittleEndian.PutUint32(h[4:], uint32(cpu)) + binary.LittleEndian.PutUint32(h[12:], uint32(macho.TypeExec)) + return h +} + +// bundleServer serves generated app bundles over HTTP. +type bundleServer struct { + t *testing.T + srv *httptest.Server + mu sync.Mutex + n int + tgz map[string][]byte +} + +func newBundleServer(t *testing.T) *bundleServer { + b := &bundleServer{t: t, tgz: map[string][]byte{}} + b.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b.mu.Lock() + body, ok := b.tgz[r.URL.Path] + b.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(b.srv.Close) + return b +} + +// publish serves a bundle for id/version with the given grant caps and binary, +// returning its catalogue variant. binSHAOverride, when set, is what the +// manifest pins instead of the binary's real sha. +func (b *bundleServer) publish(id, version string, caps []string, bin []byte, binSHAOverride string) variant { + b.t.Helper() + sum := sha256.Sum256(bin) + binSHA := hex.EncodeToString(sum[:]) + if binSHAOverride != "" { + binSHA = binSHAOverride + } + var grants []map[string]string + for _, c := range caps { + grants = append(grants, map[string]string{"cap": c, "target": "$APP/state"}) + } + mf, _ := json.Marshal(map[string]any{ + "id": id, "app_version": version, "manifest_version": 1, + "binary": map[string]string{"runtime": "go", "path": "bin/app", "sha256": binSHA}, + "grants": grants, + }) + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, f := range []struct { + name string + body []byte + }{{"./manifest.json", mf}, {"bin/app", bin}} { + if err := tw.WriteHeader(&tar.Header{Name: f.name, Size: int64(len(f.body)), Typeflag: tar.TypeReg, Mode: 0o755}); err != nil { + b.t.Fatal(err) + } + _, _ = tw.Write(f.body) + } + _ = tw.Close() + _ = gz.Close() + b.mu.Lock() + b.n++ + p := fmt.Sprintf("/bundle-%d.tar.gz", b.n) + b.tgz[p] = buf.Bytes() + b.mu.Unlock() + tsum := sha256.Sum256(buf.Bytes()) + return variant{BundleURL: b.srv.URL + p, BundleSHA: hex.EncodeToString(tsum[:])} +} + +func legacy(id, version string, v variant) entry { + return entry{ID: id, Version: version, BundleURL: v.BundleURL, BundleSHA: v.BundleSHA} +} + +func testLinter(p policy, allow bool) *linter { + return &linter{policy: p, allowBump: allow, fetch: openURL, cache: map[string]*bundleInfo{}} +} + +func errorsOf(fs []finding) []finding { + var out []finding + for _, f := range fs { + if !f.Warning { + out = append(out, f) + } + } + return out +} + +func hasFinding(fs []finding, warning bool, titleSub, msgSub string) bool { + for _, f := range fs { + if f.Warning == warning && strings.Contains(f.Title, titleSub) && strings.Contains(f.Msg, msgSub) { + return true + } + } + return false +} + +func TestStatefulAppUpdateNeedsApproval(t *testing.T) { + s := newBundleServer(t) + const id = "io.pilot.wallet" + base := &catalogue{Apps: []entry{legacy(id, "0.3.3", s.publish(id, "0.3.3", nil, scriptBin, ""))}} + head := &catalogue{Apps: []entry{legacy(id, "0.3.4", s.publish(id, "0.3.4", nil, scriptBin, ""))}} + listed := policy{StatefulApps: []string{id}} + + got := testLinter(listed, false).lint(base, head) + if len(errorsOf(got)) != 1 || !hasFinding(got, false, "needs approval", "version 0.3.3 → 0.3.4") || + !hasFinding(got, false, "", "listed in catalogue/stateful-apps.json") { + t.Fatalf("listed stateful bump: %+v", got) + } + if code := report(&bytes.Buffer{}, got, false); code != 1 { + t.Fatalf("exit = %d, want 1", code) + } + + // Approval by file entry for this exact version. + approved := listed + approved.ApprovedBumps = []approval{{ID: id, Version: "0.3.4", Reason: "fleet on fixed pilotctl", ApprovedBy: "release-owner"}} + if got := testLinter(approved, false).lint(base, head); len(errorsOf(got)) != 0 || !hasFinding(got, true, "approved", "release-owner") { + t.Fatalf("approved bump: %+v", got) + } + // An approval for a different version does not count. + approved.ApprovedBumps[0].Version = "0.3.5" + if got := testLinter(approved, false).lint(base, head); len(errorsOf(got)) != 1 { + t.Fatalf("approval for another version accepted: %+v", got) + } + // Approval by PR label. + if got := testLinter(listed, true).lint(base, head); len(errorsOf(got)) != 0 || !hasFinding(got, true, "approved by label", id) { + t.Fatalf("label-approved bump: %+v", got) + } +} + +func TestIncompleteApprovalIsAnError(t *testing.T) { + p := policy{ApprovedBumps: []approval{{ID: "io.pilot.wallet", Version: "0.3.4"}}} + if got := testLinter(p, false).lint(&catalogue{}, &catalogue{}); !hasFinding(got, false, "incomplete", "approved_by") { + t.Fatalf("got %+v", got) + } +} + +func TestManifestDeclaredStateIsFrozen(t *testing.T) { + s := newBundleServer(t) + for _, tc := range []struct { + caps []string + stateful bool + }{ + {[]string{"fs.write"}, true}, + {[]string{"net.dial", "key.sign"}, true}, + {[]string{"net.dial", "fs.read", "audit.log"}, false}, + } { + id := "io.test.app" + strings.ReplaceAll(strings.Join(tc.caps, ""), ".", "") + base := &catalogue{Apps: []entry{legacy(id, "1.0.0", s.publish(id, "1.0.0", tc.caps, scriptBin, ""))}} + head := &catalogue{Apps: []entry{legacy(id, "1.1.0", s.publish(id, "1.1.0", tc.caps, scriptBin, ""))}} + got := testLinter(policy{}, false).lint(base, head) + if tc.stateful != (len(errorsOf(got)) == 1 && hasFinding(got, false, "needs approval", "manifest grants")) { + t.Errorf("caps %v: stateful=%v but findings %+v", tc.caps, tc.stateful, got) + } + if !tc.stateful && len(got) != 0 { + t.Errorf("stateless update produced findings: %+v", got) + } + } +} + +func TestSameVersionRepublishOfStatefulAppIsFrozen(t *testing.T) { + s := newBundleServer(t) + const id = "io.pilot.smol" + base := &catalogue{Apps: []entry{legacy(id, "1.2.0", s.publish(id, "1.2.0", nil, scriptBin, ""))}} + head := &catalogue{Apps: []entry{legacy(id, "1.2.0", s.publish(id, "1.2.0", nil, []byte("#!/bin/sh\necho rebuilt\n"), ""))}} + got := testLinter(policy{StatefulApps: []string{id}}, false).lint(base, head) + if !hasFinding(got, false, "needs approval", "same-version republish") { + t.Fatalf("got %+v", got) + } +} + +func TestNewAndUntouchedEntriesAreNotFrozen(t *testing.T) { + s := newBundleServer(t) + const id = "io.pilot.wallet" + e := legacy(id, "0.3.3", s.publish(id, "0.3.3", []string{"fs.write"}, scriptBin, "")) + p := policy{StatefulApps: []string{id}} + if got := testLinter(p, false).lint(&catalogue{}, &catalogue{Apps: []entry{e}}); len(got) != 0 { + t.Fatalf("new stateful app: %+v", got) + } + if got := testLinter(p, false).lint(&catalogue{Apps: []entry{e}}, &catalogue{Apps: []entry{e}}); len(got) != 0 { + t.Fatalf("untouched entry: %+v", got) + } +} + +func TestLegacyBundleMustNotShipANativeBinary(t *testing.T) { + s := newBundleServer(t) + for name, bin := range map[string][]byte{"mach-o arm64": machArm64, "elf amd64": elfAmd64} { + id := "io.test.legacy" + strings.ReplaceAll(strings.ReplaceAll(name, " ", ""), "-", "") + head := &catalogue{Apps: []entry{legacy(id, "0.1.0", s.publish(id, "0.1.0", nil, bin, ""))}} + got := testLinter(policy{}, false).lint(&catalogue{}, head) + if !hasFinding(got, false, "legacy bundle", "every platform installs it") { + t.Errorf("%s legacy bundle: %+v", name, got) + } + } + head := &catalogue{Apps: []entry{legacy("io.test.script", "0.1.0", s.publish("io.test.script", "0.1.0", nil, scriptBin, ""))}} + if got := testLinter(policy{}, false).lint(&catalogue{}, head); len(got) != 0 { + t.Fatalf("portable legacy bundle: %+v", got) + } +} + +func TestPerPlatformBundlesMustMatchTheirPlatform(t *testing.T) { + s := newBundleServer(t) + const id = "io.test.multi" + e := entry{ID: id, Version: "1.0.0", Bundles: map[string]variant{ + "linux/amd64": s.publish(id, "1.0.0", nil, elfAmd64, ""), + "linux/arm64": s.publish(id, "1.0.0", nil, elfArm64, ""), + "darwin/arm64": s.publish(id, "1.0.0", nil, elfAmd64, ""), + "macos/arm64": s.publish(id, "1.0.0", nil, machArm64, ""), + }} + e.BundleURL, e.BundleSHA = e.Bundles["linux/amd64"].BundleURL, e.Bundles["linux/amd64"].BundleSHA + got := testLinter(policy{}, false).lint(&catalogue{}, &catalogue{Apps: []entry{e}}) + if len(errorsOf(got)) != 2 || + !hasFinding(got, false, "another platform", "darwin/arm64: binary bin/app is ELF amd64") || + !hasFinding(got, false, "unknown bundle platform", "macos/arm64") { + t.Fatalf("got %+v", got) + } +} + +func TestBundlesMustVerifyAgainstEntry(t *testing.T) { + s := newBundleServer(t) + good := s.publish("io.test.v", "1.0.0", nil, scriptBin, "") + for name, tc := range map[string]struct { + e entry + want string + }{ + "bundle sha": {legacy("io.test.v", "1.0.0", variant{good.BundleURL, strings.Repeat("0", 64)}), "the catalogue pins"}, + "bad pin": {legacy("io.test.v", "1.0.0", variant{good.BundleURL, "REPLACE_AT_RELEASE_TIME"}), "is not a sha256"}, + "manifest version": {legacy("io.test.v", "1.0.1", good), "reinstall it every hour"}, + "manifest id": {legacy("io.test.other", "1.0.0", good), `manifest id is "io.test.v"`}, + "binary pin": {legacy("io.test.b", "1.0.0", s.publish("io.test.b", "1.0.0", nil, scriptBin, strings.Repeat("a", 64))), "pilotctl refuses to install it"}, + "missing": {legacy("io.test.v", "1.0.0", variant{s.srv.URL + "/gone.tar.gz", good.BundleSHA}), "http 404"}, + } { + got := testLinter(policy{}, false).lint(&catalogue{}, &catalogue{Apps: []entry{tc.e}}) + if !hasFinding(got, false, "", tc.want) { + t.Errorf("%s: want a finding containing %q, got %+v", name, tc.want, got) + } + } +} + +func TestUninspectableUpdateIsTreatedAsStateful(t *testing.T) { + s := newBundleServer(t) + const id = "io.test.opaque" + base := &catalogue{Apps: []entry{legacy(id, "1.0.0", variant{s.srv.URL + "/gone.tar.gz", strings.Repeat("b", 64)})}} + head := &catalogue{Apps: []entry{legacy(id, "1.1.0", s.publish(id, "1.1.0", nil, scriptBin, ""))}} + got := testLinter(policy{}, false).lint(base, head) + if !hasFinding(got, false, "needs approval", "could not be inspected") { + t.Fatalf("got %+v", got) + } +} + +// The committed catalogue and policy parse, list the known stateful apps, and +// produce no findings against themselves (nothing is touched). +func TestRepoCatalogueAndPolicy(t *testing.T) { + l, err := newLinter(filepath.Join("..", "stateful-apps.json"), false, true) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"io.pilot.wallet", "io.pilot.smol", "io.pilot.agentphone", "io.pilot.bowmark", "io.pilot.orthogonal"} { + if !contains(l.policy.StatefulApps, id) { + t.Errorf("stateful-apps.json does not list %s", id) + } + } + cat, err := loadCatalogue(filepath.Join("..", "catalogue.json"), false) + if err != nil { + t.Fatal(err) + } + if len(cat.Apps) == 0 { + t.Fatal("empty catalogue") + } + if got := l.lint(cat, cat); len(got) != 0 { + t.Fatalf("unchanged catalogue produced findings: %+v", got) + } + // Offline, a bump of a listed app is still caught from the policy list. + bumped := &catalogue{Apps: append([]entry(nil), cat.Apps...)} + for i := range bumped.Apps { + if bumped.Apps[i].ID == "io.pilot.wallet" { + bumped.Apps[i].Version = "9.9.9" + } + } + if got := l.lint(cat, bumped); !hasFinding(got, false, "needs approval", "io.pilot.wallet") { + t.Fatalf("offline wallet bump: %+v", got) + } +} + +func TestReportFormats(t *testing.T) { + var buf bytes.Buffer + if code := report(&buf, []finding{{Warning: true, AppID: "a", Title: "t", Msg: "m"}}, true); code != 0 { + t.Fatalf("warnings only: exit %d", code) + } + if !strings.Contains(buf.String(), "::warning file=catalogue/catalogue.json,title=t::m") { + t.Fatalf("annotation: %q", buf.String()) + } + buf.Reset() + if code := report(&buf, []finding{{AppID: "a", Title: "t", Msg: "m"}}, false); code != 1 || !strings.Contains(buf.String(), "ERROR: [t] m") { + t.Fatalf("error: exit %d, %q", code, buf.String()) + } +} + +func TestLoadCatalogueOptionalBase(t *testing.T) { + c, err := loadCatalogue("", true) + if err != nil || len(c.Apps) != 0 { + t.Fatalf("empty base: %v %v", c, err) + } + p := filepath.Join(t.TempDir(), "empty.json") + if err := os.WriteFile(p, nil, 0o600); err != nil { + t.Fatal(err) + } + if c, err := loadCatalogue(p, true); err != nil || len(c.Apps) != 0 { + t.Fatalf("empty base file: %v %v", c, err) + } +} diff --git a/catalogue/stateful-apps.json b/catalogue/stateful-apps.json new file mode 100644 index 00000000..83cafd17 --- /dev/null +++ b/catalogue/stateful-apps.json @@ -0,0 +1,20 @@ +{ + "_comment": [ + "Release freeze for apps that keep state in their install dir ($APP).", + "Until the fleet runs a pilotctl with the app-state fix (install/upgrade carry $APP state forward),", + "the hourly `pilotctl appstore upgrade --all` on older nodes DELETES an app's saved state when the", + "catalogue publishes an update for it (the wallet's EVM key and data.db, smol's secrets.json,", + "per-app identity.json). catalogue/lint (the catalogue-lint CI job) therefore fails any PR that", + "updates an app listed here, or any app whose bundle manifest grants fs.write or key.sign.", + "To ship one anyway, add {id, version, reason, approved_by} to approved_bumps (reviewed in the PR),", + "or apply the PR label catalogue:stateful-bump-approved. See catalogue/README.md." + ], + "stateful_apps": [ + "io.pilot.wallet", + "io.pilot.smol", + "io.pilot.agentphone", + "io.pilot.bowmark", + "io.pilot.orthogonal" + ], + "approved_bumps": [] +} diff --git a/cmd/pilotctl/appstore.go b/cmd/pilotctl/appstore.go index 807bef6e..13067a44 100644 --- a/cmd/pilotctl/appstore.go +++ b/cmd/pilotctl/appstore.go @@ -29,6 +29,7 @@ import ( "net" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -118,13 +119,19 @@ Usage: pilotctl appstore uninstall --yes remove an installed app from the install root pilotctl appstore verify sha256-check a pre-install bundle against its manifest pilotctl appstore catalogue list apps available for one-command install - pilotctl appstore install [--force] - install by catalogue ID (fetches + verifies + extracts) - pilotctl appstore install --local [--force] + pilotctl appstore install [--version ] [--force [--reset-state]] + install by catalogue ID (fetches + verifies + extracts). + already installed: a no-op that points at upgrade + (another --version needs --force: conflict otherwise). + --force reinstalls in place and KEEPS the app's state + (keys, data.db, secrets, cap-state, audit log); + --reset-state (implies --force) starts it empty + pilotctl appstore install --local [--force [--reset-state]] sideload a local bundle (sandbox: fs.read/fs.write under $APP, audit.log; no net, no key.sign, no hooks) pilotctl appstore outdated list installed apps with a newer version in the catalogue - pilotctl appstore upgrade | --all re-install the catalogue's current version (verified; supervisor restarts) + pilotctl appstore upgrade | --all re-install the catalogue's current version (verified; app + state is kept; supervisor restarts) pilotctl appstore gen-key generate a fresh ed25519 publisher keypair; prints the public side pilotctl appstore sign --key sign (or re-sign) a manifest's store.signature so the supervisor accepts it @@ -140,6 +147,11 @@ Usage: default 120s — raise for slow methods) Install root is taken from $PILOT_APPSTORE_ROOT or ~/.pilot/apps. +Every install that replaces an app keeps the replaced dir as a backup under +$PILOT_APPSTORE_BACKUP_ROOT or app-backups// beside the install root +(~/.pilot/app-backups). Routine backups are rotated (the newest 3 upgrades +and 3 same-version reinstalls per app); one that holds the only copy of +state (--reset-state, state that could not be carried) is never removed. ` func appStoreHelp() { @@ -787,6 +799,12 @@ func cmdAppStoreUninstall(args []string) { "the install root layout is corrupt; inspect manually", "%s is not a directory", dir) } + // Not in the middle of an install or upgrade of the same app. + unlock, err := lockAppInstall(root, appID) + if err != nil { + fatalHint("timeout", "wait for the other install or upgrade of this app to finish, then re-run", "%v", err) + } + defer unlock() // Read the manifest BEFORE deleting it so we can snapshot the // binary sha256 + app_version into the uninstall audit record. @@ -823,7 +841,7 @@ func cmdAppStoreUninstall(args []string) { ) var rmErr error for i := 0; i < removeRetries; i++ { - if err := os.RemoveAll(dir); err == nil { + if err := removeAllForce(dir); err == nil { // read-only dirs in $APP (a Go module cache) included rmErr = nil break } else { @@ -849,17 +867,31 @@ func cmdAppStoreUninstall(args []string) { Reason: fmt.Sprintf("actor=%s removed=%s", currentActor(), dir), }) + // Replaced installs are kept as backups (appstore_state.go), wherever + // retireAppDir had to put them, and may hold the app's keys. Uninstall + // leaves them, so it says where every one of them is. + backups := listAppBackups(root, appID) if jsonOutput { - _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + out := map[string]any{ "id": appID, "removed": dir, "daemon_notice": "supervisor's next rescan (≤30s) will cancel the per-app goroutine; manifest already removed", - }) + } + if len(backups) > 0 { + out["backups"] = backups + } + _ = json.NewEncoder(os.Stdout).Encode(out) return } fmt.Printf("removed %s\n", dir) fmt.Println("note: the daemon's supervisor will cancel its per-app goroutine on its next rescan") fmt.Println(" (≤30s); no daemon restart needed") + if len(backups) > 0 { + fmt.Printf("note: %d backup(s) of earlier installs remain; they may hold the app's keys and data. Delete them by hand once you no longer need them:\n", len(backups)) + for _, b := range backups { + fmt.Printf(" %s\n", b) + } + } } // ── verify ───────────────────────────────────────────────────────────── @@ -991,6 +1023,25 @@ type installReport struct { InstalledTo string `json:"installed_to"` BinarySHA256 string `json:"binary_sha256"` DaemonNotice string `json:"daemon_notice"` + + // AlreadyInstalled marks the no-op answer to `install` of an app that is + // already installed without --force; Hint says what to run instead. + AlreadyInstalled bool `json:"already_installed,omitempty"` + Hint string `json:"hint,omitempty"` + // PreservedState lists the app-state paths (relative to the app dir) + // carried from the replaced install into the new one. + PreservedState []string `json:"preserved_state,omitempty"` + // StateReset is true when --reset-state deliberately started the app + // without its previous state. + StateReset bool `json:"state_reset,omitempty"` + // StateNotCarried lists app state that could not be carried into the + // new install; it is only in BackupDir, which is never pruned. + StateNotCarried []string `json:"state_not_carried,omitempty"` + // BackupDir is where the replaced install was kept. + BackupDir string `json:"backup_dir,omitempty"` + // BackupWarning is set when the backup is not where it was configured + // to go (or could not be completed); the same text goes to stderr. + BackupWarning string `json:"backup_warning,omitempty"` } // cmdAppStoreInstall places a verified bundle into the install root. @@ -999,10 +1050,14 @@ type installReport struct { // root never sees a partially-written app dir. // // Steps: -// 1. sha256-check the bundle (same logic as `verify`) -// 2. reject if app already installed unless --force -// 3. stage into /.staging/ -// 4. atomic rename .staging → +// 1. sha256-check the bundle (same logic as `verify`) and refuse a binary +// built for another platform +// 2. if the app is already installed: without --force, a no-op that points +// at `upgrade`; with --force, replace it (state kept, see below) +// 3. stage into /.staging/, carrying the installed app's +// state (appstore_state.go) unless --reset-state +// 4. swap .staging → , keeping the old dir at .previous until the +// new one verifies, then retire it to the backups // // The daemon's supervisor only scans on Start, so an install while the // daemon is running is invisible until the next daemon restart — the @@ -1037,17 +1092,23 @@ func resolveUnder(base, rel string) (string, error) { func cmdAppStoreInstall(args []string) { if len(args) < 1 { fatalHint("invalid_argument", - "usage: pilotctl appstore install [--force] [--local] [--version ]", + "usage: pilotctl appstore install [--force [--reset-state]] [--local] [--version ]", "missing app id or bundle dir") } target := args[0] force := false + resetState := false wantVersion := "" allowLocal := false for i := 1; i < len(args); i++ { switch args[i] { case "--force", "-f": force = true + case "--reset-state": + // The explicit destructive variant: reinstall WITHOUT carrying the + // app's state forward. Implies --force; warns loudly below. + resetState = true + force = true case "--version": // Read the value before advancing so the bound is checked against // the index actually used. @@ -1066,11 +1127,41 @@ func cmdAppStoreInstall(args []string) { allowLocal = true default: fatalHint("invalid_argument", - "available flags: --force, --local, --version", + "available flags: --force, --reset-state, --local, --version", "unknown install flag: %s", args[i]) } } + // Already installed and no --force: answer before downloading anything. + // (A local bundle path is only known to be installed once its manifest + // is read, so that case is answered after validation below.) A pinned + // --version other than the installed one is not answered here: it is + // either unavailable or a replacement that needs --force, both errors. + if !force && !allowLocal && target != "" && !strings.HasPrefix(target, ".") && !strings.ContainsAny(target, `/\`) { + if dir, err := resolveUnder(appStoreRoot(), target); err == nil { + im, _, merr := readInstalledManifest(dir) + if merr != nil { + if _, perr := os.Lstat(dir + appPreviousSuffix); perr == nil { + // No live manifest but a .previous: an install of + // this app died mid-swap, or is mid-swap right now. The + // lock waits for a running one to finish; only then is + // what is left a crash leftover to repair. + if unlock, lerr := lockAppInstall(appStoreRoot(), target); lerr == nil { + if notes, rerr := recoverInterruptedInstall(dir, target); rerr == nil { + printInstallNotes(notes) + } + unlock() + im, _, merr = readInstalledManifest(dir) + } + } + } + if merr == nil && im.ID == target && (wantVersion == "" || wantVersion == im.AppVersion) { + reportAlreadyInstalled(dir, im, target, false) + return + } + } + } + // Resolve `target` to a local bundle dir and a source tag. // Catalogue path = signed, runs the standard signature gate. // Local path = sideload, requires --local AND must satisfy the @@ -1156,28 +1247,76 @@ func cmdAppStoreInstall(args []string) { "run `pilotctl appstore verify` for a side-by-side; this bundle is tampered or built from a different source than the manifest claims", "binary sha256 mismatch: manifest=%s actual=%s", m.Binary.SHA256, got) } - if err := validateHostExecutable(srcBin); err != nil { - fatalHint("platform_mismatch", - "this catalogue bundle is not executable on the current host; use a release that publishes a matching per-platform bundle", - "refusing incompatible app binary: %v", err) + if err := checkAppBinaryPlatform(srcBin); err != nil { + hint := fmt.Sprintf("nothing was installed and any existing install of %s is untouched. The bundle ships a binary for another platform; the publisher needs to publish a per-platform `bundles` entry for %s/%s (see catalogue/README.md)", + m.ID, runtime.GOOS, runtime.GOARCH) + if source == installSourceLocal { + hint = fmt.Sprintf("nothing was installed and any existing install of %s is untouched. Rebuild the bundle's binary for %s/%s", + m.ID, runtime.GOOS, runtime.GOARCH) + } + fatalHint("platform_mismatch", hint, + "refusing to install %s v%s: %s is built for a different platform than this host (%s/%s): %v", + m.ID, m.AppVersion, m.Binary.Path, runtime.GOOS, runtime.GOARCH, err) } root := appStoreRoot() finalDir := filepath.Join(root, m.ID) - stagingDir := finalDir + ".staging" + stagingDir := finalDir + appStagingSuffix + + // Everything from here to the retire of the replaced install runs under + // the app's install lock (appstore_lock.go): a concurrent install or + // upgrade of the same app waits, instead of interleaving its swap with + // ours or mistaking our in-flight swap for a crash leftover. + unlock, err := lockAppInstall(root, m.ID) + if err != nil { + fatalHint("timeout", "wait for the other install, upgrade or uninstall of this app to finish, then re-run", "%v", err) + } + defer unlock() - // 2. Reject existing install unless --force. - if _, err := os.Stat(finalDir); err == nil { - if !force { - fatalHint("conflict", - "app already installed; uninstall first or pass --force to overwrite", - "refusing to overwrite %s without --force", finalDir) + // 2. Already installed? First repair anything a crashed install left + // behind (this can only restore or back up, never delete), then: + // without --force this is a no-op pointing at `upgrade`; with --force + // the existing install is replaced and its state carried over. + notes, err := recoverInterruptedInstall(finalDir, m.ID) + if err != nil { + fatalHint("io_error", + "a previous install of this app was interrupted and could not be repaired automatically; inspect the install root", + "%v", err) + } + printInstallNotes(notes) + replacing := false + var oldManifest *manifest.Manifest + if _, err := os.Lstat(finalDir); err == nil { + replacing = true + if im, _, merr := readInstalledManifest(finalDir); merr == nil { + oldManifest = im + if !force { + // A caller that named a version (--version, or a local + // bundle) other than the installed one asked for a + // replacement: refuse it rather than report success. + if (wantVersion != "" || source == installSourceLocal) && m.AppVersion != im.AppVersion { + how := "pass --force to replace it (app state is kept)" + if source != installSourceLocal { + how += ", or run `pilotctl appstore upgrade " + m.ID + "` for the catalogue's current version" + } + fatalHint("conflict", how, + "%s v%s is installed; refusing to replace it with v%s without --force", m.ID, im.AppVersion, m.AppVersion) + } + reportAlreadyInstalled(finalDir, im, target, source == installSourceLocal) + return + } + } else { + // A dir without a readable manifest is a broken install (e.g. an + // interrupted uninstall). Replace it, keeping whatever state it has. + fmt.Fprintf(os.Stderr, "note: %s has no readable manifest (%v); repairing it with this bundle and keeping its app state\n", finalDir, merr) } } - // 3. Stage atomically. We may have a leftover staging dir from a - // previous crashed install — blow it away unconditionally. - if err := os.RemoveAll(stagingDir); err != nil { + // 3. Stage atomically. recoverInterruptedInstall above already dropped a + // leftover staging dir (it holds only a bundle copy plus links to state + // whose originals are in finalDir), so this only matters when an older + // pilotctl, which takes no lock, is writing one right now. + if err := removeAllForce(stagingDir); err != nil { fatalHint("io_error", "check install root permissions", "clean stale staging dir %s: %v", stagingDir, err) } @@ -1185,11 +1324,10 @@ func cmdAppStoreInstall(args []string) { fatalHint("io_error", "check install root permissions", "mkdir staging %s: %v", stagingDir, err) } - // Write manifest.json (0644 — readable by everyone in the user's group; not secret). - if err := os.WriteFile(filepath.Join(stagingDir, "manifest.json"), raw, 0o644); err != nil { - _ = os.RemoveAll(stagingDir) - fatalHint("io_error", "check install root permissions", "write manifest: %v", err) - } + // manifest.json is written LAST (after the binary and the carried + // state): the supervisor adopts any dir under the install root that + // holds a manifest, so staging must not carry one while still filling. + // // Place the binary at the manifest-declared path inside staging. // Re-apply the containment guard against the staging root: defence // in depth so the write target can't escape even if bundleDir and @@ -1264,37 +1402,93 @@ func cmdAppStoreInstall(args []string) { } } - // 4. Atomic swap. Rename of directories is atomic on Linux/macOS - // as long as the destination does not already exist; with - // --force we have to step aside the previous install first. - if force { - previousDir := finalDir + ".previous" - _ = os.RemoveAll(previousDir) - if _, err := os.Stat(finalDir); err == nil { - if err := os.Rename(finalDir, previousDir); err != nil { - _ = os.RemoveAll(stagingDir) - fatalHint("io_error", "the previous install could not be moved aside", - "rename %s → %s: %v", finalDir, previousDir, err) - } + // Carry the installed app's state (keys, databases, secrets, spend-cap + // ledger, audit log — everything the new bundle does not ship) into + // staging, and check it landed, BEFORE the live dir is touched. Any + // failure here aborts with the existing install exactly as it was. + var ( + carry *appStateCarry + carried []string + oldBinary string + oldVersion string + ) + if oldManifest != nil { + oldBinary, oldVersion = oldManifest.Binary.Path, oldManifest.AppVersion + } + if replacing && !resetState { + carry, err = carryAppState(finalDir, stagingDir, oldBinary) + if err == nil { + carried = carry.Carried + err = verifyCarriedState(stagingDir, carried, false) } - if err := os.Rename(stagingDir, finalDir); err != nil { - // Best-effort restore so we don't leave the user with no app at all. - if err2 := os.Rename(previousDir, finalDir); err2 != nil { - fatalHint("io_error", "rollback also failed — inspect the install root manually", - "rename staged → final: %v; rollback also failed: %v", err, err2) - } - _ = os.RemoveAll(stagingDir) - fatalHint("io_error", "the swap failed but the previous install was restored", - "rename staged → final: %v", err) + if err != nil { + err = withStagingDiscarded(stagingDir, err) + fatalHint("io_error", + "nothing was changed: the existing install and its state are untouched. Fix the error (e.g. free disk space) and re-run", + "carry app state from %s: %v", finalDir, err) } - _ = os.RemoveAll(previousDir) - } else { - if err := os.Rename(stagingDir, finalDir); err != nil { - _ = os.RemoveAll(stagingDir) - fatalHint("io_error", "check install root permissions", - "rename staged → final: %v", err) + if len(carry.Unreadable) > 0 { + fmt.Fprintf(os.Stderr, "note: %d entr(ies) of %s's state cannot be linked or read by this user (%s); they are moved into the new install after the swap instead\n", + len(carry.Unreadable), m.ID, summarizePaths(carry.Unreadable, 4)) } } + if replacing && resetState { + fmt.Fprintf(os.Stderr, "WARNING: --reset-state: %s is being reinstalled WITHOUT its saved state.\n", m.ID) + fmt.Fprintln(os.Stderr, "WARNING: keys (identity*.json), databases (data.db*), secrets, the spend-cap ledger and the") + fmt.Fprintln(os.Stderr, "WARNING: audit log of the current install will NOT be in the new install; the app starts empty.") + fmt.Fprintf(os.Stderr, "WARNING: the current install is kept as a backup under %s, and that backup is never removed automatically.\n", filepath.Join(appStoreBackupRoot(), m.ID)) + } + + // Write manifest.json (0644 — readable by everyone in the user's group; not secret). + if err := os.WriteFile(filepath.Join(stagingDir, "manifest.json"), raw, 0o644); err != nil { // #nosec G306 G703 -- manifest is public metadata read by the daemon's supervisor; stagingDir is appStoreRoot()/.staging (m.ID reverse-DNS validated by m.Validate()), confined to the install root + fatalHint("io_error", "check install root permissions", "write manifest: %v", withStagingDiscarded(stagingDir, err)) + } + + if testHookBeforeSwap != nil { + testHookBeforeSwap(finalDir) + } + // 4. Swap. The live dir is renamed to .previous and kept there + // until the new dir verifies (exact manifest, pinned binary sha, + // carried state); on any failure the previous install is restored. + previousDir, err := swapInAppDir(finalDir, stagingDir, func(dir string) error { + return verifyInstalledApp(dir, raw, m, carried) + }) + if err != nil { + fatalHint("io_error", "check install root permissions and re-run; see the error for the state of the previous install", "%v", err) + } + // 5. Reconcile: bring into the new install what the still-running old + // process changed in the old dir since the carry, and move across the + // entries that could not be linked (appstore_state.go, step 5). + var reconciled stateReconcile + if previousDir != "" && carry != nil { + reconciled = reconcileAppState(previousDir, finalDir, oldBinary, carry) + carried = mergeSortedUnique(carried, reconciled.Updated) + if len(reconciled.Removed) > 0 { + fmt.Fprintf(os.Stderr, "note: dropped %d stale file(s) the running app deleted during the upgrade: %s\n", + len(reconciled.Removed), summarizePaths(reconciled.Removed, 4)) + } + } + // 6. Retire the replaced install out of the install root, where the + // supervisor would otherwise adopt it. Kept as a backup; retention + // never removes one that holds the only copy of something. + backupDir, backupWarning := "", "" + if previousDir != "" { + kind := replacedInstallBackupKind(resetState, reconciled.Leftover, oldVersion, m.AppVersion) + backupDir, err = retireAppDir(previousDir, m.ID, appBackupMeta{ + Kind: kind, + FromVersion: oldVersion, + ToVersion: m.AppVersion, + NotCarried: reconciled.Leftover, + }) + if err != nil { + backupWarning = err.Error() + fmt.Fprintf(os.Stderr, "warn: %v\n", err) + } + } + if len(reconciled.Leftover) > 0 { + fmt.Fprintf(os.Stderr, "warn: %d entr(ies) of %s's state could not be carried into the new install and are only in the backup %s (never removed automatically): %s. Copy what the app needs back into %s.\n", + len(reconciled.Leftover), m.ID, backupDir, summarizePaths(reconciled.Leftover, 6), finalDir) + } // Drop a forensic line into the supervisor's audit log so the // install moment is recoverable post-hoc. Without this, the @@ -1328,6 +1522,19 @@ func cmdAppStoreInstall(args []string) { if force { reason += " --force" } + if replacing { + if resetState { + reason += " --reset-state" + } else { + reason += fmt.Sprintf(" state_kept=%d", len(carried)) + } + if len(reconciled.Leftover) > 0 { + reason += fmt.Sprintf(" state_not_carried=%d", len(reconciled.Leftover)) + } + if backupDir != "" { + reason += " backup=" + backupDir + } + } writePilotctlAudit(root, pilotctlAuditEvent{ Event: "installed", AppID: m.ID, @@ -1335,6 +1542,9 @@ func cmdAppStoreInstall(args []string) { SHA256: m.Binary.SHA256, Reason: reason, }) + // Nothing below touches the app dir; let a waiting install of this app + // go ahead instead of waiting out telemetry and the demo fetch. + unlock() // Emit a telemetry event for the successful install. // Consent-gated (telemetry flag, default on). Best-effort: a send @@ -1375,6 +1585,11 @@ func cmdAppStoreInstall(args []string) { InstalledTo: finalDir, BinarySHA256: m.Binary.SHA256, DaemonNotice: "supervisor periodically rescans the install root; this app will be picked up within ~30s (no daemon restart needed)", + PreservedState: carried, + StateNotCarried: reconciled.Leftover, + StateReset: replacing && resetState, + BackupDir: backupDir, + BackupWarning: backupWarning, } if jsonOutput { _ = json.NewEncoder(os.Stdout).Encode(report) @@ -1382,6 +1597,17 @@ func cmdAppStoreInstall(args []string) { } fmt.Printf("installed %s v%s (manifest v%d) → %s\n", report.AppID, report.AppVersion, report.ManifestVersion, report.InstalledTo) + switch { + case replacing && resetState: + fmt.Println("state: RESET — the app starts without its previous state (--reset-state)") + case replacing && len(carried) > 0: + fmt.Printf("state: kept %d file(s) from the previous install (%s)\n", len(carried), summarizePaths(carried, 6)) + case replacing: + fmt.Println("state: the previous install had no app state to keep") + } + if backupDir != "" { + fmt.Printf("backup: the replaced install is kept at %s\n", backupDir) + } if source == installSourceLocal { fmt.Println("mode: SIDELOADED — manifest-level allow-list applied (audit.log, fs.read/$APP, fs.write/$APP).") fmt.Println(" this is NOT an OS sandbox: a malicious binary that ignores its manifest can still") diff --git a/cmd/pilotctl/appstore_lock.go b/cmd/pilotctl/appstore_lock.go new file mode 100644 index 00000000..b3fcd42a --- /dev/null +++ b/cmd/pilotctl/appstore_lock.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "syscall" + "time" +) + +// Per-app install lock. +// +// Installing, upgrading and uninstalling an app rename dirs in the install +// root ( → .previous, .staging → ), and crash recovery +// (recoverInterruptedInstall) reads a lone .previous as the leftover of a +// dead install and puts it back. Without a lock, recovery run by one pilotctl +// (the hourly `upgrade --all`, or an agent's `install `) could take over +// another pilotctl's swap halfway through it. Every step that looks at or +// changes those dirs therefore runs under an exclusive flock on +// /..lock. The lock file is a plain file: the supervisor and +// `list` only look at dirs. It is never deleted (deleting a lock file races +// with the next locker); it is empty and harmless. +// +// flock is released by the kernel when the process exits, so a pilotctl that +// dies or exits through fatalHint never leaves the app locked. + +// appInstallLockWait bounds how long an install waits for another one of the +// same app to finish. Linking tens of thousands of carried files can take a +// minute on a slow filesystem; the fleet installer gives a whole pilotctl run +// 10 minutes. A var so tests can shorten it. +var appInstallLockWait = 5 * time.Minute + +func appInstallLockPath(root, appID string) string { + return filepath.Join(root, "."+appID+".lock") +} + +// lockAppInstall takes the app's install lock, waiting up to +// appInstallLockWait for another holder. It returns the function that +// releases it (safe to call more than once). +func lockAppInstall(root, appID string) (func(), error) { + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, fmt.Errorf("create install root %s: %w", root, err) + } + path := appInstallLockPath(root, appID) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) // #nosec G304 -- /..lock + if err != nil { + return nil, fmt.Errorf("open lock %s: %w", path, err) + } + fd := int(f.Fd()) // #nosec G115 -- a file descriptor always fits an int + deadline := time.Now().Add(appInstallLockWait) + waiting := false + for { + err := syscall.Flock(fd, syscall.LOCK_EX|syscall.LOCK_NB) + if err == nil { + break + } + if !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EINTR) { + _ = f.Close() + return nil, fmt.Errorf("lock %s: %w", path, err) + } + if time.Now().After(deadline) { + _ = f.Close() + return nil, fmt.Errorf("another pilotctl has been installing, upgrading or removing %s for more than %s (lock %s is held)", appID, appInstallLockWait, path) + } + if !waiting { + fmt.Fprintf(os.Stderr, "note: waiting for another pilotctl to finish installing, upgrading or removing %s\n", appID) + waiting = true + } + time.Sleep(50 * time.Millisecond) + } + var once sync.Once + return func() { + once.Do(func() { + _ = syscall.Flock(fd, syscall.LOCK_UN) + _ = f.Close() + }) + }, nil +} diff --git a/cmd/pilotctl/appstore_platform.go b/cmd/pilotctl/appstore_platform.go new file mode 100644 index 00000000..86221faf --- /dev/null +++ b/cmd/pilotctl/appstore_platform.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "debug/macho" + "debug/pe" + "fmt" + "runtime" + "sort" + "strings" +) + +// checkAppBinaryPlatform refuses an app binary that cannot run on this host, +// before anything is staged. validateHostExecutable (executable_platform.go) +// covers thin ELF and Mach-O images; this adds universal (fat) Mach-O and PE, +// which it lets through unchecked. Scripts and unrecognised adapter formats +// are still accepted and left to the OS. +// +// This is what stops a legacy single-`bundle_url` catalogue entry that ships +// one platform's binary (wallet 0.3.3 and cosift 0.1.2: darwin/arm64 Mach-O; +// generallegal 0.1.0: linux/amd64 ELF) from "installing successfully" on every +// other platform and then failing at spawn with an exec-format error. +func checkAppBinaryPlatform(path string) error { + if err := checkFatOrPEPlatform(path, runtime.GOOS, runtime.GOARCH); err != nil { + return err + } + return validateHostExecutable(path) +} + +// checkFatOrPEPlatform checks universal Mach-O and PE images against +// goos/goarch. Any other format (thin ELF/Mach-O, scripts, unknown) returns nil +// for the thin-image check to handle. +func checkFatOrPEPlatform(path, goos, goarch string) error { + if ff, err := macho.OpenFat(path); err == nil { + defer ff.Close() + var slices []string + match := false + for _, a := range ff.Arches { + arch := machoCPUArch(a.Cpu) + slices = append(slices, arch) + if arch == goarch { + match = true + } + } + sort.Strings(slices) + if goos != "darwin" { + return fmt.Errorf("binary format is universal Mach-O/macOS (%s), host is %s/%s", strings.Join(slices, ", "), goos, goarch) + } + if !match { + return fmt.Errorf("universal Mach-O slices (%s) do not include host architecture %s", strings.Join(slices, ", "), goarch) + } + return nil + } + if pf, err := pe.Open(path); err == nil { + defer pf.Close() + arch := peMachineArch(pf.Machine) + if goos != "windows" { + return fmt.Errorf("binary format is PE/Windows (%s), host is %s/%s", arch, goos, goarch) + } + if arch != goarch { + return fmt.Errorf("PE machine %s does not match host architecture %s", arch, goarch) + } + } + return nil +} + +func machoCPUArch(cpu macho.Cpu) string { + switch cpu { + case macho.CpuAmd64: + return "amd64" + case macho.CpuArm64: + return "arm64" + case macho.Cpu386: + return "386" + case macho.CpuArm: + return "arm" + default: + return fmt.Sprintf("cpu %#x", uint32(cpu)) + } +} + +func peMachineArch(machine uint16) string { + switch machine { + case pe.IMAGE_FILE_MACHINE_AMD64: + return "amd64" + case pe.IMAGE_FILE_MACHINE_ARM64: + return "arm64" + case pe.IMAGE_FILE_MACHINE_I386: + return "386" + default: + return fmt.Sprintf("machine %#x", machine) + } +} diff --git a/cmd/pilotctl/appstore_platform_test.go b/cmd/pilotctl/appstore_platform_test.go new file mode 100644 index 00000000..42ecb72a --- /dev/null +++ b/cmd/pilotctl/appstore_platform_test.go @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "debug/macho" + "debug/pe" + "encoding/binary" + "encoding/json" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// thinMachO64 is a minimal parseable 64-bit Mach-O header with no load commands. +func thinMachO64(cpu macho.Cpu) []byte { + h := make([]byte, 32) + binary.LittleEndian.PutUint32(h[0:], macho.Magic64) + binary.LittleEndian.PutUint32(h[4:], uint32(cpu)) + binary.LittleEndian.PutUint32(h[12:], uint32(macho.TypeExec)) + return h +} + +// fatMachO builds a universal Mach-O with one thin slice per cpu. +func fatMachO(cpus ...macho.Cpu) []byte { + const align = 0x1000 + out := make([]byte, align*(len(cpus)+1)) + binary.BigEndian.PutUint32(out[0:], macho.MagicFat) + binary.BigEndian.PutUint32(out[4:], uint32(len(cpus))) + for i, cpu := range cpus { + off := uint32(align * (i + 1)) + slice := thinMachO64(cpu) + entry := out[8+20*i:] + binary.BigEndian.PutUint32(entry[0:], uint32(cpu)) + binary.BigEndian.PutUint32(entry[8:], off) + binary.BigEndian.PutUint32(entry[12:], uint32(len(slice))) + binary.BigEndian.PutUint32(entry[16:], 12) // 2^12 alignment + copy(out[off:], slice) + } + return out +} + +// minimalPE builds a parseable PE image header for machine. +func minimalPE(machine uint16) []byte { + b := make([]byte, 0x80+24) + copy(b, "MZ") + binary.LittleEndian.PutUint32(b[0x3c:], 0x80) + copy(b[0x80:], "PE\x00\x00") + binary.LittleEndian.PutUint16(b[0x84:], machine) + return b +} + +func writeBin(t *testing.T, body []byte) string { + t.Helper() + p := filepath.Join(t.TempDir(), "app") + if err := os.WriteFile(p, body, 0o755); err != nil { + t.Fatal(err) + } + return p +} + +func TestCheckFatOrPEPlatformUniversalMachO(t *testing.T) { + t.Parallel() + both := writeBin(t, fatMachO(macho.CpuAmd64, macho.CpuArm64)) + if err := checkFatOrPEPlatform(both, "darwin", "arm64"); err != nil { + t.Fatalf("universal amd64+arm64 on darwin/arm64: %v", err) + } + if err := checkFatOrPEPlatform(both, "darwin", "amd64"); err != nil { + t.Fatalf("universal amd64+arm64 on darwin/amd64: %v", err) + } + if err := checkFatOrPEPlatform(both, "linux", "arm64"); err == nil || !strings.Contains(err.Error(), "universal Mach-O/macOS") { + t.Fatalf("universal Mach-O on linux: %v", err) + } + armOnly := writeBin(t, fatMachO(macho.CpuArm64)) + if err := checkFatOrPEPlatform(armOnly, "darwin", "amd64"); err == nil || !strings.Contains(err.Error(), "do not include host architecture amd64") { + t.Fatalf("arm64-only universal on darwin/amd64: %v", err) + } +} + +func TestCheckFatOrPEPlatformPE(t *testing.T) { + t.Parallel() + exe := writeBin(t, minimalPE(pe.IMAGE_FILE_MACHINE_AMD64)) + if err := checkFatOrPEPlatform(exe, "windows", "amd64"); err != nil { + t.Fatalf("PE amd64 on windows/amd64: %v", err) + } + for _, host := range [][2]string{{"linux", "amd64"}, {"darwin", "arm64"}} { + if err := checkFatOrPEPlatform(exe, host[0], host[1]); err == nil || !strings.Contains(err.Error(), "PE/Windows") { + t.Fatalf("PE on %s/%s: %v", host[0], host[1], err) + } + } + if err := checkFatOrPEPlatform(exe, "windows", "arm64"); err == nil || !strings.Contains(err.Error(), "does not match host architecture arm64") { + t.Fatalf("PE amd64 on windows/arm64: %v", err) + } +} + +func TestCheckFatOrPEPlatformLeavesOtherFormatsToThinCheck(t *testing.T) { + t.Parallel() + for name, body := range map[string][]byte{ + "script": []byte("#!/bin/sh\necho hi\n"), + "thin macho": thinMachO64(macho.CpuArm64), + "java class": append([]byte{0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 0x34}, make([]byte, 64)...), + "unknown": []byte("adapter-v1\n"), + } { + if err := checkFatOrPEPlatform(writeBin(t, body), "linux", "amd64"); err != nil { + t.Errorf("%s: %v", name, err) + } + } +} + +// foreignBinary returns a native executable header for a platform that is +// not this host, and that platform's name. +func foreignBinary() ([]byte, string) { + if runtime.GOOS == "linux" && runtime.GOARCH == "amd64" { + return thinMachO64(macho.CpuArm64), "darwin/arm64" + } + elf := make([]byte, 64) + copy(elf, "\x7fELF") + elf[4], elf[5] = 2, 1 // 64-bit, little endian + binary.LittleEndian.PutUint16(elf[18:], 62) + return elf, "linux/amd64" +} + +// TestAppStoreInstallRefusesForeignPlatformBinary drives the real CLI (the +// refusal exits non-zero) with a bundle shaped like wallet 0.3.3 on Linux: a +// correctly pinned binary for the wrong platform. The install must fail with +// platform_mismatch and leave the existing install and its state untouched. +func TestAppStoreInstallRefusesForeignPlatformBinary(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.foreign" + appDir := filepath.Join(root, id) + good := writeVersionedBundle(t, id, "1.0.0", "v1") + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{good, "--local"}) }) + seedAppState(t, appDir) + + bad := writeVersionedBundle(t, id, "1.0.1", "v2") + body, platform := foreignBinary() + bin := filepath.Join(bad, "bin", "app") + if err := os.WriteFile(bin, body, 0o755); err != nil { + t.Fatal(err) + } + var mf map[string]any + if err := json.Unmarshal([]byte(mustRead(t, filepath.Join(bad, "manifest.json"))), &mf); err != nil { + t.Fatal(err) + } + mf["binary"].(map[string]any)["sha256"] = sha256File(bin) + raw, _ := json.Marshal(mf) + mustWrite(t, filepath.Join(bad, "manifest.json"), string(raw), 0o644) + + env := map[string]string{} + for _, k := range []string{"PILOT_APPSTORE_ROOT", "PILOT_SOCKET", "PILOT_TELEMETRY_URL", "PILOT_APPSTORE_CATALOG_URL"} { + env[k] = os.Getenv(k) + } + _, stderr, code := runCLI(t, []string{"--json", "appstore", "install", bad, "--local", "--force"}, env) + if code == 0 { + t.Fatalf("installing a %s binary on %s/%s succeeded", platform, runtime.GOOS, runtime.GOARCH) + } + var env2 map[string]any + line := strings.TrimSpace(stderr[strings.LastIndex(strings.TrimSpace(stderr), "\n")+1:]) + if err := json.Unmarshal([]byte(line), &env2); err != nil { + t.Fatalf("stderr is not a JSON error envelope: %v\n%s", err, stderr) + } + if env2["code"] != "platform_mismatch" { + t.Fatalf("code = %v, want platform_mismatch\n%s", env2["code"], stderr) + } + msg, _ := env2["message"].(string) + if !strings.Contains(msg, id) || !strings.Contains(msg, runtime.GOOS+"/"+runtime.GOARCH) { + t.Errorf("message should name the app and the host platform: %q", msg) + } + if hint, _ := env2["hint"].(string); !strings.Contains(hint, "untouched") { + t.Errorf("hint should say the existing install is untouched: %q", hint) + } + assertAppStateKept(t, appDir, "after a refused foreign-platform install") + if m, _, err := readInstalledManifest(appDir); err != nil || m.AppVersion != "1.0.0" { + t.Fatalf("installed app changed: %v, %v", m, err) + } +} diff --git a/cmd/pilotctl/appstore_state.go b/cmd/pilotctl/appstore_state.go new file mode 100644 index 00000000..f36cf81b --- /dev/null +++ b/cmd/pilotctl/appstore_state.go @@ -0,0 +1,1324 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +// App state preservation across `appstore install --force` and +// `appstore upgrade` (which the hourly updater runs as `upgrade --all`, and +// which reinstalls through the same --force path). +// +// An installed app keeps its state inside its own directory, // +// ($APP): the wallet's identity-evm.json and data.db, smol's secrets.json, +// each metered app's identity.json, the cap-state.jsonl spend-cap ledger the +// supervisor passes every app, and the supervisor.log audit trail. The install +// swap used to rename that dir to .previous, move the fresh bundle into +// place and then RemoveAll the old dir, so every reinstall and every upgrade +// silently deleted all of it (for the wallet: the EVM private key). +// +// Every step below runs under the app's install lock (appstore_lock.go), so +// two pilotctl processes (the hourly `upgrade --all` and an agent's +// `install`) never interleave their swaps, and one never mistakes the other's +// in-flight swap for a crash leftover. +// +// 1. The new bundle is staged in .staging (binary + aux files). +// 2. Everything in the live dir that the new bundle does not ship, and that +// is not a bundle/pilotctl/supervisor control file, is hard-linked into +// staging (copied when a link is impossible). Directories are created +// owner-writable while they are filled and get their own mode back +// afterwards, so a read-only dir (a Go module cache) carries like any +// other. An entry this user can neither link nor read (a root-owned file +// left by a daemon once run with sudo) cannot be linked or copied; it is +// moved across whole after the swap (step 5). The carried set is checked +// in staging. +// 3. manifest.json is written last, so the supervisor never sees a +// manifest-bearing staging dir that is still being filled. +// 4. The live dir is renamed to .previous and staging to . The new +// dir is verified (manifest bytes, binary sha256, carried state). On any +// failure the previous dir is put back and staging is discarded. +// 5. Reconcile. The old process keeps running until the supervisor restarts +// it (on its next rescan, up to ~30s later). Hard links carry every write +// it makes to an existing file in place, but until the swap its $APP +// still named the old dir, so a file it replaced there (write a temp file, +// rename it over) or created there after step 2 would otherwise exist only +// in the old dir. The old dir is walked again: such files are linked into +// the new dir unless the new dir's copy changed since the carry (then that +// newer write wins), volatile sidecars the app deleted there (SQLite +// -journal/-wal, lock and pid files) are dropped from the new dir too, and +// the entries step 2 could not link are moved across. Writes the old +// process makes after this through $APP land in the new install; a write +// through a path relative to its working directory (which still names the +// old dir) lands in the retired copy until the supervisor restarts it. +// 6. .previous is retired: moved OUT of the install root (the supervisor +// adopts any dir there that holds a manifest, and a same-version .previous +// would win over the live dir at daemon start) into +// //-v/, without the old binary. +// retireAppDir says where it goes when that root is unusable, and +// pruneBackupDirs how long it is kept: routine backups are rotated, a +// backup holding the only copy of something is never removed. +// +// `install --force --reset-state` is the explicit destructive variant: steps 2 +// and 5 are skipped (the new install starts empty) but step 6 still keeps the +// old dir, as a backup retention never removes, and a loud warning names it. + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "syscall" + "time" + + "github.com/pilot-protocol/app-store/pkg/manifest" +) + +const ( + appPreviousSuffix = ".previous" + appStagingSuffix = ".staging" + + // appBackupKeep is how many routine backups of each kind (see + // backupKindRotated) are kept per app and backup location. + appBackupKeep = 3 + + // appBackupDetachMax bounds which backup files are turned into + // independent copies. Carried state is hard-linked, so a retired dir + // shares inodes with the live one; small files (keys, secrets, config) + // are copied so an in-place rewrite by a later version cannot reach the + // backup. Larger files (databases) stay linked: they are protected from + // deletion and replacement, which is the failure this guards against. + appBackupDetachMax = 1 << 20 + + // appBackupMetaName is written into every backup. It records what kind + // of backup it is, which decides whether retention may ever remove it. + appBackupMetaName = ".pilot-backup.json" + + // inRootBackupDirName is the last backup location tried: a dir inside the + // install root, so a backup is a same-filesystem rename even when every + // other location is unusable. It holds no manifest.json and starts with a + // dot, so neither the supervisor nor `list` treats it as an app. + inRootBackupDirName = ".app-backups" +) + +// Backup kinds. Routine ones are rotated; every other kind holds the only +// copy of something and is never removed automatically. +const ( + backupKindUpgrade = "upgrade" // replaced by a different version (routine) + backupKindReinstall = "reinstall" // replaced by the same version (routine) + backupKindResetState = "reset-state" // --reset-state: the only copy of the dropped state + backupKindIncomplete = "incomplete" // holds state that could not be carried into the new install + backupKindRecovered = "recovered" // a leftover found by crash recovery; what it holds is unknown +) + +// backupKindRotated reports the kinds retention may prune. Anything else, +// including a backup without readable metadata, is kept until a person +// removes it. +func backupKindRotated(kind string) bool { + return kind == backupKindUpgrade || kind == backupKindReinstall +} + +// Test seams. Production code never reassigns them. +var ( + linkFile = os.Link // hard-links a carried file + renameForBackup = os.Rename // moves a replaced install into a backup location + testHookBeforeSwap func(liveDir string) + testHookAfterMoveAside func() +) + +// appDirControlFiles are top-level entries of an installed app dir that belong +// to the bundle, to pilotctl or to the supervisor rather than to the app. They +// are never carried into a new install: the new bundle or the install itself +// recreates the ones that should exist. +var appDirControlFiles = map[string]bool{ + "manifest.json": true, // bundle: always the new one + "install.json": true, // bundle aux (native-delivery spec) + "install.sh": true, // bundle aux + manifest.SideloadMarkerName: true, // set per install source; a catalogue reinstall must drop it + ".suspended": true, // supervisor crash-loop marker: a new install starts clean + ".resume": true, // one-shot restart request + bundleSHAMarker: true, // rewritten by every catalogue install + nextStepsFileName: true, // catalogue-derived cache, re-fetched at install + nsCheckedMarker: true, + nsRetryMarker: true, + "app.sock": true, // the running process's socket, recreated at spawn + appBackupMetaName: true, // backup bookkeeping, if a backup was restored by hand +} + +// appStoreBackupRoot is where retired installs are kept: $PILOT_APPSTORE_BACKUP_ROOT, +// or an "app-backups" dir beside the install root (~/.pilot/app-backups for +// the default ~/.pilot/apps). It must be outside the install root, which the +// supervisor scans for apps. +func appStoreBackupRoot() string { + if r := os.Getenv("PILOT_APPSTORE_BACKUP_ROOT"); r != "" { + return r + } + return defaultAppStoreBackupRoot() +} + +func defaultAppStoreBackupRoot() string { + return filepath.Join(filepath.Dir(filepath.Clean(appStoreRoot())), "app-backups") +} + +// backupLocations lists where retired installs go, in the order they are +// tried: the configured backup root, the default one beside the install root +// (when a different one is configured), and last the in-root fallback. +func backupLocations() []string { + locs := []string{appStoreBackupRoot()} + if def := defaultAppStoreBackupRoot(); filepath.Clean(locs[0]) != filepath.Clean(def) { + locs = append(locs, def) + } + return append(locs, filepath.Join(appStoreRoot(), inRootBackupDirName)) +} + +// readInstalledManifest returns the manifest of the app installed at dir, or an +// error when dir holds no readable, parseable manifest. +func readInstalledManifest(dir string) (*manifest.Manifest, []byte, error) { + raw, err := os.ReadFile(filepath.Join(dir, "manifest.json")) // #nosec G304 -- dir is / + if err != nil { + return nil, nil, err + } + m, err := manifest.Parse(raw) + if err != nil { + return nil, nil, err + } + return m, raw, nil +} + +// recoverInterruptedInstall repairs what a crashed or killed install can leave +// behind, before anything else touches the app dir. The caller holds the app's +// install lock, so nothing it finds belongs to an install still in progress. +// It never deletes state: +// +// - .previous without : the swap died after moving the live install +// aside. The previous install is the only copy of the app's state, so it +// is put back. +// - .previous beside : the swap finished but the old dir was never +// retired. It is moved to the backups like any other replaced install. +// - .staging: an install died before its swap. Staging only ever holds +// the new bundle plus links or copies of state whose originals are in the +// live dir, so it is discarded (a manifest-bearing staging dir would +// otherwise be adopted by the supervisor as a second copy of the app). +// +// It returns human-readable notes describing what it did. +func recoverInterruptedInstall(finalDir, appID string) ([]string, error) { + var notes []string + previousDir := finalDir + appPreviousSuffix + if _, err := os.Lstat(previousDir); err == nil { + if _, err := os.Lstat(finalDir); errors.Is(err, fs.ErrNotExist) { + if err := os.Rename(previousDir, finalDir); err != nil { + return nil, fmt.Errorf("restore interrupted install %s → %s: %w", previousDir, finalDir, err) + } + notes = append(notes, fmt.Sprintf("restored %s from an interrupted install (%s)", appID, previousDir)) + } else { + backup, err := retireAppDir(previousDir, appID, appBackupMeta{ + Kind: backupKindRecovered, + Reason: "left behind by an interrupted install", + }) + if backup == "" { + return nil, fmt.Errorf("retire leftover %s: %w", previousDir, err) + } + if err != nil { + fmt.Fprintf(os.Stderr, "warn: %v\n", err) + } + notes = append(notes, fmt.Sprintf("moved a leftover previous install of %s to %s", appID, backup)) + } + } + stagingDir := finalDir + appStagingSuffix + if _, err := os.Lstat(stagingDir); err == nil { + if err := discardStagingDir(stagingDir); err != nil { + return notes, fmt.Errorf("remove the unfinished install %s: %w", stagingDir, err) + } + notes = append(notes, fmt.Sprintf("removed an unfinished install of %s (%s)", appID, stagingDir)) + } + return notes, nil +} + +// ── carry (step 2) ────────────────────────────────────────────────────────── + +// carriedEntry is what one carried file or symlink looked like at carry time, +// in the old dir (src) and in the new one (dst). The post-swap reconcile uses +// it to tell a change the old process made in the old dir (take it) from one +// made in the new dir (keep it). +type carriedEntry struct { + src, dst fs.FileInfo + target string // symlink target +} + +// appStateCarry is the result of carryAppState. +type appStateCarry struct { + // Carried lists the files and symlinks now in the new dir, relative to + // the app dir, sorted. + Carried []string + // Unreadable lists entries this user could neither link nor read. They + // are moved into the new install after the swap (reconcileAppState). + Unreadable []string + + entries map[string]carriedEntry + dirs map[string]bool // dirs created in, or merged into, the new dir +} + +// dirModes remembers the modes of dirs that are made owner-writable while +// they are filled, to restore them afterwards (deepest first). +type dirModes []struct { + path string + perm fs.FileMode +} + +func (m *dirModes) add(path string, perm fs.FileMode) { + *m = append(*m, struct { + path string + perm fs.FileMode + }{path, perm}) +} + +func (m dirModes) restore() { + for i := len(m) - 1; i >= 0; i-- { + _ = os.Chmod(m[i].path, m[i].perm) + } +} + +// withWritableDir runs fn with dir temporarily owner-writable and searchable +// when its mode denies that, and restores the mode afterwards. +func withWritableDir(dir string, fn func() error) error { + fi, err := os.Lstat(dir) + if err != nil || !fi.IsDir() || fi.Mode().Perm()&0o300 == 0o300 { + return fn() + } + if err := os.Chmod(dir, fi.Mode().Perm()|0o700); err != nil { + return fn() + } + defer func() { _ = os.Chmod(dir, fi.Mode().Perm()) }() + return fn() +} + +func cleanRel(rel string) string { + if rel == "" { + return "" + } + return filepath.Clean(filepath.FromSlash(rel)) +} + +// skipStateEntry reports entries that are never app state: top-level control +// files and the old manifest's binary (a stale binary is not state). +func skipStateEntry(rel, oldBin string) bool { + topLevel := !strings.ContainsRune(rel, filepath.Separator) + return (topLevel && appDirControlFiles[rel]) || (oldBin != "" && rel == oldBin) +} + +// carryAppState links (or copies) every piece of app state in oldDir into +// newDir: every file, symlink and directory except +// +// - top-level control files (appDirControlFiles), +// - the old manifest's binary (oldBinaryRel), +// - anything the new bundle already placed in newDir (the bundle wins), +// - sockets, pipes, devices and *.sock files. +// +// A file that disappears while it is being carried (a transient journal the +// running app just deleted) is skipped. An entry this user can neither link +// nor read is listed in Unreadable instead of failing the install. Any other +// error (disk full, I/O) fails the carry, and nothing has been changed. +func carryAppState(oldDir, newDir, oldBinaryRel string) (*appStateCarry, error) { + oldBin := cleanRel(oldBinaryRel) + c := &appStateCarry{entries: map[string]carriedEntry{}, dirs: map[string]bool{}} + created := map[string]bool{} + var modes dirModes + defer func() { modes.restore() }() + + err := filepath.WalkDir(oldDir, func(path string, d fs.DirEntry, walkErr error) error { + rel, err := filepath.Rel(oldDir, path) + if err != nil { + return err + } + if walkErr != nil { + switch { + case rel == ".": + return walkErr + case errors.Is(walkErr, fs.ErrNotExist): + return nil + case errors.Is(walkErr, fs.ErrPermission): + // A dir this user cannot list. Drop the empty copy made on + // the first visit: the dir is moved across whole after the + // swap. + if created[rel] { + _ = os.Remove(filepath.Join(newDir, rel)) + delete(created, rel) + delete(c.dirs, rel) + } + c.Unreadable = append(c.Unreadable, rel) + if d != nil && d.IsDir() { + return fs.SkipDir + } + return nil + } + return walkErr + } + if rel == "." { + return nil + } + if skipStateEntry(rel, oldBin) { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + dst := filepath.Join(newDir, rel) + existing, existErr := os.Lstat(dst) + exists := existErr == nil + switch typ := d.Type(); { + case typ.IsDir(): + if exists { + if existing.IsDir() { + c.dirs[rel] = true + return nil // the bundle ships this dir too; merge into it + } + fmt.Fprintf(os.Stderr, "warn: not carrying %s/: the new bundle ships a file at that path\n", rel) + return fs.SkipDir + } + info, err := d.Info() + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fs.SkipDir + } + return err + } + // Owner-writable while it is filled; its own mode is restored + // once the walk is done. + if err := os.Mkdir(dst, 0o700); err != nil { + return fmt.Errorf("carry %s/: %w", rel, err) + } + modes.add(dst, info.Mode().Perm()) + c.dirs[rel] = true + created[rel] = true + case typ&fs.ModeSymlink != 0: + if exists { + return nil + } + src, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + target, err := os.Readlink(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if err := os.Symlink(target, dst); err != nil { // #nosec G122 -- dst is in our staging dir; the walked app dir belongs to the same user, so a path race gains nothing the app could not do itself + return fmt.Errorf("carry %s: %w", rel, err) + } + c.record(rel, src, dst, target) + case typ.IsRegular(): + if exists || strings.HasSuffix(d.Name(), ".sock") { + return nil + } + // Lstat BEFORE the link: if the app replaces the file in + // between, the reconcile sees a changed source and relinks. + src, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if err := linkOrCopy(path, dst); err != nil { + switch { + case errors.Is(err, fs.ErrNotExist): + return nil + case errors.Is(err, fs.ErrPermission): + _ = os.Remove(dst) // never leave a partial copy + c.Unreadable = append(c.Unreadable, rel) + return nil + } + return fmt.Errorf("carry %s: %w", rel, err) + } + c.record(rel, src, dst, "") + } + // Sockets, named pipes and devices are runtime objects, not state. + return nil + }) + sort.Strings(c.Carried) + sort.Strings(c.Unreadable) + return c, err +} + +func (c *appStateCarry) record(rel string, src fs.FileInfo, dst, target string) { + c.Carried = append(c.Carried, rel) + if di, err := os.Lstat(dst); err == nil { + c.entries[rel] = carriedEntry{src: src, dst: di, target: target} + } +} + +// linkOrCopy hard-links src to dst, falling back to a mode-preserving copy when +// the filesystem refuses the link. +func linkOrCopy(src, dst string) error { + if err := linkFile(src, dst); err == nil { + return nil + } else if errors.Is(err, fs.ErrNotExist) { + return err + } + info, err := os.Lstat(src) + if err != nil { + return err + } + return copyFile(src, dst, info.Mode().Perm()) +} + +// volatileStateFile reports files whose lifecycle the running app owns and +// that may legitimately vanish between the carry and the post-swap check +// (SQLite sidecars, lock/pid/temp files). They are carried like everything +// else but not required to still exist after the swap, and one the app deleted +// from the old dir during the swap is dropped from the new dir too (a stale +// SQLite rollback journal would otherwise roll back a committed transaction). +func volatileStateFile(rel string) bool { + name := filepath.Base(rel) + for _, suffix := range []string{"-journal", "-wal", "-shm", ".lock", ".pid", ".tmp", "~"} { + if strings.HasSuffix(name, suffix) { + return true + } + } + return false +} + +// verifyCarriedState checks that every carried path exists under dir. With +// skipVolatile it tolerates volatile files the app may have removed since. +func verifyCarriedState(dir string, carried []string, skipVolatile bool) error { + for _, rel := range carried { + if skipVolatile && volatileStateFile(rel) { + continue + } + if _, err := os.Lstat(filepath.Join(dir, rel)); err != nil { + return fmt.Errorf("carried state %s is missing: %w", rel, err) + } + } + return nil +} + +// verifyInstalledApp is the post-swap check on the new live dir: it holds the +// exact manifest we staged, the binary it pins, and the carried state. +func verifyInstalledApp(dir string, wantManifest []byte, m *manifest.Manifest, carried []string) error { + raw, err := os.ReadFile(filepath.Join(dir, "manifest.json")) // #nosec G304 -- dir is / + if err != nil { + return fmt.Errorf("read installed manifest: %w", err) + } + if string(raw) != string(wantManifest) { + return errors.New("installed manifest does not match the staged one") + } + bin, err := resolveUnder(dir, m.Binary.Path) + if err != nil { + return fmt.Errorf("binary path: %w", err) + } + if got := sha256File(bin); got != m.Binary.SHA256 { + return fmt.Errorf("installed binary sha256 %s does not match the manifest pin %s", got, m.Binary.SHA256) + } + return verifyCarriedState(dir, carried, true) +} + +// ── swap (step 4) ─────────────────────────────────────────────────────────── + +// swapInAppDir moves stagingDir into place at finalDir. A live install at +// finalDir is renamed to finalDir+".previous" first and is left there (never +// deleted) until verify accepts the new dir; on any failure the previous +// install is put back and staging is discarded. It returns the previous dir's +// path when there was one, for the caller to reconcile and retire. +func swapInAppDir(finalDir, stagingDir string, verify func(dir string) error) (string, error) { + previousDir := finalDir + appPreviousSuffix + if _, err := os.Lstat(previousDir); err == nil { + return "", withStagingDiscarded(stagingDir, fmt.Errorf("%s already exists; refusing to overwrite it (it may hold the only copy of the app's state)", previousDir)) + } + hadOld := false + if _, err := os.Lstat(finalDir); err == nil { + if err := os.Rename(finalDir, previousDir); err != nil { + return "", withStagingDiscarded(stagingDir, fmt.Errorf("move the current install aside: %w (the current install is unchanged)", err)) + } + hadOld = true + if testHookAfterMoveAside != nil { + testHookAfterMoveAside() + } + } + placed := false + rollback := func(cause error) error { + if placed { + // Park the rejected new dir back at the staging path; it only + // holds the new bundle plus links to state whose originals are + // in previousDir. + if err := os.Rename(finalDir, stagingDir); err != nil { + if hadOld { + return fmt.Errorf("%w; rollback failed: could not move the rejected new install aside (%v); %s", cause, err, locatePreviousInstall(previousDir, finalDir)) + } + return fmt.Errorf("%w; could not remove the rejected install at %s: %v", cause, finalDir, err) + } + } + var restoreErr error + if hadOld { + restoreErr = os.Rename(previousDir, finalDir) + } + // Staging never holds the only copy of anything, and must not stay + // in the install root with a manifest the supervisor would adopt. + cause = withStagingDiscarded(stagingDir, cause) + if !hadOld { + return cause + } + if restoreErr != nil { + return fmt.Errorf("%w; rollback failed: %v — %s", cause, restoreErr, locatePreviousInstall(previousDir, finalDir)) + } + return fmt.Errorf("%w (the previous install was restored unchanged)", cause) + } + if err := os.Rename(stagingDir, finalDir); err != nil { + return "", rollback(fmt.Errorf("move the new install into place: %w", err)) + } + placed = true + if verify != nil { + if err := verify(finalDir); err != nil { + return "", rollback(fmt.Errorf("verify the new install: %w", err)) + } + } + if !hadOld { + return "", nil + } + return previousDir, nil +} + +// locatePreviousInstall says where the previous install actually is after a +// failed rollback, instead of assuming it is still where the swap left it. +func locatePreviousInstall(previousDir, finalDir string) string { + if _, err := os.Lstat(previousDir); err == nil { + return fmt.Sprintf("the previous install is intact at %s; move it back to %s", previousDir, finalDir) + } + if _, err := os.Lstat(finalDir); err == nil { + return fmt.Sprintf("an install is in place at %s (another process put it back); nothing was deleted", finalDir) + } + return fmt.Sprintf("neither %s nor %s exists; look for the previous install under %s", previousDir, finalDir, appStoreBackupRoot()) +} + +// withStagingDiscarded discards stagingDir and folds a failure to do so into +// cause. +func withStagingDiscarded(stagingDir string, cause error) error { + if err := discardStagingDir(stagingDir); err != nil { + return fmt.Errorf("%w; %v", cause, err) + } + return cause +} + +// discardStagingDir removes a staging dir that is not going to be installed. +// Staging only ever holds the new bundle plus links or copies of state whose +// originals are elsewhere, so removing it loses nothing. When it cannot be +// removed whole, its manifest is disabled so the supervisor never adopts it. +func discardStagingDir(dir string) error { + if _, err := os.Lstat(dir); errors.Is(err, fs.ErrNotExist) { + return nil + } + rmErr := removeAllForce(dir) + if rmErr == nil { + return nil + } + mf := filepath.Join(dir, "manifest.json") + if err := os.Rename(mf, mf+".disabled"); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("could not remove %s (%v) or disable its manifest (%v); remove it by hand before the daemon rescans", dir, rmErr, err) + } + return fmt.Errorf("could not remove %s (%v); its manifest is disabled so the daemon will not load it", dir, rmErr) +} + +// removeAllForce is os.RemoveAll for a tree that may contain read-only dirs +// (a carried Go module cache): when the first attempt fails, every dir in the +// tree is made owner-writable and the removal is retried. +func removeAllForce(dir string) error { + if err := os.RemoveAll(dir); err == nil { + return nil + } + _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() { + if info, ierr := d.Info(); ierr == nil && info.Mode().Perm()&0o700 != 0o700 { + _ = os.Chmod(path, info.Mode().Perm()|0o700) // #nosec G122 -- a staging/backup/app dir of the same user; only adds owner bits so it can be removed + } + } + return nil + }) + return os.RemoveAll(dir) // #nosec G703 -- callers pass install-root staging/previous dirs or backup dirs we created +} + +// ── reconcile (step 5) ────────────────────────────────────────────────────── + +// stateReconcile is the result of reconcileAppState. +type stateReconcile struct { + Updated []string // entries brought into the new install (new, replaced or moved) + Removed []string // volatile files the app deleted from the old dir, dropped from the new one + Leftover []string // state that stayed only in the old dir (its backup keeps it) +} + +// reconcileAppState is step 5 (see the top of this file): after the swap and +// before the old dir is retired, it brings into newDir what the still-running +// old process changed in oldDir since carryAppState, and moves across the +// entries the carry could not link. The new dir's own changes always win. +func reconcileAppState(oldDir, newDir, oldBinaryRel string, c *appStateCarry) stateReconcile { + var r stateReconcile + oldBin := cleanRel(oldBinaryRel) + + // Entries this user cannot link or read: move them whole. A rename needs + // no read access to the entry itself. + unreadable := make(map[string]bool, len(c.Unreadable)) + for _, rel := range c.Unreadable { + unreadable[rel] = true + src := filepath.Join(oldDir, rel) + if err := moveStateEntry(src, filepath.Join(newDir, rel)); err != nil { + if _, lerr := os.Lstat(src); errors.Is(lerr, fs.ErrNotExist) { + continue // the app deleted it since + } + r.Leftover = append(r.Leftover, rel) + continue + } + r.Updated = append(r.Updated, rel) + } + + var modes dirModes + _ = filepath.WalkDir(oldDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if path == oldDir { + return err + } + return nil // vanished, or unreadable: handled above + } + rel, rerr := filepath.Rel(oldDir, path) + if rerr != nil || rel == "." { + return nil + } + if skipStateEntry(rel, oldBin) || unreadable[rel] { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + dst := filepath.Join(newDir, rel) + live, liveErr := os.Lstat(dst) + if d.IsDir() { + if liveErr == nil { + if live.IsDir() { + return nil + } + return fs.SkipDir // the new bundle ships a file at that path + } + if c.dirs[rel] || !errors.Is(liveErr, fs.ErrNotExist) { + return fs.SkipDir // carried, then removed from the new install by the app: keep it removed + } + info, ierr := d.Info() + if ierr != nil { + return fs.SkipDir + } + // Created in the old dir after the carry. + if merr := withWritableDir(filepath.Dir(dst), func() error { return os.Mkdir(dst, 0o700) }); merr != nil { + r.Leftover = append(r.Leftover, rel+string(filepath.Separator)) + return fs.SkipDir + } + modes.add(dst, info.Mode().Perm()) + c.dirs[rel] = true + return nil + } + typ := d.Type() + if !typ.IsRegular() && typ&fs.ModeSymlink == 0 { + return nil + } + if typ.IsRegular() && strings.HasSuffix(d.Name(), ".sock") { + return nil + } + cur, cerr := os.Lstat(path) + if cerr != nil { + return nil + } + e, wasCarried := c.entries[rel] + if !wasCarried { + if !errors.Is(liveErr, fs.ErrNotExist) { + return nil // the bundle ships it, or the app already wrote it in the new install + } + // Created in the old dir after the carry. + if err := placeStateEntry(path, dst, cur); err != nil { + r.Leftover = append(r.Leftover, rel) + return nil + } + r.Updated = append(r.Updated, rel) + return nil + } + if liveErr != nil { + return nil // the app removed it from the new install: keep it removed + } + if !sameStateEntry(live, e.dst) { + return nil // changed in the new install since the carry: that write is newer + } + if sameStateEntry(cur, e.src) && (typ&fs.ModeSymlink == 0 || readlinkIs(path, e.target)) { + return nil // unchanged + } + // Replaced (or, for a copied file, rewritten) in the old dir after + // the carry: take that version. + if err := replaceStateEntry(path, dst, cur); err != nil { + r.Leftover = append(r.Leftover, rel) + return nil + } + r.Updated = append(r.Updated, rel) + return nil + }) + modes.restore() + + // Volatile sidecars the app deleted from the old dir after the carry. + for rel, e := range c.entries { + if !volatileStateFile(rel) { + continue + } + if _, err := os.Lstat(filepath.Join(oldDir, rel)); !errors.Is(err, fs.ErrNotExist) { + continue + } + dst := filepath.Join(newDir, rel) + live, err := os.Lstat(dst) + if err != nil || !sameStateEntry(live, e.dst) { + continue + } + if withWritableDir(filepath.Dir(dst), func() error { return os.Remove(dst) }) == nil { + r.Removed = append(r.Removed, rel) + } + } + sort.Strings(r.Updated) + sort.Strings(r.Removed) + sort.Strings(r.Leftover) + return r +} + +// sameStateEntry reports whether two Lstat results name the same, unmodified +// file: same inode, size and modification time. +func sameStateEntry(a, b fs.FileInfo) bool { + return a != nil && b != nil && os.SameFile(a, b) && a.Size() == b.Size() && a.ModTime().Equal(b.ModTime()) +} + +func readlinkIs(path, target string) bool { + got, err := os.Readlink(path) + return err == nil && got == target +} + +// placeStateEntry puts the file or symlink at src into dst (which does not +// exist): a hard link, a copy, or, for a file this user cannot read, a move. +func placeStateEntry(src, dst string, info fs.FileInfo) error { + return withWritableDir(filepath.Dir(dst), func() error { + if info.Mode()&fs.ModeSymlink != 0 { + target, err := os.Readlink(src) + if err != nil { + return err + } + return os.Symlink(target, dst) + } + err := linkOrCopy(src, dst) + if errors.Is(err, fs.ErrPermission) { + _ = os.Remove(dst) + return moveStateEntry(src, dst) + } + return err + }) +} + +// replaceStateEntry atomically replaces dst with the file or symlink at src. +func replaceStateEntry(src, dst string, info fs.FileInfo) error { + dir := filepath.Dir(dst) + return withWritableDir(dir, func() error { + tmp := filepath.Join(dir, fmt.Sprintf(".pilot-reconcile-%d-%s", time.Now().UnixNano(), filepath.Base(dst))) + if err := placeStateEntry(src, tmp, info); err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, dst); err != nil { + _ = os.Remove(tmp) + return err + } + return nil + }) +} + +// moveStateEntry renames src to dst (which must not exist), making the parent +// dirs (and, for a dir moved to a new parent, the dir itself) temporarily +// owner-writable when their modes deny it. Only entries this user owns, or +// whose parents it can write, can move; anything else stays where it is. +func moveStateEntry(src, dst string) error { + if _, err := os.Lstat(dst); err == nil { + return fmt.Errorf("%s: %w", dst, fs.ErrExist) + } + return withWritableDir(filepath.Dir(src), func() error { + return withWritableDir(filepath.Dir(dst), func() error { + err := os.Rename(src, dst) + if !errors.Is(err, fs.ErrPermission) { + return err + } + // Moving a dir to a new parent rewrites its ".." entry, which + // needs write access to the dir itself. + fi, lerr := os.Lstat(src) + if lerr != nil || !fi.IsDir() || os.Chmod(src, fi.Mode().Perm()|0o700) != nil { + return err + } + err = os.Rename(src, dst) + if err != nil { + _ = os.Chmod(src, fi.Mode().Perm()) + return err + } + _ = os.Chmod(dst, fi.Mode().Perm()) + return nil + }) + }) +} + +// ── backups (step 6) ──────────────────────────────────────────────────────── + +// appBackupMeta is the .pilot-backup.json written into every backup. +type appBackupMeta struct { + AppID string `json:"app_id"` + Kind string `json:"kind"` + FromVersion string `json:"from_version,omitempty"` + ToVersion string `json:"to_version,omitempty"` + // Pinned backups are never removed by retention. + Pinned bool `json:"pinned"` + Reason string `json:"reason,omitempty"` + // NotCarried lists state (relative to the app dir) that is in this + // backup but could not be carried into the install that replaced it. + NotCarried []string `json:"not_carried,omitempty"` + CreatedAt string `json:"created_at"` +} + +// replacedInstallBackupKind classifies the backup of a replaced install. +func replacedInstallBackupKind(resetState bool, leftover []string, fromVersion, toVersion string) string { + switch { + case resetState: + return backupKindResetState + case len(leftover) > 0: + return backupKindIncomplete + case fromVersion == "": + return backupKindRecovered // no readable manifest: a broken install we repaired + case fromVersion != toVersion: + return backupKindUpgrade + default: + return backupKindReinstall + } +} + +var unsafeBackupNameChars = regexp.MustCompile(`[^0-9A-Za-z._+-]`) + +// retireAppDir moves a replaced install out of the install root into +// //[-v]/, strips what is not state (the +// old binary, a dead socket), makes small files independent copies, records +// meta in it, and applies retention to that app's backups there. +// +// The locations are tried in order (backupLocations): the configured backup +// root, the default one beside the install root, and a dot-dir inside the +// install root. A location on another filesystem is written by copying (then +// the original is removed). Only when every location fails does the dir stay +// in the install root, renamed .previous- with its manifest +// disabled so the supervisor can never adopt it; it gets the same stripping, +// metadata and retention. +// +// It returns where the backup is. A non-nil error with a non-empty path is a +// warning (the backup is not where it was configured to go); with an empty +// path, the dir could not be retired at all. +func retireAppDir(previousDir, appID string, meta appBackupMeta) (string, error) { + oldBinary := "" + if m, _, err := readInstalledManifest(previousDir); err == nil { + oldBinary = m.Binary.Path + if meta.FromVersion == "" { + meta.FromVersion = m.AppVersion + } + } + now := time.Now().UTC() + stamp := now.Format("20060102T150405.000000000Z") + name := stamp + if meta.FromVersion != "" { + name += "-v" + unsafeBackupNameChars.ReplaceAllString(meta.FromVersion, "_") + } + meta.AppID = appID + meta.CreatedAt = now.Format(time.RFC3339Nano) + meta.Pinned = !backupKindRotated(meta.Kind) + + root := filepath.Clean(appStoreRoot()) + inRoot := filepath.Join(root, inRootBackupDirName) + var failures []string + for i, loc := range backupLocations() { + if c := filepath.Clean(loc); c != inRoot && (c == root || strings.HasPrefix(c, root+string(filepath.Separator))) { + failures = append(failures, fmt.Sprintf("%s: inside the install root", loc)) + continue + } + dst, err := moveIntoBackupLocation(previousDir, loc, appID, name, oldBinary) + if dst == "" { + failures = append(failures, fmt.Sprintf("%s: %v", loc, err)) + continue + } + finishBackup(dst, oldBinary, meta) + reportPrunedBackups(pruneAppBackups(filepath.Dir(dst), appBackupKeep)) + switch { + case err != nil: + return dst, err + case i > 0: + return dst, fmt.Errorf("could not keep the replaced install of %s in %s (%s); it is kept at %s instead. Fix that location so later backups go there", + appID, filepath.Join(appStoreBackupRoot(), appID), strings.Join(failures, "; "), dst) + } + return dst, nil + } + + parked := previousDir + "-" + stamp + if err := os.Rename(previousDir, parked); err != nil { + parked = previousDir + } + if err := disableManifest(parked); err != nil { + return parked, fmt.Errorf("could not move the replaced install of %s to any backup location (%s), and disabling its manifest failed (%v); remove or move %s by hand before the daemon restarts", + appID, strings.Join(failures, "; "), err, parked) + } + finishBackup(parked, oldBinary, meta) + if parked != previousDir { + reportPrunedBackups(pruneBackupDirs(parkedBackups(root, appID), appBackupKeep)) + } + return parked, fmt.Errorf("could not move the replaced install of %s to any backup location (%s); it is kept at %s with its manifest disabled", + appID, strings.Join(failures, "; "), parked) +} + +// moveIntoBackupLocation moves previousDir to //. Across +// filesystems it copies (without the old binary), then removes the original; +// if only that removal fails, the copy is returned with a warning and the +// remainder is parked with its manifest disabled. +func moveIntoBackupLocation(previousDir, loc, appID, name, oldBinary string) (string, error) { + appBackups, err := resolveUnder(loc, appID) + if err != nil { + return "", err + } + if err := os.MkdirAll(appBackups, 0o700); err != nil { + return "", err + } + dst := filepath.Join(appBackups, name) + for i := 1; ; i++ { // same-timestamp collision on a coarse clock + if _, lerr := os.Lstat(dst); lerr != nil { + break + } + dst = filepath.Join(appBackups, fmt.Sprintf("%s.%d", name, i)) + } + err = renameForBackup(previousDir, dst) + if err == nil { + return dst, nil + } + if !errors.Is(err, syscall.EXDEV) { + return "", err + } + if err := copyTreeForBackup(previousDir, dst, oldBinary); err != nil { + _ = removeAllForce(dst) + return "", fmt.Errorf("copy to another filesystem: %w", err) + } + // The copy is complete. Disable the original's manifest first, so a + // removal that fails halfway can never leave an adoptable dir behind. + _ = disableManifest(previousDir) + if err := removeAllForce(previousDir); err != nil { + rest := previousDir + "-" + time.Now().UTC().Format("20060102T150405.000000000Z") + if rerr := os.Rename(previousDir, rest); rerr != nil { + rest = previousDir + } + return dst, fmt.Errorf("the replaced install was copied to %s but its original could not be removed (%v); what is left of it is at %s with its manifest disabled — remove it by hand", dst, err, rest) + } + return dst, nil +} + +// copyTreeForBackup copies an install dir to dst (which must not exist): +// regular files with their modes, symlinks and dirs. The old binary, sockets +// and other special files are left out, as stripBackup would remove them. +func copyTreeForBackup(src, dst, oldBinaryRel string) error { + oldBin := cleanRel(oldBinaryRel) + var modes dirModes + defer func() { modes.restore() }() + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + info, err := d.Info() + if err != nil { + return err + } + switch typ := d.Type(); { + case typ.IsDir(): + if err := os.Mkdir(target, 0o700); err != nil { + return err + } + modes.add(target, info.Mode().Perm()) + case rel == oldBin: + return nil + case typ&fs.ModeSymlink != 0: + link, err := os.Readlink(path) + if err != nil { + return err + } + return os.Symlink(link, target) // #nosec G122 -- target is in the backup dir being created; the source tree belongs to the same user + case typ.IsRegular(): + return copyFile(path, target, info.Mode().Perm()) + } + return nil + }) +} + +func disableManifest(dir string) error { + mf := filepath.Join(dir, "manifest.json") + if err := os.Rename(mf, mf+".disabled"); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil +} + +// finishBackup strips a retired dir and records its metadata. +func finishBackup(dir, oldBinary string, meta appBackupMeta) { + stripBackup(dir, oldBinary) + raw, err := json.MarshalIndent(meta, "", " ") + if err == nil { + err = os.WriteFile(filepath.Join(dir, appBackupMetaName), append(raw, '\n'), 0o600) // #nosec G306 G703 -- our own backup dir + } + if err != nil { + // Without metadata retention treats the backup as pinned: it is + // kept, never pruned. + fmt.Fprintf(os.Stderr, "warn: could not record backup metadata in %s: %v\n", dir, err) + } +} + +// stripBackup removes what a backup does not need (the old binary, which the +// catalogue re-serves, and any leftover socket) and detaches small files from +// the live install's inodes. Best-effort: a failure leaves a larger backup. +func stripBackup(dir, oldBinaryRel string) { + if oldBinaryRel != "" { + if bin, err := resolveUnder(dir, oldBinaryRel); err == nil { + _ = withWritableDir(filepath.Dir(bin), func() error { return os.Remove(bin) }) + } + } + _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { // #nosec G703 -- dir is a backup retireAppDir just created + if err != nil { + return nil + } + typ := d.Type() + if typ&fs.ModeSocket != 0 { + _ = withWritableDir(filepath.Dir(path), func() error { return os.Remove(path) }) + return nil + } + if !typ.IsRegular() { + return nil + } + info, err := d.Info() + if err != nil || info.Size() > appBackupDetachMax { + return nil + } + if err := withWritableDir(filepath.Dir(path), func() error { return detachFile(path, info.Mode().Perm()) }); err != nil { + fmt.Fprintf(os.Stderr, "warn: backup %s stays hard-linked to the live file: %v\n", path, err) + } + return nil + }) +} + +// detachFile replaces path with an independent copy of itself (same bytes, same +// mode), breaking any hard link to the live install. +func detachFile(path string, perm fs.FileMode) error { + in, err := os.Open(path) // #nosec G304 -- a file inside our own backup dir + if err != nil { + return err + } + defer in.Close() + tmp, err := os.CreateTemp(filepath.Dir(path), ".detach-*") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := io.Copy(tmp, in); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Chmod(tmpName, perm); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, path); err != nil { + _ = os.Remove(tmpName) + return err + } + return nil +} + +func readBackupMeta(dir string) (appBackupMeta, bool) { + var meta appBackupMeta + raw, err := os.ReadFile(filepath.Join(dir, appBackupMetaName)) // #nosec G304 G703 -- a backup dir we list ourselves + if err != nil || json.Unmarshal(raw, &meta) != nil { + return meta, false + } + return meta, true +} + +// pruneAppBackups applies retention to the backups in one app's backup dir +// (names start with a fixed-width UTC timestamp, so name order is age order). +func pruneAppBackups(dir string, keep int) []string { + entries, err := os.ReadDir(dir) + if err != nil { + return nil + } + var paths []string + for _, e := range entries { + if e.IsDir() { + paths = append(paths, filepath.Join(dir, e.Name())) + } + } + sort.Strings(paths) + return pruneBackupDirs(paths, keep) +} + +// parkedBackups lists the .previous- dirs retireAppDir left in +// the install root, oldest first. +func parkedBackups(root, appID string) []string { + entries, err := os.ReadDir(root) + if err != nil { + return nil + } + prefix := appID + appPreviousSuffix + "-" + var paths []string + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), prefix) { + paths = append(paths, filepath.Join(root, e.Name())) + } + } + sort.Strings(paths) + return paths +} + +// pruneBackupDirs keeps the newest keep backups of each routine kind in paths +// (oldest first) and removes the older ones. Backups that are pinned, of any +// other kind, or without readable metadata are never removed: they may be the +// only copy of an app's keys (a --reset-state, state that could not be +// carried, a crash leftover). It returns what it removed. +func pruneBackupDirs(paths []string, keep int) []string { + byKind := map[string][]string{} + for _, p := range paths { + meta, ok := readBackupMeta(p) + if !ok || meta.Pinned || !backupKindRotated(meta.Kind) { + continue + } + byKind[meta.Kind] = append(byKind[meta.Kind], p) + } + kinds := make([]string, 0, len(byKind)) + for k := range byKind { + kinds = append(kinds, k) + } + sort.Strings(kinds) + var removed []string + for _, kind := range kinds { + list := byKind[kind] + for len(list) > keep { + if removeAllForce(list[0]) == nil { + removed = append(removed, list[0]) + } + list = list[1:] + } + } + return removed +} + +func reportPrunedBackups(removed []string) { + for _, p := range removed { + fmt.Fprintf(os.Stderr, "note: removed the older backup %s (the newest %d routine backups of each kind are kept; backups that hold the only copy of an app's state are never removed)\n", p, appBackupKeep) + } +} + +// listAppBackups returns every backup of appID that exists in any backup +// location, including dirs parked in the install root. +func listAppBackups(root, appID string) []string { + var out []string + seen := map[string]bool{} + for _, loc := range backupLocations() { + dir, err := resolveUnder(loc, appID) + if err != nil || seen[dir] { + continue + } + seen[dir] = true + entries, err := os.ReadDir(dir) + if err != nil { + continue + } + for _, e := range entries { + if e.IsDir() { + out = append(out, filepath.Join(dir, e.Name())) + } + } + } + return append(out, parkedBackups(root, appID)...) +} + +// ── reporting ─────────────────────────────────────────────────────────────── + +// reportAlreadyInstalled is the answer to `install` of an app that is already +// installed, without --force: nothing is fetched or changed, and the output +// says how to get a newer version (`upgrade`) or reinstall in place. Both keep +// the app's state. Exit status 0: the app the caller asked for is installed. +func reportAlreadyInstalled(dir string, im *manifest.Manifest, target string, local bool) { + upgrade := "pilotctl appstore upgrade " + im.ID + reinstall := "pilotctl appstore install " + im.ID + " --force" + if local { + reinstall = "pilotctl appstore install " + target + " --local --force" + } + hint := "already installed; nothing changed. " + if !local { + hint += "To move to the catalogue's current version (app state is kept): `" + upgrade + "`. " + } + hint += "To reinstall in place (app state is kept): `" + reinstall + "`." + if jsonOutput { + _ = json.NewEncoder(os.Stdout).Encode(installReport{ + AppID: im.ID, + AppVersion: im.AppVersion, + ManifestVersion: im.ManifestVersion, + InstalledTo: dir, + BinarySHA256: im.Binary.SHA256, + AlreadyInstalled: true, + Hint: hint, + }) + return + } + fmt.Printf("%s v%s is already installed (%s); nothing changed.\n", im.ID, im.AppVersion, dir) + if !local { + fmt.Printf(" newer version from the catalogue (keeps app state): %s\n", upgrade) + } + fmt.Printf(" reinstall in place (keeps app state): %s\n", reinstall) +} + +// printInstallNotes surfaces what recoverInterruptedInstall repaired. +func printInstallNotes(notes []string) { + for _, n := range notes { + fmt.Fprintf(os.Stderr, "note: %s\n", n) + } +} + +// summarizePaths renders up to limit paths for a one-line message. +func summarizePaths(paths []string, limit int) string { + if len(paths) <= limit { + return strings.Join(paths, ", ") + } + return strings.Join(paths[:limit], ", ") + fmt.Sprintf(", +%d more", len(paths)-limit) +} + +// mergeSortedUnique merges two path lists into one sorted list without +// duplicates. +func mergeSortedUnique(a, b []string) []string { + if len(b) == 0 { + return a + } + seen := make(map[string]bool, len(a)+len(b)) + out := make([]string, 0, len(a)+len(b)) + for _, list := range [][]string{a, b} { + for _, p := range list { + if !seen[p] { + seen[p] = true + out = append(out, p) + } + } + } + sort.Strings(out) + return out +} diff --git a/cmd/pilotctl/appstore_state_regression_test.go b/cmd/pilotctl/appstore_state_regression_test.go new file mode 100644 index 00000000..9808c900 --- /dev/null +++ b/cmd/pilotctl/appstore_state_regression_test.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/pilot-protocol/app-store/pkg/manifest" +) + +// Regression tests for the app-store state wipe: `appstore install --force` +// (and `appstore upgrade`, which the hourly updater runs and which reinstalls +// with --force) used to rename the live app dir to .previous, swap in the +// fresh bundle, then RemoveAll the old dir. Everything the app had written into +// $APP (the wallet's EVM key and payment DB, smol's secrets, per-app identities, +// the spend-cap ledger, the audit trail) was deleted. On a checkout without the +// fix, TestAppStoreForceReinstallKeepsAppState fails at the first state file. + +// isolateAppStoreTest points every path an install touches at a scratch dir: +// the install root, HOME/PILOT_HOME (consent + identity), the daemon socket (so +// no live daemon is dialled), telemetry (a dead loopback port) and the +// catalogue (a missing file, so target resolution falls through to the local +// bundle without touching the network). Returns the install root; backups land +// in a sibling "app-backups" dir inside the same scratch dir. +func isolateAppStoreTest(t *testing.T) string { + t.Helper() + base := t.TempDir() + root := filepath.Join(base, "apps") + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + home := filepath.Join(base, "home") + t.Setenv("PILOT_APPSTORE_ROOT", root) + t.Setenv("PILOT_APPSTORE_BACKUP_ROOT", "") + t.Setenv("HOME", home) + t.Setenv("PILOT_HOME", home) + t.Setenv("PILOT_SOCKET", filepath.Join(base, "no-daemon.sock")) + t.Setenv("PILOT_TELEMETRY_URL", "http://127.0.0.1:9/telemetry") + t.Setenv("PILOT_APPSTORE_CATALOG_URL", "file://"+filepath.Join(base, "no-catalogue.json")) + prev := jsonOutput + t.Cleanup(func() { jsonOutput = prev }) + jsonOutput = false + return root +} + +// writeVersionedBundle builds a valid local bundle (manifest.json + bin/app) +// for id at version; binBody varies the binary so two versions differ. +func writeVersionedBundle(t *testing.T, id, version, binBody string) string { + t.Helper() + dir := t.TempDir() + bin := filepath.Join(dir, "bin", "app") + if err := os.MkdirAll(filepath.Dir(bin), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(bin, []byte("#!/bin/sh\necho "+binBody+"\n"), 0o755); err != nil { + t.Fatal(err) + } + var mf map[string]any + if err := json.Unmarshal(validManifestJSON(id, sha256File(bin)), &mf); err != nil { + t.Fatal(err) + } + mf["app_version"] = version + raw, err := json.Marshal(mf) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "manifest.json"), raw, 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +// appStateFixture is what a real install accumulates in $APP at runtime: the +// wallet's keys and DB (with a WAL sidecar), smol's secrets, the spend-cap +// ledger, and a nested app-private file. +var appStateFixture = map[string]string{ + "identity-evm.json": `{"private_key":"0xthis-wallet-key-must-survive-an-upgrade"}`, + "identity.json": `{"ed25519":"per-app identity"}`, + "data.db": "SQLite format 3\x00payments ledger", + "data.db-wal": "wal frames", + "secrets.json": `{"smol_cloud_secret":"s3cr3t"}`, + "cap-state.jsonl": `{"window":"24h","spent":"12.50"}` + "\n", + "data/cache/state.bin": "nested app-private state", +} + +// appControlFixture are files in $APP that belong to the bundle, pilotctl or +// the supervisor rather than the app; a reinstall must not carry them over. +var appControlFixture = []string{".suspended", ".resume", "app.sock", "next-steps.json", ".bundle-sha256"} + +func seedAppState(t *testing.T, appDir string) { + t.Helper() + for rel, body := range appStateFixture { + p := filepath.Join(appDir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + for _, name := range appControlFixture { + if err := os.WriteFile(filepath.Join(appDir, name), []byte("control"), 0o600); err != nil { + t.Fatal(err) + } + } + f, err := os.OpenFile(filepath.Join(appDir, "supervisor.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + _, _ = f.WriteString(`{"event":"spawn","reason":"prior-history-marker"}` + "\n") + _ = f.Close() +} + +func assertAppStateKept(t *testing.T, appDir, when string) { + t.Helper() + // Wallet key first, so a regression reports the headline loss. + rels := []string{"identity-evm.json"} + for rel := range appStateFixture { + if rel != "identity-evm.json" { + rels = append(rels, rel) + } + } + sort.Strings(rels[1:]) + for _, rel := range rels { + want := appStateFixture[rel] + p := filepath.Join(appDir, filepath.FromSlash(rel)) + got, err := os.ReadFile(p) + if err != nil { + t.Fatalf("%s: app state %s was lost: %v", when, rel, err) + } + if string(got) != want { + t.Fatalf("%s: app state %s changed: got %q want %q", when, rel, got, want) + } + fi, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Errorf("%s: %s mode = %v, want 0600 (keys must stay private)", when, rel, fi.Mode().Perm()) + } + } + log, err := os.ReadFile(filepath.Join(appDir, "supervisor.log")) + if err != nil || !strings.Contains(string(log), "prior-history-marker") { + t.Fatalf("%s: supervisor.log audit history was lost (err=%v)", when, err) + } +} + +func TestAppStoreForceReinstallKeepsAppState(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.statefulwallet" + appDir := filepath.Join(root, id) + + v1 := writeVersionedBundle(t, id, "1.0.0", "v1") + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{v1, "--local"}) }) + seedAppState(t, appDir) + + // Same-version reinstall: the docs' generic `install --force` step. + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{v1, "--local", "--force"}) }) + assertAppStateKept(t, appDir, "after install --force") + for _, name := range appControlFixture { + if _, err := os.Lstat(filepath.Join(appDir, name)); err == nil { + t.Errorf("control file %s was carried into the new install", name) + } + } + if _, err := os.Stat(filepath.Join(appDir, manifest.SideloadMarkerName)); err != nil { + t.Errorf("sideload marker missing after a --local reinstall: %v", err) + } + + // Version bump: the path `appstore upgrade` (and the hourly updater) takes. + v2 := writeVersionedBundle(t, id, "1.0.1", "v2") + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{v2, "--local", "--force"}) }) + assertAppStateKept(t, appDir, "after a version-bump reinstall") + raw, err := os.ReadFile(filepath.Join(appDir, "manifest.json")) + if err != nil { + t.Fatal(err) + } + m, err := manifest.Parse(raw) + if err != nil { + t.Fatal(err) + } + if m.AppVersion != "1.0.1" { + t.Fatalf("installed version = %s, want 1.0.1", m.AppVersion) + } + if got := sha256File(filepath.Join(appDir, "bin", "app")); got != m.Binary.SHA256 { + t.Fatalf("installed binary sha %s does not match the new manifest %s", got, m.Binary.SHA256) + } + + // Nothing half-done is left where the supervisor scans... + for _, leftover := range []string{id + ".previous", id + ".staging"} { + if _, err := os.Lstat(filepath.Join(root, leftover)); err == nil { + t.Errorf("%s left in the install root, where the supervisor would scan it", leftover) + } + } + // ...and the replaced installs are kept as backups outside it, not deleted. + backups, err := os.ReadDir(filepath.Join(filepath.Dir(root), "app-backups", id)) + if err != nil { + t.Fatalf("no backup of the previous install: %v", err) + } + if len(backups) != 2 { + t.Fatalf("got %d backups, want 2 (one per replaced install)", len(backups)) + } + newest := filepath.Join(filepath.Dir(root), "app-backups", id, backups[len(backups)-1].Name()) + if got, err := os.ReadFile(filepath.Join(newest, "identity-evm.json")); err != nil || string(got) != appStateFixture["identity-evm.json"] { + t.Fatalf("backup is missing the wallet key: %q, %v", got, err) + } + if !strings.HasSuffix(newest, "-v1.0.0") { + t.Errorf("backup %s should be tagged with the version it held (1.0.0)", newest) + } +} diff --git a/cmd/pilotctl/appstore_state_review_test.go b/cmd/pilotctl/appstore_state_review_test.go new file mode 100644 index 00000000..1a4c9873 --- /dev/null +++ b/cmd/pilotctl/appstore_state_review_test.go @@ -0,0 +1,910 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "errors" + "io/fs" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "testing" + "time" +) + +// Tests for the review findings on the app-state preservation change: +// F1 carry of read-only / unreadable state and `upgrade --all` isolation, +// F2 backup retention, F3 the install lock and swap rollback, F4 writes the +// running app makes during the swap, F5 `install --version` on an installed +// app, F6 backup locations and their fallbacks. + +// makeTreeRemovable registers a cleanup, run before t.TempDir's own, that +// makes read-only dirs under dir writable again so the temp dir can go. +func makeTreeRemovable(t *testing.T, dir string) { + t.Helper() + t.Cleanup(func() { _ = removeAllForce(dir) }) +} + +// runTrapped runs fn with fatal exits trapped, capturing stdout and stderr. +func runTrapped(t *testing.T, fn func()) (stdout, stderr string, failure *trappedFatal) { + t.Helper() + stderr = captureStderr(t, func() { + stdout = captureStdout(t, func() { failure = runTrappingFatal(fn) }) + }) + return stdout, stderr, failure +} + +// installQuiet runs an install whose output the test does not need, failing +// the test if it ends fatally. +func installQuiet(t *testing.T, args ...string) (stdout, stderr string) { + t.Helper() + stdout, stderr, f := runTrapped(t, func() { cmdAppStoreInstall(args) }) + if f != nil { + t.Fatalf("install %v failed: %s: %s\nstderr:\n%s", args, f.Code, f.Message, stderr) + } + return stdout, stderr +} + +// atomicWrite replaces path the way most apps save state: write a temp file +// beside it, rename it over. +func atomicWrite(t *testing.T, path, body string) { + t.Helper() + tmp := path + ".tmp-write" + mustWrite(t, tmp, body, 0o600) + if err := os.Rename(tmp, path); err != nil { + t.Fatal(err) + } +} + +func installedVersion(t *testing.T, appDir string) string { + t.Helper() + m, _, err := readInstalledManifest(appDir) + if err != nil { + t.Fatalf("read installed manifest in %s: %v", appDir, err) + } + return m.AppVersion +} + +// backupsByKind reads every backup of an app in dir, keyed by kind. +func backupsByKind(t *testing.T, dir string) map[string][]string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read backups %s: %v", dir, err) + } + out := map[string][]string{} + for _, e := range entries { + p := filepath.Join(dir, e.Name()) + meta, ok := readBackupMeta(p) + kind := meta.Kind + if !ok { + kind = "(no metadata)" + } + out[kind] = append(out[kind], p) + } + return out +} + +// ── F1: read-only and unreadable state no longer blocks install --force ──── + +func TestCarryAppStateCarriesReadOnlyDirs(t *testing.T) { + base := t.TempDir() + makeTreeRemovable(t, base) + oldDir, newDir := filepath.Join(base, "old"), filepath.Join(base, "new") + // A Go module cache: read-only dirs holding read-only files. + mustWrite(t, filepath.Join(oldDir, "gomodcache", "pkg", "f.go"), "package f", 0o444) + for _, d := range []string{"gomodcache/pkg", "gomodcache"} { + if err := os.Chmod(filepath.Join(oldDir, filepath.FromSlash(d)), 0o555); err != nil { + t.Fatal(err) + } + } + mustWrite(t, filepath.Join(newDir, "bin", "app"), "new binary", 0o755) + + c, err := carryAppState(oldDir, newDir, "") + if err != nil { + t.Fatalf("a read-only state dir must carry, got: %v", err) + } + if want := []string{filepath.Join("gomodcache", "pkg", "f.go")}; !reflect.DeepEqual(c.Carried, want) || len(c.Unreadable) != 0 { + t.Fatalf("carried = %q, unreadable = %q", c.Carried, c.Unreadable) + } + for _, d := range []string{"gomodcache", "gomodcache/pkg"} { + fi, err := os.Stat(filepath.Join(newDir, filepath.FromSlash(d))) + if err != nil || fi.Mode().Perm() != 0o555 { + t.Errorf("%s mode = %v (%v), want the old dir's 0555 restored", d, fi, err) + } + } + oldFi, _ := os.Stat(filepath.Join(oldDir, "gomodcache", "pkg", "f.go")) + newFi, err := os.Stat(filepath.Join(newDir, "gomodcache", "pkg", "f.go")) + if err != nil || !os.SameFile(oldFi, newFi) { + t.Fatalf("f.go not hard-linked into the new dir: %v", err) + } + // A staging dir holding read-only dirs can still be thrown away. + if err := discardStagingDir(newDir); err != nil { + t.Fatalf("discard staging with read-only dirs: %v", err) + } + assertAbsent(t, newDir) +} + +func TestAppStoreReinstallWithReadOnlyStateDirKeepsWorking(t *testing.T) { + root := isolateAppStoreTest(t) + makeTreeRemovable(t, filepath.Dir(root)) + const id = "io.test.gomodcache" + appDir := filepath.Join(root, id) + v1 := writeVersionedBundle(t, id, "1.0.0", "v1") + installQuiet(t, v1, "--local") + seedAppState(t, appDir) + mustWrite(t, filepath.Join(appDir, "gomodcache", "pkg", "f.go"), "package f", 0o444) + for _, d := range []string{"gomodcache/pkg", "gomodcache"} { + if err := os.Chmod(filepath.Join(appDir, filepath.FromSlash(d)), 0o555); err != nil { + t.Fatal(err) + } + } + + // Enough reinstalls that retention has to remove backups holding the + // read-only tree, and the last carries from a dir the carry created. + for i := 0; i < appBackupKeep+2; i++ { + installQuiet(t, v1, "--local", "--force") + } + assertAppStateKept(t, appDir, "after reinstalls with a read-only state dir") + if got := mustRead(t, filepath.Join(appDir, "gomodcache", "pkg", "f.go")); got != "package f" { + t.Fatalf("f.go = %q", got) + } + if fi, err := os.Stat(filepath.Join(appDir, "gomodcache", "pkg")); err != nil || fi.Mode().Perm() != 0o555 { + t.Errorf("gomodcache/pkg mode = %v (%v), want 0555", fi, err) + } + kinds := backupsByKind(t, filepath.Join(filepath.Dir(root), "app-backups", id)) + if len(kinds[backupKindReinstall]) != appBackupKeep || len(kinds) != 1 { + t.Fatalf("backups = %v, want exactly %d reinstall backups (the older ones removed despite read-only dirs)", kinds, appBackupKeep) + } + assertAbsent(t, appDir+appStagingSuffix) + assertAbsent(t, appDir+appPreviousSuffix) + // Uninstall removes an app dir holding read-only dirs too. + if _, stderr, f := runTrapped(t, func() { cmdAppStoreUninstall([]string{id, "--yes"}) }); f != nil { + t.Fatalf("uninstall with a read-only state dir: %+v\n%s", f, stderr) + } + assertAbsent(t, appDir) +} + +func TestAppStoreReinstallMovesStateThisUserCannotRead(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root can read every file") + } + root := isolateAppStoreTest(t) + makeTreeRemovable(t, filepath.Dir(root)) + const id = "io.test.foreignstate" + appDir := filepath.Join(root, id) + installQuiet(t, writeVersionedBundle(t, id, "1.0.0", "v1"), "--local") + seedAppState(t, appDir) + + // A root-owned file left by a daemon once run with sudo: with + // fs.protected_hardlinks this user can neither link nor read it. + // Simulated with a 0000 file whose link fails with EPERM, plus a dir + // this user cannot list. + foreign := filepath.Join(appDir, "supervisor.log.1") + mustWrite(t, foreign, "root's rotated log", 0o600) + if err := os.Chmod(foreign, 0); err != nil { + t.Fatal(err) + } + mustWrite(t, filepath.Join(appDir, "root-owned", "x.json"), "y", 0o600) + if err := os.Chmod(filepath.Join(appDir, "root-owned"), 0); err != nil { + t.Fatal(err) + } + before, err := os.Lstat(foreign) + if err != nil { + t.Fatal(err) + } + origLink := linkFile + linkFile = func(oldname, newname string) error { + if filepath.Base(oldname) == "supervisor.log.1" { + return &os.LinkError{Op: "link", Old: oldname, New: newname, Err: syscall.EPERM} + } + return origLink(oldname, newname) + } + t.Cleanup(func() { linkFile = origLink }) + + jsonOutput = true + out, stderr := installQuiet(t, writeVersionedBundle(t, id, "1.1.0", "v2"), "--local", "--force") + jsonOutput = false + var rpt installReport + if err := json.Unmarshal([]byte(out), &rpt); err != nil { + t.Fatalf("parse report: %v\n%s", err, out) + } + if len(rpt.StateNotCarried) != 0 { + t.Fatalf("state_not_carried = %q, want everything moved across", rpt.StateNotCarried) + } + for _, want := range []string{"supervisor.log.1", "root-owned"} { + found := false + for _, p := range rpt.PreservedState { + found = found || p == want + } + if !found { + t.Errorf("preserved_state does not list %s: %q", want, rpt.PreservedState) + } + } + if !strings.Contains(stderr, "cannot be linked or read") { + t.Errorf("stderr should say some state is moved instead of linked:\n%s", stderr) + } + after, err := os.Lstat(foreign) + if err != nil || !os.SameFile(before, after) { + t.Fatalf("the unreadable file was not moved into the new install: %v", err) + } + if err := os.Chmod(filepath.Join(appDir, "root-owned"), 0o700); err != nil { + t.Fatalf("the unreadable dir was not moved into the new install: %v", err) + } + if got := mustRead(t, filepath.Join(appDir, "root-owned", "x.json")); got != "y" { + t.Fatalf("root-owned/x.json = %q", got) + } + assertAppStateKept(t, appDir, "after a reinstall with unreadable state") + if v := installedVersion(t, appDir); v != "1.1.0" { + t.Fatalf("installed %s, want 1.1.0", v) + } + if meta, ok := readBackupMeta(rpt.BackupDir); !ok || meta.Kind != backupKindUpgrade { + t.Errorf("backup meta = %+v (ok=%v), want a routine upgrade backup: nothing was left behind", meta, ok) + } +} + +func TestReconcileLeftoverMakesTheBackupPinned(t *testing.T) { + base := t.TempDir() + oldDir, newDir := filepath.Join(base, "old"), filepath.Join(base, "new") + mustWrite(t, filepath.Join(oldDir, "secret.key"), "k", 0o600) + // The path is taken in the new install, so the entry the carry could not + // link cannot be moved there either: it stays in the old dir. + mustWrite(t, filepath.Join(newDir, "secret.key"), "bundle file", 0o644) + r := reconcileAppState(oldDir, newDir, "", &appStateCarry{ + Unreadable: []string{"secret.key"}, + entries: map[string]carriedEntry{}, + dirs: map[string]bool{}, + }) + if !reflect.DeepEqual(r.Leftover, []string{"secret.key"}) { + t.Fatalf("leftover = %q", r.Leftover) + } + kind := replacedInstallBackupKind(false, r.Leftover, "1.0.0", "1.1.0") + if kind != backupKindIncomplete || backupKindRotated(kind) { + t.Fatalf("kind = %s, want a pinned %s backup", kind, backupKindIncomplete) + } +} + +func TestAppStoreUpgradeAllContinuesPastAFailedApp(t *testing.T) { + root := isolateAppStoreTest(t) + catPath := filepath.Join(t.TempDir(), "catalogue.json") + t.Setenv("PILOT_APPSTORE_CATALOG_URL", "file://"+catPath) + const alpha, zulu = "io.test.alpha", "io.test.zulu" + + a1, a1SHA := tarGzBundle(t, writeVersionedBundle(t, alpha, "1.0.0", "a1")) + z1, z1SHA := tarGzBundle(t, writeVersionedBundle(t, zulu, "1.0.0", "z1")) + publishSignedCatalogueApps(t, catPath, + catalogueTestApp{alpha, "1.0.0", a1, a1SHA}, catalogueTestApp{zulu, "1.0.0", z1, z1SHA}) + installQuiet(t, alpha) + installQuiet(t, zulu) + seedAppState(t, filepath.Join(root, alpha)) + + // Both move to 1.1.0; alpha's (sorted first) cannot be installed. + a2, _ := tarGzBundle(t, writeVersionedBundle(t, alpha, "1.1.0", "a2")) + z2, z2SHA := tarGzBundle(t, writeVersionedBundle(t, zulu, "1.1.0", "z2")) + publishSignedCatalogueApps(t, catPath, + catalogueTestApp{alpha, "1.1.0", a2, strings.Repeat("0", 64)}, catalogueTestApp{zulu, "1.1.0", z2, z2SHA}) + + _, stderr, f := runTrapped(t, func() { cmdAppStoreUpgrade([]string{"--all"}) }) + if f == nil || f.Code != "upgrade_failed" || !strings.Contains(f.Message, alpha) { + t.Fatalf("upgrade --all = %+v, want upgrade_failed naming %s\nstderr:\n%s", f, alpha, stderr) + } + if v := installedVersion(t, filepath.Join(root, zulu)); v != "1.1.0" { + t.Fatalf("%s is at %s: the failed %s stopped the upgrade of the apps after it", zulu, v, alpha) + } + if v := installedVersion(t, filepath.Join(root, alpha)); v != "1.0.0" { + t.Fatalf("%s is at %s, want it unchanged at 1.0.0", alpha, v) + } + assertAppStateKept(t, filepath.Join(root, alpha), "after its failed upgrade") + if !strings.Contains(stderr, alpha+" was not upgraded") { + t.Errorf("stderr should say %s was skipped:\n%s", alpha, stderr) + } + if fatalTrapDepth != 0 { + t.Fatalf("fatal trap depth leaked: %d", fatalTrapDepth) + } +} + +// ── F2: retention never removes the only copy of an app's state ──────────── + +func TestBackupRetentionNeverRemovesTheResetStateBackup(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.wallet" + appDir := filepath.Join(root, id) + bundle := writeVersionedBundle(t, id, "1.0.0", "v1") + installQuiet(t, bundle, "--local") + mustWrite(t, filepath.Join(appDir, "identity-evm.json"), `{"private_key":"0xORIGINAL"}`, 0o600) + + installQuiet(t, bundle, "--local", "--reset-state") + // Routine reinstalls afterwards (an agent's install --force, hourly + // upgrades) must not push the only copy of the key out. + for i := 0; i < appBackupKeep+1; i++ { + installQuiet(t, bundle, "--local", "--force") + } + backups := filepath.Join(filepath.Dir(root), "app-backups", id) + kinds := backupsByKind(t, backups) + if len(kinds[backupKindResetState]) != 1 { + t.Fatalf("backups = %v: the --reset-state backup was removed", kinds) + } + if got := mustRead(t, filepath.Join(kinds[backupKindResetState][0], "identity-evm.json")); !strings.Contains(got, "0xORIGINAL") { + t.Fatalf("reset-state backup key = %q", got) + } + if meta, _ := readBackupMeta(kinds[backupKindResetState][0]); !meta.Pinned { + t.Error("the reset-state backup is not marked pinned") + } + if len(kinds[backupKindReinstall]) != appBackupKeep { + t.Errorf("reinstall backups = %d, want %d", len(kinds[backupKindReinstall]), appBackupKeep) + } +} + +func TestBackupRetentionKeepsUpgradeBackupsApartFromReinstalls(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.upgradebackup" + appDir := filepath.Join(root, id) + installQuiet(t, writeVersionedBundle(t, id, "1.0.0", "v1"), "--local") + seedAppState(t, appDir) + v2 := writeVersionedBundle(t, id, "1.1.0", "v2") + installQuiet(t, v2, "--local", "--force") // the pre-upgrade backup + var stderr string + for i := 0; i < appBackupKeep+1; i++ { + _, stderr = installQuiet(t, v2, "--local", "--force") + } + kinds := backupsByKind(t, filepath.Join(filepath.Dir(root), "app-backups", id)) + if len(kinds[backupKindUpgrade]) != 1 || !strings.HasSuffix(kinds[backupKindUpgrade][0], "-v1.0.0") { + t.Fatalf("backups = %v: same-version reinstalls pushed out the pre-upgrade backup", kinds) + } + if len(kinds[backupKindReinstall]) != appBackupKeep { + t.Fatalf("reinstall backups = %d, want %d", len(kinds[backupKindReinstall]), appBackupKeep) + } + if !strings.Contains(stderr, "note: removed the older backup") { + t.Errorf("pruning a backup must say so, stderr:\n%s", stderr) + } +} + +func TestPruneBackupDirsOnlyRotatesRoutineKinds(t *testing.T) { + dir := t.TempDir() + mk := func(name string, meta *appBackupMeta) string { + p := filepath.Join(dir, name) + if err := os.MkdirAll(p, 0o700); err != nil { + t.Fatal(err) + } + if meta != nil { + meta.Pinned = !backupKindRotated(meta.Kind) + raw, _ := json.Marshal(meta) + mustWrite(t, filepath.Join(p, appBackupMetaName), string(raw), 0o600) + } + return p + } + stamp := func(i int) string { + return "20260101T0000" + string(rune('0'+i/10)) + string(rune('0'+i%10)) + ".000000000Z" + } + noMeta := mk(stamp(0)+"-v1", nil) + r1 := mk(stamp(1)+"-v1", &appBackupMeta{Kind: backupKindReinstall}) + reset := mk(stamp(2)+"-v1", &appBackupMeta{Kind: backupKindResetState}) + r2 := mk(stamp(3)+"-v1", &appBackupMeta{Kind: backupKindReinstall}) + up := mk(stamp(4)+"-v1", &appBackupMeta{Kind: backupKindUpgrade}) + r3 := mk(stamp(5)+"-v2", &appBackupMeta{Kind: backupKindReinstall}) + r4 := mk(stamp(6)+"-v2", &appBackupMeta{Kind: backupKindReinstall}) + inc := mk(stamp(7)+"-v2", &appBackupMeta{Kind: backupKindIncomplete}) + + removed := pruneAppBackups(dir, 2) + if !reflect.DeepEqual(removed, []string{r1, r2}) { + t.Fatalf("removed = %q, want the two oldest reinstall backups", removed) + } + for _, kept := range []string{noMeta, reset, up, r3, r4, inc} { + if _, err := os.Stat(kept); err != nil { + t.Errorf("%s was removed: %v", kept, err) + } + } +} + +// ── F3: one install at a time per app; rollback never lies or leaks ──────── + +func TestLockAppInstallIsExclusivePerApp(t *testing.T) { + root := t.TempDir() + unlock, err := lockAppInstall(root, "io.test.lock") + if err != nil { + t.Fatal(err) + } + prev := appInstallLockWait + appInstallLockWait = 200 * time.Millisecond + t.Cleanup(func() { appInstallLockWait = prev }) + + var lockErr error + stderr := captureStderr(t, func() { _, lockErr = lockAppInstall(root, "io.test.lock") }) + if lockErr == nil || !strings.Contains(lockErr.Error(), "more than") { + t.Fatalf("second lock = %v, want a timeout while the first is held", lockErr) + } + if !strings.Contains(stderr, "waiting for another pilotctl") { + t.Errorf("a waiting install should say why, stderr:\n%s", stderr) + } + other, err := lockAppInstall(root, "io.test.other") + if err != nil { + t.Fatalf("another app's lock must not wait: %v", err) + } + other() + unlock() + unlock() // idempotent + again, err := lockAppInstall(root, "io.test.lock") + if err != nil { + t.Fatalf("lock after release: %v", err) + } + again() + if fi, err := os.Lstat(appInstallLockPath(root, "io.test.lock")); err != nil || !fi.Mode().IsRegular() { + t.Fatalf("lock file = %v, %v; want a plain file the supervisor ignores", fi, err) + } +} + +// startInFlightSwap puts appDir in the state another pilotctl's install is in +// halfway through its swap: the live dir moved to .previous and a +// manifest-bearing staging dir holding the new version, with that install's +// lock held. It returns the lock's release. +func startInFlightSwap(t *testing.T, root, id, newBundle string) func() { + t.Helper() + appDir := filepath.Join(root, id) + unlock, err := lockAppInstall(root, id) + if err != nil { + t.Fatal(err) + } + t.Cleanup(unlock) + if err := os.Rename(appDir, appDir+appPreviousSuffix); err != nil { + t.Fatal(err) + } + staging := appDir + appStagingSuffix + mustWrite(t, filepath.Join(staging, "bin", "app"), mustRead(t, filepath.Join(newBundle, "bin", "app")), 0o755) + mustWrite(t, filepath.Join(staging, "manifest.json"), mustRead(t, filepath.Join(newBundle, "manifest.json")), 0o644) + return unlock +} + +// finishInFlightSwap completes the other install's swap: staging becomes the +// live dir (with the state carried) and the old dir leaves the install root. +func finishInFlightSwap(t *testing.T, root, id string) { + t.Helper() + appDir := filepath.Join(root, id) + staging := appDir + appStagingSuffix + if _, err := carryAppState(appDir+appPreviousSuffix, staging, "bin/app"); err != nil { + t.Fatal(err) + } + if err := os.Rename(staging, appDir); err != nil { + t.Fatal(err) + } + if err := os.Rename(appDir+appPreviousSuffix, filepath.Join(t.TempDir(), "retired")); err != nil { + t.Fatal(err) + } +} + +func TestAppStoreNoOpInstallWaitsForAnInFlightSwap(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.inflight" + appDir := filepath.Join(root, id) + installQuiet(t, writeVersionedBundle(t, id, "1.0.0", "v1"), "--local") + seedAppState(t, appDir) + unlock := startInFlightSwap(t, root, id, writeVersionedBundle(t, id, "1.0.1", "v2")) + + // `install ` (the no-op path) while the other install is mid-swap. + // It used to read the lone .previous as a crash leftover and put it + // back, breaking the other install's swap. + type result struct { + out, errOut string + f *trappedFatal + } + done := make(chan result, 1) + go func() { + var r result + r.errOut = captureStderr(t, func() { + r.out = captureStdout(t, func() { r.f = runTrappingFatal(func() { cmdAppStoreInstall([]string{id}) }) }) + }) + done <- r + }() + time.Sleep(300 * time.Millisecond) + select { + case r := <-done: + t.Fatalf("install returned while another install held the app's lock: %+v", r) + default: + } + if _, err := os.Stat(filepath.Join(appDir+appPreviousSuffix, "identity-evm.json")); err != nil { + t.Fatalf("the in-flight install's previous dir was taken over: %v", err) + } + assertAbsent(t, appDir) + if _, err := os.Stat(filepath.Join(appDir+appStagingSuffix, "manifest.json")); err != nil { + t.Fatalf("the in-flight install's staging dir was touched: %v", err) + } + + finishInFlightSwap(t, root, id) + unlock() + r := <-done + if r.f != nil { + t.Fatalf("install failed: %+v\n%s", r.f, r.errOut) + } + if !strings.Contains(r.out, "already installed") || !strings.Contains(r.out, "v1.0.1") { + t.Fatalf("want the no-op answer about the version the other install put in place, got:\n%s", r.out) + } + if !strings.Contains(r.errOut, "waiting for another pilotctl") { + t.Errorf("stderr should say it waited:\n%s", r.errOut) + } + assertAppStateKept(t, appDir, "after an install waited for another") + assertAbsent(t, appDir+appPreviousSuffix) + assertAbsent(t, appDir+appStagingSuffix) +} + +func TestAppStoreInstallWaitsForAnInFlightSwap(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.inflightforce" + appDir := filepath.Join(root, id) + installQuiet(t, writeVersionedBundle(t, id, "1.0.0", "v1"), "--local") + seedAppState(t, appDir) + unlock := startInFlightSwap(t, root, id, writeVersionedBundle(t, id, "1.0.1", "v2")) + + // An install --force of yet another version races the in-flight one. + v3 := writeVersionedBundle(t, id, "1.0.2", "v3") + done := make(chan *trappedFatal, 1) + go func() { + var f *trappedFatal + _ = captureStderr(t, func() { + _ = captureStdout(t, func() { f = runTrappingFatal(func() { cmdAppStoreInstall([]string{v3, "--local", "--force"}) }) }) + }) + done <- f + }() + time.Sleep(300 * time.Millisecond) + if _, err := os.Stat(filepath.Join(appDir+appPreviousSuffix, "identity-evm.json")); err != nil { + t.Fatalf("the in-flight install's previous dir was taken over: %v", err) + } + if _, err := os.Stat(filepath.Join(appDir+appStagingSuffix, "manifest.json")); err != nil { + t.Fatalf("the in-flight install's staging dir was touched: %v", err) + } + finishInFlightSwap(t, root, id) + unlock() + if f := <-done; f != nil { + t.Fatalf("install failed: %+v", f) + } + if v := installedVersion(t, appDir); v != "1.0.2" { + t.Fatalf("installed %s, want 1.0.2", v) + } + assertAppStateKept(t, appDir, "after two serialized installs") + assertAbsent(t, appDir+appPreviousSuffix) + assertAbsent(t, appDir+appStagingSuffix) +} + +func TestSwapInAppDirRollbackSaysWhereThePreviousInstallIs(t *testing.T) { + base := t.TempDir() + final := filepath.Join(base, "io.test.rollback") + staging := final + appStagingSuffix + mustWrite(t, filepath.Join(final, "identity-evm.json"), "old key", 0o600) + mustWrite(t, filepath.Join(staging, "manifest.json"), "new", 0o644) + // Another process puts the old dir back right after the swap moved it + // aside (what an unlocked crash recovery used to do). + testHookAfterMoveAside = func() { + if err := os.Rename(final+appPreviousSuffix, final); err != nil { + t.Error(err) + } + } + t.Cleanup(func() { testHookAfterMoveAside = nil }) + + prev, err := swapInAppDir(final, staging, nil) + if err == nil || prev != "" { + t.Fatalf("swap = (%q, %v), want a failure", prev, err) + } + if strings.Contains(err.Error(), "intact at "+final+appPreviousSuffix) { + t.Fatalf("error claims the previous install is at %s, which does not exist: %v", final+appPreviousSuffix, err) + } + if !strings.Contains(err.Error(), "another process put it back") { + t.Errorf("error should say where the previous install is: %v", err) + } + // No manifest-bearing staging dir is left for the supervisor to adopt. + assertAbsent(t, staging) + if got := mustRead(t, filepath.Join(final, "identity-evm.json")); got != "old key" { + t.Fatalf("live install = %q", got) + } +} + +// ── F4: writes the running app makes during the swap are kept ───────────── + +func TestReconcileAppStateTakesWhatTheOldProcessChanged(t *testing.T) { + base := t.TempDir() + oldDir, newDir := filepath.Join(base, "old"), filepath.Join(base, "new") + mustWrite(t, filepath.Join(oldDir, "a-state.json"), `{"balance":100}`, 0o600) + mustWrite(t, filepath.Join(oldDir, "b.json"), "b1", 0o600) + mustWrite(t, filepath.Join(oldDir, "copied.json"), "c1", 0o600) + mustWrite(t, filepath.Join(oldDir, "data.db-journal"), "hot journal", 0o600) + mustWrite(t, filepath.Join(oldDir, "notes.txt"), "n", 0o600) + mustWrite(t, filepath.Join(oldDir, "sub", "keep.txt"), "k", 0o600) + mustWrite(t, filepath.Join(newDir, "bin", "app"), "new binary", 0o755) + origLink := linkFile + linkFile = func(oldname, newname string) error { + if filepath.Base(oldname) == "copied.json" { + return &os.LinkError{Op: "link", Old: oldname, New: newname, Err: syscall.EXDEV} + } + return origLink(oldname, newname) + } + c, err := carryAppState(oldDir, newDir, "") + linkFile = origLink + if err != nil { + t.Fatal(err) + } + + // The old process, still running, with $APP naming the old dir: + atomicWrite(t, filepath.Join(oldDir, "a-state.json"), `{"balance":42}`) // replaced + mustWrite(t, filepath.Join(oldDir, "created.json"), "created after the carry", 0o600) + mustWrite(t, filepath.Join(oldDir, "sub2", "nested.json"), "in a new dir", 0o600) + if err := os.WriteFile(filepath.Join(oldDir, "copied.json"), []byte("c2, rewritten in place"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(oldDir, "data.db-journal")); err != nil { // transaction committed + t.Fatal(err) + } + if err := os.Remove(filepath.Join(oldDir, "notes.txt")); err != nil { + t.Fatal(err) + } + // b.json changed on both sides: the new install's write wins. + atomicWrite(t, filepath.Join(oldDir, "b.json"), "b2 in the old dir") + atomicWrite(t, filepath.Join(newDir, "b.json"), "b2 in the new install") + + r := reconcileAppState(oldDir, newDir, "", c) + for rel, want := range map[string]string{ + "a-state.json": `{"balance":42}`, + "created.json": "created after the carry", + "sub2/nested.json": "in a new dir", + "copied.json": "c2, rewritten in place", + "b.json": "b2 in the new install", + "sub/keep.txt": "k", + "notes.txt": "n", // a non-volatile deletion is not mirrored + } { + if got := mustRead(t, filepath.Join(newDir, filepath.FromSlash(rel))); got != want { + t.Errorf("%s = %q, want %q", rel, got, want) + } + } + // A stale rollback journal would roll back the committed transaction. + assertAbsent(t, filepath.Join(newDir, "data.db-journal")) + wantUpdated := []string{"a-state.json", "copied.json", "created.json", filepath.Join("sub2", "nested.json")} + if !reflect.DeepEqual(r.Updated, wantUpdated) || !reflect.DeepEqual(r.Removed, []string{"data.db-journal"}) || len(r.Leftover) != 0 { + t.Fatalf("reconcile = %+v, want updated %q, removed [data.db-journal]", r, wantUpdated) + } +} + +func TestAppStoreForceReinstallKeepsWritesMadeDuringTheSwap(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.atomicwrite" + appDir := filepath.Join(root, id) + installQuiet(t, writeVersionedBundle(t, id, "1.0.0", "v1"), "--local") + seedAppState(t, appDir) + mustWrite(t, filepath.Join(appDir, "a-state.json"), `{"balance":100}`, 0o600) + + // Between the carry and the swap the running app saves its state the + // usual way (temp file + rename) and writes a new file, through $APP. + testHookBeforeSwap = func(live string) { + atomicWrite(t, filepath.Join(live, "a-state.json"), `{"balance":42}`) + mustWrite(t, filepath.Join(live, "receipts", "r1.json"), "receipt", 0o600) + } + t.Cleanup(func() { testHookBeforeSwap = nil }) + installQuiet(t, writeVersionedBundle(t, id, "1.1.0", "v2"), "--local", "--force") + + if got := mustRead(t, filepath.Join(appDir, "a-state.json")); got != `{"balance":42}` { + t.Fatalf("a-state.json = %s: the write made during the swap was reverted", got) + } + if got := mustRead(t, filepath.Join(appDir, "receipts", "r1.json")); got != "receipt" { + t.Fatalf("receipts/r1.json = %q", got) + } + assertAppStateKept(t, appDir, "after writes during the swap") + if v := installedVersion(t, appDir); v != "1.1.0" { + t.Fatalf("installed %s, want 1.1.0", v) + } +} + +// ── F5: install --version on an installed app does not fake success ──────── + +func TestAppStoreInstallPinnedVersionOnAnInstalledApp(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.pinned" + appDir := filepath.Join(root, id) + catPath := filepath.Join(t.TempDir(), "catalogue.json") + t.Setenv("PILOT_APPSTORE_CATALOG_URL", "file://"+catPath) + v1, v1SHA := tarGzBundle(t, writeVersionedBundle(t, id, "1.0.0", "v1")) + publishSignedCatalogue(t, catPath, id, "1.0.0", v1, v1SHA) + installQuiet(t, id) + seedAppState(t, appDir) + v2, v2SHA := tarGzBundle(t, writeVersionedBundle(t, id, "2.0.0", "v2")) + publishSignedCatalogue(t, catPath, id, "2.0.0", v2, v2SHA) + + jsonOutput = true + _, stderr, f := runTrapped(t, func() { cmdAppStoreInstall([]string{id, "--version", "2.0.0"}) }) + jsonOutput = false + if f == nil || f.Code != "conflict" { + t.Fatalf("install --version 2.0.0 over 1.0.0 without --force = %+v, want conflict\n%s", f, stderr) + } + if !strings.Contains(stderr, "--force") { + t.Errorf("the conflict should say how to replace it:\n%s", stderr) + } + _, _, f = runTrapped(t, func() { cmdAppStoreInstall([]string{id, "--version", "9.9.9"}) }) + if f == nil || f.Code != "version_unavailable" { + t.Fatalf("install --version 9.9.9 = %+v, want version_unavailable", f) + } + out, _, f := runTrapped(t, func() { cmdAppStoreInstall([]string{id, "--version", "1.0.0"}) }) + if f != nil || !strings.Contains(out, "already installed") { + t.Fatalf("install --version = %+v, want the no-op answer, got:\n%s", f, out) + } + // A local bundle of another version is a pinned version too. + _, _, f = runTrapped(t, func() { cmdAppStoreInstall([]string{writeVersionedBundle(t, id, "3.0.0", "v3"), "--local"}) }) + if f == nil || f.Code != "conflict" { + t.Fatalf("local bundle of another version without --force = %+v, want conflict", f) + } + if v := installedVersion(t, appDir); v != "1.0.0" { + t.Fatalf("installed %s after refused installs, want 1.0.0", v) + } + assertAppStateKept(t, appDir, "after refused installs") + // With --force the pinned version is installed, state kept. + installQuiet(t, id, "--version", "2.0.0", "--force") + if v := installedVersion(t, appDir); v != "2.0.0" { + t.Fatalf("installed %s, want 2.0.0", v) + } + assertAppStateKept(t, appDir, "after install --version --force") +} + +// ── F6: backup locations and their fallbacks ────────────────────────────── + +// writeReplacedInstall creates /.previous as a swap leaves it. +func writeReplacedInstall(t *testing.T, root, id string) string { + t.Helper() + prev := filepath.Join(root, id) + appPreviousSuffix + mustWrite(t, filepath.Join(prev, "manifest.json"), string(validManifestJSON(id, strings.Repeat("a", 64))), 0o644) + mustWrite(t, filepath.Join(prev, "bin", "app"), "old binary", 0o755) + mustWrite(t, filepath.Join(prev, "identity-evm.json"), "key", 0o600) + mustWrite(t, filepath.Join(prev, "data", "cache", "x"), "x", 0o600) + if err := os.Symlink("identity-evm.json", filepath.Join(prev, "current-key")); err != nil { + t.Fatal(err) + } + return prev +} + +func assertStrippedBackup(t *testing.T, dir, kind string) { + t.Helper() + if got := mustRead(t, filepath.Join(dir, "identity-evm.json")); got != "key" { + t.Fatalf("backup %s key = %q", dir, got) + } + if fi, err := os.Stat(filepath.Join(dir, "identity-evm.json")); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("backup key mode = %v, %v", fi, err) + } + if target, err := os.Readlink(filepath.Join(dir, "current-key")); err != nil || target != "identity-evm.json" { + t.Errorf("backup symlink = %q, %v", target, err) + } + assertAbsent(t, filepath.Join(dir, "bin", "app")) + if meta, ok := readBackupMeta(dir); !ok || meta.Kind != kind { + t.Errorf("backup meta = %+v (ok=%v), want kind %s", meta, ok, kind) + } +} + +func TestRetireAppDirCopiesToABackupRootOnAnotherFilesystem(t *testing.T) { + root := isolateAppStoreTest(t) + backupRoot := filepath.Join(t.TempDir(), "other-fs") + t.Setenv("PILOT_APPSTORE_BACKUP_ROOT", backupRoot) + orig := renameForBackup + renameForBackup = func(oldpath, newpath string) error { + if strings.HasPrefix(newpath, backupRoot) { + return &os.LinkError{Op: "rename", Old: oldpath, New: newpath, Err: syscall.EXDEV} + } + return orig(oldpath, newpath) + } + t.Cleanup(func() { renameForBackup = orig }) + const id = "io.test.exdev" + prev := writeReplacedInstall(t, root, id) + + dst, err := retireAppDir(prev, id, appBackupMeta{Kind: backupKindUpgrade}) + if err != nil { + t.Fatalf("a backup root on another filesystem must work: %v", err) + } + if !strings.HasPrefix(dst, filepath.Join(backupRoot, id)+string(filepath.Separator)) { + t.Fatalf("backup at %s, want it under the configured root %s", dst, backupRoot) + } + assertStrippedBackup(t, dst, backupKindUpgrade) + assertAbsent(t, prev) +} + +func TestRetireAppDirFallsBackWithoutGrowingOrHiding(t *testing.T) { + root := isolateAppStoreTest(t) + blocker := filepath.Join(t.TempDir(), "not-a-dir") + mustWrite(t, blocker, "x", 0o600) + t.Setenv("PILOT_APPSTORE_BACKUP_ROOT", filepath.Join(blocker, "backups")) + const id = "io.test.fallback" + defaultLoc := filepath.Join(filepath.Dir(root), "app-backups") + + // 1. The configured root is unusable: the default beside the install + // root takes the backup, with a warning naming the problem. + dst, err := retireAppDir(writeReplacedInstall(t, root, id), id, appBackupMeta{Kind: backupKindReinstall}) + if err == nil || !strings.Contains(err.Error(), "Fix that location") { + t.Fatalf("want a warning about the configured root, got %v", err) + } + if !strings.HasPrefix(dst, filepath.Join(defaultLoc, id)) { + t.Fatalf("backup at %s, want it in the default location", dst) + } + assertStrippedBackup(t, dst, backupKindReinstall) + + // 2. The default is unusable too: a dot-dir inside the install root, + // stripped and pruned like any other backup location. + if err := os.RemoveAll(defaultLoc); err != nil { + t.Fatal(err) + } + mustWrite(t, defaultLoc, "not a dir", 0o600) + inRoot := filepath.Join(root, inRootBackupDirName, id) + for i := 0; i < appBackupKeep+2; i++ { + stderr := captureStderr(t, func() { + dst, err = retireAppDir(writeReplacedInstall(t, root, id), id, appBackupMeta{Kind: backupKindReinstall}) + }) + if err == nil || !strings.HasPrefix(dst, inRoot) { + t.Fatalf("retire %d = (%s, %v), want the in-root fallback with a warning", i, dst, err) + } + if i == appBackupKeep && !strings.Contains(stderr, "note: removed the older backup") { + t.Errorf("pruning must say so:\n%s", stderr) + } + assertStrippedBackup(t, dst, backupKindReinstall) + } + if entries, _ := os.ReadDir(inRoot); len(entries) != appBackupKeep { + t.Fatalf("in-root fallback holds %d backups, want %d (pruned)", len(entries), appBackupKeep) + } + assertAbsent(t, filepath.Join(root, inRootBackupDirName, "manifest.json")) + if apps, err := scanInstalledApps(); err != nil || len(apps) != 0 { + t.Fatalf("the fallback dir shows up as an installed app: %v, %v", apps, err) + } + + // 3. Nothing is usable: parked in the install root with the manifest + // disabled, still stripped and pruned. + if err := os.RemoveAll(filepath.Join(root, inRootBackupDirName)); err != nil { + t.Fatal(err) + } + mustWrite(t, filepath.Join(root, inRootBackupDirName), "not a dir", 0o600) + for i := 0; i < appBackupKeep+2; i++ { + _ = captureStderr(t, func() { + dst, err = retireAppDir(writeReplacedInstall(t, root, id), id, appBackupMeta{Kind: backupKindReinstall}) + }) + if err == nil || !strings.HasPrefix(filepath.Base(dst), id+appPreviousSuffix+"-") { + t.Fatalf("retire %d = (%s, %v), want a parked dir with a warning", i, dst, err) + } + assertAbsent(t, filepath.Join(dst, "manifest.json")) + assertStrippedBackup(t, dst, backupKindReinstall) + } + parked := parkedBackups(root, id) + if len(parked) != appBackupKeep { + t.Fatalf("%d parked dirs in the install root, want %d (pruned)", len(parked), appBackupKeep) + } + assertAbsent(t, filepath.Join(root, id)+appPreviousSuffix) + + // Uninstall names every backup that is left, parked ones included. + mustWrite(t, filepath.Join(root, id, "manifest.json"), string(validManifestJSON(id, strings.Repeat("a", 64))), 0o644) + jsonOutput = true + out := captureStdout(t, func() { cmdAppStoreUninstall([]string{id, "--yes"}) }) + jsonOutput = false + var rpt struct { + Backups []string `json:"backups"` + } + if err := json.Unmarshal([]byte(out), &rpt); err != nil { + t.Fatalf("parse: %v\n%s", err, out) + } + if !reflect.DeepEqual(rpt.Backups, parked) { + t.Fatalf("uninstall lists %q, want the parked backups %q", rpt.Backups, parked) + } + for _, p := range parked { + if _, err := os.Stat(filepath.Join(p, "identity-evm.json")); err != nil { + t.Errorf("uninstall removed a backup (%s): %v", p, err) + } + } +} + +func TestUninstallListsBackupsInEveryLocation(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.uninstallbackups" + appDir := filepath.Join(root, id) + bundle := writeVersionedBundle(t, id, "1.0.0", "v1") + installQuiet(t, bundle, "--local") + seedAppState(t, appDir) + installQuiet(t, bundle, "--local", "--force") // → default location + inRoot := filepath.Join(root, inRootBackupDirName, id, "20260101T000000.000000000Z-v0.9.0") + mustWrite(t, filepath.Join(inRoot, "identity-evm.json"), "older key", 0o600) + + out := captureStdout(t, func() { cmdAppStoreUninstall([]string{id, "--yes"}) }) + if !strings.Contains(out, "2 backup(s) of earlier installs remain") || !strings.Contains(out, inRoot) || + !strings.Contains(out, filepath.Join(filepath.Dir(root), "app-backups", id)) { + t.Fatalf("uninstall should list both backups, got:\n%s", out) + } + if errors.Is(func() error { _, err := os.Stat(inRoot); return err }(), fs.ErrNotExist) { + t.Fatal("uninstall removed a backup") + } +} diff --git a/cmd/pilotctl/appstore_state_test.go b/cmd/pilotctl/appstore_state_test.go new file mode 100644 index 00000000..0d1b6bc5 --- /dev/null +++ b/cmd/pilotctl/appstore_state_test.go @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/pilot-protocol/app-store/pkg/manifest" + "github.com/pilot-protocol/pilotprotocol/internal/catalogtrust" +) + +func mustWrite(t *testing.T, path, body string, perm os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), perm); err != nil { + t.Fatal(err) + } +} + +func mustRead(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func assertAbsent(t *testing.T, path string) { + t.Helper() + if _, err := os.Lstat(path); err == nil { + t.Errorf("%s exists, want it absent", path) + } +} + +// ── install without --force on an installed app is a no-op ───────────────── + +func TestAppStoreInstallAlreadyInstalledIsNoOp(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.noop" + appDir := filepath.Join(root, id) + bundle := writeVersionedBundle(t, id, "1.0.0", "v1") + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{bundle, "--local"}) }) + seedAppState(t, appDir) + before, err := os.Stat(filepath.Join(appDir, "manifest.json")) + if err != nil { + t.Fatal(err) + } + + // By catalogue id: answered before any download, points at `upgrade`. + out := captureStdout(t, func() { cmdAppStoreInstall([]string{id}) }) + if !strings.Contains(out, "already installed") || !strings.Contains(out, "pilotctl appstore upgrade "+id) { + t.Fatalf("no-op output should say it is installed and point at upgrade, got:\n%s", out) + } + // By local bundle path: answered after reading the bundle's manifest. + out = captureStdout(t, func() { cmdAppStoreInstall([]string{bundle, "--local"}) }) + if !strings.Contains(out, "already installed") || !strings.Contains(out, "--local --force") { + t.Fatalf("local no-op output should point at --local --force, got:\n%s", out) + } + // JSON: a success-shaped report flagged already_installed, with a hint. + jsonOutput = true + out = captureStdout(t, func() { cmdAppStoreInstall([]string{id}) }) + jsonOutput = false + var rpt installReport + if err := json.Unmarshal([]byte(out), &rpt); err != nil { + t.Fatalf("parse: %v\n%s", err, out) + } + if !rpt.AlreadyInstalled || rpt.AppVersion != "1.0.0" || !strings.Contains(rpt.Hint, "appstore upgrade") { + t.Fatalf("JSON no-op report = %+v", rpt) + } + + after, err := os.Stat(filepath.Join(appDir, "manifest.json")) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(before, after) { + t.Error("a no-op install replaced the installed app") + } + assertAppStateKept(t, appDir, "after no-op installs") + if _, err := os.Stat(filepath.Join(appDir, ".suspended")); err != nil { + t.Error("a no-op install touched the installed app's control files") + } + assertAbsent(t, filepath.Join(filepath.Dir(root), "app-backups")) +} + +// ── --reset-state is the explicit, loud, destructive variant ─────────────── + +func TestAppStoreInstallResetStateStartsEmptyButKeepsBackup(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.reset" + appDir := filepath.Join(root, id) + bundle := writeVersionedBundle(t, id, "1.0.0", "v1") + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{bundle, "--local"}) }) + seedAppState(t, appDir) + + jsonOutput = true + var out string + stderr := captureStderr(t, func() { + out = captureStdout(t, func() { cmdAppStoreInstall([]string{bundle, "--local", "--reset-state"}) }) + }) + jsonOutput = false + if !strings.Contains(stderr, "WARNING: --reset-state") || !strings.Contains(stderr, "WITHOUT its saved state") { + t.Fatalf("--reset-state must warn loudly on stderr, got:\n%s", stderr) + } + var rpt installReport + if err := json.Unmarshal([]byte(out), &rpt); err != nil { + t.Fatalf("parse: %v\n%s", err, out) + } + if !rpt.StateReset || len(rpt.PreservedState) != 0 || rpt.BackupDir == "" { + t.Fatalf("report = %+v, want state_reset with a backup and nothing preserved", rpt) + } + for rel := range appStateFixture { + assertAbsent(t, filepath.Join(appDir, filepath.FromSlash(rel))) + } + if strings.Contains(mustRead(t, filepath.Join(appDir, "supervisor.log")), "prior-history-marker") { + t.Error("--reset-state carried the old audit log") + } + // The discarded state is still recoverable from the backup. + if got := mustRead(t, filepath.Join(rpt.BackupDir, "identity-evm.json")); got != appStateFixture["identity-evm.json"] { + t.Fatalf("backup key = %q", got) + } + // Uninstall leaves the backups and says where they are. + out = captureStdout(t, func() { cmdAppStoreUninstall([]string{id, "--yes"}) }) + if !strings.Contains(out, "backup(s) of earlier installs") { + t.Errorf("uninstall should point at the remaining backups, got:\n%s", out) + } +} + +// ── what counts as app state ──────────────────────────────────────────────── + +func TestCarryAppStateCarriesOnlyAppState(t *testing.T) { + oldDir := filepath.Join(t.TempDir(), "old") + newDir := filepath.Join(t.TempDir(), "new") + + // Old install: bundle files, pilotctl/supervisor control files, and state. + mustWrite(t, filepath.Join(oldDir, "manifest.json"), "old manifest", 0o644) + mustWrite(t, filepath.Join(oldDir, "bin", "oldapp"), "old binary", 0o755) + mustWrite(t, filepath.Join(oldDir, "bin", "helper"), "fetched helper", 0o755) + mustWrite(t, filepath.Join(oldDir, "install.json"), "old install spec", 0o644) + for name := range appDirControlFiles { + if name != "manifest.json" && name != "install.json" { + mustWrite(t, filepath.Join(oldDir, name), "control", 0o600) + } + } + mustWrite(t, filepath.Join(oldDir, "stale.sock"), "not state", 0o600) + mustWrite(t, filepath.Join(oldDir, "data.db"), "db", 0o600) + mustWrite(t, filepath.Join(oldDir, "shipped.txt"), "user-modified copy", 0o600) + mustWrite(t, filepath.Join(oldDir, "data", "cache", "state.bin"), "nested", 0o600) + if err := os.Chmod(filepath.Join(oldDir, "data"), 0o750); err != nil { + t.Fatal(err) + } + if err := os.Symlink("data.db", filepath.Join(oldDir, "current.db")); err != nil { + t.Fatal(err) + } + // A live unix socket (short path: macOS caps sun_path at 104 bytes). + sockDir, err := os.MkdirTemp("", "cas") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(sockDir) }) + if ln, err := net.Listen("unix", filepath.Join(sockDir, "s")); err == nil { + t.Cleanup(func() { _ = ln.Close() }) + if err := os.Rename(filepath.Join(sockDir, "s"), filepath.Join(oldDir, "live-socket")); err != nil { + t.Fatal(err) + } + } + + // New bundle already staged: its own binary and a file it ships. + mustWrite(t, filepath.Join(newDir, "bin", "newapp"), "new binary", 0o755) + mustWrite(t, filepath.Join(newDir, "shipped.txt"), "bundle copy", 0o644) + + c, err := carryAppState(oldDir, newDir, "bin/oldapp") + if err != nil { + t.Fatalf("carryAppState: %v", err) + } + if len(c.Unreadable) != 0 { + t.Fatalf("nothing here is unreadable, got %q", c.Unreadable) + } + carried := c.Carried + want := []string{ + filepath.Join("bin", "helper"), + "current.db", + "data.db", + filepath.Join("data", "cache", "state.bin"), + } + if !reflect.DeepEqual(carried, want) { + t.Fatalf("carried = %q\nwant %q", carried, want) + } + for name := range appDirControlFiles { + assertAbsent(t, filepath.Join(newDir, name)) + } + for _, p := range []string{"bin/oldapp", "stale.sock", "live-socket", "install.json"} { + assertAbsent(t, filepath.Join(newDir, filepath.FromSlash(p))) + } + if got := mustRead(t, filepath.Join(newDir, "shipped.txt")); got != "bundle copy" { + t.Errorf("a path the new bundle ships was overwritten by the old copy: %q", got) + } + // State is hard-linked (same inode as the running app's files) ... + oldFi, _ := os.Stat(filepath.Join(oldDir, "data.db")) + newFi, _ := os.Stat(filepath.Join(newDir, "data.db")) + if !os.SameFile(oldFi, newFi) { + t.Error("data.db was copied, want a hard link so a running app's writes are not lost") + } + // ... symlinks stay symlinks, and dir modes are kept. + if target, err := os.Readlink(filepath.Join(newDir, "current.db")); err != nil || target != "data.db" { + t.Errorf("symlink not preserved: %q, %v", target, err) + } + if fi, err := os.Stat(filepath.Join(newDir, "data")); err != nil || fi.Mode().Perm() != 0o750 { + t.Errorf("data/ mode not preserved: %v, %v", fi, err) + } + if err := verifyCarriedState(newDir, carried, false); err != nil { + t.Fatal(err) + } +} + +func TestVerifyCarriedStateToleratesOnlyVolatileFiles(t *testing.T) { + dir := t.TempDir() + mustWrite(t, filepath.Join(dir, "identity.json"), "k", 0o600) + carried := []string{"identity.json", "data.db-journal"} + if err := verifyCarriedState(dir, carried, true); err != nil { + t.Fatalf("a vanished SQLite journal must not fail the post-swap check: %v", err) + } + if err := verifyCarriedState(dir, carried, false); err == nil { + t.Fatal("the pre-swap check must require every carried path") + } + if err := verifyCarriedState(dir, []string{"secrets.json"}, true); err == nil { + t.Fatal("a missing secrets file must fail verification") + } +} + +// ── the swap keeps the previous install until the new one verifies ──────── + +func TestSwapInAppDirRestoresPreviousWhenVerifyFails(t *testing.T) { + base := t.TempDir() + final := filepath.Join(base, "io.test.swap") + staging := final + appStagingSuffix + mustWrite(t, filepath.Join(final, "identity-evm.json"), "old key", 0o600) + mustWrite(t, filepath.Join(staging, "manifest.json"), "new", 0o644) + + prev, err := swapInAppDir(final, staging, func(dir string) error { + if got := mustRead(t, filepath.Join(dir, "manifest.json")); got != "new" { + t.Errorf("verify saw %q, want the staged dir", got) + } + if _, err := os.Stat(final + appPreviousSuffix); err != nil { + t.Errorf("previous install not kept while verifying: %v", err) + } + return errors.New("binary sha mismatch") + }) + if err == nil || prev != "" || !strings.Contains(err.Error(), "restored") { + t.Fatalf("swap = (%q, %v), want a restore error", prev, err) + } + if got := mustRead(t, filepath.Join(final, "identity-evm.json")); got != "old key" { + t.Fatalf("previous install not restored: %q", got) + } + assertAbsent(t, final+appPreviousSuffix) + assertAbsent(t, staging) +} + +func TestSwapInAppDirSuccessAndPreviousGuard(t *testing.T) { + base := t.TempDir() + final := filepath.Join(base, "io.test.swapok") + staging := final + appStagingSuffix + mustWrite(t, filepath.Join(final, "v"), "old", 0o600) + mustWrite(t, filepath.Join(staging, "v"), "new", 0o600) + prev, err := swapInAppDir(final, staging, func(string) error { return nil }) + if err != nil || prev != final+appPreviousSuffix { + t.Fatalf("swap = (%q, %v)", prev, err) + } + if mustRead(t, filepath.Join(final, "v")) != "new" || mustRead(t, filepath.Join(prev, "v")) != "old" { + t.Fatal("swap did not place new and keep old") + } + // A second swap must refuse to clobber an unretired previous dir. + mustWrite(t, filepath.Join(staging, "v"), "newer", 0o600) + if _, err := swapInAppDir(final, staging, nil); err == nil { + t.Fatal("swap overwrote an existing .previous dir") + } + if mustRead(t, filepath.Join(prev, "v")) != "old" { + t.Fatal("previous dir changed") + } + assertAbsent(t, staging) // a refused swap does not leave staging for the supervisor + // Fresh install (nothing to replace) returns no previous dir. + fresh := filepath.Join(base, "io.test.fresh") + mustWrite(t, filepath.Join(fresh+appStagingSuffix, "v"), "x", 0o600) + if prev, err := swapInAppDir(fresh, fresh+appStagingSuffix, nil); err != nil || prev != "" { + t.Fatalf("fresh swap = (%q, %v)", prev, err) + } +} + +// ── crash recovery never deletes state ───────────────────────────────────── + +func TestRecoverInterruptedInstall(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.crash" + final := filepath.Join(root, id) + prev := final + appPreviousSuffix + + // Died after moving the live install aside: put it back. + mustWrite(t, filepath.Join(prev, "identity-evm.json"), "key", 0o600) + notes, err := recoverInterruptedInstall(final, id) + if err != nil || len(notes) != 1 { + t.Fatalf("recover = (%v, %v)", notes, err) + } + if mustRead(t, filepath.Join(final, "identity-evm.json")) != "key" { + t.Fatal("interrupted install not restored") + } + assertAbsent(t, prev) + + // Died after the swap but before retiring the old dir: back it up. + mustWrite(t, filepath.Join(prev, "identity-evm.json"), "older key", 0o600) + if _, err := recoverInterruptedInstall(final, id); err != nil { + t.Fatal(err) + } + assertAbsent(t, prev) + backups, err := os.ReadDir(filepath.Join(appStoreBackupRoot(), id)) + if err != nil || len(backups) != 1 { + t.Fatalf("leftover previous dir not backed up: %v, %v", backups, err) + } + if got := mustRead(t, filepath.Join(appStoreBackupRoot(), id, backups[0].Name(), "identity-evm.json")); got != "older key" { + t.Fatalf("backup = %q", got) + } + if meta, ok := readBackupMeta(filepath.Join(appStoreBackupRoot(), id, backups[0].Name())); !ok || meta.Kind != backupKindRecovered || !meta.Pinned { + t.Fatalf("a crash leftover must be kept as a pinned backup, meta = %+v (ok=%v)", meta, ok) + } + // Died before its swap: a manifest-bearing staging dir the supervisor + // would adopt as a second copy of the app. Discarded. + mustWrite(t, filepath.Join(final+appStagingSuffix, "manifest.json"), "{}", 0o644) + mustWrite(t, filepath.Join(final+appStagingSuffix, "bin", "app"), "new", 0o755) + if notes, err := recoverInterruptedInstall(final, id); err != nil || len(notes) != 1 { + t.Fatalf("recover staging = (%v, %v)", notes, err) + } + assertAbsent(t, final+appStagingSuffix) + if mustRead(t, filepath.Join(final, "identity-evm.json")) != "key" { + t.Fatal("recovery touched the live install") + } + // Nothing to do on a clean layout. + if notes, err := recoverInterruptedInstall(final, id); err != nil || notes != nil { + t.Fatalf("clean recover = (%v, %v)", notes, err) + } +} + +func TestAppStoreForceInstallAfterCrashKeepsState(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.crashinstall" + appDir := filepath.Join(root, id) + v1 := writeVersionedBundle(t, id, "1.0.0", "v1") + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{v1, "--local"}) }) + seedAppState(t, appDir) + // Simulate the old pilotctl dying mid-swap: live dir moved aside, a + // half-built staging dir next to it, nothing at . + if err := os.Rename(appDir, appDir+appPreviousSuffix); err != nil { + t.Fatal(err) + } + mustWrite(t, filepath.Join(appDir+appStagingSuffix, "bin", "app"), "partial", 0o755) + + v2 := writeVersionedBundle(t, id, "1.1.0", "v2") + _ = captureStderr(t, func() { + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{v2, "--local", "--force"}) }) + }) + assertAppStateKept(t, appDir, "after recovering an interrupted install") + if m, _, err := readInstalledManifest(appDir); err != nil || m.AppVersion != "1.1.0" { + t.Fatalf("installed manifest = %v, %v", m, err) + } + assertAbsent(t, appDir+appPreviousSuffix) + assertAbsent(t, appDir+appStagingSuffix) +} + +// ── backups: out of the install root, detached, pruned, never lost ───────── + +func TestRetireAppDirKeepsNewestBackupsDetached(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.retire" + live := filepath.Join(root, id) + mustWrite(t, filepath.Join(live, "identity-evm.json"), "key", 0o600) + + var last string + for i := 0; i < appBackupKeep+2; i++ { + prev := live + appPreviousSuffix + mf := validManifestJSON(id, strings.Repeat("a", 64)) + mustWrite(t, filepath.Join(prev, "manifest.json"), string(mf), 0o644) + mustWrite(t, filepath.Join(prev, "bin", "app"), "old binary", 0o755) + if err := os.Link(filepath.Join(live, "identity-evm.json"), filepath.Join(prev, "identity-evm.json")); err != nil { + t.Fatal(err) + } + dst, err := retireAppDir(prev, id, appBackupMeta{Kind: backupKindReinstall}) + if err != nil { + t.Fatalf("retire %d: %v", i, err) + } + last = dst + assertAbsent(t, prev) + } + backups, err := os.ReadDir(filepath.Join(appStoreBackupRoot(), id)) + if err != nil || len(backups) != appBackupKeep { + t.Fatalf("got %d backups (%v), want the newest %d", len(backups), err, appBackupKeep) + } + if filepath.Base(last) != backups[len(backups)-1].Name() || !strings.HasSuffix(last, "-v1.0.0") { + t.Fatalf("newest backup %s not kept (have %v)", last, backups) + } + assertAbsent(t, filepath.Join(last, "bin", "app")) // binaries are not state + liveFi, _ := os.Stat(filepath.Join(live, "identity-evm.json")) + bakFi, err := os.Stat(filepath.Join(last, "identity-evm.json")) + if err != nil { + t.Fatal(err) + } + if os.SameFile(liveFi, bakFi) { + t.Error("backup key still shares an inode with the live key") + } + if bakFi.Mode().Perm() != 0o600 || mustRead(t, filepath.Join(last, "identity-evm.json")) != "key" { + t.Errorf("backup key mode/content wrong: %v", bakFi.Mode()) + } +} + +// ── end to end through a signed catalogue: install, no-op, upgrade ───────── + +func tarGzBundle(t *testing.T, dir string) (path, sha string) { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, rel := range []string{"manifest.json", "bin/app"} { + body, err := os.ReadFile(filepath.Join(dir, rel)) + if err != nil { + t.Fatal(err) + } + if err := tw.WriteHeader(&tar.Header{Name: rel, Size: int64(len(body)), Typeflag: tar.TypeReg, Mode: 0o755}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(body); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(buf.Bytes()) + path = filepath.Join(t.TempDir(), "bundle.tar.gz") + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return path, hex.EncodeToString(sum[:]) +} + +// catalogueTestApp is one entry of a test catalogue. +type catalogueTestApp struct { + id, version, bundlePath, bundleSHA string +} + +// publishSignedCatalogue writes a one-app catalogue signed with an ephemeral +// catalogue key (restored at test end) where loadCatalogue will fetch it. +func publishSignedCatalogue(t *testing.T, catPath, id, version, bundlePath, bundleSHA string) { + t.Helper() + publishSignedCatalogueApps(t, catPath, catalogueTestApp{id, version, bundlePath, bundleSHA}) +} + +// publishSignedCatalogueApps is publishSignedCatalogue for several apps. +func publishSignedCatalogueApps(t *testing.T, catPath string, apps ...catalogueTestApp) { + t.Helper() + entries := make([]map[string]any, 0, len(apps)) + for _, a := range apps { + entries = append(entries, map[string]any{ + "id": a.id, "version": a.version, "description": "stateful test app", + "bundle_url": "file://" + a.bundlePath, "bundle_sha256": a.bundleSHA, + }) + } + body, err := json.Marshal(map[string]any{"version": 2, "apps": entries}) + if err != nil { + t.Fatal(err) + } + sig, restore := catalogtrust.SignWithEphemeralKey(body) + t.Cleanup(restore) + mustWrite(t, catPath, string(body), 0o644) + mustWrite(t, catPath+".sig", base64.StdEncoding.EncodeToString(sig), 0o644) +} + +func TestAppStoreUpgradeThroughCatalogueKeepsAppState(t *testing.T) { + root := isolateAppStoreTest(t) + const id = "io.test.catalogueupgrade" + appDir := filepath.Join(root, id) + catPath := filepath.Join(t.TempDir(), "catalogue.json") + t.Setenv("PILOT_APPSTORE_CATALOG_URL", "file://"+catPath) + + v1Tar, v1SHA := tarGzBundle(t, writeVersionedBundle(t, id, "1.0.0", "v1")) + publishSignedCatalogue(t, catPath, id, "1.0.0", v1Tar, v1SHA) + _ = captureStdout(t, func() { cmdAppStoreInstall([]string{id}) }) + if _, err := os.Stat(filepath.Join(appDir, manifest.SideloadMarkerName)); err == nil { + t.Fatal("catalogue install was marked sideloaded") + } + seedAppState(t, appDir) + + // The docs' generic step, re-run on an installed app: a no-op. + if out := captureStdout(t, func() { cmdAppStoreInstall([]string{id}) }); !strings.Contains(out, "already installed") { + t.Fatalf("re-install without --force should be a no-op, got:\n%s", out) + } + + // A new release lands in the catalogue; the hourly updater runs + // `pilotctl appstore upgrade --all`. + v2Tar, v2SHA := tarGzBundle(t, writeVersionedBundle(t, id, "1.1.0", "v2")) + publishSignedCatalogue(t, catPath, id, "1.1.0", v2Tar, v2SHA) + out := captureStdout(t, func() { cmdAppStoreUpgrade([]string{"--all"}) }) + if !strings.Contains(out, "state: kept") { + t.Errorf("upgrade output should report kept state, got:\n%s", out) + } + assertAppStateKept(t, appDir, "after appstore upgrade --all") + if m, _, err := readInstalledManifest(appDir); err != nil || m.AppVersion != "1.1.0" { + t.Fatalf("upgraded manifest = %v, %v", m, err) + } + if got := strings.TrimSpace(mustRead(t, filepath.Join(appDir, bundleSHAMarker))); got != v2SHA { + t.Errorf("bundle sha marker = %s, want the new bundle's %s", got, v2SHA) + } + if outdated, err := findOutdated(); err != nil || len(outdated) != 0 { + t.Errorf("still outdated after upgrade: %v, %v", outdated, err) + } + backups, err := os.ReadDir(filepath.Join(filepath.Dir(root), "app-backups", id)) + if err != nil || len(backups) != 1 || !strings.HasSuffix(backups[0].Name(), "-v1.0.0") { + t.Fatalf("upgrade backup = %v, %v", backups, err) + } +} diff --git a/cmd/pilotctl/appstore_update.go b/cmd/pilotctl/appstore_update.go index a7999b91..600fb4e2 100644 --- a/cmd/pilotctl/appstore_update.go +++ b/cmd/pilotctl/appstore_update.go @@ -234,11 +234,31 @@ func cmdAppStoreUpgrade(args []string) { targets = []outdatedApp{o} } + var failed []string for _, o := range targets { fmt.Printf("==> upgrading %s %s → %s\n", o.ID, o.Installed, o.Available) // --force: install over the existing app dir; the supervisor applies the - // version bump (and refuses a downgrade) on its next rescan. - cmdAppStoreInstall([]string{o.ID, "--force"}) + // version bump (and refuses a downgrade) on its next rescan. The app's + // state (keys, data.db, secrets, cap-state, audit log) is carried into + // the new install and the replaced dir is kept as a backup — see + // appstore_state.go. This is the path the hourly updater drives. + install := func() { cmdAppStoreInstall([]string{o.ID, "--force"}) } + if len(targets) == 1 { + install() + continue + } + // One app that cannot be upgraded (its error is printed, and it is + // left exactly as it was) must not stop the upgrade of every app + // after it, security fixes included. + if f := runTrappingFatal(install); f != nil { + failed = append(failed, o.ID) + fmt.Fprintf(os.Stderr, "==> %s was not upgraded (%s) and stays at %s; continuing with the remaining apps\n", o.ID, f.Code, o.Installed) + } + } + if len(failed) > 0 { + fatalHint("upgrade_failed", + "every app that failed is unchanged and its error is printed above; fix the cause and re-run `pilotctl appstore upgrade `", + "upgraded %s; %d failed: %s", pluralApps(len(targets)-len(failed)), len(failed), strings.Join(failed, ", ")) } fmt.Printf("\nupgraded %s\n", strings.TrimSpace(pluralApps(len(targets)))) } diff --git a/cmd/pilotctl/fatal_trap.go b/cmd/pilotctl/fatal_trap.go new file mode 100644 index 00000000..ddf97aa3 --- /dev/null +++ b/cmd/pilotctl/fatal_trap.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import "os" + +// A fatal error ends the process: fatalCode and fatalHint print it and exit 1. +// runTrappingFatal lets a command run another command's code path, which may +// end that way, and carry on afterwards. `appstore upgrade --all` uses it so +// one app that cannot be upgraded does not stop the upgrade of every app +// sorted after it (the hourly updater runs `upgrade --all`). + +// trappedFatal is the error a trapped fatalCode/fatalHint call ended with. The +// message and hint have already been printed. +type trappedFatal struct { + Code string + Message string +} + +// fatalTrapDepth > 0 while runTrappingFatal is running. +var fatalTrapDepth int + +// exitAfterFatal ends the process after fatalCode/fatalHint printed an +// error, or, inside runTrappingFatal, unwinds to it (running deferred calls, +// such as releasing an app's install lock, on the way). +func exitAfterFatal(code, msg string) { + if fatalTrapDepth > 0 { + panic(trappedFatal{Code: code, Message: msg}) + } + os.Exit(1) +} + +// runTrappingFatal runs fn and returns the fatal error it ended with, or nil +// when it returned normally. Any other panic propagates. +func runTrappingFatal(fn func()) (failure *trappedFatal) { + fatalTrapDepth++ + defer func() { + fatalTrapDepth-- + if r := recover(); r != nil { + tf, ok := r.(trappedFatal) + if !ok { + panic(r) + } + failure = &tf + } + }() + fn() + return nil +} diff --git a/cmd/pilotctl/main.go b/cmd/pilotctl/main.go index 58fa017e..0dec3553 100644 --- a/cmd/pilotctl/main.go +++ b/cmd/pilotctl/main.go @@ -157,7 +157,7 @@ func fatalCode(code string, format string, args ...interface{}) { } else { fmt.Fprintf(os.Stderr, "error: %s\n", msg) } - os.Exit(1) + exitAfterFatal(code, msg) } // classifyDaemonError inspects an error string from the daemon and, when it @@ -214,7 +214,7 @@ func fatalHint(code, hint, format string, args ...interface{}) { // fatalHint knows where the output ends, because only it exits. printNextSteps(exitNextSteps) } - os.Exit(1) + exitAfterFatal(code, msg) } func fatal(format string, args ...interface{}) {