diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37ceb87d..c25f3219 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,8 @@ jobs: node --test scripts/test-check-secrets.mjs - name: Verify publisher authentication and private release snapshots run: node --test scripts/test-native-release-auth.mjs + - name: Signed APT repository builder + run: node --test scripts/test-build-apt-repo.mjs - name: Managed llama.cpp packaging (pins, notices, relative links, deb repack) run: node --test scripts/test-llama-runtime.mjs scripts/test-llama-build-inputs.mjs - name: Packaging PATH ignores host node hijacks @@ -52,3 +54,124 @@ jobs: run: | node --test --test-reporter=tap scripts/test-install-auth.mjs bash scripts/test-install-appimage.sh + # Pull requests that touch packaging inputs also run the packaging checks on + # both architectures and build the Flatpak. Change detection uses Git only, + # so no third-party path-filter action has to be trusted. + packaging-changes: + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + packaging: ${{ steps.diff.outputs.packaging }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + persist-credentials: false + - id: diff + name: Detect packaging, scripts, desktop, core or workflow changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA") + printf '%s\n' "$changed" + if printf '%s\n' "$changed" | grep -Eq '^(packaging|scripts|src-tauri|native|\.github/workflows)/'; then + echo packaging=true >> "$GITHUB_OUTPUT" + else + echo packaging=false >> "$GITHUB_OUTPUT" + fi + packaging-dry-run: + needs: packaging-changes + if: needs.packaging-changes.outputs.packaging == 'true' + name: packaging dry run (${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runner: ubuntu-24.04 + - arch: aarch64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + - name: Debian packaging tools + run: sudo apt-get update && sudo apt-get install -y lintian desktop-file-utils appstream + - name: Architecture names, release authentication and signed targets + run: node --test scripts/test-native-arch.mjs scripts/test-native-release-auth.mjs + - name: Debian package assembly, runtime repack and desktop metadata + run: node --test scripts/test-native-deb.mjs scripts/test-llama-runtime.mjs scripts/test-native-desktop-metadata.mjs scripts/test-native-app-run.mjs + - name: Packaging PATH and pinned sources + run: node --test scripts/test-native-packaging-env.mjs scripts/test-native-source-fetch.mjs + - name: Installer on this architecture + run: | + node --test --test-reporter=tap scripts/test-install-auth.mjs + bash scripts/test-install-appimage.sh + - name: Shell syntax of packaging scripts + run: | + for script in packaging/native-app-run.sh packaging/native-runtime/build.sh packaging/flatpak/build-in-sdk.sh scripts/*.sh; do + bash -n "$script" + done + python3 -m py_compile scripts/build-flatpak.py + flatpak: + needs: packaging-changes + if: needs.packaging-changes.outputs.packaging == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 150 + env: + RUSTUP_TOOLCHAIN: '1.95.0' + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + - name: Install the pinned Rust toolchain + run: rustup toolchain install 1.95.0 --profile minimal + - name: Flatpak tools and Flathub + run: | + sudo apt-get update + sudo apt-get install -y flatpak flatpak-builder dbus-daemon + # flatpak-builder's bubblewrap needs unprivileged user namespaces, + # which Ubuntu 24.04 restricts through AppArmor by default. + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + flatpak remote-add --user --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo + - name: Restore the flatpak-builder and SDK build caches + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.flatpak-builder + target/flatpak/sdk-cargo-cache + target/flatpak/sdk-native-cache + key: flatpak-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Cargo.lock', 'ui/package-lock.json', 'tools/llama.cpp.pin', 'packaging/flatpak/**', 'scripts/build-flatpak.py') }} + restore-keys: | + flatpak-${{ runner.os }}-${{ runner.arch }}- + - name: Build the Flatpak with the GNOME 50 SDK (CPU-only llama.cpp) + # Releases also build the Vulkan module; it is skipped here for time. + run: python3 scripts/build-flatpak.py --cpu-only-llama --install-deps-from=flathub --state-dir ~/.flatpak-builder + - name: Install and start the bundle + run: | + flatpak install --user --noninteractive target/flatpak/ShadowCode_*_x86_64.flatpak + # `flatpak run` needs a session bus; the runner has no desktop session. + dbus-run-session -- flatpak run com.shadowfetch.shadowcode --version + dbus-run-session -- flatpak run --command=/app/lib/shadowcode/llama-server com.shadowfetch.shadowcode --version + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: flatpak-pr-build + retention-days: 7 + if-no-files-found: ignore + path: | + target/flatpak/*.flatpak + target/flatpak/SHA256SUMS + target/flatpak/SOURCE-INPUTS.json + target/flatpak/com.shadowfetch.shadowcode.json diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 9523a0ff..220d2ff0 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -8,7 +8,24 @@ permissions: contents: read jobs: verify: - runs-on: ubuntu-24.04 + # x86_64 and aarch64 (GitHub's Arm runners) build and qualify natively. + # The x86_64 leg keeps its historical artifact names (empty suffix). + name: verify (${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runner: ubuntu-24.04 + appimage: amd64 + deb: amd64 + suffix: '' + - arch: aarch64 + runner: ubuntu-24.04-arm + appimage: aarch64 + deb: arm64 + suffix: '-aarch64' + runs-on: ${{ matrix.runner }} timeout-minutes: 120 env: RUSTUP_TOOLCHAIN: '1.95.0' @@ -23,15 +40,15 @@ jobs: cache: npm cache-dependency-path: ui/package-lock.json - name: Native window dependencies - run: sudo apt-get update && sudo apt-get install -y libwebkit2gtk-4.1-dev libgtk-3-dev librsvg2-dev libayatana-appindicator3-dev libasound2-dev patchelf webkit2gtk-driver xvfb appstream dbus-daemon dbus-bin lintian desktop-file-utils + run: sudo apt-get update && sudo apt-get install -y libwebkit2gtk-4.1-dev libgtk-3-dev librsvg2-dev libayatana-appindicator3-dev libasound2-dev patchelf webkit2gtk-driver xvfb appstream dbus-daemon dbus-bin lintian desktop-file-utils xdg-utils - name: Test release staging and immutable retry behavior - run: node --test scripts/test-native-release.mjs scripts/test-native-release-verification.mjs scripts/test-publisher-auth.mjs scripts/test-native-release-signing.mjs scripts/test-native-test-session.mjs scripts/test-clean-host-mcp-receipt.mjs scripts/test-clean-host-container.mjs scripts/test-native-app-run.mjs scripts/test-llama-build-inputs.mjs && node scripts/test-native-process-ownership.mjs + run: node --test scripts/test-native-release.mjs scripts/test-native-release-verification.mjs scripts/test-publisher-auth.mjs scripts/test-native-release-signing.mjs scripts/test-native-test-session.mjs scripts/test-clean-host-mcp-receipt.mjs scripts/test-clean-host-container.mjs scripts/test-native-app-run.mjs scripts/test-llama-build-inputs.mjs scripts/test-native-arch.mjs && node scripts/test-native-process-ownership.mjs - name: Verify publisher authentication and private release snapshots run: node --test scripts/test-native-release-auth.mjs scripts/test-check-secrets.mjs - name: Package version run: echo "VERSION=$(node -p "require('./src-tauri/tauri.conf.json').version")" >> "$GITHUB_ENV" - name: Managed llama.cpp build dependencies - run: sudo apt-get install -y cmake ninja-build libssl-dev libvulkan-dev glslc spirv-headers + run: sudo apt-get install -y cmake ninja-build libssl-dev libvulkan-dev glslc spirv-headers gcc-14 g++-14 - run: npm --prefix ui ci && npm --prefix ui run build && npm --prefix ui test - name: Install pinned Rust toolchain run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy @@ -50,7 +67,7 @@ jobs: if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: built-project-cleanup-diagnostics + name: built-project-cleanup-diagnostics${{ matrix.suffix }} path: artifacts/built-project-cleanup - name: Exercise the native CLI without a display run: node scripts/test-native-cli.mjs @@ -59,7 +76,7 @@ jobs: - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() with: - name: native-stress-diagnostics + name: native-stress-diagnostics${{ matrix.suffix }} path: artifacts/native-stress - name: Exercise the native terminal through a real PTY run: node scripts/test-native-tui.mjs @@ -74,14 +91,14 @@ jobs: - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() with: - name: native-mcp-server-diagnostics + name: native-mcp-server-diagnostics${{ matrix.suffix }} path: | artifacts/native-mcp-server* artifacts/native-mcp-peer - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() with: - name: native-cli-diagnostics + name: native-cli-diagnostics${{ matrix.suffix }} path: | artifacts/native-cli artifacts/native-tui @@ -95,7 +112,7 @@ jobs: - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() with: - name: native-window-diagnostics + name: native-window-diagnostics${{ matrix.suffix }} path: | artifacts/native artifacts/native-markdown @@ -113,7 +130,7 @@ jobs: tools/llama.cpp tools/spirv-headers tools/spirv-headers-prefix - key: llama-cpp-${{ runner.os }}-${{ hashFiles('tools/llama.cpp.pin', 'scripts/build-llama.cpp.sh') }} + key: llama-cpp-${{ runner.os }}${{ matrix.suffix }}-${{ hashFiles('tools/llama.cpp.pin', 'scripts/build-llama.cpp.sh') }} - name: Build the managed llama.cpp runtime (Vulkan + all CPU variants) run: bash scripts/build-llama.cpp.sh --no-user-install - name: Check the managed runtime packaging steps @@ -121,11 +138,11 @@ jobs: - name: Build native development packages with dependency notices run: node scripts/build-native.mjs - name: Inspect both packages - run: node scripts/check-native-package.mjs "target/release/bundle/appimage/ShadowCode_${VERSION}_amd64.AppImage" "target/release/bundle/deb/ShadowCode_${VERSION}_amd64.deb" + run: node scripts/check-native-package.mjs "target/release/bundle/appimage/ShadowCode_${VERSION}_${{ matrix.appimage }}.AppImage" "target/release/bundle/deb/ShadowCode_${VERSION}_${{ matrix.deb }}.deb" - name: Build runtime-only clean-host qualification image - run: docker build -f scripts/clean-host-runtime.Dockerfile -t shadowcode-clean-runtime:ci scripts + run: docker build --build-arg TARGETARCH="$(dpkg --print-architecture)" -f scripts/clean-host-runtime.Dockerfile -t shadowcode-clean-runtime:ci scripts - name: Install and first-launch both packages in an offline clean host - run: node scripts/test-clean-host-packages.mjs "target/release/bundle/appimage/ShadowCode_${VERSION}_amd64.AppImage" "target/release/bundle/deb/ShadowCode_${VERSION}_amd64.deb" artifacts/native-package/SHA256SUMS + run: node scripts/test-clean-host-packages.mjs "target/release/bundle/appimage/ShadowCode_${VERSION}_${{ matrix.appimage }}.AppImage" "target/release/bundle/deb/ShadowCode_${VERSION}_${{ matrix.deb }}.deb" artifacts/native-package/SHA256SUMS - name: Reject AppImage extraction writes that fail run: node scripts/test-native-runtime-write-errors.mjs - name: Rebuild the shipped runtime sources without network access @@ -138,58 +155,58 @@ jobs: bash scripts/test-install-appimage.sh - name: Exercise the packaged CLI without a display env: - SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_amd64.AppImage + SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_${{ matrix.appimage }}.AppImage SHADOW_CLI_ARGS: '["--appimage-extract-and-run"]' SHADOW_CLI_ARTIFACTS: artifacts/native-package/cli run: node scripts/test-native-cli.mjs - name: Exercise the packaged native terminal env: - SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_amd64.AppImage + SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_${{ matrix.appimage }}.AppImage SHADOW_CLI_ARGS: '["--appimage-extract-and-run"]' SHADOW_TUI_ARTIFACTS: artifacts/native-package/tui run: node scripts/test-native-tui.mjs - name: Exercise the packaged native window env: - SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_amd64.AppImage + SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_${{ matrix.appimage }}.AppImage SHADOW_DESKTOP_ARGS: '["--appimage-extract-and-run","ui"]' SHADOW_NATIVE_DEFAULT_PROFILE: '1' SHADOW_NATIVE_ARTIFACTS: artifacts/native-package/window run: node scripts/run-native-x11.mjs node scripts/test-native-desktop.mjs - name: Exercise the packaged Markdown worker under its native security policy env: - SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_amd64.AppImage + SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_${{ matrix.appimage }}.AppImage SHADOW_DESKTOP_ARGS: '["--appimage-extract-and-run"]' SHADOW_NATIVE_ARTIFACTS: artifacts/native-package/markdown run: node scripts/run-native-x11.mjs node scripts/test-native-markdown.mjs - name: Exercise the packaged native MCP server env: - SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_amd64.AppImage + SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_${{ matrix.appimage }}.AppImage SHADOW_CLI_ARGS: '["--appimage-extract-and-run"]' SHADOW_MCP_ARTIFACTS: artifacts/native-package/mcp-server run: node scripts/test-native-mcp-server.mjs - name: Exercise the packaged native MCP HTTP server env: - SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_amd64.AppImage + SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_${{ matrix.appimage }}.AppImage SHADOW_CLI_ARGS: '["--appimage-extract-and-run"]' SHADOW_MCP_ARTIFACTS: artifacts/native-package/mcp-server-http SHADOW_MCP_TRANSPORT: http run: node scripts/test-native-mcp-server.mjs - name: Verify packaged MCP interoperability with official TypeScript clients env: - SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_amd64.AppImage + SHADOW_DESKTOP_BINARY: ${{ github.workspace }}/target/release/bundle/appimage/ShadowCode_${{ env.VERSION }}_${{ matrix.appimage }}.AppImage SHADOW_CLI_ARGS: '["--appimage-extract-and-run"]' SHADOW_MCP_PEER_ARTIFACTS: artifacts/native-package/mcp-peer run: node scripts/test-native-mcp-peer.mjs - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() with: - name: native-package-diagnostics + name: native-package-diagnostics${{ matrix.suffix }} path: | artifacts/native-package artifacts/native-x11 - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: native-linux-development + name: native-linux-development${{ matrix.suffix }} retention-days: 14 path: | target/release/bundle/appimage/*.AppImage diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59b6ef6a..02918d0c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -135,8 +135,68 @@ jobs: path: ${{ runner.temp }}/unsigned-release if-no-files-found: error compression-level: 0 + # aarch64 packages are built and qualified natively on GitHub's Arm runner + # with the same required gates and handed off unsigned exactly like x86_64. + # The signer authenticates each target's bundle; its release metadata carries + # an -aarch64 suffix so both targets publish into one release. + build-aarch64: + needs: configuration + runs-on: ubuntu-24.04-arm + timeout-minutes: 180 + env: + RUSTUP_TOOLCHAIN: '1.95.0' + SHADOWCODE_BUILD_JOBS: '4' + SHADOWCODE_REQUIRE_LINTIAN: '1' + outputs: + artifact-id: ${{ steps.unsigned.outputs.artifact-id }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: ui/package-lock.json + - name: Native package dependencies + run: sudo apt-get update && sudo apt-get install -y libwebkit2gtk-4.1-dev libgtk-3-dev librsvg2-dev libayatana-appindicator3-dev libasound2-dev patchelf webkit2gtk-driver xvfb appstream dbus-daemon dbus-bin lintian desktop-file-utils xdg-utils cmake ninja-build libssl-dev libvulkan-dev glslc spirv-headers gcc-14 g++-14 + - name: Install the pinned Rust toolchain + run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy + - name: Run every required release gate in order on aarch64 + run: | + set -euo pipefail + gates=$(node --input-type=module -e "const { REQUIRED_GATES } = await import('./scripts/native-release-verification.mjs'); console.log(REQUIRED_GATES.join(' '))") + for gate in $gates; do + # The installer gate verifies the checksums the packages gate wrote. + if [ "$gate" = installer ]; then cp artifacts/native-package/SHA256SUMS target/release/bundle/SHA256SUMS; fi + node scripts/native-release-verification.mjs run "$gate" + done + - name: Preserve aarch64 qualification diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: native-release-aarch64-diagnostics + path: | + artifacts/release-verification + artifacts/built-project-cleanup + artifacts/native + artifacts/native-markdown + artifacts/native-endurance + artifacts/native-x11 + artifacts/native-package + - name: Prepare immutable unsigned aarch64 handoff with all required receipts + run: node scripts/prepare-native-release.mjs "$RUNNER_TEMP/unsigned-release" + - id: unsigned + name: Retain this run's exact unsigned aarch64 handoff + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: unsigned-release-aarch64-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/unsigned-release + if-no-files-found: error + compression-level: 0 + sign: - needs: [configuration, build] + needs: [configuration, build, build-aarch64] runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: @@ -155,21 +215,31 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: '22' - - name: Require one build artifact identity + - name: Require one build artifact identity per target env: RELEASE_ARTIFACT_ID: ${{ needs.build.outputs.artifact-id }} - run: node -e "require('node:assert/strict').match(process.env.RELEASE_ARTIFACT_ID || '', /^[1-9][0-9]*$/)" + AARCH64_ARTIFACT_ID: ${{ needs.build-aarch64.outputs.artifact-id }} + run: node -e "const assert = require('node:assert/strict'); for (const id of [process.env.RELEASE_ARTIFACT_ID, process.env.AARCH64_ARTIFACT_ID]) assert.match(id || '', /^[1-9][0-9]*$/); assert.notEqual(process.env.RELEASE_ARTIFACT_ID, process.env.AARCH64_ARTIFACT_ID)" - name: Download only the immutable artifact ID produced by this build uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: artifact-ids: ${{ needs.build.outputs.artifact-id }} path: ${{ runner.temp }}/unsigned-download merge-multiple: true - - name: Sign and reverify qualified data with the protected key + - name: Download only the immutable artifact ID produced by the aarch64 build + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + artifact-ids: ${{ needs.build-aarch64.outputs.artifact-id }} + path: ${{ runner.temp }}/unsigned-download-aarch64 + merge-multiple: true + - name: Sign and reverify each target's qualified data with the protected key working-directory: trusted env: NATIVE_RELEASE_SIGNING_KEY_PEM: ${{ secrets.NATIVE_RELEASE_SIGNING_KEY_PEM }} - run: node scripts/sign-native-release.mjs "$RUNNER_TEMP/unsigned-download" release/trust "$RUNNER_TEMP/signed-release" + run: | + mkdir -m 700 "$RUNNER_TEMP/signed-release" + node scripts/sign-native-release.mjs "$RUNNER_TEMP/unsigned-download" release/trust "$RUNNER_TEMP/signed-release/x86_64" + node scripts/sign-native-release.mjs "$RUNNER_TEMP/unsigned-download-aarch64" release/trust "$RUNNER_TEMP/signed-release/aarch64" aarch64-unknown-linux-gnu - id: signed name: Retain only authenticated public release data uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -209,4 +279,81 @@ jobs: working-directory: trusted env: GH_TOKEN: ${{ github.token }} - run: node scripts/publish-native-release.mjs "$RUNNER_TEMP/signed-download" release/trust + run: node scripts/publish-native-release.mjs "$RUNNER_TEMP/signed-download/x86_64" release/trust "$RUNNER_TEMP/signed-download/aarch64" + # Signed APT repository on GitHub Pages (gh-pages branch, path apt/). Only + # the .deb that passed publisher authentication is indexed; the repository + # signing key is a separate OpenPGP key held in the protected environment. + apt-repository: + needs: [configuration, sign, publish] + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: write + environment: ${{ needs.configuration.outputs.environment }} + steps: + - name: Checkout independently approved tooling and public trust + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ needs.configuration.outputs.tooling }} + path: trusted + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + - name: Require a pinned APT signing fingerprint + env: + APT_FPR: ${{ vars.NATIVE_APT_SIGNING_FINGERPRINT }} + run: node -e "require('node:assert/strict').match(process.env.APT_FPR || '', /^[0-9A-F]{40}$/, 'Provision NATIVE_APT_SIGNING_FINGERPRINT')" + - name: Download only this signing job's immutable public bundle + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + artifact-ids: ${{ needs.sign.outputs.artifact-id }} + path: ${{ runner.temp }}/signed-download + merge-multiple: true + - name: Authenticate each target's .deb against the tracked trust bundle + working-directory: trusted + run: | + version="${GITHUB_REF_NAME#v}" + mkdir -m 700 "$RUNNER_TEMP/verified-deb" + for pair in x86_64:amd64 aarch64:arm64; do + bash scripts/verify-native-release.sh \ + --bundle-dir "$RUNNER_TEMP/signed-download/${pair%%:*}" \ + --trust-dir release/trust \ + --artifact "ShadowCode_${version}_${pair#*:}.deb" \ + --stage-dir "$RUNNER_TEMP/verified-deb/${pair#*:}" \ + --expect-version "$version" + done + - name: Checkout the published Pages branch + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + continue-on-error: true + id: pages + with: + ref: gh-pages + path: pages + - name: Build and sign the repository, keeping earlier pool packages + working-directory: trusted + env: + APT_REPO_SIGNING_KEY: ${{ secrets.NATIVE_APT_SIGNING_KEY }} + APT_FPR: ${{ vars.NATIVE_APT_SIGNING_FINGERPRINT }} + run: | + previous=() + [ -d "$GITHUB_WORKSPACE/pages/apt" ] && previous=(--previous "$GITHUB_WORKSPACE/pages/apt") + node scripts/build-apt-repo.mjs --out "$RUNNER_TEMP/apt" --fingerprint "$APT_FPR" "${previous[@]}" "$RUNNER_TEMP"/verified-deb/*/*.deb + gpgv --keyring "$RUNNER_TEMP/apt/shadowcode-archive-keyring.gpg" "$RUNNER_TEMP/apt/dists/stable/InRelease" + - name: Publish to gh-pages + env: + GH_TOKEN: ${{ github.token }} + run: | + if [ "${{ steps.pages.outcome }}" != success ]; then + rm -rf pages && mkdir pages && cd pages + git init -q -b gh-pages + git remote add origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + touch .nojekyll + else + cd pages + fi + rm -rf apt && cp -a "$RUNNER_TEMP/apt" apt + git add -A + git -c user.name=github-actions -c user.email=github-actions@users.noreply.github.com \ + commit -q -m "APT repository for ${GITHUB_REF_NAME}" + git push origin HEAD:gh-pages diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a01e604..c4a2935b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,61 @@ # Changelog +## 2.1.0: ARM64, a signed APT repository and a safer installer + +- **Signed APT repository.** Each release's authenticated `.deb` is indexed in + an APT repository on GitHub Pages (`stable`/`main`, amd64 and arm64), signed + with a dedicated OpenPGP key held in the protected release environment. + Install it with a deb822 `.sources` file and a `Signed-By` keyring in + `/etc/apt/keyrings`; updates and downgrade protection then come from APT. + See the README's Debian section and `docs/DISTRIBUTING.md`. +- **ARM64 (aarch64) builds.** The AppImage (`_aarch64.AppImage`), Debian package (`_arm64.deb`, `Architecture: arm64`), Flatpak (`_aarch64.flatpak`), the source-built AppImage runtime and the bundled llama.cpp runtime are built natively on aarch64. x86_64 package names and contents are unchanged. +- Native CI runs on x86_64 and on GitHub's `ubuntu-24.04-arm` runners. The release workflow qualifies aarch64 packages with every required gate, signs them with their own envelope and publishes them in the same release as x86_64. Their signature files carry an `-aarch64` suffix (`RELEASE-AUTH-aarch64`, `RELEASE-AUTH-aarch64.sig`, `RELEASE-MANIFEST-aarch64.json`, `SHA256SUMS-aarch64`); the x86_64 names are unchanged. The APT repository indexes the arm64 `.deb` too. +- On ARM64, the in-app update notice names the aarch64 AppImage and its signature files, and treats a release as signed only when they are present. +- Release authentication accepts `aarch64-unknown-linux-gnu` envelopes with their own asset names. Trust policies may list several targets; releases for a target the policy does not name, or that switch target against accepted history, are refused. The publisher's trust policy now names both `x86_64-unknown-linux-gnu` and `aarch64-unknown-linux-gnu`. +- The AppImage installer installs only the AppImage for the machine's architecture and requires the signed target to match it. +- `build-flatpak.py` gains `--arch`, `--state-dir`, `--install-deps-from` and `--cpu-only-llama`. +- Pull requests that touch packaging, scripts, the desktop, the core or workflows build the Flatpak with the GNOME 50 SDK and run the packaging checks on both architectures. +- Flatpak: hostname allow-list mode for agent shell commands now works. The host bubblewrap gets its own private network namespace, and a small packaged relay forwards loopback port 3128 to ShadowCode's allow-list proxy over a Unix socket. If the relay or proxy socket cannot be set up, the command is still refused. +- **The window remembers its size again.** ShadowCode now saves the size of + the window's content, without the title bar or the shadow the desktop + adds, and reopens at that size. On Wayland desktops such as KDE Plasma + the window opens at the size you left it and no longer grows from launch + to launch. The remembered size is still fitted to the screen (at most 90% + of it, never below 520x640), so it fits a smaller screen too. Sizes saved + by earlier versions are ignored, so the first launch after updating opens + at the default size. +- **The AppImage installer checks the first window and can roll back the + database.** Before activation it snapshots the profile database with + SQLite's `VACUUM INTO` and refuses while ShadowCode is running. With a + display, it then starts the new version, which must report within 120 + seconds that its database opened (and migrated) and its main window loaded. + Otherwise the new version is stopped and the previous AppImage, llama.cpp + runtime, launchers, desktop entry and database snapshot are restored. + `--recover` does the same after an installer crash during the check. The + durable highest accepted release is not lowered by this rollback. + `SHADOWCODE_FIRST_START_CHECK` (`auto`, `require`, `skip`) and + `SHADOWCODE_FIRST_START_TIMEOUT` control the check; without a display it is + skipped. +- The desktop app writes the installer's first-start report after the + database opened and the main window finished loading, and a hidden + `shadowcode install-snapshot FILE` command makes the installer's database + snapshot. An older version still refuses a database a newer version changed, + without changing it. +- **Model history per project in the picker.** Every finished task records + the model that ran it, how long it took, its tokens, its cost when the + provider, vendor CLI or this computer reported it (otherwise shown as + *cost unavailable*, never estimated), and whether observed verification + passed. Picker rows show the median time, median reported cost, pass count + and number of tasks in this project, and **Compare history** shows all + models side by side. Stored in a new `model_outcomes` table (database + format 28, migrated with a backup like earlier formats). +- **Opt-in suggestion: the cheapest model that passed here before.** Turn on + **Suggest the cheapest model that passed here** (`routing.suggest_from_history`) + to see a suggestion you can accept or dismiss. ShadowCode never switches + models on its own; offline only local models are suggested, and spending + limits for paid API models are respected. +- New experimental endpoint `GET /api/models/outcomes`. + ## 2.0.0: Flatpak, brand identity, and remote-access fixes - Update repository, contributor, release and package links for the ShadowfetchStudio GitHub account rename. Preserve verification of existing signed releases with the same pinned repository and publisher IDs. diff --git a/Cargo.lock b/Cargo.lock index 47c65b0a..03eac00b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4477,7 +4477,7 @@ dependencies = [ [[package]] name = "shadowcode-core" -version = "2.0.0" +version = "2.1.0" dependencies = [ "anyhow", "base64 0.22.1", @@ -4535,7 +4535,7 @@ dependencies = [ [[package]] name = "shadowcode-desktop" -version = "2.0.0" +version = "2.1.0" dependencies = [ "anyhow", "gdk", diff --git a/Cargo.toml b/Cargo.toml index 2242191d..7cb6c3f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["vendor/glib-0.18.5"] resolver = "2" [workspace.package] -version = "2.0.0" +version = "2.1.0" edition = "2021" rust-version = "1.95" license = "Apache-2.0" diff --git a/README.md b/README.md index 19e0d502..d6979566 100644 --- a/README.md +++ b/README.md @@ -70,20 +70,27 @@ The code is open source; the Shadowfetch and ShadowCode product identity is cove own terminal, go from commit to pull request, preview the app you're building, dictate with your voice, and follow along from your phone. -The source tree is now **2.0.0**, with a locally buildable Flatpak and fixes for private-profile access and secret-file diff redaction. See [Flatpak installation and build instructions](docs/FLATPAK.md). The 2.0.0 source is available here; its Flatpak bundle is a local build and has not been published as a GitHub release or on Flathub. - -The last published release, ShadowCode **1.0.1**, is available in [GitHub releases](https://github.com/ShadowfetchStudio/ShadowCode/releases/tag/v1.0.1). -See the [1.0.1 release notes](docs/RELEASE_NOTES.md) for changes and qualification +ShadowCode **2.1.0** is available in [GitHub releases](https://github.com/ShadowfetchStudio/ShadowCode/releases/tag/v2.1.0) for x86_64 and, for the first time, 64-bit ARM, and from the signed APT repository below. +See the [2.1.0 release notes](docs/RELEASE_NOTES.md) for changes and qualification limits, and the [release history](CHANGELOG.md). ## Install Releases target x86_64 Linux with glibc 2.39 or newer (Ubuntu 24.04 or later). +From 2.1.0, aarch64 (64-bit ARM) packages are built and qualified too: +`ShadowCode_VERSION_aarch64.AppImage`, `ShadowCode_VERSION_arm64.deb` and +`ShadowCode_VERSION_aarch64.flatpak`. Each is signed for its own target and +published next to the x86_64 files, with its signature files named +`RELEASE-AUTH-aarch64`, `RELEASE-AUTH-aarch64.sig`, +`RELEASE-MANIFEST-aarch64.json` and `SHA256SUMS-aarch64`; download those with +the package and use the same installer and verifier commands with the aarch64 +file name. See +[Distributing](docs/DISTRIBUTING.md#arm64-aarch64). Check [GitHub releases](https://github.com/ShadowfetchStudio/ShadowCode/releases) -for published versions. The 1.0.1 packages and checksums are: +for published versions. The 2.1.0 x86_64 packages and checksums are: -- `ShadowCode_1.0.1_amd64.AppImage` -- `ShadowCode_1.0.1_amd64.deb` +- `ShadowCode_2.1.0_amd64.AppImage` +- `ShadowCode_2.1.0_amd64.deb` - `SHA256SUMS`, `RELEASE-MANIFEST.json`, `RELEASE-AUTH` and `RELEASE-AUTH.sig` (download these beside the package; the installer checks the publisher's signature before running anything) @@ -95,7 +102,7 @@ assets are checksum-only and cannot be installed by this new entry point; use the instructions shipped with that released version for those existing assets. The 0.33.0 release attempt failed before signing and publication. Its tag stays unchanged. Version 0.33.1 was the first signed release; the public trust policy -in `release/trust` now authorizes 0.33.0 through 1.0.1. +in `release/trust` now authorizes 0.33.0 through 2.1.0. For development, use [Build from source](#build-from-source). Obtain an independently authenticated installer bundle as described below, @@ -109,11 +116,13 @@ bundle and bootstrap requirements. From that independently authenticated installer bundle, run: ```bash -bash /path/to/trusted-bundle/scripts/install-appimage.sh /path/to/downloads/ShadowCode_1.0.1_amd64.AppImage +bash /path/to/trusted-bundle/scripts/install-appimage.sh /path/to/downloads/ShadowCode_2.1.0_amd64.AppImage ``` The installer takes the AppImage path (or `--recover`), resolves its own trusted -policy and keys, and verifies the adjacent release metadata. Verifier flags such +policy and keys, and verifies the adjacent release metadata. It installs only +the AppImage for the machine it runs on: `_amd64.AppImage` on x86_64 and +`_aarch64.AppImage` on aarch64, whose signed target must match the machine. Verifier flags such as `--trust-dir` are not installer options. [`scripts/install-appimage.sh`](scripts/install-appimage.sh): @@ -139,30 +148,83 @@ as `--trust-dir` are not installer options. AppImage/runtime on retry when all recorded identities still match. Run `./ShadowCode/scripts/install-appimage.sh --recover` to attempt that recovery without the original download, using the same trusted installer bundle. Changed, ambiguous or activation-stage states - are preserved for manual recovery; first-window and database rollback are - not yet qualified. + are preserved for manual recovery. +- when a display is available, checks the first window: before activation it + snapshots the profile database with SQLite's `VACUUM INTO` (close ShadowCode + first; the installer refuses while it runs), then starts the new version, + which must report that its database opened and its main window loaded + within 120 seconds. Otherwise it is stopped and the AppImage, runtime, + launchers and database snapshot are all restored; `--recover` does the same + after an installer crash during the check. `SHADOWCODE_FIRST_START_CHECK` + (`auto`, `require`, `skip`) and `SHADOWCODE_FIRST_START_TIMEOUT` (seconds) + change this. Without a display the check is skipped and says so. An older + version never opens a database a newer version changed, so a rollback + cannot corrupt it. After obtaining and verifying the intended package, run the AppImage without installing it: -`./ShadowCode_1.0.1_amd64.AppImage --appimage-extract-and-run`. FUSE is not +`./ShadowCode_2.1.0_amd64.AppImage --appimage-extract-and-run`. FUSE is not required. ### Debian package -Verify the Debian artifact using the independently trusted verifier bundle. -`verified-deb-1.0.1` must not already exist; install the verified snapshot: +The recommended route is the signed APT repository, which delivers updates +through `apt upgrade`: + +```sh +sudo install -d -m 0755 /etc/apt/keyrings +curl -fsSLo /tmp/shadowcode-archive-keyring.gpg \ + https://shadowfetchstudio.github.io/ShadowCode/apt/shadowcode-archive-keyring.gpg +gpg --show-keys --with-fingerprint /tmp/shadowcode-archive-keyring.gpg +# Compare the fingerprint with release/trust/APT-SIGNING-KEY in a reviewed +# copy of the repository at a release tag; stop if it differs. +sudo install -m 0644 /tmp/shadowcode-archive-keyring.gpg /etc/apt/keyrings/ +sudo tee /etc/apt/sources.list.d/shadowcode.sources >/dev/null <<'EOF' +Types: deb +URIs: https://shadowfetchstudio.github.io/ShadowCode/apt +Suites: stable +Components: main +Architectures: amd64 arm64 +Signed-By: /etc/apt/keyrings/shadowcode-archive-keyring.gpg +EOF +sudo apt update && sudo apt install shadow-code +``` + +The keyring is fetched once, over HTTPS, and must be checked against the +fingerprint in a reviewed copy of the repository: like the AppImage +installer's trust bundle, a key downloaded beside the packages does not +authenticate itself. `Signed-By` scopes the key to this one source; it is not +added to APT's global trusted keys. The repository key is a separate OpenPGP +key from the Ed25519 publisher key: CI indexes only a `.deb` that already +passed `verify-native-release.sh` against `release/trust`, then signs +`InRelease` and `Release.gpg`. + +Downgrade and rollback protection comes from APT. It verifies `InRelease` with +the keyring, checks every index and package against the SHA-256 sums in the +signed `Release`, and never replaces an installed version with a lower one +during `apt upgrade`; a downgrade needs an explicit `shadow-code=VERSION` and +`--allow-downgrades`. Earlier versions stay in the pool so an administrator can +roll back deliberately. `Release` carries no `Valid-Until` by default, so a +mirror or network attacker could keep serving an old signed `Release` (a +freeze); that withholds updates but cannot install anything older than what is +installed. The repository is published by the release workflow only; nothing +in ShadowCode adds or contacts it by itself. + +To install one release by hand instead, verify the Debian artifact using the independently trusted verifier bundle. +`verified-deb-2.1.0` must not already exist; install the verified snapshot: ```bash bash /path/to/trusted-bundle/scripts/verify-native-release.sh \ --bundle-dir /path/to/downloads \ --trust-dir /path/to/trusted-bundle/release/trust \ - --artifact ShadowCode_1.0.1_amd64.deb \ - --stage-dir ./verified-deb-1.0.1 \ - --expect-version 1.0.1 -sudo apt install ./verified-deb-1.0.1/ShadowCode_1.0.1_amd64.deb + --artifact ShadowCode_2.1.0_amd64.deb \ + --stage-dir ./verified-deb-2.1.0 \ + --expect-version 2.1.0 +sudo apt install ./verified-deb-2.1.0/ShadowCode_2.1.0_amd64.deb ``` -This manual Debian verification does not provide the AppImage installer's durable -accepted-version and rollback state. Checksums alone do not authenticate a publisher. +A `.deb` installed by hand this way gets no updates and none of the APT +repository's rollback protection; add the repository above to get both. +Checksums alone do not authenticate a publisher. The deb installs `shadowcode` and the same llama.cpp runtime in `/usr/lib/shadowcode`, with a manual page (`man shadowcode`) and bash, zsh and @@ -183,8 +245,8 @@ About**, ShadowCode asks GitHub for the latest release. The request carries no version or identifier, and nothing is downloaded or installed for you. When a newer version exists, **Update available** appears in the status bar; **Settings › About** links the release notes and shows how to update this -installation (the authenticated installer for the AppImage, your package -manager for the deb). Turn it off there with **Check for updates once a +installation (the authenticated installer for the AppImage, `apt upgrade` from the +ShadowCode APT repository or your package manager for the deb). Turn it off there with **Check for updates once a day**; Offline mode pauses it. Distributions can turn it off for everyone. ## First run diff --git a/config.example.yaml b/config.example.yaml index c22e8927..73c1a458 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -139,6 +139,9 @@ routing: small_edits: "" vision: "" local: "" + # Offer the cheapest model that passed verification in this project + # before (a suggestion only; it never switches the model). + suggest_from_history: false # MCP servers (Settings › Advanced › MCP). An entry is # {name, command: [program, args…] or url, api_key_env, description, diff --git a/docs/API_CONTRACT.md b/docs/API_CONTRACT.md index 8c889a74..614f8108 100644 --- a/docs/API_CONTRACT.md +++ b/docs/API_CONTRACT.md @@ -466,6 +466,7 @@ need a Unix build. ShadowCode 1.x ships for Linux, where all of them exist. | `POST` | `/api/mcp/servers/delete` | stable | allowed | [Extensions](#extensions) | | `POST` | `/api/memory` | stable | allowed | [Slash commands and memory](#slash-commands-and-memory) | | `GET` | `/api/models` | stable | allowed | [Accounts and models](#accounts-and-models) | +| `GET` | `/api/models/outcomes` | experimental | allowed | [Accounts and models](#accounts-and-models) | | `POST` | `/api/models/register` | stable | allowed | [Accounts and models](#accounts-and-models) | | `POST` | `/api/models/select` | stable | allowed | [Accounts and models](#accounts-and-models) | | `POST` | `/api/models/test` | stable | allowed | [Accounts and models](#accounts-and-models) | @@ -2123,6 +2124,46 @@ models that take no effort (Claude Haiku). A loaded local row's `reason` starts with `Loaded ·` (and says `CPU fallback (GPU load failed)` when the GPU start failed). +### Model outcome history and suggestion + +`GET /api/models/outcomes?workspace=¤t=` (experimental) → +`{workspace, suggest_from_history: boolean, models: ModelOutcomeStats[], +suggestion: OutcomeSuggestion|null}`. `workspace` defaults to the current +project; `current` is the picker id now selected (never suggested). + +``` +ModelOutcomeStats = { + model_id, // exact picker id that ran the tasks + model_label, + samples, // finished top-level tasks counted (newest 50) + median_seconds, median_tokens, + median_cost_usd, // null = unavailable: no cost was reported + cost_samples, // tasks whose cost was reported + verified_samples, // tasks with an observed pass/fail + passes, pass_rate, // pass_rate null when nothing was verified + last_finished_at +} +OutcomeSuggestion = {model_id, name, median_cost_usd, pass_rate, passes, samples, reason} +``` + +One row per finished task is written to `model_outcomes` (schema 28) in the +same transaction as `agent.completed`. Cancelled tasks, `command` mode and +subagent jobs are not recorded. `cost_usd` is stored only when the provider, +the vendor CLI or this computer reported it; price-list estimates and +unknown prices are stored as `NULL` and shown as "unavailable". A task passes +only on observed verification (`verification.verified`); a model's claim +alone is "not verified". Deleting a conversation deletes its outcomes. + +`suggestion` is `null` unless `routing.suggest_from_history` is `true` (set +with `PUT /api/config {values: {routing: {suggest_from_history: true}}}`). +It is the cheapest model by median **reported** cost that passed +verification in this project at least once and is `ready` in the picker now. +Models whose cost was never reported are not ranked. When the app is +offline only `inference: "local"` rows qualify; paid API rows are skipped +once today's spending limit is reached or when their median cost is above +the per-task limit. The endpoint never changes the selected model: the +picker shows the suggestion and the user accepts it (`Use …`) or not. + ### Accounts (subscriptions) - `GET /api/accounts?refresh=1|cached=1` → `{vendors: {[vendor]: diff --git a/docs/DATA.md b/docs/DATA.md index c6b46711..4b10ba6a 100644 --- a/docs/DATA.md +++ b/docs/DATA.md @@ -40,6 +40,18 @@ that every upgrade finds your data. Don't rename them. made by the older version, or one of the copies made before the upgrade (below), with `shadowcode restore FOLDER` in a terminal: the older app cannot open Settings on that profile. +- **AppImage installer rollback.** Before it activates a new version (with a + display, or `SHADOWCODE_FIRST_START_CHECK=require`), the + [AppImage installer](../README.md#appimage-recommended) copies + `~/.local/state/shadow-agent/shadow-agent.db` with SQLite's `VACUUM INTO` + (a consistent copy of the live WAL database, without migrating it) into its + private staging folder. It refuses while ShadowCode runs on the profile. If + the new version does not report that its database opened and its window + loaded in time, the installer stops it and puts the copy back (removing the + `-wal`/`-shm` files first) together with the previous AppImage and runtime, + so the older version gets the database format it reads. The copy is deleted + after a successful install; data written by the new version during a failed + first start is not kept. ## Back up diff --git a/docs/DISTRIBUTING.md b/docs/DISTRIBUTING.md index 2e4877a3..b11c4098 100644 --- a/docs/DISTRIBUTING.md +++ b/docs/DISTRIBUTING.md @@ -17,9 +17,97 @@ Ubuntu 24.04 and later): | `ShadowCode_VERSION_amd64.deb` (package name `shadow-code`) | You install system-wide through dpkg/APT. **Recommended for distributions.** | | `ShadowCode_VERSION_amd64.AppImage` | One user installs it in their home folder with the authenticated installer. It carries its own WebKit and does not need FUSE (`libfuse2`). | +### ARM64 (aarch64) + +From 2.1.0 the same packages are built natively for 64-bit ARM (aarch64) on +GitHub's `ubuntu-24.04-arm` runners and pass the same release gates: + +| x86_64 | aarch64 | +| --- | --- | +| `ShadowCode_VERSION_amd64.deb` | `ShadowCode_VERSION_arm64.deb` (`Architecture: arm64`) | +| `ShadowCode_VERSION_amd64.AppImage` | `ShadowCode_VERSION_aarch64.AppImage` | +| `ShadowCode_VERSION_appimage-runtime-sources.tar.gz` | `ShadowCode_VERSION_aarch64_appimage-runtime-sources.tar.gz` | +| `ShadowCode_VERSION_x86_64.flatpak` | `ShadowCode_VERSION_aarch64.flatpak` | + +The publisher's trust policy (`release/trust/policy`) names both +`x86_64-unknown-linux-gnu` and `aarch64-unknown-linux-gnu`. Each release +publishes both targets' packages, each with its own signed metadata; the +aarch64 files carry an `-aarch64` suffix: `RELEASE-AUTH-aarch64`, +`RELEASE-AUTH-aarch64.sig`, `RELEASE-MANIFEST-aarch64.json` and +`SHA256SUMS-aarch64`. Verify an aarch64 package exactly like an x86_64 one, +naming the aarch64 package; the verifier reads the suffixed metadata itself: + +```bash +bash /path/to/trusted-bundle/scripts/verify-native-release.sh \ + --bundle-dir /path/to/downloads \ + --trust-dir /path/to/trusted-bundle/release/trust \ + --artifact ShadowCode_VERSION_arm64.deb \ + --stage-dir ./verified-deb-VERSION +``` + +Releases before 2.1.0 have no aarch64 packages; build them from source (below) +on an aarch64 machine. Cross-building is not supported: each architecture is built +on its own hardware. + Both contain the same compiled application and the same pinned llama.cpp -runtime. There is no ShadowCode APT repository: a distribution that ships the -`.deb` delivers updates itself (see [Updates](#updates)). +runtime. Each release is also indexed in a signed APT repository (see +[APT repository](#apt-repository)); a distribution that ships the `.deb` +itself delivers updates itself (see [Updates](#updates)). + +## APT repository + +The release workflow publishes `https://shadowfetchstudio.github.io/ShadowCode/apt` +(suite `stable`, component `main`, architectures `amd64 arm64`) from the +`gh-pages` branch. Configure it with a deb822 source and a scoped keyring in +`/etc/apt/keyrings`: + +```sh +sudo install -d -m 0755 /etc/apt/keyrings +curl -fsSLo /tmp/shadowcode-archive-keyring.gpg \ + https://shadowfetchstudio.github.io/ShadowCode/apt/shadowcode-archive-keyring.gpg +gpg --show-keys --with-fingerprint /tmp/shadowcode-archive-keyring.gpg +# Compare the fingerprint with release/trust/APT-SIGNING-KEY in a reviewed +# copy of the repository at a release tag; stop if it differs. +sudo install -m 0644 /tmp/shadowcode-archive-keyring.gpg /etc/apt/keyrings/ +sudo tee /etc/apt/sources.list.d/shadowcode.sources >/dev/null <<'EOF' +Types: deb +URIs: https://shadowfetchstudio.github.io/ShadowCode/apt +Suites: stable +Components: main +Architectures: amd64 arm64 +Signed-By: /etc/apt/keyrings/shadowcode-archive-keyring.gpg +EOF +sudo apt update && sudo apt install shadow-code +``` + +`shadowcode.sources` in the repository root is the same source file. + +The keyring is fetched once, over HTTPS, and must be checked against the +fingerprint in a reviewed copy of the repository: like the AppImage +installer's trust bundle, a key downloaded beside the packages does not +authenticate itself. `Signed-By` scopes the key to this one source; it is not +added to APT's global trusted keys. The repository key is a separate OpenPGP +key from the Ed25519 publisher key: CI indexes only a `.deb` that already +passed `verify-native-release.sh` against `release/trust`, then signs +`InRelease` and `Release.gpg`. + +Downgrade and rollback protection comes from APT. It verifies `InRelease` with +the keyring, checks every index and package against the SHA-256 sums in the +signed `Release`, and never replaces an installed version with a lower one +during `apt upgrade`; a downgrade needs an explicit `shadow-code=VERSION` and +`--allow-downgrades`. Earlier versions stay in the pool so an administrator can +roll back deliberately. `Release` carries no `Valid-Until` by default, so a +mirror or network attacker could keep serving an old signed `Release` (a +freeze); that withholds updates but cannot install anything older than what is +installed. The repository is published by the release workflow only; nothing +in ShadowCode adds or contacts it by itself. + +`scripts/build-apt-repo.mjs` builds the repository (a plain pool plus +`Packages`, `Release`, `InRelease` and `Release.gpg`, no reprepro needed) and +`scripts/test-build-apt-repo.mjs` checks it with a throwaway key, `gpgv` and +`apt-get` in a private root. From 2.1.0 each release adds both the amd64 and +the arm64 `.deb`, each verified against its own signed envelope first, so one +`.sources` file serves both architectures. ## Verify a release before you ship it @@ -61,6 +149,18 @@ node scripts/check-native-package.mjs \ target/release/bundle/deb/ShadowCode_VERSION_amd64.deb ``` +On an aarch64 host the same commands produce +`ShadowCode_VERSION_aarch64.AppImage` and `ShadowCode_VERSION_arm64.deb`; pass +those names to `check-native-package.mjs`. The AppImage runtime is built in a +`linux/arm64` container from the same pinned Alpine image (a multi-arch +index), so Docker or Podman must run natively there. The llama.cpp runtime's +aarch64 CPU variants include Armv9.2 SME, which needs GCC 14 or newer; on +Ubuntu 24.04 install `gcc-14` and `g++-14` (`scripts/build-llama.cpp.sh` uses +them when the default compiler cannot target `armv9.2-a+sme`), or set +`SHADOWCODE_CC` and `SHADOWCODE_CXX`. The AppImage bundler copies +`/usr/bin/xdg-open` into the AppImage, so install `xdg-utils` on a minimal +build host. + Build-time switches, read from the environment of the build: | Variable | Effect | @@ -153,7 +253,7 @@ the release notes and what to do next, chosen by how ShadowCode was installed: | Installed as | Next step shown | | --- | --- | | AppImage | Download the AppImage and its four signature files, then run the authenticated installer. The installer verifies the signature before it runs anything. | -| Debian package | Update through the package manager or the distribution's updates; a `.deb` installed by hand is verified with the trusted verifier and installed with apt. | +| Debian package | `apt upgrade` from the ShadowCode APT repository, or the distribution's updates; a `.deb` installed by hand is verified with the trusted verifier and installed with apt. | | Other system package | Update through the package manager. | | Built from source | `git fetch --tags && git checkout vVERSION`, then build. | diff --git a/docs/FLATPAK.md b/docs/FLATPAK.md index 5df663a5..698a48b3 100644 --- a/docs/FLATPAK.md +++ b/docs/FLATPAK.md @@ -1,17 +1,20 @@ -# ShadowCode 2.0 Flatpak +# ShadowCode 2.1 Flatpak The Flatpak runs the native GTK/WebKit desktop, application engine, SQLite profile, and bundled CPU/Vulkan llama.cpp runtime on GNOME Platform 50. It is -an x86_64 package built from source inside the SDK. Version 2.0.0 is a local +an x86_64 or aarch64 package built from source inside the SDK on a host of +the same architecture. Version 2.1.0 is a local build until a publisher signs and publishes it; it is not listed on Flathub. ## Install the local bundle ```sh -flatpak install --user ./ShadowCode_2.0.0_x86_64.flatpak +flatpak install --user ./ShadowCode_2.1.0_x86_64.flatpak flatpak run com.shadowfetch.shadowcode ``` +On 64-bit ARM, install `ShadowCode_VERSION_aarch64.flatpak` instead. + The bundle points to Flathub for the GNOME runtime. Installation exports the application launcher and icons into the desktop's Applications menu. @@ -45,9 +48,15 @@ The package includes the SDK's glibc, GCC, and Git license notices. Agent shell commands require working bubblewrap on the host. Network off uses bubblewrap's private network namespace; network on leaves network -access enabled. Hostname allow-list mode refuses to run, because the -current bridge cannot transfer the native network namespace to a host -process. It does not silently widen the selected permission. +access enabled. Hostname allow-list mode runs the command in a host +bubblewrap with its own private network namespace (`--unshare-net`); the +packaged `shadowcode-proxy-relay` (built statically from +`packaging/flatpak/proxy-relay.c`, source under +`/app/share/licenses/shadowcode/proxy-relay.c`) forwards loopback port 3128 +to ShadowCode's allow-list proxy over a Unix socket in +`$XDG_RUNTIME_DIR/app/com.shadowfetch.shadowcode`. If the relay, the shared +socket, or its startup probe is unavailable, the command is refused rather +than run with wider access. See [NATIVE_SANDBOX.md](NATIVE_SANDBOX.md). Flatpak stores its application profile below `~/.var/app/com.shadowfetch.shadowcode`. Existing AppImage/Debian profile @@ -86,7 +95,18 @@ application code is rebuilt; packaging-only retries retain the finished link. Use `--prepare-only` to prepare inputs, `--skip-fetch` to reuse prepared dependency caches, or `--install` to install the finished bundle for the -current user. SDK extensions and the host compiler may differ; the +current user. The bundle is built for the host architecture +(`ShadowCode_VERSION_x86_64.flatpak` or `ShadowCode_VERSION_aarch64.flatpak`); +`--arch` names it explicitly and is passed to flatpak-builder, which needs +QEMU user emulation for a foreign architecture. `--state-dir DIR` moves +flatpak-builder's cache (CI uses `~/.flatpak-builder`), +`--install-deps-from=flathub` lets flatpak-builder install the SDK and +extensions itself, and `--cpu-only-llama` skips the Vulkan module for faster +test builds (the runtime then records `backend=cpu`; never ship such a bundle). + +Pull requests that change `packaging/`, `scripts/`, `src-tauri/`, `native/` or +the workflows run the `flatpak` job in `.github/workflows/ci.yml`: a +CPU-only-llama build with the GNOME 50 SDK, then install and start the bundle. SDK extensions and the host compiler may differ; the packaged notices identify the compiler that actually built the app. After installing, verify command ownership with: diff --git a/docs/MACOS_PORT_ESTIMATE.md b/docs/MACOS_PORT_ESTIMATE.md new file mode 100644 index 00000000..794211ab --- /dev/null +++ b/docs/MACOS_PORT_ESTIMATE.md @@ -0,0 +1,52 @@ +# macOS port: build estimate + +Status: estimate only, nothing built. Figures are judgement from a source survey +(~135k lines of Rust in `native/core` and `src-tauri`), not measurements. + +## Summary + +- Total: **17–36 engineer-weeks** (midpoint ~24), plus QA and buffer. +- Tauri 2 and most dependencies (rusqlite, reqwest/rustls, tokio, ratatui) are + already cross-platform, and much code is gated `cfg(unix)` rather than Linux. +- First blocker: the ungated `gdk` dependency in `src-tauri/Cargo.toml`, plus + `/proc`, D-Bus and namespace code that needs `cfg(target_os = "linux")`. + +## Work items + +| Area | Main files | macOS approach | Weeks | Risk | +|---|---|---|---|---| +| Sandbox (bwrap, Landlock, netns) | `native/core/src/sandbox.rs`, `sandbox/{netns,lsm,proxy}.rs` | Seatbelt (`sandbox-exec`) profile + localhost proxy; no PID/IPC isolation | 3–6 | Highest: security semantics differ | +| Keyring | `native/core/src/keyring.rs` | Keychain (`security-framework`) | 1–2 | Needs stable code signing | +| PTY relay | `native/core/src/terminal.rs` | posix_openpt fd-flag/ioctl fixes, zsh default | 0.5–1.5 | Low–med | +| `/proc` usage | `preview/ports.rs`, `process.rs`, `worktrees/cleanup.rs` | `libproc`, `sysctl` | 1–2 | Medium | +| Paths | `paths.rs` | `~/Library/...` | 0.5–1 | Low | +| Window logic | `src-tauri/src/window_size.rs` | Gate GDK code; app menu, WKWebView QA | 1–2 | Low–med | +| llama.cpp | `scripts/build-llama.cpp.sh`, `local_runtime.rs`, `local_engine.rs` | Metal build, arm64 only | 1.5–3 | Medium | +| Packaging, signing, updates | `scripts/`, `packaging/`, `updates.rs` | `.app`/`.dmg`, codesign + notarytool, updater | 3–5 | Med–high | +| Shell / GUI `PATH` | `git_hooks.rs`, `updates.rs`, others | BSD tools, bash 3.2, Homebrew paths | 1–2 | Medium | +| Process control | `process.rs`, `cli_agent/runner.rs`, `control.rs` | kqueue `EVFILT_PROC` instead of PDEATHSIG | 1–2 | Medium | +| Voice | `voice/capture.rs` | cpal CoreAudio, mic entitlement | 0.5–1 | Low | +| Notifications | `src-tauri/src/notices.rs` | tauri-plugin-notification | 0.25–0.5 | Low | +| Desktop metadata | `packaging/*.desktop`, appdata | Info.plist, `.icns` | 0.5 | Low | +| CI | `.github/workflows/*` | `macos-14/15` jobs, signing secrets | 1.5–3 | ~10x runner cost | +| Tests and docs | tests across crates, docs | Gate or port Linux-specific tests | 1–2 | Low–med | + +## Phases + +1. **Compile on macOS (1–2 wk):** gate Linux-only code; add a macOS CI build. +2. **Unsigned MVP (6–11 wk):** paths, PTY, GUI `PATH`, Keychain, libproc, + kqueue, window gating, Metal llama.cpp, a Seatbelt sandbox shown as + reduced isolation (or every command asks) — never silently unsandboxed. +3. **Signed, notarized beta (4–8 wk):** Developer ID ($99/yr), hardened + runtime, sign nested binaries inside-out, notarize in CI, updater, + hardened Seatbelt, SECURITY.md update. +4. **Parity (6–15 wk):** full macOS test matrix, clean-VM smoke tests, + network-isolation parity, Metal tuning, optional Intel/universal, Homebrew cask. + +## Key risks + +1. Sandbox guarantees can't be matched exactly; needs a product decision. +2. Notarization of nested executables (llama-server, helpers). +3. GUI-launched apps lack the shell `PATH`, breaking CLI detection. +4. ~17k lines of Linux-specific release tooling to parallel and maintain. +5. macOS CI cost and no Docker-style clean-host testing. diff --git a/docs/NATIVE_DESKTOP.md b/docs/NATIVE_DESKTOP.md index 95add35f..a81b42b6 100644 --- a/docs/NATIVE_DESKTOP.md +++ b/docs/NATIVE_DESKTOP.md @@ -200,7 +200,12 @@ Prove it with `node --test scripts/test-native-packaging-env.mjs`. - For the default profile the window reopens maximized or full screen if it was, and on X11 at its last place (moved up or left until it fits that screen's work area); a Wayland compositor chooses the place itself. Its - size is not remembered (`window_size.rs`): it opens at 1380x920, or at 90% + content size (inside the title bar, in logical pixels) is remembered in + `window-content-size.json` (`window_size.rs`); whatever the window reports + beyond the size it asked for at launch (GTK's header bar and shadow on + Wayland) is taken off before saving, so reopening never grows the window, + and sizes saved before 2.1.0 are ignored. It opens at that size or 1380x920, + capped at 90% of a smaller screen with room for the title bar (the X11 window manager's frame, or GTK's header bar inside the size on Wayland), on the screen it reopens on (X11) or fitting every screen (Wayland). It opens maximized only diff --git a/docs/NATIVE_ROUTING.md b/docs/NATIVE_ROUTING.md index d5e27ef6..7424fcfa 100644 --- a/docs/NATIVE_ROUTING.md +++ b/docs/NATIVE_ROUTING.md @@ -22,6 +22,27 @@ Test for required command verification. Planning and review retain read-only permissions regardless of the selected model. A different model never grants additional workspace or command permissions. +## Suggestions from outcome history + +Routing never learns on its own. With `routing.suggest_from_history: true` +(off by default; the picker's **Suggest the cheapest model that passed +here** checkbox sets it) the picker asks `GET /api/models/outcomes` for a +suggestion: the model with the lowest median reported cost that passed +observed verification in this project before and is ready in the picker +now. The user accepts it with **Use …**; nothing changes otherwise, and +explicit selections, routes and running tasks are never altered. + +- Only reported costs count (provider, vendor CLI, or $0 on this computer). + Price-list estimates and unknown prices are "unavailable" and such models + are not ranked. +- Offline (`network.mode: offline`): only models on this computer. +- Spending limits: paid API models are skipped once today's limit is reached + and when their median cost is above the per-task limit. +- The model already selected is never suggested. + +Outcomes are stored per task in `model_outcomes` (schema 28); see +[the API contract](API_CONTRACT.md#model-outcome-history-and-suggestion). + ## Models with the same name Discovered and newly configured models receive IDs derived from their provider, diff --git a/docs/NATIVE_SANDBOX.md b/docs/NATIVE_SANDBOX.md index dc998f73..31b50d23 100644 --- a/docs/NATIVE_SANDBOX.md +++ b/docs/NATIVE_SANDBOX.md @@ -26,5 +26,25 @@ isolation when neither layer is available. The conversation shows a warning. Once execution begins, errors are returned without replaying the command outside the sandbox. Doctor reports the probe result. +## Allow-list mode in the Flatpak package + +In the Flatpak, bubblewrap runs on the host through `flatpak-spawn --host`, so +ShadowCode cannot hand it a network namespace it created itself. Instead the +host bubblewrap is started with `--unshare-net`, and the command is wrapped by +`shadowcode-proxy-relay`, a small static program shipped in the package and +mounted read-only at `/shadowcode-proxy-relay`. The relay listens on +`127.0.0.1:3128` inside that namespace and forwards each connection to a Unix +socket served by ShadowCode's allow-list proxy. The socket lives in a private +folder that the application and the host see at the same path +(`$XDG_RUNTIME_DIR/app/`), mounted read-only at `/shadowcode-proxy`. + +Enforcement still comes from the network namespace: the relay makes no policy +decision, and the proxy decides every host exactly as in the native build. It +fails closed: if the relay is missing from the package, no shared folder exists, +the socket path is too long, the relay cannot listen on loopback, or its probe +connection to the proxy fails, the command is not started (the relay exits with +status 125). Tool results report `network_enforcement: +host_allowlist_proxy_relay`. + This optional profile is not a complete operating-system security boundary. Commands run as your account. Review approvals and resulting changes. diff --git a/docs/RELEASE_AUTHENTICATION.md b/docs/RELEASE_AUTHENTICATION.md index 141dc21a..d1a2aa39 100644 --- a/docs/RELEASE_AUTHENTICATION.md +++ b/docs/RELEASE_AUTHENTICATION.md @@ -1,6 +1,6 @@ # Publisher authentication contract -The helpers in `scripts/native-release-auth.mjs` and `scripts/verify-native-release.sh` verify offline publisher signatures. They do not install, execute, download or publish an application, update trusted keys, or generate production keys. CI tests their behavior with disposable fixture keys. Public trust for 0.33.0–1.0.1 is tracked and the protected signing environment is configured. The installer uses these helpers before executing a candidate and explicitly refuses `--unverified`. [Version 0.33.1](https://github.com/ShadowfetchStudio/ShadowCode/releases/tag/v0.33.1), published on September 28, 2026, is the first signed release. +The helpers in `scripts/native-release-auth.mjs` and `scripts/verify-native-release.sh` verify offline publisher signatures. They do not install, execute, download or publish an application, update trusted keys, or generate production keys. CI tests their behavior with disposable fixture keys. Public trust for 0.33.0–2.1.0 is tracked and the protected signing environment is configured. The installer uses these helpers before executing a candidate and explicitly refuses `--unverified`. [Version 0.33.1](https://github.com/ShadowfetchStudio/ShadowCode/releases/tag/v0.33.1), published on September 28, 2026, is the first signed release. The Bash verifier loads `scripts/native-release-auth-lib.sh` from its own script directory. Keep both reviewed code files together in the trusted tooling bundle; do not obtain the library from the candidate download or source metadata as shell code. The installer uses the same reviewed routines for current candidates and signed historical receipts; it never sources release metadata as code. @@ -50,7 +50,7 @@ The line order is: 5. `version=MAJOR.MINOR.PATCH`. 6. `tag=vMAJOR.MINOR.PATCH`, exactly matching version. 7. `commit=COMMIT_SHA`. -8. `target=x86_64-unknown-linux-gnu`. +8. `target=x86_64-unknown-linux-gnu` or `target=aarch64-unknown-linux-gnu`. 9. `channel=stable`. 10. `key-id=SHA256_OF_ED25519_SPKI_DER`. 11. `key-epoch=POSITIVE_INTEGER`. @@ -60,6 +60,17 @@ The line order is: 15. `asset=debShadowCode_VERSION_amd64.debSIZESHA256`. 16. `asset=runtime-sourcesShadowCode_VERSION_appimage-runtime-sources.tar.gzSIZESHA256`. +Those are the x86_64 names. An `aarch64-unknown-linux-gnu` envelope names `ShadowCode_VERSION_aarch64.AppImage`, `ShadowCode_VERSION_arm64.deb` and `ShadowCode_VERSION_aarch64_appimage-runtime-sources.tar.gz` instead, so both targets' packages can sit side by side. Each target has its own envelope, signature, manifest and checksums, published in the same GitHub release under per-target names: + +| | x86_64 | aarch64 | +| --- | --- | --- | +| Envelope | `RELEASE-AUTH` | `RELEASE-AUTH-aarch64` | +| Signature | `RELEASE-AUTH.sig` | `RELEASE-AUTH-aarch64.sig` | +| Manifest | `RELEASE-MANIFEST.json` | `RELEASE-MANIFEST-aarch64.json` | +| Checksums | `SHA256SUMS` | `SHA256SUMS-aarch64` | + +The x86_64 names are historical and unchanged, so existing installers keep working. The requested package basename selects which names a verifier reads (an `_aarch64`/`_arm64` package reads the `-aarch64` metadata), and the envelope found there must name that same target; an envelope filed under another target's names is refused before any digest check. The names describe only the published release: a verified stage and the installer's accepted-release receipt always keep the historical names, whatever the target. The rest of this document says `RELEASE-AUTH`, `SHA256SUMS` and so on for either target's file. + `RELEASE-AUTH.sig` is exactly the 64-byte raw Ed25519 signature over all envelope bytes. There is no home-grown cryptography, prehash signature variant, URL field, arbitrary asset path or shell-sourced content. The domain marker prevents reuse as a different application statement. The complete three-asset list is always authenticated. `SHA256SUMS` must contain exactly those entries in that order, matching the current `check-native-package.mjs` producer: each `HASH`, two spaces, basename, LF. Duplicate, extra, path-bearing and contradictory entries fail even when the checksum-file digest is correctly signed. Each artifact is nonempty and at most 32 GiB. The exact JSON manifest is hash-bound and limited to 1 MiB. Its audit contents are opaque to the shell verifier; the envelope is the authoritative install identity. The Node construction API additionally checks the manifest's current schema/tag/commit/target and exact asset map and requires the existing pretty-printed JSON serialization, which rejects duplicate JSON fields. Publisher receipt validation remains the responsibility of the existing release gates and publisher before signing. @@ -76,7 +87,7 @@ The explicit trust directory contains `policy` and one `KEY_ID.pem` public key p 2. `repository=ShadowfetchStudio/ShadowCode` (or the historical signed name above). 3. `repository-id=1377099349`. 4. `owner-id=209457103`. -5. `target=x86_64-unknown-linux-gnu`. +5. `target=TARGETS`: `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu`, or both separated by a comma (no spaces, no duplicates). An envelope whose target the policy does not name is refused, and a release cannot switch target against its accepted history. 6. `channel=stable`. 7. `minimum-epoch=POSITIVE_INTEGER`. 8. `minimum-version=MAJOR.MINOR.PATCH`. @@ -87,7 +98,7 @@ Each key has an inclusive version range and exactly one epoch. Unknown keys, dif For a reviewed rotation, distribute a new authenticated trust bundle with the new public key and higher minimum epoch/version. Retain the old public key with its historical version range only when needed to verify an existing accepted-state receipt. The internal history check may ignore current minimum floors, but it still verifies the old signature, identity, key epoch and historical key range. This exception is not exposed as an install-verification API. Deleting an old key outright can make old state unverifiable; the installer must then fail pending explicit migration, not erase the state or assume a fresh install. -The tracked `release/trust` bundle contains the public Ed25519 key with ID `f0c60ff8228314616985712f8918c9798c0ecf72b7271e19343a041c25663b7f`. Its epoch is 1, minimum version is 0.33.0, and its inclusive key range is 0.33.0–1.0.1. It authorizes no later version without a reviewed policy update. The private key is not distributed in the repository or installer bundle. Fixture keys remain temporary and separate. Public fingerprint distribution, custody and future rotation/revocation remain owner responsibilities. +The tracked `release/trust` bundle contains the public Ed25519 key with ID `f0c60ff8228314616985712f8918c9798c0ecf72b7271e19343a041c25663b7f`. Its epoch is 1, minimum version is 0.33.0, and its inclusive key range is 0.33.0–2.1.0. It authorizes no later version without a reviewed policy update. The private key is not distributed in the repository or installer bundle. Fixture keys remain temporary and separate. Public fingerprint distribution, custody and future rotation/revocation remain owner responsibilities. ## Downgrade, replay and accepted state @@ -134,8 +145,8 @@ A separate publisher runner receives the signing job's immutable artifact ID and The workflow deliberately refuses missing owner configuration. The following records the 0.33.1 setup; recheck it before dispatching a release: - The tracked `release/trust` public policy and key bundle must be included in the reviewed tooling commit and distributed through an authenticated channel. It contains public material only; no default fixture key is trusted. -- Repository variable `NATIVE_RELEASE_TOOLING_COMMIT` must pin the exact 40-character reviewed and pushed commit containing the current gate definitions, signing/publisher code, public trust and release-appropriate notes. The reviewed pin for 0.33.1 was `e15c4480e65db5650af012bb2a9773dbe89acf84`; for 1.0.1 it is the tagged `v1.0.1` release commit (`v1.0.0` for 1.0.0, `v0.34.2` for 0.34.2). Never infer this pin from an unreviewed tag. -- Repository variable `NATIVE_RELEASE_SIGNING_ENVIRONMENT` is set to `release-signing`. Its verified configuration requires the `ShadowfetchStudio` User reviewer (ID `209457103`, unchanged after the account rename), allows self-review and retains GitHub's default administrator override. Its sole deployment rule allows the tag being released (`v1.0.1` for 1.0.1; `v1.0.0`, `v0.34.2` and `v0.33.1` before). This is an owner-reviewed gate, not a two-person approval guarantee; a later release needs an explicit rule/policy update. +- Repository variable `NATIVE_RELEASE_TOOLING_COMMIT` must pin the exact 40-character reviewed and pushed commit containing the current gate definitions, signing/publisher code, public trust and release-appropriate notes. The reviewed pin for 0.33.1 was `e15c4480e65db5650af012bb2a9773dbe89acf84`; for 2.1.0 it is the tagged `v2.1.0` release commit (`v1.0.1` for 1.0.1, `v1.0.0` for 1.0.0, `v0.34.2` for 0.34.2). Never infer this pin from an unreviewed tag. +- Repository variable `NATIVE_RELEASE_SIGNING_ENVIRONMENT` is set to `release-signing`. Its verified configuration requires the `ShadowfetchStudio` User reviewer (ID `209457103`, unchanged after the account rename), allows self-review and retains GitHub's default administrator override. Its sole deployment rule allows the tag being released (`v2.1.0` for 2.1.0; `v1.0.1`, `v1.0.0`, `v0.34.2` and `v0.33.1` before). This is an owner-reviewed gate, not a two-person approval guarantee; a later release needs an explicit rule/policy update. - Environment secret `NATIVE_RELEASE_SIGNING_KEY_PEM` has been uploaded to `release-signing`. It is scoped to the signing environment, not repository-wide secrets or the build job. Its presence does not prove that a candidate has been signed. Owner review of the release workflow/source and repository protections remains necessary: another authorized workflow could request the same environment secret. Local gate receipts are validated assertions, not independent attestations that an adversarial producer executed the commands. Workflow dependencies and exact artifact IDs bind the proposed job flow; local fixtures do not prove GitHub control-plane behavior. Remote checks are observations, not atomic compare-and-swap against independently authorized concurrent changes. diff --git a/docs/RELEASE_NOTES.md b/docs/RELEASE_NOTES.md index 8d896a25..9f49728e 100644 --- a/docs/RELEASE_NOTES.md +++ b/docs/RELEASE_NOTES.md @@ -1,3 +1,45 @@ +# ShadowCode 2.1.0 + +ShadowCode 2.1.0 runs on 64-bit ARM computers, installs and updates from a +signed APT repository, and its installer now checks that the new version +starts before it keeps it. The 2.0.0 notes follow below. + +## New + +- **64-bit ARM.** The AppImage, the Debian package and the Flatpak are built + and tested on ARM64 (aarch64) as well as x86_64, and each is signed for its + own architecture. Download the files ending in `_aarch64` or `_arm64`. +- **Updates through APT.** Add the signed ShadowCode repository once and + `apt upgrade` keeps ShadowCode current, on both architectures. The README's + Debian section shows how, including how to check the repository's key. +- **An installer that checks before it commits.** The AppImage installer + saves a copy of your database, starts the new version and waits for its + window. If it hangs, crashes or reports the wrong version, everything is put + back as it was, database included. +- **Model history in the picker.** Each model shows how long tasks took in + this project, what they cost when the provider reported it, and how often + their checks passed. **Compare history** shows them side by side, and an + optional suggestion points to the cheapest model that passed here before. + ShadowCode never switches models on its own. + +## Fixed + +- **The window remembers its size again**, including on Wayland desktops + such as KDE Plasma, without growing from launch to launch. +- **Website allow-list in the Flatpak.** Agent shell commands in the Flatpak + can now be limited to the websites you allow; if that can't be set up, the + command is refused rather than run unrestricted. + +## Download + +For x86_64: `ShadowCode_2.1.0_amd64.AppImage`, `ShadowCode_2.1.0_amd64.deb` +and `ShadowCode_2.1.0_appimage-runtime-sources.tar.gz`, with `SHA256SUMS`, +`RELEASE-MANIFEST.json`, `RELEASE-AUTH` and `RELEASE-AUTH.sig`. For ARM64 the +packages end in `_aarch64` (AppImage) or `_arm64` (`.deb`), with signature +files ending in `-aarch64`. Install the AppImage with the authenticated +installer described in the [README](../README.md#install); the signature is +checked before anything runs. + # ShadowCode 2.0.0 Version 2.0.0 adds a source-built Flatpak, closes three reproduced remote-access defects, and updates the affected development dependency. See [Flatpak](FLATPAK.md) for installation, host-tool permissions, and package-specific limits. The source is published in this repository; binary release publication and Flathub submission remain pending. About ShadowCode shows the version, Apache license, publisher, official website, repository, and contributors. The [trademark and branding policy](../TRADEMARKS.md) covers the official identity and permits forks and accurate community attribution. Previous release notes follow. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 3c850bb7..b1732cfb 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -7,7 +7,7 @@ and authentication receipts to the exact candidate before publication. Pushing a `v*` tag starts `.github/workflows/release.yml`. Publication follows build/qualification, protected signing review and verification of all seven public assets. The signing environment permits only the tag being released -(`v1.0.1` for 1.0.1); every later version needs reviewed trust-policy, +(`v2.1.0` for 2.1.0); every later version needs reviewed trust-policy, deployment-rule and tooling-pin updates. The workflow uses `docs/RELEASE_NOTES.md` as the release text. @@ -138,7 +138,16 @@ bash scripts/test-install-appimage.sh - **`test-install-appimage.sh`** uses fake AppImages in a temporary `HOME`. It covers checksum refusal, broken runtimes, `--unverified`, rollback after the runtime swap and a changed download. If `packaging/llama.cpp/bin` exists, it - also runs the real `llama-server`. + also runs the real `llama-server`. Scenarios 20–24 cover the first-window + check: a version that hangs, exits or reports another version is stopped and + the AppImage, runtime, launchers and database snapshot are restored without + lowering the accepted-release record; the check refuses while ShadowCode + holds the profile lock; and `--recover` after an installer killed during the + check waits for the started version to close, then restores all of it. Run + it as an ordinary user: as root, permission-based failure injection + (scenario 5) cannot fail. The real `VACUUM INTO` snapshot and the + first-start marker are covered by `cargo test -p shadowcode-core --lib -- + install_snapshot first_start`. ## 5. Protected signing review and tag @@ -159,14 +168,32 @@ run green. Existing 0.33.0 receipts do not qualify the new package bytes. The first-authenticated install boundary remains 0.33.0; do not silently move this durable boundary. -Public trust in `release/trust` currently allows epoch 1 and versions 0.33.0–1.0.1; +The release workflow also runs every required gate on aarch64 (`build-aarch64`, +on `ubuntu-24.04-arm`) and hands off its packages and receipts unsigned, exactly +like x86_64 (`unsigned-release-aarch64-RUN-ATTEMPT`). The trust policy trusts +both targets (`target=x86_64-unknown-linux-gnu,aarch64-unknown-linux-gnu`). +`sign` waits for both builds and signs each target separately with the same +key: `node scripts/sign-native-release.mjs BUNDLE TRUST OUTPUT [TARGET]`, where +the aarch64 bundle's receipts are checked against the aarch64 gate scripts (the +package names in three gates differ by architecture) and its metadata is +written as `RELEASE-AUTH-aarch64`, `RELEASE-AUTH-aarch64.sig`, +`RELEASE-MANIFEST-aarch64.json` and `SHA256SUMS-aarch64`. `publish` takes the +x86_64 bundle and then the aarch64 bundle, authenticates each for its own +target and stages all fourteen assets in one draft before publishing; the +x86_64 bundle is required and no asset name may repeat. `apt-repository` +verifies and indexes both `.deb` files. The pinned tooling commit must include +this multi-target signer and publisher; an older pin refuses the aarch64 +signing call, so the release fails rather than shipping x86_64 alone. + +Public trust in `release/trust` currently allows the x86_64 and aarch64 targets, epoch 1 and versions 0.33.0–2.1.0 +(raise the key's maximum version in the release commit, as for earlier releases); `release/install-policy` fixes the first authenticated version at 0.33.0. The `release-signing` environment has the signing secret and requires review by `Shadowfetchapps` (User ID `209457103`). Self-review is allowed and GitHub's -default administrator override remains unchanged. For 1.0.1 its sole -deployment rule is the tag `v1.0.1` (it was `v1.0.0` for 1.0.0, `v0.34.2` for -0.34.2 and `v0.33.1` for 0.33.1); `NATIVE_RELEASE_SIGNING_ENVIRONMENT` points -to that environment. For 1.0.1, +default administrator override remains unchanged. For 2.1.0 its sole +deployment rule is the tag `v2.1.0` (it was `v1.0.1` for 1.0.1, `v1.0.0` for +1.0.0, `v0.34.2` for 0.34.2 and `v0.33.1` for 0.33.1); +`NATIVE_RELEASE_SIGNING_ENVIRONMENT` points to that environment. For 2.1.0, `NATIVE_RELEASE_TOOLING_COMMIT` pins the tagged release commit itself, as it did for 0.33.1 (`e15c4480e65db5650af012bb2a9773dbe89acf84`). @@ -211,6 +238,36 @@ version are refused. Do not overwrite published versioned assets. Don't announce a tag until this job succeeds. Afterwards, compare the uploaded checksums and the release text with what you expect. +### APT repository + +After `publish`, the `apt-repository` job re-verifies the `.deb` with +`verify-native-release.sh` against `release/trust`, rebuilds the repository +with `scripts/build-apt-repo.mjs` (keeping every package already in the +`gh-pages` pool; changed bytes under an existing file name are refused), checks +`InRelease` with `gpgv` and pushes `apt/` to `gh-pages`. Pages must be set to +deploy from that branch. + +It needs, in the protected signing environment: + +- `secrets.NATIVE_APT_SIGNING_KEY`: the ASCII-armored OpenPGP secret key + (`gpg --armor --export-secret-keys FPR`), without a passphrase. Generate it + offline; never commit it or a copy of it. +- `vars.NATIVE_APT_SIGNING_FINGERPRINT`: its 40-digit uppercase primary + fingerprint. The builder refuses a secret holding any other key. + +Record the public fingerprint in `release/trust/APT-SIGNING-KEY` through a +reviewed commit before the first repository publication, so users can check +the keyring they download. Rotating the key is a reviewed change to that file +and the two settings; announce it, since every installed source must import +the new keyring. + +Check afterwards: + +```sh +node --test scripts/test-build-apt-repo.mjs # locally, before tagging +curl -fsSL https://shadowfetchstudio.github.io/ShadowCode/apt/dists/stable/InRelease | gpgv --keyring ./shadowcode-archive-keyring.gpg +``` + ## 6. Install check Use the complete independently authenticated installer bundle, including its @@ -227,7 +284,10 @@ The installer accepts a single AppImage path or `--recover`; it does not accept read-only verifier. No checksum-only or `--unverified` bypass is supported. For manual Debian verification/install, see the [README](../README.md#debian-package). -Then open ShadowCode from the desktop entry and check: +With a display, the installer starts the new version itself and waits for its +first window (see the [README](../README.md#appimage-recommended)); a failed first start +rolls back the AppImage, runtime and database snapshot. Then use the +ShadowCode it opened, or open it from the desktop entry, and check: - Settings, conversations and accounts are still there. - **Settings › Local models › Runtime** shows the pinned commit and the diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 59a9a001..b2f26b6e 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -1,6 +1,6 @@ # ShadowCode user guide -This guide describes the 1.0.1 workflow: open a +This guide describes the 2.1.0 workflow: open a project, pick a model, describe the task, watch the agent work, then review what changed. Installation and a feature overview are in the [README](../README.md). @@ -87,6 +87,26 @@ A row that isn't ready still opens something useful: *Sign in* opens ShadowCode stores the choice for the conversation and uses it as the project's default for new conversations. +### How each model did in this project + +After tasks finish, each row that ran in this project shows a short history: +median time, median cost, how many verified tasks passed and how many tasks +it ran, for example *Here: 2m 5s · $0.04 · 3/4 passed · 5 tasks*. The cost is +shown only when the provider, the vendor CLI or this computer reported it; +otherwise it reads *cost unavailable*. ShadowCode never fills in a guess. +A task counts as passed only when its checks actually ran and passed; a task +without observed checks is *not verified*. **Compare history** in the picker +shows every model side by side. Cancelled tasks, plain commands and +subagents are not counted, and deleting a conversation removes its history. + +Tick **Suggest the cheapest model that passed here** to get a suggestion at +the top of the picker: the model with the lowest median reported cost that +passed in this project before and is ready now. It is only a suggestion: +choose **Use …** to switch, or **Not now**. ShadowCode never switches for +you. Offline, only models on this computer are suggested; paid API models +are left out once today's spending limit is reached or when they usually +cost more than the per-task limit. + Until a subscription is connected, **On this computer** comes first, marked *Free · your code stays on this computer*, so the first row (and Enter) is a free model. diff --git a/integrations/github-action/README.md b/integrations/github-action/README.md index 949caee9..e61f0537 100644 --- a/integrations/github-action/README.md +++ b/integrations/github-action/README.md @@ -50,7 +50,7 @@ command line runs directly. | `github-token` | `github.token` | Used only for the comment or pull request, after ShadowCode finished. | | `issue-number` | the event's issue or pull request | Where to comment. | | `base` | the checked-out branch | Target branch of the pull request. | -| `version` | `1.0.1` | ShadowCode release to run. `approval: approve` needs the release that ships this action or later. | +| `version` | `2.1.0` | ShadowCode release to run. `approval: approve` needs the release that ships this action or later. | | `appimage-sha256` | empty | Optional extra pin for the AppImage's SHA-256. | | `working-directory` | the workspace | Project folder. | diff --git a/integrations/github-action/action.yml b/integrations/github-action/action.yml index 414766df..2f7c1957 100644 --- a/integrations/github-action/action.yml +++ b/integrations/github-action/action.yml @@ -81,7 +81,7 @@ inputs: version: description: ShadowCode release to download (without the leading v). required: false - default: 1.0.1 + default: 2.1.0 appimage-sha256: description: >- Optional SHA-256 of the AppImage. When set it must match as well as the diff --git a/integrations/github-action/examples/issue-comment.yml b/integrations/github-action/examples/issue-comment.yml index df37be7d..f35b0b14 100644 --- a/integrations/github-action/examples/issue-comment.yml +++ b/integrations/github-action/examples/issue-comment.yml @@ -39,7 +39,7 @@ jobs: persist-credentials: false # Pin to a release tag or, better, a full commit SHA. - - uses: ShadowfetchStudio/ShadowCode/integrations/github-action@v1.0.1 + - uses: ShadowfetchStudio/ShadowCode/integrations/github-action@v2.1.0 with: task: ${{ github.event.comment.body }} api-key: ${{ secrets.OPENROUTER_API_KEY }} diff --git a/integrations/github-action/examples/pr-label.yml b/integrations/github-action/examples/pr-label.yml index c006cfe6..0bb7da02 100644 --- a/integrations/github-action/examples/pr-label.yml +++ b/integrations/github-action/examples/pr-label.yml @@ -35,7 +35,7 @@ jobs: persist-credentials: false fetch-depth: 0 - - uses: ShadowfetchStudio/ShadowCode/integrations/github-action@v1.0.1 + - uses: ShadowfetchStudio/ShadowCode/integrations/github-action@v2.1.0 with: task: >- Review the changes on this branch against diff --git a/native/core/src/cli/args.rs b/native/core/src/cli/args.rs index 71604be4..a07f90df 100644 --- a/native/core/src/cli/args.rs +++ b/native/core/src/cli/args.rs @@ -81,6 +81,15 @@ pub enum Command { #[arg(long)] repair: bool, }, + /// Installer use: copy the profile database to a new file with SQLite's + /// `VACUUM INTO` (never a raw copy of a live WAL database), without + /// opening or migrating it. Prints the copied database format, or + /// `absent` when the profile has no database yet. + #[command(hide = true)] + InstallSnapshot { + /// The new file to write; it must not exist. + output: PathBuf, + }, /// Back up your conversations, jobs, goals and settings to a folder. Backup { /// Also copy API keys (secrets.env and keys kept in the desktop diff --git a/native/core/src/cli/mod.rs b/native/core/src/cli/mod.rs index cca701f4..52f77a87 100644 --- a/native/core/src/cli/mod.rs +++ b/native/core/src/cli/mod.rs @@ -436,6 +436,14 @@ pub async fn run(options: Options) -> Result { )?)? .path; let paths = options.paths()?; + if let Some(Command::InstallSnapshot { output }) = &options.command { + let output = std::env::current_dir()?.join(expand(output)?); + match crate::data::install_snapshot(&paths, &output)? { + Some(version) => println!("{version}"), + None => println!("absent"), + } + return Ok(0); + } if let Some(command @ (Command::Restore { .. } | Command::Reset { .. })) = &options.command { return data_command(&paths, &workspace, command, options.json).await; } @@ -1141,8 +1149,8 @@ async fn execute(backend: &Backend, workspace: &Path, options: &Options) -> Resu value["items"] = json!(items); value } - Command::Restore { .. } | Command::Reset { .. } => { - unreachable!("Restore and reset are handled before the engine opens") + Command::Restore { .. } | Command::Reset { .. } | Command::InstallSnapshot { .. } => { + unreachable!("Restore, reset and install snapshots are handled before the engine opens") } Command::Understand { save } => { backend diff --git a/native/core/src/config.rs b/native/core/src/config.rs index a9f6b769..53c905a3 100644 --- a/native/core/src/config.rs +++ b/native/core/src/config.rs @@ -339,7 +339,7 @@ impl Default for Config { // config.example.yaml documents it (tests/config_keys.rs). ui: json!({"theme":"light","notify":true,"notify_approval":true,"notify_failed":true,"notify_limit":true,"notify_finished":true,"notify_sound":false}), onboarding: json!({"completed":false,"workspace":""}), - routing: json!({"enabled":false,"planner":"","coder":"","reviewer":"","tester":"","architecture":"","small_edits":"","vision":"","local":""}), + routing: json!({"enabled":false,"planner":"","coder":"","reviewer":"","tester":"","architecture":"","small_edits":"","vision":"","local":"","suggest_from_history":false}), mcp: json!({"servers":[],"approved":[],"share_with_cli_agents":true}), hooks: crate::hooks::HookConfig::default(), verification: crate::verification::CheckConfig::default(), diff --git a/native/core/src/data.rs b/native/core/src/data.rs index 782b7914..fd026c98 100644 --- a/native/core/src/data.rs +++ b/native/core/src/data.rs @@ -499,6 +499,25 @@ fn copy_database(source: &Path, target: &Path) -> Result { Ok(version) } +/// The installer's pre-activation snapshot of the profile database: a +/// consistent `VACUUM INTO` copy (the live database is neither migrated nor +/// written). `None` when the profile has no database yet. The installer +/// restores it only after the new version failed its first start and was +/// stopped, so the older version is not handed a database format newer than +/// it reads. +pub fn install_snapshot(paths: &AppPaths, target: &Path) -> Result> { + let source = paths.database(); + if !source.exists() { + return Ok(None); + } + ensure!( + fs::symlink_metadata(target).is_err(), + "{} already exists", + target.display() + ); + copy_database(&source, target).map(Some) +} + #[derive(Clone, Copy, Debug)] pub struct BackupOptions<'a> { pub include_secrets: bool, @@ -1623,6 +1642,52 @@ pub fn restart_hint(mode: Option<&str>, pid: u32) -> String { mod tests { use super::*; + /// The installer's snapshot is a consistent copy of a live WAL database + /// (committed rows still in the log included), does not migrate or + /// write the source, and puts back a database the current version opens + /// after a newer version marked the live one as a newer format. + #[test] + fn install_snapshot_copies_a_live_database_for_rollback() { + let temp = tempfile::tempdir().unwrap(); + let paths = AppPaths::isolated(&temp.path().join("profile")).unwrap(); + let target = temp.path().join("snapshot.db"); + assert_eq!(install_snapshot(&paths, &target).unwrap(), None); + assert!(!target.exists()); + + let store = store::Store::open(&paths.database()).unwrap(); + // Held open in WAL mode with a committed row not yet checkpointed. + let live = Connection::open(paths.database()).unwrap(); + live.execute_batch( + "CREATE TABLE install_probe(v TEXT); INSERT INTO install_probe VALUES('kept');", + ) + .unwrap(); + assert_eq!( + install_snapshot(&paths, &target).unwrap(), + Some(store::SCHEMA_VERSION) + ); + let copy = Connection::open(&target).unwrap(); + let kept: String = copy + .query_row("SELECT v FROM install_probe", [], |r| r.get(0)) + .unwrap(); + assert_eq!(kept, "kept"); + drop(copy); + // An existing target is never overwritten. + assert!(install_snapshot(&paths, &target).is_err()); + + // A newer version migrated the live database; this version refuses it. + live.pragma_update(None, "user_version", store::SCHEMA_VERSION + 1) + .unwrap(); + drop(live); + drop(store); + assert!(store::Store::open(&paths.database()).is_err()); + // Rollback: the snapshot replaces the database and its WAL files. + for suffix in ["-wal", "-shm"] { + let _ = fs::remove_file(format!("{}{suffix}", paths.database().display())); + } + fs::rename(&target, paths.database()).unwrap(); + store::Store::open(&paths.database()).unwrap(); + } + /// Someone who can change the chosen folder renames the new backup /// folder and puts a symlink to their own folder in its place: every /// file still lands in the folder this process made, the backup is diff --git a/native/core/src/engine.rs b/native/core/src/engine.rs index 6cb6ec2e..6821dd05 100644 --- a/native/core/src/engine.rs +++ b/native/core/src/engine.rs @@ -1871,7 +1871,10 @@ impl Engine { } ); let mut saved = json!(*job); - let event = self.0.store.finish_job(&mut saved)?; + let event = self + .0 + .store + .finish_job_with(&mut saved, running.child.is_none())?; job.event_cursor = event["id"].as_i64().unwrap_or(0); let _ = self.0.sender.send(event); running.finished.store(true, Ordering::Release); diff --git a/native/core/src/flatpak.rs b/native/core/src/flatpak.rs index 46d754ae..81e1107a 100644 --- a/native/core/src/flatpak.rs +++ b/native/core/src/flatpak.rs @@ -13,3 +13,56 @@ pub const HOST_ENV: &[&str] = &[ "SHADOWCODE_HOST_PATH", "SHADOWCODE_HOST_DBUS", ]; + +/// Name of the allow-list relay installed beside the host bridge. +pub const RELAY_NAME: &str = "shadowcode-proxy-relay"; + +/// The `app-path` of `/.flatpak-info`: where this application's `/app` lives +/// on the host, so a host process can execute a file shipped in it. +pub fn app_path_from_info(info: &str) -> Option { + info.lines() + .find_map(|line| line.trim().strip_prefix("app-path=")) + .map(|value| PathBuf::from(value.trim())) + .filter(|path| path.is_absolute()) +} + +/// The relay binary as the host sees it, if this package ships one. +pub fn host_relay() -> Option { + let local = Path::new("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/app/libexec").join(RELAY_NAME); + if !local.is_file() { + return None; + } + let info = std::fs::read_to_string("/.flatpak-info").ok()?; + Some(app_path_from_info(&info)?.join("libexec").join(RELAY_NAME)) +} + +/// A folder this process and host commands see at the same absolute path: +/// Flatpak's per-application runtime folder, or the launcher's host-visible +/// temporary folder. The relay's probe connection proves it either way. +pub fn shared_socket_base() -> Option { + let runtime = std::env::var_os("XDG_RUNTIME_DIR") + .zip(std::env::var_os("FLATPAK_ID")) + .map(|(run, id)| PathBuf::from(run).join("app").join(id)) + .filter(|path| path.is_absolute() && path.is_dir()); + runtime.or_else(|| { + let tmp = std::env::temp_dir(); + // The Flatpak's own /tmp is private to the application sandbox. + (!tmp.starts_with("/tmp") && !tmp.starts_with("/var/tmp") && tmp.is_dir()).then_some(tmp) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn app_path_is_read_from_flatpak_info() { + let info = "[Application]\nname=com.shadowfetch.shadowcode\n\n[Instance]\napp-path=/var/lib/flatpak/app/com.shadowfetch.shadowcode/x86_64/master/abc/files\n"; + assert_eq!( + app_path_from_info(info).unwrap(), + Path::new("/var/lib/flatpak/app/com.shadowfetch.shadowcode/x86_64/master/abc/files") + ); + assert_eq!(app_path_from_info("[Instance]\napp-path=relative\n"), None); + assert_eq!(app_path_from_info("[Instance]\n"), None); + } +} diff --git a/native/core/src/lib.rs b/native/core/src/lib.rs index 735bddb1..95d10eba 100644 --- a/native/core/src/lib.rs +++ b/native/core/src/lib.rs @@ -50,6 +50,7 @@ pub mod models; pub mod notify; pub mod ollama_store; pub mod openrouter; +pub mod outcomes; pub mod page_context; pub mod parallel; pub mod patch; diff --git a/native/core/src/lifecycle.rs b/native/core/src/lifecycle.rs index 699b4a6e..30f55aee 100644 --- a/native/core/src/lifecycle.rs +++ b/native/core/src/lifecycle.rs @@ -73,3 +73,51 @@ pub async fn parent_exited(parent: Option) { tokio::time::sleep(Duration::from_millis(50)).await; } } + +/// Set by the AppImage installer when it starts a newly installed version +/// for its first-window check. The installer rolls back the AppImage, +/// runtime and database snapshot unless this file appears in time. +pub const FIRST_START_MARKER_ENV: &str = "SHADOWCODE_FIRST_START_MARKER"; + +/// Reports a successful first start: the profile database opened (and any +/// migration committed) and the main window finished loading its page. +/// Writes `ShadowCode VERSION` once to the file named by +/// [`FIRST_START_MARKER_ENV`]; does nothing when it is unset. Best effort: +/// a failure only makes the installer roll back. +pub fn report_first_start() { + static REPORTED: std::sync::Once = std::sync::Once::new(); + REPORTED.call_once(|| { + if let Some(path) = std::env::var_os(FIRST_START_MARKER_ENV).map(PathBuf::from) { + if let Err(error) = write_first_start(&path) { + tracing::warn!("could not report the first start: {error:#}"); + } + } + }); +} + +fn write_first_start(path: &std::path::Path) -> anyhow::Result<()> { + anyhow::ensure!( + path.is_absolute(), + "the first-start marker path must be absolute" + ); + crate::paths::atomic_write( + path, + format!("ShadowCode {}\n", crate::VERSION).as_bytes(), + true, + ) +} + +#[cfg(test)] +mod first_start_tests { + #[test] + fn first_start_marker_names_this_version() { + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join("first-start"); + super::write_first_start(&marker).unwrap(); + assert_eq!( + std::fs::read_to_string(&marker).unwrap(), + format!("ShadowCode {}\n", crate::VERSION) + ); + assert!(super::write_first_start(std::path::Path::new("relative")).is_err()); + } +} diff --git a/native/core/src/outcomes.rs b/native/core/src/outcomes.rs new file mode 100644 index 00000000..6e5a32a2 --- /dev/null +++ b/native/core/src/outcomes.rs @@ -0,0 +1,471 @@ +//! Per-model, per-project outcome history ("how did this model do here?"). +//! +//! Each finished top-level task records one row in `model_outcomes` +//! (schema 28): the exact picker id that ran it, the project, wall time, +//! tokens, the cost **only when the provider, the vendor CLI or this computer +//! reported it** (an estimate from a price list, or an unknown price, is +//! stored as `NULL` and shown as "unavailable"), and whether observed +//! verification passed. +//! +//! [`summarize`] turns rows into per-model statistics (medians, pass rate, +//! sample counts), and [`suggest`] picks the cheapest model that passed in +//! this project before. A suggestion is only ever *offered*; the picker +//! never switches models on its own. Candidates must be ready in the picker, +//! run on this computer when the app is offline, and stay within the +//! spending limits for paid API models. +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Rows read per project when summarizing (newest first). +pub const MAX_ROWS: usize = 2000; +/// Samples per model that count toward its statistics (newest first). +pub const MAX_SAMPLES_PER_MODEL: usize = 50; + +/// One finished task, as stored. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Outcome { + pub task_id: String, + pub session_id: String, + pub workspace: String, + /// Exact picker id (`run.model_id`, else `routing.model_id`). + pub model_id: String, + /// What the job showed as its model ("Codex", "qwen3-coder" …). + pub model_label: String, + pub mode: String, + /// `completed`, `failed` … + pub status: String, + pub success: bool, + pub started_at: f64, + pub finished_at: f64, + pub duration_seconds: f64, + pub total_tokens: u64, + /// Some token counts were estimated by ShadowCode. + pub tokens_estimated: bool, + /// US dollars, only when reported (never an estimate). + pub cost_usd: Option, + /// `Some(true)` observed verification passed, `Some(false)` it failed or + /// the task failed, `None` nothing was verified. + pub passed: Option, + pub recorded_at: f64, +} + +/// The outcome of a finished job record (the desktop job JSON with its +/// `result`), or `None` when it should not count: cancelled, a plain +/// command, or no known model. Subagents are skipped by the caller. +pub fn from_job(job: &Value, session_id: &str) -> Option { + let result = job.get("result")?; + if result["cancelled"] == true || job["status"] == "cancelled" { + return None; + } + let mode = job["mode"].as_str().unwrap_or("code"); + if mode == "command" { + return None; + } + let model_id = job["run"]["model_id"] + .as_str() + .or_else(|| job["routing"]["model_id"].as_str()) + .filter(|id| !id.is_empty() && *id != "mock")? + .to_owned(); + let started_at = job["started_at"].as_f64()?; + let finished_at = job["finished_at"].as_f64()?; + let success = result["success"] == true; + let usage = crate::usage::parse(&job["usage"]); + Some(Outcome { + task_id: job["task_id"].as_str()?.to_owned(), + session_id: session_id.to_owned(), + workspace: job["workspace"].as_str()?.to_owned(), + model_label: job["model"].as_str().unwrap_or(&model_id).to_owned(), + model_id, + mode: mode.to_owned(), + status: job["status"].as_str().unwrap_or("").to_owned(), + success, + started_at, + finished_at, + duration_seconds: (finished_at - started_at).max(0.0), + total_tokens: usage.total_tokens, + tokens_estimated: usage.estimated || job["usage_is_estimated"] == true, + cost_usd: reported_cost(&usage), + passed: passed(success, &result["verification"]), + recorded_at: crate::now(), + }) +} + +/// The cost only when it was reported: not from a price list, not partly +/// unknown, and a real amount. +pub fn reported_cost(usage: &crate::models::Usage) -> Option { + usage + .cost_usd + .filter(|cost| !usage.cost_estimated && cost.is_finite() && *cost >= 0.0) +} + +/// Pass/fail from the task's verification summary. Only observed command +/// evidence counts; a model's claim alone is "not verified" (`None`). +pub fn passed(success: bool, verification: &Value) -> Option { + let observed = verification["commands"] + .as_array() + .is_some_and(|commands| !commands.is_empty()); + if verification["verified"] == true && success { + Some(true) + } else if observed || (!success && verification["status"] == "failed") { + Some(false) + } else { + None + } +} + +/// Statistics for one model in one project. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ModelStats { + pub model_id: String, + pub model_label: String, + /// Finished tasks counted. + pub samples: usize, + pub median_seconds: Option, + pub median_tokens: Option, + /// Median over tasks whose cost was reported; `None` = unavailable. + pub median_cost_usd: Option, + /// Tasks whose cost was reported. + pub cost_samples: usize, + /// Tasks with a pass/fail verification result. + pub verified_samples: usize, + pub passes: usize, + /// `passes / verified_samples`; `None` when nothing was verified. + pub pass_rate: Option, + pub last_finished_at: f64, +} + +fn median(mut values: Vec) -> Option { + if values.is_empty() { + return None; + } + values.sort_by(f64::total_cmp); + let mid = values.len() / 2; + Some(if values.len().is_multiple_of(2) { + (values[mid - 1] + values[mid]) / 2.0 + } else { + values[mid] + }) +} + +/// Per-model statistics, most samples first. `rows` may be in any order; +/// the newest [`MAX_SAMPLES_PER_MODEL`] per model count. +pub fn summarize(rows: &[Outcome]) -> Vec { + let mut sorted: Vec<&Outcome> = rows.iter().collect(); + sorted.sort_by(|a, b| b.finished_at.total_cmp(&a.finished_at)); + let mut groups: Vec<(String, Vec<&Outcome>)> = Vec::new(); + for row in sorted { + match groups.iter_mut().find(|(id, _)| *id == row.model_id) { + Some((_, list)) if list.len() >= MAX_SAMPLES_PER_MODEL => {} + Some((_, list)) => list.push(row), + None => groups.push((row.model_id.clone(), vec![row])), + } + } + let mut stats: Vec = groups + .into_iter() + .map(|(model_id, list)| { + let costs: Vec = list.iter().filter_map(|r| r.cost_usd).collect(); + let verified: Vec = list.iter().filter_map(|r| r.passed).collect(); + let passes = verified.iter().filter(|p| **p).count(); + ModelStats { + model_label: list[0].model_label.clone(), + samples: list.len(), + median_seconds: median(list.iter().map(|r| r.duration_seconds).collect()), + median_tokens: median(list.iter().map(|r| r.total_tokens as f64).collect()) + .map(|v| v.round() as u64), + cost_samples: costs.len(), + median_cost_usd: median(costs), + verified_samples: verified.len(), + passes, + pass_rate: (!verified.is_empty()).then(|| passes as f64 / verified.len() as f64), + last_finished_at: list[0].finished_at, + model_id, + } + }) + .collect(); + stats.sort_by(|a, b| { + b.samples + .cmp(&a.samples) + .then(b.last_finished_at.total_cmp(&a.last_finished_at)) + }); + stats +} + +/// What the suggestion may choose from: the picker rows and the limits in +/// force right now. +#[derive(Clone, Debug, Default)] +pub struct Constraints { + /// The app is offline: only models on this computer. + pub offline: bool, + /// Today's spending limit for paid API models is reached. + pub daily_limit_reached: bool, + /// The per-task limit for paid API models, when set. + pub task_limit_usd: Option, + /// The model currently chosen in the picker (never suggested again). + pub current: Option, +} + +/// A model the user may switch to. Never applied without the user's click. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Suggestion { + pub model_id: String, + pub name: String, + pub median_cost_usd: f64, + pub pass_rate: f64, + pub passes: usize, + pub samples: usize, + pub reason: String, +} + +/// A paid API model (billed per token, under the spending limits). +fn is_paid_row(row: &Value) -> bool { + row["group"] == "api" || row["billing"] == "api_key" +} + +/// The cheapest model, by median reported cost, that passed verification in +/// this project at least once and can run now. Models whose cost was never +/// reported are not ranked: an unknown price is not cheap. +pub fn suggest( + stats: &[ModelStats], + targets: &[Value], + limits: &Constraints, +) -> Option { + let mut best: Option<(&ModelStats, &Value, f64)> = None; + for stat in stats { + if stat.passes == 0 { + continue; + } + let Some(cost) = stat.median_cost_usd else { + continue; + }; + let Some(row) = targets.iter().find(|row| row["id"] == stat.model_id) else { + continue; + }; + if row["availability"] != "ready" { + continue; + } + if limits.offline && row["inference"] != "local" { + continue; + } + if is_paid_row(row) { + if limits.daily_limit_reached { + continue; + } + if limits.task_limit_usd.is_some_and(|limit| cost > limit) { + continue; + } + } + let rate = stat.pass_rate.unwrap_or(0.0); + let better = match best { + None => true, + Some((current, _, best_cost)) => { + cost < best_cost + || (cost == best_cost + && (rate > current.pass_rate.unwrap_or(0.0) + || (rate == current.pass_rate.unwrap_or(0.0) + && stat.samples > current.samples))) + } + }; + if better { + best = Some((stat, row, cost)); + } + } + let (stat, row, cost) = best?; + if limits.current.as_deref() == Some(stat.model_id.as_str()) { + return None; + } + let name = row["name"] + .as_str() + .filter(|n| !n.is_empty()) + .unwrap_or(&stat.model_label) + .to_owned(); + Some(Suggestion { + reason: format!( + "Cheapest model that passed verification in this project: {} of {} verified tasks passed, median reported cost {}.", + stat.passes, + stat.verified_samples, + crate::spending::money(cost) + ), + model_id: stat.model_id.clone(), + name, + median_cost_usd: cost, + pass_rate: stat.pass_rate.unwrap_or(0.0), + passes: stat.passes, + samples: stat.samples, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn row( + model: &str, + finished: f64, + secs: f64, + cost: Option, + passed: Option, + ) -> Outcome { + Outcome { + task_id: format!("{model}-{finished}"), + workspace: "/p".into(), + model_id: model.into(), + model_label: model.into(), + started_at: finished - secs, + finished_at: finished, + duration_seconds: secs, + total_tokens: 1000, + cost_usd: cost, + passed, + success: true, + ..Default::default() + } + } + + fn job(cost: Value, estimated: bool, verification: Value) -> Value { + json!({ + "task_id":"t1","workspace":"/p","mode":"code","status":"completed","model":"Qwen", + "started_at":100.0,"finished_at":160.0, + "usage":{"total_tokens":1234,"cost_usd":cost,"cost_estimated":estimated}, + "run":{"model_id":"api:openrouter:qwen"}, + "result":{"success":true,"cancelled":false,"verification":verification} + }) + } + + #[test] + fn records_only_reported_costs() { + let verified = json!({"verified":true,"commands":[{"exit_code":0}]}); + let reported = from_job(&job(json!(0.12), false, verified.clone()), "s").unwrap(); + assert_eq!(reported.cost_usd, Some(0.12)); + assert_eq!(reported.duration_seconds, 60.0); + assert_eq!(reported.total_tokens, 1234); + assert_eq!(reported.passed, Some(true)); + assert_eq!(reported.model_id, "api:openrouter:qwen"); + let estimated = from_job(&job(json!(0.12), true, verified.clone()), "s").unwrap(); + assert_eq!( + estimated.cost_usd, None, + "price-list estimates are not reported costs" + ); + let unknown = from_job(&job(Value::Null, false, verified), "s").unwrap(); + assert_eq!(unknown.cost_usd, None); + } + + #[test] + fn skips_cancelled_and_command_jobs() { + let mut cancelled = job(json!(0.1), false, json!({})); + cancelled["result"]["cancelled"] = json!(true); + assert!(from_job(&cancelled, "s").is_none()); + let mut command = job(json!(0.1), false, json!({})); + command["mode"] = json!("command"); + assert!(from_job(&command, "s").is_none()); + } + + #[test] + fn pass_requires_observed_verification() { + assert_eq!(passed(true, &json!({"verified":true})), Some(true)); + assert_eq!( + passed(true, &json!({"claim":"model_claim","commands":[]})), + None + ); + assert_eq!( + passed(true, &json!({"verified":false,"commands":[{}]})), + Some(false) + ); + assert_eq!( + passed(false, &json!({"verified":true,"commands":[{}]})), + Some(false) + ); + assert_eq!(passed(false, &json!({"commands":[]})), None); + } + + #[test] + fn summarizes_medians_and_pass_rate() { + let rows = vec![ + row("a", 10.0, 10.0, Some(0.10), Some(true)), + row("a", 20.0, 30.0, None, Some(false)), + row("a", 30.0, 20.0, Some(0.30), None), + row("b", 40.0, 5.0, None, None), + ]; + let stats = summarize(&rows); + assert_eq!(stats[0].model_id, "a"); + assert_eq!(stats[0].samples, 3); + assert_eq!(stats[0].median_seconds, Some(20.0)); + assert_eq!(stats[0].cost_samples, 2); + assert!((stats[0].median_cost_usd.unwrap() - 0.20).abs() < 1e-9); + assert_eq!(stats[0].verified_samples, 2); + assert_eq!(stats[0].pass_rate, Some(0.5)); + assert_eq!( + stats[1].median_cost_usd, None, + "no reported cost stays unavailable" + ); + assert_eq!(stats[1].pass_rate, None); + } + + fn target(id: &str, group: &str, inference: &str) -> Value { + json!({"id":id,"name":id.to_uppercase(),"group":group,"inference":inference,"availability":"ready"}) + } + + #[test] + fn suggests_cheapest_passing_model_within_rules() { + let rows = vec![ + row("api:cheap", 1.0, 10.0, Some(0.05), Some(true)), + row("api:dear", 2.0, 10.0, Some(0.50), Some(true)), + row("api:never-passed", 3.0, 10.0, Some(0.01), Some(false)), + row("cli:codex", 4.0, 10.0, None, Some(true)), + row("local:gguf:x", 5.0, 10.0, Some(0.0), Some(true)), + ]; + let stats = summarize(&rows); + let targets = vec![ + target("api:cheap", "api", "cloud"), + target("api:dear", "api", "cloud"), + target("api:never-passed", "api", "cloud"), + target("cli:codex", "subscriptions", "cloud"), + ]; + let pick = suggest(&stats, &targets, &Constraints::default()).unwrap(); + assert_eq!(pick.model_id, "api:cheap"); + assert!(pick.reason.contains("$0.05"), "{}", pick.reason); + + // Daily limit reached: paid models are out, unknown cost is not ranked. + let limits = Constraints { + daily_limit_reached: true, + ..Default::default() + }; + assert!(suggest(&stats, &targets, &limits).is_none()); + + // Per-task limit below the median cost excludes that model. + let limits = Constraints { + task_limit_usd: Some(0.01), + ..Default::default() + }; + assert!(suggest(&stats, &targets, &limits).is_none()); + + // Offline: only models on this computer. + let mut with_local = targets.clone(); + with_local.push(target("local:gguf:x", "local", "local")); + let limits = Constraints { + offline: true, + ..Default::default() + }; + assert_eq!( + suggest(&stats, &with_local, &limits).unwrap().model_id, + "local:gguf:x" + ); + + // Already selected: nothing to suggest. + let limits = Constraints { + current: Some("api:cheap".into()), + ..Default::default() + }; + assert!(suggest(&stats, &targets, &limits).is_none()); + + // Not ready in the picker: skipped. + let mut not_ready = targets.clone(); + not_ready[0]["availability"] = json!("sign_in"); + assert_eq!( + suggest(&stats, ¬_ready, &Constraints::default()) + .unwrap() + .model_id, + "api:dear" + ); + } +} diff --git a/native/core/src/routing.rs b/native/core/src/routing.rs index 0f9b52c1..68940e05 100644 --- a/native/core/src/routing.rs +++ b/native/core/src/routing.rs @@ -44,8 +44,8 @@ pub fn validate(value: &Value) -> Result<()> { ) })?; for (key, value) in object { - if key == "enabled" { - ensure!(value.is_boolean(), "routing.enabled must be a boolean"); + if key == "enabled" || key == "suggest_from_history" { + ensure!(value.is_boolean(), "routing.{key} must be a boolean"); } else { ensure!( PURPOSES.contains(&key.as_str()), diff --git a/native/core/src/sandbox.rs b/native/core/src/sandbox.rs index df49e0e0..c77c781d 100644 --- a/native/core/src/sandbox.rs +++ b/native/core/src/sandbox.rs @@ -475,6 +475,94 @@ pub struct ChildSetup { #[cfg(target_os = "linux")] landlock: Option>, gate: Option>, + /// Flatpak host commands: the proxy's Unix socket, reached through the + /// relay inside the host bubblewrap network namespace. + relay: Option>, +} +/// The proxy's listening Unix socket and the private folder holding it, +/// removed when the command's setup is dropped. +pub struct RelaySocket { + listener: Mutex>, + dir: tempfile::TempDir, +} +impl RelaySocket { + /// Bind a fresh socket in a private folder below `base`. + pub fn bind(base: &Path) -> Result { + let dir = tempfile::Builder::new() + .prefix("scp-") + .tempdir_in(base) + .context("Could not create the allow-list proxy folder")?; + let path = dir.path().join(RELAY_SOCKET); + // sockaddr_un holds 107 bytes and a terminating NUL. + ensure!( + path.as_os_str().len() < 108, + "The allow-list proxy socket path is too long ({})", + path.display() + ); + let listener = std::os::unix::net::UnixListener::bind(&path) + .context("Could not create the allow-list proxy socket")?; + Ok(Self { + listener: Mutex::new(Some(listener)), + dir, + }) + } + pub fn dir(&self) -> &Path { + self.dir.path() + } +} +impl std::fmt::Debug for RelaySocket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RelaySocket") + .field("dir", &self.dir.path()) + .finish() + } +} +/// Socket file name inside the relay folder. +const RELAY_SOCKET: &str = "p"; +/// Where the relay binary and the proxy folder appear inside bubblewrap. +const RELAY_IN_SANDBOX: &str = "/shadowcode-proxy-relay"; +const RELAY_DIR_IN_SANDBOX: &str = "/shadowcode-proxy"; + +/// Turn an [`isolated_args`] profile built with [`Net::Proxy`] into one that +/// a host bubblewrap enforces by itself: a private network namespace whose +/// only way out is `relay`, forwarding loopback port [`proxy::PROXY_PORT`] to +/// the proxy socket in `socket_dir`. Refuses rather than guess if the profile +/// does not have the expected shape. +pub fn host_relay_args( + mut args: Vec, + relay: &Path, + socket_dir: &Path, +) -> Result> { + let separator = args + .iter() + .rposition(|arg| arg == "--") + .context("Sandbox profile has no command separator")?; + ensure!( + args[separator + 1..].first().map(String::as_str) == Some("/bin/sh"), + "Sandbox profile has an unexpected command" + ); + ensure!( + !args[..separator].iter().any(|arg| arg == "--share-net"), + "Sandbox profile shares the network" + ); + let s = |p: &Path| p.to_string_lossy().into_owned(); + let insert = [ + "--unshare-net".to_owned(), + "--ro-bind".to_owned(), + s(relay), + RELAY_IN_SANDBOX.to_owned(), + "--ro-bind".to_owned(), + s(socket_dir), + RELAY_DIR_IN_SANDBOX.to_owned(), + ]; + let relay_command = [ + RELAY_IN_SANDBOX.to_owned(), + format!("{RELAY_DIR_IN_SANDBOX}/{RELAY_SOCKET}"), + proxy::PROXY_PORT.to_string(), + ]; + args.splice(separator + 1..separator + 1, relay_command); + args.splice(separator..separator, insert); + Ok(args) } impl std::fmt::Debug for ChildSetup { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -482,7 +570,9 @@ impl std::fmt::Debug for ChildSetup { #[cfg(target_os = "linux")] d.field("netns", &self.netns.is_some()) .field("landlock", &self.landlock.is_some()); - d.field("proxy", &self.gate.is_some()).finish() + d.field("proxy", &self.gate.is_some()) + .field("relay", &self.relay) + .finish() } } /// Stops the allow-list proxy when the command's process is finished. @@ -514,6 +604,15 @@ impl ChildSetup { let listener = netns.receive()?; return Ok(Some(ProxyGuard(proxy::spawn(listener, gate.clone())?))); } + if let (Some(relay), Some(gate)) = (&self.relay, &self.gate) { + let listener = relay + .listener + .lock() + .map_err(|_| anyhow::anyhow!("Proxy socket lock poisoned"))? + .take() + .context("The allow-list proxy socket was already used")?; + return Ok(Some(ProxyGuard(proxy::spawn_unix(listener, gate.clone())?))); + } Ok(None) } } @@ -569,9 +668,6 @@ pub fn prepare_shell( command: &str, policy: &ShellPolicy, ) -> Result { - if crate::flatpak::host_bridge().is_some() && policy.network == ShellNetwork::Allowlist { - bail!("The command did not run: host-name network restrictions are unavailable for Flatpak host commands. Choose network off or on in Settings > Permissions & network."); - } let layout = HomeLayout::current(&policy.home_binds, workspace); match working_bwrap(workspace) { Ok(program) => { @@ -582,7 +678,37 @@ pub fn prepare_shell( }; let mut child = None; let mut gate = None; - if net == Net::Proxy { + let mut relay = None; + if net == Net::Proxy && crate::flatpak::host_bridge().is_some() { + // bubblewrap runs on the host, so this process cannot place it + // in a namespace. The host bubblewrap makes its own private + // network namespace instead, and the relay shipped in the + // package is its only route, to this process's proxy. + let unavailable = + "The command did not run: the network allow-list for Flatpak host commands"; + let program = crate::flatpak::host_relay().with_context(|| { + format!( + "{unavailable} needs {} in this package", + crate::flatpak::RELAY_NAME + ) + })?; + let base = crate::flatpak::shared_socket_base().with_context(|| { + format!("{unavailable} needs a folder shared with the host") + })?; + let socket = + Arc::new(RelaySocket::bind(&base).with_context(|| unavailable.to_owned())?); + let shared = proxy::Gate::new(policy.allow.clone()); + child = Some(ChildSetup { + #[cfg(target_os = "linux")] + netns: None, + #[cfg(target_os = "linux")] + landlock: None, + gate: Some(shared.clone()), + relay: Some(socket.clone()), + }); + gate = Some(shared); + relay = Some((program, socket)); + } else if net == Net::Proxy { #[cfg(target_os = "linux")] { let shared = proxy::Gate::new(policy.allow.clone()); @@ -590,6 +716,7 @@ pub fn prepare_shell( netns: Some(Arc::new(netns::Netns::new()?)), landlock: None, gate: Some(shared.clone()), + relay: None, }); gate = Some(shared); } @@ -601,6 +728,9 @@ pub fn prepare_shell( if let Some(index) = args.iter().position(|arg| arg == "--chdir") { args[index + 1] = cwd.to_string_lossy().into_owned(); } + if let Some((program, socket)) = &relay { + args = host_relay_args(args, program, socket.dir())?; + } let note = json!({ "mode": "bubblewrap", "network": network_label(policy.network), @@ -608,6 +738,7 @@ pub fn prepare_shell( "network_enforcement": match net { Net::Off => "private_namespace", Net::On => "none", + Net::Proxy if relay.is_some() => "host_allowlist_proxy_relay", Net::Proxy => "host_allowlist_proxy", }, "allow": if net == Net::Proxy { json!(policy.allow_text) } else { Value::Null }, @@ -708,6 +839,7 @@ pub fn prepare_shell( netns: None, landlock: Some(Arc::new(fd)), gate: None, + relay: None, }), #[cfg(not(target_os = "linux"))] child: None, @@ -901,6 +1033,172 @@ mod tests { assert!(position(&proxy, &["--unsetenv", "NO_PROXY"]).is_some()); } + #[test] + fn flatpak_relay_profile_unshares_the_network_and_wraps_the_command() { + let (_root, home, ws) = fake_home(); + let layout = HomeLayout::build(Some(home), &[], &ws); + let base = isolated_args(&ws, "curl https://crates.io", Net::Proxy, None, &layout); + let relay = Path::new("/var/lib/flatpak/app/x/files/libexec/shadowcode-proxy-relay"); + let dir = Path::new("/run/user/1000/app/x/scp-abc"); + let args = host_relay_args(base, relay, dir).unwrap(); + let separator = args.iter().rposition(|a| a == "--").unwrap(); + assert!(args[..separator].iter().any(|a| a == "--unshare-net")); + assert!( + position(&args, &["--ro-bind", &text(relay), RELAY_IN_SANDBOX]).unwrap() < separator + ); + assert!( + position(&args, &["--ro-bind", &text(dir), RELAY_DIR_IN_SANDBOX]).unwrap() < separator + ); + let port = proxy::PROXY_PORT.to_string(); + assert_eq!( + &args[separator + 1..], + &[ + RELAY_IN_SANDBOX, + "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/shadowcode-proxy/p", + port.as_str(), + "/bin/sh", + "-c", + "curl https://crates.io" + ] + ); + // Proxy environment is still set for the command. + let url = format!("http://127.0.0.1:{}", proxy::PROXY_PORT); + assert!(position(&args, &["--setenv", "HTTPS_PROXY", &url]).is_some()); + } + + #[test] + fn flatpak_relay_profile_refuses_unexpected_shapes() { + let relay = Path::new("/r"); + let dir = Path::new("/d"); + assert!(host_relay_args(vec!["--die-with-parent".into()], relay, dir).is_err()); + assert!(host_relay_args(vec!["--".into(), "/usr/bin/env".into()], relay, dir).is_err()); + let shared = vec!["--share-net".into(), "--".into(), "/bin/sh".into()]; + assert!(host_relay_args(shared, relay, dir).is_err()); + } + + #[test] + fn relay_socket_refuses_paths_the_kernel_cannot_hold() { + let root = tempfile::tempdir().unwrap(); + let deep = root.path().join("d".repeat(120)); + fs::create_dir(&deep).unwrap(); + assert!(RelaySocket::bind(&deep).is_err()); + let ok = RelaySocket::bind(root.path()).unwrap(); + assert!(ok.dir().join(RELAY_SOCKET).exists()); + let dir = ok.dir().to_owned(); + drop(ok); + assert!( + !dir.exists(), + "the socket folder is removed with the command" + ); + } + + /// Build the relay shipped in the Flatpak, or skip when no C compiler. + fn build_relay(out: &Path) -> Option { + let source = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../packaging/flatpak/proxy-relay.c"); + let binary = out.join("relay"); + let status = Command::new("cc") + .args(["-Os", "-Wall", "-Wextra", "-Werror", "-o"]) + .arg(&binary) + .arg(&source) + .status() + .ok()?; + status.success().then_some(binary) + } + + fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() + } + + #[tokio::test] + async fn flatpak_relay_forwards_only_through_the_allow_list_proxy() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let temp = tempfile::tempdir().unwrap(); + let Some(binary) = build_relay(temp.path()) else { + eprintln!("skipping: no C compiler"); + return; + }; + let socket = Arc::new(RelaySocket::bind(temp.path()).unwrap()); + let gate = proxy::Gate::new(proxy::parse_list(&["allowed.invalid".into()]).unwrap()); + let setup = ChildSetup { + gate: Some(gate.clone()), + relay: Some(socket.clone()), + ..ChildSetup::default() + }; + let _guard = setup.after_spawn().unwrap().unwrap(); + let port = free_port(); + let mut relay = tokio::process::Command::new(&binary) + .arg(socket.dir().join(RELAY_SOCKET)) + .arg(port.to_string()) + .args(["/bin/sh", "-c", "sleep 5; exit 7"]) + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut stream = None; + for _ in 0..100 { + if let Ok(s) = tokio::net::TcpStream::connect(("127.0.0.1", port)).await { + stream = Some(s); + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let mut stream = stream.expect("relay listens on loopback"); + stream + .write_all(b"CONNECT example.com:443 HTTP/1.1\r\n\r\n") + .await + .unwrap(); + let mut reply = Vec::new(); + let _ = tokio::time::timeout(Duration::from_secs(5), stream.read_to_end(&mut reply)).await; + let reply = String::from_utf8_lossy(&reply); + assert!(reply.starts_with("HTTP/1.1 403"), "{reply}"); + assert!(gate.report()["blocked"] + .as_array() + .unwrap() + .iter() + .any(|b| b == "example.com:443")); + let status = relay.wait().await.unwrap(); + assert_eq!( + status.code(), + Some(7), + "the relay returns the command's status" + ); + } + + #[test] + fn flatpak_relay_fails_closed_without_the_proxy() { + let temp = tempfile::tempdir().unwrap(); + let Some(binary) = build_relay(temp.path()) else { + eprintln!("skipping: no C compiler"); + return; + }; + let marker = temp.path().join("ran"); + let script = format!("touch '{}'", marker.display()); + let output = Command::new(&binary) + .arg(temp.path().join("missing.sock")) + .arg(free_port().to_string()) + .args(["/bin/sh", "-c", &script]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(125)); + assert!(!marker.exists(), "the command must not run"); + assert!(String::from_utf8_lossy(&output.stderr).contains("did not run")); + // A port that is already taken is refused the same way. + let busy = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let socket = RelaySocket::bind(temp.path()).unwrap(); + let output = Command::new(&binary) + .arg(socket.dir().join(RELAY_SOCKET)) + .arg(busy.local_addr().unwrap().port().to_string()) + .args(["/bin/sh", "-c", &script]) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(125)); + assert!(!marker.exists(), "the command must not run"); + } + #[test] fn protected_home_entries_are_never_mounted() { for bad in [ diff --git a/native/core/src/sandbox/proxy.rs b/native/core/src/sandbox/proxy.rs index 1b9962e8..ffd27ef3 100644 --- a/native/core/src/sandbox/proxy.rs +++ b/native/core/src/sandbox/proxy.rs @@ -13,7 +13,7 @@ use std::{ time::Duration, }; use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, net::{TcpListener, TcpStream}, }; @@ -293,7 +293,7 @@ fn usable(addr: &SocketAddr, literal: bool) -> bool { } } -async fn deny(stream: &mut TcpStream, status: &str, text: &str) -> Result<()> { +async fn deny(stream: &mut S, status: &str, text: &str) -> Result<()> { let body = format!("ShadowCode sandbox: {text}\n"); let reply = format!( "HTTP/1.1 {status}\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", @@ -303,7 +303,7 @@ async fn deny(stream: &mut TcpStream, status: &str, text: &str) -> Result<()> { Ok(()) } -async fn handle(mut client: TcpStream, gate: &Gate) -> Result<()> { +async fn handle(mut client: S, gate: &Gate) -> Result<()> { let mut buffer = Vec::with_capacity(4096); let end = tokio::time::timeout(Duration::from_secs(15), async { loop { @@ -394,6 +394,37 @@ pub fn spawn( Ok(tokio::spawn(serve(listener, gate))) } +/// Serve the same proxy on a Unix socket until the returned task is aborted. +/// +/// Used for Flatpak host commands: the command runs in a host bubblewrap +/// network namespace whose loopback relay forwards each connection to this +/// socket, so the allow-list decision still happens here. +#[cfg(unix)] +pub fn spawn_unix( + listener: std::os::unix::net::UnixListener, + gate: Arc, +) -> Result> { + listener.set_nonblocking(true)?; + let listener = tokio::net::UnixListener::from_std(listener)?; + Ok(tokio::spawn(async move { + let slots = Arc::new(tokio::sync::Semaphore::new(MAX_CONNECTIONS)); + loop { + let Ok((stream, _)) = listener.accept().await else { + tokio::time::sleep(Duration::from_millis(20)).await; + continue; + }; + let Ok(permit) = slots.clone().acquire_owned().await else { + return; + }; + let gate = gate.clone(); + tokio::spawn(async move { + let _ = handle(stream, &gate).await; + drop(permit); + }); + } + })) +} + #[cfg(test)] mod tests { use super::*; diff --git a/native/core/src/service/model_catalog.rs b/native/core/src/service/model_catalog.rs index 6e028d51..d31642e2 100644 --- a/native/core/src/service/model_catalog.rs +++ b/native/core/src/service/model_catalog.rs @@ -107,6 +107,7 @@ impl Service { let unloaded = self.engine.local_runtime().unload().await?; Ok(json!({"ok":true,"unloaded":unloaded})) } + ("GET", "/api/models/outcomes") => self.model_outcomes(call).await, ("POST", "/api/models/test") => self.test_model(&call.body).await, _ => self.blocking(call, Self::model_routes_sync).await, } @@ -177,6 +178,46 @@ impl Service { _ => Err(call.unavailable()), } } + /// GET /api/models/outcomes?workspace=¤t=: per-model outcome + /// history for one project, and (only when `routing.suggest_from_history` + /// is on) the cheapest model that passed there before. Never switches + /// the model; the picker shows the suggestion for the user to accept. + async fn model_outcomes(&self, call: &Call) -> Result { + let workspace = if call.q("workspace").is_empty() { + self.workspace()? + } else { + Workspace::open(&expand_path(call.q("workspace"))?)?.path + }; + let cfg = Config::load(self.engine.paths(), Some(&workspace))?; + let store = self.engine.store(); + let rows = { + let workspace = workspace.clone(); + store.run(move |s| s.model_outcomes(&workspace)).await? + }; + let stats = crate::outcomes::summarize(&rows); + let opted_in = cfg.routing["suggest_from_history"] == true; + let suggestion = if opted_in && !stats.is_empty() { + let picker = self.picker_catalog_with(&cfg, false, true).await?; + let targets = picker["targets"].as_array().cloned().unwrap_or_default(); + let day = crate::spending::today(&store, crate::now())?; + let limits = crate::outcomes::Constraints { + offline: cfg.offline(), + daily_limit_reached: crate::spending::daily_limit(&cfg.spending, &day) + .is_some_and(|limit| day.usd >= limit), + task_limit_usd: cfg.spending.task_usd, + current: Some(call.q("current").to_owned()).filter(|c| !c.is_empty()), + }; + crate::outcomes::suggest(&stats, &targets, &limits) + } else { + None + }; + Ok(json!({ + "workspace": workspace, + "suggest_from_history": opted_in, + "models": stats, + "suggestion": suggestion, + })) + } /// POST /api/models/test: one short completion (vendor CLIs report /// their install/login state instead), bounded to 45 seconds. async fn test_model(&self, body: &Value) -> Result { diff --git a/native/core/src/store.rs b/native/core/src/store.rs index e33022ae..c8409a20 100644 --- a/native/core/src/store.rs +++ b/native/core/src/store.rs @@ -16,6 +16,7 @@ mod goals; pub mod keys; mod memory; mod meta; +mod outcomes; pub use automations::AutomationRun; pub use goals::MilestoneSpec; pub use meta::MetaTransaction; @@ -94,7 +95,22 @@ CREATE TABLE IF NOT EXISTS task_notes ( /// Current schema version. Older databases are backed up, then migrated /// forward one step at a time inside one transaction. -pub const SCHEMA_VERSION: i64 = 27; +pub const SCHEMA_VERSION: i64 = 28; + +/// Version 28: per-model, per-project outcome history (`crate::outcomes`). +/// `cost_usd` is NULL unless the cost was reported; `passed` is NULL when +/// nothing was verified. +const MIGRATION_28: &str = r#" +CREATE TABLE IF NOT EXISTS model_outcomes ( + task_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, workspace TEXT NOT NULL, + model_id TEXT NOT NULL, model_label TEXT NOT NULL, mode TEXT NOT NULL, + status TEXT NOT NULL, success INTEGER NOT NULL, started_at REAL NOT NULL, + finished_at REAL NOT NULL, duration_seconds REAL NOT NULL, + total_tokens INTEGER NOT NULL, tokens_estimated INTEGER NOT NULL, + cost_usd REAL, passed INTEGER, recorded_at REAL NOT NULL +); +CREATE INDEX IF NOT EXISTS model_outcomes_workspace ON model_outcomes(workspace,finished_at); +"#; /// Version 27: bounded, profile-private editor drafts. Draft bytes are never /// written into the project until the user explicitly saves a reviewed file. @@ -341,6 +357,9 @@ impl Store { if version < 27 { tx.execute_batch(MIGRATION_27)?; } + if version < 28 { + tx.execute_batch(MIGRATION_28)?; + } tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; tx.commit()?; } @@ -1010,7 +1029,13 @@ impl Store { runs.push(id); } tx.execute("DELETE FROM native_meta WHERE key=?", [&index])?; - for table in ["events", "pins", "session_meta", "queued_tasks"] { + for table in [ + "events", + "pins", + "session_meta", + "queued_tasks", + "model_outcomes", + ] { tx.execute( &format!("DELETE FROM {table} WHERE session_id=?"), [session], @@ -1047,10 +1072,15 @@ impl Store { Ok(task) } pub fn finish_task(&self, tid: &str, status: &str, summary: &str, usage: &Value) -> Result<()> { - self.finish_transaction(tid, status, summary, usage, None) + self.finish_transaction(tid, status, summary, usage, None, false) .map(|_| ()) } pub fn finish_job(&self, job: &mut Value) -> Result { + self.finish_job_with(job, true) + } + /// Finish a job; `record_outcome` adds it to the per-model outcome + /// history (`crate::outcomes`). Subagent jobs pass `false`. + pub fn finish_job_with(&self, job: &mut Value, record_outcome: bool) -> Result { let tid = job["task_id"] .as_str() .context("Missing task ID")? @@ -1058,7 +1088,7 @@ impl Store { let status = job["status"].as_str().context("Missing status")?.to_owned(); let summary = job["summary"].as_str().unwrap_or("").to_owned(); let usage = job["usage"].clone(); - self.finish_transaction(&tid, &status, &summary, &usage, Some(job))? + self.finish_transaction(&tid, &status, &summary, &usage, Some(job), record_outcome)? .context("Missing completion event") } fn finish_transaction( @@ -1068,6 +1098,7 @@ impl Store { summary: &str, usage: &Value, job: Option<&mut Value>, + record_outcome: bool, ) -> Result> { let mut db = self.lock()?; let tx = db.transaction()?; @@ -1078,6 +1109,11 @@ impl Store { tx.execute("INSERT INTO events(ts,type,session_id,task_id,payload) VALUES(?,'agent.completed',?,?,?)",params![ts,sid,tid,payload.to_string()])?; let cursor = tx.last_insert_rowid(); job["event_cursor"] = json!(cursor); + if record_outcome { + if let Some(outcome) = crate::outcomes::from_job(job, &sid) { + outcomes::record_on(&tx, &outcome)?; + } + } tx.execute( "UPDATE desktop_jobs SET payload=? WHERE id=?", params![job.to_string(), job["id"].as_str()], diff --git a/native/core/src/store/outcomes.rs b/native/core/src/store/outcomes.rs new file mode 100644 index 00000000..b0746b57 --- /dev/null +++ b/native/core/src/store/outcomes.rs @@ -0,0 +1,132 @@ +//! `model_outcomes` (schema 28): see [`crate::outcomes`]. +use super::*; +use crate::outcomes::Outcome; + +/// Insert or replace one finished task's outcome (a re-finished task keeps +/// only its latest outcome). +pub(super) fn record_on(db: &Connection, o: &Outcome) -> Result<()> { + db.execute( + "INSERT OR REPLACE INTO model_outcomes(task_id,session_id,workspace,model_id,model_label,mode,status,success,started_at,finished_at,duration_seconds,total_tokens,tokens_estimated,cost_usd,passed,recorded_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + params![ + o.task_id, + o.session_id, + o.workspace, + o.model_id, + o.model_label, + o.mode, + o.status, + o.success, + o.started_at, + o.finished_at, + o.duration_seconds, + i64::try_from(o.total_tokens).unwrap_or(i64::MAX), + o.tokens_estimated, + o.cost_usd, + o.passed, + o.recorded_at, + ], + )?; + Ok(()) +} + +impl Store { + pub fn record_outcome(&self, outcome: &Outcome) -> Result<()> { + record_on(&*self.lock()?, outcome) + } + + /// The newest outcomes recorded in this project. + pub fn model_outcomes(&self, workspace: &Path) -> Result> { + let db = self.lock()?; + let mut query = db.prepare( + "SELECT task_id,session_id,workspace,model_id,model_label,mode,status,success,started_at,finished_at,duration_seconds,total_tokens,tokens_estimated,cost_usd,passed,recorded_at FROM model_outcomes WHERE workspace=? ORDER BY finished_at DESC LIMIT ?", + )?; + let rows = query.query_map( + params![ + workspace.to_string_lossy(), + crate::outcomes::MAX_ROWS as i64 + ], + |r| { + Ok(Outcome { + task_id: r.get(0)?, + session_id: r.get(1)?, + workspace: r.get(2)?, + model_id: r.get(3)?, + model_label: r.get(4)?, + mode: r.get(5)?, + status: r.get(6)?, + success: r.get(7)?, + started_at: r.get(8)?, + finished_at: r.get(9)?, + duration_seconds: r.get(10)?, + total_tokens: r.get::<_, i64>(11)?.max(0) as u64, + tokens_estimated: r.get(12)?, + cost_usd: r.get(13)?, + passed: r.get(14)?, + recorded_at: r.get(15)?, + }) + }, + )?; + Ok(rows.collect::>()?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn records_and_reads_back_per_project() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(&dir.path().join("s.sqlite")).unwrap(); + let mut outcome = Outcome { + task_id: "t1".into(), + session_id: "s1".into(), + workspace: "/a".into(), + model_id: "cli:codex".into(), + model_label: "Codex".into(), + mode: "code".into(), + status: "completed".into(), + success: true, + finished_at: 5.0, + duration_seconds: 5.0, + total_tokens: 42, + cost_usd: None, + passed: Some(true), + ..Default::default() + }; + store.record_outcome(&outcome).unwrap(); + outcome.task_id = "t2".into(); + outcome.workspace = "/b".into(); + outcome.cost_usd = Some(0.25); + outcome.passed = None; + store.record_outcome(&outcome).unwrap(); + let a = store.model_outcomes(Path::new("/a")).unwrap(); + assert_eq!(a.len(), 1); + assert_eq!(a[0].cost_usd, None); + assert_eq!(a[0].passed, Some(true)); + assert_eq!(a[0].total_tokens, 42); + let b = store.model_outcomes(Path::new("/b")).unwrap(); + assert_eq!(b[0].cost_usd, Some(0.25)); + assert_eq!(b[0].passed, None); + } + + #[test] + fn migrates_a_schema_27_database() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("s.sqlite"); + drop(Store::open(&path).unwrap()); + { + let db = Connection::open(&path).unwrap(); + db.execute_batch("DROP TABLE model_outcomes; PRAGMA user_version=27;") + .unwrap(); + } + let store = Store::open(&path).unwrap(); + assert!(store.model_outcomes(Path::new("/a")).unwrap().is_empty()); + let version: i64 = store + .lock() + .unwrap() + .pragma_query_value(None, "user_version", |r| r.get(0)) + .unwrap(); + assert_eq!(version, SCHEMA_VERSION); + } +} diff --git a/native/core/src/updates.rs b/native/core/src/updates.rs index fb7aeaeb..648da6c3 100644 --- a/native/core/src/updates.rs +++ b/native/core/src/updates.rs @@ -52,13 +52,27 @@ const STATE_FILE: &str = "update-check.json"; const MAX_RESPONSE_BYTES: usize = 2_000_000; const MAX_POLICY_BYTES: u64 = 64 * 1024; const MAX_MESSAGE_CHARS: usize = 300; -/// Files a release needs before the authenticated installer accepts it. -const SIGNATURE_FILES: [&str; 4] = [ - "SHA256SUMS", - "RELEASE-MANIFEST.json", - "RELEASE-AUTH", - "RELEASE-AUTH.sig", -]; +/// Files a release needs before the authenticated installer accepts it on +/// `arch`. x86_64 keeps the historical names; aarch64 release metadata carries +/// an `-aarch64` suffix so both architectures fit one release. +fn signature_files(arch: &str) -> [String; 4] { + let suffix = if arch == "aarch64" { "-aarch64" } else { "" }; + [ + format!("SHA256SUMS{suffix}"), + format!("RELEASE-MANIFEST{suffix}.json"), + format!("RELEASE-AUTH{suffix}"), + format!("RELEASE-AUTH{suffix}.sig"), + ] +} + +/// The AppImage name part for `arch` (`ShadowCode_VERSION_.AppImage`). +fn appimage_arch(arch: &str) -> &'static str { + if arch == "aarch64" { + "aarch64" + } else { + "amd64" + } +} /// `updates` in config.yaml. #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] @@ -205,6 +219,10 @@ pub struct Release { /// Drafts, pre-releases and tags other than `vMAJOR.MINOR.PATCH` are /// refused. Links are built from the tag, never taken from the response. pub fn parse_release(value: &Value) -> Result { + parse_release_for(value, std::env::consts::ARCH) +} + +fn parse_release_for(value: &Value, arch: &str) -> Result { ensure!(value.is_object(), "GitHub returned an unexpected answer"); ensure!( value["draft"] != Value::Bool(true) && value["prerelease"] != Value::Bool(true), @@ -233,7 +251,9 @@ pub fn parse_release(value: &Value) -> Result { .collect() }) .unwrap_or_default(); - let signed = SIGNATURE_FILES.iter().all(|name| assets.contains(name)); + let signed = signature_files(arch) + .iter() + .all(|name| assets.contains(&name.as_str())); let published_at = value["published_at"] .as_str() .filter(|text| { @@ -683,6 +703,15 @@ pub fn dismiss(paths: &AppPaths, version: &str) -> Result<()> { /// The steps to update this kind of installation to `release`. pub fn next_step(install: InstallKind, policy: &Policy, release: Option<&Release>) -> Value { + next_step_for(install, policy, release, std::env::consts::ARCH) +} + +fn next_step_for( + install: InstallKind, + policy: &Policy, + release: Option<&Release>, + arch: &str, +) -> Value { if let Some(message) = &policy.message { return json!({"text": message, "command": null, "link": null}); } @@ -695,11 +724,15 @@ pub fn next_step(install: InstallKind, policy: &Policy, release: Option<&Release "command": null, "link": null, }), - InstallKind::Appimage => json!({ - "text": format!("Download ShadowCode_{version}_amd64.AppImage and its four signature files (SHA256SUMS, RELEASE-MANIFEST.json, RELEASE-AUTH, RELEASE-AUTH.sig) from the release page into one folder. Then run the installer from a trusted copy of ShadowCode's installer bundle. It checks the publisher signature before it runs anything and keeps your settings."), - "command": format!("bash /path/to/trusted-bundle/scripts/install-appimage.sh ~/Downloads/ShadowCode_{version}_amd64.AppImage"), - "link": readme("appimage-recommended"), - }), + InstallKind::Appimage => { + let image = format!("ShadowCode_{version}_{}.AppImage", appimage_arch(arch)); + let files = signature_files(arch).join(", "); + json!({ + "text": format!("Download {image} and its four signature files ({files}) from the release page into one folder. Then run the installer from a trusted copy of ShadowCode's installer bundle. It checks the publisher signature before it runs anything and keeps your settings."), + "command": format!("bash /path/to/trusted-bundle/scripts/install-appimage.sh ~/Downloads/{image}"), + "link": readme("appimage-recommended"), + }) + } InstallKind::Deb => json!({ "text": "Update through your package manager or your distribution's software updates. If you installed the .deb from GitHub yourself, download the new .deb and its four signature files, verify them, then install it with apt.", "command": null, @@ -852,7 +885,7 @@ mod tests { #[test] fn release_json_is_read_from_github_fixtures() { - let release = parse_release(&fixture("latest-signed.json")).unwrap(); + let release = parse_release_for(&fixture("latest-signed.json"), "x86_64").unwrap(); assert_eq!(release.version, "0.34.0"); assert_eq!(release.tag, "v0.34.0"); // The link is built from the tag, not taken from html_url. @@ -866,7 +899,7 @@ mod tests { ); assert!(release.signed); - let unsigned = parse_release(&fixture("latest-unsigned.json")).unwrap(); + let unsigned = parse_release_for(&fixture("latest-unsigned.json"), "x86_64").unwrap(); assert_eq!(unsigned.version, "0.32.0"); assert!(!unsigned.signed); @@ -1089,11 +1122,42 @@ mod tests { assert_eq!(InstallKind::System.label(), "System package"); } + #[test] + fn aarch64_needs_its_own_signature_files_and_appimage() { + // A release signed only for x86_64 is not installable on aarch64. + let mut value = fixture("latest-signed.json"); + assert!(!parse_release_for(&value, "aarch64").unwrap().signed); + let assets = value["assets"].as_array_mut().unwrap(); + for name in [ + "SHA256SUMS-aarch64", + "RELEASE-MANIFEST-aarch64.json", + "RELEASE-AUTH-aarch64", + "RELEASE-AUTH-aarch64.sig", + ] { + assets.push(json!({"name": name})); + } + let release = parse_release_for(&value, "aarch64").unwrap(); + assert!(release.signed); + let open = policy_from(None, None, &[]); + let step = next_step_for(InstallKind::Appimage, &open, Some(&release), "aarch64"); + let text = step["text"].as_str().unwrap(); + assert!( + text.contains("ShadowCode_0.34.0_aarch64.AppImage"), + "{text}" + ); + assert!(text.contains("RELEASE-AUTH-aarch64.sig"), "{text}"); + assert!(!text.contains("amd64"), "{text}"); + assert_eq!( + step["command"], + "bash /path/to/trusted-bundle/scripts/install-appimage.sh ~/Downloads/ShadowCode_0.34.0_aarch64.AppImage" + ); + } + #[test] fn update_steps_match_the_install_type() { let open = policy_from(None, None, &[]); - let release = parse_release(&fixture("latest-signed.json")).unwrap(); - let appimage = next_step(InstallKind::Appimage, &open, Some(&release)); + let release = parse_release_for(&fixture("latest-signed.json"), "x86_64").unwrap(); + let appimage = next_step_for(InstallKind::Appimage, &open, Some(&release), "x86_64"); let text = appimage["text"].as_str().unwrap(); assert!(text.contains("ShadowCode_0.34.0_amd64.AppImage")); assert!(text.contains("RELEASE-AUTH.sig")); @@ -1109,7 +1173,7 @@ mod tests { signed: false, ..release.clone() }; - let refused = next_step(InstallKind::Appimage, &open, Some(&unsigned)); + let refused = next_step_for(InstallKind::Appimage, &open, Some(&unsigned), "x86_64"); assert!(refused["text"].as_str().unwrap().contains("will refuse it")); assert!(refused["command"].is_null()); diff --git a/native/core/src/worktrees/cleanup.rs b/native/core/src/worktrees/cleanup.rs index 6c1e59bb..a3ab444d 100644 --- a/native/core/src/worktrees/cleanup.rs +++ b/native/core/src/worktrees/cleanup.rs @@ -1378,6 +1378,14 @@ mod tests { .await; fs::set_permissions(&blocked, fs::Permissions::from_mode(0o755)).unwrap(); assert!(removed.is_err()); + // Git stops at the first entry it cannot delete, so whether it reached + // q/.git first depends on the filesystem's directory order (tmpfs and + // some ext4 hash seeds list `blocked` first). Model the removal that + // got past q/.git explicitly. + match fs::remove_file(q.join(".git")) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => panic!("{error}"), + _ => {} + } let admin = f.session.admin_path(); let mut pending = f.session.intent.as_ref().unwrap().clone(); let parent = admin.parent().unwrap(); diff --git a/packaging/com.shadowfetch.shadowcode.appdata.xml b/packaging/com.shadowfetch.shadowcode.appdata.xml index eb95951f..7d2ebccc 100644 --- a/packaging/com.shadowfetch.shadowcode.appdata.xml +++ b/packaging/com.shadowfetch.shadowcode.appdata.xml @@ -29,6 +29,9 @@ shadowcode + +

Signed ARM64 builds, a signed APT repository, an installer that checks the new version's first window and can roll back, the hostname allow-list in the Flatpak, a window that remembers its size on Wayland, and per-model history in the picker.

 +

Flatpak distribution and corrected remote-access protection for private settings and secret-file diffs.

 diff --git a/packaging/flatpak/build-in-sdk.sh b/packaging/flatpak/build-in-sdk.sh index 088fc327..84014f99 100644 --- a/packaging/flatpak/build-in-sdk.sh +++ b/packaging/flatpak/build-in-sdk.sh @@ -19,22 +19,35 @@ install -Dm755 packaging/flatpak/shadowcode-host /app/libexec/shadowcode-host cc -static -Os -Wall -Wextra -Werror packaging/flatpak/host-monitor.c -o /app/libexec/shadowcode-host-monitor strip /app/libexec/shadowcode-host-monitor install -Dm644 packaging/flatpak/host-monitor.c /app/share/licenses/shadowcode/host-monitor.c +cc -static -Os -Wall -Wextra -Werror packaging/flatpak/proxy-relay.c -o /app/libexec/shadowcode-proxy-relay +strip /app/libexec/shadowcode-proxy-relay +install -Dm644 packaging/flatpak/proxy-relay.c /app/share/licenses/shadowcode/proxy-relay.c native_cache="${SHADOWCODE_NATIVE_CACHE:-$PWD/target}" spirv_commit=$(sed -n 's/^spirv_headers_commit=//p' tools/llama.cpp.pin) llama_commit=$(sed -n 's/^commit=//p' tools/llama.cpp.pin) spirv_prefix="$native_cache/spirv-$spirv_commit" llama_build="$native_cache/llama-$llama_commit" -cmake -S tools/spirv-headers -B "$native_cache/spirv-build-$spirv_commit" -G Ninja -DCMAKE_INSTALL_PREFIX="$spirv_prefix" -cmake --install "$native_cache/spirv-build-$spirv_commit" +# SHADOWCODE_LLAMA_VULKAN=0 (scripts/build-flatpak.py --cpu-only-llama) is a +# faster CI build without the Vulkan module; releases always build it. +llama_vulkan=( + -DGGML_VULKAN=ON -DVulkan_GLSLC_EXECUTABLE=/usr/bin/glslc + -DCMAKE_PREFIX_PATH="$spirv_prefix" + -DCMAKE_C_FLAGS="-isystem $spirv_prefix/include" + -DCMAKE_CXX_FLAGS="-isystem $spirv_prefix/include" +) +if [[ "${SHADOWCODE_LLAMA_VULKAN:-1}" == 0 ]]; then + llama_vulkan=(-DGGML_VULKAN=OFF) + llama_build="$llama_build-cpu" +else + cmake -S tools/spirv-headers -B "$native_cache/spirv-build-$spirv_commit" -G Ninja -DCMAKE_INSTALL_PREFIX="$spirv_prefix" + cmake --install "$native_cache/spirv-build-$spirv_commit" +fi cmake -S tools/llama.cpp -B "$llama_build" -G Ninja \ -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON \ -DLLAMA_BUILD_COMMIT="$llama_commit" \ -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_NATIVE=OFF \ - -DGGML_CUDA=OFF -DGGML_VULKAN=ON -DVulkan_GLSLC_EXECUTABLE=/usr/bin/glslc \ - -DCMAKE_PREFIX_PATH="$spirv_prefix" \ - -DCMAKE_C_FLAGS="-isystem $spirv_prefix/include" \ - -DCMAKE_CXX_FLAGS="-isystem $spirv_prefix/include" \ + -DGGML_CUDA=OFF "${llama_vulkan[@]}" \ -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TOOLS=ON \ -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_UI=OFF -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_CURL=OFF \ -DCMAKE_INSTALL_RPATH='$ORIGIN' -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON @@ -43,6 +56,11 @@ mkdir -p /app/lib/shadowcode/NOTICES install -m755 "$llama_build/bin/llama-server" "$llama_build/bin/llama-cli" /app/lib/shadowcode/ cp -a "$llama_build/bin/"*.so* /app/lib/shadowcode/ cp tools/llama.cpp.pin /app/lib/shadowcode/COMMIT +if [[ "${SHADOWCODE_LLAMA_VULKAN:-1}" == 0 ]]; then + # Record the CPU-only CI runtime honestly; notices then omit SPIRV-Headers. + sed -i 's/^backend=.*/backend=cpu/; /^spirv_headers_commit=/d' /app/lib/shadowcode/COMMIT + export SHADOWCODE_ALLOW_CPU_ONLY_RUNTIME=1 +fi printf 'built=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> /app/lib/shadowcode/COMMIT cp tools/llama.cpp/LICENSE /app/lib/shadowcode/NOTICES/llama.cpp-LICENSE cp tools/llama.cpp/licenses/LICENSE-jsonhpp /app/lib/shadowcode/NOTICES/nlohmann-json-LICENSE @@ -50,7 +68,7 @@ cp tools/llama.cpp/vendor/cpp-httplib/LICENSE /app/lib/shadowcode/NOTICES/cpp-ht sed -n '/This software is available under 2 licenses/,$p' tools/llama.cpp/vendor/stb/stb_image.h > /app/lib/shadowcode/NOTICES/stb_image-LICENSE sed -n '/This software is available as a choice of the following licenses/,$p' tools/llama.cpp/vendor/miniaudio/miniaudio.h > /app/lib/shadowcode/NOTICES/miniaudio-LICENSE sed -n '1,/For more information, please refer to/p' tools/llama.cpp/vendor/sheredom/subprocess.h > /app/lib/shadowcode/NOTICES/subprocess.h-LICENSE -cp tools/spirv-headers/LICENSE /app/lib/shadowcode/NOTICES/SPIRV-Headers-LICENSE +[[ "${SHADOWCODE_LLAMA_VULKAN:-1}" == 0 ]] || cp tools/spirv-headers/LICENSE /app/lib/shadowcode/NOTICES/SPIRV-Headers-LICENSE sed -n '/LLM_ARCH_NAMES = {/,/^};/p' tools/llama.cpp/src/llama-arch.cpp | sed -n 's/.*"\([^"]*\)".*/\1/p' > /app/lib/shadowcode/architectures.txt # Git and Node are fallbacks when a host installation is absent. Host tools diff --git a/packaging/flatpak/com.shadowfetch.shadowcode.desktop b/packaging/flatpak/com.shadowfetch.shadowcode.desktop index e1e9647d..357de985 100644 --- a/packaging/flatpak/com.shadowfetch.shadowcode.desktop +++ b/packaging/flatpak/com.shadowfetch.shadowcode.desktop @@ -11,4 +11,4 @@ Keywords=agent;llm;coding;shadowfetch; StartupNotify=true StartupWMClass=shadowcode X-GNOME-UsesNotifications=true -X-ShadowCode-Version=2.0.0 +X-ShadowCode-Version=2.1.0 diff --git a/packaging/flatpak/proxy-relay.c b/packaging/flatpak/proxy-relay.c new file mode 100644 index 00000000..4d70ba34 --- /dev/null +++ b/packaging/flatpak/proxy-relay.c @@ -0,0 +1,135 @@ +/* ShadowCode allow-list relay for Flatpak host commands. + * + * Runs inside a host bubblewrap sandbox created with --unshare-net, so the + * only interface is loopback. It listens on 127.0.0.1:PORT and forwards each + * connection to SOCKET, a Unix socket served by ShadowCode's allow-list proxy + * inside the Flatpak. The relay makes no policy decision: the network + * namespace is what keeps every other route closed, and the proxy decides + * which hosts are reached. + * + * Fails closed: if the loopback listener or the proxy socket is unavailable, + * the command is not started and the relay exits with status 125. + * + * Usage: shadowcode-proxy-relay SOCKET PORT PROGRAM [ARG...] + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REFUSED 125 + +static int refuse(const char *what) { + fprintf(stderr, "ShadowCode sandbox: the allow-list relay could not start (%s: %s); the command did not run.\n", + what, strerror(errno)); + return REFUSED; +} + +static int connect_proxy(const char *path) { + struct sockaddr_un address = {0}; + address.sun_family = AF_UNIX; + if (strlen(path) >= sizeof address.sun_path) { errno = ENAMETOOLONG; return -1; } + strcpy(address.sun_path, path); + int fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + if (fd < 0) return -1; + if (connect(fd, (struct sockaddr *)&address, sizeof address) < 0) { + int saved = errno; + close(fd); + errno = saved; + return -1; + } + return fd; +} + +/* Copy both directions until both sides have finished. */ +static void pump(int a, int b) { + char buffer[65536]; + int open_ab = 1, open_ba = 1; + while (open_ab || open_ba) { + struct pollfd fds[] = {{open_ab ? a : -1, POLLIN, 0}, {open_ba ? b : -1, POLLIN, 0}}; + if (poll(fds, 2, -1) < 0) { if (errno == EINTR) continue; return; } + for (int i = 0; i < 2; i++) { + if (!fds[i].revents) continue; + int from = i ? b : a, to = i ? a : b; + ssize_t count = read(from, buffer, sizeof buffer); + if (count <= 0) { + shutdown(to, SHUT_WR); + if (i) open_ba = 0; else open_ab = 0; + continue; + } + for (ssize_t done = 0; done < count;) { + ssize_t wrote = write(to, buffer + done, (size_t)(count - done)); + if (wrote < 0) { if (errno == EINTR) continue; return; } + done += wrote; + } + } + } +} + +int main(int argc, char **argv) { + if (argc < 4) { + fputs("Usage: shadowcode-proxy-relay SOCKET PORT PROGRAM [ARG...]\n", stderr); + return REFUSED; + } + const char *socket_path = argv[1]; + char *end = NULL; + long port = strtol(argv[2], &end, 10); + if (!*argv[2] || *end || port < 1 || port > 65535) { + fputs("ShadowCode sandbox: invalid relay port; the command did not run.\n", stderr); + return REFUSED; + } + signal(SIGPIPE, SIG_IGN); + + int listener = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); + if (listener < 0) return refuse("socket"); + int yes = 1; + setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof yes); + struct sockaddr_in local = {0}; + local.sin_family = AF_INET; + local.sin_port = htons((uint16_t)port); + local.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (bind(listener, (struct sockaddr *)&local, sizeof local) < 0) return refuse("loopback listener"); + if (listen(listener, 64) < 0) return refuse("listen"); + + /* Prove the proxy is reachable before the command can run. */ + int probe = connect_proxy(socket_path); + if (probe < 0) return refuse("proxy socket"); + close(probe); + + pid_t command = fork(); + if (command < 0) return refuse("fork"); + if (command == 0) { + execvp(argv[3], argv + 3); + perror(argv[3]); + _exit(127); + } + for (;;) { + int status; + pid_t done; + while ((done = waitpid(-1, &status, WNOHANG)) > 0) { + if (done == command) + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); + } + struct pollfd fds[] = {{listener, POLLIN, 0}}; + if (poll(fds, 1, 50) <= 0) continue; + int client = accept4(listener, NULL, NULL, SOCK_CLOEXEC); + if (client < 0) continue; + pid_t worker = fork(); + if (worker == 0) { + close(listener); + int upstream = connect_proxy(socket_path); + if (upstream >= 0) pump(client, upstream); + _exit(0); + } + close(client); + } +} diff --git a/packaging/native-app-run.sh b/packaging/native-app-run.sh index 5eead913..64dc5bc2 100644 --- a/packaging/native-app-run.sh +++ b/packaging/native-app-run.sh @@ -12,5 +12,12 @@ if [ -n "$shadowcode_gdk_backend" ]; then export GDK_BACKEND="$shadowcode_gdk_backend" fi unset shadowcode_gdk_backend -export LD_LIBRARY_PATH="$APPDIR/usr/lib:$APPDIR/usr/lib/x86_64-linux-gnu:$APPDIR/usr/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +# The bundler copies host libraries into the Debian multiarch directory of the +# architecture the AppImage was built on (x86_64 or aarch64). +shadowcode_multiarch=x86_64-linux-gnu +if [ -d "$APPDIR/usr/lib/aarch64-linux-gnu" ]; then + shadowcode_multiarch=aarch64-linux-gnu +fi +export LD_LIBRARY_PATH="$APPDIR/usr/lib:$APPDIR/usr/lib/$shadowcode_multiarch:$APPDIR/usr/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +unset shadowcode_multiarch exec "$APPDIR/usr/bin/shadowcode" "$@" diff --git a/packaging/native-runtime/apk-packages.txt b/packaging/native-runtime/apk-packages.txt index 6f5de9b9..f7ee10ab 100644 --- a/packaging/native-runtime/apk-packages.txt +++ b/packaging/native-runtime/apk-packages.txt @@ -104,10 +104,10 @@ perl=5.40.4-r0 perl-error=0.17029-r2 perl-git=2.47.3-r0 pkgconf=2.3.0-r0 -pyc=3.12.14-r0 -python3=3.12.14-r0 -python3-pyc=3.12.14-r0 -python3-pycache-pyc0=3.12.14-r0 +pyc=3.12.15-r0 +python3=3.12.15-r0 +python3-pyc=3.12.15-r0 +python3-pycache-pyc0=3.12.15-r0 readline=8.2.13-r0 runuser=2.40.4-r1 samurai=1.2-r6 @@ -125,8 +125,8 @@ uuidgen=2.40.4-r1 wipefs=2.40.4-r1 xz=5.8.4-r0 xz-libs=5.8.4-r0 -zlib-dev=1.3.2-r0 -zlib-static=1.3.2-r0 +zlib-dev=1.3.2-r1 +zlib-static=1.3.2-r1 zstd=1.5.6-r2 zstd-dev=1.5.6-r2 zstd-libs=1.5.6-r2 diff --git a/packaging/native-runtime/build.sh b/packaging/native-runtime/build.sh index 1032ad11..a86fa591 100644 --- a/packaging/native-runtime/build.sh +++ b/packaging/native-runtime/build.sh @@ -50,17 +50,26 @@ while IFS=$'\t' read -r name archive directory; do done < <(jq -r --arg name "$name" '.packages[] | select(.name == $name) | .notices[]' alpine-sources.json) done < <(jq -r '.packages[] | [.name,.archive,.sourceDirectory] | @tsv' alpine-sources.json) +# The container runs natively for the target (docker --platform linux/amd64 or +# linux/arm64), so the runtime is named after this machine: runtime-x86_64 or +# runtime-aarch64. +ARCH="$(uname -m)" +case "$ARCH" in + x86_64|aarch64) ;; + *) echo "Unsupported AppImage runtime architecture: $ARCH" >&2; exit 1 ;; +esac + cd "$RUNTIME_SOURCE" printf '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/AppImage/type2-runtime/commit/%s\n' "$UPSTREAM_COMMIT" > src/runtime/version sed -i 's/-static-pie/-static-pie -Wl,-Map,\/out\/runtime.map/' src/runtime/Makefile bash scripts/build-runtime.sh -cp out/runtime-x86_64 out/runtime-x86_64.debug /out/ +cp "out/runtime-$ARCH" "out/runtime-$ARCH.debug" /out/ cp /lib/apk/db/installed /out/build-packages.db apk info -vv | sort > /out/build-packages.txt clang --version > /out/compiler.txt -readelf -d /out/runtime-x86_64 > /out/runtime-dynamic.txt +readelf -d "/out/runtime-$ARCH" > /out/runtime-dynamic.txt if grep -q NEEDED /out/runtime-dynamic.txt; then echo 'The AppImage runtime must link statically.' >&2 exit 1 fi -/out/runtime-x86_64 --appimage-version +"/out/runtime-$ARCH" --appimage-version diff --git a/packaging/shadow-agent.desktop b/packaging/shadow-agent.desktop index e258b102..34bdc195 100644 --- a/packaging/shadow-agent.desktop +++ b/packaging/shadow-agent.desktop @@ -13,4 +13,4 @@ Keywords=agent;llm;coding;shadowfetch; StartupNotify=true StartupWMClass=shadowcode X-GNOME-UsesNotifications=true -X-ShadowCode-Version=2.0.0 +X-ShadowCode-Version=2.1.0 diff --git a/release/trust/APT-SIGNING-KEY b/release/trust/APT-SIGNING-KEY new file mode 100644 index 00000000..db3a42ae --- /dev/null +++ b/release/trust/APT-SIGNING-KEY @@ -0,0 +1,39 @@ +ShadowCode APT repository signing key + +Fingerprint: 1BC07B5ECFDB514673A296FB021CAB3F2F7A1F61 + (1BC0 7B5E CFDB 5146 73A2 96FB 021C AB3F 2F7A 1F61) + +Key: RSA 4096, created 2026-10-11, no expiry, signs the repository's InRelease +and Release.gpg only. Compare this fingerprint with the keyring you download +before trusting it; a change of key is announced in a reviewed commit here. + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGrLlrwBEADZ0SDyrw48pu8+LWpJ43J61zLmk+0u1Qr7IaXDowDcEzId2Bov ++YiesDyAIdaI/K2ouvdCKw4M97do8y6VS0jS6VTY0q3bYbQTjIybkLPCJwON2EXi +UW8IFg0r13wPBqB9W9Nxpnq+Jbsu2tv4fFj/05lHmli+bO3V8KoMt9LwkudGdHfF +j8a+u3HRRumQMJlLbgnQDxo/brZyiqkvQY2gv0fYl1vCSH+zKyoah9VWXyx/M/3c +8MnckFRQQn2SXRYtHkr1plfeZjh7EN5FdAErTVC8CuEtIiplcdfauw08wrmedvoH +AoGbCblgI8kk/sb+WW74sqba9t8MPF1TSatVDBkMAqAYwZhju88KbLZAfZum2Uz9 +FUybeTlf4P8ZlOysiGg6u7x0f38gnk1IaizclBnQ3t9TP5FhSzRCn/2A6c7ybpdU +MqFNur5Ed8gazolncKKHE47epgb+bgb8H8D5k5nDfcqp0tyyv2OKLy2ZBwJY+pWQ +u4THhO1kexVRFNBtK5x7M0Bj9gC2sMJscCKTKT29Xp88zCNMK90FBC282yAFPBQf +YonY0oWzV3609E7Iw0t5iMTJ5WbclPxcGxbLhUvtwTd4enUHEss4DqOgrzcnrbUN +wOGMze8ioqGcHn5e4ytAiMhY64yJ5VzontiSq/3i7nQaIQQTeOYkfCGRwQARAQAB +tE5TaGFkb3dDb2RlIEFQVCByZXBvc2l0b3J5IDwyMDk0NTcxMDMrU2hhZG93ZmV0 +Y2hhcHBzQHVzZXJzLm5vcmVwbHkuZ2l0aHViLmNvbT6JAlEEEwEKADsWIQQbwHte +z9tRRnOilvsCHKs/L3ofYQUCasuWvAIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIe +BwIXgAAKCRACHKs/L3ofYbT+EACOyd7OxXC/HS4W3ARWUn+tXmKYtwkTimyb0CMM +rughf/AWBt/YGZjkgSBnFsr0/Vm9huhLO514TEcFcYY6bi1K1Aburdf/ORwihaZL +m63Bg6pwiGm8+8INBy53B2sXzWqgSKt3f9uapQJK4bETuFz1luiC6uTPyaDkJGKG +rP2NoAFmJbpfPPzKH6AkM7gBxwuwcAyOnWo1tm0DPYqbJUxJsqnPvC/XikISAR/J +chNCGoJ9Rq1i0TqhJ7epj1pt523XHx7t2zKmhBCxXcF1R9QGTW7HAvoGHoyCgQZ7 +ZR9FwGoplo/FJfW+iljLEh/cyqNxnylf/P5S3+4MP3w0tOR29dCAJW5akiPYWFpq +ugO3IuttuZ30L8HFcrC8P8cTBUreuVvkGVWDQDByuVe0eHgekhEOtISKik3kwaU8 +eJI/ckrV+pnwJz/YkRWdH7uxFd17SnSpKXeXqUUEky13eHvz5cjA/aQIpeIZ9UZ+ +LxChhcKY9expfSAUTqfMAVoE8VyG2fw1YRWt7zzhzo+V9xrOARHRs8peQNRSPRAo +6zu31mqeKUCgMIPseFXgXd1ZlxdHbSWTQc91t9nFaKzUdWA6C9jd9zm5gPWKGO7r +uTwyFc7xr20QKpFrLXoBD4vtLfO3Sin4w04keUFZtxGV+XidiO3gpWrm7hkD7Y1b +HV7E5g== +=XvBJ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/release/trust/policy b/release/trust/policy index a18804e5..cf8feed2 100644 --- a/release/trust/policy +++ b/release/trust/policy @@ -2,9 +2,9 @@ ShadowCode-Release-Trust-v1 repository=Shadowfetchapps/ShadowCode repository-id=1377099349 owner-id=209457103 -target=x86_64-unknown-linux-gnu +target=x86_64-unknown-linux-gnu,aarch64-unknown-linux-gnu channel=stable minimum-epoch=1 minimum-version=0.33.0 keys=ed25519-spki-sha256 -key=1 f0c60ff8228314616985712f8918c9798c0ecf72b7271e19343a041c25663b7f 0.33.0 1.0.1 +key=1 f0c60ff8228314616985712f8918c9798c0ecf72b7271e19343a041c25663b7f 0.33.0 2.1.0 diff --git a/scripts/build-apt-repo.mjs b/scripts/build-apt-repo.mjs new file mode 100644 index 00000000..f955b33c --- /dev/null +++ b/scripts/build-apt-repo.mjs @@ -0,0 +1,238 @@ +#!/usr/bin/env node +// Builds a signed, flat-pool APT repository from release .debs that were +// already authenticated with scripts/verify-native-release.sh (or an +// equivalent trusted verification). It does not verify publisher signatures +// itself and must never be fed a candidate download directly. +// +// node scripts/build-apt-repo.mjs --out DIR --fingerprint FPR \ +// [--previous DIR] [--valid-days N] [--date RFC2822] DEB... +// +// The OpenPGP signing key is read from APT_REPO_SIGNING_KEY (an ASCII-armored +// secret key, provided by a protected CI secret) and imported into a private, +// temporary GNUPGHOME that is deleted afterwards. The imported key must have +// exactly the primary fingerprint given by --fingerprint, so a swapped secret +// cannot silently re-root the repository. No key is ever written to --out +// except the public key (shadowcode-archive-keyring.gpg/.asc). +// +// Layout (suite "stable", component "main", architectures amd64 arm64): +// pool/main/s/shadow-code/.deb +// dists/stable/{Release,InRelease,Release.gpg} +// dists/stable/main/binary-{amd64,arm64}/{Packages,Packages.gz,Release} +// --previous keeps .debs already in an earlier published pool, so older +// versions stay installable; an identical name with different bytes fails. +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + copyFile, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { gzipSync } from "node:zlib"; + +const run = promisify(execFile); + +export const SUITE = "stable"; +export const COMPONENT = "main"; +export const ARCHITECTURES = ["amd64", "arm64"]; +export const PACKAGE = "shadow-code"; +export const ORIGIN = "ShadowCode"; +export const LABEL = "ShadowCode"; +export const POOL = `pool/${COMPONENT}/s/${PACKAGE}`; +export const KEYRING_NAME = "shadowcode-archive-keyring"; +const MAX_DEB = 32 * 1024 ** 3; + +const sha = (algo, data) => createHash(algo).update(data).digest("hex"); + +function fail(message) { + throw new Error(`build-apt-repo: ${message}`); +} + +/** Reads the control fields APT needs from a .deb with dpkg-deb. */ +export async function readControl(deb) { + const { stdout } = await run("dpkg-deb", ["-f", deb], { maxBuffer: 1 << 20 }); + return stdout.replace(/\n+$/, ""); +} + +function field(control, name) { + const m = control.match(new RegExp(`^${name}:[ \\t]*(.*)$`, "mi")); + return m ? m[1].trim() : undefined; +} + +/** Validates one package and returns its index stanza data. */ +export async function describeDeb(deb) { + const info = await stat(deb); + if (!info.isFile() || info.size === 0 || info.size > MAX_DEB) fail(`${deb}: not a bounded regular file`); + const control = await readControl(deb); + const pkg = field(control, "Package"); + const version = field(control, "Version"); + const arch = field(control, "Architecture"); + if (pkg !== PACKAGE) fail(`${deb}: package ${pkg ?? "(none)"} is not ${PACKAGE}`); + if (!version || !/^[0-9][A-Za-z0-9.+~-]*$/.test(version)) fail(`${deb}: bad version ${version}`); + if (!ARCHITECTURES.includes(arch)) fail(`${deb}: architecture ${arch} is not one of ${ARCHITECTURES.join(", ")}`); + if (/^(Filename|Size|MD5sum|SHA1|SHA256|SHA512):/im.test(control)) fail(`${deb}: control carries index fields`); + const bytes = await readFile(deb); + const name = `${pkg}_${version.replace(/^[0-9]+:/, "")}_${arch}.deb`; + return { deb, control, pkg, version, arch, name, size: bytes.length, sha256: sha("sha256", bytes), md5: sha("md5", bytes), bytes }; +} + +async function listPool(dir) { + try { + return (await readdir(dir)).filter((n) => n.endsWith(".deb")).map((n) => path.join(dir, n)); + } catch (error) { + if (error.code === "ENOENT") return []; + throw error; + } +} + +function stanza(d) { + return [ + d.control, + `Filename: ${POOL}/${d.name}`, + `Size: ${d.size}`, + `MD5sum: ${d.md5}`, + `SHA256: ${d.sha256}`, + ].join("\n"); +} + +async function gpg(home, args, input) { + const child = run("gpg", ["--homedir", home, "--batch", "--no-tty", "--pinentry-mode", "loopback", ...args], { maxBuffer: 1 << 24 }); + if (input !== undefined) { + child.child.stdin.end(input); + } + return child; +} + +/** Imports the armored secret key and checks it is exactly `fingerprint`. */ +async function importKey(home, armored, fingerprint) { + await gpg(home, ["--import"], armored); + const { stdout } = await gpg(home, ["--with-colons", "--list-secret-keys"]); + const fprs = []; + let primary = false; + for (const line of stdout.split("\n")) { + const f = line.split(":"); + if (f[0] === "sec") { primary = true; continue; } + if (f[0] === "ssb") { primary = false; continue; } + if (f[0] === "fpr" && primary) fprs.push(f[9]); + } + if (fprs.length !== 1) fail(`signing key must contain exactly one primary secret key, found ${fprs.length}`); + if (fprs[0] !== fingerprint) fail(`signing key fingerprint ${fprs[0]} does not match the pinned ${fingerprint}`); +} + +export async function buildRepository({ debs, out, fingerprint, armoredKey, previous, validDays, date = new Date() }) { + if (!/^[0-9A-F]{40}$/.test(fingerprint ?? "")) fail("--fingerprint must be a 40-digit uppercase OpenPGP v4 fingerprint"); + if (!armoredKey || !/-----BEGIN PGP PRIVATE KEY BLOCK-----/.test(armoredKey)) fail("APT_REPO_SIGNING_KEY must hold an ASCII-armored secret key"); + if (!debs.length) fail("no .deb given"); + + const entries = new Map(); + const add = (d, origin) => { + const prior = entries.get(d.name); + if (prior && prior.sha256 !== d.sha256) fail(`${d.name}: ${origin} differs from an existing package with the same name`); + entries.set(d.name, d); + }; + if (previous) for (const deb of await listPool(path.join(previous, POOL))) add(await describeDeb(deb), "previous pool"); + for (const deb of debs) add(await describeDeb(deb), "new package"); + + await rm(out, { recursive: true, force: true }); + const pool = path.join(out, POOL); + await mkdir(pool, { recursive: true }); + for (const d of entries.values()) await writeFile(path.join(pool, d.name), d.bytes); + + const dist = path.join(out, "dists", SUITE); + const indexFiles = []; + for (const arch of ARCHITECTURES) { + const rel = `${COMPONENT}/binary-${arch}`; + const dir = path.join(dist, rel); + await mkdir(dir, { recursive: true }); + const items = [...entries.values()].filter((d) => d.arch === arch).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + const packages = Buffer.from(items.map(stanza).join("\n\n") + (items.length ? "\n" : "")); + const archRelease = Buffer.from(`Archive: ${SUITE}\nSuite: ${SUITE}\nComponent: ${COMPONENT}\nOrigin: ${ORIGIN}\nLabel: ${LABEL}\nArchitecture: ${arch}\n`); + for (const [name, data] of [["Packages", packages], ["Packages.gz", gzipSync(packages, { level: 9 })], ["Release", archRelease]]) { + await writeFile(path.join(dir, name), data); + indexFiles.push({ name: `${rel}/${name}`, data }); + } + } + + const rfc2822 = (d) => d.toUTCString().replace("GMT", "UTC"); + const lines = [ + `Origin: ${ORIGIN}`, + `Label: ${LABEL}`, + `Suite: ${SUITE}`, + `Codename: ${SUITE}`, + `Date: ${rfc2822(date)}`, + ]; + if (validDays) lines.push(`Valid-Until: ${rfc2822(new Date(date.getTime() + validDays * 86400000))}`); + lines.push( + `Architectures: ${ARCHITECTURES.join(" ")}`, + `Components: ${COMPONENT}`, + "Acquire-By-Hash: no", + `Description: ShadowCode release packages`, + "MD5Sum:", + ...indexFiles.map((f) => ` ${sha("md5", f.data)} ${String(f.data.length).padStart(10)} ${f.name}`), + "SHA256:", + ...indexFiles.map((f) => ` ${sha("sha256", f.data)} ${String(f.data.length).padStart(10)} ${f.name}`), + ); + const release = lines.join("\n") + "\n"; + await writeFile(path.join(dist, "Release"), release); + + const home = await mkdtemp(path.join(tmpdir(), "apt-repo-gnupg-")); + try { + await importKey(home, armoredKey, fingerprint); + const signer = ["--local-user", `${fingerprint}!`, "--digest-algo", "SHA512"]; + await gpg(home, [...signer, "--clearsign", "--output", path.join(dist, "InRelease"), path.join(dist, "Release")]); + await gpg(home, [...signer, "--armor", "--detach-sign", "--output", path.join(dist, "Release.gpg"), path.join(dist, "Release")]); + await gpg(home, ["--export", "--output", path.join(out, `${KEYRING_NAME}.gpg`), fingerprint]); + await gpg(home, ["--export", "--armor", "--output", path.join(out, `${KEYRING_NAME}.asc`), fingerprint]); + } finally { + await rm(home, { recursive: true, force: true }); + } + await writeFile(path.join(out, "shadowcode.sources"), sourcesFile()); + return { packages: [...entries.keys()].sort(), release }; +} + +/** The deb822 source users place in /etc/apt/sources.list.d/. */ +export function sourcesFile(uri = "https://shadowfetchstudio.github.io/ShadowCode/apt") { + return [ + "Types: deb", + `URIs: ${uri}`, + `Suites: ${SUITE}`, + `Components: ${COMPONENT}`, + `Architectures: ${ARCHITECTURES.join(" ")}`, + `Signed-By: /etc/apt/keyrings/${KEYRING_NAME}.gpg`, + "", + ].join("\n"); +} + +async function main(argv) { + const opts = { debs: [] }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--out") opts.out = argv[++i]; + else if (a === "--fingerprint") opts.fingerprint = argv[++i]; + else if (a === "--previous") opts.previous = argv[++i]; + else if (a === "--valid-days") opts.validDays = Number(argv[++i]); + else if (a === "--date") opts.date = new Date(argv[++i]); + else if (a.startsWith("--")) fail(`unknown option ${a}`); + else opts.debs.push(a); + } + if (!opts.out) fail("--out is required"); + if (opts.validDays !== undefined && !(Number.isInteger(opts.validDays) && opts.validDays > 0)) fail("--valid-days must be a positive integer"); + opts.armoredKey = process.env.APT_REPO_SIGNING_KEY; + const result = await buildRepository(opts); + process.stdout.write(`APT repository in ${opts.out}:\n${result.packages.map((p) => ` ${p}\n`).join("")}`); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)).catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); + }); +} diff --git a/scripts/build-flatpak.py b/scripts/build-flatpak.py index db3a7e07..a7cac2bc 100644 --- a/scripts/build-flatpak.py +++ b/scripts/build-flatpak.py @@ -9,6 +9,7 @@ import hashlib import json import os +import platform from pathlib import Path import shutil import subprocess @@ -16,6 +17,15 @@ ROOT = Path(__file__).resolve().parent.parent APP_ID = "com.shadowfetch.shadowcode" +# Flatpak architecture names; amd64/arm64 are accepted as aliases. +ARCHES = {"x86_64": "x86_64", "amd64": "x86_64", "aarch64": "aarch64", "arm64": "aarch64"} + + +def host_arch(): + machine = platform.machine() + if machine not in ARCHES: + raise SystemExit(f"Unsupported build host architecture: {machine}") + return ARCHES[machine] def run(args, **kwargs): @@ -52,7 +62,12 @@ def main(): parser.add_argument("--prepare-only", action="store_true") parser.add_argument("--install", action="store_true") parser.add_argument("--skip-fetch", action="store_true", help="Reuse already prepared locked dependency caches") + parser.add_argument("--arch", choices=sorted(ARCHES), help="Flatpak architecture (default: the build host's; x86_64 or aarch64)") + parser.add_argument("--state-dir", help="flatpak-builder state/cache directory (default: target/flatpak/state-v2-final)") + parser.add_argument("--install-deps-from", metavar="REMOTE", help="Let flatpak-builder install the GNOME SDK and extensions from REMOTE (e.g. flathub)") + parser.add_argument("--cpu-only-llama", action="store_true", help="Build the bundled llama.cpp without the Vulkan module (faster CI builds; not for releases)") args = parser.parse_args() + arch = ARCHES[args.arch] if args.arch else host_arch() for tool in ["flatpak", "flatpak-builder", "cargo", "npm", "git"]: if shutil.which(tool) is None: parser.error(f"Missing build tool: {tool}") @@ -124,14 +139,20 @@ def main(): commit = run(["git", "rev-parse", "HEAD"], stdout=subprocess.PIPE).stdout.decode().strip() dirty = run(["git", "status", "--porcelain", "--untracked-files=no"], stdout=subprocess.PIPE).stdout.strip() manifest["build-options"]["env"]["SHADOWCODE_COMMIT"] = commit + ("-dirty" if dirty else "") + if args.cpu_only_llama: + manifest["build-options"]["env"]["SHADOWCODE_LLAMA_VULKAN"] = "0" generated = work / f"{APP_ID}.json" generated.write_text(json.dumps(manifest, indent=2) + "\n") - (work / "SOURCE-INPUTS.json").write_text(json.dumps({"version": version, "commit": commit, "dirty": bool(dirty), "sources": sources, "runtime": "org.gnome.Platform//50"}, indent=2) + "\n") + (work / "SOURCE-INPUTS.json").write_text(json.dumps({"version": version, "commit": commit, "dirty": bool(dirty), "sources": sources, "runtime": "org.gnome.Platform//50", "arch": arch}, indent=2) + "\n") if args.prepare_only: return - run(["flatpak-builder", "--user", "--disable-rofiles-fuse", "--keep-build-dirs", "--force-clean", "--repo=" + str(work / "repo"), "--state-dir=" + str(work / "state-v2-final"), work / "build", generated]) - bundle = work / f"ShadowCode_{version}_x86_64.flatpak" - run(["flatpak", "build-bundle", "--runtime-repo=https://flathub.org/repo/flathub.flatpakrepo", work / "repo", bundle, APP_ID]) + state_dir = Path(args.state_dir).expanduser().resolve() if args.state_dir else work / "state-v2-final" + builder = ["flatpak-builder", "--user", "--disable-rofiles-fuse", "--keep-build-dirs", "--force-clean", "--arch=" + arch, "--repo=" + str(work / "repo"), "--state-dir=" + str(state_dir)] + if args.install_deps_from: + builder.append("--install-deps-from=" + args.install_deps_from) + run([*builder, work / "build", generated]) + bundle = work / f"ShadowCode_{version}_{arch}.flatpak" + run(["flatpak", "build-bundle", "--arch=" + arch, "--runtime-repo=https://flathub.org/repo/flathub.flatpakrepo", work / "repo", bundle, APP_ID]) (work / "SHA256SUMS").write_text(hashlib.sha256(bundle.read_bytes()).hexdigest() + " " + bundle.name + "\n") if args.install: run(["flatpak", "install", "--user", "--noninteractive", bundle]) diff --git a/scripts/build-llama.cpp.sh b/scripts/build-llama.cpp.sh index 8199711f..b8a9754c 100755 --- a/scripts/build-llama.cpp.sh +++ b/scripts/build-llama.cpp.sh @@ -211,7 +211,28 @@ if [[ "$WANT_VULKAN" == "1" ]]; then BACKEND="vulkan+cpu" fi -"$CMAKE" -S "$SRC" -B "$BUILD_DIR" "${GENERATOR[@]}" \ +# On aarch64, GGML_CPU_ALL_VARIANTS includes an Armv9.2 SME variant +# (-march=armv9.2-a+…+sme), which needs GCC 14 or newer; Ubuntu 24.04's default +# compiler is GCC 13. Use gcc-14/g++-14 when the default cannot target it. +# SHADOWCODE_CC and SHADOWCODE_CXX choose the compilers explicitly. +COMPILER_FLAGS=() +if [[ "$(uname -m)" == aarch64 ]]; then + targets_sme() { printf 'int shadowcode_probe;\n' | "$1" -x "$2" -march=armv9.2-a+sme -c -o /dev/null - >/dev/null 2>&1; } + LLAMA_CC="${SHADOWCODE_CC:-${CC:-cc}}" + LLAMA_CXX="${SHADOWCODE_CXX:-${CXX:-c++}}" + if ! { targets_sme "$LLAMA_CC" c && targets_sme "$LLAMA_CXX" c++; }; then + if [[ -z "${SHADOWCODE_CC:-}${SHADOWCODE_CXX:-}" ]] && command -v gcc-14 >/dev/null 2>&1 && command -v g++-14 >/dev/null 2>&1 \ + && targets_sme gcc-14 c && targets_sme g++-14 c++; then + LLAMA_CC=gcc-14 LLAMA_CXX=g++-14 + else + echo "llama.cpp's aarch64 CPU variants need a C/C++ compiler that targets armv9.2-a+sme (GCC 14 or newer): install gcc-14 and g++-14, or set SHADOWCODE_CC and SHADOWCODE_CXX." >&2 + exit 1 + fi + fi + COMPILER_FLAGS=(-DCMAKE_C_COMPILER="$LLAMA_CC" -DCMAKE_CXX_COMPILER="$LLAMA_CXX") +fi + +"$CMAKE" -S "$SRC" -B "$BUILD_DIR" "${GENERATOR[@]}" "${COMPILER_FLAGS[@]}" \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON \ -DGGML_BACKEND_DL=ON \ diff --git a/scripts/build-native.mjs b/scripts/build-native.mjs index 8e33d96d..aac4ef0d 100644 --- a/scripts/build-native.mjs +++ b/scripts/build-native.mjs @@ -30,6 +30,7 @@ import { normalizeDesktopEntry, } from "./native-desktop-metadata.mjs"; import { finishDebianPackage } from "./native-deb.mjs"; +import { hostArch, packageNames } from "./native-arch.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); const packagingPath = applyPackagingPath(root); console.log(`Using sanitized packaging PATH: ${packagingPath}`); @@ -51,11 +52,15 @@ async function command(binary, args, env = {}) { }); } const run = (args, env = {}) => command(process.execPath, [cli, ...args], env); -if (process.platform !== "linux" || process.arch !== "x64") - throw new Error("This packaging workflow currently supports Linux x86_64"); +if (process.platform !== "linux") + throw new Error("This packaging workflow supports Linux x86_64 and aarch64"); +// Packages are built natively on each architecture (no cross packaging). +const arch = hostArch(); const version = JSON.parse( await readFile(path.join(root, "src-tauri/tauri.conf.json"), "utf8"), ).version; +const names = packageNames(version, arch.arch); +console.log(`Packaging for ${arch.arch} (${arch.target})`); // Settings › About shows the commit. A distribution building from a source // archive without Git history may set SHADOWCODE_COMMIT itself. if (!process.env.SHADOWCODE_COMMIT) { @@ -85,7 +90,7 @@ console.log( `Bundling managed llama.cpp ${llama.commit} (${llama.backend}) as /${RUNTIME_LOCATION}`, ); const notices = path.join(root, "target/native-notices"); -const nativeRuntime = await buildRuntime(); +const nativeRuntime = await buildRuntime(arch.arch); await applicationNotices(notices, llama); const files = { "/usr/share/doc/shadowcode/notices": notices, @@ -145,7 +150,7 @@ try { await addRuntimeToDeb( path.join( root, - `target/release/bundle/deb/ShadowCode_${version}_amd64.deb`, + `target/release/bundle/deb/${names.deb}`, ), llama, scratch, @@ -165,7 +170,7 @@ try { if (format === "appimage") { const appimage = path.join( root, - `target/release/bundle/appimage/ShadowCode_${version}_amd64.AppImage`, + `target/release/bundle/appimage/${names.appimage}`, ); const appdir = path.join( root, @@ -196,14 +201,14 @@ try { { APPIMAGE_EXTRACT_AND_RUN: "1", OUTPUT: repacked, - ARCH: "x86_64", + ARCH: arch.arch, LDAI_RUNTIME_FILE: nativeRuntime.runtime, }, ); await rename(repacked, appimage); const sources = path.join( root, - `target/release/bundle/appimage/ShadowCode_${version}_appimage-runtime-sources.tar.gz`, + `target/release/bundle/appimage/${names.runtimeSources}`, ); const pendingSources = path.join(scratch, path.basename(sources)); await command("tar", [ @@ -218,7 +223,7 @@ try { "build-packages.txt", "compiler.txt", "runtime-dynamic.txt", - "runtime-x86_64.debug", + `runtime-${arch.arch}.debug`, ]); await rename(pendingSources, sources); } diff --git a/scripts/check-native-package.mjs b/scripts/check-native-package.mjs index 8f323995..336d5c7f 100644 --- a/scripts/check-native-package.mjs +++ b/scripts/check-native-package.mjs @@ -21,6 +21,7 @@ import { verifyRuntimeDirectory, } from "./llama-runtime.mjs"; import { DESKTOP_FILE, METAINFO_FILE } from "./native-desktop-metadata.mjs"; +import { hostArch, packageNames } from "./native-arch.mjs"; import { lintDebianPackage, verifyDebianPackage } from "./native-deb.mjs"; const run = promisify(execFile); const [appimagePath, debPath] = process.argv @@ -254,7 +255,7 @@ try { ); const sourcesPath = path.join( path.dirname(appimagePath), - `ShadowCode_${version.split(" ")[1]}_appimage-runtime-sources.tar.gz`, + packageNames(version.split(" ")[1], hostArch().arch).runtimeSources, ); const sources = path.join(scratch, "runtime-sources"); await mkdir(sources); diff --git a/scripts/clean-host-runtime.Dockerfile b/scripts/clean-host-runtime.Dockerfile index 49189de8..66ea65c1 100644 --- a/scripts/clean-host-runtime.Dockerfile +++ b/scripts/clean-host-runtime.Dockerfile @@ -1,7 +1,16 @@ # Debian qualification image: application runtime libraries plus a virtual # display harness. The app must not rely on the repository, Node, Cargo, or # the CI runner's development packages. -FROM --platform=linux/amd64 debian@sha256:38a76d01668772e381ad2826d876627c89e7133e2f8a0f5d567306798b0f2a16 +# debian:trixie-20260803-slim, pinned per architecture. BuildKit only pulls the +# stage for the target platform (the host's unless --platform is given). +# TARGETARCH has no default: a declared default would win over the value +# BuildKit sets, selecting the amd64 stage on an arm64 host. The build +# commands also pass the host's architecture, because Docker's classic +# builder sets no TARGETARCH at all. +ARG TARGETARCH +FROM --platform=linux/amd64 debian@sha256:38a76d01668772e381ad2826d876627c89e7133e2f8a0f5d567306798b0f2a16 AS debian-amd64 +FROM --platform=linux/arm64 debian@sha256:c94f5ddd41327aa2d4a7cfba7889056c02936182fd76a513fec6160c97181fc0 AS debian-arm64 +FROM debian-${TARGETARCH} RUN apt-get update -qq \ && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends \ git libgomp1 libssl3t64 libasound2t64 libwebkit2gtk-4.1-0 libgtk-3-0 \ diff --git a/scripts/install-appimage.sh b/scripts/install-appimage.sh index 705dc7fd..cd0a2bf6 100755 --- a/scripts/install-appimage.sh +++ b/scripts/install-appimage.sh @@ -20,12 +20,16 @@ # recovered on retry or with --recover. An activation-started crash is also # recoverable while every recorded launcher/desktop integration is unchanged; # ambiguous or changed state is preserved for review. -# * Settings and task history are never touched. Previous ShadowCode +# * With a display, the profile database is snapshotted (VACUUM INTO) before +# activation and the new version must report its first window in time; +# otherwise it is stopped and AppImage, runtime, integrations and the +# database snapshot are restored (also by --recover after a crash). +# * Settings and task history are otherwise never touched. Previous ShadowCode # AppImages are removed only after success. set -euo pipefail umask 077 usage() { - echo 'Usage: install-appimage.sh /path/to/ShadowCode_VERSION_amd64.AppImage | --recover' >&2 + echo 'Usage: install-appimage.sh /path/to/ShadowCode_VERSION_amd64.AppImage (x86_64) | ShadowCode_VERSION_aarch64.AppImage (aarch64) | --recover' >&2 exit 2 } fail() { echo "$*" >&2; exit 1; } @@ -46,6 +50,14 @@ else [[ -n "$SOURCE_ARG" ]] || usage fi +# The host selects the only installable AppImage role and its signed target. +# x86_64 keeps its historical names (download _amd64, installed -x86_64). +case "$(uname -m)" in + x86_64) HOST_TARGET=x86_64-unknown-linux-gnu PACKAGE_ARCH=amd64 INSTALLED_ARCH=x86_64 ;; + aarch64|arm64) HOST_TARGET=aarch64-unknown-linux-gnu PACKAGE_ARCH=aarch64 INSTALLED_ARCH=aarch64 ;; + *) fail "ShadowCode AppImages are published for x86_64 and aarch64, not $(uname -m)." ;; +esac + [[ "$UNVERIFIED" == 0 ]] || fail '--unverified is not supported by this signed-release installer. Publisher signatures are required; unsigned development installation needs a separate isolated development path.' APPS="$HOME/Applications" @@ -57,6 +69,17 @@ JOURNAL="$(dirname "$LIB")/.shadowcode-install-intent" # A convenience symlink next to downloads must not choose a substitute bundle. INSTALLER_FILE="$(realpath -- "${BASH_SOURCE[0]}")" ROOT="$(cd "$(dirname "$INSTALLER_FILE")/.." && pwd)" +# The profile database, as the app finds it (an XDG base must be absolute). +STATE_HOME="${XDG_STATE_HOME:-}" +[[ "$STATE_HOME" == /* ]] || STATE_HOME="$HOME/.local/state" +PROFILE_STATE="$STATE_HOME/shadow-agent" +DATABASE="$PROFILE_STATE/shadow-agent.db" +# First-window check after activation: auto (when a display is available), +# require, or skip. The new version must report its first start in time. +FIRST_START_CHECK="${SHADOWCODE_FIRST_START_CHECK:-auto}" +FIRST_START_TIMEOUT="${SHADOWCODE_FIRST_START_TIMEOUT:-120}" +[[ "$FIRST_START_CHECK" =~ ^(auto|require|skip)$ ]] || fail 'SHADOWCODE_FIRST_START_CHECK must be auto, require or skip.' +[[ "$FIRST_START_TIMEOUT" =~ ^[1-9][0-9]{0,3}$ ]] || fail 'SHADOWCODE_FIRST_START_TIMEOUT must be a number of seconds.' mkdir -p "$APPS" "$BIN" "$(dirname "$LIB")" "$DATA/applications" "$DATA/icons/hicolor/scalable/apps" # Keep one stable lock inode, including between failed attempts. Two installers @@ -103,6 +126,32 @@ runtime_hash() { fi } private_dir() { [[ -d "$1" && ! -L "$1" && "$(stat -c '%u:%a' -- "$1")" == "$(id -u):700" ]]; } +# No ShadowCode holds the profile lock (the engine flocks native.lock). +profile_idle() { + local lock="$PROFILE_STATE/native.lock" + ! exists "$lock" && return 0 + [[ -f "$lock" && ! -L "$lock" ]] || return 1 + flock -n -- "$lock" true +} +# Put back the pre-activation database snapshot (or the recorded absence of +# a database) once the new version has stopped. The log files go first: a +# snapshot must never be paired with another database's WAL. +restore_database() { + local snapshot=$1 expected=$2 + profile_idle || return 1 + if [[ "$expected" == - ]]; then + rm -f -- "$DATABASE-wal" "$DATABASE-shm" "$DATABASE" || return 1 + else + [[ "$(file_hash "$snapshot")" == "$expected" ]] || return 1 + rm -f -- "$DATABASE.rollback.pending" || return 1 + cp -- "$snapshot" "$DATABASE.rollback.pending" || return 1 + chmod 600 -- "$DATABASE.rollback.pending" || return 1 + [[ "$(file_hash "$DATABASE.rollback.pending")" == "$expected" ]] || return 1 + rm -f -- "$DATABASE-wal" "$DATABASE-shm" || return 1 + mv -Tf -- "$DATABASE.rollback.pending" "$DATABASE" || return 1 + fi + sync -f "$PROFILE_STATE" 2>/dev/null || true +} recovery_refusal() { fail "Interrupted install needs manual review: $*. Preserve $JOURNAL and $LIB.previous."; } read_field() { local field="$JOURNAL/$1" @@ -123,18 +172,33 @@ recover_install() { candidate="$(read_field candidate)"; candidate_hash="$(read_field candidate-sha256)" existed="$(read_field candidate-existed)"; new_hash="$(read_field new-runtime-sha256)" stage_name="$(read_field stage)" - [[ ( "$schema" == 1 || "$schema" == 2 || "$schema" == 3 ) && ( "$phase" == prepared || ( "$schema" == 3 && "$phase" == activation_started ) ) ]] || recovery_refusal 'unsupported intent or activation already started' + [[ ( "$schema" == 1 || "$schema" == 2 || "$schema" == 3 ) && ( "$phase" == prepared || ( "$schema" == 3 && ( "$phase" == activation_started || "$phase" == first_start_check ) ) ) ]] || recovery_refusal 'unsupported intent or activation already started' + # A first-window check may have started the new version, which may have + # migrated the database: its snapshot is restored with the package. + local restore_db=0 db_snapshot_hash=- first_start=0 candidate_fp + local -a integration_restores=() + if [[ "$phase" == first_start_check ]]; then + phase=activation_started + first_start=1 + if [[ "$(read_field database-check)" == 1 ]]; then + restore_db=1 + [[ "$(read_field database-path)" == "$DATABASE" ]] || recovery_refusal 'profile database location changed' + db_snapshot_hash="$(read_field database-snapshot-sha256)" + [[ "$db_snapshot_hash" == - || "$db_snapshot_hash" =~ ^[a-f0-9]{64}$ ]] || recovery_refusal 'invalid database snapshot identity' + profile_idle || recovery_refusal 'ShadowCode is running on this profile; close it, then retry' + fi + fi if [[ "$schema" == 3 ]]; then [[ "$(read_field bin-root)" == "$(realpath -- "$BIN")" && "$(read_field applications-root)" == "$(realpath -- "$DATA/applications")" && "$(read_field icons-root)" == "$(realpath -- "$DATA/icons/hicolor/scalable/apps")" ]] || recovery_refusal 'launcher or desktop installation root changed' fi if [[ "$schema" == 1 ]] && exists "$STATE"; then recovery_refusal 'legacy intent conflicts with authenticated state'; fi - [[ "$candidate" =~ ^ShadowCode-[0-9]+\.[0-9]+\.[0-9]+-x86_64\.AppImage$ && "$candidate_hash" =~ ^[a-f0-9]{64}$ && "$new_hash" =~ ^[a-f0-9]{64}$ ]] || recovery_refusal 'invalid candidate identity' + [[ "$candidate" =~ ^ShadowCode-[0-9]+\.[0-9]+\.[0-9]+-${INSTALLED_ARCH}\.AppImage$ && "$candidate_hash" =~ ^[a-f0-9]{64}$ && "$new_hash" =~ ^[a-f0-9]{64}$ ]] || recovery_refusal 'invalid candidate identity' [[ "$had" =~ ^[01]$ && "$existed" =~ ^[01]$ && "$stage_name" =~ ^\.shadowcode-install\.[A-Za-z0-9]{6}$ ]] || recovery_refusal 'invalid transaction identity' [[ ( "$had" == 1 && "$old_hash" =~ ^[a-f0-9]{64}$ ) || ( "$had" == 0 && "$old_hash" == - ) ]] || recovery_refusal 'invalid prior runtime identity' if [[ "$prior" == - ]]; then [[ "$prior_hash" == - ]] || recovery_refusal 'active AppImage changed' else - [[ "$prior" =~ ^ShadowCode-[0-9]+\.[0-9]+\.[0-9]+-x86_64\.AppImage$ && "$prior_hash" =~ ^[a-f0-9]{64}$ ]] || recovery_refusal 'invalid prior AppImage identity' + [[ "$prior" =~ ^ShadowCode-[0-9]+\.[0-9]+\.[0-9]+-${INSTALLED_ARCH}\.AppImage$ && "$prior_hash" =~ ^[a-f0-9]{64}$ ]] || recovery_refusal 'invalid prior AppImage identity' [[ "$(file_hash "$APPS/$prior")" == "$prior_hash" ]] || recovery_refusal 'prior AppImage changed' fi if exists "$APPS/ShadowCode.AppImage"; then @@ -194,12 +258,28 @@ recover_install() { expected="$(read_field "integration-$name")" [[ "$expected" == - || "$expected" =~ ^[a-f0-9]{64}$ ]] || recovery_refusal 'invalid integration identity' actual="$(integration_fingerprint "$location")" || recovery_refusal "$name integration cannot be identified" - [[ "$actual" == "$expected" ]] || recovery_refusal "$name integration changed after activation began" + if [[ "$first_start" == 1 ]]; then + # Activation completed before the check: each file is the recorded + # candidate (or already restored), and its prior copy is intact. + candidate_fp="$(read_field "integration-new-$name")" + [[ "$candidate_fp" =~ ^[a-f0-9]{64}$ ]] || recovery_refusal 'invalid integration identity' + [[ "$actual" == "$expected" || "$actual" == "$candidate_fp" ]] || recovery_refusal "$name integration changed after activation began" + if [[ "$expected" != - ]]; then + [[ "$(integration_fingerprint "$stage/integration-prior/$name")" == "$expected" ]] || recovery_refusal "$name prior integration copy changed or missing" + fi + ! exists "$location.rollback.pending" || recovery_refusal "$name integration rollback copy already exists" + [[ "$actual" == "$expected" ]] || integration_restores+=("$name") + else + [[ "$actual" == "$expected" ]] || recovery_refusal "$name integration changed after activation began" + fi done fi if [[ "$active_link_candidate" == 1 ]]; then [[ "$(file_hash "$APPS/$candidate")" == "$candidate_hash" ]] || recovery_refusal 'active candidate AppImage changed' fi + if [[ "$restore_db" == 1 && "$db_snapshot_hash" != - ]]; then + [[ "$(file_hash "$stage/database-snapshot.db")" == "$db_snapshot_hash" ]] || recovery_refusal 'database snapshot changed or missing' + fi if exists "$stage/recovery-link"; then [[ "$active_link_candidate" == 1 && "$prior" != - && -L "$stage/recovery-link" && "$(readlink -- "$stage/recovery-link")" == "$prior" ]] || recovery_refusal 'recovery link changed' fi @@ -223,6 +303,24 @@ recover_install() { [[ "$existed" == 1 ]] || rm -f -- "$APPS/$candidate" rm -f -- "$APPS/$candidate.pending" rm -f -- "$APPS/.ShadowCode.AppImage.pending" + for name in "${integration_restores[@]}"; do + case "$name" in + shadow) location="$BIN/shadow" ;; + shadowcode) location="$BIN/shadowcode" ;; + icon) location="$DATA/icons/hicolor/scalable/apps/shadow-agent.svg" ;; + desktop) location="$DATA/applications/shadow-agent.desktop" ;; + esac + expected="$(read_field "integration-$name")" + if [[ "$expected" == - ]]; then + rm -f -- "$location" || recovery_refusal "could not remove the $name integration" + else + cp -a --no-dereference -- "$stage/integration-prior/$name" "$location.rollback.pending" \ + && mv -Tf -- "$location.rollback.pending" "$location" || recovery_refusal "could not restore the $name integration" + fi + done + if [[ "$restore_db" == 1 ]]; then + restore_database "$stage/database-snapshot.db" "$db_snapshot_hash" || recovery_refusal 'could not restore the database snapshot' + fi sync -f "$(dirname "$LIB")" # Retire the active record by rename, so interruption of recursive staging # cleanup cannot leave a partially deleted active journal blocking recovery. @@ -246,7 +344,7 @@ DOWNLOAD="$(realpath -- "$SOURCE_ARG")" [[ -f "$DOWNLOAD" && ! -L "$DOWNLOAD" ]] || fail 'AppImage not found.' SOURCE_DIR="$(dirname "$DOWNLOAD")" SOURCE_NAME="$(basename "$DOWNLOAD")" -[[ "$SOURCE_NAME" =~ ^ShadowCode_[0-9]+\.[0-9]+\.[0-9]+_amd64\.AppImage$ ]] || fail 'Only the authenticated AppImage role can be installed by this script.' +[[ "$SOURCE_NAME" =~ ^ShadowCode_[0-9]+\.[0-9]+\.[0-9]+_${PACKAGE_ARCH}\.AppImage$ ]] || fail "Only the authenticated AppImage role can be installed by this script (ShadowCode_VERSION_${PACKAGE_ARCH}.AppImage on this $INSTALLED_ARCH machine)." STAGE="$(mktemp -d "$(dirname "$LIB")/.shadowcode-install.XXXXXX")" VERIFIED="$STAGE/verified-release" DEST='' @@ -259,6 +357,23 @@ LINK_REPLACING=0 DEST_CREATING=0 INTEGRATION_ACTIVE=0 PREVIOUS_LINK="$(readlink "$APPS/ShadowCode.AppImage" 2>/dev/null || true)" +APP_PID='' +DATABASE_ACTIVE=0 +DB_SNAPSHOT_HASH=- +# Stop the whole session of the started version (it runs under setsid). +stop_first_start() { + local tries=0 + kill -TERM -- "-$APP_PID" 2>/dev/null || true + while kill -0 -- "-$APP_PID" 2>/dev/null && (( tries < 100 )); do + sleep 0.1; tries=$((tries + 1)) + done + kill -KILL -- "-$APP_PID" 2>/dev/null || true + wait "$APP_PID" 2>/dev/null || true + tries=0 + while ! profile_idle && (( tries < 50 )); do sleep 0.1; tries=$((tries + 1)); done + APP_PID='' + profile_idle +} restore_integration() { local name=$1 destination=$2 prior=$3 candidate=$4 actual backup pending backup="$STAGE/integration-prior/$name" @@ -284,6 +399,10 @@ finish() { local restored=1 trap - EXIT INT TERM if [[ "$status" != 0 ]]; then + # A new version that did not report its first start is stopped before + # anything it uses is put back. + local app_stopped=1 + [[ -z "$APP_PID" ]] || stop_first_start || { restored=0; app_stopped=0; } if [[ "$RUNTIME_REPLACING" == 1 ]]; then if [[ -e "$LIB.previous" || -L "$LIB.previous" ]]; then if rm -rf -- "$LIB"; then @@ -312,6 +431,14 @@ finish() { restore_integration shadowcode "$BIN/shadowcode" "$SHADOWCODE_INTEGRATION" "$POST_SHADOWCODE_INTEGRATION" || restored=0 restore_integration shadow "$BIN/shadow" "$SHADOW_INTEGRATION" "$POST_SHADOW_INTEGRATION" || restored=0 fi + if [[ "$DATABASE_ACTIVE" == 1 ]]; then + if [[ "$app_stopped" == 1 ]] && restore_database "$STAGE/database-snapshot.db" "$DB_SNAPSHOT_HASH"; then + echo 'The database snapshot taken before activation was restored.' >&2 + else + restored=0 + echo "The database snapshot could not be restored; it is kept in $STAGE/database-snapshot.db." >&2 + fi + fi if [[ "$DEST_CREATING" == 1 && "$restored" == 1 ]]; then rm -f -- "$DEST" || restored=0 fi @@ -355,12 +482,14 @@ SOURCE="$VERIFIED/$SOURCE_NAME" ACTUAL="$(file_hash "$SOURCE")" || fail 'Verified AppImage disappeared.' VERSION="$(awk -F= '$1 == "version" {print $2}' "$VERIFIED/RELEASE-AUTH")" VERIFIED_COMMIT="$(awk -F= '$1 == "commit" {print $2}' "$VERIFIED/RELEASE-AUTH")" +VERIFIED_TARGET="$(awk -F= '$1 == "target" {print $2}' "$VERIFIED/RELEASE-AUTH")" +[[ "$VERIFIED_TARGET" == "$HOST_TARGET" ]] || fail "The signed release targets $VERIFIED_TARGET, not this $HOST_TARGET machine." version_compare "$VERSION" "$FIRST_AUTH_VERSION" (( CMP >= 0 )) || fail 'Signed candidate is before the first authenticated release boundary.' chmod 500 "$SOURCE" VERSION_LINE="$("$SOURCE" --appimage-extract-and-run --version)" || fail 'The AppImage did not start.' [[ "$VERSION_LINE" == "ShadowCode $VERSION" ]] || fail 'The AppImage version differs from its signed release identity.' -DEST="$APPS/ShadowCode-${VERSION}-x86_64.AppImage" +DEST="$APPS/ShadowCode-${VERSION}-${INSTALLED_ARCH}.AppImage" [[ ! -e "$APPS/ShadowCode.AppImage" || -L "$APPS/ShadowCode.AppImage" ]] || fail 'ShadowCode.AppImage is not a symlink; refusing to replace it.' DEST_EXISTED=0 if [[ -e "$DEST" || -L "$DEST" ]]; then @@ -395,7 +524,7 @@ RUNTIME_VERSION="$(cd / && env -u LD_LIBRARY_PATH "$NEW_RUNTIME/llama-server" -- # These are data files, never shell-sourced commands or arbitrary paths. PRIOR_HASH=- if [[ -n "$PREVIOUS_LINK" ]]; then - [[ "$PREVIOUS_LINK" =~ ^ShadowCode-[0-9]+\.[0-9]+\.[0-9]+-x86_64\.AppImage$ ]] \ + [[ "$PREVIOUS_LINK" =~ ^ShadowCode-[0-9]+\.[0-9]+\.[0-9]+-${INSTALLED_ARCH}\.AppImage$ ]] \ || fail 'Active AppImage link is not a versioned ShadowCode basename; preserve it and review before installing.' PRIOR_HASH="$(file_hash "$APPS/$PREVIOUS_LINK")" || fail 'Prior AppImage cannot be identified.' fi @@ -444,6 +573,30 @@ POST_SHADOW_INTEGRATION="$(integration_fingerprint "$STAGE/integration-new/shado POST_SHADOWCODE_INTEGRATION="$(integration_fingerprint "$STAGE/integration-new/shadowcode")" || fail 'Candidate launcher alias cannot be identified.' POST_ICON_INTEGRATION="$(integration_fingerprint "$STAGE/integration-new/icon")" || fail 'Candidate icon cannot be identified.' POST_DESKTOP_INTEGRATION="$(integration_fingerprint "$STAGE/integration-new/desktop")" || fail 'Candidate desktop entry cannot be identified.' +# First-window check and database snapshot. The check starts the new version +# on this profile, which may migrate its database; an older version refuses a +# newer database format, so rollback needs the pre-activation database. +CHECK_FIRST_START=0 +case "$FIRST_START_CHECK" in + require) CHECK_FIRST_START=1 ;; + auto) [[ -z "${DISPLAY:-}" && -z "${WAYLAND_DISPLAY:-}" ]] || CHECK_FIRST_START=1 ;; +esac +if [[ "$CHECK_FIRST_START" == 1 ]]; then + profile_idle || fail 'ShadowCode is running on this profile. Close it, then run the installer again.' + ! exists "$DATABASE.rollback.pending" || fail "A database rollback copy already exists: $DATABASE.rollback.pending" + # SQLite's VACUUM INTO through the verified candidate: a consistent copy, + # never a raw copy of a live WAL database, and the source is not migrated. + SNAPSHOT_RESULT="$(cd / && "$SOURCE" --appimage-extract-and-run install-snapshot "$STAGE/database-snapshot.db")" \ + || fail 'Could not snapshot the profile database; nothing was replaced.' + if [[ "$SNAPSHOT_RESULT" == absent ]]; then + ! exists "$STAGE/database-snapshot.db" || fail 'Unexpected database snapshot.' + else + [[ "$SNAPSHOT_RESULT" =~ ^[0-9]+$ ]] || fail 'Unexpected database snapshot result.' + chmod 600 "$STAGE/database-snapshot.db" + DB_SNAPSHOT_HASH="$(file_hash "$STAGE/database-snapshot.db")" || fail 'Database snapshot missing.' + fi + profile_idle || fail 'ShadowCode started during the database snapshot. Close it, then run the installer again.' +fi sync -f "$STAGE" [[ "$(file_hash "$SOURCE")" == "$ACTUAL" ]] || fail 'AppImage changed during validation.' prepare_accepted_record @@ -460,6 +613,10 @@ write_field integration-shadow "$SHADOW_INTEGRATION" write_field integration-shadowcode "$SHADOWCODE_INTEGRATION" write_field integration-icon "$ICON_INTEGRATION" write_field integration-desktop "$DESKTOP_INTEGRATION" +write_field integration-new-shadow "$POST_SHADOW_INTEGRATION" +write_field integration-new-shadowcode "$POST_SHADOWCODE_INTEGRATION" +write_field integration-new-icon "$POST_ICON_INTEGRATION" +write_field integration-new-desktop "$POST_DESKTOP_INTEGRATION" write_field prior-link "${PREVIOUS_LINK:--}" write_field prior-app-sha256 "$PRIOR_HASH" write_field had-runtime "$HAD_RUNTIME" @@ -473,6 +630,9 @@ write_field accepted-prior "$STATE_PRIOR" write_field accepted-candidate "$CANDIDATE_ID" write_field accepted-record-sha256 "$CANDIDATE_RECORD_HASH" write_field state-root "$STATE" +write_field database-check "$CHECK_FIRST_START" +write_field database-path "$DATABASE" +write_field database-snapshot-sha256 "$DB_SNAPSHOT_HASH" sync -f "$JOURNAL_TEMP" JOURNAL_CREATED=1 mv -T -- "$JOURNAL_TEMP" "$JOURNAL" @@ -516,6 +676,39 @@ mv -Tf -- "$DATA/applications/shadow-agent.desktop.pending" "$DATA/applications/ "$(integration_fingerprint "$DATA/icons/hicolor/scalable/apps/shadow-agent.svg")" == "$POST_ICON_INTEGRATION" && \ "$(integration_fingerprint "$DATA/applications/shadow-agent.desktop")" == "$POST_DESKTOP_INTEGRATION" ]] \ || fail 'Desktop integration changed during activation; rollback requires review if an unrelated edit occurred.' +if [[ "$CHECK_FIRST_START" == 1 ]]; then + # Recorded before the start: a crash from here on also restores the + # database snapshot on recovery. + printf 'first_start_check\n' > "$JOURNAL/phase.pending" + chmod 600 "$JOURNAL/phase.pending" + mv -f -- "$JOURNAL/phase.pending" "$JOURNAL/phase" + sync -f "$JOURNAL" + DATABASE_ACTIVE=1 + MARKER="$STAGE/first-start" + printf 'Starting ShadowCode %s to check its first window (up to %ss)...\n' "$VERSION" "$FIRST_START_TIMEOUT" + # Its own session, so a failed start is stopped with every child process. + # It must not inherit the installer lock: it outlives a successful install. + (exec {INSTALL_LOCK}>&- && cd "$HOME" && SHADOWCODE_FIRST_START_MARKER="$MARKER" exec setsid "$BIN/shadow" ui) \ + < /dev/null > "$STAGE/first-start.log" 2>&1 & + APP_PID=$! + deadline=$((SECONDS + FIRST_START_TIMEOUT)) + until [[ -f "$MARKER" ]]; do + if ! kill -0 "$APP_PID" 2>/dev/null; then + tail -n 20 "$STAGE/first-start.log" >&2 || true + fail "ShadowCode $VERSION exited before its first window opened; rolling back." + fi + if (( SECONDS >= deadline )); then + tail -n 20 "$STAGE/first-start.log" >&2 || true + fail "ShadowCode $VERSION did not report its first window within ${FIRST_START_TIMEOUT}s; rolling back." + fi + sleep 0.2 + done + [[ "$(cat -- "$MARKER")" == "ShadowCode $VERSION" ]] || fail 'The first-start report is not from the installed version; rolling back.' + # Healthy: leave the new version running for the user. + disown "$APP_PID" 2>/dev/null || true + APP_PID='' + DATABASE_ACTIVE=0 +fi # Success: drop the previous runtime and older AppImages. mv -T -- "$JOURNAL" "$STAGE/completed-intent" JOURNAL_CREATED=0 @@ -523,9 +716,14 @@ RUNTIME_REPLACING=0 LINK_REPLACING=0 DEST_CREATING=0 rm -rf -- "$LIB.previous" -for old in "$APPS"/ShadowCode-*-x86_64.AppImage; do +for old in "$APPS"/ShadowCode-*-"$INSTALLED_ARCH".AppImage; do [[ "$old" == "$DEST" || ! -f "$old" ]] || rm -- "$old" done command -v update-desktop-database >/dev/null && update-desktop-database "$DATA/applications" || true -printf 'Installed %s\nInstalled llama.cpp %s to %s\nConfig and task history are preserved. Restart the app to use the new release.\n' \ +printf 'Installed %s\nInstalled llama.cpp %s to %s\nConfig and task history are preserved.\n' \ "$DEST" "${RUNTIME_COMMIT:0:9}" "$LIB" +if [[ "$CHECK_FIRST_START" == 1 ]]; then + printf 'ShadowCode %s opened its first window and is running.\n' "$VERSION" +else + printf 'The first-window check was skipped (no display, or SHADOWCODE_FIRST_START_CHECK=skip). Restart the app to use the new release.\n' +fi diff --git a/scripts/install-release-state.sh b/scripts/install-release-state.sh index fd1ddd69..7cff09ae 100644 --- a/scripts/install-release-state.sh +++ b/scripts/install-release-state.sh @@ -71,7 +71,7 @@ check_legacy_identity() { if exists "$APPS/ShadowCode.AppImage"; then [[ -L "$APPS/ShadowCode.AppImage" ]] || fail 'Active AppImage is not a supported link.' previous=$(readlink -- "$APPS/ShadowCode.AppImage") - [[ "$previous" =~ ^ShadowCode-((0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8}))-x86_64\.AppImage$ ]] || fail 'Active AppImage has an unsupported identity.' + [[ "$previous" =~ ^ShadowCode-((0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8}))-${INSTALLED_ARCH}\.AppImage$ ]] || fail 'Active AppImage has an unsupported identity.' version=${BASH_REMATCH[1]} # This compares data only; no installed executable is queried. version_compare "$version" "$FIRST_AUTH_VERSION" @@ -142,7 +142,7 @@ validate_recovery_state() { result=$(validate_receipt "$directory" "$target") || recovery_refusal 'candidate receipt changed' [[ "$(runtime_hash "$directory")" == "$record_hash" ]] || recovery_refusal 'candidate receipt bytes changed' IFS=$'\t' read -r version _commit _artifact hash <<< "$result" - [[ "$candidate" == "ShadowCode-${version}-x86_64.AppImage" && "$candidate_hash" == "$hash" ]] || recovery_refusal 'candidate receipt does not bind AppImage' + [[ "$candidate" == "ShadowCode-${version}-${INSTALLED_ARCH}.AppImage" && "$candidate_hash" == "$hash" ]] || recovery_refusal 'candidate receipt does not bind AppImage' found=1 fi done diff --git a/scripts/installer-auth-fixtures.mjs b/scripts/installer-auth-fixtures.mjs index f4f969b0..1ee1d62f 100644 --- a/scripts/installer-auth-fixtures.mjs +++ b/scripts/installer-auth-fixtures.mjs @@ -2,10 +2,10 @@ import { generateKeyPairSync } from 'node:crypto'; import { mkdir, writeFile, readFile } from 'node:fs/promises'; import path from 'node:path'; -import { assetNames, createEnvelope, signEnvelope, publicKeyIdentity, digestBytes, IDENTITY } from './native-release-auth.mjs'; +import { assetNames, metadataNames, createEnvelope, signEnvelope, publicKeyIdentity, digestBytes, IDENTITY } from './native-release-auth.mjs'; export const COMMIT = 'c'.repeat(40); -export async function provision(root, privateFile) { +export async function provision(root, privateFile, targets = [IDENTITY.target]) { const pair = generateKeyPairSync('ed25519'); const publicPem = pair.publicKey.export({ format: 'pem', type: 'spki' }); const privatePem = pair.privateKey.export({ format: 'pem', type: 'pkcs8' }); @@ -13,24 +13,30 @@ export async function provision(root, privateFile) { await mkdir(trust, { recursive: true }); await writeFile(privateFile, privatePem, { mode: 0o600 }); await writeFile(path.join(trust, `${id}.pem`), publicPem); - await writeFile(path.join(trust, 'policy'), `ShadowCode-Release-Trust-v1\nrepository=${IDENTITY.repository}\nrepository-id=${IDENTITY.repositoryId}\nowner-id=${IDENTITY.ownerId}\ntarget=${IDENTITY.target}\nchannel=stable\nminimum-epoch=1\nminimum-version=0.28.0\nkeys=ed25519-spki-sha256\nkey=1\t${id}\t0.28.0\t999.999.999\n`); + await writeFile(path.join(trust, 'policy'), `ShadowCode-Release-Trust-v1\nrepository=${IDENTITY.repository}\nrepository-id=${IDENTITY.repositoryId}\nowner-id=${IDENTITY.ownerId}\ntarget=${targets.join(',')}\nchannel=stable\nminimum-epoch=1\nminimum-version=0.28.0\nkeys=ed25519-spki-sha256\nkey=1\t${id}\t0.28.0\t999.999.999\n`); await writeFile(path.join(root, 'release/install-policy'), 'ShadowCode-Install-Policy-v1\nfirst-authenticated-version=0.28.0\n'); } -export async function signFixture(file, privateFile) { - const version = /^ShadowCode_(\d+\.\d+\.\d+)_amd64\.AppImage$/.exec(path.basename(file))?.[1]; - if (!version) throw new Error('Invalid fixture filename'); +const FIXTURE_TARGETS = { amd64: 'x86_64-unknown-linux-gnu', aarch64: 'aarch64-unknown-linux-gnu' }; +// The AppImage name selects the signed target unless a test overrides it. +export async function signFixture(file, privateFile, target) { + const match = /^ShadowCode_(\d+\.\d+\.\d+)_(amd64|aarch64)\.AppImage$/.exec(path.basename(file)); + if (!match) throw new Error('Invalid fixture filename'); + const version = match[1]; + target ??= FIXTURE_TARGETS[match[2]]; const directory = path.dirname(file), assets = {}; - for (const [role, name] of assetNames(version)) { - const target = path.join(directory, name); - if (role !== 'appimage') await writeFile(target, `Fixture ${role} ${version}\n`); - assets[name] = digestBytes(await readFile(target)); + for (const [role, name] of assetNames(version, target)) { + const assetFile = path.join(directory, name); + if (role !== 'appimage') await writeFile(assetFile, `Fixture ${role} ${version}\n`); + assets[name] = digestBytes(await readFile(assetFile)); } - await writeFile(path.join(directory, 'SHA256SUMS'), Object.entries(assets).map(([name, hash]) => `${hash} ${name}\n`).join('')); - await writeFile(path.join(directory, 'RELEASE-MANIFEST.json'), `${JSON.stringify({ schema: 1, tag: `v${version}`, commit: COMMIT, target: IDENTITY.target, assets, fixture_only: true }, null, 2)}\n`); + // Release asset names: aarch64 metadata carries its target suffix. + const meta = metadataNames(target); + await writeFile(path.join(directory, meta.sums), Object.entries(assets).map(([name, hash]) => `${hash} ${name}\n`).join('')); + await writeFile(path.join(directory, meta.manifest), `${JSON.stringify({ schema: 1, tag: `v${version}`, commit: COMMIT, target, assets, fixture_only: true }, null, 2)}\n`); const privatePem = await readFile(privateFile); - const envelope = await createEnvelope({ bundleDir: directory, version, commit: COMMIT, keyId: publicKeyIdentity(privatePem), keyEpoch: 1 }); - await writeFile(path.join(directory, 'RELEASE-AUTH'), envelope); - await writeFile(path.join(directory, 'RELEASE-AUTH.sig'), signEnvelope(envelope, privatePem)); + const envelope = await createEnvelope({ bundleDir: directory, version, commit: COMMIT, keyId: publicKeyIdentity(privatePem), keyEpoch: 1, target }); + await writeFile(path.join(directory, meta.auth), envelope); + await writeFile(path.join(directory, meta.signature), signEnvelope(envelope, privatePem)); } -if (process.argv[2] === 'provision') await provision(process.argv[3], process.argv[4]); +if (process.argv[2] === 'provision') await provision(process.argv[3], process.argv[4], process.argv[5]?.split(',')); if (process.argv[2] === 'sign') await signFixture(process.argv[3], process.argv[4]); diff --git a/scripts/llama-runtime.mjs b/scripts/llama-runtime.mjs index bae8f009..30b44310 100644 --- a/scripts/llama-runtime.mjs +++ b/scripts/llama-runtime.mjs @@ -219,6 +219,7 @@ export async function copyManagedRuntime(runtime, destination) { export const ALLOWED_SYSTEM_LIBRARIES = new Set([ "linux-vdso.so.1", "ld-linux-x86-64.so.2", + "ld-linux-aarch64.so.1", "libc.so.6", "libm.so.6", "libdl.so.2", diff --git a/scripts/native-arch.mjs b/scripts/native-arch.mjs new file mode 100644 index 00000000..d5c9cd88 --- /dev/null +++ b/scripts/native-arch.mjs @@ -0,0 +1,65 @@ +// One table for every architecture-dependent packaging name. x86_64 values are +// the historical names and must never change; aarch64 follows Tauri's bundler +// (AppImage `_aarch64`, Debian `_arm64`) and Debian/Flatpak conventions. +import assert from "node:assert/strict"; + +export const ARCHES = Object.freeze({ + x86_64: Object.freeze({ + arch: "x86_64", + node: "x64", + target: "x86_64-unknown-linux-gnu", + deb: "amd64", + appimage: "amd64", + // Name of the AppImage once installed under ~/Applications. + installed: "x86_64", + flatpak: "x86_64", + docker: "linux/amd64", + multiarch: "x86_64-linux-gnu", + loader: "ld-linux-x86-64.so.2", + // Historical: x86_64 runtime sources carry no architecture in their name. + runtimeSources: "appimage-runtime-sources", + }), + aarch64: Object.freeze({ + arch: "aarch64", + node: "arm64", + target: "aarch64-unknown-linux-gnu", + deb: "arm64", + appimage: "aarch64", + installed: "aarch64", + flatpak: "aarch64", + docker: "linux/arm64", + multiarch: "aarch64-linux-gnu", + loader: "ld-linux-aarch64.so.1", + runtimeSources: "aarch64_appimage-runtime-sources", + }), +}); + +const ALIASES = { x86_64: "x86_64", amd64: "x86_64", x64: "x86_64", aarch64: "aarch64", arm64: "aarch64" }; + +/** Resolve an architecture name (x86_64/amd64/x64, aarch64/arm64). */ +export function archInfo(name) { + const key = ALIASES[name]; + assert(key, `Unsupported architecture: ${name}`); + return ARCHES[key]; +} + +/** The build host's architecture, or SHADOWCODE_ARCH when it names the same one. */ +export function hostArch(env = process.env, nodeArch = process.arch) { + const host = Object.values(ARCHES).find((info) => info.node === nodeArch); + assert(host, `This packaging workflow supports Linux x86_64 and aarch64, not ${nodeArch}`); + if (env.SHADOWCODE_ARCH) { + // Packages embed host-built binaries; cross packaging is not supported. + assert.equal(archInfo(env.SHADOWCODE_ARCH).arch, host.arch, "SHADOWCODE_ARCH must match the build host (no cross packaging)"); + } + return host; +} + +export function packageNames(version, arch) { + const info = archInfo(arch); + return { + appimage: `ShadowCode_${version}_${info.appimage}.AppImage`, + deb: `ShadowCode_${version}_${info.deb}.deb`, + runtimeSources: `ShadowCode_${version}_${info.runtimeSources}.tar.gz`, + flatpak: `ShadowCode_${version}_${info.flatpak}.flatpak`, + }; +} diff --git a/scripts/native-release-assets.mjs b/scripts/native-release-assets.mjs index 90d0b20e..5e085d70 100644 --- a/scripts/native-release-assets.mjs +++ b/scripts/native-release-assets.mjs @@ -5,17 +5,29 @@ import { open, mkdtemp, mkdir, rm, readFile, writeFile, readdir } from 'node:fs/ import { createHash } from 'node:crypto'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { assetNames, LIMITS, IDENTITY, verifyBundle } from './native-release-auth.mjs'; +import { assetNames, metadataNames, LIMITS, IDENTITY, TARGETS, verifyBundle } from './native-release-auth.mjs'; import { validateVerification } from './native-release-verification.mjs'; +import { ARCHES } from './native-arch.mjs'; export const METADATA = Object.freeze(['SHA256SUMS', 'RELEASE-MANIFEST.json', 'RELEASE-AUTH', 'RELEASE-AUTH.sig']); export const RECEIPTS_LIMIT = 1024 * 1024; export const jsonBytes = value => Buffer.from(`${JSON.stringify(value, null, 2)}\n`); -export function releaseNames(version, signed = true) { - return [...assetNames(version).map(([, name]) => name), ...METADATA.slice(0, signed ? 4 : 2)]; +// Every name below is a release asset name too: one target's bundle uploads +// as-is next to the other targets' bundles in the same GitHub release. +export function metadataList(target = IDENTITY.target) { + const names = metadataNames(target); + return [names.sums, names.manifest, names.auth, names.signature]; } -export function classifyFiles(files, version, signed = true) { - const expected = releaseNames(version, signed); +export function releaseNames(version, signed = true, target = IDENTITY.target) { + return [...assetNames(version, target).map(([, name]) => name), ...metadataList(target).slice(0, signed ? 4 : 2)]; +} +// Release gates name packages by build architecture; signed targets map to it. +export function targetArch(target = IDENTITY.target) { + assert(Object.hasOwn(TARGETS, target), 'Unsupported release target'); + return Object.values(ARCHES).find(info => info.target === target).arch; +} +export function classifyFiles(files, version, signed = true, target = IDENTITY.target) { + const expected = releaseNames(version, signed, target); assert(Array.isArray(files), 'Required signed release assets missing'); const result = new Map(); for (const file of files) { @@ -29,7 +41,12 @@ export function classifyFiles(files, version, signed = true) { return result; } function limitFor(name) { - return { 'SHA256SUMS': LIMITS.checksums, 'RELEASE-MANIFEST.json': LIMITS.manifest, 'RELEASE-AUTH': LIMITS.envelope, 'RELEASE-AUTH.sig': LIMITS.signature, 'VERIFICATION.json': RECEIPTS_LIMIT }[name] ?? LIMITS.artifact; + for (const target of Object.keys(TARGETS)) { + const names = metadataNames(target); + const limit = { [names.sums]: LIMITS.checksums, [names.manifest]: LIMITS.manifest, [names.auth]: LIMITS.envelope, [names.signature]: LIMITS.signature }[name]; + if (limit !== undefined) return limit; + } + return name === 'VERIFICATION.json' ? RECEIPTS_LIMIT : LIMITS.artifact; } export async function snapshotFile(source, destination, limit) { const input = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); @@ -52,8 +69,8 @@ export async function snapshotFile(source, destination, limit) { return { hash: hash.digest('hex'), size }; } finally { await output?.close(); await input.close(); } } -export async function withReleaseSnapshot({ files, version, signed = true }, use) { - const sources = classifyFiles(files, version, signed); +export async function withReleaseSnapshot({ files, version, signed = true, target = IDENTITY.target }, use) { + const sources = classifyFiles(files, version, signed, target); const directory = await mkdtemp(path.join(tmpdir(), 'shadowcode-publisher-snapshot-')); try { const observed = new Map(); @@ -83,30 +100,30 @@ export function validateContext({ repo, tag, commit, runId, runAttempt }) { assert.match(runAttempt, /^[1-9][0-9]*$/, 'Expected workflow attempt required'); return tag.slice(1); } -export async function validateStagedRelease({ directory, observed, verification, context, trustDir, signed }) { - const version = validateContext(context); - const packages = Object.fromEntries(assetNames(version).map(([, name]) => [name, observed.get(name).hash])); - const gates = validateVerification(verification, context.commit, packages); +export async function validateStagedRelease({ directory, observed, verification, context, trustDir, signed, target = IDENTITY.target }) { + const version = validateContext(context), names = metadataNames(target); + const packages = Object.fromEntries(assetNames(version, target).map(([, name]) => [name, observed.get(name).hash])); + const gates = validateVerification(verification, context.commit, packages, targetArch(target)); assert.equal(verification.run_id, context.runId, 'Unexpected workflow run'); assert.equal(verification.run_attempt, context.runAttempt, 'Unexpected workflow attempt'); - const manifest = await readJson(path.join(directory, 'RELEASE-MANIFEST.json'), LIMITS.manifest); + const manifest = await readJson(path.join(directory, names.manifest), LIMITS.manifest); assert.deepEqual(Object.keys(manifest).sort(), ['schema', 'tag', 'commit', 'target', 'cargo_lock_sha256', 'ui_lock_sha256', 'runtime_pin', 'verification', 'assets'].sort(), 'Manifest field set mismatch'); - assert.equal(manifest.schema, 1); assert.equal(manifest.tag, context.tag); assert.equal(manifest.commit, context.commit); assert.equal(manifest.target, IDENTITY.target); + assert.equal(manifest.schema, 1); assert.equal(manifest.tag, context.tag); assert.equal(manifest.commit, context.commit); assert.equal(manifest.target, target); assert.match(manifest.cargo_lock_sha256, /^[a-f0-9]{64}$/); assert.match(manifest.ui_lock_sha256, /^[a-f0-9]{64}$/); assert(typeof manifest.runtime_pin === 'string' && manifest.runtime_pin.length > 0 && manifest.runtime_pin.length <= 65536, 'Invalid runtime pin'); assert.deepEqual(manifest.assets, packages, 'Manifest package identities mismatch'); assert.deepEqual(manifest.verification, gates, 'Signed manifest gate claims differ from required receipts'); const sums = Buffer.from(Object.entries(packages).map(([name, hash]) => `${hash} ${name}\n`).join('')); - assert((await readFile(path.join(directory, 'SHA256SUMS'))).equals(sums), 'Checksums must contain the exact canonical package set'); + assert((await readFile(path.join(directory, names.sums))).equals(sums), 'Checksums must contain the exact canonical package set'); if (signed) { assert(trustDir, 'Publisher authentication requires independently trusted public keys'); - for (const [name] of Object.entries(packages)) await verifyBundle({ bundleDir: directory, trustDir, artifact: name, expectedVersion: version, expectedCommit: context.commit }); + for (const [name] of Object.entries(packages)) await verifyBundle({ bundleDir: directory, trustDir, artifact: name, expectedVersion: version, expectedCommit: context.commit, expectedTarget: target }); } return { packages, gates, manifest }; } -export async function requireTransportLayout(directory, version, signed = false) { - assert.deepEqual((await readdir(directory)).sort(), [...releaseNames(version, signed), 'VERIFICATION.json'].sort(), 'Unexpected transport file set'); - return releaseNames(version, signed).map(name => path.join(directory, name)); +export async function requireTransportLayout(directory, version, signed = false, target = IDENTITY.target) { + assert.deepEqual((await readdir(directory)).sort(), [...releaseNames(version, signed, target), 'VERIFICATION.json'].sort(), 'Unexpected transport file set'); + return releaseNames(version, signed, target).map(name => path.join(directory, name)); } export async function retainBundle(directory, destination, names, verification) { await mkdir(destination, { mode: 0o700 }); // Refuse an existing output, including a symlink. diff --git a/scripts/native-release-auth-lib.sh b/scripts/native-release-auth-lib.sh index 8e3475b6..c8ac1efe 100644 --- a/scripts/native-release-auth-lib.sh +++ b/scripts/native-release-auth-lib.sh @@ -4,6 +4,30 @@ HEX='^[a-f0-9]{64}$' VERSION_PATTERN='^(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})$' EPOCH_PATTERN='^[1-9][0-9]{0,8}$' MAX_ARTIFACT=34359738368 +# Supported signed targets. x86_64 package and metadata names are historical +# and fixed; other targets' release metadata carries TARGET_META as a suffix +# (RELEASE-AUTH-aarch64 …) so every target fits one GitHub release. +target_names() { + case "$1" in + x86_64-unknown-linux-gnu) TARGET_APPIMAGE=amd64 TARGET_DEB=amd64 TARGET_SOURCES=appimage-runtime-sources TARGET_META='' ;; + aarch64-unknown-linux-gnu) TARGET_APPIMAGE=aarch64 TARGET_DEB=arm64 TARGET_SOURCES=aarch64_appimage-runtime-sources TARGET_META=-aarch64 ;; + *) return 1 ;; + esac +} +# The target whose package set contains this basename (x86_64 otherwise); the +# signed envelope read for it must name the same target. +target_for_artifact() { + local name=$1 target + ARTIFACT_TARGET=x86_64-unknown-linux-gnu + [[ "$name" =~ ^ShadowCode_((0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8})\.(0|[1-9][0-9]{0,8}))_ ]] || return 0 + local version=${BASH_REMATCH[1]} + for target in aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu; do + target_names "$target" + if [[ "$name" == "ShadowCode_${version}_${TARGET_APPIMAGE}.AppImage" || "$name" == "ShadowCode_${version}_${TARGET_DEB}.deb" || "$name" == "ShadowCode_${version}_${TARGET_SOURCES}.tar.gz" ]]; then + ARTIFACT_TARGET=$target; return 0 + fi + done +} copy_file() { local source=$1 destination=$2 limit=$3 size copied [[ -f "$source" && ! -L "$source" ]] || fail 'input must be a regular non-symlink file' @@ -56,13 +80,13 @@ parse_envelope() { take 4 version; EN_VERSION=$VALUE; [[ "$EN_VERSION" =~ $VERSION_PATTERN ]] || fail 'invalid stable version' take 5 tag; [[ "$VALUE" == "v$EN_VERSION" ]] || fail 'version/tag mismatch' take 6 commit; EN_COMMIT=$VALUE; [[ "$EN_COMMIT" =~ ^[a-f0-9]{40}$ ]] || fail 'invalid commit' - take 7 target; [[ "$VALUE" == x86_64-unknown-linux-gnu ]] || fail 'wrong target' + take 7 target; EN_TARGET=$VALUE; target_names "$EN_TARGET" || fail 'wrong target' take 8 channel; [[ "$VALUE" == stable ]] || fail 'wrong channel' take 9 key-id; EN_KEY=$VALUE; [[ "$EN_KEY" =~ $HEX ]] || fail 'invalid key ID' take 10 key-epoch; EN_EPOCH=$VALUE; [[ "$EN_EPOCH" =~ $EPOCH_PATTERN ]] || fail 'invalid key epoch' take 11 manifest-sha256; EN_MANIFEST=$VALUE; [[ "$EN_MANIFEST" =~ $HEX ]] || fail 'invalid manifest digest' take 12 checksums-sha256; EN_SUMS=$VALUE; [[ "$EN_SUMS" =~ $HEX ]] || fail 'invalid checksums digest' - EN_NAMES=("ShadowCode_${EN_VERSION}_amd64.AppImage" "ShadowCode_${EN_VERSION}_amd64.deb" "ShadowCode_${EN_VERSION}_appimage-runtime-sources.tar.gz") + EN_NAMES=("ShadowCode_${EN_VERSION}_${TARGET_APPIMAGE}.AppImage" "ShadowCode_${EN_VERSION}_${TARGET_DEB}.deb" "ShadowCode_${EN_VERSION}_${TARGET_SOURCES}.tar.gz") EN_HASHES=(); EN_SIZES=() local -a roles=(appimage deb runtime-sources) local i value role name size hash extra @@ -80,7 +104,17 @@ load_trust_policy() { read_lines "$SCRATCH/trust-policy" 4096 [[ "${#ROWS[@]}" -ge 10 && "${#ROWS[@]}" -le 17 && "${ROWS[0]}" == ShadowCode-Release-Trust-v1 ]] || fail 'trust policy schema/field count' identity - take 4 target; [[ "$VALUE" == x86_64-unknown-linux-gnu ]] || fail 'wrong trusted target' + # One target, or a comma-separated list of distinct supported targets. + take 4 target + local trusted + TRUST_TARGETS=() + IFS=, read -r -a TRUST_TARGETS <<< "$VALUE" + [[ "${#TRUST_TARGETS[@]}" -ge 1 && "$VALUE" != *, ]] || fail 'wrong trusted target' + declare -A trusted_seen=() + for trusted in "${TRUST_TARGETS[@]}"; do + target_names "$trusted" && [[ -z "${trusted_seen[$trusted]:-}" ]] || fail 'wrong trusted target' + trusted_seen[$trusted]=1 + done take 5 channel; [[ "$VALUE" == stable ]] || fail 'wrong trusted channel' take 6 minimum-epoch; MIN_EPOCH=$VALUE; [[ "$MIN_EPOCH" =~ $EPOCH_PATTERN ]] || fail 'invalid epoch floor' take 7 minimum-version; MIN_VERSION=$VALUE; [[ "$MIN_VERSION" =~ $VERSION_PATTERN ]] || fail 'invalid version floor' @@ -100,6 +134,9 @@ load_trust_policy() { verify_envelope() { local file=$1 signature=$2 historical=$3 i found=-1 der_hex parse_envelope "$file" + local trusted_target=0 candidate + for candidate in "${TRUST_TARGETS[@]}"; do [[ "$candidate" != "$EN_TARGET" ]] || trusted_target=1; done + [[ "$trusted_target" == 1 ]] || fail 'release target is not trusted' [[ "$(stat -c %s -- "$signature")" == 64 ]] || fail 'signature must be exactly 64 bytes' for i in "${!KEY_IDS[@]}"; do [[ "${KEY_IDS[$i]}" != "$EN_KEY" ]] || found=$i; done (( found >= 0 )) || fail 'unknown signing key' diff --git a/scripts/native-release-auth.mjs b/scripts/native-release-auth.mjs index 1c3dce83..3d40674f 100644 --- a/scripts/native-release-auth.mjs +++ b/scripts/native-release-auth.mjs @@ -8,6 +8,13 @@ import path from 'node:path'; export const IDENTITY = Object.freeze({ repository: 'ShadowfetchStudio/ShadowCode', repositoryId: '1377099349', ownerId: '209457103', target: 'x86_64-unknown-linux-gnu', channel: 'stable' }); // Releases through 1.0.1 retain their original signed bytes after the rename. // Both names require the same pinned repository and owner IDs and trusted key. +// Signed release targets and their package names. x86_64 names are historical +// and fixed; aarch64 follows Tauri (AppImage `_aarch64`, Debian `_arm64`) and +// keeps its runtime sources distinct so both targets fit one GitHub release. +export const TARGETS = Object.freeze({ + 'x86_64-unknown-linux-gnu': Object.freeze({ appimage: 'amd64', deb: 'amd64', sources: 'appimage-runtime-sources' }), + 'aarch64-unknown-linux-gnu': Object.freeze({ appimage: 'aarch64', deb: 'arm64', sources: 'aarch64_appimage-runtime-sources' }), +}); const REPOSITORY_NAMES = new Set([IDENTITY.repository, 'Shadowfetchapps/ShadowCode']); export const LIMITS = Object.freeze({ envelope: 4096, trust: 4096, manifest: 1024 * 1024, checksums: 4096, publicKey: 1024, signature: 64, artifact: 32 * 1024 ** 3, keys: 8 }); const HEX = /^[a-f0-9]{64}$/; @@ -37,15 +44,34 @@ export function compareVersions(a, b) { for (let i = 0; i < 3; i++) if (left[i] !== right[i]) return left[i] < right[i] ? -1 : 1; return 0; } -export function assetNames(version) { +export function assetNames(version, target = IDENTITY.target) { assert.match(version, VERSION, 'Invalid stable version'); assert.equal(version.trim(), version, 'Version whitespace'); + assert(Object.hasOwn(TARGETS, target), 'Unsupported release target'); + const names = TARGETS[target]; return [ - ['appimage', `ShadowCode_${version}_amd64.AppImage`], - ['deb', `ShadowCode_${version}_amd64.deb`], - ['runtime-sources', `ShadowCode_${version}_appimage-runtime-sources.tar.gz`], + ['appimage', `ShadowCode_${version}_${names.appimage}.AppImage`], + ['deb', `ShadowCode_${version}_${names.deb}.deb`], + ['runtime-sources', `ShadowCode_${version}_${names.sources}.tar.gz`], ]; } +// Release metadata names. x86_64 keeps the historical unsuffixed names, which +// existing installers read; other targets carry a suffix so every target's +// envelope, signature, manifest and checksums fit one GitHub release. +const METADATA_SUFFIX = Object.freeze({ 'x86_64-unknown-linux-gnu': '', 'aarch64-unknown-linux-gnu': '-aarch64' }); +export function metadataNames(target = IDENTITY.target) { + assert(Object.hasOwn(TARGETS, target), 'Unsupported release target'); + const suffix = METADATA_SUFFIX[target]; + return Object.freeze({ sums: `SHA256SUMS${suffix}`, manifest: `RELEASE-MANIFEST${suffix}.json`, auth: `RELEASE-AUTH${suffix}`, signature: `RELEASE-AUTH${suffix}.sig` }); +} +// The target whose package set contains this basename. Package names differ +// per target, so the requested artifact selects which metadata to read; the +// signed envelope must then name the same target. +export function targetForArtifact(name) { + const version = /^ShadowCode_((?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8})\.(?:0|[1-9][0-9]{0,8}))_/.exec(String(name))?.[1]; + if (version) for (const target of Object.keys(TARGETS)) if (assetNames(version, target).some(([, asset]) => asset === name)) return target; + return IDENTITY.target; +} function fixedIdentity(rows, offset = 1) { assert(REPOSITORY_NAMES.has(field(rows[offset], 'repository')), 'Wrong publisher repository'); assert.equal(field(rows[offset + 1], 'repository-id'), IDENTITY.repositoryId, 'Wrong repository ID'); @@ -61,13 +87,14 @@ export function parseEnvelope(bytes) { assert.equal(field(rows[5], 'tag'), `v${version}`, 'Version/tag mismatch'); const commit = field(rows[6], 'commit'); assert.match(commit, /^[a-f0-9]{40}$/); - assert.equal(field(rows[7], 'target'), IDENTITY.target, 'Wrong target'); + const target = field(rows[7], 'target'); + assert(Object.hasOwn(TARGETS, target), 'Wrong target'); assert.equal(field(rows[8], 'channel'), IDENTITY.channel, 'Wrong channel'); const keyId = field(rows[9], 'key-id'), keyEpoch = field(rows[10], 'key-epoch'); assert.match(keyId, HEX); assert.match(keyEpoch, EPOCH); const manifestHash = field(rows[11], 'manifest-sha256'), checksumsHash = field(rows[12], 'checksums-sha256'); assert.match(manifestHash, HEX); assert.match(checksumsHash, HEX); - const assets = assetNames(version).map(([role, name], index) => { + const assets = assetNames(version, target).map(([role, name], index) => { assert(rows[13 + index].startsWith('asset='), 'Missing asset'); const parts = rows[13 + index].slice(6).split('\t'); assert.equal(parts.length, 4, 'Asset field count'); @@ -79,14 +106,17 @@ export function parseEnvelope(bytes) { assert.match(parts[3], HEX); return { role, name, size, hash: parts[3] }; }); - return { version, tag: `v${version}`, commit, keyId, keyEpoch: Number(keyEpoch), manifestHash, checksumsHash, assets, envelopeHash: HASH(bytes) }; + return { version, tag: `v${version}`, commit, target, keyId, keyEpoch: Number(keyEpoch), manifestHash, checksumsHash, assets, envelopeHash: HASH(bytes) }; } export function parseTrustPolicy(bytes) { const rows = lines(bytes, LIMITS.trust, 'Trust policy'); assert(rows.length >= 10 && rows.length <= 9 + LIMITS.keys, 'Trust policy field count'); assert.equal(rows[0], 'ShadowCode-Release-Trust-v1', 'Trust policy schema'); fixedIdentity(rows); - assert.equal(field(rows[4], 'target'), IDENTITY.target); + // One target, or a comma-separated list of distinct supported targets. + const targets = field(rows[4], 'target').split(','); + assert(targets.every(target => Object.hasOwn(TARGETS, target)), 'Unsupported trusted target'); + assert.equal(new Set(targets).size, targets.length, 'Duplicate trusted target'); assert.equal(field(rows[5], 'channel'), IDENTITY.channel); const minimumEpoch = field(rows[6], 'minimum-epoch'), minimumVersion = field(rows[7], 'minimum-version'); assert.match(minimumEpoch, EPOCH); assert.match(minimumVersion, VERSION); @@ -101,7 +131,7 @@ export function parseTrustPolicy(bytes) { assert(!seen.has(id), 'Duplicate trust key'); seen.add(id); return { epoch: Number(epoch), id, minimum, maximum }; }); - return { minimumEpoch: Number(minimumEpoch), minimumVersion, keys }; + return { targets, minimumEpoch: Number(minimumEpoch), minimumVersion, keys }; } export function publicKeyIdentity(pem) { const key = createPublicKey(pem); @@ -142,19 +172,19 @@ async function assetHash(file) { } finally { await handle.close(); } } function canonicalChecksums(assets) { return Buffer.from(assets.map(asset => `${asset.hash} ${asset.name}\n`).join('')); } -export async function createEnvelope({ bundleDir, version, commit, keyId, keyEpoch }) { - const assets = []; - for (const [role, name] of assetNames(version)) assets.push({ role, name, ...await assetHash(path.join(bundleDir, name)) }); - const sums = await boundedFile(path.join(bundleDir, 'SHA256SUMS'), LIMITS.checksums); +export async function createEnvelope({ bundleDir, version, commit, keyId, keyEpoch, target = IDENTITY.target }) { + const assets = [], names = metadataNames(target); + for (const [role, name] of assetNames(version, target)) assets.push({ role, name, ...await assetHash(path.join(bundleDir, name)) }); + const sums = await boundedFile(path.join(bundleDir, names.sums), LIMITS.checksums); assert(sums.equals(canonicalChecksums(assets)), 'Checksums must contain the exact canonical asset set'); - const manifestBytes = await boundedFile(path.join(bundleDir, 'RELEASE-MANIFEST.json'), LIMITS.manifest); + const manifestBytes = await boundedFile(path.join(bundleDir, names.manifest), LIMITS.manifest); const manifest = JSON.parse(manifestBytes.toString('utf8')); assert(manifestBytes.equals(Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`)), 'Manifest must be canonical JSON without duplicate fields'); assert.equal(manifest.schema, 1); assert.equal(manifest.tag, `v${version}`); - assert.equal(manifest.commit, commit); assert.equal(manifest.target, IDENTITY.target); + assert.equal(manifest.commit, commit); assert.equal(manifest.target, target); assert.deepEqual(manifest.assets, Object.fromEntries(assets.map(a => [a.name, a.hash])), 'Manifest asset bindings'); const rows = ['ShadowCode-Release-Auth-v1', `repository=${IDENTITY.repository}`, `repository-id=${IDENTITY.repositoryId}`, `owner-id=${IDENTITY.ownerId}`, - `version=${version}`, `tag=v${version}`, `commit=${commit}`, `target=${IDENTITY.target}`, `channel=${IDENTITY.channel}`, `key-id=${keyId}`, `key-epoch=${keyEpoch}`, + `version=${version}`, `tag=v${version}`, `commit=${commit}`, `target=${target}`, `channel=${IDENTITY.channel}`, `key-id=${keyId}`, `key-epoch=${keyEpoch}`, `manifest-sha256=${HASH(manifestBytes)}`, `checksums-sha256=${HASH(sums)}`, ...assets.map(a => `asset=${a.role}\t${a.name}\t${a.size}\t${a.hash}`)]; const bytes = Buffer.from(`${rows.join('\n')}\n`); parseEnvelope(bytes); return bytes; @@ -168,6 +198,7 @@ async function verifyEnvelopeInternal({ bytes, signature, trustDir, historical = const metadata = parseEnvelope(bytes); assert(Buffer.isBuffer(signature) && signature.length === LIMITS.signature, 'Signature must be exactly 64 bytes'); const trust = parseTrustPolicy(await boundedFile(path.join(trustDir, 'policy'), LIMITS.trust)); + assert(trust.targets.includes(metadata.target), 'Release target is not trusted'); const entry = trust.keys.find(key => key.id === metadata.keyId); assert(entry && entry.epoch === metadata.keyEpoch, 'Unknown signing key or epoch'); assert(compareVersions(metadata.version, entry.minimum) >= 0 && compareVersions(metadata.version, entry.maximum) <= 0, 'Outside key version interval'); @@ -187,22 +218,27 @@ export async function verifyEnvelope({ bytes, signature, trustDir }) { // Historical floor exceptions are deliberately not exposed as an install API. return verifyEnvelopeInternal({ bytes, signature, trustDir }); } -export async function verifyBundle({ bundleDir, trustDir, artifact, previousDir, expectedVersion, expectedCommit }) { +export async function verifyBundle({ bundleDir, trustDir, artifact, previousDir, expectedVersion, expectedCommit, expectedTarget }) { assert(trustDir, 'An independently trusted key directory is required'); - const bytes = await boundedFile(path.join(bundleDir, 'RELEASE-AUTH'), LIMITS.envelope); - const signature = await boundedFile(path.join(bundleDir, 'RELEASE-AUTH.sig'), LIMITS.signature); + const target = expectedTarget ?? targetForArtifact(artifact), names = metadataNames(target); + const bytes = await boundedFile(path.join(bundleDir, names.auth), LIMITS.envelope); + const signature = await boundedFile(path.join(bundleDir, names.signature), LIMITS.signature); const metadata = await verifyEnvelope({ bytes, signature, trustDir }); if (expectedVersion !== undefined) assert.equal(metadata.version, expectedVersion, 'Unexpected version'); if (expectedCommit !== undefined) assert.equal(metadata.commit, expectedCommit, 'Unexpected commit'); + assert.equal(metadata.target, target, expectedTarget === undefined ? 'Release metadata names another target' : 'Unexpected target'); if (previousDir) { + // The previously accepted release is a verified stage or installer receipt, + // which keeps the historical metadata names for every target. const previous = await verifyEnvelopeInternal({ bytes: await boundedFile(path.join(previousDir, 'RELEASE-AUTH'), LIMITS.envelope), signature: await boundedFile(path.join(previousDir, 'RELEASE-AUTH.sig'), LIMITS.signature), trustDir, historical: true }); + assert.equal(metadata.target, previous.target, 'Release target changed'); assert(metadata.keyEpoch >= previous.keyEpoch, 'Signing epoch rollback'); const comparison = compareVersions(metadata.version, previous.version); assert(comparison >= 0, 'Release downgrade'); if (comparison === 0) assert.equal(metadata.envelopeHash, previous.envelopeHash, 'Changed release under an accepted version'); } - const manifest = await boundedFile(path.join(bundleDir, 'RELEASE-MANIFEST.json'), LIMITS.manifest); - const sums = await boundedFile(path.join(bundleDir, 'SHA256SUMS'), LIMITS.checksums); + const manifest = await boundedFile(path.join(bundleDir, names.manifest), LIMITS.manifest); + const sums = await boundedFile(path.join(bundleDir, names.sums), LIMITS.checksums); assert.equal(HASH(manifest), metadata.manifestHash, 'Manifest digest mismatch'); assert.equal(HASH(sums), metadata.checksumsHash, 'Checksums digest mismatch'); assert(sums.equals(canonicalChecksums(metadata.assets)), 'Checksums asset set mismatch'); diff --git a/scripts/native-release-verification.mjs b/scripts/native-release-verification.mjs index 07040006..c09050b6 100644 --- a/scripts/native-release-verification.mjs +++ b/scripts/native-release-verification.mjs @@ -7,10 +7,17 @@ import { tmpdir } from 'node:os'; import { spawn, execFileSync } from 'node:child_process'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; +import { archInfo, hostArch } from './native-arch.mjs'; const version = String.raw`VERSION=$(node -p "require('./src-tauri/tauri.conf.json').version")`; -const packaged = `${version}\nBINARY="$GITHUB_WORKSPACE/target/release/bundle/appimage/ShadowCode_${'${VERSION}'}_amd64.AppImage"`; -export const GATES = { +// Gate scripts name one architecture's packages. On x86_64 they are +// byte-identical to the historical scripts (and their digests). A signer on one +// architecture checks another architecture's receipts against that +// architecture's scripts, never its own. +export function gatesFor(arch = hostArch().arch) { + const ARCH = archInfo(arch); + const packaged = `${version}\nBINARY="$GITHUB_WORKSPACE/target/release/bundle/appimage/ShadowCode_${'${VERSION}'}_${ARCH.appimage}.AppImage"`; + return { 'release-tests': { script: 'node --test --test-reporter=tap scripts/test-native-release.mjs scripts/test-native-release-verification.mjs scripts/test-publisher-auth.mjs scripts/test-native-release-signing.mjs scripts/test-native-test-session.mjs scripts/test-clean-host-mcp-receipt.mjs scripts/test-clean-host-container.mjs scripts/test-native-app-run.mjs scripts/test-llama-build-inputs.mjs\nnode scripts/test-native-process-ownership.mjs' }, 'release-auth': { script: 'node scripts/check-secrets.mjs\nnode --test --test-reporter=tap scripts/test-native-release-auth.mjs scripts/test-check-secrets.mjs', scope: 'Publisher authentication parser, crypto, replay and private-snapshot fixtures with ephemeral keys; production signing and installer integration are separate.' }, 'release-tag': { script: `${version}\n` + String.raw`test "$GITHUB_REF_NAME" = "v$VERSION" @@ -30,12 +37,12 @@ node --input-type=module -e 'import assert from "node:assert/strict"; import { r 'managed-runtime': { script: 'bash scripts/build-llama.cpp.sh --no-user-install\nnode --test --test-reporter=tap scripts/test-llama-runtime.mjs' }, packages: { artifacts: true, script: `${version}\n` + String.raw`node scripts/build-native.mjs node --test --test-reporter=tap scripts/test-native-packaging-env.mjs scripts/test-native-deb.mjs -node scripts/check-native-package.mjs "target/release/bundle/appimage/ShadowCode_${'${VERSION}'}_amd64.AppImage" "target/release/bundle/deb/ShadowCode_${'${VERSION}'}_amd64.deb" +node scripts/check-native-package.mjs "target/release/bundle/appimage/ShadowCode_${'${VERSION}'}_${ARCH.appimage}.AppImage" "target/release/bundle/deb/ShadowCode_${'${VERSION}'}_${ARCH.deb}.deb" node scripts/test-native-runtime-write-errors.mjs node scripts/test-native-runtime-sources.mjs node scripts/test-native-runtime.mjs` }, - 'clean-host-packages': { artifacts: true, unchanged: true, scope: 'Both packages installed or extracted in a network-disabled Debian 13 runtime container; offline CLI status, Debian launcher/icon metadata, packaged MCP stdio initialize/tool discovery and a visible first GUI window under Xvfb. Interactive GUI tasks, physical Wayland, local inference, model downloads and installer rollback remain separate.', script: `${version}\n` + String.raw`docker build -f scripts/clean-host-runtime.Dockerfile -t shadowcode-clean-runtime:ci scripts -node scripts/test-clean-host-packages.mjs "target/release/bundle/appimage/ShadowCode_${'${VERSION}'}_amd64.AppImage" "target/release/bundle/deb/ShadowCode_${'${VERSION}'}_amd64.deb" artifacts/native-package/SHA256SUMS` }, + 'clean-host-packages': { artifacts: true, unchanged: true, scope: 'Both packages installed or extracted in a network-disabled Debian 13 runtime container; offline CLI status, Debian launcher/icon metadata, packaged MCP stdio initialize/tool discovery and a visible first GUI window under Xvfb. Interactive GUI tasks, physical Wayland, local inference, model downloads and installer rollback remain separate.', script: `${version}\n` + String.raw`docker build --build-arg TARGETARCH="$(dpkg --print-architecture)" -f scripts/clean-host-runtime.Dockerfile -t shadowcode-clean-runtime:ci scripts +node scripts/test-clean-host-packages.mjs "target/release/bundle/appimage/ShadowCode_${'${VERSION}'}_${ARCH.appimage}.AppImage" "target/release/bundle/deb/ShadowCode_${'${VERSION}'}_${ARCH.deb}.deb" artifacts/native-package/SHA256SUMS` }, 'packaged-behavior': { artifacts: true, unchanged: true, scope: 'Packaged fixture behavior on the CI build host; authenticated cloud-consent check is optional and reported separately.', script: `${packaged}\n` + String.raw`SHADOW_DESKTOP_BINARY="$BINARY" SHADOW_CLI_ARGS='["--appimage-extract-and-run"]' node scripts/test-native-cli.mjs SHADOW_DESKTOP_BINARY="$BINARY" SHADOW_CLI_ARGS='["--appimage-extract-and-run"]' node scripts/test-native-tui.mjs SHADOW_DESKTOP_BINARY="$BINARY" SHADOW_DESKTOP_ARGS='["--appimage-extract-and-run","ui"]' SHADOW_NATIVE_DEFAULT_PROFILE=1 node scripts/run-native-x11.mjs node scripts/test-native-desktop.mjs @@ -44,7 +51,9 @@ SHADOW_DESKTOP_BINARY="$BINARY" SHADOW_CLI_ARGS='["--appimage-extract-and-run"]' npm --prefix scripts/native-mcp-peer ci --ignore-scripts --no-audit --no-fund SHADOW_DESKTOP_BINARY="$BINARY" SHADOW_CLI_ARGS='["--appimage-extract-and-run"]' node scripts/test-native-mcp-peer.mjs` }, installer: { artifacts: true, unchanged: true, script: 'node --test --test-reporter=tap scripts/test-install-auth.mjs\nbash scripts/test-install-appimage.sh' }, -}; + }; +} +export const GATES = gatesFor(); export const REQUIRED_GATES = Object.keys(GATES); const OPTIONAL_RUST_TESTS = new Set([ 'live_qwen3_agent_and_gemma4_vision_from_the_ollama_store', @@ -63,16 +72,17 @@ const OPTIONAL_RUST_TESTS = new Set([ // A real desktop keyring in a private D-Bus session (docs/OPENROUTER.md). 'keyring::tests::live_secret_service_round_trip', ]); -export const scriptDigest = gate => createHash('sha256').update(GATES[gate].script).digest('hex'); +export const scriptDigest = (gate, arch) => createHash('sha256').update((arch === undefined ? GATES : gatesFor(arch))[gate].script).digest('hex'); export async function digest(file) { const hash = createHash('sha256'); for await (const chunk of createReadStream(file)) hash.update(chunk); return hash.digest('hex'); } -export async function packageHashes() { +export async function packageHashes(arch = hostArch().arch) { + const ARCH = archInfo(arch); const { version: releaseVersion } = JSON.parse(await readFile('src-tauri/tauri.conf.json', 'utf8')); const base = 'target/release/bundle'; - const files = [`${base}/appimage/ShadowCode_${releaseVersion}_amd64.AppImage`, `${base}/appimage/ShadowCode_${releaseVersion}_appimage-runtime-sources.tar.gz`, `${base}/deb/ShadowCode_${releaseVersion}_amd64.deb`]; + const files = [`${base}/appimage/ShadowCode_${releaseVersion}_${ARCH.appimage}.AppImage`, `${base}/appimage/ShadowCode_${releaseVersion}_${ARCH.runtimeSources}.tar.gz`, `${base}/deb/ShadowCode_${releaseVersion}_${ARCH.deb}.deb`]; return Object.fromEntries(await Promise.all(files.map(async file => [path.basename(file), await digest(file)]))); } @@ -94,7 +104,8 @@ export function inspectLine(gate, raw, requiredSkips, optional) { || /^SKIP\b/.test(line)) requiredSkips.add(line); } -export function validateVerification(verification, commit, assets) { +export function validateVerification(verification, commit, assets, arch) { + const gateTable = arch === undefined ? GATES : gatesFor(arch); assert(verification?.schema === 1, 'Required release verification receipts are missing'); assert.equal(verification.commit, commit, 'Release verification commit mismatch'); assert.match(verification.run_id, /^\d+$/, 'Release verification run identity missing'); @@ -114,25 +125,25 @@ export function validateVerification(verification, commit, assets) { assert.equal(receipt.commit, commit, `Stale verification receipt: ${gate}`); assert.equal(receipt.run_id, verification.run_id, `Wrong workflow run: ${gate}`); assert.equal(receipt.run_attempt, verification.run_attempt, `Wrong workflow attempt: ${gate}`); - assert.equal(receipt.script_sha256, scriptDigest(gate), `Verification command changed: ${gate}`); + assert.equal(receipt.script_sha256, scriptDigest(gate, arch), `Verification command changed: ${gate}`); assert.equal(receipt.status, 'passed', `Required verification gate did not pass: ${gate}`); assert.equal(receipt.exit_code, 0, `Required verification command failed: ${gate}`); assert.deepEqual(receipt.required_skips, [], `Required verification checks skipped: ${gate}`); assert(Array.isArray(receipt.optional_checks), `Optional verification scope missing: ${gate}`); - if (GATES[gate].artifacts) assert.deepEqual(receipt.artifacts, verification.artifacts, `Verification artifact mismatch: ${gate}`); - gates[gate] = { status: 'passed', script_sha256: receipt.script_sha256, scope: GATES[gate].scope || 'All checks in this required gate.', optional_checks: receipt.optional_checks }; + if (gateTable[gate].artifacts) assert.deepEqual(receipt.artifacts, verification.artifacts, `Verification artifact mismatch: ${gate}`); + gates[gate] = { status: 'passed', script_sha256: receipt.script_sha256, scope: gateTable[gate].scope || 'All checks in this required gate.', optional_checks: receipt.optional_checks }; } return { schema: 1, status: 'required_gates_passed', scope: 'Declared Linux CI gates only; excluded checks are not verified. This does not establish publisher authentication, first GUI launch, local-model operation or complete clean-host usability.', gates }; } -export async function readVerification(directory, commit, runId, attempt, artifacts) { +export async function readVerification(directory, commit, runId, attempt, artifacts, arch) { const gates = {}; for (const gate of REQUIRED_GATES) { try { gates[gate] = JSON.parse(await readFile(path.join(directory, `${gate}.json`), 'utf8')); } catch (error) { throw new Error(`Required verification receipt missing or invalid: ${gate}`, { cause: error }); } } const result = { schema: 1, commit, run_id: runId, run_attempt: attempt, artifacts, gates }; - validateVerification(result, commit, artifacts); + validateVerification(result, commit, artifacts, arch); return result; } diff --git a/scripts/native-runtime.mjs b/scripts/native-runtime.mjs index 40ba0d21..6208baf9 100644 --- a/scripts/native-runtime.mjs +++ b/scripts/native-runtime.mjs @@ -15,6 +15,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { applyPackagingPath } from "./native-packaging-env.mjs"; +import { archInfo, hostArch } from "./native-arch.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); const exec = promisify(execFile); const sha = (value) => createHash("sha256").update(value).digest("hex"); @@ -45,7 +46,8 @@ async function command(binary, args) { ); }); } -export async function buildRuntime() { +export async function buildRuntime(arch = hostArch().arch) { + const info = archInfo(arch); applyPackagingPath(root); const recipe = path.join(root, "packaging/native-runtime"); const files = [ @@ -63,6 +65,9 @@ export async function buildRuntime() { .update(name) .update("\0") .update(await readFile(path.join(recipe, name))); + // x86_64 hashes only the recipe files, as it always has; every other + // architecture is a distinct build input. + if (info.arch !== "x86_64") input.update("arch\0").update(info.arch); const inputHash = input.digest("hex"); const manifest = JSON.parse( await readFile(path.join(recipe, "manifest.json"), "utf8"), @@ -78,7 +83,7 @@ export async function buildRuntime() { "target/native-runtime", inputHash.slice(0, 24), ); - const runtime = path.join(directory, "runtime-x86_64"); + const runtime = path.join(directory, `runtime-${info.arch}`); const receiptPath = path.join(directory, "receipt.json"); try { const receipt = JSON.parse(await readFile(receiptPath, "utf8")); @@ -138,7 +143,7 @@ export async function buildRuntime() { await command(engine, [ "build", "--platform", - "linux/amd64", + info.docker, "--build-arg", `BASE_IMAGE=${manifest.container}`, "--tag", @@ -179,7 +184,7 @@ export async function buildRuntime() { "build-packages.txt", "compiler.txt", "runtime-dynamic.txt", - "runtime-x86_64.debug", + `runtime-${info.arch}.debug`, ]) artifacts.push({ file, @@ -253,4 +258,4 @@ if ( process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) ) - await buildRuntime(); + await buildRuntime(process.argv[2] || hostArch().arch); diff --git a/scripts/prepare-native-release.mjs b/scripts/prepare-native-release.mjs index fc8edf0c..c7502dcf 100644 --- a/scripts/prepare-native-release.mjs +++ b/scripts/prepare-native-release.mjs @@ -5,12 +5,13 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { execFileSync } from 'node:child_process'; -import { assetNames, IDENTITY, LIMITS } from './native-release-auth.mjs'; +import { assetNames, metadataNames, IDENTITY, LIMITS } from './native-release-auth.mjs'; import { digest, packageHashes, readVerification, validateVerification } from './native-release-verification.mjs'; -import { snapshotFile, validateContext, validateStagedRelease, retainBundle, releaseNames, jsonBytes } from './native-release-assets.mjs'; +import { snapshotFile, validateContext, validateStagedRelease, retainBundle, releaseNames, jsonBytes, targetArch } from './native-release-assets.mjs'; +import { hostArch } from './native-arch.mjs'; -export async function prepareRelease({ files, checksums, source, verification, context, outputDir }) { - const version = validateContext(context), names = assetNames(version).map(([, name]) => name); +export async function prepareRelease({ files, checksums, source, verification, context, outputDir, target = IDENTITY.target }) { + const version = validateContext(context), names = assetNames(version, target).map(([, name]) => name); assert.deepEqual(files.map(file => path.basename(file)).sort(), [...names].sort(), 'Build package set mismatch'); const directory = await mkdtemp(path.join(tmpdir(), 'shadowcode-build-handoff-')); try { @@ -19,12 +20,13 @@ export async function prepareRelease({ files, checksums, source, verification, c const name = path.basename(file); observed.set(name, { file: path.join(directory, name), ...await snapshotFile(file, path.join(directory, name), LIMITS.artifact) }); } - await snapshotFile(checksums, path.join(directory, 'SHA256SUMS'), LIMITS.checksums); + const metadata = metadataNames(target); + await snapshotFile(checksums, path.join(directory, metadata.sums), LIMITS.checksums); const assets = Object.fromEntries(names.map(name => [name, observed.get(name).hash])); - const manifest = { schema: 1, tag: context.tag, commit: context.commit, target: IDENTITY.target, cargo_lock_sha256: source.cargo_lock_sha256, ui_lock_sha256: source.ui_lock_sha256, runtime_pin: source.runtime_pin, verification: validateVerification(verification, context.commit, assets), assets }; - await writeFile(path.join(directory, 'RELEASE-MANIFEST.json'), jsonBytes(manifest), { mode: 0o400 }); - await validateStagedRelease({ directory, observed, verification, context, signed: false }); - await retainBundle(directory, outputDir, releaseNames(version, false), verification); + const manifest = { schema: 1, tag: context.tag, commit: context.commit, target, cargo_lock_sha256: source.cargo_lock_sha256, ui_lock_sha256: source.ui_lock_sha256, runtime_pin: source.runtime_pin, verification: validateVerification(verification, context.commit, assets, targetArch(target)), assets }; + await writeFile(path.join(directory, metadata.manifest), jsonBytes(manifest), { mode: 0o400 }); + await validateStagedRelease({ directory, observed, verification, context, signed: false, target }); + await retainBundle(directory, outputDir, releaseNames(version, false, target), verification); } finally { await rm(directory, { recursive: true, force: true }); } } async function main() { @@ -35,10 +37,12 @@ async function main() { assert.equal(git(['rev-parse', 'HEAD']), context.commit); assert.equal(git(['rev-parse', `${context.tag}^{commit}`]), context.commit); assert.equal(JSON.parse(await readFile('src-tauri/tauri.conf.json', 'utf8')).version, version); + // The handoff is for the packages this host built; cross packaging is refused. + const { arch, target } = hostArch(); const base = 'target/release/bundle'; - const files = assetNames(version).map(([role, name]) => `${base}/${role === 'deb' ? 'deb' : 'appimage'}/${name}`); - const verification = await readVerification('artifacts/release-verification', context.commit, context.runId, context.runAttempt, await packageHashes()); + const files = assetNames(version, target).map(([role, name]) => `${base}/${role === 'deb' ? 'deb' : 'appimage'}/${name}`); + const verification = await readVerification('artifacts/release-verification', context.commit, context.runId, context.runAttempt, await packageHashes(arch), arch); const source = { cargo_lock_sha256: await digest('Cargo.lock'), ui_lock_sha256: await digest('ui/package-lock.json'), runtime_pin: (await readFile('tools/llama.cpp.pin', 'utf8')).trim() }; - await prepareRelease({ files, checksums: `${base}/SHA256SUMS`, source, verification, context, outputDir }); + await prepareRelease({ files, checksums: `${base}/SHA256SUMS`, source, verification, context, outputDir, target }); } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) await main(); diff --git a/scripts/publish-native-release.mjs b/scripts/publish-native-release.mjs index f71d7571..4a0a6e2b 100644 --- a/scripts/publish-native-release.mjs +++ b/scripts/publish-native-release.mjs @@ -7,19 +7,38 @@ import { execFileSync } from 'node:child_process'; import assert from 'node:assert/strict'; import { setTimeout as delay } from 'node:timers/promises'; import { digest } from './native-release-verification.mjs'; -import { IDENTITY } from './native-release-auth.mjs'; +import { IDENTITY, TARGETS } from './native-release-auth.mjs'; import { withReleaseSnapshot, validateContext, validateStagedRelease, readReceipts, requireTransportLayout } from './native-release-assets.mjs'; export { digest } from './native-release-verification.mjs'; const command = (args) => execFileSync('gh', args, { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] }); -export async function publish({ repo, tag, commit, files, verification, trustDir, runId, runAttempt, gh = command, sleep = delay }) { +// One release carries every target's signed bundle. Each bundle is snapshotted +// and authenticated for its own target before any release operation; their +// asset names are disjoint (non-x86_64 metadata carries a target suffix). +export async function publish({ repo, tag, commit, files, verification, bundles, trustDir, runId, runAttempt, gh = command, sleep = delay }) { const context = { repo, tag, commit, runId, runAttempt }; const version = validateContext(context); - return withReleaseSnapshot({ files, version }, async (directory, expected) => { - await validateStagedRelease({ directory, observed: expected, verification, context, trustDir, signed: true }); - return publishVerified({ repo, tag, commit, expected, gh, sleep }); - }); + bundles ??= [{ target: IDENTITY.target, files, verification }]; + assert(Array.isArray(bundles) && bundles.length >= 1, 'At least one signed release bundle is required'); + const targets = bundles.map(bundle => bundle.target); + assert(targets.every(target => Object.hasOwn(TARGETS, target)), 'Unsupported release target'); + assert.equal(new Set(targets).size, targets.length, 'Duplicate release target'); + assert(targets.includes(IDENTITY.target), 'The x86_64 release bundle is required'); + const expected = new Map(); + const stage = async index => { + if (index === bundles.length) return publishVerified({ repo, tag, commit, expected, gh, sleep }); + const { target, files: bundleFiles, verification: receipts } = bundles[index]; + return withReleaseSnapshot({ files: bundleFiles, version, target }, async (directory, observed) => { + await validateStagedRelease({ directory, observed, verification: receipts, context, trustDir, signed: true, target }); + for (const [name, asset] of observed) { + assert(!expected.has(name), `Duplicate release asset across targets: ${name}`); + expected.set(name, asset); + } + return stage(index + 1); + }); + }; + return stage(0); } async function publishVerified({ repo, tag, commit, expected, gh, sleep }) { const identity = JSON.parse(gh(['api', `repos/${repo}`])); @@ -92,12 +111,13 @@ async function publishVerified({ repo, tag, commit, expected, gh, sleep }) { } async function main() { - const [bundle, trustDir] = process.argv.slice(2); - assert(bundle && trustDir && process.argv.length === 4, 'Usage: node publish-native-release.mjs SIGNED_BUNDLE TRUST_DIR'); + const [bundle, trustDir, aarch64Bundle] = process.argv.slice(2); + assert(bundle && trustDir && (process.argv.length === 4 || (process.argv.length === 5 && aarch64Bundle)), 'Usage: node publish-native-release.mjs SIGNED_BUNDLE TRUST_DIR [AARCH64_SIGNED_BUNDLE]'); const context = { repo: process.env.GITHUB_REPOSITORY, tag: process.env.GITHUB_REF_NAME, commit: process.env.GITHUB_SHA, runId: process.env.GITHUB_RUN_ID, runAttempt: process.env.GITHUB_RUN_ATTEMPT }; const version = validateContext(context); - const files = await requireTransportLayout(bundle, version, true); - const verification = await readReceipts(bundle); - console.log(await publish({ ...context, files, verification, trustDir })); + const load = async (directory, target) => ({ target, files: await requireTransportLayout(directory, version, true, target), verification: await readReceipts(directory) }); + const bundles = [await load(bundle, IDENTITY.target)]; + if (aarch64Bundle) bundles.push(await load(aarch64Bundle, 'aarch64-unknown-linux-gnu')); + console.log(await publish({ ...context, bundles, trustDir })); } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) await main(); diff --git a/scripts/publisher-fixtures.mjs b/scripts/publisher-fixtures.mjs index 440e07f1..35b523a8 100644 --- a/scripts/publisher-fixtures.mjs +++ b/scripts/publisher-fixtures.mjs @@ -4,12 +4,15 @@ import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'; import { readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { assetNames, createEnvelope, signEnvelope, publicKeyIdentity, IDENTITY } from './native-release-auth.mjs'; +import { assetNames, metadataNames, createEnvelope, signEnvelope, publicKeyIdentity, IDENTITY } from './native-release-auth.mjs'; import { digest, GATES, REQUIRED_GATES, scriptDigest, validateVerification } from './native-release-verification.mjs'; +import { targetArch } from './native-release-assets.mjs'; export const COMMIT = 'a'.repeat(40); export const VERSION = '1.0.0'; // Fixture only; not a production release or cutover. -export async function fixture(run, publish) { +export const ARM = 'aarch64-unknown-linux-gnu'; +// `targets` adds further signed bundles (aarch64) to the same fixture release. +export async function fixture(run, publish, { targets = [IDENTITY.target] } = {}) { const root = await mkdtemp(path.join(tmpdir(), 'shadowcode-publisher-fixture-')); const bundle = path.join(root, 'bundle'), trustDir = path.join(root, 'trust'); await mkdir(bundle); await mkdir(trustDir); @@ -17,26 +20,39 @@ export async function fixture(run, publish) { const publicPem = publicKey.export({ format: 'pem', type: 'spki' }); const keyId = publicKeyIdentity(publicPem); await writeFile(path.join(trustDir, `${keyId}.pem`), publicPem); - await writeFile(path.join(trustDir, 'policy'), `ShadowCode-Release-Trust-v1\nrepository=${IDENTITY.repository}\nrepository-id=${IDENTITY.repositoryId}\nowner-id=${IDENTITY.ownerId}\ntarget=${IDENTITY.target}\nchannel=${IDENTITY.channel}\nminimum-epoch=1\nminimum-version=1.0.0\nkeys=ed25519-spki-sha256\nkey=1\t${keyId}\t1.0.0\t999.999.999\n`); - const packages = assetNames(VERSION).map(([, name]) => path.join(bundle, name)); - await writeFile(packages[0], `#!/bin/sh\nprintf executed > '${root}/candidate-executed'\n`, { mode: 0o755 }); - for (const file of packages.slice(1)) await writeFile(file, `fixture ${path.basename(file)}`); - const verification = { schema: 1, commit: COMMIT, run_id: '123', run_attempt: '2', artifacts: {}, gates: {} }; + await writeFile(path.join(trustDir, 'policy'), `ShadowCode-Release-Trust-v1\nrepository=${IDENTITY.repository}\nrepository-id=${IDENTITY.repositoryId}\nowner-id=${IDENTITY.ownerId}\ntarget=${targets.join(',')}\nchannel=${IDENTITY.channel}\nminimum-epoch=1\nminimum-version=1.0.0\nkeys=ed25519-spki-sha256\nkey=1\t${keyId}\t1.0.0\t999.999.999\n`); const source = { cargo_lock_sha256: 'b'.repeat(64), ui_lock_sha256: 'c'.repeat(64), runtime_pin: 'commit=' + 'd'.repeat(40) }; + // One signed bundle per target; x86_64 keeps the historical `bundle` layout. + const bundles = []; + for (const target of targets) { + const directory = target === IDENTITY.target ? bundle : path.join(root, `bundle-${targetArch(target)}`); + await mkdir(directory, { recursive: true }); + const packages = assetNames(VERSION, target).map(([, name]) => path.join(directory, name)); + await writeFile(packages[0], `#!/bin/sh\nprintf executed > '${root}/candidate-executed'\n`, { mode: 0o755 }); + for (const file of packages.slice(1)) await writeFile(file, `fixture ${path.basename(file)}`); + const meta = metadataNames(target); + bundles.push({ target, directory, packages, verification: { schema: 1, commit: COMMIT, run_id: '123', run_attempt: '2', artifacts: {}, gates: {} }, + files: [...packages, ...[meta.sums, meta.manifest, meta.auth, meta.signature].map(name => path.join(directory, name))] }); + } + const [{ packages, verification, files }] = bundles; async function rebuild() { - const artifacts = Object.fromEntries(await Promise.all(packages.map(async file => [path.basename(file), await digest(file)]))); - verification.artifacts = artifacts; - verification.gates = Object.fromEntries(REQUIRED_GATES.map(gate => [gate, { schema: 1, gate, commit: COMMIT, run_id: verification.run_id, run_attempt: verification.run_attempt, script_sha256: scriptDigest(gate), status: 'passed', exit_code: 0, required_skips: [], optional_checks: [], ...(GATES[gate].artifacts ? { artifacts } : {}) }])); - await writeFile(path.join(bundle, 'SHA256SUMS'), Object.entries(artifacts).map(([name, sha]) => `${sha} ${name}\n`).join('')); - const manifest = { schema: 1, tag: `v${VERSION}`, commit: COMMIT, target: IDENTITY.target, ...source, verification: validateVerification(verification, COMMIT, artifacts), assets: artifacts }; - await writeFile(path.join(bundle, 'RELEASE-MANIFEST.json'), `${JSON.stringify(manifest, null, 2)}\n`); - await writeFile(path.join(bundle, 'VERIFICATION.json'), `${JSON.stringify(verification, null, 2)}\n`); - const envelope = await createEnvelope({ bundleDir: bundle, version: VERSION, commit: COMMIT, keyId, keyEpoch: 1 }); - await writeFile(path.join(bundle, 'RELEASE-AUTH'), envelope); - await writeFile(path.join(bundle, 'RELEASE-AUTH.sig'), signEnvelope(envelope, privateKey)); + for (const { target, directory, packages, verification } of bundles) { + // Receipts carry the gate digests of the architecture that built them. + const arch = targetArch(target), meta = metadataNames(target); + const artifacts = Object.fromEntries(await Promise.all(packages.map(async file => [path.basename(file), await digest(file)]))); + verification.artifacts = artifacts; + verification.gates = Object.fromEntries(REQUIRED_GATES.map(gate => [gate, { schema: 1, gate, commit: COMMIT, run_id: verification.run_id, run_attempt: verification.run_attempt, script_sha256: scriptDigest(gate, arch), status: 'passed', exit_code: 0, required_skips: [], optional_checks: [], ...(GATES[gate].artifacts ? { artifacts } : {}) }])); + await writeFile(path.join(directory, meta.sums), Object.entries(artifacts).map(([name, sha]) => `${sha} ${name}\n`).join('')); + const manifest = { schema: 1, tag: `v${VERSION}`, commit: COMMIT, target, ...source, verification: validateVerification(verification, COMMIT, artifacts, arch), assets: artifacts }; + await writeFile(path.join(directory, meta.manifest), `${JSON.stringify(manifest, null, 2)}\n`); + await writeFile(path.join(directory, 'VERIFICATION.json'), `${JSON.stringify(verification, null, 2)}\n`); + const envelope = await createEnvelope({ bundleDir: directory, version: VERSION, commit: COMMIT, keyId, keyEpoch: 1, target }); + await writeFile(path.join(directory, meta.auth), envelope); + await writeFile(path.join(directory, meta.signature), signEnvelope(envelope, privateKey)); + } } await rebuild(); - const files = [...packages, ...['SHA256SUMS', 'RELEASE-MANIFEST.json', 'RELEASE-AUTH', 'RELEASE-AUTH.sig'].map(name => path.join(bundle, name))]; + const assetCount = bundles.reduce((count, bundle) => count + bundle.files.length, 0); const state = { release: null, assets: new Map(), calls: [], failUpload: false, apiFailure: false }; const gh = args => { state.calls.push(args); assert(!args.includes('--clobber')); @@ -55,12 +71,13 @@ export async function fixture(run, publish) { assert(!state.assets.has(path.basename(args[3]))); state.assets.set(path.basename(args[3]), state.corruptUpload ? Buffer.from('corrupt upload') : readFileSync(args[3])); break; case 'download': writeFileSync(args[args.indexOf('--output') + 1], state.assets.get(args[args.indexOf('--pattern') + 1])); break; - case 'edit': assert.equal(state.assets.size, files.length); state.release.draft = false; break; + case 'edit': assert.equal(state.assets.size, assetCount); state.release.draft = false; break; default: throw new Error(`Unexpected command: ${args}`); } return ''; }; - const execute = (overrides = {}) => publish({ repo: IDENTITY.repository, tag: `v${VERSION}`, commit: COMMIT, files, verification, trustDir, runId: '123', runAttempt: '2', gh, ...overrides }); - try { await run({ root, bundle, trustDir, packages, files, state, execute, verification, rebuild, privateKey, keyId, source, gh }); } + const signedBundles = bundles.map(({ target, files, verification }) => ({ target, files, verification })); + const execute = (overrides = {}) => publish({ repo: IDENTITY.repository, tag: `v${VERSION}`, commit: COMMIT, ...(bundles.length > 1 ? { bundles: signedBundles } : { files, verification }), trustDir, runId: '123', runAttempt: '2', gh, ...overrides }); + try { await run({ root, bundle, trustDir, packages, files, state, execute, verification, rebuild, privateKey, keyId, source, gh, bundles, signedBundles }); } finally { await rm(root, { recursive: true, force: true }); } } diff --git a/scripts/sign-native-release.mjs b/scripts/sign-native-release.mjs index b9b8f5c3..49540e56 100644 --- a/scripts/sign-native-release.mjs +++ b/scripts/sign-native-release.mjs @@ -3,36 +3,39 @@ import assert from 'node:assert/strict'; import { writeFile } from 'node:fs/promises'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; -import { createEnvelope, signEnvelope, publicKeyIdentity, parseTrustPolicy, LIMITS } from './native-release-auth.mjs'; +import { createEnvelope, signEnvelope, publicKeyIdentity, parseTrustPolicy, metadataNames, IDENTITY, TARGETS, LIMITS } from './native-release-auth.mjs'; import { withReleaseSnapshot, validateContext, validateStagedRelease, readReceipts, requireTransportLayout, retainBundle, releaseNames, readBytes } from './native-release-assets.mjs'; -export async function signRelease({ bundleDir, trustDir, privateKey, context, outputDir }) { +export async function signRelease({ bundleDir, trustDir, privateKey, context, outputDir, target = IDENTITY.target }) { assert(trustDir && privateKey, 'Provisioned publisher trust and private signing key required'); - const version = validateContext(context), files = await requireTransportLayout(bundleDir, version, false); - const verification = await readReceipts(bundleDir); - await withReleaseSnapshot({ files, version, signed: false }, async (directory, observed) => { - await validateStagedRelease({ directory, observed, verification, context, signed: false }); + assert(Object.hasOwn(TARGETS, target), 'Unsupported release target'); + const version = validateContext(context), files = await requireTransportLayout(bundleDir, version, false, target); + const verification = await readReceipts(bundleDir), names = metadataNames(target); + await withReleaseSnapshot({ files, version, signed: false, target }, async (directory, observed) => { + await validateStagedRelease({ directory, observed, verification, context, signed: false, target }); const keyId = publicKeyIdentity(privateKey); // The independently reviewed tooling checkout supplies this bounded public policy. - const policy = await readBytes(path.join(trustDir, 'policy'), LIMITS.trust); - const entry = parseTrustPolicy(policy).keys.find(key => key.id === keyId); + const trust = parseTrustPolicy(await readBytes(path.join(trustDir, 'policy'), LIMITS.trust)); + assert(trust.targets.includes(target), 'Release target is not in the provisioned trust policy'); + const entry = trust.keys.find(key => key.id === keyId); assert(entry, 'Private signing key is not in the provisioned trust policy'); - const bytes = await createEnvelope({ bundleDir: directory, version, commit: context.commit, keyId, keyEpoch: entry.epoch }); - await writeFile(path.join(directory, 'RELEASE-AUTH'), bytes, { flag: 'wx', mode: 0o400 }); - await writeFile(path.join(directory, 'RELEASE-AUTH.sig'), signEnvelope(bytes, privateKey), { flag: 'wx', mode: 0o400 }); + const bytes = await createEnvelope({ bundleDir: directory, version, commit: context.commit, keyId, keyEpoch: entry.epoch, target }); + await writeFile(path.join(directory, names.auth), bytes, { flag: 'wx', mode: 0o400 }); + await writeFile(path.join(directory, names.signature), signEnvelope(bytes, privateKey), { flag: 'wx', mode: 0o400 }); // Applies current floors/key ranges and verifies all three exact package snapshots. - await validateStagedRelease({ directory, observed, verification, context, trustDir, signed: true }); - await retainBundle(directory, outputDir, releaseNames(version), verification); + await validateStagedRelease({ directory, observed, verification, context, trustDir, signed: true, target }); + await retainBundle(directory, outputDir, releaseNames(version, true, target), verification); }); } async function main() { - const [bundleDir, trustDir, outputDir] = process.argv.slice(2); + // An optional fourth argument names the target; x86_64 when omitted. + const [bundleDir, trustDir, outputDir, target = IDENTITY.target] = process.argv.slice(2); const privateKey = process.env.NATIVE_RELEASE_SIGNING_KEY_PEM; delete process.env.NATIVE_RELEASE_SIGNING_KEY_PEM; try { - assert(bundleDir && trustDir && outputDir && process.argv.length === 5); + assert(bundleDir && trustDir && outputDir && (process.argv.length === 5 || process.argv.length === 6)); const context = { repo: process.env.GITHUB_REPOSITORY, tag: process.env.GITHUB_REF_NAME, commit: process.env.GITHUB_SHA, runId: process.env.GITHUB_RUN_ID, runAttempt: process.env.GITHUB_RUN_ATTEMPT }; - await signRelease({ bundleDir, trustDir, privateKey, context, outputDir }); + await signRelease({ bundleDir, trustDir, privateKey, context, outputDir, target }); console.log('Publisher signature created and verified for the required release asset set.'); } catch { // Do not serialize crypto exceptions/arguments from a secret-bearing process. diff --git a/scripts/test-build-apt-repo.mjs b/scripts/test-build-apt-repo.mjs new file mode 100644 index 00000000..19c672e4 --- /dev/null +++ b/scripts/test-build-apt-repo.mjs @@ -0,0 +1,202 @@ +// Signed APT repository builder (scripts/build-apt-repo.mjs), exercised with a +// throwaway GPG key, dummy packages made with dpkg-deb, gpgv and, when +// available, apt-get in a private root. Run: node --test scripts/test-build-apt-repo.mjs +import assert from "node:assert/strict"; +import { execFile, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { gunzipSync } from "node:zlib"; +import { KEYRING_NAME, POOL, buildRepository, sourcesFile } from "./build-apt-repo.mjs"; + +const run = promisify(execFile); +const here = path.dirname(fileURLToPath(import.meta.url)); +const has = (cmd) => spawnSync("sh", ["-c", `command -v ${cmd}`]).status === 0; +const needed = ["gpg", "gpgv", "dpkg-deb"].filter((c) => !has(c)); +const skip = needed.length ? `missing ${needed.join(", ")}` : false; + +async function makeKey(dir, uid) { + const home = path.join(dir, `gnupg-${uid}`); + await mkdir(home, { mode: 0o700 }); + const g = (args) => run("gpg", ["--homedir", home, "--batch", "--pinentry-mode", "loopback", "--passphrase", "", ...args]); + await g(["--quick-gen-key", `${uid} `, "ed25519", "sign", "1d"]); + const { stdout } = await g(["--with-colons", "--list-secret-keys"]); + const fpr = stdout.split("\n").find((l) => l.startsWith("fpr:")).split(":")[9]; + const { stdout: armored } = await g(["--armor", "--export-secret-keys", fpr]); + return { fpr, armored }; +} + +async function makeDeb(dir, { pkg = "shadow-code", version, arch, payload = "x" }) { + const root = path.join(dir, `${pkg}-${version}-${arch}`); + await mkdir(path.join(root, "DEBIAN"), { recursive: true }); + await mkdir(path.join(root, "usr/share/doc", pkg), { recursive: true }); + await writeFile(path.join(root, "usr/share/doc", pkg, "marker"), `${version} ${payload}\n`); + await writeFile( + path.join(root, "DEBIAN/control"), + `Package: ${pkg}\nVersion: ${version}\nArchitecture: ${arch}\nMaintainer: Test \nDescription: dummy\n`, + ); + const deb = path.join(dir, `${pkg}_${version}_${arch}-${payload}.deb`); + await run("dpkg-deb", ["--root-owner-group", "-Zgzip", "--build", root, deb]); + return deb; +} + +async function setup() { + const dir = await mkdtemp(path.join(tmpdir(), "apt-repo-test-")); + const key = await makeKey(dir, "ShadowCode Test Archive"); + return { dir, key }; +} + +test("builds a signed repository that gpgv and the index hashes accept", { skip }, async () => { + const { dir, key } = await setup(); + try { + const debs = [ + await makeDeb(dir, { version: "2.0.0", arch: "amd64" }), + await makeDeb(dir, { version: "2.1.0", arch: "amd64" }), + await makeDeb(dir, { version: "2.1.0", arch: "arm64" }), + ]; + const out = path.join(dir, "repo"); + await buildRepository({ debs, out, fingerprint: key.fpr, armoredKey: key.armored, date: new Date("2026-10-11T00:00:00Z") }); + const dist = path.join(out, "dists/stable"); + const keyring = path.join(out, `${KEYRING_NAME}.gpg`); + await run("gpgv", ["--keyring", keyring, path.join(dist, "InRelease")]); + await run("gpgv", ["--keyring", keyring, path.join(dist, "Release.gpg"), path.join(dist, "Release")]); + + const release = await readFile(path.join(dist, "Release"), "utf8"); + assert.match(release, /^Suite: stable$/m); + assert.match(release, /^Architectures: amd64 arm64$/m); + assert.doesNotMatch(release, /Valid-Until/); + const sha = release.split("SHA256:\n")[1].trim().split("\n"); + assert.equal(sha.length, 6); + for (const line of sha) { + const [hash, size, name] = line.trim().split(/\s+/); + const data = await readFile(path.join(dist, name)); + assert.equal(data.length, Number(size)); + assert.equal(createHash("sha256").update(data).digest("hex"), hash); + } + const amd = gunzipSync(await readFile(path.join(dist, "main/binary-amd64/Packages.gz"))).toString(); + assert.equal((amd.match(/^Package: shadow-code$/gm) || []).length, 2); + for (const m of amd.matchAll(/^Filename: (.*)\nSize: (\d+)\nMD5sum: .*\nSHA256: (.*)$/gm)) { + const data = await readFile(path.join(out, m[1])); + assert.equal(data.length, Number(m[2])); + assert.equal(createHash("sha256").update(data).digest("hex"), m[3]); + } + const arm = await readFile(path.join(dist, "main/binary-arm64/Packages"), "utf8"); + assert.match(arm, /^Filename: pool\/main\/s\/shadow-code\/shadow-code_2\.1\.0_arm64\.deb$/m); + assert.equal(await readFile(path.join(out, "shadowcode.sources"), "utf8"), sourcesFile()); + const files = await readdir(out); + assert.ok(!files.some((f) => /secret|private/i.test(f))); + assert.doesNotMatch(await readFile(path.join(out, `${KEYRING_NAME}.asc`), "utf8"), /PRIVATE/); + + // A tampered index no longer matches the signed Release. + const tampered = path.join(dir, "Release.tampered"); + await writeFile(tampered, release.replace("Suite: stable", "Suite: unstable")); + await assert.rejects(run("gpgv", ["--keyring", keyring, path.join(dist, "Release.gpg"), tampered])); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("refuses a mismatched key, foreign packages and conflicting pool files", { skip }, async () => { + const { dir, key } = await setup(); + try { + const out = path.join(dir, "repo"); + const good = await makeDeb(dir, { version: "2.1.0", arch: "amd64" }); + const other = await makeKey(dir, "Other"); + await assert.rejects( + buildRepository({ debs: [good], out, fingerprint: key.fpr, armoredKey: other.armored }), + /does not match the pinned/, + ); + await assert.rejects(buildRepository({ debs: [good], out, fingerprint: key.fpr.toLowerCase(), armoredKey: key.armored }), /fingerprint/); + await assert.rejects(buildRepository({ debs: [good], out, fingerprint: key.fpr, armoredKey: "" }), /secret key/); + const foreign = await makeDeb(dir, { pkg: "evil", version: "1.0", arch: "amd64" }); + await assert.rejects(buildRepository({ debs: [foreign], out, fingerprint: key.fpr, armoredKey: key.armored }), /is not shadow-code/); + const i386 = await makeDeb(dir, { version: "2.1.0", arch: "i386" }); + await assert.rejects(buildRepository({ debs: [i386], out, fingerprint: key.fpr, armoredKey: key.armored }), /architecture i386/); + + // --previous keeps older pool entries but refuses changed bytes under the same name. + const old = await makeDeb(dir, { version: "2.0.0", arch: "amd64" }); + const first = path.join(dir, "first"); + await buildRepository({ debs: [old], out: first, fingerprint: key.fpr, armoredKey: key.armored }); + const merged = await buildRepository({ debs: [good], out, fingerprint: key.fpr, armoredKey: key.armored, previous: first }); + assert.deepEqual(merged.packages, ["shadow-code_2.0.0_amd64.deb", "shadow-code_2.1.0_amd64.deb"]); + const changed = await makeDeb(dir, { version: "2.0.0", arch: "amd64", payload: "changed" }); + await assert.rejects( + buildRepository({ debs: [changed], out: path.join(dir, "again"), fingerprint: key.fpr, armoredKey: key.armored, previous: first }), + /differs from an existing package/, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("Valid-Until is opt-in and the CLI reads the key from the environment", { skip }, async () => { + const { dir, key } = await setup(); + try { + const deb = await makeDeb(dir, { version: "2.1.0", arch: "arm64" }); + const out = path.join(dir, "repo"); + await run(process.execPath, [path.join(here, "build-apt-repo.mjs"), "--out", out, "--fingerprint", key.fpr, "--valid-days", "30", "--date", "2026-10-11T00:00:00Z", deb], { + env: { ...process.env, APT_REPO_SIGNING_KEY: key.armored }, + }); + const release = await readFile(path.join(out, "dists/stable/Release"), "utf8"); + assert.match(release, /^Date: Sun, 11 Oct 2026 00:00:00 UTC$/m); + assert.match(release, /^Valid-Until: Tue, 10 Nov 2026 00:00:00 UTC$/m); + await assert.rejects( + run(process.execPath, [path.join(here, "build-apt-repo.mjs"), "--out", out, "--fingerprint", key.fpr, deb], { env: { ...process.env, APT_REPO_SIGNING_KEY: "" } }), + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("apt-get in a private root trusts only the signed-by keyring and never downgrades", { skip: skip || (!has("apt-get") && "apt-get missing") }, async () => { + const { dir, key } = await setup(); + try { + const debs = [await makeDeb(dir, { version: "2.0.0", arch: "amd64" }), await makeDeb(dir, { version: "2.1.0", arch: "amd64" })]; + const out = path.join(dir, "repo"); + await buildRepository({ debs, out, fingerprint: key.fpr, armoredKey: key.armored }); + + const root = path.join(dir, "root"); + for (const d of ["etc/apt/sources.list.d", "etc/apt/keyrings", "etc/apt/preferences.d", "etc/apt/apt.conf.d", "var/lib/apt/lists/partial", "var/cache/apt/archives/partial", "var/lib/dpkg"]) { + await mkdir(path.join(root, d), { recursive: true }); + } + await writeFile(path.join(root, "var/lib/dpkg/status"), ""); + await writeFile(path.join(root, "etc/apt/sources.list"), ""); + const keyring = path.join(root, "etc/apt/keyrings", `${KEYRING_NAME}.gpg`); + await writeFile(keyring, await readFile(path.join(out, `${KEYRING_NAME}.gpg`))); + const sources = sourcesFile(`file:${out}`).replace(`/etc/apt/keyrings/${KEYRING_NAME}.gpg`, keyring).replace("amd64 arm64", "amd64"); + await writeFile(path.join(root, "etc/apt/sources.list.d/shadowcode.sources"), sources); + const apt = (args) => + run("apt-get", [ + "-o", `Dir=${root}`, "-o", "Dir::State::status=" + path.join(root, "var/lib/dpkg/status"), + "-o", "APT::Architecture=amd64", "-o", "APT::Architectures::=amd64", + "-o", "Debug::NoLocking=1", "-o", "APT::Sandbox::User=root", + "-o", "Dir::Etc::trusted=/dev/null", "-o", "Dir::Etc::trustedparts=" + path.join(root, "etc/apt/trusted.gpg.d"), + ...args, + ]); + const { stdout: update, stderr } = await apt(["update"]); + assert.doesNotMatch(update + stderr, /NO_PUBKEY|not signed|insecure|^W:|^E:/m); + const { stdout: sim } = await apt(["--simulate", "install", "shadow-code"]); + assert.match(sim, /Inst shadow-code \(2\.1\.0 /); + // Downgrades are refused unless the administrator asks for one explicitly. + await writeFile(path.join(root, "var/lib/dpkg/status"), "Package: shadow-code\nStatus: install ok installed\nVersion: 2.1.0\nArchitecture: amd64\nMaintainer: T \nDescription: dummy\n"); + const { stdout: up } = await apt(["--simulate", "upgrade"]); + assert.doesNotMatch(up, /Inst shadow-code/); + await assert.rejects(apt(["--simulate", "-y", "--no-allow-downgrades", "install", "shadow-code=2.0.0"])); + + // A Release signed by another key is rejected. + const other = await makeKey(dir, "Other"); + await buildRepository({ debs, out, fingerprint: other.fpr, armoredKey: other.armored }); + await rm(path.join(root, "var/lib/apt/lists"), { recursive: true }); + await mkdir(path.join(root, "var/lib/apt/lists/partial"), { recursive: true }); + const res = await apt(["update"]).then((r) => r.stdout + r.stderr, (e) => e.stdout + e.stderr); + assert.match(res, /NO_PUBKEY|not signed|signature/i); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +assert.equal(POOL, "pool/main/s/shadow-code"); diff --git a/scripts/test-flatpak-package.mjs b/scripts/test-flatpak-package.mjs index 073faddd..3f0ce794 100644 --- a/scripts/test-flatpak-package.mjs +++ b/scripts/test-flatpak-package.mjs @@ -58,7 +58,7 @@ await writeFile(path.join(profile, "config/config.yaml"), JSON.stringify({ })); let owner, lease; try { - assert.equal(await cli(["--version"]), "ShadowCode 2.0.0"); + assert.equal(await cli(["--version"]), "ShadowCode 2.1.0"); assert.equal((await cli(["health"])).runtime, "rust"); await cli(["trust"]); const exec = await cli(["exec", "printf host-command > host.txt; git --version; node --version; readlink /proc/self/ns/net"]); @@ -98,7 +98,7 @@ try { } const view = (await exchange("POST", "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/api/views", null, null, true)).view; const about = await exchange("GET", "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/api/about", null, view); - assert.equal(about.version, "2.0.0"); assert.equal(about.publisher, "Shadowfetch"); + assert.equal(about.version, "2.1.0"); assert.equal(about.publisher, "Shadowfetch"); assert.equal(about.install.kind, "flatpak"); assert.equal(about.links.website, "https://www.shadowfetchlinux.org/shadowcode"); assert.equal(about.links.publisher, "https://shadowfetch.com"); @@ -136,7 +136,7 @@ try { return result.stdout === "dead"; }, 5000); checks.push("persistent engine, attached view, real host PTY, shell job control, resize, Ctrl-C and descendant cleanup"); - await writeFile(path.join(output, "package-result.json"), JSON.stringify({ version: "2.0.0", profile, project, checks, terminal, terminal_child_pid: terminalChild, host_network: hostNetwork }, null, 2) + "\n"); + await writeFile(path.join(output, "package-result.json"), JSON.stringify({ version: "2.1.0", profile, project, checks, terminal, terminal_child_pid: terminalChild, host_network: hostNetwork }, null, 2) + "\n"); console.log(JSON.stringify(checks, null, 2)); } finally { if (lease) lease.destroy(); diff --git a/scripts/test-install-appimage.sh b/scripts/test-install-appimage.sh index 94833e74..53daab16 100755 --- a/scripts/test-install-appimage.sh +++ b/scripts/test-install-appimage.sh @@ -4,10 +4,18 @@ # links, refusal of broken runtimes, rollback, and preserved profile data. # Never touches the real HOME. set -euo pipefail +trap 'echo "test-install-appimage.sh failed at line $LINENO" >&2' ERR SOURCE_ROOT="$(cd "$(dirname "$0")/.." && pwd)" SCRATCH="$(mktemp -d)" cleanup() { chmod -R u+w -- "$SCRATCH" 2>/dev/null || true; rm -rf -- "$SCRATCH"; } trap cleanup EXIT +# Pin the host to x86_64 so the suite means the same on any runner; the +# aarch64 scenario puts its own shim ahead of this one. +mkdir -p "$SCRATCH/x86-uname" +printf '#!/bin/sh\nif [ "${1:-}" = -m ]; then echo x86_64; else exec %s "$@"; fi\n' "$(command -v uname)" > "$SCRATCH/x86-uname/uname" +chmod 755 "$SCRATCH/x86-uname/uname" +PATH="$SCRATCH/x86-uname:$PATH" +export PATH # Self-contained disposable trusted installer bundle; never provision the checkout. ROOT="$SCRATCH/installer" mkdir -p "$ROOT/scripts" "$ROOT/assets/icons" "$ROOT/packaging" @@ -26,7 +34,10 @@ REAL_BIN="$SOURCE_ROOT/packaging/llama.cpp/bin" HOME="$SCRATCH/home" export HOME export XDG_DATA_HOME="$HOME/.local/share" -unset SHADOWCODE_SHA256SUMS SHADOWCODE_GIT_SHA +unset SHADOWCODE_SHA256SUMS SHADOWCODE_GIT_SHA XDG_STATE_HOME SHADOWCODE_FIRST_START_TIMEOUT +# Scenarios 1-19 run without a display, so the first-window check is skipped; +# scenarios 20 onward require it. +unset DISPLAY WAYLAND_DISPLAY SHADOWCODE_FIRST_START_CHECK COMMIT_A="$(printf 'a1%.0s' {1..20})" COMMIT_B="$(printf 'b2%.0s' {1..20})" LIB="$HOME/.local/lib/shadowcode" @@ -38,8 +49,11 @@ printf 'old runtime' > "$LIB/old-runtime-marker" # fake_appimage FILE VERSION COMMIT MODE # MODE: good | absolute-link | dangling-link | broken-server | wrong-commit | no-runtime +# START (first window, optional): ok | hang | crash | wrong-version +# The fake profile database is a text file; the real VACUUM INTO snapshot is +# covered by the native core tests. fake_appimage() { - local file="$1" version="$2" commit="$3" mode="$4" + local file="$1" version="$2" commit="$3" mode="$4" start="${5:-ok}" mkdir -p "$(dirname "$file")" cat > "$file" <"\$profile/native.lock" + flock -n 9 + # The new version migrates the database before its window loads. + printf 'format of $version\\n' > "\$profile/shadow-agent.db" + printf 'wal of $version\\n' > "\$profile/shadow-agent.db-wal" + printf '%s\\n' "\$\$" > "\$profile/fake-app.pid" + case "$start" in + ok) printf 'ShadowCode $version\\n' > "\$SHADOWCODE_FIRST_START_MARKER.tmp" + mv "\$SHADOWCODE_FIRST_START_MARKER.tmp" "\$SHADOWCODE_FIRST_START_MARKER" ;; + wrong-version) printf 'ShadowCode 0.0.1\\n' > "\$SHADOWCODE_FIRST_START_MARKER"; sleep 600 ;; + hang) sleep 600 ;; + crash) echo 'fake window failed' >&2; exit 1 ;; + esac + exit 0 +fi if [[ "\${1:-}" == "--appimage-extract" && "\${2:-}" == "usr/lib/shadowcode" ]]; then [[ "$mode" == "no-runtime" ]] && exit 0 dir=squashfs-root/usr/lib/shadowcode @@ -463,4 +499,148 @@ cp -p "$SCRATCH/prior-desktop" "$XDG_DATA_HOME/applications/shadow-agent.desktop grep -Fq 'Recovered the previous runtime and AppImage' "$SCRATCH/recovered-after-link.txt" assert_state_unchanged "$PRIOR_LINK" "$PRIOR_COMMIT" test "$(cat "$XDG_DATA_HOME/shadow-agent/profile.txt")" = 'keep this profile data' + +# 20. First-window check: a new version that never reports its first start is +# stopped, and the AppImage, runtime, launchers and pre-activation database +# snapshot are all restored. The accepted-release record is not lowered. +PROFILE_STATE="$HOME/.local/state/shadow-agent" +DB="$PROFILE_STATE/shadow-agent.db" +mkdir -p "$PROFILE_STATE" +printf 'format of the prior version\n' > "$DB" +first_start_env=(env SHADOWCODE_FIRST_START_CHECK=require SHADOWCODE_FIRST_START_TIMEOUT=2) +assert_database_restored() { + test "$(cat "$DB")" = 'format of the prior version' + test ! -e "$DB-wal" + test ! -e "$DB.rollback.pending" +} +ACCEPTED_BEFORE="$(readlink "$HOME/.local/lib/.shadowcode-release-state/accepted")" +HANGS="$SCRATCH/release/0.29.0/ShadowCode_0.29.0_amd64.AppImage" +fake_appimage "$HANGS" 0.29.0 "$COMMIT_B" good hang +checksum "$HANGS" +expect_refusal 'first window never reported' "${first_start_env[@]}" "$INSTALLER" "$HANGS" +grep -Fq 'did not report its first window within 2s' "$SCRATCH/refused.txt" +grep -Fq 'database snapshot taken before activation was restored' "$SCRATCH/refused.txt" +grep -Fq 'previous runtime and AppImage were restored' "$SCRATCH/refused.txt" +assert_database_restored +assert_state_unchanged "$PRIOR_LINK" "$PRIOR_COMMIT" +test ! -e "$HOME/Applications/ShadowCode-0.29.0-x86_64.AppImage" +flock -n "$PROFILE_STATE/native.lock" true +# Rollback keeps the durable highest accepted release: 0.28.5 is now a downgrade. +test "$(readlink "$HOME/.local/lib/.shadowcode-release-state/accepted")" != "$ACCEPTED_BEFORE" +expect_refusal 'downgrade below a rolled-back release' "$INSTALLER" "$NEXT" +grep -Fq 'release downgrade' "$SCRATCH/refused.txt" + +# 21. A version that exits before its window, or reports another version, +# is rolled back the same way. +for case_version in crash:0.29.1 wrong-version:0.29.2; do + start=${case_version%%:*}; version=${case_version#*:} + rm -f "$SCRATCH/release/SHA256SUMS" + FAILS="$SCRATCH/release/$version/ShadowCode_${version}_amd64.AppImage" + fake_appimage "$FAILS" "$version" "$COMMIT_B" good "$start" + checksum "$FAILS" + expect_refusal "first start $start" "${first_start_env[@]}" "$INSTALLER" "$FAILS" + grep -Fq 'database snapshot taken before activation was restored' "$SCRATCH/refused.txt" + assert_database_restored + assert_state_unchanged "$PRIOR_LINK" "$PRIOR_COMMIT" +done +grep -Fq 'not from the installed version' "$SCRATCH/refused.txt" + +# 22. The check refuses to start while ShadowCode holds the profile lock; +# nothing is replaced and the database is not touched. +GOOD="$SCRATCH/release/0.29.3/ShadowCode_0.29.3_amd64.AppImage" +fake_appimage "$GOOD" 0.29.3 "$COMMIT_A" good ok +checksum "$GOOD" +exec {HOLDER}>"$PROFILE_STATE/native.lock"; flock -n "$HOLDER" +expect_refusal 'profile in use' "${first_start_env[@]}" "$INSTALLER" "$GOOD" +exec {HOLDER}>&- +grep -Fq 'ShadowCode is running on this profile' "$SCRATCH/refused.txt" +assert_database_restored +assert_state_unchanged "$PRIOR_LINK" "$PRIOR_COMMIT" + +# 23. A healthy first start completes the install and keeps the migrated data. +"${first_start_env[@]}" "$INSTALLER" "$GOOD" > "$SCRATCH/first-start-ok.txt" +grep -Fq 'ShadowCode 0.29.3 opened its first window' "$SCRATCH/first-start-ok.txt" +assert_state_unchanged ShadowCode-0.29.3-x86_64.AppImage "$COMMIT_A" +test "$(cat "$DB")" = 'format of 0.29.3' +PRIOR_LINK=ShadowCode-0.29.3-x86_64.AppImage +PRIOR_COMMIT="$COMMIT_A" +rm -f "$DB-wal" +printf 'format of the prior version\n' > "$DB" + +# 24. An installer killed during the check leaves a first_start_check intent. +# Recovery waits until the started version is closed, then restores the +# package and the database snapshot together. +KILLED="$SCRATCH/release/0.29.4/ShadowCode_0.29.4_amd64.AppImage" +fake_appimage "$KILLED" 0.29.4 "$COMMIT_B" good hang +checksum "$KILLED" +rm -f "$PROFILE_STATE/fake-app.pid" +SHADOWCODE_FIRST_START_CHECK=require SHADOWCODE_FIRST_START_TIMEOUT=60 "$INSTALLER" "$KILLED" > "$SCRATCH/killed-check.txt" 2>&1 & +KILLED_INSTALLER=$! +for _ in $(seq 100); do [[ -s "$PROFILE_STATE/fake-app.pid" ]] && break; sleep 0.1; done +test "$(cat "$JOURNAL/phase")" = first_start_check +kill -KILL "$KILLED_INSTALLER"; wait "$KILLED_INSTALLER" 2>/dev/null || true +# Its short-lived children (a polling sleep) may hold the installer lock briefly. +for _ in $(seq 50); do flock -n "$HOME/.local/lib/.shadowcode-install-lock" true && break; sleep 0.1; done +expect_refusal 'recover while the new version runs' "$INSTALLER" --recover +grep -Fq 'ShadowCode is running on this profile' "$SCRATCH/refused.txt" +test -d "$JOURNAL" +kill -TERM -- "-$(cat "$PROFILE_STATE/fake-app.pid")" +for _ in $(seq 50); do flock -n "$PROFILE_STATE/native.lock" true && break; sleep 0.1; done +"$INSTALLER" --recover > "$SCRATCH/recovered-check.txt" +grep -Fq 'Recovered the previous runtime and AppImage' "$SCRATCH/recovered-check.txt" +grep -Fxq "X-ShadowCode-Version=0.29.3" "$DESKTOP" +assert_database_restored +assert_state_unchanged "$PRIOR_LINK" "$PRIOR_COMMIT" + +# 25. An aarch64 host installs only the signed aarch64 AppImage, under +# -aarch64 names, and an x86_64 host refuses it. A uname shim selects the +# host; the fake AppImages are architecture-neutral scripts. +ARM_ROOT="$SCRATCH/installer-arm" +mkdir -p "$ARM_ROOT" +cp -a "$ROOT/scripts" "$ROOT/assets" "$ROOT/packaging" "$ARM_ROOT/" +ARM_KEY="$SCRATCH/fixture-arm-private.pem" +node "$FIXTURE_TOOL" provision "$ARM_ROOT" "$ARM_KEY" x86_64-unknown-linux-gnu,aarch64-unknown-linux-gnu +ARM_INSTALLER="$ARM_ROOT/scripts/install-appimage.sh" +ARM_SHIM="$SCRATCH/arm-uname" +mkdir -p "$ARM_SHIM" +printf '#!/bin/sh\nif [ "${1:-}" = -m ]; then echo aarch64; else exec %s "$@"; fi\n' "$(command -v uname)" > "$ARM_SHIM/uname" +chmod 755 "$ARM_SHIM/uname" +ARM_HOME="$SCRATCH/arm-home" +mkdir -p "$ARM_HOME/Applications" "$ARM_HOME/.local/share/shadow-agent" +printf 'keep this profile data' > "$ARM_HOME/.local/share/shadow-agent/profile.txt" +arm_install() { env HOME="$ARM_HOME" XDG_DATA_HOME="$ARM_HOME/.local/share" PATH="$ARM_SHIM:$PATH" "$ARM_INSTALLER" "$@"; } +ARM_APP="$SCRATCH/arm-release/0.29.0/ShadowCode_0.29.0_aarch64.AppImage" +fake_appimage "$ARM_APP" 0.29.0 "$COMMIT_A" good +node "$FIXTURE_TOOL" sign "$ARM_APP" "$ARM_KEY" +test -f "$SCRATCH/arm-release/0.29.0/ShadowCode_0.29.0_arm64.deb" +test -f "$SCRATCH/arm-release/0.29.0/ShadowCode_0.29.0_aarch64_appimage-runtime-sources.tar.gz" +# The signed x86_64 AppImage is not installable on aarch64. +X86_APP="$SCRATCH/arm-release/x86/ShadowCode_0.29.0_amd64.AppImage" +fake_appimage "$X86_APP" 0.29.0 "$COMMIT_A" good +node "$FIXTURE_TOOL" sign "$X86_APP" "$ARM_KEY" +expect_refusal 'x86_64 AppImage on aarch64' arm_install "$X86_APP" +grep -Fq 'Only the authenticated AppImage role' "$SCRATCH/refused.txt" +grep -Fq 'ShadowCode_VERSION_aarch64.AppImage' "$SCRATCH/refused.txt" +test -z "$(ls -A "$ARM_HOME/Applications")" +# An aarch64 AppImage is not installable on x86_64. +expect_refusal 'aarch64 AppImage on x86_64' env HOME="$ARM_HOME" XDG_DATA_HOME="$ARM_HOME/.local/share" "$ARM_INSTALLER" "$ARM_APP" +grep -Fq 'Only the authenticated AppImage role' "$SCRATCH/refused.txt" +test -z "$(ls -A "$ARM_HOME/Applications")" +# A file renamed to the aarch64 role is not an authenticated asset of the x86_64 release. +cp -p "$X86_APP" "$SCRATCH/arm-release/x86/ShadowCode_0.29.0_aarch64.AppImage" +expect_refusal 'renamed x86_64 AppImage on aarch64' arm_install "$SCRATCH/arm-release/x86/ShadowCode_0.29.0_aarch64.AppImage" +test -z "$(ls -A "$ARM_HOME/Applications")" +arm_install "$ARM_APP" > "$SCRATCH/arm-installed.txt" +grep -Fq "Installed $ARM_HOME/Applications/ShadowCode-0.29.0-aarch64.AppImage" "$SCRATCH/arm-installed.txt" +test "$(readlink "$ARM_HOME/Applications/ShadowCode.AppImage")" = ShadowCode-0.29.0-aarch64.AppImage +grep -Fxq "commit=$COMMIT_A" "$ARM_HOME/.local/lib/shadowcode/COMMIT" +test "$(cat "$ARM_HOME/.local/share/shadow-agent/profile.txt")" = 'keep this profile data' +# Upgrades keep the aarch64 identity and remove the older aarch64 AppImage. +ARM_NEXT="$SCRATCH/arm-release/0.29.1/ShadowCode_0.29.1_aarch64.AppImage" +fake_appimage "$ARM_NEXT" 0.29.1 "$COMMIT_B" good +node "$FIXTURE_TOOL" sign "$ARM_NEXT" "$ARM_KEY" +arm_install "$ARM_NEXT" > /dev/null +test "$(readlink "$ARM_HOME/Applications/ShadowCode.AppImage")" = ShadowCode-0.29.1-aarch64.AppImage +test ! -e "$ARM_HOME/Applications/ShadowCode-0.29.0-aarch64.AppImage" +grep -Fxq "commit=$COMMIT_B" "$ARM_HOME/.local/lib/shadowcode/COMMIT" printf 'AppImage installer checks passed.\n' diff --git a/scripts/test-install-auth.mjs b/scripts/test-install-auth.mjs index 36915e7a..75fa1a68 100644 --- a/scripts/test-install-auth.mjs +++ b/scripts/test-install-auth.mjs @@ -26,14 +26,21 @@ async function fixture(t) { const libraryParent = path.join(home, '.local/lib'), library = path.join(libraryParent, 'shadowcode'); const marker = path.join(root, 'candidate-executed'); const installer = path.join(bundle, 'scripts/install-appimage.sh'); - const env = { ...process.env, HOME: home, XDG_DATA_HOME: path.join(home, '.local/share') }; + // Pin the host to x86_64 so the suite means the same on any runner; + // armHost() puts its aarch64 shim ahead of this one. + const x86 = path.join(root, 'x86-uname'); await mkdir(x86); + await writeFile(path.join(x86, 'uname'), `#!/bin/sh\nif [ "\${1:-}" = -m ]; then echo x86_64; else exec /usr/bin/uname "$@"; fi\n`, { mode: 0o755 }); + const env = { ...process.env, PATH: `${x86}:${process.env.PATH}`, HOME: home, XDG_DATA_HOME: path.join(home, '.local/share') }; delete env.SHADOWCODE_SHA256SUMS; delete env.SHADOWCODE_GIT_SHA; + // A developer desktop's display would turn on the first-window check, which + // these fixture AppImages do not implement (as test-install-appimage.sh). + delete env.DISPLAY; delete env.WAYLAND_DISPLAY; delete env.SHADOWCODE_FIRST_START_CHECK; const run = (args, extra = {}) => spawnSync('/bin/bash', [installer, ...args], { env: { ...env, ...extra }, encoding: 'utf8', timeout: 20000 }); return { root, bundle, home, key, library, libraryParent, marker, installer, env, run, state: path.join(libraryParent, '.shadowcode-release-state'), journal: path.join(libraryParent, '.shadowcode-install-intent') }; } -async function candidate(f, version = '0.28.0', suffix = '') { +async function candidate(f, version = '0.28.0', suffix = '', arch = 'amd64') { const directory = path.join(f.root, `release-${version}${suffix}`); await mkdir(directory, { recursive: true }); - const file = path.join(directory, `ShadowCode_${version}_amd64.AppImage`); + const file = path.join(directory, `ShadowCode_${version}_${arch}.AppImage`); await writeFile(file, `#!/usr/bin/env bash set -euo pipefail printf '%s\\n' "$0" >> '${f.marker}' @@ -128,7 +135,7 @@ set -euo pipefail if [[ "$AUTH_FAULT" == accepted-sync && "\${@: -1}" == "$HOME/.local/lib/.shadowcode-release-state" ]]; then echo 'Injected accepted state flush failure' >&2; exit 75; fi exec /usr/bin/sync "$@" `, { mode: 0o755 }); - return { PATH: `${bin}:${process.env.PATH}`, AUTH_FAULT_MODE: 'kill' }; + return { PATH: `${bin}:${f.env.PATH}`, AUTH_FAULT_MODE: 'kill' }; } test('rejects jointly replaced checksums and candidate before any executable runs', async t => { @@ -285,7 +292,7 @@ if [[ "\${1:-}" == 500 && "\${2:-}" == */verified-release/*.AppImage ]]; then fi exec /usr/bin/chmod "$@" `, { mode: 0o755 }); - passed(f.run([file], { PATH: `${bin}:${process.env.PATH}`, AUTH_MUTATE_DOWNLOAD: file })); + passed(f.run([file], { PATH: `${bin}:${f.env.PATH}`, AUTH_MUTATE_DOWNLOAD: file })); assert.notDeepEqual(await readFile(file), original); assert.deepEqual(await readFile(path.join(f.home, 'Applications/ShadowCode-0.28.0-x86_64.AppImage')), original); for (const executed of (await readFile(f.marker, 'utf8')).trim().split('\n')) assert(executed.includes('/verified-release/')); @@ -367,7 +374,7 @@ test('current publisher handoff and signer produce an installable authenticated // execution or real package qualification is claimed by this fixture. const verification = { schema: 1, commit: COMMIT, run_id: context.runId, run_attempt: context.runAttempt, artifacts, gates: Object.fromEntries(REQUIRED_GATES.map(gate => [gate, { schema: 1, gate, commit: COMMIT, run_id: context.runId, run_attempt: context.runAttempt, - script_sha256: scriptDigest(gate), status: 'passed', exit_code: 0, required_skips: [], optional_checks: [], ...(GATES[gate].artifacts ? { artifacts } : {}), + script_sha256: scriptDigest(gate, 'x86_64'), status: 'passed', exit_code: 0, required_skips: [], optional_checks: [], ...(GATES[gate].artifacts ? { artifacts } : {}), }])), }; const unsigned = path.join(f.root, 'unsigned-handoff'), signed = path.join(f.root, 'signed-handoff'); @@ -379,6 +386,34 @@ test('current publisher handoff and signer produce an installable authenticated assert.deepEqual(await readFile(path.join(f.home, 'Applications/ShadowCode-0.28.0-x86_64.AppImage')), await readFile(file)); }); +test('an aarch64 handoff signed on an x86_64 signer installs on an aarch64 host', async t => { + const f = await fixture(t), env = await armHost(f), version = '0.28.0', target = 'aarch64-unknown-linux-gnu'; + await provision(f.bundle, f.key, [IDENTITY.target, target]); + const file = await candidate(f, version, '', 'aarch64'); + const packages = assetNames(version, target).map(([, name]) => path.join(path.dirname(file), name)); + const artifacts = Object.fromEntries(await Promise.all(packages.map(async file => [path.basename(file), digestBytes(await readFile(file))]))); + const context = { repo: IDENTITY.repository, tag: `v${version}`, commit: COMMIT, runId: '992', runAttempt: '1' }; + // Receipts from the aarch64 runner carry the aarch64 gate scripts' digests. + const receipts = arch => ({ schema: 1, commit: COMMIT, run_id: context.runId, run_attempt: context.runAttempt, artifacts, + gates: Object.fromEntries(REQUIRED_GATES.map(gate => [gate, { schema: 1, gate, commit: COMMIT, run_id: context.runId, run_attempt: context.runAttempt, + script_sha256: scriptDigest(gate, arch), status: 'passed', exit_code: 0, required_skips: [], optional_checks: [], ...(GATES[gate].artifacts ? { artifacts } : {}), + }])), + }); + const source = { cargo_lock_sha256: 'd'.repeat(64), ui_lock_sha256: 'e'.repeat(64), runtime_pin: `commit=${'f'.repeat(40)}` }; + const checksums = path.join(path.dirname(file), 'SHA256SUMS-aarch64'); + // x86_64 gate digests do not qualify aarch64 packages. + await assert.rejects(prepareRelease({ files: packages, checksums, source, verification: receipts('x86_64'), context, outputDir: path.join(f.root, 'wrong-arch'), target }), /Verification command changed/); + const unsigned = path.join(f.root, 'unsigned-aarch64'), signed = path.join(f.root, 'signed-aarch64'); + await prepareRelease({ files: packages, checksums, source, verification: receipts('aarch64'), context, outputDir: unsigned, target }); + await signRelease({ bundleDir: unsigned, trustDir: path.join(f.bundle, 'release/trust'), privateKey: await readFile(f.key), context, outputDir: signed, target }); + assert.deepEqual((await readdir(signed)).sort(), [...assetNames(version, target).map(([, name]) => name), 'RELEASE-AUTH-aarch64', 'RELEASE-AUTH-aarch64.sig', 'RELEASE-MANIFEST-aarch64.json', 'SHA256SUMS-aarch64', 'VERIFICATION.json'].sort()); + passed(f.run([path.join(signed, path.basename(file))], env)); + assert.equal(await readlink(path.join(f.home, 'Applications/ShadowCode.AppImage')), 'ShadowCode-0.28.0-aarch64.AppImage'); + // The accepted-release receipt keeps the historical metadata names. + const receipt = await accepted(f); + assert.deepEqual(await readFile(path.join(f.state, receipt, 'RELEASE-AUTH')), await readFile(path.join(signed, 'RELEASE-AUTH-aarch64'))); +}); + test('candidate-adjacent and environment-provided trust cannot replace the installer bundle policy', async t => { const f = await fixture(t), file = await candidate(f), adjacent = path.dirname(file), otherKey = path.join(f.root, 'adjacent-fixture-private.pem'); @@ -412,3 +447,32 @@ for (const role of ['deb', 'runtime-sources']) test(`signed ${role} asset never const result = f.run([target]); refused(result); assert.match(result.stderr, /Only the authenticated AppImage role/); await missing(f.marker); await missing(f.state); }); + +// aarch64 hosts: a uname shim selects the host; fixture AppImages are scripts. +async function armHost(f) { + const bin = path.join(f.root, 'arm-uname'); await mkdir(bin); + await writeFile(path.join(bin, 'uname'), `#!/bin/sh\nif [ "\${1:-}" = -m ]; then echo aarch64; else exec /usr/bin/uname "$@"; fi\n`, { mode: 0o755 }); + return { PATH: `${bin}:${f.env.PATH}` }; +} +test('aarch64 host refuses a signed aarch64 release while the trust policy names only x86_64', async t => { + const f = await fixture(t), env = await armHost(f), app = await candidate(f, '0.28.0', '', 'aarch64'); + const result = f.run([app], env); refused(result); assert.match(result.stderr, /release target is not trusted/); + await missing(f.state); await missing(path.join(f.home, 'Applications/ShadowCode.AppImage')); + assert.equal((await readFile(f.marker, 'utf8').catch(() => '')), '', 'Unauthenticated candidate executed'); +}); +test('aarch64 host installs the signed aarch64 AppImage under aarch64 names and refuses x86_64', async t => { + const f = await fixture(t), env = await armHost(f); + await provision(f.bundle, f.key, [IDENTITY.target, 'aarch64-unknown-linux-gnu']); + const x86 = await candidate(f, '0.28.0', '-x86'); + let result = f.run([x86], env); refused(result); assert.match(result.stderr, /Only the authenticated AppImage role/); + await missing(f.marker); + const arm = await candidate(f, '0.28.0', '', 'aarch64'); + result = f.run([arm]); refused(result); assert.match(result.stderr, /Only the authenticated AppImage role/); + result = f.run([arm], env); passed(result); + assert.equal(await readlink(path.join(f.home, 'Applications/ShadowCode.AppImage')), 'ShadowCode-0.28.0-aarch64.AppImage'); + await missing(path.join(f.home, 'Applications/ShadowCode-0.28.0-x86_64.AppImage')); + const next = await candidate(f, '0.28.1', '', 'aarch64'); + result = f.run([next], env); passed(result); + assert.equal(await readlink(path.join(f.home, 'Applications/ShadowCode.AppImage')), 'ShadowCode-0.28.1-aarch64.AppImage'); + await missing(path.join(f.home, 'Applications/ShadowCode-0.28.0-aarch64.AppImage')); +}); diff --git a/scripts/test-native-arch.mjs b/scripts/test-native-arch.mjs new file mode 100644 index 00000000..1fcfbddd --- /dev/null +++ b/scripts/test-native-arch.mjs @@ -0,0 +1,89 @@ +// Architecture names used by the AppImage/Debian/Flatpak packaging scripts. +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { ARCHES, archInfo, hostArch, packageNames } from "./native-arch.mjs"; +import { TARGETS, assetNames } from "./native-release-auth.mjs"; + +test("x86_64 package names are the historical ones", () => { + assert.deepEqual(packageNames("2.1.0", "x86_64"), { + appimage: "ShadowCode_2.1.0_amd64.AppImage", + deb: "ShadowCode_2.1.0_amd64.deb", + runtimeSources: "ShadowCode_2.1.0_appimage-runtime-sources.tar.gz", + flatpak: "ShadowCode_2.1.0_x86_64.flatpak", + }); + assert.equal(ARCHES.x86_64.docker, "linux/amd64"); + assert.equal(ARCHES.x86_64.loader, "ld-linux-x86-64.so.2"); +}); + +test("aarch64 package names follow Tauri, Debian and Flatpak", () => { + assert.deepEqual(packageNames("2.1.0", "aarch64"), { + appimage: "ShadowCode_2.1.0_aarch64.AppImage", + deb: "ShadowCode_2.1.0_arm64.deb", + runtimeSources: "ShadowCode_2.1.0_aarch64_appimage-runtime-sources.tar.gz", + flatpak: "ShadowCode_2.1.0_aarch64.flatpak", + }); + assert.equal(ARCHES.aarch64.docker, "linux/arm64"); + assert.equal(ARCHES.aarch64.multiarch, "aarch64-linux-gnu"); + assert.equal(ARCHES.aarch64.loader, "ld-linux-aarch64.so.1"); +}); + +test("aliases resolve and unsupported architectures are refused", () => { + for (const name of ["x86_64", "amd64", "x64"]) assert.equal(archInfo(name).arch, "x86_64"); + for (const name of ["aarch64", "arm64"]) assert.equal(archInfo(name).arch, "aarch64"); + for (const name of ["riscv64", "armhf", "i686", ""]) assert.throws(() => archInfo(name), /Unsupported architecture/); +}); + +test("the host decides; SHADOWCODE_ARCH cannot request cross packaging", () => { + assert.equal(hostArch({}, "x64").arch, "x86_64"); + assert.equal(hostArch({}, "arm64").arch, "aarch64"); + assert.equal(hostArch({ SHADOWCODE_ARCH: "amd64" }, "x64").arch, "x86_64"); + assert.equal(hostArch({ SHADOWCODE_ARCH: "arm64" }, "arm64").arch, "aarch64"); + assert.throws(() => hostArch({ SHADOWCODE_ARCH: "aarch64" }, "x64"), /no cross packaging/); + assert.throws(() => hostArch({}, "riscv64"), /x86_64 and aarch64/); +}); + +test("signed release asset names agree with the packaging names", () => { + for (const info of Object.values(ARCHES)) { + assert.ok(Object.hasOwn(TARGETS, info.target), info.target); + const names = packageNames("2.1.0", info.arch); + assert.deepEqual(assetNames("2.1.0", info.target), [ + ["appimage", names.appimage], + ["deb", names.deb], + ["runtime-sources", names.runtimeSources], + ]); + } +}); + +test("the installer and shell verifier name the same targets and packages", async () => { + const installer = await readFile(new URL("./install-appimage.sh", import.meta.url), "utf8"); + const library = await readFile(new URL("./native-release-auth-lib.sh", import.meta.url), "utf8"); + for (const info of Object.values(ARCHES)) { + assert.match(installer, new RegExp(`HOST_TARGET=${info.target} PACKAGE_ARCH=${info.appimage} INSTALLED_ARCH=${info.installed} `)); + assert.match(library, new RegExp(`${info.target}\\) TARGET_APPIMAGE=${info.appimage} TARGET_DEB=${info.deb} TARGET_SOURCES=${info.runtimeSources} `)); + } +}); + +test("the AppImage runtime build and launcher handle both architectures", async () => { + const build = await readFile(new URL("../packaging/native-runtime/build.sh", import.meta.url), "utf8"); + assert.match(build, /x86_64\|aarch64\) ;;/); + assert.doesNotMatch(build, /out\/runtime-x86_64/); + const apprun = await readFile(new URL("../packaging/native-app-run.sh", import.meta.url), "utf8"); + assert.match(apprun, /shadowcode_multiarch=x86_64-linux-gnu/); + assert.match(apprun, /usr\/lib\/aarch64-linux-gnu/); + const docker = await readFile(new URL("./clean-host-runtime.Dockerfile", import.meta.url), "utf8"); + assert.match(docker, /FROM --platform=linux\/amd64 debian@sha256:[a-f0-9]{64} AS debian-amd64/); + assert.match(docker, /FROM --platform=linux\/arm64 debian@sha256:[a-f0-9]{64} AS debian-arm64/); + // BuildKit's own TARGETARCH must select the stage; a default would override it. + assert.match(docker, /^ARG TARGETARCH$/m); + assert.doesNotMatch(docker, /ARG TARGETARCH=/); + // Docker's classic builder sets no TARGETARCH, so the image's build + // commands pass the host architecture themselves. + const gates = await readFile(new URL("./native-release-verification.mjs", import.meta.url), "utf8"); + const ci = await readFile(new URL("../.github/workflows/native.yml", import.meta.url), "utf8"); + for (const text of [gates, ci]) { + const builds = text.match(/docker build [^\n]*clean-host-runtime\.Dockerfile[^\n]*/g) ?? []; + assert.ok(builds.length > 0); + for (const line of builds) assert.match(line, /--build-arg TARGETARCH="\$\(dpkg --print-architecture\)"/); + } +}); diff --git a/scripts/test-native-deb.mjs b/scripts/test-native-deb.mjs index 4aac62a4..64e89d57 100644 --- a/scripts/test-native-deb.mjs +++ b/scripts/test-native-deb.mjs @@ -44,6 +44,7 @@ import { wrapDescription, } from "./native-deb.mjs"; import { DESKTOP_FILE, METAINFO_FILE } from "./native-desktop-metadata.mjs"; +import { hostArch } from "./native-arch.mjs"; const run = promisify(execFile); const root = fileURLToPath(new URL("../", import.meta.url)); @@ -52,6 +53,8 @@ const metainfo = await readFile( "utf8", ); const releases = appstreamReleases(metainfo); +// Package the fixture for the build host, as Tauri does (amd64 or arm64). +const ARCH = hostArch(); const version = releases[0].version; const notice = await readFile(path.join(root, "NOTICE"), "utf8"); const iconBytes = await readFile( @@ -69,7 +72,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND. `; const TAURI_CONTROL = `Package: shadow-code Version: ${version} -Architecture: amd64 +Architecture: ${ARCH.deb} Installed-Size: 1 Maintainer: Shadowfetch <209457103+ShadowfetchStudio@users.noreply.github.com> Priority: optional @@ -280,7 +283,7 @@ async function fixture(scratch) { await put("usr/share/doc/shadowcode/notices/README.txt", "notices\n"); for (const directory of ["usr", "usr/share", "usr/share/doc"]) await chmod(path.join(tree, directory), 0o775); - const deb = path.join(scratch, `ShadowCode_${version}_amd64.deb`); + const deb = path.join(scratch, `ShadowCode_${version}_${ARCH.deb}.deb`); await run("dpkg-deb", ["--root-owner-group", "--build", tree, deb]); const runtime = path.join(scratch, "runtime"); @@ -294,7 +297,7 @@ async function fixture(scratch) { LLAMA_LICENSE, ); await copyFile( - "/usr/lib/x86_64-linux-gnu/libz.so.1", + `/usr/lib/${ARCH.multiarch}/libz.so.1`, path.join(runtime, "libggml.so.0.25.0"), ); await chmod(path.join(runtime, "libggml.so.0.25.0"), 0o755); diff --git a/scripts/test-native-glib-variant.mjs b/scripts/test-native-glib-variant.mjs index 465daddc..cefdf3e9 100644 --- a/scripts/test-native-glib-variant.mjs +++ b/scripts/test-native-glib-variant.mjs @@ -32,7 +32,10 @@ const directory = await mkdtemp(path.join(os.tmpdir(), 'shadowcode-glib-regressi try { const binary = path.join(directory, 'variant-tests'); console.log(`${compiler.trim()}\nOptimized iterator fixture: ${libraries[0]}`); - execFileSync('rustc', ['--edition=2021', '--test', '-O', '--extern', `glib=${libraries[0]}`, '-L', `dependency=${dependencies}`, ...nativeSearch, path.join(root, 'scripts/fixtures/native-glib-variant.rs'), '-o', binary], { ...options, stdio: 'inherit' }); + // The release profile's LTO makes Cargo emit bitcode-only rlibs. rust-lld + // (the x86_64 default) accepts them, but the system linker on aarch64 does + // not, so link through LTO here to produce real objects on every host. + execFileSync('rustc', ['--edition=2021', '--test', '-O', '-C', 'lto=fat', '--extern', `glib=${libraries[0]}`, '-L', `dependency=${dependencies}`, ...nativeSearch, path.join(root, 'scripts/fixtures/native-glib-variant.rs'), '-o', binary], { ...options, stdio: 'inherit' }); execFileSync(binary, ['--test-threads=1'], { ...options, timeout: 30000, stdio: 'inherit' }); } finally { await rm(directory, { recursive: true, force: true }); diff --git a/scripts/test-native-packaging-env.mjs b/scripts/test-native-packaging-env.mjs index 8df0aa2d..ca30698f 100644 --- a/scripts/test-native-packaging-env.mjs +++ b/scripts/test-native-packaging-env.mjs @@ -19,12 +19,13 @@ import { GTK_PLUGIN_SHA256, installPinnedGtkPlugin, } from "./native-packaging-env.mjs"; +import { hostArch } from "./native-arch.mjs"; const exec = promisify(execFile); const root = fileURLToPath(new URL("../", import.meta.url)); const linuxdeploy = path.join( root, - "target/.tauri/linuxdeploy-x86_64.AppImage", + `target/.tauri/linuxdeploy-${hostArch().arch}.AppImage`, ); const hermesNode = "/usr/local/bin/node"; const dirtyPath = `${path.dirname(hermesNode)}:/snap/bin:/usr/bin:/bin`; diff --git a/scripts/test-native-release-auth.mjs b/scripts/test-native-release-auth.mjs index 9163a53e..b9645fa6 100644 --- a/scripts/test-native-release-auth.mjs +++ b/scripts/test-native-release-auth.mjs @@ -6,7 +6,9 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { spawnSync } from 'node:child_process'; -import { assetNames, createEnvelope, signEnvelope, verifyBundle, parseEnvelope, parseTrustPolicy, publicKeyIdentity, digestBytes, IDENTITY, LIMITS } from './native-release-auth.mjs'; +import { assetNames, metadataNames, createEnvelope, signEnvelope, verifyBundle, parseEnvelope, parseTrustPolicy, publicKeyIdentity, digestBytes, IDENTITY, LIMITS, TARGETS } from './native-release-auth.mjs'; + +const ARM = 'aarch64-unknown-linux-gnu'; const verifier = fileURLToPath(new URL('./verify-native-release.sh', import.meta.url)); function keyPair() { @@ -15,29 +17,30 @@ function keyPair() { const publicPem = publicKey.export({ type: 'spki', format: 'pem' }); return { privatePem, publicPem, id: publicKeyIdentity(publicPem) }; } -async function trustPolicy(directory, keys, { minimumVersion = '0.32.0', minimumEpoch = 1 } = {}) { +async function trustPolicy(directory, keys, { minimumVersion = '0.32.0', minimumEpoch = 1, target = IDENTITY.target } = {}) { await mkdir(directory, { recursive: true }); - const rows = ['ShadowCode-Release-Trust-v1', `repository=${IDENTITY.repository}`, `repository-id=${IDENTITY.repositoryId}`, `owner-id=${IDENTITY.ownerId}`, `target=${IDENTITY.target}`, `channel=${IDENTITY.channel}`, `minimum-epoch=${minimumEpoch}`, `minimum-version=${minimumVersion}`, 'keys=ed25519-spki-sha256']; + const rows = ['ShadowCode-Release-Trust-v1', `repository=${IDENTITY.repository}`, `repository-id=${IDENTITY.repositoryId}`, `owner-id=${IDENTITY.ownerId}`, `target=${target}`, `channel=${IDENTITY.channel}`, `minimum-epoch=${minimumEpoch}`, `minimum-version=${minimumVersion}`, 'keys=ed25519-spki-sha256']; for (const key of keys) { rows.push(`key=${key.epoch ?? 1}\t${key.id}\t${key.minimum ?? '0.0.0'}\t${key.maximum ?? '999999999.999999999.999999999'}`); await writeFile(path.join(directory, `${key.id}.pem`), key.publicPem); } await writeFile(path.join(directory, 'policy'), `${rows.join('\n')}\n`); } -async function makeBundle(directory, key, { version = '0.32.0', commit = 'a'.repeat(40), epoch = 1, marker } = {}) { +async function makeBundle(directory, key, { version = '0.32.0', commit = 'a'.repeat(40), epoch = 1, marker, target = IDENTITY.target } = {}) { await mkdir(directory, { recursive: true }); const hashes = {}; - for (const [role, name] of assetNames(version)) { + for (const [role, name] of assetNames(version, target)) { const bytes = role === 'appimage' ? Buffer.from(`#!/bin/sh\nprintf executed > '${marker}'\n`) : Buffer.from(`fixture ${role} ${version} ${commit}\n`); hashes[name] = digestBytes(bytes); await writeFile(path.join(directory, name), bytes, { mode: 0o755 }); } - await writeFile(path.join(directory, 'SHA256SUMS'), Object.entries(hashes).map(([name, hash]) => `${hash} ${name}\n`).join('')); - await writeFile(path.join(directory, 'RELEASE-MANIFEST.json'), `${JSON.stringify({ schema: 1, tag: `v${version}`, commit, target: IDENTITY.target, assets: hashes }, null, 2)}\n`); - const bytes = await createEnvelope({ bundleDir: directory, version, commit, keyId: key.id, keyEpoch: epoch }); - await writeFile(path.join(directory, 'RELEASE-AUTH'), bytes); - await writeFile(path.join(directory, 'RELEASE-AUTH.sig'), signEnvelope(bytes, key.privatePem)); - return { version, commit, artifact: assetNames(version)[0][1] }; + const meta = metadataNames(target); + await writeFile(path.join(directory, meta.sums), Object.entries(hashes).map(([name, hash]) => `${hash} ${name}\n`).join('')); + await writeFile(path.join(directory, meta.manifest), `${JSON.stringify({ schema: 1, tag: `v${version}`, commit, target, assets: hashes }, null, 2)}\n`); + const bytes = await createEnvelope({ bundleDir: directory, version, commit, keyId: key.id, keyEpoch: epoch, target }); + await writeFile(path.join(directory, meta.auth), bytes); + await writeFile(path.join(directory, meta.signature), signEnvelope(bytes, key.privatePem)); + return { version, commit, artifact: assetNames(version, target)[0][1] }; } async function fixture(t) { const root = await mkdtemp(path.join(tmpdir(), 'shadowcode-auth-test-')); @@ -346,3 +349,84 @@ test('parser rejects unsupported schemas and signing refuses a different key', a assert.throws(() => parseTrustPolicy(Buffer.from('ShadowCode-Release-Trust-v2\n'))); assert.throws(() => signEnvelope(bytes, keyPair().privatePem), /fingerprint mismatch/); }); + +test('x86_64 release names and envelope target are unchanged', () => { + assert.equal(IDENTITY.target, 'x86_64-unknown-linux-gnu'); + assert.deepEqual(assetNames('1.2.3'), [ + ['appimage', 'ShadowCode_1.2.3_amd64.AppImage'], + ['deb', 'ShadowCode_1.2.3_amd64.deb'], + ['runtime-sources', 'ShadowCode_1.2.3_appimage-runtime-sources.tar.gz'], + ]); + assert.deepEqual(assetNames('1.2.3', IDENTITY.target), assetNames('1.2.3')); + assert.deepEqual(Object.keys(TARGETS).sort(), [ARM, IDENTITY.target].sort()); +}); +test('aarch64 releases use Tauri and Debian names that cannot collide with x86_64 assets', () => { + const arm = assetNames('1.2.3', ARM); + assert.deepEqual(arm, [ + ['appimage', 'ShadowCode_1.2.3_aarch64.AppImage'], + ['deb', 'ShadowCode_1.2.3_arm64.deb'], + ['runtime-sources', 'ShadowCode_1.2.3_aarch64_appimage-runtime-sources.tar.gz'], + ]); + const x86 = new Set(assetNames('1.2.3').map(([, name]) => name)); + assert(arm.every(([, name]) => !x86.has(name))); + assert.throws(() => assetNames('1.2.3', 'riscv64gc-unknown-linux-gnu'), /Unsupported release target/); +}); +test('an aarch64 release verifies in Node and OpenSSL when the policy trusts both targets', async t => { + const f = await fixture(t), armDir = path.join(f.root, 'arm'); + await trustPolicy(f.trustDir, [f.key], { target: `${IDENTITY.target},${ARM}` }); + const arm = await makeBundle(armDir, f.key, { marker: f.marker, target: ARM }); + assert.equal(arm.artifact, 'ShadowCode_0.32.0_aarch64.AppImage'); + const { metadata } = await accepted(f, { bundleDir: armDir, artifact: arm.artifact, stageDir: path.join(f.root, 'arm-stage'), expectedTarget: ARM }); + assert.equal(metadata.target, ARM); + assert.equal(parseEnvelope(await readFile(path.join(armDir, 'RELEASE-AUTH-aarch64'))).target, ARM); + await notExists(path.join(armDir, 'RELEASE-AUTH')); + for (const [role, artifact] of assetNames(f.version, ARM)) await accepted(f, { bundleDir: armDir, artifact, stageDir: path.join(f.root, `arm-${role}`) }); + // The x86_64 release still verifies under the two-target policy. + await accepted(f, { stageDir: path.join(f.root, 'x86-stage'), expectedTarget: IDENTITY.target }); + // An x86_64 bundle has no aarch64 metadata to authenticate. + await assert.rejects(verifyBundle({ ...f, expectedTarget: ARM }), { code: 'ENOENT' }); + // An x86_64 basename selects x86_64 metadata, which an aarch64 bundle lacks. + await refused(f, { bundleDir: armDir, artifact: f.artifact, stageDir: path.join(f.root, 'arm-wrong-name') }, /input must be a regular non-symlink file/); +}); +test('the target suffix on release metadata must match the signed envelope', async t => { + const f = await fixture(t), armDir = path.join(f.root, 'arm'); + await trustPolicy(f.trustDir, [f.key], { target: `${IDENTITY.target},${ARM}` }); + const arm = await makeBundle(armDir, f.key, { marker: f.marker, target: ARM }); + // A valid x86_64 envelope, signature, manifest and checksums placed under the + // aarch64 names are refused for an aarch64 package before any digest check. + for (const [from, to] of [['RELEASE-AUTH', 'RELEASE-AUTH-aarch64'], ['RELEASE-AUTH.sig', 'RELEASE-AUTH-aarch64.sig'], ['RELEASE-MANIFEST.json', 'RELEASE-MANIFEST-aarch64.json'], ['SHA256SUMS', 'SHA256SUMS-aarch64']]) { + await writeFile(path.join(armDir, to), await readFile(path.join(f.bundleDir, from))); + } + await refused(f, { bundleDir: armDir, artifact: arm.artifact }, /release metadata names another target/); + await assert.rejects(verifyBundle({ bundleDir: armDir, trustDir: f.trustDir, artifact: arm.artifact }), /Release metadata names another target/); + await assert.rejects(verifyBundle({ bundleDir: armDir, trustDir: f.trustDir, artifact: arm.artifact, expectedTarget: ARM }), /Unexpected target/); + // The x86_64 package still reads only the historical names beside them. + await accepted(f); +}); +test('an aarch64 release is refused by an x86_64-only trust policy', async t => { + const f = await fixture(t), armDir = path.join(f.root, 'arm'); + const arm = await makeBundle(armDir, f.key, { marker: f.marker, target: ARM }); + await refused(f, { bundleDir: armDir, artifact: arm.artifact }, /release target is not trusted/); + await trustPolicy(f.trustDir, [f.key], { target: ARM }); + await refused(f, {}, /release target is not trusted/); + await accepted(f, { bundleDir: armDir, artifact: arm.artifact }); +}); +test('trust policies refuse unknown, duplicate and empty targets', async t => { + const f = await fixture(t); + for (const target of ['riscv64gc-unknown-linux-gnu', `${IDENTITY.target},${IDENTITY.target}`, `${IDENTITY.target},`, `,${IDENTITY.target}`, `${IDENTITY.target} ,${ARM}`]) { + await trustPolicy(f.trustDir, [f.key], { target }); + assert.throws(() => parseTrustPolicy(Buffer.from(`ShadowCode-Release-Trust-v1\nrepository=${IDENTITY.repository}\nrepository-id=${IDENTITY.repositoryId}\nowner-id=${IDENTITY.ownerId}\ntarget=${target}\nchannel=stable\nminimum-epoch=1\nminimum-version=0.1.0\nkeys=ed25519-spki-sha256\nkey=1\t${f.key.id}\t0.0.0\t1.0.0\n`))); + await refused(f, {}, /wrong trusted target/); + } +}); +test('a signed envelope cannot claim an unsupported target or switch target against accepted history', async t => { + const f = await fixture(t); + await mutateEnvelope(f, text => text.replace(`target=${IDENTITY.target}`, 'target=riscv64gc-unknown-linux-gnu')); + await refused(f, {}, /wrong target/); + const g = await fixture(t), previousDir = path.join(g.root, 'previous'), armDir = path.join(g.root, 'arm'); + await trustPolicy(g.trustDir, [g.key], { target: `${IDENTITY.target},${ARM}` }); + await makeBundle(previousDir, g.key, { version: '0.31.0', marker: g.marker }); + const arm = await makeBundle(armDir, g.key, { marker: g.marker, target: ARM }); + await refused(g, { bundleDir: armDir, artifact: arm.artifact, previousDir }, /release target changed/); + await accepted(g, { previousDir }); +}); diff --git a/scripts/test-native-release-signing.mjs b/scripts/test-native-release-signing.mjs index 6c0630bb..0bf7eeb8 100644 --- a/scripts/test-native-release-signing.mjs +++ b/scripts/test-native-release-signing.mjs @@ -121,17 +121,31 @@ test('release workflow separates untrusted build from protected signing and writ const workflow = await readFile(new URL('../.github/workflows/release.yml', import.meta.url), 'utf8'); const build = workflow.split('\n build:')[1].split('\n sign:')[0]; const sign = workflow.split('\n sign:')[1].split('\n publish:')[0]; - const publication = workflow.split('\n publish:')[1]; + const publication = workflow.split('\n publish:')[1].split('\n apt-repository:')[0]; + const apt = workflow.split('\n apt-repository:')[1]; assert(workflow.includes('permissions:\n contents: read')); assert(!/secrets\.|contents: write|publish-native-release/.test(build)); - assert.match(sign, /needs: \[configuration, build\]/); assert.match(sign, /environment: \$\{\{ needs.configuration.outputs.environment \}\}/); + assert.match(sign, /needs: \[configuration, build, build-aarch64\]/); assert.match(sign, /environment: \$\{\{ needs.configuration.outputs.environment \}\}/); assert(!/contents: write|npm |cargo |cache:/.test(sign)); assert.match(sign, /ref: \$\{\{ needs.configuration.outputs.tooling \}\}/); assert.match(sign, /artifact-ids: \$\{\{ needs.build.outputs.artifact-id \}\}/); + // Each target is built, handed off and signed separately; aarch64 is named. + const arm = workflow.split('\n build-aarch64:')[1].split('\n sign:')[0]; + assert.match(arm, /runs-on: ubuntu-24\.04-arm/); assert.match(arm, /prepare-native-release\.mjs/); + assert.match(sign, /artifact-ids: \$\{\{ needs.build-aarch64.outputs.artifact-id \}\}/); + assert.match(sign, /sign-native-release\.mjs "\$RUNNER_TEMP\/unsigned-download" release\/trust "\$RUNNER_TEMP\/signed-release\/x86_64"\n/); + assert.match(sign, /sign-native-release\.mjs "\$RUNNER_TEMP\/unsigned-download-aarch64" release\/trust "\$RUNNER_TEMP\/signed-release\/aarch64" aarch64-unknown-linux-gnu/); + assert.match(publication, /publish-native-release\.mjs "\$RUNNER_TEMP\/signed-download\/x86_64" release\/trust "\$RUNNER_TEMP\/signed-download\/aarch64"/); + assert.match(apt, /x86_64:amd64 aarch64:arm64/); assert.match(publication, /artifact-ids: \$\{\{ needs.sign.outputs.artifact-id \}\}/); assert.match(publication, /needs: \[configuration, build, sign\]/); assert.match(publication, /contents: write/); assert(!/SIGNING_KEY|secrets\./.test(publication)); assert.equal([...workflow.matchAll(/secrets\.NATIVE_RELEASE_SIGNING_KEY_PEM/g)].length, 1); + assert.match(apt, /needs: \[configuration, sign, publish\]/); assert.match(apt, /environment: \$\{\{ needs.configuration.outputs.environment \}\}/); + assert.match(apt, /artifact-ids: \$\{\{ needs.sign.outputs.artifact-id \}\}/); + assert.match(apt, /verify-native-release\.sh[\s\S]*build-apt-repo\.mjs/); + assert.equal([...workflow.matchAll(/secrets\.NATIVE_APT_SIGNING_KEY\b/g)].length, 1); + assert(!/NATIVE_RELEASE_SIGNING_KEY_PEM|npm |cargo |cache:/.test(apt)); for (const [, ref] of workflow.matchAll(/uses: ([^\s#]+)/g)) assert.match(ref, /@[a-f0-9]{40}$/); }); test('workflow configuration fails closed without real provisioning; fixture-only inputs exercise its parser', () => testFixture(async f => { @@ -154,3 +168,27 @@ for (const status of ['missing', 'failed', 'skipped']) test(`signing and publish await assert.rejects(f.execute({ verification: receipts }), /gate|verification/i); assert.equal(f.state.calls.length, 0); })); +test('each target is handed off, signed under its own metadata names and published together', () => fixture(async f => { + const ARM = 'aarch64-unknown-linux-gnu', [x86, arm] = f.bundles, sign = (input, target, outputDir, trustDir = f.trustDir) => signRelease({ bundleDir: input, trustDir, privateKey: f.privateKey, context, outputDir, target }); + const handoff = async ({ target, directory, packages, verification }) => { + const outputDir = path.join(f.root, `unsigned-${target}`); + await prepareRelease({ files: packages, checksums: path.join(directory, target === ARM ? 'SHA256SUMS-aarch64' : 'SHA256SUMS'), source: f.source, verification, context, outputDir, target }); + return outputDir; + }; + const x86Input = await handoff(x86), armInput = await handoff(arm); + // The signer refuses a target the provisioned policy does not trust. + const policy = path.join(f.trustDir, 'policy'), trusted = await readFile(policy, 'utf8'); + await writeFile(policy, trusted.replace(`,${ARM}`, '')); + await assert.rejects(sign(armInput, ARM, path.join(f.root, 'refused')), /not in the provisioned trust policy/); + await assert.rejects(stat(path.join(f.root, 'refused')), { code: 'ENOENT' }); + await writeFile(policy, trusted); + // An aarch64 handoff is not an x86_64 transport layout. + await assert.rejects(sign(armInput, IDENTITY.target, path.join(f.root, 'wrong-target')), /transport file set/); + await sign(x86Input, IDENTITY.target, path.join(f.root, 'signed-x86_64')); + await sign(armInput, ARM, path.join(f.root, 'signed-aarch64')); + assert.deepEqual((await readdir(path.join(f.root, 'signed-aarch64'))).sort(), [...releaseNames(VERSION, true, ARM), 'VERIFICATION.json'].sort()); + const load = async (directory, target) => ({ target, files: await requireTransportLayout(directory, VERSION, true, target), verification: await readReceipts(directory) }); + const bundles = [await load(path.join(f.root, 'signed-x86_64'), IDENTITY.target), await load(path.join(f.root, 'signed-aarch64'), ARM)]; + assert.equal((await f.execute({ bundles })).status, 'published'); + assert.equal(f.state.assets.size, 14); +}, publish, { targets: ['x86_64-unknown-linux-gnu', 'aarch64-unknown-linux-gnu'] })); diff --git a/scripts/test-native-release-verification.mjs b/scripts/test-native-release-verification.mjs index bd114352..9983afff 100644 --- a/scripts/test-native-release-verification.mjs +++ b/scripts/test-native-release-verification.mjs @@ -143,6 +143,7 @@ async function sourceFixture(directory) { }; for (const name of ['scripts', 'ui', 'src-tauri', 'packaging', 'artifacts/bin']) await mkdir(path.join(directory, name), { recursive: true }); await copyFile(new URL('./native-release-verification.mjs', import.meta.url), path.join(directory, 'scripts/native-release-verification.mjs')); + await copyFile(new URL('./native-arch.mjs', import.meta.url), path.join(directory, 'scripts/native-arch.mjs')); await writeFile(path.join(directory, '.gitignore'), '/artifacts/\n'); await writeFile(path.join(directory, 'tracked.txt'), 'committed source\n'); await utimes(path.join(directory, 'tracked.txt'), 946684800, 946684800); @@ -264,7 +265,7 @@ test('clean-host package qualification is required and bound to the verified art const gate = 'clean-host-packages'; assert(REQUIRED_GATES.includes(gate)); assert.match(GATES[gate].scope, /packaged MCP stdio.*visible first GUI window.*Interactive GUI tasks.*remain separate/); - assert.match(GATES[gate].script, /docker build -f scripts\/clean-host-runtime\.Dockerfile/); + assert.match(GATES[gate].script, /docker build --build-arg TARGETARCH="\$\(dpkg --print-architecture\)" -f scripts\/clean-host-runtime\.Dockerfile/); assert.match(GATES[gate].script, /test-clean-host-packages\.mjs.*SHA256SUMS/); const missing = complete(); delete missing.gates[gate]; assert.throws(() => validateVerification(missing, commit, artifacts), /gate set mismatch/); diff --git a/scripts/test-native-release.mjs b/scripts/test-native-release.mjs index 7f610d64..a997f616 100644 --- a/scripts/test-native-release.mjs +++ b/scripts/test-native-release.mjs @@ -1,10 +1,11 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { writeFile } from 'node:fs/promises'; +import { writeFile, readFile } from 'node:fs/promises'; +import path from 'node:path'; import { readFileSync } from 'node:fs'; import { publish } from './publish-native-release.mjs'; -import { fixture as signedFixture } from './publisher-fixtures.mjs'; +import { fixture as signedFixture, ARM } from './publisher-fixtures.mjs'; const fixture = run => signedFixture(run, publish); test('uploads to a draft, verifies all bytes, then publishes', () => fixture(async ({ state, execute }) => { @@ -113,3 +114,39 @@ for (const status of ['failed', 'skipped', 'running']) { assert.equal(state.calls.length, 0); })); } + +const twoTargets = run => signedFixture(run, publish, { targets: ['x86_64-unknown-linux-gnu', ARM] }); +test('one release carries the x86_64 and aarch64 bundles under disjoint names', () => twoTargets(async ({ state, execute }) => { + assert.equal((await execute()).status, 'published'); + assert.deepEqual([...state.assets.keys()].sort(), [ + 'RELEASE-AUTH', 'RELEASE-AUTH-aarch64', 'RELEASE-AUTH-aarch64.sig', 'RELEASE-AUTH.sig', + 'RELEASE-MANIFEST-aarch64.json', 'RELEASE-MANIFEST.json', 'SHA256SUMS', 'SHA256SUMS-aarch64', + 'ShadowCode_1.0.0_aarch64.AppImage', 'ShadowCode_1.0.0_aarch64_appimage-runtime-sources.tar.gz', 'ShadowCode_1.0.0_amd64.AppImage', 'ShadowCode_1.0.0_amd64.deb', + 'ShadowCode_1.0.0_appimage-runtime-sources.tar.gz', 'ShadowCode_1.0.0_arm64.deb', + ].sort()); + const publication = state.calls.findIndex(args => args[1] === 'edit'); + assert.equal(state.calls.slice(0, publication).filter(args => args[1] === 'download').length, 14); + // A retry reads the published release back and changes nothing. + const before = state.calls.length; + assert.equal((await execute()).status, 'already-published'); + assert(state.calls.slice(before).every(args => args[0] === 'api' || args[1] === 'download')); +})); +test('an aarch64 bundle is authenticated for its own target before any release operation', () => twoTargets(async ({ state, execute, signedBundles, bundles, trustDir }) => { + const [x86, arm] = signedBundles; + // The x86_64 bundle is required; targets cannot repeat or be unknown. + await assert.rejects(execute({ bundles: [arm] }), /x86_64 release bundle is required/); + await assert.rejects(execute({ bundles: [x86, x86] }), /Duplicate release target/); + await assert.rejects(execute({ bundles: [x86, { ...arm, target: 'riscv64gc-unknown-linux-gnu' }] }), /Unsupported release target/); + // x86_64 receipts or files offered as the aarch64 bundle are refused. + await assert.rejects(execute({ bundles: [x86, { ...x86, target: ARM }] }), /Unexpected release asset|asset set mismatch/); + await assert.rejects(execute({ bundles: [x86, { ...arm, verification: x86.verification }] }), /Verification package set or bytes changed/); + // An aarch64 envelope is invalid once the policy stops trusting aarch64. + const policy = path.join(trustDir, 'policy'), trusted = await readFile(policy, 'utf8'); + await writeFile(policy, trusted.replace(`,${ARM}`, '')); + await assert.rejects(execute(), /Release target is not trusted/); + await writeFile(policy, trusted); + // A tampered aarch64 package fails its own signed checksums. + await writeFile(bundles[1].packages[1], 'tampered'); + await assert.rejects(execute(), /Checksums|digest|Verification package/); + assert.equal(state.calls.length, 0); +})); diff --git a/scripts/test-native-runtime-sources.mjs b/scripts/test-native-runtime-sources.mjs index 04b9dd73..e165304f 100644 --- a/scripts/test-native-runtime-sources.mjs +++ b/scripts/test-native-runtime-sources.mjs @@ -8,6 +8,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { hostArch, packageNames } from "./native-arch.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); const version = JSON.parse( @@ -17,7 +18,7 @@ const archive = path.resolve( process.argv[2] || path.join( root, - `target/release/bundle/appimage/ShadowCode_${version}_appimage-runtime-sources.tar.gz`, + `target/release/bundle/appimage/${packageNames(version, hostArch().arch).runtimeSources}`, ), ); const artifacts = path.resolve( @@ -98,7 +99,7 @@ try { const runtime = path.join(scratch, "runtime"); await exec( engine, - ["cp", `${container}:/out/runtime-x86_64`, runtime], + ["cp", `${container}:/out/runtime-${hostArch().arch}`, runtime], options, ); await exec( diff --git a/scripts/test-native-runtime.mjs b/scripts/test-native-runtime.mjs index bdc6f5a9..ec2b2038 100644 --- a/scripts/test-native-runtime.mjs +++ b/scripts/test-native-runtime.mjs @@ -15,6 +15,7 @@ import { import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { hostArch, packageNames } from "./native-arch.mjs"; const root = fileURLToPath(new URL("../", import.meta.url)); const expectedVersion = JSON.parse( await readFile(path.join(root, "src-tauri/tauri.conf.json"), "utf8"), @@ -23,7 +24,7 @@ const binary = path.resolve( process.argv[2] || path.join( root, - `target/release/bundle/appimage/ShadowCode_${expectedVersion}_amd64.AppImage`, + `target/release/bundle/appimage/${packageNames(expectedVersion, hostArch().arch).appimage}`, ), ); const artifacts = path.resolve( diff --git a/scripts/test-native-test-session.mjs b/scripts/test-native-test-session.mjs index 4fb00bf0..2511d0a4 100644 --- a/scripts/test-native-test-session.mjs +++ b/scripts/test-native-test-session.mjs @@ -364,7 +364,13 @@ try { await writePrivateSessionReport(session,cleanup||{ok:false,services:[],failures:['Fixture stopped before cleanup']}); }\n`); const env = { ...process.env, PATH:`${directory}/bin:${process.env.PATH}`, SHADOW_X11_ARTIFACTS:path.join(directory,'artifacts') }; - const result = await execute(process.execPath,[wrapper,process.execPath,path.join(directory,'harness.mjs'),directory],{env,timeout:15000,maxBuffer:1024*1024}); + const result = await execute(process.execPath,[wrapper,process.execPath,path.join(directory,'harness.mjs'),directory],{env,timeout:15000,maxBuffer:1024*1024}).catch(async error => { + // Show the wrapper's own report: stderr alone hides why it failed. + const runs=await readdir(path.join(directory,'artifacts')).catch(()=>[]); + const reports=await Promise.all(runs.map(run=>readFile(path.join(directory,'artifacts',run,'result.json'),'utf8').catch(e=>String(e)))); + error.message+=`\nstdout:\n${error.stdout}\nresult.json:\n${reports.join('\n')}`; + throw error; + }); const runs=await readdir(path.join(directory,'artifacts')); assert.equal(runs.length,1); const report=JSON.parse(await readFile(path.join(directory,'artifacts',runs[0],'result.json'),'utf8')); runtime=report.runtime; assert.equal(report.ok,true,result.stdout+'\n'+result.stderr); diff --git a/scripts/verify-native-release.sh b/scripts/verify-native-release.sh index 0f8f3ccc..260d881d 100755 --- a/scripts/verify-native-release.sh +++ b/scripts/verify-native-release.sh @@ -39,23 +39,29 @@ trap 'exit 143' TERM # shellcheck source=native-release-auth-lib.sh source "$(dirname "$0")/native-release-auth-lib.sh" load_trust_policy -copy_file "$BUNDLE/RELEASE-AUTH" "$SCRATCH/RELEASE-AUTH" 4096 -copy_file "$BUNDLE/RELEASE-AUTH.sig" "$SCRATCH/RELEASE-AUTH.sig" 64 +# The requested package selects its target's metadata names; the verified stage +# keeps the historical names, so installers and receipts read one layout. +target_for_artifact "$ARTIFACT"; META_TARGET=$ARTIFACT_TARGET +target_names "$META_TARGET"; META=$TARGET_META +copy_file "$BUNDLE/RELEASE-AUTH$META" "$SCRATCH/RELEASE-AUTH" 4096 +copy_file "$BUNDLE/RELEASE-AUTH$META.sig" "$SCRATCH/RELEASE-AUTH.sig" 64 verify_envelope "$SCRATCH/RELEASE-AUTH" "$SCRATCH/RELEASE-AUTH.sig" 0 -CURRENT_VERSION=$EN_VERSION; CURRENT_EPOCH=$EN_EPOCH; CURRENT_HASH=$(hash_file "$SCRATCH/RELEASE-AUTH") +[[ "$EN_TARGET" == "$META_TARGET" ]] || fail 'release metadata names another target' +CURRENT_VERSION=$EN_VERSION; CURRENT_TARGET=$EN_TARGET; CURRENT_EPOCH=$EN_EPOCH; CURRENT_HASH=$(hash_file "$SCRATCH/RELEASE-AUTH") [[ -z "$EXPECT_VERSION" || "$EXPECT_VERSION" == "$EN_VERSION" ]] || fail 'unexpected version' [[ -z "$EXPECT_COMMIT" || "$EXPECT_COMMIT" == "$EN_COMMIT" ]] || fail 'unexpected commit' if [[ -n "$PREVIOUS" ]]; then copy_file "$PREVIOUS/RELEASE-AUTH" "$SCRATCH/previous-auth" 4096 copy_file "$PREVIOUS/RELEASE-AUTH.sig" "$SCRATCH/previous-auth.sig" 64 verify_envelope "$SCRATCH/previous-auth" "$SCRATCH/previous-auth.sig" 1 + [[ "$CURRENT_TARGET" == "$EN_TARGET" ]] || fail 'release target changed' (( CURRENT_EPOCH >= EN_EPOCH )) || fail 'signing epoch rollback' version_compare "$CURRENT_VERSION" "$EN_VERSION"; (( CMP >= 0 )) || fail 'release downgrade' if (( CMP == 0 )); then [[ "$CURRENT_HASH" == "$(hash_file "$SCRATCH/previous-auth")" ]] || fail 'changed release under accepted version'; fi parse_envelope "$SCRATCH/RELEASE-AUTH" fi -copy_file "$BUNDLE/RELEASE-MANIFEST.json" "$SCRATCH/RELEASE-MANIFEST.json" 1048576 -copy_file "$BUNDLE/SHA256SUMS" "$SCRATCH/SHA256SUMS" 4096 +copy_file "$BUNDLE/RELEASE-MANIFEST$META.json" "$SCRATCH/RELEASE-MANIFEST.json" 1048576 +copy_file "$BUNDLE/SHA256SUMS$META" "$SCRATCH/SHA256SUMS" 4096 [[ "$(hash_file "$SCRATCH/RELEASE-MANIFEST.json")" == "$EN_MANIFEST" ]] || fail 'manifest digest mismatch' [[ "$(hash_file "$SCRATCH/SHA256SUMS")" == "$EN_SUMS" ]] || fail 'checksums digest mismatch' SELECTED=-1 diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index d36d9c73..bd1ffaa5 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -25,6 +25,13 @@ mod web_ui; mod window_size; use backend::Backend; +/// WebKit's helper process inside an AppImage built on this architecture; the +/// bundler copies it into the build host's Debian multiarch directory. +#[cfg(target_arch = "aarch64")] +const APPIMAGE_WEBKIT_PROCESS: &str = "usr/lib/aarch64-linux-gnu/webkit2gtk-4.1/WebKitWebProcess"; +#[cfg(not(target_arch = "aarch64"))] +const APPIMAGE_WEBKIT_PROCESS: &str = "usr/lib/x86_64-linux-gnu/webkit2gtk-4.1/WebKitWebProcess"; + #[derive(Default)] struct Lifecycle { requested: AtomicBool, @@ -396,9 +403,7 @@ fn run() -> Result<()> { ); if let Some(appdir) = std::env::var_os("APPDIR").map(PathBuf::from) { if std::env::current_exe()?.starts_with(&appdir) - && appdir - .join("usr/lib/x86_64-linux-gnu/webkit2gtk-4.1/WebKitWebProcess") - .is_file() + && appdir.join(APPIMAGE_WEBKIT_PROCESS).is_file() { // No GTK or asynchronous runtime has been started at this point. std::env::set_current_dir(appdir.join("usr"))?; @@ -481,6 +486,18 @@ fn run() -> Result<()> { // window that fits it, title bar and panel included: the screen // the window-state plugin moves it back to on X11, every screen // on Wayland. + // The content size remembered from the last launch (not for an + // isolated profile), then fitted to the screen like the default. + let size_file = (!isolated) + .then(|| app.path().app_config_dir().ok()) + .flatten() + .map(|dir| dir.join(window_size::CONTENT_SIZE_FILE)); + window_size::restore_config( + &mut config, + size_file + .as_deref() + .and_then(window_size::SizeMemory::load), + ); let opening = window_size::opening_on(app, !isolated); if let Some(opening) = &opening { window_size::fit_config(&mut config, opening); @@ -499,7 +516,19 @@ fn run() -> Result<()> { && url.host_str() == Some("127.0.0.1") && url.port() == Some(5174)) }) + .on_page_load(|_, payload| { + // The backend (and its database migration) opened before + // the window was built; a loaded page completes the first + // start the AppImage installer waits for. + if matches!(payload.event(), tauri::webview::PageLoadEvent::Finished) { + shadowcode_core::lifecycle::report_first_start(); + } + }) .build()?; + app.manage(window_size::SizeMemory::new( + size_file, + (config.width, config.height), + )); // Building the window queued the plugin's move back to where it // was (X11); this move follows it and keeps the fitted window // inside that screen's work area. @@ -572,6 +601,18 @@ fn run() -> Result<()> { Ok(()) }) .on_window_event(|window, event| { + if window.label() == "main" { + if let Some(memory) = window.try_state::() { + match event { + tauri::WindowEvent::Resized(_) => memory.observe(window), + tauri::WindowEvent::CloseRequested { .. } => { + memory.observe(window); + memory.save(); + } + _ => {} + } + } + } if let tauri::WindowEvent::CloseRequested { api, .. } = event { api.prevent_close(); request_shutdown(window.app_handle()); diff --git a/src-tauri/src/window_size.rs b/src-tauri/src/window_size.rs index 1eb6041f..f66bfd91 100644 --- a/src-tauri/src/window_size.rs +++ b/src-tauri/src/window_size.rs @@ -14,9 +14,9 @@ use tauri_plugin_window_state::StateFlags; /// plugin saved the size GTK reports in its configure event, which includes /// that header bar and the shadow margins, and restoring it added them /// again, so the window grew on every launch (about 52 px wide and 99 px -/// tall). The window now opens at its configured size fitted to the screen -/// ([`fit_config`]) on every desktop, and a size saved by an earlier version -/// is ignored. +/// tall). The plugin's size is therefore never used: the content size is +/// remembered separately ([`SizeMemory`], [`ContentSize`]) without those +/// decorations, and fitted to the screen ([`fit_config`]). pub const REMEMBERED: StateFlags = StateFlags::all().difference(StateFlags::SIZE); /// The window-state plugin, remembering [`REMEMBERED`]. @@ -419,6 +419,168 @@ impl Opening { } } +/// File, in the app's config directory, holding the remembered content +/// size. It is separate from the window-state plugin's file, whose size is +/// not used (see [`REMEMBERED`]). +pub const CONTENT_SIZE_FILE: &str = "window-content-size.json"; + +/// Version of [`ContentSize`]'s file. A file without it, or with another +/// version, is ignored, so no size saved before 2.1.0 is ever restored. +pub const CONTENT_SIZE_VERSION: u32 = 2; + +/// A larger difference between the size the window asked for and the size +/// it reports is not a title bar or shadow (a tiling window manager, say), +/// so it is not taken off. +const MAX_DECORATION: f64 = 200.0; + +/// The window's content (inner) size in logical pixels, as remembered +/// between launches. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, Deserialize)] +pub struct ContentSize { + pub version: u32, + pub width: f64, + pub height: f64, +} + +impl ContentSize { + pub fn new(width: f64, height: f64) -> Self { + Self { + version: CONTENT_SIZE_VERSION, + width: width.round(), + height: height.round(), + } + } + + /// Read a saved file: None for another version or a size that is not a + /// positive, finite number. + pub fn parse(file: &str) -> Option { + let saved: Self = serde_json::from_str(file).ok()?; + let valid = |length: f64| length.is_finite() && length > 0.0; + (saved.version == CONTENT_SIZE_VERSION && valid(saved.width) && valid(saved.height)) + .then_some(saved) + } + + pub fn to_json(self) -> String { + serde_json::to_string(&self).unwrap_or_default() + } +} + +/// What the window adds to the content size it was opened with, when it +/// reports its size: on Wayland GTK can count its header bar and shadow, +/// on X11 nothing. Measured once, from the first size the window reports +/// while neither maximized nor full screen, against the size it was built +/// with. Negative or implausibly large differences count as none. +pub fn decoration(requested: (f64, f64), reported: (f64, f64)) -> (f64, f64) { + let extra = |asked: f64, got: f64| { + let extra = got - asked; + if extra.is_finite() && (0.0..=MAX_DECORATION).contains(&extra) { + extra + } else { + 0.0 + } + }; + ( + extra(requested.0, reported.0), + extra(requested.1, reported.1), + ) +} + +/// The content size to remember for a window that reports `reported` +/// (logical pixels), taking off its [`decoration`]. +pub fn content_size(reported: (f64, f64), decoration: (f64, f64)) -> ContentSize { + ContentSize::new(reported.0 - decoration.0, reported.1 - decoration.1) +} + +/// The inner size to open with: the remembered content size, or the +/// configured one, never below the minimum. [`fit_config`] then fits it to +/// the screen. +pub fn restored_size( + configured: (f64, f64), + min: (f64, f64), + saved: Option, +) -> (f64, f64) { + match saved { + Some(saved) => (saved.width.max(min.0), saved.height.max(min.1)), + None => configured, + } +} + +/// Use the remembered content size, if any, as the window's configured +/// inner size. Call before [`fit_config`]. +pub fn restore_config(config: &mut WindowConfig, saved: Option) { + let min = ( + config.min_width.unwrap_or(0.0), + config.min_height.unwrap_or(0.0), + ); + (config.width, config.height) = restored_size((config.width, config.height), min, saved); +} + +/// Remembers the main window's content size: tracks the window's reported +/// size and writes the content size when the window closes. +pub struct SizeMemory { + file: Option, + requested: (f64, f64), + state: std::sync::Mutex, +} + +#[derive(Default)] +struct MemoryState { + decoration: Option<(f64, f64)>, + last: Option, +} + +impl SizeMemory { + /// `file` is None for an isolated profile, which remembers nothing. + /// `requested` is the inner size the window was built with. + pub fn new(file: Option, requested: (f64, f64)) -> Self { + Self { + file, + requested, + state: Default::default(), + } + } + + /// The saved content size in `file`, if it is of this version. + pub fn load(file: &std::path::Path) -> Option { + ContentSize::parse(&std::fs::read_to_string(file).ok()?) + } + + /// Note the window's current size. A maximized or full-screen window's + /// size is not its own and is skipped. + pub fn observe(&self, window: &tauri::Window) { + if window.is_maximized().unwrap_or(true) || window.is_fullscreen().unwrap_or(true) { + return; + } + let Ok(size) = window.inner_size() else { + return; + }; + let scale = valid_scale(window.scale_factor().unwrap_or(1.0)); + let reported = ( + f64::from(size.width) / scale, + f64::from(size.height) / scale, + ); + let Ok(mut state) = self.state.lock() else { + return; + }; + let decoration = *state + .decoration + .get_or_insert_with(|| decoration(self.requested, reported)); + state.last = Some(content_size(reported, decoration)); + } + + /// Write the last content size noted, if any. + pub fn save(&self) { + let Some(file) = &self.file else { return }; + let last = self.state.lock().ok().and_then(|state| state.last); + if let Some(last) = last { + if let Some(dir) = file.parent() { + let _ = std::fs::create_dir_all(dir); + } + let _ = std::fs::write(file, last.to_json()); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -891,4 +1053,76 @@ mod tests { ); assert_eq!(Saved::main_window("not json"), None); } + + /// A window manager that reports `extra` more than the content size, + /// as GTK's header bar and shadow did on Plasma (52x99). + fn launch(saved: Option, screen: Screen, extra: (f64, f64)) -> ContentSize { + let mut config = shipped_config(); + restore_config(&mut config, saved); + let opening = Opening { + desktop: Desktop::Wayland, + screens: vec![screen], + restored: None, + }; + fit_config(&mut config, &opening); + let requested = (config.width, config.height); + let reported = (requested.0 + extra.0, requested.1 + extra.1); + content_size(reported, decoration(requested, reported)) + } + + #[test] + fn save_then_restore_is_a_fixed_point() { + for extra in [(0.0, 0.0), (52.0, 99.0)] { + let first = launch(None, wayland(1920.0, 1080.0), extra); + assert_eq!((first.width, first.height), DEFAULT); + let user = ContentSize::new(1000.0, 700.0); + let again = launch(Some(user), wayland(1920.0, 1080.0), extra); + assert_eq!(again, user); + } + } + + #[test] + fn repeated_launches_never_grow() { + let mut saved = None; + for _ in 0..20 { + let next = launch(saved, wayland(2560.0, 1440.0), (52.0, 99.0)); + if let Some(previous) = saved { + assert_eq!(next, previous); + } + saved = Some(next); + } + assert_eq!(saved.map(|s| (s.width, s.height)), Some(DEFAULT)); + } + + #[test] + fn a_remembered_size_still_fits_a_small_screen() { + let big = ContentSize::new(1800.0, 1000.0); + let small = launch(Some(big), wayland(1366.0, 768.0), (0.0, 0.0)); + assert_eq!((small.width, small.height), (1229.0, 691.0)); + let tiny = ContentSize::new(100.0, 100.0); + let floor = launch(Some(tiny), wayland(1920.0, 1080.0), (0.0, 0.0)); + assert_eq!((floor.width, floor.height), MIN); + } + + #[test] + fn sizes_from_other_versions_are_ignored() { + assert_eq!(ContentSize::parse(r#"{"width":1432,"height":1019}"#), None); + assert_eq!( + ContentSize::parse(r#"{"version":1,"width":1432,"height":1019}"#), + None + ); + assert_eq!( + ContentSize::parse(r#"{"version":2,"width":-1,"height":1019}"#), + None + ); + let size = ContentSize::new(1000.4, 700.6); + assert_eq!(ContentSize::parse(&size.to_json()), Some(size)); + assert_eq!((size.width, size.height), (1000.0, 701.0)); + } + + #[test] + fn implausible_decorations_are_not_taken_off() { + assert_eq!(decoration((1000.0, 700.0), (900.0, 1400.0)), (0.0, 0.0)); + assert_eq!(decoration((1000.0, 700.0), (1052.0, 799.0)), (52.0, 99.0)); + } } diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 44e53fcf..ce53fa79 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "ShadowCode", - "version": "2.0.0", + "version": "2.1.0", "identifier": "com.shadowfetch.shadowcode", "build": { "beforeDevCommand": { diff --git a/ui/package-lock.json b/ui/package-lock.json index ac6bac6b..89a4f0f4 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "shadow-agent-ui", - "version": "2.0.0", + "version": "2.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "shadow-agent-ui", - "version": "2.0.0", + "version": "2.1.0", "license": "Apache-2.0", "dependencies": { "@codemirror/commands": "6.11.1", diff --git a/ui/package.json b/ui/package.json index 3b2b8bf8..87a5b27a 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "shadow-agent-ui", - "version": "2.0.0", + "version": "2.1.0", "private": true, "license": "Apache-2.0", "type": "module", diff --git a/ui/src/api.ts b/ui/src/api.ts index b84432db..ae340ec3 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -1,6 +1,7 @@ import { request, ApiError } from "./lib/transport"; import { createVerificationReader } from "./lib/verificationRefresh"; import type { BillingMode, PickerTarget, UsageSnapshot } from "./lib/picker"; +import type { ModelOutcomes } from "./lib/outcomes"; import type { ContextAttachment } from "./lib/pendingAttachments"; import type { PreviewOpened, PreviewServer } from "./lib/preview"; @@ -1728,6 +1729,12 @@ export const api = { get(`/api/picker${refresh ? "?refresh=1" : ""}`), /** Local and cached rows only; does not start provider or network probes. */ pickerCached: () => get("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/api/picker?cached=1"), + /** Per-model outcome history for a project, plus the opt-in suggestion + * (never applied automatically). */ + modelOutcomes: (workspace: string, current = "") => + get( + `/api/models/outcomes?workspace=${encodeURIComponent(workspace)}¤t=${encodeURIComponent(current)}`, + ), accounts: (refresh = false) => get(`/api/accounts${refresh ? "?refresh=1" : ""}`), accountsCached: () => get("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/api/accounts?cached=1"), diff --git a/ui/src/components.css b/ui/src/components.css index 2d483c16..30d79730 100644 --- a/ui/src/components.css +++ b/ui/src/components.css @@ -645,6 +645,47 @@ button.link { .unified-picker-hint { color: var(--warn) !important; } +.unified-picker-suggestion { + display: grid; + gap: 4px; + margin: 4px 4px 6px; + padding: 6px 8px; + border: 1px solid var(--border); + border-radius: 6px; + font-size: 12px; +} +.unified-picker-suggestion .row { + gap: 6px; +} +.picker-history { + color: var(--text-2); +} +.unified-picker-history { + border-top: 1px solid var(--border); + padding: 6px 4px; + font-size: 12px; + overflow-x: auto; +} +.unified-picker-history table { + width: 100%; + border-collapse: collapse; +} +.unified-picker-history th, +.unified-picker-history td { + text-align: left; + padding: 2px 6px; + white-space: nowrap; +} +.unified-picker-history p { + margin: 4px 6px 0; + color: var(--text-2); +} +.unified-picker-optin { + display: inline-flex; + align-items: center; + gap: 4px; + font-size: 12px; +} .unified-picker-actions { display: flex; flex-wrap: wrap; diff --git a/ui/src/components/UnifiedPicker.outcomes.test.tsx b/ui/src/components/UnifiedPicker.outcomes.test.tsx new file mode 100644 index 00000000..e1ff40bd --- /dev/null +++ b/ui/src/components/UnifiedPicker.outcomes.test.tsx @@ -0,0 +1,152 @@ +import { useState } from "react"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, expect, it, vi } from "vitest"; +import { UnifiedPicker } from "./UnifiedPicker"; +import type { PickerTarget } from "../lib/picker"; +import type { ModelOutcomes } from "../lib/outcomes"; + +afterEach(() => cleanup()); + +const codex: PickerTarget = { + id: "cli:codex", + provider: "cli:codex", + group: "subscriptions", + name: "Codex", + inference: "cloud", + availability: "ready", + availability_label: "Ready", + tools: true, +}; +const cheap: PickerTarget = { + id: "api:openrouter:cheap", + provider: "openrouter", + group: "api", + name: "Cheap Coder", + inference: "cloud", + availability: "ready", + availability_label: "Ready", + tools: true, +}; + +const outcomes: ModelOutcomes = { + workspace: "/p", + suggest_from_history: true, + models: [ + { + model_id: "cli:codex", + model_label: "Codex", + samples: 3, + median_seconds: 90, + median_tokens: 1000, + median_cost_usd: null, + cost_samples: 0, + verified_samples: 2, + passes: 2, + pass_rate: 1, + last_finished_at: 2, + }, + { + model_id: "api:openrouter:cheap", + model_label: "cheap", + samples: 2, + median_seconds: 40, + median_tokens: 800, + median_cost_usd: 0.03, + cost_samples: 2, + verified_samples: 2, + passes: 1, + pass_rate: 0.5, + last_finished_at: 1, + }, + ], + suggestion: { + model_id: "api:openrouter:cheap", + name: "Cheap Coder", + median_cost_usd: 0.03, + pass_rate: 0.5, + passes: 1, + samples: 2, + reason: "Cheapest model that passed verification in this project", + }, +}; + +function Harness({ + onSelect = vi.fn(), + onSuggestionsChange, + data = outcomes, +}: { + onSelect?: (id: string) => void; + onSuggestionsChange?: (on: boolean) => void; + data?: ModelOutcomes; +}) { + const [value, setValue] = useState("cli:codex"); + const [open, setOpen] = useState(true); + return ( + { + setValue(id); + onSelect(id); + }} + onConnect={vi.fn()} + onSetup={vi.fn()} + onAddLocal={vi.fn()} + outcomes={data} + onSuggestionsChange={onSuggestionsChange} + /> + ); +} + +it("shows a compact per-row history with unavailable costs", () => { + render(); + expect( + screen.getByText(/Here: 1m 30s · cost unavailable · 2\/2 passed · 3 tasks/), + ).toBeTruthy(); + expect( + screen.getByText(/Here: 40s · \$0.03 · 1\/2 passed · 2 tasks/), + ).toBeTruthy(); +}); + +it("offers the suggestion and switches only when accepted", () => { + const onSelect = vi.fn(); + render(); + const note = screen.getByRole("note", { name: "Suggested model" }); + expect(note.textContent).toContain("Cheap Coder"); + expect(onSelect).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Use Cheap Coder" })); + expect(onSelect).toHaveBeenCalledWith("api:openrouter:cheap"); +}); + +it("can dismiss the suggestion without switching", () => { + const onSelect = vi.fn(); + render(); + fireEvent.click(screen.getByRole("button", { name: "Not now" })); + expect(screen.queryByRole("note", { name: "Suggested model" })).toBeNull(); + expect(onSelect).not.toHaveBeenCalled(); +}); + +it("shows no suggestion when the server offers none", () => { + render(); + expect(screen.queryByRole("note", { name: "Suggested model" })).toBeNull(); +}); + +it("opens the side-by-side history and toggles the opt-in", () => { + const onSuggestionsChange = vi.fn(); + render(); + fireEvent.click(screen.getByRole("button", { name: "Compare history" })); + const table = screen.getByRole("table", { + name: "Model history in this project", + }); + expect(table.textContent).toContain("Median reported cost"); + expect(table.textContent).toContain("cost unavailable"); + expect(table.textContent).toContain("$0.03"); + const box = screen.getByRole("checkbox", { + name: /Suggest the cheapest model that passed here/, + }); + expect((box as HTMLInputElement).checked).toBe(true); + fireEvent.click(box); + expect(onSuggestionsChange).toHaveBeenCalledWith(false); +}); diff --git a/ui/src/components/UnifiedPicker.tsx b/ui/src/components/UnifiedPicker.tsx index e40343a6..fcc51d61 100644 --- a/ui/src/components/UnifiedPicker.tsx +++ b/ui/src/components/UnifiedPicker.tsx @@ -37,6 +37,14 @@ import { vendorSections, type PickerTarget, } from "../lib/picker"; +import { + compactHistory, + formatCost, + formatDuration, + formatPassRate, + statsFor, + type ModelOutcomes, +} from "../lib/outcomes"; type Item = | { kind: "row"; key: string; target: PickerTarget } @@ -76,6 +84,8 @@ export function UnifiedPicker({ note, loading, label = "Model for this task", + outcomes, + onSuggestionsChange, }: { targets: PickerTarget[]; value: string; @@ -92,6 +102,10 @@ export function UnifiedPicker({ loading?: boolean; /** Accessible name of the trigger, before the chosen row's name. */ label?: string; + /** This project's per-model outcome history and the opt-in suggestion. */ + outcomes?: ModelOutcomes | null; + /** Turn "suggest the cheapest model that passed here" on or off. */ + onSuggestionsChange?: (on: boolean) => void; }) { const [query, setQuery] = useState(""); const [active, setActive] = useState(0); @@ -99,6 +113,16 @@ export function UnifiedPicker({ const [details, setDetails] = useState(null); const [recent] = useState(recentTargets); const [refreshing, setRefreshing] = useState(false); + const [comparing, setComparing] = useState(false); + const [dismissed, setDismissed] = useState(null); + const offered = outcomes?.suggestion ?? null; + const suggestion = + offered && + offered.model_id !== value && + offered.model_id !== dismissed && + targets.some((t) => t.id === offered.model_id && isReady(t)) + ? offered + : null; const trigger = useRef(null); const root = useRef(null); const search = useRef(null); @@ -376,6 +400,39 @@ export function UnifiedPicker({ onKeyDown={onKeyDown} /> + {suggestion && ( +
+ + Suggestion: {suggestion.name} passed here + before ({suggestion.passes} passed, median{" "} + {formatCost(suggestion.median_cost_usd)}). + + + + + +
+ )}
)} + {comparing && outcomes && ( + + )}
{onRefresh && ( + )} + {onSuggestionsChange && outcomes && ( + + )}