From e46c8a808d5dac4bbd121c1ea34c6d6c57d8cf5c Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 02:35:30 -0500 Subject: [PATCH 01/23] feat(canonical): sign attestations with a key held elsewhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `attest_value` splits into `prepare_attestation` (author, content hash and the exact bytes to sign) and `attach_proof`, so a signer that keeps its key outside this process — a browser's WebCrypto, a hardware token — produces the same `eddsa-jcs-2022` attestation the CLI does. `attest_value` is now those two halves around a local `Signer`. `atomic-canonical-wasm` exposes that to the browser: prepare, attach and verify attestations, plus the DID and canonical JSON of a document, all over JSON strings and byte arrays. `build.sh` builds it with wasm-bindgen; `smoke.mjs` signs with a non-extractable WebCrypto key, verifies, and checks tampering is caught. --- Cargo.lock | 11 ++++ Cargo.toml | 1 + atomic-canonical-wasm/.gitignore | 1 + atomic-canonical-wasm/Cargo.toml | 24 +++++++ atomic-canonical-wasm/README.md | 20 ++++++ atomic-canonical-wasm/build.sh | 11 ++++ atomic-canonical-wasm/smoke.mjs | 13 ++++ atomic-canonical-wasm/src/lib.rs | 107 +++++++++++++++++++++++++++++++ atomic-canonical/src/proof.rs | 64 ++++++++++++++++-- 9 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 atomic-canonical-wasm/.gitignore create mode 100644 atomic-canonical-wasm/Cargo.toml create mode 100644 atomic-canonical-wasm/README.md create mode 100755 atomic-canonical-wasm/build.sh create mode 100644 atomic-canonical-wasm/smoke.mjs create mode 100644 atomic-canonical-wasm/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 830d8ca4..a23eac48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,6 +172,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "atomic-canonical-wasm" +version = "0.18.3" +dependencies = [ + "atomic-canonical", + "atomic-identity", + "getrandom 0.2.17", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "atomic-cli" version = "0.18.3" diff --git a/Cargo.toml b/Cargo.toml index 36780037..17e609ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "atomic-agent", "atomic-canonical", + "atomic-canonical-wasm", "atomic-cli", "atomic-core", "atomic-config", diff --git a/atomic-canonical-wasm/.gitignore b/atomic-canonical-wasm/.gitignore new file mode 100644 index 00000000..01d0a084 --- /dev/null +++ b/atomic-canonical-wasm/.gitignore @@ -0,0 +1 @@ +pkg/ diff --git a/atomic-canonical-wasm/Cargo.toml b/atomic-canonical-wasm/Cargo.toml new file mode 100644 index 00000000..8847e900 --- /dev/null +++ b/atomic-canonical-wasm/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "atomic-canonical-wasm" +description = "atomic-canonical for the browser: build and check atomic attestations where the signing key lives in WebCrypto" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +atomic-canonical = { workspace = true } +atomic-identity = { workspace = true } +serde_json = { workspace = true } +wasm-bindgen = "0.2" + +# The browser supplies randomness; nothing here draws any, but the +# dependency tree links getrandom, which refuses wasm32 without this. +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.2", features = ["js"] } diff --git a/atomic-canonical-wasm/README.md b/atomic-canonical-wasm/README.md new file mode 100644 index 00000000..a5b15f48 --- /dev/null +++ b/atomic-canonical-wasm/README.md @@ -0,0 +1,20 @@ +# atomic-canonical-wasm + +`atomic-canonical` for the browser. A page that holds an atomic identity's +Ed25519 key in WebCrypto signs atomic documents without the key ever leaving +the browser: this module builds the canonical document and the bytes to sign +(with the same code the CLI uses), WebCrypto signs them, and the result is an +ordinary `eddsa-jcs-2022` attestation that `atomic` verifies. + +```js +import init, { prepareAttestation, attachProof } from "./pkg/atomic_canonical_wasm.js"; +await init(); +const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); +const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); +const attested = JSON.parse(attachProof(prepared.document, publicKeyBytes, new Uint8Array(sig))); +``` + +Also: `verifyAttestation`, `didForPublicKey`, `publicKeyBase32`, `canonicalize`. + +Build with `./build.sh [out-dir]`; `node smoke.mjs` (Node ≥ 22, which has +WebCrypto Ed25519) signs, verifies and tamper-checks a document end to end. diff --git a/atomic-canonical-wasm/build.sh b/atomic-canonical-wasm/build.sh new file mode 100755 index 00000000..94f6390e --- /dev/null +++ b/atomic-canonical-wasm/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Build the browser module: pkg/atomic_canonical_wasm{.js,_bg.wasm}. +# Needs the wasm32 target and a wasm-bindgen CLI matching the locked +# wasm-bindgen crate: +# rustup target add wasm32-unknown-unknown +# cargo install wasm-bindgen-cli --version +set -eu +here=$(cd "$(dirname "$0")" && pwd) +cargo build -p atomic-canonical-wasm --target wasm32-unknown-unknown --release +wasm-bindgen --target web --out-dir "${1:-$here/pkg}" \ + "$here/../target/wasm32-unknown-unknown/release/atomic_canonical_wasm.wasm" diff --git a/atomic-canonical-wasm/smoke.mjs b/atomic-canonical-wasm/smoke.mjs new file mode 100644 index 00000000..b5a8250d --- /dev/null +++ b/atomic-canonical-wasm/smoke.mjs @@ -0,0 +1,13 @@ +import { readFileSync } from "node:fs"; +import init, { prepareAttestation, attachProof, verifyAttestation, didForPublicKey } from "./pkg/atomic_canonical_wasm.js"; +await init({ module_or_path: readFileSync(new URL("./pkg/atomic_canonical_wasm_bg.wasm", import.meta.url)) }); +const kp = await crypto.subtle.generateKey("Ed25519", false, ["sign", "verify"]); +const pub = new Uint8Array(await crypto.subtle.exportKey("raw", kp.publicKey)); +const doc = { "@type": "ExampleLogin", nonce: "abc", "é": [1, 2.5, "x"] }; +const p = prepareAttestation(JSON.stringify(doc), pub); +const sig = new Uint8Array(await crypto.subtle.sign("Ed25519", kp.privateKey, p.signingBytes)); +const attested = attachProof(p.document, pub, sig); +verifyAttestation(attested, pub); +console.log(didForPublicKey(pub)); +console.log(attested); +try { verifyAttestation(attested.replace('"abc"', '"abd"'), pub); console.log("TAMPER NOT DETECTED"); } catch (e) { console.log("tamper rejected:", e.message); } diff --git a/atomic-canonical-wasm/src/lib.rs b/atomic-canonical-wasm/src/lib.rs new file mode 100644 index 00000000..e4c2d55d --- /dev/null +++ b/atomic-canonical-wasm/src/lib.rs @@ -0,0 +1,107 @@ +//! atomic-canonical for the browser. +//! +//! A web page that holds an atomic identity's key in WebCrypto (as a +//! non-extractable Ed25519 key) signs atomic documents without the key ever +//! reaching Rust — or leaving the browser. This crate does everything *but* +//! the signing, with the same code the CLI uses, so the result is an ordinary +//! atomic attestation (`eddsa-jcs-2022`) that `atomic` verifies: +//! +//! ```js +//! const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); +//! const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); +//! const attested = attachProof(prepared.document, publicKeyBytes, new Uint8Array(sig)); +//! ``` +//! +//! Every function takes and returns JSON as strings and keys/signatures as +//! bytes, so there is no JS object model to keep in step with the Rust types. + +use atomic_canonical::{did, jcs, proof}; +use atomic_identity::keypair::PublicKey; +use atomic_identity::signing::Signature; +use wasm_bindgen::prelude::*; + +fn public_key(bytes: &[u8]) -> Result { + let bytes: &[u8; 32] = bytes + .try_into() + .map_err(|_| JsError::new("an Ed25519 public key is 32 bytes"))?; + PublicKey::from_bytes(bytes).map_err(|e| JsError::new(&e.to_string())) +} + +fn parse(json: &str) -> Result { + serde_json::from_str(json).map_err(|e| JsError::new(&format!("not JSON: {e}"))) +} + +/// A document ready to sign: the document to hand back to [`attach_proof`], +/// and the bytes the signature must cover. +#[wasm_bindgen] +pub struct Prepared { + document: String, + signing_bytes: Vec, +} + +#[wasm_bindgen] +impl Prepared { + /// The document with `attributedTo` and `contentHash` filled in (JSON). + #[wasm_bindgen(getter)] + pub fn document(&self) -> String { + self.document.clone() + } + + /// What to sign with the identity's Ed25519 key. + #[wasm_bindgen(getter, js_name = signingBytes)] + pub fn signing_bytes(&self) -> Vec { + self.signing_bytes.clone() + } +} + +/// Fill in the author and content hash of `document` (JSON) for the key +/// `public_key`, and return it with the bytes to sign. +#[wasm_bindgen(js_name = prepareAttestation)] +pub fn prepare_attestation(document: &str, public_key_bytes: &[u8]) -> Result { + let pk = public_key(public_key_bytes)?; + let prepared = proof::prepare_attestation(parse(document)?, &pk); + Ok(Prepared { + document: prepared.value.to_string(), + signing_bytes: prepared.signing_bytes, + }) +} + +/// Attach the proof for `signature` (64 bytes, over the prepared signing +/// bytes) and check it verifies. Returns the attested document (JSON). +#[wasm_bindgen(js_name = attachProof)] +pub fn attach_proof( + document: &str, + public_key_bytes: &[u8], + signature: &[u8], +) -> Result { + let pk = public_key(public_key_bytes)?; + let sig = Signature::from_slice(signature).map_err(|e| JsError::new(&e.to_string()))?; + let attested = proof::attach_proof(parse(document)?, &pk, &sig); + proof::verify_value(&attested, &pk).map_err(|e| JsError::new(&e.to_string()))?; + Ok(attested.to_string()) +} + +/// Check an attested document (JSON) against a public key. +#[wasm_bindgen(js_name = verifyAttestation)] +pub fn verify_attestation(document: &str, public_key_bytes: &[u8]) -> Result<(), JsError> { + let pk = public_key(public_key_bytes)?; + proof::verify_value(&parse(document)?, &pk).map_err(|e| JsError::new(&e.to_string())) +} + +/// The `did:atomic` identifier for a public key. +#[wasm_bindgen(js_name = didForPublicKey)] +pub fn did_for_public_key(public_key_bytes: &[u8]) -> Result { + Ok(did::did_for_public_key(&public_key(public_key_bytes)?)) +} + +/// The key's base32 form — what atomic writes as the `kid` of its tokens. +#[wasm_bindgen(js_name = publicKeyBase32)] +pub fn public_key_base32(public_key_bytes: &[u8]) -> Result { + Ok(public_key(public_key_bytes)?.to_base32()) +} + +/// RFC 8785 canonical JSON of `document` — the bytes atomic hashes and signs. +#[wasm_bindgen] +pub fn canonicalize(document: &str) -> Result { + Ok(jcs::canonicalize(&parse(document)?)) +} diff --git a/atomic-canonical/src/proof.rs b/atomic-canonical/src/proof.rs index afd3710d..ab45b012 100644 --- a/atomic-canonical/src/proof.rs +++ b/atomic-canonical/src/proof.rs @@ -110,8 +110,32 @@ pub fn substance_view(value: &Value) -> Value { /// path all typed nodes share. Fills `attributedTo` (from the identity's /// `did:atomic`) when absent, computes the content hash over `hashing_view`, /// signs `jcs(signing_view)`, and attaches the proof. Returns the value. -pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> Value { - let did = did::did_for_public_key(&identity.public_key); +/// +/// It is [`prepare_attestation`] → sign → [`attach_proof`]; a signer that +/// holds its key outside this process uses those two halves directly. +pub fn attest_value(value: Value, identity: &Identity, keypair: &KeyPair) -> Value { + let prepared = prepare_attestation(value, &identity.public_key); + let signature = Signer::new(keypair).sign(&prepared.signing_bytes); + attach_proof(prepared.value, &identity.public_key, &signature) +} + +/// A value made ready to sign: `attributedTo` and `contentHash` filled in, +/// and the exact bytes the signature must cover. +#[derive(Debug, Clone)] +pub struct PreparedAttestation { + /// The value to sign — pass it back to [`attach_proof`] unchanged. + pub value: Value, + /// `jcs(signing_view(value))`: what the Ed25519 signature covers. + pub signing_bytes: Vec, +} + +/// First half of [`attest_value`], for signers that hold the key somewhere +/// else — a browser's WebCrypto, a hardware token, a remote signing service. +/// Everything that must agree with [`verify_value`] (the author, the content +/// hash, the canonical bytes) is computed here, so the external signer only +/// ever signs bytes, and the result is an ordinary atomic attestation. +pub fn prepare_attestation(mut value: Value, public_key: &PublicKey) -> PreparedAttestation { + let did = did::did_for_public_key(public_key); if let Some(obj) = value.as_object_mut() { // Fill attributedTo only if there is no non-empty value already. @@ -121,7 +145,7 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> .map(|s| !s.is_empty()) .unwrap_or(false); if !has_author { - obj.insert(PROP_ATTRIBUTED_TO.to_string(), Value::String(did.clone())); + obj.insert(PROP_ATTRIBUTED_TO.to_string(), Value::String(did)); } } @@ -133,13 +157,24 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> } let signing_bytes = jcs::canonicalize(&signing_view(&value)).into_bytes(); - let signature = Signer::new(keypair).sign(&signing_bytes); + PreparedAttestation { + value, + signing_bytes, + } +} + +/// Second half of [`attest_value`]: attach the `eddsa-jcs-2022` proof for a +/// signature over [`PreparedAttestation::signing_bytes`] made by +/// `public_key`'s private key. Does not check the signature — run +/// [`verify_value`] on the result for that. +pub fn attach_proof(mut value: Value, public_key: &PublicKey, signature: &Signature) -> Value { + let did = did::did_for_public_key(public_key); let proof = Proof { type_: PROOF_TYPE.to_string(), cryptosuite: CRYPTOSUITE.to_string(), verification_method: did::verification_method(&did), proof_purpose: PROOF_PURPOSE.to_string(), - proof_value: encode_proof_value(&signature), + proof_value: encode_proof_value(signature), }; if let Some(obj) = value.as_object_mut() { obj.insert( @@ -281,6 +316,25 @@ mod tests { }) } + /// An external signer — handed only the prepared bytes — produces + /// exactly the attestation `attest_value` would, and it verifies. + #[test] + fn prepare_sign_attach_matches_attest_value() { + let (id, kp) = dev_identity(); + let prepared = prepare_attestation(minimal_value(), &kp.public); + let signature = Signer::new(&kp).sign(&prepared.signing_bytes); + let external = attach_proof(prepared.value, &kp.public, &signature); + + assert_eq!(external, attest_value(minimal_value(), &id, &kp)); + verify_value(&external, &kp.public).expect("externally signed value verifies"); + + // A signature over anything else does not. + let wrong = Signer::new(&kp).sign(b"not the prepared bytes"); + let prepared = prepare_attestation(minimal_value(), &kp.public); + let bad = attach_proof(prepared.value, &kp.public, &wrong); + assert!(verify_value(&bad, &kp.public).is_err()); + } + #[test] fn attest_value_then_verify_value_roundtrips() { let (id, kp) = dev_identity(); From c643ed77067aa666cf5667d6d424260cb71a54ef Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 03:19:34 -0500 Subject: [PATCH 02/23] feat(intent): attest with a key held elsewhere `atomic intent attest --prepare` prints the document an attestation signs and the exact bytes to sign, needing only the identity's public key. `--signed ` records an attestation a key holder produced from it: it must be signed by `--identity`'s key and attest the intent as it is now, so a signature over a stale or altered intent is refused. This lets a sandbox that holds only an agent's public identity attest as that agent, with the key kept by a signing service outside it. --- atomic-cli/src/commands/intent/attest.rs | 89 +++++++-- .../intent_attest_external_signer_test.rs | 185 ++++++++++++++++++ 2 files changed, 260 insertions(+), 14 deletions(-) create mode 100644 atomic-cli/tests/intent_attest_external_signer_test.rs diff --git a/atomic-cli/src/commands/intent/attest.rs b/atomic-cli/src/commands/intent/attest.rs index d879adc0..a374834e 100644 --- a/atomic-cli/src/commands/intent/attest.rs +++ b/atomic-cli/src/commands/intent/attest.rs @@ -4,6 +4,7 @@ use clap::Parser; use serde_json::Value; +use atomic_canonical::proof::prepare_attestation; use atomic_canonical::{lift_and_attest, validate_intent, verify}; use atomic_core::pristine::VaultEntryType; use atomic_identity::IdentityStore; @@ -28,6 +29,20 @@ pub struct IntentAttest { /// Output the attested node as JSON-LD. #[arg(long)] pub json: bool, + + /// Don't sign: print what a signer holding the identity's key elsewhere + /// (a browser, a hardware token, a remote signing service) must sign — + /// `{"document": …, "signingBytes": ""}`. Needs only the + /// identity's public key. Complete with `--signed`. + #[arg(long, conflicts_with = "signed")] + pub prepare: bool, + + /// Record an attestation signed elsewhere: a JSON file holding the + /// attested node (the `--prepare` document with its proof attached). It + /// must be signed by `--identity`'s key and attest the intent as it is + /// now. + #[arg(long, value_name = "PATH")] + pub signed: Option, } impl Command for IntentAttest { @@ -59,7 +74,7 @@ impl Command for IntentAttest { ))); } - // Resolve identity + keypair the way `atomic identity sign` does. + // Resolve the identity the way `atomic identity sign` does. let store = IdentityStore::open_default().map_err(|e| { CliError::Internal(anyhow::anyhow!("Failed to open identity store: {}", e)) })?; @@ -79,20 +94,66 @@ impl Command for IntentAttest { .to_string(), })? }; - let keypair = store.load_keypair(&identity.id, None).map_err(|e| { - CliError::Internal(anyhow::anyhow!( - "Failed to load keypair for '{}': {}", - identity.name, - e - )) - })?; - // Attest: lift + fill attributedTo (from the identity's did:atomic when - // absent) + hash + sign. - let node = lift_and_attest(&inputs.frontmatter, &inputs.body, &identity, &keypair) - .map_err(|e| CliError::InvalidArgument { - message: format!("could not attest intent: {e}"), + // Signing elsewhere, step 1: say what to sign. The same preparation + // `lift_and_attest` does (author, content hash, canonical bytes), with + // no private key involved. + if self.prepare { + let prepared = prepare_attestation(unattested.to_value(), &identity.public_key); + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "document": prepared.value, + "signingBytes": data_encoding::BASE64.encode(&prepared.signing_bytes), + })) + .unwrap() + ); + return Ok(()); + } + + let node = if let Some(path) = &self.signed { + // Signing elsewhere, step 2: the signature must be over exactly + // what `--prepare` produces for this intent now — so a signature + // over a stale or altered intent is refused, not recorded. + let text = std::fs::read_to_string(path).map_err(CliError::Io)?; + let signed: Value = + serde_json::from_str(&text).map_err(|e| CliError::InvalidArgument { + message: format!("{} is not JSON: {e}", path.display()), + })?; + let expected = prepare_attestation(unattested.to_value(), &identity.public_key).value; + let mut unsigned = signed.clone(); + if let Some(obj) = unsigned.as_object_mut() { + obj.remove("proof"); + } + if unsigned != expected { + return Err(CliError::InvalidArgument { + message: format!( + "the signed attestation is not of intent {} as it is now (re-run --prepare)", + self.id + ), + }); + } + serde_json::from_value::(signed).map_err(|e| { + CliError::InvalidArgument { + message: format!("not an attested intent: {e}"), + } + })? + } else { + let keypair = store.load_keypair(&identity.id, None).map_err(|e| { + CliError::Internal(anyhow::anyhow!( + "Failed to load keypair for '{}': {}", + identity.name, + e + )) })?; + // Attest: lift + fill attributedTo (from the identity's + // did:atomic when absent) + hash + sign. + lift_and_attest(&inputs.frontmatter, &inputs.body, &identity, &keypair).map_err( + |e| CliError::InvalidArgument { + message: format!("could not attest intent: {e}"), + }, + )? + }; // Belt-and-suspenders: re-gate the ATTESTED node — proof + attributedTo // must now satisfy the gate. @@ -107,7 +168,7 @@ impl Command for IntentAttest { // Self-check: the proof verifies against the signing key before we // write anything to disk. - verify(&node, &keypair.public).map_err(|e| CliError::InvalidArgument { + verify(&node, &identity.public_key).map_err(|e| CliError::InvalidArgument { message: format!("attested intent failed self-verification: {e}"), })?; diff --git a/atomic-cli/tests/intent_attest_external_signer_test.rs b/atomic-cli/tests/intent_attest_external_signer_test.rs new file mode 100644 index 00000000..d6871257 --- /dev/null +++ b/atomic-cli/tests/intent_attest_external_signer_test.rs @@ -0,0 +1,185 @@ +//! `atomic intent attest --prepare` / `--signed`: attesting an intent with a +//! key that is not on this machine. The identity here is public-only — as an +//! agent identity is inside a sandbox whose key lives with a signing +//! service — so plain `attest` can't sign; `--prepare` says what to sign, +//! the key holder signs it, and `--signed` checks and records the result. + +use std::path::Path; +use std::process::{Command, Output}; + +use atomic_identity::{Identity, IdentityStore, IdentityType, IdentityUsage, KeyPair}; + +fn atomic(home: &Path, cwd: &Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(args) + .current_dir(cwd) + .env("HOME", home) + .env("ATOMIC_CONFIG_DIR", home.join(".atomic")) + .output() + .unwrap() +} + +fn ok(out: &Output) -> String { + assert!( + out.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +/// A repo with one intent, a default human identity, and a public-only +/// agent identity whose key only the test holds. +fn setup() -> (tempfile::TempDir, tempfile::TempDir, String, KeyPair) { + let home = tempfile::tempdir().unwrap(); + let repo = tempfile::tempdir().unwrap(); + ok(&atomic( + home.path(), + repo.path(), + &[ + "identity", + "new", + "ada", + "--email", + "ada@example.com", + "--set-default", + ], + )); + ok(&atomic(home.path(), repo.path(), &["init"])); + ok(&atomic( + home.path(), + repo.path(), + &["intent", "new", "Add a greeting"], + )); + + let key = KeyPair::generate(); + let agent = Identity::builder("ada+bot") + .identity_type(IdentityType::Agent) + .usage(IdentityUsage::Bot) + .public_key(key.public.clone()) + .build() + .unwrap(); + IdentityStore::open(&home.path().join(".atomic").join("identities")) + .unwrap() + .save(&agent) + .unwrap(); + + let list = ok(&atomic( + home.path(), + repo.path(), + &["intent", "list", "--json"], + )); + let intents: serde_json::Value = serde_json::from_str(&list).unwrap(); + let id = intents + .as_array() + .and_then(|a| a.first()) + .and_then(|i| i.get("id")) + .and_then(|v| v.as_str()) + .expect("one intent") + .to_string(); + (home, repo, id, key) +} + +/// What the key holder does with `--prepare`'s output. +fn sign_elsewhere(prepared: &str, key: &KeyPair) -> serde_json::Value { + let prepared: serde_json::Value = serde_json::from_str(prepared).unwrap(); + let bytes = data_encoding::BASE64 + .decode(prepared["signingBytes"].as_str().unwrap().as_bytes()) + .unwrap(); + let signature = atomic_identity::signing::Signer::new(key).sign(&bytes); + atomic_canonical::proof::attach_proof(prepared["document"].clone(), &key.public, &signature) +} + +#[test] +fn an_intent_is_attested_by_a_key_held_elsewhere() { + let (home, repo, id, key) = setup(); + let (home, repo) = (home.path(), repo.path()); + + // No key here: plain attest can't sign as the agent. + let out = atomic( + home, + repo, + &["intent", "attest", &id, "--identity", "ada+bot"], + ); + assert!(!out.status.success(), "attest without a key must fail"); + + let prepared = ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--prepare", + ], + )); + let signed = sign_elsewhere(&prepared, &key); + let file = home.join("signed.json"); + std::fs::write(&file, signed.to_string()).unwrap(); + ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--signed", + file.to_str().unwrap(), + ], + )); + + let report: serde_json::Value = serde_json::from_str(&ok(&atomic( + home, + repo, + &["intent", "validate", &id, "--json"], + ))) + .unwrap(); + assert_eq!(report["conforms"], true, "{report}"); + assert_eq!( + signed["attributedTo"], + atomic_canonical::did::did_for_public_key(&key.public) + ); +} + +#[test] +fn a_signature_by_another_key_is_refused() { + let (home, repo, id, _key) = setup(); + let (home, repo) = (home.path(), repo.path()); + let prepared = ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--prepare", + ], + )); + let signed = sign_elsewhere(&prepared, &KeyPair::generate()); + let file = home.join("signed.json"); + std::fs::write(&file, signed.to_string()).unwrap(); + let out = atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--signed", + file.to_str().unwrap(), + ], + ); + assert!( + !out.status.success(), + "a signature by the wrong key must not be recorded" + ); +} From 8f5509597291105b88fbef5e7dc8ad69ae59bfc3 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 08:55:30 -0500 Subject: [PATCH 03/23] fix(agent): never record agent work under the human's key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an agent identity was named (option or ATOMIC_AGENT_IDENTITY) but couldn't be used — missing, not an agent, unreadable — recording fell back to the plus-tag author on the human's default key, putting a person's key on work an agent did, exactly when the caller said it was an agent's. It now records unkeyed instead, and says so in the log. --- atomic-agent/src/identity.rs | 61 +++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/atomic-agent/src/identity.rs b/atomic-agent/src/identity.rs index 0367c60c..b4f44dbd 100644 --- a/atomic-agent/src/identity.rs +++ b/atomic-agent/src/identity.rs @@ -125,7 +125,20 @@ pub fn resolve_agent_author(options: &AgentAuthorOptions<'_>) -> Author { return author; } - // Otherwise: plus-tag the human's identity. Legible, not provable. + // An agent identity was named but can't be used (missing, not an agent, + // unreadable). Never fall back to the human's key then: that would put a + // person's key on work they didn't do, exactly when the caller said it + // was an agent's. Record it unkeyed — honestly anonymous — instead. + if requested_agent_identity(options).is_some() { + log::warn!( + "agent identity {:?} is not usable; recording without a key rather than as the human", + requested_agent_identity(options) + ); + return fallback_agent_author(options); + } + + // No agent identity asked for: plus-tag the human's identity. Legible, + // not provable. match load_default_user_identity(options.identity_dir.as_deref()) { Some(user) => derive_agent_author(&user, options), None => fallback_agent_author(options), @@ -173,13 +186,19 @@ pub const AGENT_IDENTITY_ENV: &str = "ATOMIC_AGENT_IDENTITY"; /// unreadable. Recording a turn must not break because an agent identity was /// mistyped; falling back to the plus-tag author keeps the work attributed to /// *someone* and leaves a debug log explaining why it is not keyed. -fn delegated_agent_author(options: &AgentAuthorOptions<'_>) -> Option { - let name = options +/// The agent identity the caller asked for, explicitly or via +/// `ATOMIC_AGENT_IDENTITY`. +fn requested_agent_identity(options: &AgentAuthorOptions<'_>) -> Option { + options .agent_identity .clone() .or_else(|| std::env::var(AGENT_IDENTITY_ENV).ok()) .map(|n| n.trim().to_string()) - .filter(|n| !n.is_empty())?; + .filter(|n| !n.is_empty()) +} + +fn delegated_agent_author(options: &AgentAuthorOptions<'_>) -> Option { + let name = requested_agent_identity(options)?; let store = match options.identity_dir.as_deref() { Some(dir) => atomic_identity::IdentityStore::open(dir), @@ -784,6 +803,40 @@ mod tests { assert_eq!(author.name, "Claude Code"); } + /// Named but unusable, with a human default present: the work must not + /// be put on the human's key. + #[test] + fn an_unusable_agent_identity_never_falls_back_to_the_human_key() { + use atomic_identity::{Identity, IdentityStore, KeyPair}; + + let dir = TempDir::new().unwrap(); + let mut store = IdentityStore::open(dir.path()).unwrap(); + let key = KeyPair::generate(); + let human = Identity::builder("alice") + .email("alice@example.com") + .public_key(key.public.clone()) + .build() + .unwrap(); + store.save_with_keypair(&human, &key, None).unwrap(); + store.set_default(&human.id).unwrap(); + + for requested in ["nobody+here", "alice"] { + let author = resolve_agent_author(&AgentAuthorOptions { + agent_name: "claude-code", + agent_display_name: "Claude Code", + session_id: "sess1234", + identity_dir: Some(dir.path().to_path_buf()), + agent_identity: Some(requested.to_string()), + }); + assert_ne!( + author.identity.as_deref(), + Some(human.public_key_base32().as_str()), + "{requested}: agent work must not carry the human's key" + ); + assert_eq!(author.name, "Claude Code"); + } + } + /// No agent identity configured: unchanged behavior. #[test] fn no_agent_identity_means_no_keyed_path() { From d06dee6af0dc92a8a3346bd9833bc4b55ddcb995 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 08:56:55 -0500 Subject: [PATCH 04/23] feat(token): name the server a token is for (aud) Self-signed request tokens now carry `aud`: the bare server URL they were minted for, normalised. A server that checks it refuses a token minted for somewhere else, so one leaked from one server can't be replayed at another within its five minutes. Verifiers that ignore unknown claims are unaffected. --- atomic-cli/src/commands/token.rs | 39 +++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/atomic-cli/src/commands/token.rs b/atomic-cli/src/commands/token.rs index 8ff725cb..86124cbf 100644 --- a/atomic-cli/src/commands/token.rs +++ b/atomic-cli/src/commands/token.rs @@ -11,7 +11,8 @@ //! `base64url(header).base64url(claims).base64url(signature)`: //! //! - header: `{"alg":"EdDSA","typ":"JWT","kid":""}` -//! - claims: `{ sub, iat, exp, jti }` (`sub` mirrors the `kid` public key) +//! - claims: `{ sub, aud, iat, exp, jti }` (`sub` mirrors the `kid` public key; +//! `aud` is the server the token is for) //! - signature: `Ed25519_sign(private_key, "header.claims")` //! //! # Acting on behalf of someone (agent identities) @@ -78,6 +79,12 @@ struct Claims { exp: i64, jti: String, + /// The server this token is for (RFC 7519 `aud`): the bare server URL, + /// without a trailing slash. A server that checks it refuses a token + /// minted for somewhere else, so a token leaked from one server can't + /// be replayed at another within its lifetime. + aud: String, + /// RFC 8693 actor claim — present only when an agent is acting. #[serde(skip_serializing_if = "Option::is_none")] act: Option, @@ -160,6 +167,7 @@ fn mint_token(server: &str, identity: &Identity) -> CliResult { let now = Utc::now(); let claims = Claims { sub, + aud: audience(server), iat: now.timestamp(), exp: (now + TOKEN_TTL).timestamp(), jti: Uuid::new_v4().to_string(), @@ -196,10 +204,36 @@ fn mint_token(server: &str, identity: &Identity) -> CliResult { Ok(format!("{signing_input}.{sig_b64}")) } +/// The `aud` for a server URL: scheme and host (and port), no trailing +/// slash, lowercased — so `https://Atomic.Storage/` and +/// `https://atomic.storage` are the same audience. +pub fn audience(server: &str) -> String { + server.trim().trim_end_matches('/').to_ascii_lowercase() +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_token_names_the_server_it_is_for() { + assert_eq!( + audience("https://Atomic.Storage/"), + "https://atomic.storage" + ); + let claims = Claims { + sub: "S".into(), + aud: audience("https://atomic.storage"), + iat: 0, + exp: 1, + jti: "j".into(), + act: None, + dlg: None, + }; + let v: serde_json::Value = serde_json::to_value(&claims).unwrap(); + assert_eq!(v["aud"], "https://atomic.storage"); + } + #[test] fn header_is_eddsa_jwt_with_kid() { let header = JwtHeader { @@ -218,6 +252,7 @@ mod tests { fn a_non_delegated_token_omits_the_actor_claims() { let claims = Claims { sub: "ABCDEF".to_string(), + aud: "https://a".to_string(), iat: 0, exp: 1, jti: "j".to_string(), @@ -237,6 +272,7 @@ mod tests { let agent = "AGENTKEY"; let claims = Claims { sub: human.to_string(), + aud: "https://a".to_string(), iat: 0, exp: 1, jti: "j".to_string(), @@ -277,6 +313,7 @@ mod tests { let now = Utc::now(); let claims = Claims { sub: public_key_b32.clone(), + aud: audience("https://atomic.storage"), iat: now.timestamp(), exp: (now + TOKEN_TTL).timestamp(), jti: Uuid::new_v4().to_string(), From 238384f2df497be09d9a3f5b543e9e281bc92b50 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 13:15:45 -0500 Subject: [PATCH 05/23] feat(repository): render a view's tree without touching disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Repository::materialize_view_entries` hands each entry of a view — path, inode, kind, mode, bytes, content hash, conflict-marker line — to a sink, in path order with directories first, plus the view's Merkle state. Read-only: no working tree, stat cache or conflict state is written, so it runs beside other readers. It serves a remote sandbox its tree and baseline. The per-file rendering is now one function, `render_view_file`, shared with `materialize_parallel`, which keeps writing to disk as before. --- Cargo.lock | 1 + atomic-repository/Cargo.toml | 1 + .../src/repository/materialize.rs | 390 +++++++++++++----- atomic-repository/src/repository/mod.rs | 1 + .../tests/materialize_view_entries_test.rs | 104 +++++ 5 files changed, 404 insertions(+), 93 deletions(-) create mode 100644 atomic-repository/tests/materialize_view_entries_test.rs diff --git a/Cargo.lock b/Cargo.lock index a23eac48..988f7e56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,6 +345,7 @@ dependencies = [ "atomic-identity", "atomic-semantic", "chrono", + "data-encoding", "dirs", "flate2", "fs2", diff --git a/atomic-repository/Cargo.toml b/atomic-repository/Cargo.toml index 397db592..96680d45 100644 --- a/atomic-repository/Cargo.toml +++ b/atomic-repository/Cargo.toml @@ -16,6 +16,7 @@ atomic-identity = { workspace = true } atomic-semantic = { workspace = true } serde = { workspace = true } +data-encoding = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } postcard = { workspace = true } diff --git a/atomic-repository/src/repository/materialize.rs b/atomic-repository/src/repository/materialize.rs index 3a2916a1..24a2f81b 100644 --- a/atomic-repository/src/repository/materialize.rs +++ b/atomic-repository/src/repository/materialize.rs @@ -545,7 +545,6 @@ impl Repository { use atomic_core::output::repo::{ collect_children, FileOutputOptions, MaterializeOptions, OutputItem, }; - use atomic_core::output::RetrieveOptions; use rayon::prelude::*; use std::collections::HashSet as StdHashSet; @@ -725,82 +724,17 @@ impl Repository { .map(|item| { let file_start = std::time::Instant::now(); - // Build retrieve options with the shared change filter - let retrieve_opts = - RetrieveOptions::default().with_change_filter_arc(change_filter_arc.clone()); - - // Inline the output pipeline so we can trace each phase. - use atomic_core::output::repo::{ - output_graph_content_resolved, resolve_conflicts_semantically, - }; - use atomic_core::output::{compute_order, retrieve_graph, Writer}; - use atomic_core::pristine::InodePreloadTxn; - - // Pre-load ALL edges for this file's inode from INODE_GRAPH - // in a single range scan, then run retrieve_graph over the - // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) - // individual B-tree probes. - let preloaded = InodePreloadTxn::from_table(&txn, item.inode, &inode_graph_table) - .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; - - let t_retrieve = std::time::Instant::now(); - let retrieve_result = retrieve_graph(&preloaded, item.position, retrieve_opts) - .map_err(|e| format!("{}: retrieve: {:?}", item.path, e))?; - - if retrieve_result.graph.is_empty() { - return Ok(None); - } - - let vertices = retrieve_result.graph.len_vertices(); - let edges = retrieve_result.edges_traversed; - let retrieve_ms = t_retrieve.elapsed(); - - let t_order = std::time::Instant::now(); - let mut graph = retrieve_result.graph; - let order = compute_order(&mut graph); - let order_ms = t_order.elapsed(); - - let t_content = std::time::Instant::now(); - let resolved = resolve_conflicts_semantically(&preloaded, store, &graph, &order); - let buffer = Vec::with_capacity(graph.total_bytes()); - let mut writer = Writer::new(buffer); - let hash_fn = |node_id: NodeId| -> Option { - if node_id.is_root() { - return None; - } - preloaded.get_external(node_id).ok().flatten() - }; - output_graph_content_resolved( + let Some(content) = render_view_file( + &txn, store, - hash_fn, - &graph, - &order, - &mut writer, - &resolved, - ) - .map_err(|e| format!("{}: content: {:?}", item.path, e))?; - let content = writer.into_inner(); - let content_ms = t_content.elapsed(); - - if content.is_empty() { + &inode_graph_table, + &change_filter_arc, + &name_conflicts, + item, + trace_mat.then_some(file_start), + )? + else { return Ok(None); - } - - // Name-conflict override (rare): when ≥ 2 inodes are alive at - // this path on the view, replace the single-inode content with - // a marker-wrapped rendering of every side so the conflict is - // surfaced instead of silently collapsed (rubric A12). - let content = match name_conflicts.get(&item.path) { - Some(sides) => render_name_conflict( - &txn, - store, - &inode_graph_table, - &change_filter_arc, - &item.path, - sides, - ) - .unwrap_or(content), - None => content, }; // Detect conflict markers in the materialized bytes. This is @@ -860,24 +794,6 @@ impl Repository { std::fs::write(&abs_path, &content) .map_err(|e| format!("{}: write: {}", item.path, e))?; - if trace_mat { - let elapsed = file_start.elapsed(); - if elapsed > std::time::Duration::from_millis(50) { - eprintln!( - "[materialize] SLOW {} bytes={} vertices={} edges={} \ - retrieve={:?} order={:?} content={:?} total={:?}", - item.path, - bytes_written, - vertices, - edges, - retrieve_ms, - order_ms, - content_ms, - elapsed, - ); - } - } - Ok(Some(( item.path.clone(), bytes_written, @@ -1114,6 +1030,294 @@ impl Repository { } } +/// One entry of a view's tree, rendered in memory by +/// [`Repository::materialize_view_entries`] — enough for a client to write +/// the working tree and a baseline index (`status` against it) without the +/// repository. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ViewEntry { + pub path: String, + pub inode: u64, + pub kind: ViewEntryKind, + /// Unix permission bits as recorded. + pub mode: u16, + /// The file's bytes as a checkout of the view would write them (empty + /// for directories). + #[serde(with = "serde_bytes_vec")] + pub content: Vec, + /// `Hash::of(content)` — the baseline a client compares against. + pub hash: Hash, + /// 1-based line of the first conflict marker, if the file is conflicted. + pub conflict_marker_line: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ViewEntryKind { + File, + Directory, + Symlink, +} + +/// What a view looked like when it was rendered. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ViewSnapshot { + pub view: String, + /// The view's Merkle state, base32: a change recorded against this + /// snapshot applies only while the view is still here. + pub state: String, + pub change_count: u64, +} + +mod serde_bytes_vec { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(v: &[u8], s: S) -> Result { + s.serialize_str(&data_encoding::BASE64.encode(v)) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + data_encoding::BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } +} + +impl Repository { + /// Render every entry of `view` and hand each to `sink`, in path order + /// (a directory before what it contains), touching nothing: no working + /// tree, no stat cache, no conflict state. Read-only, so it runs beside + /// other readers — this is what serves a remote sandbox its tree. + /// + /// Files are rendered with the same kernel as + /// [`Repository::materialize_parallel`], in parallel chunks, so memory + /// holds one chunk of contents at a time. + pub fn materialize_view_entries( + &self, + view_name: &str, + mut sink: impl FnMut(ViewEntry) -> Result<(), E>, + ) -> Result, RepositoryError> { + use atomic_core::output::repo::{collect_children, MaterializeOptions}; + use atomic_core::types::Base32; + use rayon::prelude::*; + + const CHUNK: usize = 256; + + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let change_filter_arc = Arc::new(collect_visible_change_ids(&txn, &view)?); + let options = MaterializeOptions::new().with_change_filter_arc(change_filter_arc.clone()); + let mut items = collect_children(&txn, Inode::ROOT, "", &options) + .map_err(|e| RepositoryError::Database(e.to_string()))?; + items.sort_by(|a, b| a.path.cmp(&b.path)); + + let visible = |item: &atomic_core::output::repo::OutputItem| { + item.position.change.is_root() || change_filter_arc.contains(&item.position.change) + }; + let files: Vec<&atomic_core::output::repo::OutputItem> = items + .iter() + .filter(|i| !i.is_directory && visible(i)) + .collect(); + let name_conflicts = collect_name_conflicts( + &txn, + &self.change_store, + &files.iter().map(|i| i.path.as_str()).collect(), + &change_filter_arc, + )?; + for id in change_filter_arc.iter().filter(|id| !id.is_root()) { + if let Ok(Some(hash)) = txn.get_external(*id) { + let _ = self.change_store.load_change(&hash); + } + } + let inode_graph_table = txn + .open_inode_graph_table() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + + // Directories that hold at least one visible file, before their files. + let file_paths: std::collections::HashSet<&str> = + files.iter().map(|i| i.path.as_str()).collect(); + let mut dirs: Vec<&atomic_core::output::repo::OutputItem> = items + .iter() + .filter(|i| i.is_directory) + .filter(|d| { + let prefix = format!("{}/", d.path); + file_paths.iter().any(|p| p.starts_with(&prefix)) + }) + .collect(); + dirs.sort_by(|a, b| a.path.cmp(&b.path)); + for d in dirs { + let entry = ViewEntry { + path: d.path.clone(), + inode: d.inode.get(), + kind: ViewEntryKind::Directory, + mode: d.metadata.permissions, + content: Vec::new(), + hash: Hash::of(&[]), + conflict_marker_line: None, + }; + if let Err(e) = sink(entry) { + return Ok(Err(e)); + } + } + + let store = &self.change_store; + for chunk in files.chunks(CHUNK) { + let rendered: Vec, String>> = chunk + .par_iter() + .map(|item| { + let content = render_view_file( + &txn, + store, + &inode_graph_table, + &change_filter_arc, + &name_conflicts, + item, + None, + )?; + Ok(content.map(|content| ViewEntry { + path: item.path.clone(), + inode: item.inode.get(), + kind: if item.metadata.is_symlink { + ViewEntryKind::Symlink + } else { + ViewEntryKind::File + }, + mode: item.metadata.permissions, + conflict_marker_line: first_conflict_marker_line(&content), + hash: Hash::of(&content), + content, + })) + }) + .collect(); + for r in rendered { + match r { + Ok(Some(entry)) => { + if let Err(e) = sink(entry) { + return Ok(Err(e)); + } + } + Ok(None) => {} + Err(e) => return Err(RepositoryError::Output(e)), + } + } + } + + Ok(Ok(ViewSnapshot { + view: view.name.clone(), + state: view.state.to_base32(), + change_count: view.change_count, + })) + } +} + +/// Render one file of a view in memory: retrieve its graph under the view's +/// change filter, order it, resolve conflicts, and write the bytes a +/// checkout would produce — with a name conflict rendered as markers. The +/// one rendering kernel behind [`Repository::materialize_parallel`] (which +/// writes the result to disk) and [`Repository::materialize_view_entries`] +/// (which hands it to a caller, touching nothing). `Ok(None)`: the file has +/// no content on this view. +fn render_view_file( + txn: &atomic_core::pristine::ReadTxn, + store: &C, + inode_graph_table: &redb::ReadOnlyMultimapTable<&'static [u8; 32], &'static [u8; 24]>, + change_filter_arc: &Arc>, + name_conflicts: &NameConflicts, + item: &atomic_core::output::repo::OutputItem, + trace_from: Option, +) -> Result>, String> { + use atomic_core::output::RetrieveOptions; + // Build retrieve options with the shared change filter + let retrieve_opts = + RetrieveOptions::default().with_change_filter_arc(change_filter_arc.clone()); + + // Inline the output pipeline so we can trace each phase. + use atomic_core::output::repo::{ + output_graph_content_resolved, resolve_conflicts_semantically, + }; + use atomic_core::output::{compute_order, retrieve_graph, Writer}; + use atomic_core::pristine::InodePreloadTxn; + + // Pre-load ALL edges for this file's inode from INODE_GRAPH + // in a single range scan, then run retrieve_graph over the + // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) + // individual B-tree probes. + let preloaded = InodePreloadTxn::from_table(&txn, item.inode, &inode_graph_table) + .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; + + let t_retrieve = std::time::Instant::now(); + let retrieve_result = retrieve_graph(&preloaded, item.position, retrieve_opts) + .map_err(|e| format!("{}: retrieve: {:?}", item.path, e))?; + + if retrieve_result.graph.is_empty() { + return Ok(None); + } + + let vertices = retrieve_result.graph.len_vertices(); + let edges = retrieve_result.edges_traversed; + let retrieve_ms = t_retrieve.elapsed(); + + let t_order = std::time::Instant::now(); + let mut graph = retrieve_result.graph; + let order = compute_order(&mut graph); + let order_ms = t_order.elapsed(); + + let t_content = std::time::Instant::now(); + let resolved = resolve_conflicts_semantically(&preloaded, store, &graph, &order); + let buffer = Vec::with_capacity(graph.total_bytes()); + let mut writer = Writer::new(buffer); + let hash_fn = |node_id: NodeId| -> Option { + if node_id.is_root() { + return None; + } + preloaded.get_external(node_id).ok().flatten() + }; + output_graph_content_resolved(store, hash_fn, &graph, &order, &mut writer, &resolved) + .map_err(|e| format!("{}: content: {:?}", item.path, e))?; + let content = writer.into_inner(); + let content_ms = t_content.elapsed(); + + if content.is_empty() { + return Ok(None); + } + + // Name-conflict override (rare): when ≥ 2 inodes are alive at + // this path on the view, replace the single-inode content with + // a marker-wrapped rendering of every side so the conflict is + // surfaced instead of silently collapsed (rubric A12). + let content = match name_conflicts.get(&item.path) { + Some(sides) => render_name_conflict( + &txn, + store, + &inode_graph_table, + &change_filter_arc, + &item.path, + sides, + ) + .unwrap_or(content), + None => content, + }; + + if let Some(file_start) = trace_from { + let elapsed = file_start.elapsed(); + if elapsed > std::time::Duration::from_millis(50) { + eprintln!( + "[materialize] SLOW {} vertices={vertices} edges={edges} retrieve={retrieve_ms:?} \ + order={order_ms:?} content={content_ms:?} total={elapsed:?}", + item.path, + ); + } + } + Ok(Some(content)) +} + #[cfg(test)] mod conflict_marker_tests { use super::first_conflict_marker_line; diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 4c98e8fe..ec11bfea 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -79,6 +79,7 @@ use crate::RepositoryError; mod deferred_tree; mod filter; mod materialize; +pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; mod sandbox; mod semantic_materialize; mod split; diff --git a/atomic-repository/tests/materialize_view_entries_test.rs b/atomic-repository/tests/materialize_view_entries_test.rs new file mode 100644 index 00000000..31d0fce7 --- /dev/null +++ b/atomic-repository/tests/materialize_view_entries_test.rs @@ -0,0 +1,104 @@ +//! `Repository::materialize_view_entries`: a view's tree, rendered in memory +//! for a remote sandbox — the same bytes a checkout writes, per view, and +//! nothing touched on disk. + +use std::fs; +use std::path::Path; + +use atomic_core::change::{Author, ChangeHeader}; +use atomic_core::types::Hash; +use atomic_repository::{RecordOptions, Repository, SplitOptions, ViewEntry, ViewEntryKind}; +use tempfile::TempDir; + +fn write(repo_path: &Path, name: &str, content: &str) { + let path = repo_path.join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); +} + +fn record(repo: &Repository, message: &str) -> Hash { + let header = ChangeHeader::builder() + .message(message) + .author(Author::new("Test", Some("test@example.com"))) + .build(); + *repo + .record(header, RecordOptions::default()) + .expect("record") + .hash() +} + +fn entries(repo: &Repository, view: &str) -> (Vec, atomic_repository::ViewSnapshot) { + let mut out = Vec::new(); + let snapshot = repo + .materialize_view_entries::<()>(view, |e| { + out.push(e); + Ok(()) + }) + .expect("materialize") + .expect("sink"); + (out, snapshot) +} + +#[test] +fn entries_are_the_view_s_recorded_tree() { + let temp = TempDir::new().unwrap(); + let root = temp.path().to_path_buf(); + let repo = Repository::init(&root).expect("init"); + let view = repo.current_view().to_string(); + + write(&root, "README.md", "hello\n"); + write(&root, "src/main.rs", "fn main() {}\n"); + repo.add("README.md", Default::default()).unwrap(); + repo.add("src/main.rs", Default::default()).unwrap(); + record(&repo, "first"); + write(&root, "README.md", "hello, world\n"); + let second = record(&repo, "second"); + // Present on disk, never recorded: not part of the view. + write(&root, "scratch.txt", "not recorded\n"); + + let before: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + let (list, snapshot) = entries(&repo, &view); + let after: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + assert_eq!( + before.len(), + after.len(), + "nothing written to the working tree" + ); + + let paths: Vec<&str> = list.iter().map(|e| e.path.as_str()).collect(); + assert_eq!( + paths, + vec!["src", "README.md", "src/main.rs"], + "directories first, then files in order" + ); + let readme = list.iter().find(|e| e.path == "README.md").unwrap(); + assert_eq!(readme.kind, ViewEntryKind::File); + assert_eq!(readme.content, b"hello, world\n"); + assert_eq!(readme.hash, Hash::of(b"hello, world\n")); + assert_eq!(readme.conflict_marker_line, None); + assert!(readme.inode > 0); + assert_eq!(snapshot.view, view); + assert_eq!(snapshot.change_count, 2); + assert!(!snapshot.state.is_empty()); + + // Another view without the second change renders its own content. + let mut repo = repo; + repo.split_view(SplitOptions::new("older", vec![second])) + .expect("split"); + let (older_on_source, older_snapshot) = entries(&repo, &view); + let readme = older_on_source + .iter() + .find(|e| e.path == "README.md") + .unwrap(); + assert_eq!( + readme.content, b"hello\n", + "the source view no longer has the second change" + ); + assert_ne!(older_snapshot.state, snapshot.state); +} From ac56c301ad85ff28ce17f042d190848872aecfde Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 13:22:01 -0500 Subject: [PATCH 06/23] Never treat a sandbox's pointer as untracked `.atomic-sandbox` showed up as untracked inside a sandbox, so `record --all` would record it; a remote pointer carries a token. Ignore it, and the sandbox's local cache `.atomic-sandbox.d`, like `.atomic`. --- atomic-repository/src/repository/sandbox.rs | 19 +++++++++++++++++++ atomic-repository/src/status.rs | 11 +++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index adb5cebf..e598d62d 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -525,4 +525,23 @@ mod tests { "the canonical graph must not be cloned into the sandbox" ); } + + #[test] + fn a_sandbox_never_sees_its_pointer_as_untracked() { + let dir = tempdir().unwrap(); + let repo = repo_with_recorded_file(&dir.path().join("repo"), "hello.txt", b"hi\n"); + let sandbox = dir.path().join("agent-1"); + repo.provision_sandbox(&sandbox, "dev").unwrap(); + std::fs::write(sandbox.join("new.txt"), b"new\n").unwrap(); + drop(repo); + + let opened = Repository::open_existing(&sandbox).unwrap(); + let status = opened.status(Default::default()).unwrap(); + let untracked: Vec<_> = status.untracked().map(|e| e.path().to_path_buf()).collect(); + assert_eq!( + untracked, + vec![PathBuf::from("new.txt")], + "the pointer can carry a credential; `record --all` must never pick it up" + ); + } } diff --git a/atomic-repository/src/status.rs b/atomic-repository/src/status.rs index 6c470d28..0cdbf9dd 100644 --- a/atomic-repository/src/status.rs +++ b/atomic-repository/src/status.rs @@ -89,8 +89,10 @@ use crate::ignore::IgnoreRules; // Constants -/// Patterns that are always ignored (internal directories) -const ALWAYS_IGNORED: &[&str] = &[".atomic", ".git"]; +/// Patterns that are always ignored: internal directories, and a sandbox's +/// pointer and local cache — the pointer can carry a credential, and neither +/// belongs in history. +const ALWAYS_IGNORED: &[&str] = &[".atomic", ".git", ".atomic-sandbox", ".atomic-sandbox.d"]; // Error Types @@ -1255,10 +1257,15 @@ mod tests { assert!(is_always_ignored(Path::new("src/.atomic/test"))); assert!(is_always_ignored(Path::new(".git"))); assert!(is_always_ignored(Path::new(".git/objects"))); + assert!(is_always_ignored(Path::new(".atomic-sandbox"))); + assert!(is_always_ignored(Path::new( + ".atomic-sandbox.d/pristine.redb" + ))); assert!(!is_always_ignored(Path::new("src"))); assert!(!is_always_ignored(Path::new("src/main.rs"))); assert!(!is_always_ignored(Path::new("atomic"))); + assert!(!is_always_ignored(Path::new(".atomic-sandbox-notes"))); } #[test] From 5270b046f0fe5decb95923039d9c5c407ee1b94d Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 13:59:49 -0500 Subject: [PATCH 07/23] Reach a repository's owner from a remote sandbox, over the same protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remote sandboxes use the database owner's protocol rather than a second one: iroh is another transport beside the local socket, answered by the same handler. Each remote frame carries a view-scoped sandbox token (in memory only, hashed); a remote caller gets Ping, the provenance requests and Materialize, each checked against its token — its own view, the sessions and turns it started, checkpoints only for changes its view can see. Opening, renewing and closing sandboxes, and shutdown, stay local. The client picks its route from .atomic-sandbox, so provenance from a remote sandbox goes through the same OwnerJournalSink as a local hook. `atomic sandbox create --remote` mints the token and writes the pointer (the owner's iroh address, the view, the token; 0600); `materialize` writes the view's tree inside the sandbox; `renew` and `close` manage the token. The owner's iroh key persists in .atomic so pointers survive a restart. --- Cargo.lock | 2091 ++++++++++++++++- Cargo.toml | 1 + atomic-cli/Cargo.toml | 2 + atomic-cli/src/commands/agent/mod.rs | 2 +- .../commands/agent/{owner.rs => owner/mod.rs} | 327 ++- atomic-cli/src/commands/agent/owner/remote.rs | 410 ++++ .../src/commands/agent/owner/sandbox.rs | 471 ++++ atomic-cli/src/commands/sandbox.rs | 140 +- .../tests/remote_sandbox_integration_test.rs | 197 ++ 9 files changed, 3552 insertions(+), 89 deletions(-) rename atomic-cli/src/commands/agent/{owner.rs => owner/mod.rs} (86%) create mode 100644 atomic-cli/src/commands/agent/owner/remote.rs create mode 100644 atomic-cli/src/commands/agent/owner/sandbox.rs create mode 100644 atomic-cli/tests/remote_sandbox_integration_test.rs diff --git a/Cargo.lock b/Cargo.lock index 988f7e56..a2715754 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -127,6 +162,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async_io_stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" +dependencies = [ + "futures 0.3.32", + "pharos", + "rustc_version", +] + [[package]] name = "atomic-agent" version = "0.18.3" @@ -148,7 +194,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 1.0.69", "tokio", @@ -211,10 +257,12 @@ dependencies = [ "fs2", "git2", "indicatif", + "iroh", "log", "open", + "rand 0.8.7", "rayon", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serial_test", @@ -278,7 +326,7 @@ dependencies = [ "chrono", "data-encoding", "dirs", - "ed25519-dalek", + "ed25519-dalek 2.2.0", "quickcheck", "quickcheck_macros", "rand 0.8.7", @@ -322,7 +370,7 @@ dependencies = [ "chrono", "env_logger", "log", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "tempfile", @@ -356,10 +404,10 @@ dependencies = [ "rayon", "redb 4.2.0", "reflink-copy", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "syntext", "tar", "tempfile", @@ -408,18 +456,53 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64 0.22.1", + "http", + "log", + "url", +] + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -464,6 +547,24 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bs58" version = "0.5.1" @@ -522,6 +623,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -530,9 +637,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" @@ -559,6 +666,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -603,7 +720,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -612,6 +729,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cobs" version = "0.3.0" @@ -627,6 +750,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compression-codecs" version = "0.4.38" @@ -663,12 +796,37 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cordyceps" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9ab7e0ca1d179628fa0172b2b97203c7fa0cd81be2448bd446fb9559ca9261" +dependencies = [ + "loom", + "tracing", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -763,6 +921,34 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -772,9 +958,27 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", - "fiat-crypto", + "digest 0.10.7", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto 0.3.0", + "rand_core 0.10.1", "rustc_version", + "serde", "subtle", "zeroize", ] @@ -787,14 +991,34 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "data-encoding-macro" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6a127ecbb3c4632e1525380e04c0c3fcf8dcb44d32a79ea290d8a36906edcd8" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "c54e03a951783e8b327515db3f2a2fd0e3bed362a96b066f341ce66ed49b4ead" +dependencies = [ + "data-encoding", + "syn 2.0.118", +] [[package]] name = "defmt" @@ -815,7 +1039,7 @@ dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -833,10 +1057,50 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "pem-rfc7468", "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.118", + "unicode-xid", +] + [[package]] name = "dialoguer" version = "0.11.0" @@ -850,14 +1114,30 @@ dependencies = [ "zeroize", ] +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", ] [[package]] @@ -881,6 +1161,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -889,7 +1181,18 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "dlopen2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2c5bd4158e66d1e215c49b837e11d62f3267b30c92f1d171c4d3105e3dc4d4" +dependencies = [ + "libc", + "once_cell", + "winapi", ] [[package]] @@ -898,9 +1201,20 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "serde", - "signature", + "signature 2.2.0", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8 0.11.0", + "serdect", + "signature 3.0.0", ] [[package]] @@ -909,10 +1223,26 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ - "curve25519-dalek", - "ed25519", + "curve25519-dalek 4.1.3", + "ed25519 2.2.3", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" +dependencies = [ + "curve25519-dalek 5.0.0", + "ed25519 3.0.0", + "rand_core 0.10.1", "serde", - "sha2", + "sha2 0.11.0", + "signature 3.0.0", "subtle", "zeroize", ] @@ -950,6 +1280,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-assoc" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0590c4a94da3372e83493b956755a6e2266830b6e4e3b101afe66e3f39477b91" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "env_filter" version = "2.0.0" @@ -1019,6 +1360,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "filetime" version = "0.2.29" @@ -1051,6 +1398,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1106,6 +1459,19 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.1", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -1139,6 +1505,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.32" @@ -1147,7 +1526,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1182,16 +1561,31 @@ dependencies = [ ] [[package]] -name = "generic-array" -version = "0.14.7" +name = "generator" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "54ade96dc9003043bce7c035c85a9df5a858bfb2039c5a2e6fdf00f324f6c551" dependencies = [ - "typenum", - "version_check", -] - -[[package]] + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] name = "getrandom" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1230,6 +1624,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "git2" version = "0.19.0" @@ -1258,6 +1662,18 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.15" @@ -1300,6 +1716,9 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" @@ -1320,7 +1739,7 @@ dependencies = [ "hash32", "rustc_version", "serde", - "spin", + "spin 0.9.8", "stable_deref_trait", ] @@ -1369,6 +1788,21 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -1383,6 +1817,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1428,7 +1863,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -1440,7 +1875,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", - "system-configuration", + "system-configuration 0.7.0", "tokio", "tower-service", "tracing", @@ -1553,6 +1988,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "identity-hash" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfdd7caa900436d8f13b2346fe10257e0c05c1f1f9e351f4f5d57c03bd5f45da" + [[package]] name = "idna" version = "1.1.0" @@ -1574,6 +2015,26 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "igd-next" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7238d487a9aff61f81b5ab41c0a841532a115a398b5fa92a2fadd0885e2581" +dependencies = [ + "attohttpc", + "bytes", + "futures 0.3.32", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "rand 0.10.2", + "tokio", + "url", + "xmltree", +] + [[package]] name = "ignore" version = "0.4.28" @@ -1613,12 +2074,197 @@ dependencies = [ "web-time", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "iroh" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f8d1cfffc83efe39a1031aab423ce09cb8048071baba550508931e9a81ce46" +dependencies = [ + "backon", + "blake3", + "bytes", + "cfg_aliases", + "ctutils", + "data-encoding", + "derive_more", + "ed25519-dalek 3.0.0", + "futures-util", + "getrandom 0.4.3", + "http", + "ipnet", + "iroh-base", + "iroh-dns", + "iroh-metrics", + "iroh-relay", + "n0-error", + "n0-future", + "n0-watcher", + "netwatch", + "noq", + "noq-proto", + "noq-udp", + "papaya", + "pin-project", + "portable-atomic", + "portmapper", + "rand 0.10.2", + "reqwest 0.13.5", + "rustc-hash", + "rustls", + "rustls-pki-types", + "serde", + "smallvec", + "strum", + "time", + "tokio", + "tokio-stream", + "tokio-util 0.7.18", + "tracing", + "url", + "wasm-bindgen-futures", +] + +[[package]] +name = "iroh-base" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd1efd1aaecd68d4d1b0727a30c5811f43fb822843e48f65f6e5db3b7256cc9" +dependencies = [ + "curve25519-dalek 5.0.0", + "data-encoding", + "data-encoding-macro", + "derive_more", + "ed25519-dalek 3.0.0", + "getrandom 0.4.3", + "n0-error", + "rand 0.10.2", + "serde", + "url", + "zeroize", +] + +[[package]] +name = "iroh-dns" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80276761b56e904e26ea71d0863beef0bb8d5ab6b639cbd1338e4b615209e3a1" +dependencies = [ + "arc-swap", + "cfg_aliases", + "derive_more", + "iroh-base", + "n0-dns-resolver", + "n0-error", + "n0-future", + "portable-atomic", + "rand 0.10.2", + "rustls", + "simple-dns", + "strum", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "iroh-metrics" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291065721ad7c477b972e581bbc528df031dc8eb5e39fe1ff3300ae5dfb157ef" +dependencies = [ + "iroh-metrics-derive", + "itoa", + "n0-error", + "portable-atomic", + "ryu", + "serde", + "tracing", +] + +[[package]] +name = "iroh-metrics-derive" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae5f0c4405d1fbc9fb16ff422ca40620e93dc36c30ecaba0c2aee3992b7bd48" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "iroh-relay" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beb2294a9749d6a25fd7cd8bcf0fccd932f20716d4f967d85d3f23c7135ae6a2" +dependencies = [ + "blake3", + "bytes", + "cfg_aliases", + "data-encoding", + "derive_more", + "getrandom 0.4.3", + "http", + "http-body-util", + "hyper", + "hyper-util", + "iroh-base", + "iroh-dns", + "iroh-metrics", + "lru", + "n0-error", + "n0-future", + "noq", + "noq-proto", + "num_enum", + "pin-project", + "postcard", + "rand 0.10.2", + "reqwest 0.13.5", + "rustls", + "rustls-pki-types", + "serde", + "serde_bytes", + "strum", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", + "tokio-websockets", + "tracing", + "url", + "webpki-roots", + "ws_stream_wasm", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -1681,7 +2327,81 @@ checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.118", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", ] [[package]] @@ -1705,6 +2425,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.186" @@ -1798,18 +2524,55 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "mac-addr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3d25b0e0b648a86960ac23b7ad4abb9717601dec6f66c165f5b037f3f03065f" + [[package]] name = "maplit" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1852,6 +2615,84 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "n0-dns-resolver" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535c99c9a786bee714155d2ee4fc2477405c2c43536e1bc6a51de09a0d1448f7" +dependencies = [ + "derive_more", + "ipconfig", + "jni 0.22.4", + "lru", + "n0-error", + "n0-future", + "ndk-context", + "rand 0.10.2", + "reqwest 0.13.5", + "rustc-hash", + "rustls", + "simple-dns", + "system-configuration 0.8.0", + "tokio", + "tokio-rustls", + "tracing", + "webpki-roots", +] + +[[package]] +name = "n0-error" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c59ea174675ce6bc196878851e5049e11b8b1f9af3ba87e4d6df8d828bb001" +dependencies = [ + "n0-error-macros", + "spez", +] + +[[package]] +name = "n0-error-macros" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4412346410d6616f3668c40e53c298e5320c7b856f7a960e5914e442601be79f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "n0-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ab99dfb861450e68853d34ae665243a88b8c493d01ba957321a1e9b2312bbe" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util 0.7.18", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "n0-watcher" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc618745ad0b7414b149d0517ad8b5573b2fb4d4e2717add3d2446ce1fdd826" +dependencies = [ + "derive_more", + "n0-error", + "n0-future", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -1869,6 +2710,257 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "netdev" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096c44b66a9b09b99b4dd36b1c295ff3a492039ccecaf55466bd6ddcdba59968" +dependencies = [ + "block2", + "dispatch2", + "dlopen2", + "ipnet", + "jni 0.21.1", + "libc", + "mac-addr", + "ndk-context", + "netlink-packet-core 0.8.2", + "netlink-packet-route 0.31.0", + "netlink-sys 0.8.8", + "objc2", + "objc2-core-foundation", + "objc2-core-wlan", + "objc2-foundation", + "objc2-system-configuration", + "once_cell", + "plist", + "windows-sys 0.61.2", +] + +[[package]] +name = "netdev" +version = "0.46.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c52c2584961c68f5a6e3356797344061f818c93b8acc9cd6d7e284795e563e" +dependencies = [ + "block2", + "dispatch2", + "dlopen2", + "ipnet", + "jni 0.21.1", + "libc", + "mac-addr", + "ndk-context", + "netlink-packet-core 0.9.0", + "netlink-packet-route 0.33.0", + "netlink-sys 0.9.0", + "objc2-core-foundation", + "objc2-system-configuration", + "once_cell", + "plist", + "windows-sys 0.61.2", +] + +[[package]] +name = "netlink-packet-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b897d7bd4f0af82e68d40d0344cf37e97f9c97ddf74a098de3e4da05e96ca395" +dependencies = [ + "paste", +] + +[[package]] +name = "netlink-packet-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6e2daf9a8c2e21302714706860a0e6be02e03d17294ecc66b354ea7d4059dd" + +[[package]] +name = "netlink-packet-route" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2288fcb784eb3defd5fb16f4c4160d5f477de192eac730f43e1d11c24d9a007" +dependencies = [ + "bitflags 2.13.0", + "libc", + "log", + "netlink-packet-core 0.8.2", +] + +[[package]] +name = "netlink-packet-route" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59d48ffc8b73a506e1ff0878528c72668f2bfbc3d875b6be890a96be5d0d5576" +dependencies = [ + "bitflags 2.13.0", + "libc", + "log", + "netlink-packet-core 0.9.0", + "zerocopy", +] + +[[package]] +name = "netlink-proto" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93af8261786086024cd5e96e0a991dd65ced07bbf7c233a487bbc96b971d5539" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "log", + "netlink-packet-core 0.8.2", + "netlink-sys 0.8.8", + "thiserror 2.0.18", +] + +[[package]] +name = "netlink-sys" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6c30ed10fa69cc491d491b85cc971f6bdeb8e7367b7cde2ee6cc878d583fae" +dependencies = [ + "bytes", + "futures-util", + "libc", + "log", + "tokio", +] + +[[package]] +name = "netlink-sys" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c99d38e00b420df49e940fe0826e667c3dad82ac68eb4c2bbf754c1c14a83a10" +dependencies = [ + "bytes", + "libc", + "log", +] + +[[package]] +name = "netwatch" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39da9cad5f23aa43401f09497d3b280e51b5feba115d9ffbf38ca35d6ff95e96" +dependencies = [ + "atomic-waker", + "bytes", + "cfg_aliases", + "derive_more", + "ipnet", + "js-sys", + "libc", + "n0-error", + "n0-future", + "n0-watcher", + "netdev 0.45.1", + "netdev 0.46.3", + "netlink-packet-core 0.8.2", + "netlink-packet-route 0.31.0", + "netlink-proto", + "netlink-sys 0.8.8", + "noq-udp", + "objc2-core-foundation", + "objc2-system-configuration", + "pin-project-lite", + "serde", + "socket2", + "time", + "tokio", + "tokio-util 0.7.18", + "tracing", + "web-sys", + "windows", + "windows-result", + "wmi", +] + +[[package]] +name = "noq" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b78be567e796cfa74bb9bdc4117790af2d505eb887019cf8803244353eb09d89" +dependencies = [ + "bytes", + "cfg_aliases", + "derive_more", + "noq-proto", + "noq-udp", + "pin-project-lite", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "web-time", +] + +[[package]] +name = "noq-proto" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c1e5b6fe668491eca022f745a0a9402585626c73a7b839b3424ace15d6a9c8f" +dependencies = [ + "aes-gcm", + "bytes", + "derive_more", + "enum-assoc", + "getrandom 0.4.3", + "identity-hash", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "sorted-index-buffer", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "noq-udp" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc50afea61aff926e150a36c31f06094608848e114a3fe667d76ec233163b1c" +dependencies = [ + "cfg_aliases", + "libc", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-traits" version = "0.2.19" @@ -1878,12 +2970,124 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "number_prefix" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.0", + "block2", + "dispatch2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-core-wlan" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e34919aba0d701380d911702455038a8a3587467fe0141d6a71501e7ffe48" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-security", + "objc2-security-foundation", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-security" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe137109bd1e8b5a99390f77a7d8b2961dafc1a1c5db8f2e60329ad6d895a" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-security-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef76382e9cedd18123099f17638715cc3d81dba3637d4c0d39ab69df2ef345a5" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "bitflags 2.13.0", + "dispatch2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-security", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1896,6 +3100,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "open" version = "5.4.0" @@ -1928,7 +3138,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1969,7 +3179,23 @@ dependencies = [ name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "papaya" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da2442474a9404698c42509b8967f437249dbc7b50493e83020333d3943ec0ae" +dependencies = [ + "equivalent", + "seize", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -1994,12 +3220,57 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures 0.3.32", + "rustc_version", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2012,8 +3283,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.0", ] [[package]] @@ -2022,11 +3303,39 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plist" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896bade328c13f7042a297ea5ac5b0951f6cf989dea5f32c2fd98da398195cb" +dependencies = [ + "base64 0.23.1", + "indexmap", + "quick-xml", + "serde", + "time", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +dependencies = [ + "serde", +] [[package]] name = "portable-atomic-util" @@ -2037,6 +3346,35 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "portmapper" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca97242f016e090a25330613bc2382aab65eb22f9149dbe84d8f8e711d49a530" +dependencies = [ + "base64 0.22.1", + "bytes", + "derive_more", + "hyper-util", + "igd-next", + "iroh-metrics", + "libc", + "n0-error", + "n0-future", + "netwatch", + "num_enum", + "rand 0.10.2", + "serde", + "smallvec", + "socket2", + "time", + "tokio", + "tokio-util 0.7.18", + "tower-layer", + "tracing", + "url", +] + [[package]] name = "postcard" version = "1.1.3" @@ -2047,9 +3385,21 @@ dependencies = [ "embedded-io 0.4.0", "embedded-io 0.6.1", "heapless", + "postcard-derive", "serde", ] +[[package]] +name = "postcard-derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0232bd009a197ceec9cc881ba46f727fcd8060a2d8d6a9dde7a69030a6fe2bb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -2059,6 +3409,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2068,6 +3424,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.15+spec-1.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -2077,6 +3442,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", +] + [[package]] name = "quickcheck" version = "1.1.0" @@ -2096,7 +3470,7 @@ checksum = "a9a28b8493dd664c8b171dd944da82d933f7d456b829bfb236738e1fe06c5ba4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2366,7 +3740,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -2402,11 +3776,48 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -2479,6 +3890,7 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -2487,6 +3899,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.15.0" @@ -2497,6 +3921,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -2538,6 +3989,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -2567,12 +4024,24 @@ dependencies = [ "libc", ] +[[package]] +name = "seize" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" + [[package]] name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + [[package]] name = "serde" version = "1.0.228" @@ -2624,7 +4093,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2662,6 +4131,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "serial_test" version = "3.5.0" @@ -2684,9 +4163,15 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -2695,7 +4180,27 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", ] [[package]] @@ -2729,18 +4234,52 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "simple-dns" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b6f884fa9a8d48101774bfbd3aeb81e968dd22cffd19a372da69f183db22c1a" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "slab" version = "0.4.12" @@ -2763,6 +4302,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sorted-index-buffer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea06cc588e43c632923a55450401b8f25e628131571d4e1baea1bdfdb2b5ed06" + +[[package]] +name = "spez" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87e960f4dca2788eeb86bbdde8dd246be8948790b7618d656e68f9b720a86e8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "spin" version = "0.9.8" @@ -2772,6 +4328,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -2779,7 +4341,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der 0.8.2", ] [[package]] @@ -2795,10 +4367,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" [[package]] -name = "strsim" -version = "0.11.1" +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "subtle" @@ -2817,6 +4410,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -2834,7 +4438,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2870,6 +4474,17 @@ dependencies = [ "system-configuration-sys", ] +[[package]] +name = "system-configuration" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501336eb7ba9e417300a6a0fa985721065467aa83a6dcf0422a8e43e4c0328fa" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.10.1", + "system-configuration-sys", +] + [[package]] name = "system-configuration-sys" version = "0.6.0" @@ -2940,7 +4555,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2951,7 +4566,47 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "js-sys", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", ] [[package]] @@ -3004,7 +4659,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3027,6 +4682,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util 0.7.18", +] + [[package]] name = "tokio-util" version = "0.6.10" @@ -3052,10 +4719,34 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] +[[package]] +name = "tokio-websockets" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52efb639344a7c6adb8e62c6f3d2c19c001ff1b79a5041ba1c6ed42e19c6aa5" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-sink", + "getrandom 0.4.3", + "http", + "httparse", + "rand 0.10.2", + "ring", + "rustls-pki-types", + "sha1_smol", + "simdutf8", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", +] + [[package]] name = "toml" version = "0.8.23" @@ -3064,8 +4755,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", ] [[package]] @@ -3077,6 +4768,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -3086,9 +4786,30 @@ dependencies = [ "indexmap", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "toml_write", - "winnow", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", ] [[package]] @@ -3153,10 +4874,23 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -3164,6 +4898,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -3294,12 +5058,34 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -3316,6 +5102,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -3348,6 +5135,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3436,7 +5229,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -3462,6 +5255,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "watchman_client" version = "0.9.0" @@ -3500,6 +5306,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.8" @@ -3509,6 +5324,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -3593,7 +5414,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3604,7 +5425,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3652,6 +5473,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -3688,6 +5518,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -3728,6 +5573,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3740,6 +5591,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3752,6 +5609,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3770,6 +5633,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3782,6 +5651,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3794,6 +5669,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3806,6 +5687,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3827,18 +5714,61 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wmi" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c81b85c57a57500e56669586496bf2abd5cf082b9d32995251185d105208b64" +dependencies = [ + "chrono", + "futures 0.3.32", + "log", + "serde", + "thiserror 2.0.18", + "windows", + "windows-core", +] + [[package]] name = "writeable" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "ws_stream_wasm" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc" +dependencies = [ + "async_io_stream", + "futures 0.3.32", + "js-sys", + "log", + "pharos", + "rustc_version", + "send_wrapper", + "thiserror 2.0.18", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "xattr" version = "1.6.1" @@ -3849,6 +5779,21 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + +[[package]] +name = "xmltree" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" +dependencies = [ + "xml-rs", +] + [[package]] name = "xxhash-rust" version = "0.8.16" @@ -3874,7 +5819,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -3895,7 +5840,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3915,7 +5860,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -3924,6 +5869,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "zerotrie" @@ -3955,7 +5914,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 17e609ad..f928bbb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ bs58 = "0.5" blake3 = "1.5" ed25519-dalek = { version = "2.1", features = ["serde"] } rand = "0.8" +iroh = "1" # Compression zstd = "0.13" diff --git a/atomic-cli/Cargo.toml b/atomic-cli/Cargo.toml index a60709a4..2f58f426 100644 --- a/atomic-cli/Cargo.toml +++ b/atomic-cli/Cargo.toml @@ -65,6 +65,8 @@ data-encoding = "2.6" # Hashing (source-content digest for canonical intent sidecars) blake3 = { workspace = true } +iroh = { workspace = true } +rand = { workspace = true } # Open URLs in default browser (for device auth flow) open = "5" diff --git a/atomic-cli/src/commands/agent/mod.rs b/atomic-cli/src/commands/agent/mod.rs index 67fa5202..5f1055f5 100644 --- a/atomic-cli/src/commands/agent/mod.rs +++ b/atomic-cli/src/commands/agent/mod.rs @@ -43,7 +43,7 @@ mod enable; mod explain; mod hooks; mod lifecycle; -mod owner; +pub(crate) mod owner; mod status; use clap::{Args, Subcommand}; diff --git a/atomic-cli/src/commands/agent/owner.rs b/atomic-cli/src/commands/agent/owner/mod.rs similarity index 86% rename from atomic-cli/src/commands/agent/owner.rs rename to atomic-cli/src/commands/agent/owner/mod.rs index 537a0592..84a95bf1 100644 --- a/atomic-cli/src/commands/agent/owner.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -3,6 +3,14 @@ //! redb permits one writable process to open a database. This service elects //! that process with an OS-backed file lock and exposes a small, versioned local //! protocol so short-lived agent hooks never open `changes.redb` themselves. +//! +//! The same protocol reaches remote sandboxes over iroh (see [`remote`] and +//! [`sandbox`]): a sandbox on another machine records provenance and reads its +//! view through the owner of the repository it belongs to, with every request +//! checked against its sandbox token. + +mod remote; +mod sandbox; use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; @@ -34,6 +42,9 @@ use uuid::Uuid; use crate::commands::Command; use crate::error::CliResult; +use self::remote::{RemotePointer, TokenRegistry}; +use self::sandbox::Caller; + const PROTOCOL_VERSION: u16 = 1; // Local IPC only; a single oversized envelope (e.g. a huge recovered reasoning // block) must still fit one frame even after client-side chunking. @@ -106,6 +117,9 @@ struct ReserveArgs { struct RequestFrame { version: u16, request_id: String, + /// A remote sandbox's token; local callers send none. + #[serde(default, skip_serializing_if = "Option::is_none")] + token: Option, request: OwnerRequest, } @@ -175,6 +189,28 @@ enum OwnerRequest { turn_number: u32, }, Shutdown, + /// Local only: mint a sandbox token for `view`, reachable over iroh. + OpenSandbox { + view: String, + #[serde(default)] + acting_as: Option, + ttl_secs: i64, + }, + /// Local only: extend `view`'s sandbox token. + RenewSandbox { + view: String, + ttl_secs: i64, + }, + /// Local only: end `view`'s sandbox token now. + CloseSandbox { + view: String, + }, + /// A view's tree: `MaterializeEntry` frames, then `Materialized`. A + /// remote caller gets its token's view; a local one names it. + Materialize { + #[serde(default)] + view: Option, + }, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -197,6 +233,8 @@ pub(crate) enum OwnerResponse { // Additive v1 capability: old clients ignore it, old owners omit it. #[serde(default)] frozen_envelope_paging: bool, + #[serde(default)] + remote_sandboxes: bool, }, ProvenanceTurn { turn: StoredProvenanceTurn, @@ -221,6 +259,25 @@ pub(crate) enum OwnerResponse { ShuttingDown { pid: u32, }, + SandboxOpened { + remote: iroh::EndpointAddr, + view: String, + token: String, + expires: String, + }, + SandboxRenewed { + expires: String, + }, + SandboxClosed { + revoked: bool, + }, + MaterializeEntry { + entry: atomic_repository::ViewEntry, + }, + Materialized { + snapshot: atomic_repository::ViewSnapshot, + entries: u64, + }, Error { code: String, message: String, @@ -784,6 +841,10 @@ fn runtime() -> anyhow::Result { /// Start the repository owner or reconnect when another process won election. pub(crate) fn start_or_reconnect(repository: &Path) -> anyhow::Result { + // A remote sandbox's owner runs elsewhere; there is nothing to start. + if remote::find_remote_pointer(repository).is_some() { + return request(repository, OwnerRequest::Ping); + } if let Ok(response) = request(repository, OwnerRequest::Ping) { return Ok(response); } @@ -860,16 +921,29 @@ where .map_err(|_| anyhow!("database owner client thread panicked"))? } -/// Send one request to an already-running owner. +/// Send one request to an already-running owner: the local one, or — in a +/// remote sandbox — the one its pointer names, with its token. fn request(repository: &Path, request_body: OwnerRequest) -> anyhow::Result { - let dot_dir = Repository::canonical_dot_dir(repository)?; - let endpoint = endpoint_name(&dot_dir); + let remote = remote::find_remote_pointer(repository).map(|(_, pointer)| pointer); let frame = RequestFrame { version: PROTOCOL_VERSION, request_id: Uuid::new_v4().to_string(), + token: remote.as_ref().map(|pointer| pointer.token.clone()), request: request_body, }; - let response = runtime()?.block_on(exchange(&endpoint, &frame))?; + let response = match &remote { + Some(pointer) => runtime()?.block_on(async { + let mut stream = remote::BiStream::dial(pointer.remote.clone()).await?; + write_frame(&mut stream, &frame).await?; + let response = read_frame::(&mut stream).await; + stream.close().await; + response + })?, + None => { + let dot_dir = Repository::canonical_dot_dir(repository)?; + runtime()?.block_on(exchange(&endpoint_name(&dot_dir), &frame))? + } + }; if response.version != PROTOCOL_VERSION { return Err(anyhow!( "database owner protocol mismatch: client {}, server {}", @@ -892,6 +966,98 @@ fn request(repository: &Path, request_body: OwnerRequest) -> anyhow::Result, + ttl_secs: i64, +) -> anyhow::Result { + start_or_reconnect(repository)?; + match request( + repository, + OwnerRequest::OpenSandbox { + view: view.to_string(), + acting_as, + ttl_secs, + }, + )? { + OwnerResponse::SandboxOpened { + remote, + view, + token, + expires, + } => Ok(OpenedSandbox { + pointer: serde_json::to_value(RemotePointer { + remote, + view, + token, + })?, + expires, + }), + other => Err(unexpected_response("open sandbox", other)), + } +} + +/// Extend `view`'s sandbox token; returns the new expiry. +pub(crate) fn renew_sandbox( + repository: &Path, + view: &str, + ttl_secs: i64, +) -> anyhow::Result { + match request( + repository, + OwnerRequest::RenewSandbox { + view: view.to_string(), + ttl_secs, + }, + )? { + OwnerResponse::SandboxRenewed { expires } => Ok(expires), + other => Err(unexpected_response("renew sandbox", other)), + } +} + +/// End `view`'s sandbox token; whether there was one. +pub(crate) fn close_sandbox(repository: &Path, view: &str) -> anyhow::Result { + match request( + repository, + OwnerRequest::CloseSandbox { + view: view.to_string(), + }, + )? { + OwnerResponse::SandboxClosed { revoked } => Ok(revoked), + other => Err(unexpected_response("close sandbox", other)), + } +} + +/// In a remote sandbox: write its view's tree into the sandbox root. +/// Returns the root and the number of entries written. +pub(crate) fn materialize_remote_sandbox(start: &Path) -> anyhow::Result<(PathBuf, u64)> { + let (root, pointer) = remote::find_remote_pointer(start) + .ok_or_else(|| anyhow!("{} is not in a remote sandbox", start.display()))?; + let frame = RequestFrame { + version: PROTOCOL_VERSION, + request_id: Uuid::new_v4().to_string(), + token: Some(pointer.token.clone()), + request: OwnerRequest::Materialize { view: None }, + }; + let written = runtime()?.block_on(async { + let mut stream = remote::BiStream::dial(pointer.remote.clone()).await?; + write_frame(&mut stream, &frame).await?; + let written = sandbox::receive_materialized(&mut stream, &frame.request_id, &root).await; + stream.close().await; + written + })?; + Ok((root, written)) +} + fn serve(repository: &Path) -> CliResult<()> { let dot_dir = Repository::canonical_dot_dir(repository)?; let owner_lock = acquire_owner_lock(&dot_dir)?; @@ -901,7 +1067,8 @@ fn serve(repository: &Path) -> CliResult<()> { .with_context(|| format!("failed to open {}", store_path.display()))?, ); let endpoint = endpoint_name(&dot_dir); - runtime()?.block_on(run_server(&endpoint, store))?; + let owner = OwnerState::new(store, dot_dir.clone()); + runtime()?.block_on(run_server(&endpoint, owner))?; FileExt::unlock(&owner_lock).context("failed to release database-owner lock")?; Ok(()) } @@ -965,6 +1132,7 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram OwnerResponse::Pong { pid: std::process::id(), frozen_envelope_paging: true, + remote_sandboxes: true, }, false, ), @@ -1298,6 +1466,17 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram }, true, ), + // Answered by `handle_connection`, which needs the whole owner. + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + | OwnerRequest::Materialize { .. } => ( + OwnerResponse::Error { + code: "internal".to_string(), + message: "sandbox requests need the owner's connection handler".to_string(), + }, + false, + ), }; ( @@ -1310,22 +1489,119 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram ) } -async fn handle_connection( - mut stream: S, +/// Everything a running owner holds. +pub(crate) struct OwnerState { store: Arc, + /// The repository root. + root: PathBuf, + dot_dir: PathBuf, + tokens: TokenRegistry, + /// Bound on the first `OpenSandbox`; accepts remote callers from then on. + remote: tokio::sync::OnceCell, shutdown: Arc, +} + +impl OwnerState { + fn new(store: Arc, dot_dir: PathBuf) -> Arc { + let root = dot_dir + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| dot_dir.clone()); + Arc::new(Self { + store, + root, + dot_dir, + tokens: TokenRegistry::default(), + remote: tokio::sync::OnceCell::new(), + shutdown: Arc::new(Notify::new()), + }) + } + + /// The owner's iroh endpoint, bound (and accepting) on first use. + async fn remote_endpoint(self: &Arc) -> anyhow::Result<&iroh::Endpoint> { + self.remote + .get_or_try_init(|| async { + let endpoint = remote::bind(&self.dot_dir, remote::offline()).await?; + spawn_accept_remote(endpoint.clone(), Arc::clone(self)); + Ok(endpoint) + }) + .await + } +} + +/// A plain function between binding and accepting: the accept loop answers +/// `OpenSandbox`, which binds, so the two futures can't contain each other. +fn spawn_accept_remote(endpoint: iroh::Endpoint, owner: Arc) { + tokio::spawn(accept_remote(endpoint, owner)); +} + +/// Answer remote callers: each bi-stream is one request, as on the socket. +async fn accept_remote(endpoint: iroh::Endpoint, owner: Arc) { + while let Some(incoming) = endpoint.accept().await { + let owner = Arc::clone(&owner); + tokio::spawn(async move { + let Ok(connection) = incoming.await else { + return; + }; + while let Some(stream) = remote::BiStream::accept(&connection).await { + let owner = Arc::clone(&owner); + tokio::spawn(async move { + if let Err(error) = handle_connection(stream, owner, Caller::Remote).await { + log::warn!("database-owner remote request failed: {error}"); + } + }); + } + }); + } +} + +async fn handle_connection( + mut stream: S, + owner: Arc, + caller: Caller, ) -> anyhow::Result<()> where S: AsyncRead + AsyncWrite + Unpin, { let request: RequestFrame = read_frame(&mut stream).await?; - let (response, should_shutdown) = handle_request(&store, request); - write_frame(&mut stream, &response).await?; + let respond = |request_id: String, response| ResponseFrame { + version: PROTOCOL_VERSION, + request_id, + response, + }; + let grant = match sandbox::authorize(&owner, caller, &request) { + Ok(grant) => grant, + Err(refusal) => { + write_frame(&mut stream, &respond(request.request_id, *refusal)).await?; + stream.shutdown().await?; + return Ok(()); + } + }; + let mut should_shutdown = false; + match &request.request { + OwnerRequest::Materialize { .. } if request.version == PROTOCOL_VERSION => { + sandbox::materialize(&owner, grant, request, &mut stream).await?; + } + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + if request.version == PROTOCOL_VERSION => + { + let response = sandbox::admin(&owner, request.request.clone()).await; + write_frame(&mut stream, &respond(request.request_id, response)).await?; + } + _ => { + let (response, shutdown) = handle_request(&owner.store, request); + sandbox::note_response(&owner.tokens, grant.as_ref(), &response.response); + write_frame(&mut stream, &response).await?; + should_shutdown = shutdown; + } + } stream.shutdown().await?; if should_shutdown { // `notify_one` retains a permit if the accept loop is between polls, // preventing a shutdown request from being acknowledged but lost. - shutdown.notify_one(); + owner.shutdown.notify_one(); } Ok(()) } @@ -1459,7 +1735,7 @@ async fn connect_owner_pipe( } #[cfg(unix)] -async fn run_server(endpoint: &str, store: Arc) -> anyhow::Result<()> { +async fn run_server(endpoint: &str, owner: Arc) -> anyhow::Result<()> { use tokio::net::UnixListener; let path = Path::new(endpoint); @@ -1469,16 +1745,15 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } let listener = UnixListener::bind(path) .with_context(|| format!("failed to bind database owner at {endpoint}"))?; - let shutdown = Arc::new(Notify::new()); + let shutdown = Arc::clone(&owner.shutdown); loop { tokio::select! { accepted = listener.accept() => { let (stream, _) = accepted?; - let store = Arc::clone(&store); - let shutdown = Arc::clone(&shutdown); + let owner = Arc::clone(&owner); tokio::spawn(async move { - if let Err(error) = handle_connection(stream, store, shutdown).await { + if let Err(error) = handle_connection(stream, owner, Caller::Local).await { log::warn!("database-owner connection failed: {error}"); } }); @@ -1488,6 +1763,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } drop(listener); + if let Some(remote) = owner.remote.get() { + remote.close().await; + } if path.exists() { std::fs::remove_file(path) .with_context(|| format!("failed to remove endpoint {endpoint}"))?; @@ -1496,10 +1774,10 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } #[cfg(windows)] -async fn run_server(endpoint: &str, store: Arc) -> anyhow::Result<()> { +async fn run_server(endpoint: &str, owner: Arc) -> anyhow::Result<()> { use tokio::net::windows::named_pipe::ServerOptions; - let shutdown = Arc::new(Notify::new()); + let shutdown = Arc::clone(&owner.shutdown); let mut server = ServerOptions::new() .first_pipe_instance(true) .create(endpoint) @@ -1515,10 +1793,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu .create(endpoint) .with_context(|| format!("failed to create database owner pipe {endpoint}"))?; let connected = std::mem::replace(&mut server, next); - let store = Arc::clone(&store); - let shutdown = Arc::clone(&shutdown); + let owner = Arc::clone(&owner); tokio::spawn(async move { - if let Err(error) = handle_connection(connected, store, shutdown).await { + if let Err(error) = handle_connection(connected, owner, Caller::Local).await { log::warn!("database-owner connection failed: {error}"); } }); @@ -1526,6 +1803,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu () = shutdown.notified() => break, } } + if let Some(remote) = owner.remote.get() { + remote.close().await; + } Ok(()) } @@ -1698,6 +1978,7 @@ mod tests { let request = RequestFrame { version: PROTOCOL_VERSION, request_id: request_id.clone(), + token: None, request: OwnerRequest::LoadFrozenEnvelopesPage { provenance_id: turn.provenance_id.get(), attempt_generation: attempt.attempt_generation, @@ -1737,7 +2018,8 @@ mod tests { .contains("database-owner shutdown")); assert!(require_frozen_paging(OwnerResponse::Pong { pid: 42, - frozen_envelope_paging: true + frozen_envelope_paging: true, + remote_sandboxes: false, }) .is_ok()); } @@ -1765,6 +2047,7 @@ mod tests { RequestFrame { version: PROTOCOL_VERSION, request_id: "retry".into(), + token: None, request: OwnerRequest::PrepareCheckpoint { provenance_id: turn.provenance_id.get(), expected_generation: generation, @@ -1812,6 +2095,7 @@ mod tests { let expected = RequestFrame { version: PROTOCOL_VERSION, request_id: "request-1".to_string(), + token: None, request: OwnerRequest::Ping, }; let sent = expected.clone(); @@ -1845,6 +2129,7 @@ mod tests { let frame = RequestFrame { version: PROTOCOL_VERSION, request_id: "stale-request".to_string(), + token: None, request: OwnerRequest::AppendProvenanceEnvelopes { provenance_id: running.provenance_id.get(), expected_generation: running.generation, diff --git a/atomic-cli/src/commands/agent/owner/remote.rs b/atomic-cli/src/commands/agent/owner/remote.rs new file mode 100644 index 00000000..94fbbb93 --- /dev/null +++ b/atomic-cli/src/commands/agent/owner/remote.rs @@ -0,0 +1,410 @@ +//! The owner's remote transport: the same protocol over iroh, for sandboxes +//! on another machine. +//! +//! The local socket trusts its caller (the filesystem guards it). A remote +//! caller is trusted only as far as its sandbox token: every frame carries +//! it, and it reaches exactly one view, optionally as one identity, until it +//! expires. Tokens live **in memory only** — they end when the owner exits — +//! and only their hashes are kept. +//! +//! Provenance a remote sandbox records is bound to its token the same way: a +//! session it starts is its own, and it can touch no other. + +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use anyhow::{anyhow, Context}; +use chrono::{DateTime, Duration, Utc}; +use iroh::endpoint::{presets, Connection, RecvStream, SendStream}; +use iroh::{Endpoint, EndpointAddr, SecretKey}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; + +/// ALPN for the owner protocol over iroh. +pub(crate) const ALPN: &[u8] = b"atomic-owner/1"; + +/// The owner's iroh key, kept beside its lock so its address survives a +/// restart and existing pointers keep working. +const IROH_KEY_FILE: &str = "owner-iroh.key"; + +/// What a sandbox token grants. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Grant { + pub(crate) view: String, + /// The identity the sandbox's work is attributed to (e.g. an agent's). + pub(crate) acting_as: Option, + pub(crate) expires: DateTime, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub(crate) enum TokenError { + #[error("unknown or revoked sandbox token")] + Unknown, + #[error("sandbox token expired at {0}")] + Expired(DateTime), + #[error("no live token for view '{0}'")] + NoTokenForView(String), +} + +#[derive(Default)] +struct Tokens { + /// Token digest → grant. A view has at most one live token. + grants: HashMap, + /// Provenance sessions and turns each view's sandbox started. + sessions: HashMap, + provenance: HashMap, +} + +/// In-memory, view-scoped sandbox tokens. Cheap to clone (shared). +#[derive(Clone, Default)] +pub(crate) struct TokenRegistry { + inner: Arc>, +} + +fn digest(token: &str) -> String { + blake3::hash(token.as_bytes()).to_hex().to_string() +} + +impl TokenRegistry { + /// Mint a token for `view`, valid for `ttl`. Minting again replaces the + /// view's token. + pub(crate) fn mint( + &self, + view: &str, + acting_as: Option, + ttl: Duration, + ) -> (String, Grant) { + let mut bytes = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut bytes); + let token = format!("ast_{}", data_encoding::BASE64URL_NOPAD.encode(&bytes)); + let grant = Grant { + view: view.to_string(), + acting_as, + expires: Utc::now() + ttl, + }; + let mut t = self.inner.lock().unwrap(); + t.grants.retain(|_, g| g.view != view); + t.grants.insert(digest(&token), grant.clone()); + (token, grant) + } + + /// Extend `view`'s live token to now + `ttl`; the same token keeps working. + pub(crate) fn renew(&self, view: &str, ttl: Duration) -> Result, TokenError> { + let mut t = self.inner.lock().unwrap(); + let now = Utc::now(); + let grant = t + .grants + .values_mut() + .find(|g| g.view == view && g.expires > now) + .ok_or_else(|| TokenError::NoTokenForView(view.to_string()))?; + grant.expires = now + ttl; + Ok(grant.expires) + } + + /// End `view`'s token now. + pub(crate) fn revoke(&self, view: &str) -> bool { + let mut t = self.inner.lock().unwrap(); + let before = t.grants.len(); + t.grants.retain(|_, g| g.view != view); + t.grants.len() != before + } + + /// What `token` grants, if it is live. + pub(crate) fn check(&self, token: &str) -> Result { + let t = self.inner.lock().unwrap(); + let grant = t.grants.get(&digest(token)).ok_or(TokenError::Unknown)?; + if grant.expires <= Utc::now() { + return Err(TokenError::Expired(grant.expires)); + } + Ok(grant.clone()) + } + + /// Whether `view`'s sandbox may use provenance session `session_id`: + /// one it started, or one nobody has (`fresh`), which becomes its own. + pub(crate) fn claim_session(&self, view: &str, session_id: &str, fresh: bool) -> bool { + let mut t = self.inner.lock().unwrap(); + match t.sessions.get(session_id) { + Some(owner) => owner == view, + None if fresh => { + t.sessions.insert(session_id.to_string(), view.to_string()); + true + } + None => false, + } + } + + pub(crate) fn owns_session(&self, view: &str, session_id: &str) -> bool { + self.inner + .lock() + .unwrap() + .sessions + .get(session_id) + .map(String::as_str) + == Some(view) + } + + pub(crate) fn bind_provenance(&self, view: &str, provenance_id: u64) { + self.inner + .lock() + .unwrap() + .provenance + .insert(provenance_id, view.to_string()); + } + + pub(crate) fn owns_provenance(&self, view: &str, provenance_id: u64) -> bool { + self.inner + .lock() + .unwrap() + .provenance + .get(&provenance_id) + .map(String::as_str) + == Some(view) + } +} + +/// A remote sandbox's `.atomic-sandbox`: where its repository's owner is, +/// the view it works on, and its token. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct RemotePointer { + pub(crate) remote: EndpointAddr, + pub(crate) view: String, + pub(crate) token: String, +} + +/// Walk up from `start` to the nearest `.atomic-sandbox`; if it names a +/// remote owner, return the sandbox root and the pointer. +pub(crate) fn find_remote_pointer(start: &Path) -> Option<(std::path::PathBuf, RemotePointer)> { + let mut dir = std::path::absolute(start).ok()?; + loop { + let path = dir.join(atomic_repository::SANDBOX_POINTER); + if path.is_file() { + let bytes = std::fs::read(&path).ok()?; + return serde_json::from_slice(&bytes).ok().map(|p| (dir, p)); + } + if dir.join(".atomic").is_dir() || !dir.pop() { + return None; + } + } +} + +/// The owner's iroh endpoint, with a key persisted in `dot_dir`. +pub(crate) async fn bind(dot_dir: &Path, offline: bool) -> anyhow::Result { + let key_path = dot_dir.join(IROH_KEY_FILE); + let key = match std::fs::read(&key_path) { + Ok(bytes) => { + let bytes: [u8; 32] = bytes + .try_into() + .map_err(|_| anyhow!("{} is not a 32-byte key", key_path.display()))?; + SecretKey::from_bytes(&bytes) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let key = SecretKey::generate(); + write_private(&key_path, &key.to_bytes())?; + key + } + Err(e) => return Err(e).with_context(|| format!("failed to read {}", key_path.display())), + }; + let builder = if offline { + Endpoint::builder(presets::Minimal) + } else { + Endpoint::builder(presets::N0) + }; + Ok(builder + .secret_key(key) + .alpns(vec![ALPN.to_vec()]) + .bind() + .await?) +} + +/// The address to put in a pointer: once online, what iroh knows (relay and +/// direct addresses); offline, the bound sockets — unspecified ones as +/// loopback, which is all an offline owner can promise. +pub(crate) async fn pointer_addr(endpoint: &Endpoint, offline: bool) -> EndpointAddr { + if !offline { + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), endpoint.online()).await; + } + let addr = endpoint.addr(); + if addr.ip_addrs().next().is_some() || !offline { + return addr; + } + endpoint + .bound_sockets() + .into_iter() + .fold(addr, |addr, mut socket| { + if socket.ip().is_unspecified() { + socket.set_ip(match socket { + std::net::SocketAddr::V4(_) => std::net::Ipv4Addr::LOCALHOST.into(), + std::net::SocketAddr::V6(_) => std::net::Ipv6Addr::LOCALHOST.into(), + }); + } + addr.with_ip_addr(socket) + }) +} + +/// A client endpoint for dialing an owner. +async fn dialer(offline: bool) -> anyhow::Result { + let builder = if offline { + Endpoint::builder(presets::Minimal) + } else { + Endpoint::builder(presets::N0) + }; + Ok(builder.bind().await?) +} + +fn write_private(path: &Path, bytes: &[u8]) -> anyhow::Result<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(path) + .with_context(|| format!("failed to create {}", path.display()))?; + file.write_all(bytes)?; + Ok(()) +} + +/// Whether owners and clients skip iroh's relays and address lookup, and +/// dial by the direct addresses a pointer carries (tests, air-gapped hosts). +pub(crate) fn offline() -> bool { + std::env::var_os("ATOMIC_OWNER_IROH_OFFLINE").is_some() +} + +/// One iroh bi-stream as `AsyncRead + AsyncWrite`: one request, as on the +/// local socket. +pub(crate) struct BiStream { + send: SendStream, + recv: RecvStream, + _connection: Connection, + /// A dialing client's own endpoint, which must outlive the stream. + dialer: Option, +} + +impl BiStream { + /// Dial the owner at `addr` and open one request stream. + pub(crate) async fn dial(addr: EndpointAddr) -> anyhow::Result { + let endpoint = dialer(offline()).await?; + let id = addr.id; + let connection = endpoint + .connect(addr, ALPN) + .await + .with_context(|| format!("database owner {id} is not reachable"))?; + let (send, recv) = connection.open_bi().await?; + Ok(Self { + send, + recv, + _connection: connection, + dialer: Some(endpoint), + }) + } + + /// Done with the request: close the dialing endpoint gracefully. + pub(crate) async fn close(self) { + if let Some(endpoint) = self.dialer { + endpoint.close().await; + } + } + + pub(crate) async fn accept(connection: &Connection) -> Option { + let (send, recv) = connection.accept_bi().await.ok()?; + Some(Self { + send, + recv, + _connection: connection.clone(), + dialer: None, + }) + } +} + +impl tokio::io::AsyncRead for BiStream { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.recv).poll_read(cx, buf) + } +} + +impl tokio::io::AsyncWrite for BiStream { + fn poll_write( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_write(cx, buf) + .map_err(std::io::Error::other) + } + + fn poll_flush( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_flush(cx) + .map_err(std::io::Error::other) + } + + fn poll_shutdown( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_shutdown(cx) + .map_err(std::io::Error::other) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_token_reaches_its_view_until_it_ends() { + let reg = TokenRegistry::default(); + let (t, g) = reg.mint("exp-1", Some("did:key:zAgent".into()), Duration::hours(24)); + assert_eq!(reg.check(&t).unwrap().view, "exp-1"); + assert_eq!(reg.check("ast_made_up"), Err(TokenError::Unknown)); + + std::thread::sleep(std::time::Duration::from_millis(5)); + let after = reg.renew("exp-1", Duration::hours(24)).unwrap(); + assert!(after > g.expires); + assert_eq!(reg.check(&t).unwrap().expires, after); + + assert!(reg.revoke("exp-1")); + assert_eq!(reg.check(&t), Err(TokenError::Unknown)); + assert!(reg.renew("exp-1", Duration::hours(1)).is_err()); + } + + #[test] + fn expired_and_replaced_tokens_are_refused() { + let reg = TokenRegistry::default(); + let (t, _) = reg.mint("exp-1", None, Duration::milliseconds(-1)); + assert!(matches!(reg.check(&t), Err(TokenError::Expired(_)))); + assert!(reg.renew("exp-1", Duration::hours(1)).is_err()); + + let (old, _) = reg.mint("exp-2", None, Duration::hours(1)); + let (new, _) = reg.mint("exp-2", None, Duration::hours(1)); + assert_eq!(reg.check(&old), Err(TokenError::Unknown)); + assert!(reg.check(&new).is_ok()); + } + + #[test] + fn a_session_belongs_to_the_view_that_started_it() { + let reg = TokenRegistry::default(); + assert!(reg.claim_session("exp-1", "s1", true)); + assert!(reg.claim_session("exp-1", "s1", false), "its own, again"); + assert!(!reg.claim_session("exp-2", "s1", true), "another view's"); + assert!( + !reg.claim_session("exp-2", "s-local", false), + "an existing session it didn't start" + ); + assert!(reg.owns_session("exp-1", "s1")); + assert!(!reg.owns_session("exp-2", "s1")); + } +} diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs new file mode 100644 index 00000000..33ad6345 --- /dev/null +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -0,0 +1,471 @@ +//! Remote sandboxes on the owner protocol: who may ask for what, the +//! requests that open and close a sandbox, and materializing its view. +//! +//! A local caller is trusted as it always was. A remote caller (iroh) gets +//! the provenance requests and `Materialize`, each checked against its +//! token: its own view, its own sessions and turns, and checkpoints only for +//! changes its view can see. Opening, renewing and closing sandboxes, and +//! shutting the owner down, are local only. + +use std::path::Path; + +use atomic_core::pristine::{GraphTxnT, ViewTxnT}; +use atomic_core::types::{Base32, Hash}; +use atomic_repository::redb_change_store::RedbChangeStore; +use atomic_repository::{Repository, ViewEntryKind}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; + +use super::remote::{Grant, TokenRegistry}; +use super::{ + read_frame, write_frame, OwnerRequest, OwnerResponse, OwnerState, RequestFrame, ResponseFrame, + PROTOCOL_VERSION, +}; + +/// Who is on the other end of a connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Caller { + /// The local socket or pipe. + Local, + /// iroh: trusted only as far as the frame's token. + Remote, +} + +fn error(code: &str, message: impl Into) -> OwnerResponse { + OwnerResponse::Error { + code: code.to_string(), + message: message.into(), + } +} + +fn refuse(code: &str, message: impl Into) -> Box { + Box::new(error(code, message)) +} + +/// The grant `frame` acts under (`None` for a local caller), or the +/// refusal to send instead. +pub(crate) fn authorize( + owner: &OwnerState, + caller: Caller, + frame: &RequestFrame, +) -> Result, Box> { + if caller == Caller::Local { + return Ok(None); + } + let token = frame + .token + .as_deref() + .ok_or_else(|| refuse("unauthorized", "a remote request needs a sandbox token"))?; + let grant = owner + .tokens + .check(token) + .map_err(|e| refuse("unauthorized", e.to_string()))?; + let view = grant.view.as_str(); + let tokens = &owner.tokens; + let forbidden = |what: &str| { + refuse( + "forbidden", + format!("{what} is not this sandbox's (view '{view}')"), + ) + }; + match &frame.request { + OwnerRequest::Ping => {} + OwnerRequest::Materialize { view: asked } => { + if asked.as_deref().is_some_and(|v| v != view) { + return Err(forbidden("that view")); + } + } + OwnerRequest::ReserveProvenanceTurn { + session_id, + turn_number, + .. + } => { + let fresh = session_is_fresh(&owner.store, session_id, *turn_number) + .map_err(|e| refuse("provenance-store", e.to_string()))?; + if !tokens.claim_session(view, session_id, fresh) { + return Err(forbidden("that session")); + } + } + OwnerRequest::AppendProvenanceEnvelopes { provenance_id, .. } + | OwnerRequest::LoadFrozenEnvelopes { provenance_id } + | OwnerRequest::LoadFrozenEnvelopesPage { provenance_id, .. } + | OwnerRequest::AcknowledgeCheckpoint { provenance_id, .. } => { + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + } + OwnerRequest::PrepareCheckpoint { + provenance_id, + source, + .. + } => { + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + require_on_view(&owner.root, view, &source.change_hashes)?; + } + OwnerRequest::BindCheckpointHash { + provenance_id, + hash, + .. + } => { + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + require_on_view(&owner.root, view, std::slice::from_ref(hash))?; + } + OwnerRequest::StopTurn { session_id, .. } + | OwnerRequest::ResumeTurn { session_id, .. } + | OwnerRequest::AbandonTurn { session_id, .. } + | OwnerRequest::TurnStatus { session_id, .. } => { + if !tokens.owns_session(view, session_id) { + return Err(forbidden("that session")); + } + } + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + | OwnerRequest::Shutdown => { + return Err(refuse("forbidden", "only a local caller may do that")); + } + } + Ok(Some(grant)) +} + +/// Whether no one has recorded provenance for `session_id` up to this turn. +fn session_is_fresh(store: &RedbChangeStore, session_id: &str, turn: u32) -> anyhow::Result { + for t in 1..=turn { + if store.get_provenance_turn_for(session_id, t)?.is_some() { + return Ok(false); + } + } + Ok(true) +} + +/// Refuse unless every change in `hashes` is visible on `view`: a sandbox's +/// provenance can only describe its own view's work. +fn require_on_view(root: &Path, view: &str, hashes: &[Hash]) -> Result<(), Box> { + if hashes.is_empty() { + return Ok(()); + } + let check = || -> anyhow::Result> { + let repo = Repository::open_readonly(root)?; + let txn = repo.pristine().read_txn()?; + let state = txn + .get_view(view)? + .ok_or_else(|| anyhow::anyhow!("view '{view}' no longer exists"))?; + let visible = atomic_repository::collect_visible_change_ids(&txn, &state)?; + for hash in hashes { + match txn.get_internal(hash)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Some(*hash)), + } + } + Ok(None) + }; + match check() { + Ok(None) => Ok(()), + Ok(Some(hash)) => Err(refuse( + "forbidden", + format!("change {} is not on view '{view}'", hash.to_base32()), + )), + Err(e) => Err(refuse("repository", e.to_string())), + } +} + +/// After a remote reservation succeeds, the turn is the sandbox's. +pub(crate) fn note_response( + tokens: &TokenRegistry, + grant: Option<&Grant>, + response: &OwnerResponse, +) { + if let (Some(grant), OwnerResponse::ProvenanceTurn { turn, .. }) = (grant, response) { + tokens.bind_provenance(&grant.view, turn.provenance_id.get()); + } +} + +/// `OpenSandbox`, `RenewSandbox`, `CloseSandbox` (local callers only). +pub(crate) async fn admin( + owner: &std::sync::Arc, + request: OwnerRequest, +) -> OwnerResponse { + match request { + OwnerRequest::OpenSandbox { + view, + acting_as, + ttl_secs, + } => { + match Repository::open_readonly(&owner.root) + .map_err(anyhow::Error::from) + .and_then(|repo| Ok(repo.pristine().read_txn()?.get_view(&view)?)) + { + Ok(Some(_)) => {} + Ok(None) => return error("view", format!("no view '{view}'")), + Err(e) => return error("repository", e.to_string()), + } + let endpoint = match owner.remote_endpoint().await { + Ok(endpoint) => endpoint, + Err(e) => return error("remote", format!("{e:#}")), + }; + let (token, grant) = + owner + .tokens + .mint(&view, acting_as, chrono::Duration::seconds(ttl_secs)); + OwnerResponse::SandboxOpened { + remote: super::remote::pointer_addr(endpoint, super::remote::offline()).await, + view, + token, + expires: grant.expires.to_rfc3339(), + } + } + OwnerRequest::RenewSandbox { view, ttl_secs } => { + match owner + .tokens + .renew(&view, chrono::Duration::seconds(ttl_secs)) + { + Ok(expires) => OwnerResponse::SandboxRenewed { + expires: expires.to_rfc3339(), + }, + Err(e) => error("sandbox-token", e.to_string()), + } + } + OwnerRequest::CloseSandbox { view } => OwnerResponse::SandboxClosed { + revoked: owner.tokens.revoke(&view), + }, + other => error("internal", format!("not a sandbox request: {other:?}")), + } +} + +/// `Materialize`: every entry of the view as its own frame, in path order, +/// then `Materialized`. The one request answered with more than one frame. +pub(crate) async fn materialize( + owner: &OwnerState, + grant: Option, + frame: RequestFrame, + stream: &mut S, +) -> anyhow::Result<()> +where + S: AsyncWrite + Unpin, +{ + let request_id = frame.request_id; + let respond = |response| ResponseFrame { + version: PROTOCOL_VERSION, + request_id: request_id.clone(), + response, + }; + let view = match (grant, frame.request) { + (Some(grant), _) => grant.view, + (None, OwnerRequest::Materialize { view: Some(view) }) => view, + (None, _) => { + return write_frame( + stream, + &respond(error("view", "name the view to materialize")), + ) + .await; + } + }; + let (tx, mut rx) = tokio::sync::mpsc::channel::(64); + let root = owner.root.clone(); + let render = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let repo = Repository::open_readonly(&root)?; + let mut entries = 0u64; + let rendered = repo.materialize_view_entries(&view, |entry| { + entries += 1; + tx.blocking_send(OwnerResponse::MaterializeEntry { entry }) + .map_err(|_| ()) + })?; + match rendered { + Ok(snapshot) => { + let _ = tx.blocking_send(OwnerResponse::Materialized { snapshot, entries }); + Ok(()) + } + Err(()) => Err(anyhow::anyhow!("the client went away")), + } + }); + while let Some(response) = rx.recv().await { + write_frame(stream, &respond(response)).await?; + } + match render.await? { + Ok(()) => Ok(()), + Err(e) => write_frame(stream, &respond(error("materialize", format!("{e:#}")))).await, + } +} + +/// Client side of `Materialize`: write the view's tree into `dir` and return +/// how many entries it had. Files already at those paths are overwritten; +/// nothing else in `dir` is touched. +pub(crate) async fn receive_materialized( + stream: &mut S, + request_id: &str, + dir: &Path, +) -> anyhow::Result +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut written = 0u64; + loop { + let frame: ResponseFrame = read_frame(stream).await?; + if frame.request_id != request_id { + anyhow::bail!("database owner response request id mismatch"); + } + match frame.response { + OwnerResponse::MaterializeEntry { entry } => { + let path = dir.join(&entry.path); + match entry.kind { + ViewEntryKind::Directory => std::fs::create_dir_all(&path)?, + #[cfg(unix)] + ViewEntryKind::Symlink => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let target = String::from_utf8(entry.content.clone())?; + match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(e.into()) + } + _ => {} + } + std::os::unix::fs::symlink(target, &path)?; + } + #[cfg(not(unix))] + ViewEntryKind::Symlink => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, &entry.content)?; + } + ViewEntryKind::File => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, &entry.content)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + &path, + std::fs::Permissions::from_mode(entry.mode as u32), + )?; + } + } + } + written += 1; + } + OwnerResponse::Materialized { entries, .. } => { + stream.shutdown().await.ok(); + if entries != written { + anyhow::bail!("owner sent {written} entries but said {entries}"); + } + return Ok(written); + } + OwnerResponse::Error { code, message } => { + anyhow::bail!("database owner materialize failed [{code}]: {message}") + } + other => anyhow::bail!("unexpected materialize response: {other:?}"), + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use atomic_core::change::ChangeHeader; + use atomic_repository::redb_change_store::ProvenanceCheckpointSource; + use atomic_repository::{InsertOptions, RecordOptions, TrackingOptions}; + + use super::*; + + /// A repository with one change on `dev`, and an owner for it. + fn owner() -> (tempfile::TempDir, Arc, Hash) { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("repo"); + let repo = Repository::init(&root).unwrap(); + std::fs::write(root.join("a.txt"), "a\n").unwrap(); + repo.add("a.txt", TrackingOptions::default()).unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(false); + let outcome = repo.record(ChangeHeader::new("a"), options).unwrap(); + repo.write_recorded(&outcome, InsertOptions::default()) + .unwrap(); + let hash = *outcome.hash(); + drop(repo); + let store = Arc::new(RedbChangeStore::open(dir.path().join("journal.redb")).unwrap()); + (dir, OwnerState::new(store, root.join(".atomic")), hash) + } + + fn frame(token: Option<&str>, request: OwnerRequest) -> RequestFrame { + RequestFrame { + version: PROTOCOL_VERSION, + request_id: "r".into(), + token: token.map(str::to_string), + request, + } + } + + fn prepare(provenance_id: u64, change_hashes: Vec) -> OwnerRequest { + OwnerRequest::PrepareCheckpoint { + provenance_id, + expected_generation: 1, + source: ProvenanceCheckpointSource { + agent_name: "a".into(), + agent_display_name: "A".into(), + agent_vendor: "v".into(), + change_hashes, + previous_provenance: None, + plan_id: None, + ledger_turn_number: 1, + }, + reuse_frozen_changes: false, + now: 0, + } + } + + fn refusal(result: Result, Box>) -> String { + match result.map_err(|e| *e) { + Err(OwnerResponse::Error { code, .. }) => code, + other => panic!("expected a refusal, got {other:?}"), + } + } + + #[test] + fn a_remote_checkpoint_covers_only_its_own_turn_and_its_view_s_changes() { + let (_dir, owner, on_view) = owner(); + let (token, _) = owner.tokens.mint("dev", None, chrono::Duration::hours(1)); + owner.tokens.bind_provenance("dev", 7); + let remote = |request| authorize(&owner, Caller::Remote, &frame(Some(&token), request)); + + assert!(remote(prepare(7, vec![on_view])).unwrap().is_some()); + assert_eq!( + refusal(remote(prepare(8, vec![on_view]))), + "forbidden", + "not its turn" + ); + let elsewhere = Hash::of(b"a change on no view"); + assert_eq!( + refusal(remote(prepare(7, vec![elsewhere]))), + "forbidden", + "not its view's change" + ); + + assert_eq!( + refusal(authorize( + &owner, + Caller::Remote, + &frame(None, OwnerRequest::Ping) + )), + "unauthorized" + ); + assert!( + authorize( + &owner, + Caller::Local, + &frame(None, prepare(8, vec![elsewhere])) + ) + .unwrap() + .is_none(), + "a local caller is trusted as before" + ); + } +} diff --git a/atomic-cli/src/commands/sandbox.rs b/atomic-cli/src/commands/sandbox.rs index 8c91949d..1797e161 100644 --- a/atomic-cli/src/commands/sandbox.rs +++ b/atomic-cli/src/commands/sandbox.rs @@ -14,7 +14,16 @@ //! //! ```text //! atomic sandbox create [--dest ] [--view ] +//! atomic sandbox create --remote [--dest ] [--acting-as ] [--ttl ] +//! atomic sandbox materialize +//! atomic sandbox renew [--ttl ] +//! atomic sandbox close //! ``` +//! +//! A **remote** sandbox is for another machine (a VM, say): it holds only a +//! pointer — the repository owner's iroh address, its view, and a token that +//! reaches that view alone — and talks to the owner over the same protocol +//! local hooks use. `materialize`, run inside it, writes the view's tree. use std::path::PathBuf; @@ -22,6 +31,7 @@ use clap::{Parser, Subcommand}; use atomic_repository::{Repository, SealOptions, StageOptions}; +use crate::commands::agent::owner; use crate::commands::{find_repository_root, Command}; use crate::error::{CliError, CliResult}; @@ -50,6 +60,15 @@ pub enum SandboxCommands { /// Produces a single-layer deployable image of the full merged state — /// "run this exact version" anywhere an OCI runtime is available. Seal(Seal), + + /// Inside a remote sandbox: write its view's tree from the owner. + Materialize(Materialize), + + /// Extend a remote sandbox's token. + Renew(Renew), + + /// End a remote sandbox's token now. + Close(Close), } impl Command for Sandbox { @@ -58,6 +77,9 @@ impl Command for Sandbox { SandboxCommands::Create(cmd) => cmd.run(), SandboxCommands::Stage(cmd) => cmd.run(), SandboxCommands::Seal(cmd) => cmd.run(), + SandboxCommands::Materialize(cmd) => cmd.run(), + SandboxCommands::Renew(cmd) => cmd.run(), + SandboxCommands::Close(cmd) => cmd.run(), } } } @@ -93,8 +115,26 @@ pub struct Create { /// view — isolating each agent's history as well as its files. #[arg(long, value_name = "VIEW")] pub from: Option, + + /// Make a remote sandbox: no working tree here, only a pointer (written + /// into `--dest`, or printed) for a machine that reaches this + /// repository's owner over iroh. + #[arg(long)] + pub remote: bool, + + /// The identity the remote sandbox's work is attributed to. + #[arg(long, value_name = "DID", requires = "remote")] + pub acting_as: Option, + + /// How long the remote sandbox's token lasts, in seconds (renew it with + /// `atomic sandbox renew`). + #[arg(long, value_name = "SECS", default_value_t = DEFAULT_TTL_SECS, requires = "remote")] + pub ttl: i64, } +/// Two hours: long enough to outlast a renewal missed or two. +const DEFAULT_TTL_SECS: i64 = 2 * 60 * 60; + impl Create { fn default_dest(repo_root: &std::path::Path, name: &str) -> PathBuf { let repo_name = repo_root @@ -116,7 +156,7 @@ impl Command for Create { .clone() .unwrap_or_else(|| Self::default_dest(&root, &self.name)); - if dest.exists() { + if !self.remote && dest.exists() { return Err(CliError::InvalidArgument { message: format!("destination already exists: {}", dest.display()), }); @@ -138,6 +178,11 @@ impl Command for Create { (v, false) }; + if self.remote { + drop(repo); + return self.create_remote(&root, &view); + } + let count = repo .provision_sandbox(&dest, &view) .map_err(CliError::Repository)?; @@ -162,6 +207,99 @@ impl Command for Create { } } +impl Create { + fn create_remote(&self, root: &std::path::Path, view: &str) -> CliResult<()> { + let opened = owner::open_sandbox(root, view, self.acting_as.clone(), self.ttl) + .map_err(CliError::Internal)?; + let pointer = serde_json::to_vec_pretty(&opened.pointer).map_err(anyhow::Error::from)?; + match &self.dest { + Some(dest) => { + std::fs::create_dir_all(dest).map_err(anyhow::Error::from)?; + let path = dest.join(atomic_repository::SANDBOX_POINTER); + write_private(&path, &pointer)?; + println!("Remote sandbox '{}' created", self.name); + println!(" Pointer: {}", path.display()); + println!(" View: {view}"); + println!(" Expires: {}", opened.expires); + println!(" Next: run `atomic sandbox materialize` there"); + } + None => println!("{}", String::from_utf8_lossy(&pointer)), + } + Ok(()) + } +} + +/// The pointer carries a token: readable by its owner only. +fn write_private(path: &std::path::Path, bytes: &[u8]) -> CliResult<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path).map_err(anyhow::Error::from)?; + file.write_all(bytes).map_err(anyhow::Error::from)?; + Ok(()) +} + +/// Write a remote sandbox's view from its repository's owner. +#[derive(Parser, Debug)] +pub struct Materialize {} + +impl Command for Materialize { + fn run(&self) -> CliResult<()> { + let cwd = std::env::current_dir().map_err(anyhow::Error::from)?; + let (root, entries) = + owner::materialize_remote_sandbox(&cwd).map_err(CliError::Internal)?; + println!("Materialized {entries} entries into {}", root.display()); + Ok(()) + } +} + +/// Extend a remote sandbox's token. +#[derive(Parser, Debug)] +pub struct Renew { + /// The sandbox's view. + #[arg(value_name = "VIEW")] + pub view: String, + + /// New lifetime from now, in seconds. + #[arg(long, value_name = "SECS", default_value_t = DEFAULT_TTL_SECS)] + pub ttl: i64, +} + +impl Command for Renew { + fn run(&self) -> CliResult<()> { + let root = find_repository_root()?; + let expires = + owner::renew_sandbox(&root, &self.view, self.ttl).map_err(CliError::Internal)?; + println!("Sandbox token for '{}' now expires {expires}", self.view); + Ok(()) + } +} + +/// End a remote sandbox's token now. +#[derive(Parser, Debug)] +pub struct Close { + /// The sandbox's view. + #[arg(value_name = "VIEW")] + pub view: String, +} + +impl Command for Close { + fn run(&self) -> CliResult<()> { + let root = find_repository_root()?; + if owner::close_sandbox(&root, &self.view).map_err(CliError::Internal)? { + println!("Sandbox token for '{}' revoked", self.view); + } else { + println!("No live sandbox token for '{}'", self.view); + } + Ok(()) + } +} + /// Stage a view as a layered OCI image (base + delta). #[derive(Parser, Debug)] pub struct Stage { diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs new file mode 100644 index 00000000..f15ad41e --- /dev/null +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -0,0 +1,197 @@ +//! A remote sandbox talks to its repository's database owner over iroh — +//! the same protocol, with every request checked against its token. +//! +//! Runs offline: the owner and the sandbox dial each other directly on +//! loopback (`ATOMIC_OWNER_IROH_OFFLINE`), no relays. + +use std::path::Path; +use std::process::{Command, Output}; + +use atomic_core::change::ChangeHeader; +use atomic_repository::{InsertOptions, RecordOptions, Repository, TrackingOptions}; +use serde_json::Value; +use tempfile::TempDir; + +fn atomic(cwd: &Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(args) + .arg("--no-color") + .current_dir(cwd) + .env("ATOMIC_OWNER_IROH_OFFLINE", "1") + .output() + .expect("run atomic") +} + +fn ok(output: Output, what: &str) -> String { + assert!( + output.status.success(), + "{what} failed: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn failure(output: Output, what: &str) -> String { + assert!(!output.status.success(), "{what} should have failed"); + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// A repository with two recorded files on `dev`. +fn repository(root: &Path) { + let repo = Repository::init(root).unwrap(); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("README.md"), "hello\n").unwrap(); + std::fs::write(root.join("src/lib.rs"), "pub fn f() {}\n").unwrap(); + repo.add("README.md", TrackingOptions::default()).unwrap(); + repo.add("src/lib.rs", TrackingOptions::default()).unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(false); + let outcome = repo.record(ChangeHeader::new("first"), options).unwrap(); + repo.write_recorded(&outcome, InsertOptions::default()) + .unwrap(); +} + +struct Owner<'a>(&'a Path); + +impl Drop for Owner<'_> { + fn drop(&mut self) { + let _ = atomic( + self.0, + &["agent", "database-owner", "shutdown", "--repository", "."], + ); + } +} + +fn reserve(cwd: &Path, session: &str, turn: &str) -> Output { + atomic( + cwd, + &[ + "agent", + "database-owner", + "reserve", + "--repository", + ".", + "--session-id", + session, + "--turn", + turn, + "--now", + "1700000000", + "--json", + ], + ) +} + +#[test] +fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + + let out = ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--view", + "dev", + "--acting-as", + "did:key:zAgent", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + assert!(out.contains("Remote sandbox 'exp-1' created"), "{out}"); + let pointer_path = vm_dir.join(".atomic-sandbox"); + let pointer: Value = serde_json::from_slice(&std::fs::read(&pointer_path).unwrap()).unwrap(); + assert_eq!(pointer["view"], "dev"); + assert!(pointer["token"].as_str().unwrap().starts_with("ast_")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&pointer_path) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o077, 0, "the pointer holds a token: owner-only"); + } + + // The view's tree, and nothing of the repository. + let out = ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + assert!(out.contains("Materialized 3 entries"), "{out}"); + assert_eq!( + std::fs::read_to_string(vm_dir.join("README.md")).unwrap(), + "hello\n" + ); + assert_eq!( + std::fs::read_to_string(vm_dir.join("src/lib.rs")).unwrap(), + "pub fn f() {}\n" + ); + assert!(!vm_dir.join(".atomic").exists()); + + // Provenance goes over the same protocol, bound to the sandbox's view: + // its own sessions, never the host's. + let reserved: Value = + serde_json::from_str(&ok(reserve(&vm_dir, "vm-session", "1"), "remote reserve")).unwrap(); + assert_eq!(reserved["committed"], true); + ok(reserve(host.path(), "host-session", "1"), "local reserve"); + let refused = failure( + reserve(&vm_dir, "host-session", "2"), + "reserving the host's session", + ); + assert!(refused.contains("forbidden"), "{refused}"); + ok( + reserve(&vm_dir, "vm-session", "2"), + "its own session's next turn", + ); + + // Only a local caller administers sandboxes or the owner. + let refused = failure( + atomic( + &vm_dir, + &["agent", "database-owner", "shutdown", "--repository", "."], + ), + "remote shutdown", + ); + assert!(refused.contains("forbidden"), "{refused}"); + + // A tampered token reaches nothing. + let mut forged = pointer.clone(); + forged["token"] = Value::String("ast_forged".into()); + std::fs::write(&pointer_path, serde_json::to_vec(&forged).unwrap()).unwrap(); + let refused = failure(atomic(&vm_dir, &["sandbox", "materialize"]), "forged token"); + assert!(refused.contains("unauthorized"), "{refused}"); + std::fs::write(&pointer_path, serde_json::to_vec(&pointer).unwrap()).unwrap(); + + // Renewal keeps the same token; closing ends it. + let out = ok( + atomic(host.path(), &["sandbox", "renew", "dev", "--ttl", "600"]), + "renew", + ); + assert!(out.contains("now expires"), "{out}"); + ok( + atomic(&vm_dir, &["sandbox", "materialize"]), + "materialize after renew", + ); + let out = ok(atomic(host.path(), &["sandbox", "close", "dev"]), "close"); + assert!(out.contains("revoked"), "{out}"); + let refused = failure( + atomic(&vm_dir, &["sandbox", "materialize"]), + "closed sandbox", + ); + assert!(refused.contains("unauthorized"), "{refused}"); +} From cfa915df600c1e6c76326c99ca5c9a0733b700a2 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:10:56 -0500 Subject: [PATCH 08/23] Record in a remote sandbox's cache exactly as on the repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A change carries the repository's internal node ids and inode numbers, so a sandbox that records without the repository must read the repository's own rows. atomic-core's pristine::slice exports and imports them byte for byte: a skeleton (the view's tree, inodes and positions, directories, ids of every visible change) and, per record, a graph slice for the inodes it touches (each file's content and name chain, one hop of neighbours so find_block resolves as it does on the repository, its CRDT rows and conflicts). The view is imported flattened, with the repository's own Merkle state. Inodes the cache adds itself start at 2^62, clear of the repository's. The change store can hold content spans without their change files, and consults them first. Repository::{export,import}_sandbox_{skeleton,slice} wrap both sides; sandbox_slice_inodes is what a cache asks for — changed files and the directories on the way to changed or new paths. status: a tracked file missing from disk is "already deleted" only when the graph holds its vertex; a cache has the tree before the graph. The parity test records the same edit on a repository and on a cache built from it — modify, deletions spanning changes, delete, add (existing and new directory), move, move and edit, and a second record after the first lands — and requires identical hashes and bytes. --- atomic-core/src/pristine/mod.rs | 1 + atomic-core/src/pristine/slice.rs | 625 ++++++++++++++++++ atomic-core/src/pristine/txn/mod.rs | 1 + atomic-repository/src/changestore/mod.rs | 47 ++ atomic-repository/src/repository/mod.rs | 2 + .../src/repository/remote_cache.rs | 216 ++++++ atomic-repository/src/repository/status.rs | 9 +- .../tests/remote_cache_parity_test.rs | 252 +++++++ 8 files changed, 1152 insertions(+), 1 deletion(-) create mode 100644 atomic-core/src/pristine/slice.rs create mode 100644 atomic-repository/src/repository/remote_cache.rs create mode 100644 atomic-repository/tests/remote_cache_parity_test.rs diff --git a/atomic-core/src/pristine/mod.rs b/atomic-core/src/pristine/mod.rs index a013e1cf..eb52a072 100644 --- a/atomic-core/src/pristine/mod.rs +++ b/atomic-core/src/pristine/mod.rs @@ -150,6 +150,7 @@ mod error; mod inode_graph; pub mod ontology; +pub mod slice; pub(crate) mod span_index; pub mod tables; mod traits; diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs new file mode 100644 index 00000000..30765acb --- /dev/null +++ b/atomic-core/src/pristine/slice.rs @@ -0,0 +1,625 @@ +//! Graph slices: a repository's own pristine rows, copied for a remote +//! sandbox's local cache. +//! +//! A remote sandbox never holds its repository, but `record` has to read the +//! graph to compute a change — and the change it computes carries the +//! repository's internal node ids and inode numbers, so the sandbox must read +//! exactly the rows the repository has. A slice is those rows, byte for byte: +//! +//! - the **skeleton** ([`ReadTxn::export_skeleton`]): the view's tree — +//! paths, inodes and their positions, directories — and the id tables for +//! every change the view can see. Enough for `status` with no graph at all. +//! - a **graph slice** ([`ReadTxn::export_graph_slice`]): for a set of +//! inodes, the GRAPH / INODE_GRAPH rows of each file's content and of its +//! name chain up to the root, one hop of neighbours (so `find_block` finds +//! what it would find on the repository), the file's CRDT rows, and its +//! recorded conflicts on the view. +//! +//! [`WriteTxn::import_skeleton`] and [`WriteTxn::import_graph_slice`] write +//! them into a fresh pristine; [`WriteTxn::import_view_snapshot`] writes the +//! view itself, flattened and with the repository's own Merkle state. +//! +//! Rows are copied, never re-derived, so a read on the cache returns what the +//! same read returns on the repository for every row the slice covers. +//! +//! Edges are not filtered by view: a vertex's key exists only while it has an +//! edge, and `find_block` must see the same vertices the repository sees. +//! Reads that matter filter by view themselves. Content is another matter — +//! the slice carries none; span bytes travel separately, and only for changes +//! the view can see. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use redb::ReadableTable; +use serde::{Deserialize, Serialize}; + +use crate::crdt::tables::{ + BRANCHES, BRANCH_AFTER, BRANCH_LEAVES, BRANCH_VERTEX, INODE_TRUNK, LEAVES, PATH_TRUNK, TRUNKS, + TRUNK_BRANCHES, VERTEX_BRANCH, +}; +use crate::pristine::error::PristineResult; +use crate::pristine::tables::*; +use crate::pristine::traits::{GraphTxnT, ViewScope, ViewState}; +use crate::pristine::txn::{ReadTxn, WriteTxn}; +use crate::types::{ChangePosition, EdgeFlags, GraphNode, Merkle, NodeId, Position}; + +/// Inodes a remote sandbox allocates for itself (`atomic add`) start here, +/// far above any the repository hands out, so the two never collide. +pub const LOCAL_INODE_FLOOR: u64 = 1 << 62; + +/// An EXTERNAL / INTERNAL / NODE_TYPES row: `(id, hash, node type)`. +pub type IdRow = (u64, [u8; 32], Option); + +/// Raw pristine rows. Every field is one table's rows, key and value exactly +/// as stored. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphSlice { + /// EXTERNAL / INTERNAL / NODE_TYPES: `(id, hash, node type)`. + pub ids: Vec, + /// TREE: `path → inode`. + pub tree: Vec<(String, u64)>, + /// REV_TREE: `inode → path`. + pub rev_tree: Vec<(u64, String)>, + /// INODES (REV_INODES is its inverse): `inode → position`. + pub inodes: Vec<(u64, [u8; 16])>, + /// DIRECTORIES: `inode → flags`. + pub directories: Vec<(u64, u8)>, + /// GRAPH: `vertex → edges`. + pub graph: Vec<([u8; 24], Vec<[u8; 24]>)>, + /// INODE_GRAPH: `(inode, vertex) → edges`. + pub inode_graph: Vec<([u8; 32], Vec<[u8; 24]>)>, + pub crdt: CrdtRows, + /// CONFLICTS for the view: `inode → serialized conflicts`. Keyed by inode + /// alone; the view id is the importer's. + pub conflicts: Vec<(u64, Vec)>, +} + +/// A file's CRDT rows (see `crdt::tables`). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct CrdtRows { + pub trunks: Vec<([u8; 12], Vec)>, + pub inode_trunk: Vec<(u64, [u8; 12])>, + pub path_trunk: Vec<(String, [u8; 12])>, + pub trunk_branches: Vec<([u8; 12], Vec<[u8; 12]>)>, + pub branches: Vec<([u8; 12], [u8; 24])>, + pub branch_after: Vec<([u8; 12], [u8; 12])>, + pub branch_vertex: Vec<([u8; 12], [u8; 24])>, + pub vertex_branch: Vec<([u8; 24], [u8; 12])>, + pub branch_leaves: Vec<([u8; 12], Vec<[u8; 12]>)>, + pub leaves: Vec<([u8; 12], [u8; 22])>, +} + +/// A view as a remote sandbox sees it: its name, the repository's Merkle +/// state and change count for it, and the ids of every change it can see. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ViewSnapshotRows { + pub id: u64, + pub name: String, + pub state: Merkle, + pub change_count: u64, + /// In application order. + pub visible: Vec, +} + +fn change_of_edge(edge: &[u8; 24]) -> (u64, u64) { + ( + u64::from_le_bytes(edge[8..16].try_into().unwrap()), + u64::from_le_bytes(edge[16..24].try_into().unwrap()), + ) +} + +fn edge_flags(edge: &[u8; 24]) -> EdgeFlags { + let flag_and_pos = u64::from_le_bytes(edge[0..8].try_into().unwrap()); + EdgeFlags::from_bits_truncate((flag_and_pos >> 56) as u8) +} + +fn edge_dest(edge: &[u8; 24]) -> Position { + let flag_and_pos = u64::from_le_bytes(edge[0..8].try_into().unwrap()); + let change = u64::from_le_bytes(edge[8..16].try_into().unwrap()); + Position::new( + NodeId::new(change), + ChangePosition::new(flag_and_pos & ((1 << 56) - 1)), + ) +} + +fn vertex_key(node: GraphNode) -> [u8; 24] { + encode_vertex(node.change.get(), node.start.get(), node.end.get()) +} + +/// How far a traversal expands from a vertex. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Reach { + /// Its row only. + Row, + /// Its row and its name chain towards the root (a name or a directory). + Up, + /// Its row, and every edge onward except into another inode's content + /// (a file's content). + File, +} + +impl ReadTxn { + /// The view's tree and ids: TREE for the `live` inodes (the paths the + /// view actually has), REV_TREE / INODES / DIRECTORIES for every inode + /// whose position a visible change introduced, and the id rows for every + /// visible change. + pub fn export_skeleton( + &self, + visible: &BTreeSet, + live: &BTreeSet, + ) -> PristineResult { + let mut out = GraphSlice::default(); + let inodes = self.txn.open_table(INODES)?; + let mut kept = BTreeSet::new(); + for row in inodes.iter()? { + let (k, v) = row?; + let (change, _) = decode_position(v.value()); + if change == 0 || visible.contains(&change) { + kept.insert(k.value()); + out.inodes.push((k.value(), *v.value())); + } + } + let rev_tree = self.txn.open_table(REV_TREE)?; + for row in rev_tree.iter()? { + let (k, v) = row?; + if kept.contains(&k.value()) { + out.rev_tree.push((k.value(), v.value().to_string())); + } + } + let tree = self.txn.open_table(TREE)?; + for row in tree.iter()? { + let (k, v) = row?; + if live.contains(&v.value()) && kept.contains(&v.value()) { + out.tree.push((k.value().to_string(), v.value())); + } + } + let directories = self.txn.open_table(DIRECTORIES)?; + for row in directories.iter()? { + let (k, v) = row?; + if kept.contains(&k.value()) { + out.directories.push((k.value(), v.value())); + } + } + out.ids = self.id_rows(visible.iter().copied())?; + Ok(out) + } + + /// The rows `record` reads for `inodes`, on the view `view_id`. + pub fn export_graph_slice(&self, inodes: &[u64], view_id: u64) -> PristineResult { + let mut out = GraphSlice::default(); + let inode_table = self.txn.open_table(INODES)?; + let rev_inodes = self.txn.open_table(REV_INODES)?; + let graph = self.txn.open_multimap_table(GRAPH)?; + let inode_graph = self.txn.open_multimap_table(INODE_GRAPH)?; + let conflicts = self.txn.open_table(CONFLICTS)?; + + let edges_of = |key: &[u8; 24]| -> PristineResult> { + let mut edges = Vec::new(); + for e in graph.get(key)? { + edges.push(*e?.value()); + } + Ok(edges) + }; + let is_inode_vertex = |node: GraphNode| -> PristineResult { + Ok(node.start == node.end + && rev_inodes + .get(&encode_position(node.change.get(), node.start.get()))? + .is_some()) + }; + + // Each vertex once, at the widest reach any path gives it. + let mut reach: BTreeMap<[u8; 24], Reach> = BTreeMap::new(); + let mut queue: VecDeque<(GraphNode, Reach)> = VecDeque::new(); + let enqueue = |node: GraphNode, + r: Reach, + reach: &mut BTreeMap<[u8; 24], Reach>, + queue: &mut VecDeque<(GraphNode, Reach)>| { + if node.is_root() { + return; + } + let key = vertex_key(node); + if reach.get(&key).is_some_and(|have| *have >= r) { + return; + } + reach.insert(key, r); + queue.push_back((node, r)); + }; + + for &inode in inodes { + let Some(pos) = inode_table.get(inode)? else { + continue; + }; + let (change, p) = decode_position(pos.value()); + let seed = GraphNode { + change: NodeId::new(change), + start: ChangePosition::new(p), + end: ChangePosition::new(p), + }; + enqueue(seed, Reach::File, &mut reach, &mut queue); + // INODE_GRAPH may hold content the forward walk doesn't reach. + let lo = encode_inode_vertex(inode, 0, 0, 0); + let hi = encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX); + for row in inode_graph.range::<&[u8; 32]>(&lo..=&hi)? { + let (k, values) = row?; + let key = *k.value(); + let mut edges = Vec::new(); + for e in values { + edges.push(*e?.value()); + } + let (_, c, s, e) = decode_inode_vertex(&key); + out.inode_graph.push((key, edges)); + enqueue( + GraphNode { + change: NodeId::new(c), + start: ChangePosition::new(s), + end: ChangePosition::new(e), + }, + Reach::File, + &mut reach, + &mut queue, + ); + } + if let Some(c) = conflicts.get(&encode_view_seq(view_id, inode))? { + out.conflicts.push((inode, c.value().to_vec())); + } + self.export_crdt(inode, &mut out.crdt)?; + } + + let resolve = |pos: Position, parent: bool| -> Vec> { + let mut found = Vec::new(); + if parent { + if let Ok(n) = self.find_block_end(pos) { + found.push(n); + } + if pos.pos.get() > 0 { + if let Ok(n) = self.find_block(Position::new( + pos.change, + ChangePosition::new(pos.pos.get() - 1), + )) { + found.push(n); + } + } + } else if let Ok(n) = self.find_block(pos) { + found.push(n); + } + found + }; + + while let Some((node, r)) = queue.pop_front() { + if r == Reach::Row { + continue; + } + for edge in edges_of(&vertex_key(node))? { + let flags = edge_flags(&edge); + let parent = flags.contains(EdgeFlags::PARENT); + let folder = flags.contains(EdgeFlags::FOLDER); + for next in resolve(edge_dest(&edge), parent) { + let onward = match r { + Reach::File if parent && folder => Reach::Up, + Reach::File if folder => Reach::Row, + Reach::File if is_inode_vertex(next)? => Reach::Row, + Reach::File => Reach::File, + Reach::Up if parent && folder => Reach::Up, + _ => Reach::Row, + }; + enqueue(next, onward, &mut reach, &mut queue); + } + } + } + + let mut ids = BTreeSet::new(); + for key in reach.keys() { + let edges = edges_of(key)?; + if edges.is_empty() { + continue; + } + ids.insert(decode_vertex(key).0); + for e in &edges { + let (c, by) = change_of_edge(e); + ids.insert(c); + ids.insert(by); + } + out.graph.push((*key, edges)); + } + for (_, edges) in &out.inode_graph { + for e in edges { + let (c, by) = change_of_edge(e); + ids.insert(c); + ids.insert(by); + } + } + for (id, _) in &out.crdt.trunks { + ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + } + for (id, _) in &out.crdt.branches { + ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + } + for (id, _) in &out.crdt.leaves { + ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + } + ids.remove(&0); + out.ids = self.id_rows(ids.into_iter())?; + Ok(out) + } + + fn export_crdt(&self, inode: u64, out: &mut CrdtRows) -> PristineResult<()> { + let inode_trunk = self.txn.open_table(INODE_TRUNK)?; + let Some(trunk) = inode_trunk.get(inode)? else { + return Ok(()); + }; + let trunk = *trunk.value(); + out.inode_trunk.push((inode, trunk)); + if let Some(t) = self.txn.open_table(TRUNKS)?.get(&trunk)? { + out.trunks.push((trunk, t.value().to_vec())); + } + for row in self.txn.open_table(PATH_TRUNK)?.iter()? { + let (k, v) = row?; + if *v.value() == trunk { + out.path_trunk.push((k.value().to_string(), trunk)); + } + } + let trunk_branches = self.txn.open_multimap_table(TRUNK_BRANCHES)?; + let branches = self.txn.open_table(BRANCHES)?; + let branch_after = self.txn.open_table(BRANCH_AFTER)?; + let branch_vertex = self.txn.open_table(BRANCH_VERTEX)?; + let vertex_branch = self.txn.open_table(VERTEX_BRANCH)?; + let branch_leaves = self.txn.open_multimap_table(BRANCH_LEAVES)?; + let leaves = self.txn.open_table(LEAVES)?; + let mut ids = Vec::new(); + for b in trunk_branches.get(&trunk)? { + ids.push(*b?.value()); + } + for b in &ids { + if let Some(v) = branches.get(b)? { + out.branches.push((*b, *v.value())); + } + if let Some(v) = branch_after.get(b)? { + out.branch_after.push((*b, *v.value())); + } + if let Some(v) = branch_vertex.get(b)? { + let vertex = *v.value(); + out.branch_vertex.push((*b, vertex)); + if let Some(back) = vertex_branch.get(&vertex)? { + out.vertex_branch.push((vertex, *back.value())); + } + } + let mut ls = Vec::new(); + for l in branch_leaves.get(b)? { + let l = *l?.value(); + if let Some(v) = leaves.get(&l)? { + out.leaves.push((l, *v.value())); + } + ls.push(l); + } + if !ls.is_empty() { + out.branch_leaves.push((*b, ls)); + } + } + out.trunk_branches.push((trunk, ids)); + Ok(()) + } + + fn id_rows(&self, ids: impl Iterator) -> PristineResult> { + let external = self.txn.open_table(EXTERNAL)?; + let node_types = self.txn.open_table(NODE_TYPES)?; + let mut out = Vec::new(); + for id in ids { + if let Some(h) = external.get(id)? { + let t = node_types.get(id)?.map(|t| t.value()); + out.push((id, *h.value(), t)); + } + } + Ok(out) + } + + /// The view's snapshot: its own state, and every change it can see. + pub fn export_view_snapshot(&self, view: &ViewState, visible: Vec) -> ViewSnapshotRows { + ViewSnapshotRows { + id: view.id, + name: view.name.clone(), + state: view.state, + change_count: view.change_count, + visible, + } + } +} + +impl WriteTxn<'_> { + /// Replace the tree tables with `slice`'s, keeping inodes this cache + /// allocated itself (at or above [`LOCAL_INODE_FLOOR`]); add its ids. + pub fn import_skeleton(&mut self, slice: &GraphSlice) -> PristineResult<()> { + let local = |inode: u64| inode >= LOCAL_INODE_FLOOR; + { + let mut tree = self.txn.open_table(TREE)?; + tree.retain(|_, inode| local(inode))?; + for (path, inode) in &slice.tree { + tree.insert(path.as_str(), *inode)?; + } + } + { + let mut rev_tree = self.txn.open_table(REV_TREE)?; + rev_tree.retain(|inode, _| local(inode))?; + for (inode, path) in &slice.rev_tree { + rev_tree.insert(*inode, path.as_str())?; + } + } + { + let mut inodes = self.txn.open_table(INODES)?; + inodes.retain(|inode, _| local(inode))?; + let mut rev = self.txn.open_table(REV_INODES)?; + rev.retain(|_, inode| local(inode))?; + for (inode, pos) in &slice.inodes { + inodes.insert(*inode, pos)?; + rev.insert(pos, *inode)?; + } + } + { + let mut dirs = self.txn.open_table(DIRECTORIES)?; + dirs.retain(|inode, _| local(inode))?; + for (inode, flags) in &slice.directories { + dirs.insert(*inode, *flags)?; + } + } + self.import_ids(&slice.ids)?; + // From here on this cache's own inodes (`atomic add`) come from above + // the floor, clear of every inode the repository has or will hand out. + self.next_inode + .fetch_max(LOCAL_INODE_FLOOR, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + /// Drop every graph, CRDT and conflict row, then write `slice`'s. + /// Conflicts are written for `view_id`. + pub fn import_graph_slice(&mut self, slice: &GraphSlice, view_id: u64) -> PristineResult<()> { + self.txn.delete_multimap_table(GRAPH)?; + self.txn.delete_multimap_table(INODE_GRAPH)?; + self.txn.delete_table(CONFLICTS)?; + self.txn.delete_table(TRUNKS)?; + self.txn.delete_table(INODE_TRUNK)?; + self.txn.delete_table(PATH_TRUNK)?; + self.txn.delete_multimap_table(TRUNK_BRANCHES)?; + self.txn.delete_table(BRANCHES)?; + self.txn.delete_table(BRANCH_AFTER)?; + self.txn.delete_table(BRANCH_VERTEX)?; + self.txn.delete_table(VERTEX_BRANCH)?; + self.txn.delete_multimap_table(BRANCH_LEAVES)?; + self.txn.delete_table(LEAVES)?; + + { + let mut t = self.txn.open_multimap_table(GRAPH)?; + for (k, edges) in &slice.graph { + for e in edges { + t.insert(k, e)?; + } + } + } + { + let mut t = self.txn.open_multimap_table(INODE_GRAPH)?; + for (k, edges) in &slice.inode_graph { + for e in edges { + t.insert(k, e)?; + } + } + } + { + let mut t = self.txn.open_table(CONFLICTS)?; + for (inode, c) in &slice.conflicts { + t.insert(&encode_view_seq(view_id, *inode), c.as_slice())?; + } + } + let c = &slice.crdt; + { + let mut t = self.txn.open_table(TRUNKS)?; + for (k, v) in &c.trunks { + t.insert(k, v.as_slice())?; + } + let mut t = self.txn.open_table(INODE_TRUNK)?; + for (k, v) in &c.inode_trunk { + t.insert(*k, v)?; + } + let mut t = self.txn.open_table(PATH_TRUNK)?; + for (k, v) in &c.path_trunk { + t.insert(k.as_str(), v)?; + } + let mut t = self.txn.open_multimap_table(TRUNK_BRANCHES)?; + for (k, vs) in &c.trunk_branches { + for v in vs { + t.insert(k, v)?; + } + } + let mut t = self.txn.open_table(BRANCHES)?; + for (k, v) in &c.branches { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(BRANCH_AFTER)?; + for (k, v) in &c.branch_after { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(BRANCH_VERTEX)?; + for (k, v) in &c.branch_vertex { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(VERTEX_BRANCH)?; + for (k, v) in &c.vertex_branch { + t.insert(k, v)?; + } + let mut t = self.txn.open_multimap_table(BRANCH_LEAVES)?; + for (k, vs) in &c.branch_leaves { + for v in vs { + t.insert(k, v)?; + } + } + let mut t = self.txn.open_table(LEAVES)?; + for (k, v) in &c.leaves { + t.insert(k, v)?; + } + } + self.import_ids(&slice.ids) + } + + fn import_ids(&mut self, ids: &[IdRow]) -> PristineResult<()> { + let mut external = self.txn.open_table(EXTERNAL)?; + let mut internal = self.txn.open_table(INTERNAL)?; + let mut types = self.txn.open_table(NODE_TYPES)?; + for (id, hash, t) in ids { + external.insert(*id, hash)?; + internal.insert(hash, *id)?; + if let Some(t) = t { + types.insert(*id, *t)?; + } + } + let max = ids.iter().map(|(id, _, _)| *id).max().unwrap_or(0); + self.next_node_id + .fetch_max(max + 1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + /// Write `snapshot` as a shared, parentless view with the repository's + /// own id, state and change count — replacing any view with that name or + /// id — and its visible changes as the view's change log. + pub fn import_view_snapshot(&mut self, snapshot: &ViewSnapshotRows) -> PristineResult<()> { + let state = ViewState { + id: snapshot.id, + name: snapshot.name.clone(), + state: snapshot.state, + change_count: snapshot.change_count, + kind: ViewScope::Shared, + parent: None, + }; + { + let mut views = self.txn.open_table(VIEWS)?; + let mut stale = Vec::new(); + for row in views.iter()? { + let (k, v) = row?; + let existing = crate::pristine::txn::deserialize_view_state(v.value())?; + if existing.id == snapshot.id || k.value() == snapshot.name { + stale.push(k.value().to_string()); + } + } + for name in stale { + views.remove(name.as_str())?; + } + let bytes = crate::pristine::txn::serialize_view_state(&state); + views.insert(snapshot.name.as_str(), bytes.as_slice())?; + } + let lo = encode_view_seq(snapshot.id, 0); + let hi = encode_view_seq(snapshot.id, u64::MAX); + { + let mut log = self.txn.open_table(VIEW_CHANGES)?; + log.retain_in::<&[u8; 16], _>(&lo..=&hi, |_, _| false)?; + for (seq, change) in snapshot.visible.iter().enumerate() { + log.insert(&encode_view_seq(snapshot.id, seq as u64), *change)?; + } + } + { + let mut rev = self.txn.open_table(REV_VIEW_CHANGES)?; + rev.retain_in::<&[u8; 16], _>(&lo..=&hi, |_, _| false)?; + for (seq, change) in snapshot.visible.iter().enumerate() { + rev.insert(&encode_view_seq(snapshot.id, *change), seq as u64)?; + } + } + self.next_view_id + .fetch_max(snapshot.id + 1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } +} diff --git a/atomic-core/src/pristine/txn/mod.rs b/atomic-core/src/pristine/txn/mod.rs index 178a2570..bcc62b8c 100644 --- a/atomic-core/src/pristine/txn/mod.rs +++ b/atomic-core/src/pristine/txn/mod.rs @@ -14,6 +14,7 @@ mod read; mod write; pub use helpers::AdjIterator; +pub(crate) use helpers::{deserialize_view_state, serialize_view_state}; pub use pristine::Pristine; pub use read::CachedGraphTxn; pub use read::InodePreloadTxn; diff --git a/atomic-repository/src/changestore/mod.rs b/atomic-repository/src/changestore/mod.rs index 22cc916f..973caf1e 100644 --- a/atomic-repository/src/changestore/mod.rs +++ b/atomic-repository/src/changestore/mod.rs @@ -121,6 +121,32 @@ pub struct ChangeStore { /// We use `RwLock` for thread-safe interior mutability, allowing /// concurrent read access while ensuring exclusive write access. pub(crate) cache: RwLock>, + + /// Content spans held without their change files — a remote sandbox's + /// cache has the bytes of the vertices it was sent, never whole changes. + /// Consulted before the change files. + pub(crate) spans: RwLock, +} + +/// Content bytes by `(change, start)`, each covering `start..end`. +#[derive(Debug, Default)] +pub struct SpanOverlay { + spans: std::collections::BTreeMap<(Hash, u64), (u64, Vec)>, +} + +impl SpanOverlay { + /// The bytes of `start..end` in `hash`'s content, if one held span covers them. + fn copy(&self, hash: &Hash, start: usize, end: usize, buf: &mut [u8]) -> Option { + let (&(h, s), (e, bytes)) = self.spans.range(..=(*hash, start as u64)).next_back()?; + if h != *hash || (end as u64) > *e { + return None; + } + let from = start - s as usize; + let len = end - start; + buf.get_mut(..len)? + .copy_from_slice(bytes.get(from..from + len)?); + Some(len) + } } impl std::fmt::Debug for ChangeStore { @@ -189,6 +215,7 @@ impl ChangeStore { Ok(Self { changes_dir, cache: RwLock::new(LruCache::new(cache_capacity)), + spans: RwLock::new(SpanOverlay::default()), }) } @@ -403,6 +430,21 @@ impl ChangeStore { Ok(change) } + /// Hold `bytes` as `hash`'s content at `start..start + bytes.len()`. + pub fn hold_span(&self, hash: Hash, start: u64, bytes: Vec) { + if let Ok(mut spans) = self.spans.write() { + let end = start + bytes.len() as u64; + spans.spans.insert((hash, start), (end, bytes)); + } + } + + /// Drop every held span. + pub fn clear_spans(&self) { + if let Ok(mut spans) = self.spans.write() { + spans.spans.clear(); + } + } + /// Copy a content span from a change without cloning the full `Change`. /// /// Graph output calls this for every vertex it materializes. Using @@ -416,6 +458,11 @@ impl ChangeStore { end: usize, buf: &mut [u8], ) -> ChangeStoreResult { + if let Ok(spans) = self.spans.read() { + if let Some(n) = spans.copy(hash, start, end, buf) { + return Ok(n); + } + } // Fast path: shared read lock — multiple threads can read concurrently. // peek() doesn't update LRU order, which is an acceptable trade-off // to avoid serializing all readers on a write lock. diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index ec11bfea..7dd98ee9 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -80,6 +80,8 @@ mod deferred_tree; mod filter; mod materialize; pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; +mod remote_cache; +pub use remote_cache::{SandboxSkeleton, SandboxSlice, SpanBytes}; mod sandbox; mod semantic_materialize; mod split; diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs new file mode 100644 index 00000000..cd4e896d --- /dev/null +++ b/atomic-repository/src/repository/remote_cache.rs @@ -0,0 +1,216 @@ +//! A remote sandbox's local cache: the repository's own rows for one view, +//! enough for `status` and `record` without the repository. +//! +//! The serving repository exports ([`Repository::export_sandbox_skeleton`], +//! [`Repository::export_sandbox_slice`]); the sandbox's cache imports +//! ([`Repository::import_sandbox_skeleton`], +//! [`Repository::import_sandbox_slice`]). See `atomic_core::pristine::slice` +//! for what the rows are and why they are copied rather than re-derived. + +use std::collections::BTreeSet; + +use atomic_core::change::ChangeStore as _; +use atomic_core::pristine::slice::{GraphSlice, ViewSnapshotRows}; +use atomic_core::pristine::{decode_vertex, GraphTxnT, MutTxnT, ViewTxnT}; +use atomic_core::types::{GraphNode, Hash, NodeId}; +use serde::{Deserialize, Serialize}; + +use super::Repository; +use crate::RepositoryError; + +/// A view's tree for a fresh cache: the skeleton rows, the view itself, and +/// every entry (with content) the view has. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SandboxSkeleton { + pub view: ViewSnapshotRows, + pub rows: GraphSlice, +} + +/// The rows and content `record` reads for some paths. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SandboxSlice { + pub rows: GraphSlice, + pub spans: Vec, +} + +/// Content bytes of one vertex: `change`'s content at `start..start + len`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpanBytes { + pub change: Hash, + pub start: u64, + #[serde(with = "serde_bytes_b64")] + pub bytes: Vec, +} + +mod serde_bytes_b64 { + use data_encoding::BASE64; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(bytes: &[u8], s: S) -> Result { + s.serialize_str(&BASE64.encode(bytes)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } +} + +fn db(e: impl std::fmt::Display) -> RepositoryError { + RepositoryError::Database(e.to_string()) +} + +impl Repository { + /// The visible change ids of `view`, sorted. + fn sandbox_visible( + &self, + txn: &atomic_core::pristine::ReadTxn, + view: &atomic_core::pristine::ViewState, + ) -> Result, RepositoryError> { + let mut ids: Vec = super::collect_visible_change_ids(txn, view)? + .into_iter() + .map(|id| id.get()) + .collect(); + ids.sort_unstable(); + Ok(ids) + } + + /// Serve side: `view`'s skeleton, for the inodes `live` (the ones its + /// materialized tree holds). + pub fn export_sandbox_skeleton( + &self, + view: &str, + live: &BTreeSet, + ) -> Result { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible = self.sandbox_visible(&txn, &state)?; + let set: BTreeSet = visible.iter().copied().collect(); + let rows = txn.export_skeleton(&set, live).map_err(db)?; + Ok(SandboxSkeleton { + view: txn.export_view_snapshot(&state, visible), + rows, + }) + } + + /// Serve side: what `record` on `view` reads for `inodes`, with the + /// content of every vertex a visible change introduced. + pub fn export_sandbox_slice( + &self, + view: &str, + inodes: &[u64], + ) -> Result { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible: BTreeSet = self.sandbox_visible(&txn, &state)?.into_iter().collect(); + let rows = txn.export_graph_slice(inodes, state.id).map_err(db)?; + let mut spans = Vec::new(); + for (key, _) in &rows.graph { + let (change, start, end) = decode_vertex(key); + if start >= end || !visible.contains(&change) { + continue; + } + let Some(hash) = txn.get_external(NodeId::new(change)).map_err(db)? else { + continue; + }; + let node = GraphNode { + change: NodeId::new(change), + start: atomic_core::types::ChangePosition::new(start), + end: atomic_core::types::ChangePosition::new(end), + }; + let mut bytes = vec![0u8; (end - start) as usize]; + self.change_store + .get_contents(|_| Some(hash), node, &mut bytes) + .map_err(db)?; + spans.push(SpanBytes { + change: hash, + start, + bytes, + }); + } + Ok(SandboxSlice { rows, spans }) + } + + /// Cache side: replace this cache's tree and view with `skeleton`'s. + pub fn import_sandbox_skeleton( + &self, + skeleton: &SandboxSkeleton, + ) -> Result<(), RepositoryError> { + let mut txn = self.pristine.write_txn().map_err(db)?; + txn.import_view_snapshot(&skeleton.view).map_err(db)?; + txn.import_skeleton(&skeleton.rows).map_err(db)?; + txn.commit().map_err(db)?; + Ok(()) + } + + /// Cache side: the inodes `record` will read — each tracked file that + /// differs from its baseline, and every tracked directory on the way to + /// any changed or new path (a new file's parent is where it is added). + /// After [`Repository::reindex_working_copy`] on a fresh tree, a clean + /// file costs nothing here. + pub fn sandbox_slice_inodes(&self) -> Result, RepositoryError> { + use crate::status::{FileStatus, StatusOptions}; + use atomic_core::pristine::slice::LOCAL_INODE_FLOOR; + + let status = self.status(StatusOptions::default())?; + let mut out = BTreeSet::new(); + let keep = |inode: atomic_core::types::Inode, out: &mut BTreeSet| { + if inode.get() < LOCAL_INODE_FLOOR { + out.insert(inode.get()); + } + }; + for entry in status.entries() { + if entry.status() == FileStatus::Clean { + continue; + } + if let Some(inode) = entry.inode() { + keep(inode, &mut out); + } else if let Some(inode) = self.get_file_inode(entry.path())? { + keep(inode, &mut out); + } + let mut dir = entry.path().parent(); + while let Some(d) = dir.filter(|d| !d.as_os_str().is_empty()) { + if let Some(inode) = self.get_file_inode(d)? { + keep(inode, &mut out); + } + dir = d.parent(); + } + } + Ok(out.into_iter().collect()) + } + + /// Cache side: replace this cache's graph rows and held content with + /// `slice`'s. + pub fn import_sandbox_slice(&self, slice: &SandboxSlice) -> Result<(), RepositoryError> { + let view_id = { + let txn = self.pristine.read_txn().map_err(db)?; + txn.get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })? + .id + }; + let mut txn = self.pristine.write_txn().map_err(db)?; + txn.import_graph_slice(&slice.rows, view_id).map_err(db)?; + txn.commit().map_err(db)?; + self.change_store.clear_spans(); + for span in &slice.spans { + self.change_store + .hold_span(span.change, span.start, span.bytes.clone()); + } + Ok(()) + } +} diff --git a/atomic-repository/src/repository/status.rs b/atomic-repository/src/repository/status.rs index af1dfabe..c694263c 100644 --- a/atomic-repository/src/repository/status.rs +++ b/atomic-repository/src/repository/status.rs @@ -285,11 +285,18 @@ impl Repository { // materialize uses) to ask: does this file have // content from this view's perspective? If not, the // deletion is already recorded. + // + // Only a graph that holds the file can say so: a remote + // sandbox's cache has its tree before any of its graph, + // and there no vertex means "ask", not "deleted". if has_graph { if let Some(inode_val) = inode { if let Ok(Some(position)) = txn.inode_position(inode_val) { if let Some(ref ids) = current_view_change_ids { - if !is_file_alive_via_retrieval(&txn, inode_val, position, ids) + if txn.has_vertex(position.inode_node()).unwrap_or(false) + && !is_file_alive_via_retrieval( + &txn, inode_val, position, ids, + ) { // Deletion already recorded — skip found_on_disk.insert(path.clone()); diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs new file mode 100644 index 00000000..e58faf26 --- /dev/null +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -0,0 +1,252 @@ +//! A remote sandbox's cache records the same change the repository would. +//! +//! The cache holds only the repository's own rows for the view (the +//! skeleton) and for the paths being recorded (a slice), plus the content +//! bytes those need. `record` runs unchanged against it, and the change it +//! produces — which carries the repository's internal ids — must be byte for +//! byte the change `record` produces on the repository itself. + +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; + +use atomic_core::change::{Author, ChangeHeader}; +use atomic_repository::{InsertOptions, RecordOptions, Repository, TrackingOptions}; +use tempfile::TempDir; + +fn write(root: &Path, rel: &str, content: &str) { + let path = root.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); +} + +fn header(message: &str) -> ChangeHeader { + ChangeHeader::builder() + .message(message) + .author(Author::new("Test", Some("test@example.com"))) + .build() +} + +fn record_and_apply(repo: &Repository, message: &str) { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(false); + let outcome = repo.record(header(message), options).unwrap(); + repo.write_recorded(&outcome, InsertOptions::default()) + .unwrap(); +} + +/// Record without saving or applying: just the change. +fn change_bytes(repo: &Repository, header: ChangeHeader) -> (String, Vec) { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = repo + .record(header, options) + .unwrap_or_else(|e| panic!("record in {}: {e}", repo.root().display())); + ( + atomic_core::types::Base32::to_base32(outcome.hash()), + outcome.v3_bytes().expect("v3 bytes").to_vec(), + ) +} + +struct Pair { + _dirs: (TempDir, TempDir), + host: Repository, + cache: Repository, +} + +impl Pair { + fn host_root(&self) -> &Path { + self.host.root() + } + fn cache_root(&self) -> &Path { + self.cache.root() + } + + /// Apply the same edit to both working trees. + fn edit(&self, f: impl Fn(&Path)) { + f(self.host_root()); + f(self.cache_root()); + } + + /// Ship the slice for what `record` will touch, as FileStates does: + /// only the inodes the cache asks for. + fn hydrate(&self, view: &str, live: &BTreeSet) { + let inodes = self.cache.sandbox_slice_inodes().unwrap(); + assert!( + inodes.len() < live.len(), + "the cache asks for what changed, not everything: {inodes:?} of {live:?}" + ); + let slice = self.host.export_sandbox_slice(view, &inodes).unwrap(); + self.cache.import_sandbox_slice(&slice).unwrap(); + } +} + +/// A host repository with some history, and a cache materialized from it. +fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeSet) { + let host_dir = TempDir::new().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let host = Repository::init(host_dir.path()).unwrap(); + history(&host); + let view = host.current_view().to_string(); + + // Materialize: the view's files into the cache's working tree, and the + // inodes they are. + let cache = Repository::init(cache_dir.path()).unwrap(); + let mut live = BTreeSet::new(); + host.materialize_view_entries::<()>(&view, |entry| { + live.insert(entry.inode); + let path = cache_dir.path().join(&entry.path); + match entry.kind { + atomic_repository::ViewEntryKind::Directory => fs::create_dir_all(path).unwrap(), + _ => { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + } + Ok(()) + }) + .unwrap() + .unwrap(); + let skeleton = host.export_sandbox_skeleton(&view, &live).unwrap(); + cache.import_sandbox_skeleton(&skeleton).unwrap(); + // The baseline: what was just written is clean. + cache.reindex_working_copy().unwrap(); + ( + Pair { + _dirs: (host_dir, cache_dir), + host, + cache, + }, + view, + live, + ) +} + +fn assert_parity(pair: &Pair, view: &str, live: &BTreeSet, what: &str) { + pair.hydrate(view, live); + let h = header(what); + let host = change_bytes(&pair.host, h.clone()); + let cache = change_bytes(&pair.cache, h); + assert_eq!( + host.0, cache.0, + "{what}: the cache records a different change" + ); + assert_eq!(host.1, cache.1, "{what}: same hash, different bytes"); +} + +fn two_files(repo: &Repository) { + let root = repo.root().to_path_buf(); + write(&root, "README.md", "hello\nworld\n"); + write(&root, "src/lib.rs", "pub fn a() {}\npub fn b() {}\n"); + repo.add("README.md", TrackingOptions::default()).unwrap(); + repo.add("src/lib.rs", TrackingOptions::default()).unwrap(); + record_and_apply(repo, "first"); + write(&root, "README.md", "hello\nthere\nworld\n"); + record_and_apply(repo, "second"); +} + +#[test] +fn modifying_a_file() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "hello\nthere\nbig\nworld\n")); + assert_parity(&pair, &view, &live, "modify"); +} + +#[test] +fn modifying_a_file_changed_by_several_changes() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "world\n")); + assert_parity(&pair, &view, &live, "delete lines from two changes"); +} + +#[test] +fn deleting_a_file() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| fs::remove_file(root.join("src/lib.rs")).unwrap()); + assert_parity(&pair, &view, &live, "delete"); +} + +#[test] +fn adding_a_file_to_an_existing_directory() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "src/new.rs", "pub fn c() {}\n")); + for repo in [&pair.host, &pair.cache] { + repo.add("src/new.rs", TrackingOptions::default()).unwrap(); + } + assert_parity(&pair, &view, &live, "add"); +} + +#[test] +fn adding_a_file_in_a_new_directory() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "docs/guide.md", "# guide\n")); + for repo in [&pair.host, &pair.cache] { + repo.add("docs/guide.md", TrackingOptions::default()) + .unwrap(); + } + assert_parity(&pair, &view, &live, "add in new dir"); +} + +#[test] +fn without_the_slice_the_cache_cannot_record_the_same_change() { + let (pair, _view, _live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "hello\nthere\nbig\nworld\n")); + let h = header("no slice"); + let host = change_bytes(&pair.host, h.clone()); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + match pair.cache.record(h, options) { + Err(_) => {} + Ok(outcome) => assert_ne!( + atomic_core::types::Base32::to_base32(outcome.hash()), + host.0, + "the graph rows are what make the change right" + ), + } + assert!(host.1.len() > 200, "a real change: {} bytes", host.1.len()); +} + +#[test] +fn moving_a_file() { + // As `atomic mv` does: the rename on disk, detected by content. + let (pair, view, live) = pair(two_files); + for repo in [&pair.host, &pair.cache] { + fs::rename( + repo.root().join("src/lib.rs"), + repo.root().join("src/core.rs"), + ) + .unwrap(); + } + assert_parity(&pair, &view, &live, "move"); +} + +#[test] +fn moving_and_editing_a_file() { + let (pair, view, live) = pair(two_files); + for repo in [&pair.host, &pair.cache] { + fs::rename(repo.root().join("README.md"), repo.root().join("READ.md")).unwrap(); + write(repo.root(), "READ.md", "hello\nthere\nworld\nagain\n"); + } + assert_parity(&pair, &view, &live, "move and edit"); +} + +#[test] +fn a_second_record_on_top_of_the_first() { + // After the first change lands on the repository, the cache takes a fresh + // skeleton and slice and records the next one identically. + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "src/lib.rs", "pub fn a() {}\n")); + assert_parity(&pair, &view, &live, "first"); + record_and_apply(&pair.host, "first"); + let skeleton = pair.host.export_sandbox_skeleton(&view, &live).unwrap(); + pair.cache.import_sandbox_skeleton(&skeleton).unwrap(); + pair.cache.reindex_working_copy().unwrap(); + pair.edit(|root| write(root, "src/lib.rs", "pub fn a() { 1 }\n")); + assert_parity(&pair, &view, &live, "second"); +} From 62ebab776f770c3ec6f6af28861138b895528eaf Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:23:38 -0500 Subject: [PATCH 09/23] Record in a remote sandbox and land the change through the owner Two more requests on the owner protocol, checked against the sandbox's token like the rest: FileStates hands the cache the repository's rows and content for the inodes record will read (only inodes on the view), and SubmitChange takes the V3 bytes the cache recorded. The owner applies a submitted change only if it is what it claims and names nothing outside the view: the bytes hash to the claimed hash; the view is still at the state it was recorded against; every change it depends on or refers to that the repository knows is visible on the view; every node its file operations name is a visible change's (FileOps::referenced_node_ids); no path touches .atomic, .atomic-sandbox or .atomic-sandbox.d; and it is new. Submissions go in one at a time. Refusals write nothing. In the sandbox, materialize also builds the cache: an ordinary repository under .atomic-sandbox.d/cache whose working tree is the sandbox, so a remote pointer opens it and status, log and diff work locally. record hydrates first, records without saving or applying, submits, then takes the view's new skeleton and marks the tree clean. When a change lands, a path the sandbox added gets the repository's inode. --- atomic-cli/src/commands/agent/owner/mod.rs | 106 +++++++++- .../src/commands/agent/owner/sandbox.rs | 96 ++++++++- atomic-cli/src/commands/record/command.rs | 28 ++- .../tests/remote_sandbox_integration_test.rs | 58 ++++++ atomic-core/src/change/ops.rs | 59 ++++++ atomic-core/src/pristine/slice.rs | 12 ++ atomic-repository/src/repository/mod.rs | 7 +- .../src/repository/remote_cache.rs | 169 +++++++++++++++- atomic-repository/src/repository/sandbox.rs | 51 ++++- .../tests/remote_cache_parity_test.rs | 188 ++++++++++++++++++ 10 files changed, 759 insertions(+), 15 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/mod.rs b/atomic-cli/src/commands/agent/owner/mod.rs index 84a95bf1..17eb7494 100644 --- a/atomic-cli/src/commands/agent/owner/mod.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -211,6 +211,38 @@ enum OwnerRequest { #[serde(default)] view: Option, }, + /// The repository's rows (and content) `record` reads for `inodes` — + /// what a sandbox's cache loads before recording. + FileStates { + #[serde(default)] + view: Option, + inodes: Vec, + }, + /// A change a sandbox recorded against `base_state`, as its V3 bytes. + SubmitChange { + #[serde(default)] + view: Option, + base_state: String, + hash: Hash, + #[serde(with = "base64_bytes")] + bytes: Vec, + }, +} + +/// Bytes as base64 in the JSON frames. +mod base64_bytes { + use serde::{Deserialize, Deserializer, Serializer}; + + pub(super) fn serialize(bytes: &[u8], s: S) -> Result { + s.serialize_str(&data_encoding::BASE64.encode(bytes)) + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + data_encoding::BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -277,6 +309,19 @@ pub(crate) enum OwnerResponse { Materialized { snapshot: atomic_repository::ViewSnapshot, entries: u64, + /// The view's rows, for the sandbox's cache. + skeleton: Box, + }, + FileStates { + slice: Box, + }, + ChangeSubmitted { + submitted: atomic_repository::Submitted, + /// The view's rows now, for the sandbox's cache. + skeleton: Box, + }, + ChangeRefused { + rejection: atomic_repository::SubmitRejection, }, Error { code: String, @@ -1037,8 +1082,8 @@ pub(crate) fn close_sandbox(repository: &Path, view: &str) -> anyhow::Result anyhow::Result<(PathBuf, u64)> { let (root, pointer) = remote::find_remote_pointer(start) .ok_or_else(|| anyhow!("{} is not in a remote sandbox", start.display()))?; @@ -1055,9 +1100,51 @@ pub(crate) fn materialize_remote_sandbox(start: &Path) -> anyhow::Result<(PathBu stream.close().await; written })?; + let (written, skeleton) = written; + Repository::create_remote_sandbox_cache(&root, &skeleton)?; Ok((root, written)) } +/// In a remote sandbox: load what `record` will read into its cache. +pub(crate) fn hydrate_remote_sandbox(repo: &Repository) -> anyhow::Result<()> { + let inodes = repo.sandbox_slice_inodes()?; + match request(repo.root(), OwnerRequest::FileStates { view: None, inodes })? { + OwnerResponse::FileStates { slice } => Ok(repo.import_sandbox_slice(&slice)?), + other => Err(unexpected_response("file states", other)), + } +} + +/// In a remote sandbox: hand a recorded change to the repository's owner. +/// On success the cache takes the view as it now is. +pub(crate) fn submit_from_remote_sandbox( + repo: &Repository, + hash: Hash, + bytes: Vec, +) -> anyhow::Result { + let base_state = repo.remote_sandbox_view_state()?; + match request( + repo.root(), + OwnerRequest::SubmitChange { + view: None, + base_state, + hash, + bytes, + }, + )? { + OwnerResponse::ChangeSubmitted { + submitted, + skeleton, + } => { + repo.import_sandbox_skeleton(&skeleton)?; + Ok(submitted) + } + OwnerResponse::ChangeRefused { rejection } => { + Err(anyhow!("the repository refused the change: {rejection}")) + } + other => Err(unexpected_response("submit change", other)), + } +} + fn serve(repository: &Path) -> CliResult<()> { let dot_dir = Repository::canonical_dot_dir(repository)?; let owner_lock = acquire_owner_lock(&dot_dir)?; @@ -1470,7 +1557,9 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram OwnerRequest::OpenSandbox { .. } | OwnerRequest::RenewSandbox { .. } | OwnerRequest::CloseSandbox { .. } - | OwnerRequest::Materialize { .. } => ( + | OwnerRequest::Materialize { .. } + | OwnerRequest::FileStates { .. } + | OwnerRequest::SubmitChange { .. } => ( OwnerResponse::Error { code: "internal".to_string(), message: "sandbox requests need the owner's connection handler".to_string(), @@ -1498,6 +1587,9 @@ pub(crate) struct OwnerState { tokens: TokenRegistry, /// Bound on the first `OpenSandbox`; accepts remote callers from then on. remote: tokio::sync::OnceCell, + /// Submitted changes go in one at a time: each is checked against the + /// view's state as it is when it's applied. + submissions: tokio::sync::Mutex<()>, shutdown: Arc, } @@ -1513,6 +1605,7 @@ impl OwnerState { dot_dir, tokens: TokenRegistry::default(), remote: tokio::sync::OnceCell::new(), + submissions: tokio::sync::Mutex::new(()), shutdown: Arc::new(Notify::new()), }) } @@ -1590,6 +1683,13 @@ where let response = sandbox::admin(&owner, request.request.clone()).await; write_frame(&mut stream, &respond(request.request_id, response)).await?; } + OwnerRequest::FileStates { .. } | OwnerRequest::SubmitChange { .. } + if request.version == PROTOCOL_VERSION => + { + let request_id = request.request_id.clone(); + let response = sandbox::record_request(&owner, grant, request.request).await; + write_frame(&mut stream, &respond(request_id, response)).await?; + } _ => { let (response, shutdown) = handle_request(&owner.store, request); sandbox::note_response(&owner.tokens, grant.as_ref(), &response.response); diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index 33ad6345..18e61684 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -69,7 +69,9 @@ pub(crate) fn authorize( }; match &frame.request { OwnerRequest::Ping => {} - OwnerRequest::Materialize { view: asked } => { + OwnerRequest::Materialize { view: asked } + | OwnerRequest::FileStates { view: asked, .. } + | OwnerRequest::SubmitChange { view: asked, .. } => { if asked.as_deref().is_some_and(|v| v != view) { return Err(forbidden("that view")); } @@ -268,14 +270,21 @@ where let render = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { let repo = Repository::open_readonly(&root)?; let mut entries = 0u64; + let mut live = std::collections::BTreeSet::new(); let rendered = repo.materialize_view_entries(&view, |entry| { entries += 1; + live.insert(entry.inode); tx.blocking_send(OwnerResponse::MaterializeEntry { entry }) .map_err(|_| ()) })?; match rendered { Ok(snapshot) => { - let _ = tx.blocking_send(OwnerResponse::Materialized { snapshot, entries }); + let skeleton = Box::new(repo.export_sandbox_skeleton(&view, &live)?); + let _ = tx.blocking_send(OwnerResponse::Materialized { + snapshot, + entries, + skeleton, + }); Ok(()) } Err(()) => Err(anyhow::anyhow!("the client went away")), @@ -290,6 +299,81 @@ where } } +/// `FileStates` and `SubmitChange`: the view is the token's (remote) or +/// the one named (local). +pub(crate) async fn record_request( + owner: &OwnerState, + grant: Option, + request: OwnerRequest, +) -> OwnerResponse { + let named = match &request { + OwnerRequest::FileStates { view, .. } | OwnerRequest::SubmitChange { view, .. } => { + view.clone() + } + _ => None, + }; + let Some(view) = grant.map(|g| g.view).or(named) else { + return error("view", "name the view"); + }; + let root = owner.root.clone(); + match request { + OwnerRequest::FileStates { inodes, .. } => { + let read = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = Repository::open_readonly(&root)?; + Ok(repo.export_sandbox_slice(&view, &inodes)?) + }); + match read.await { + Ok(Ok(slice)) => OwnerResponse::FileStates { + slice: Box::new(slice), + }, + Ok(Err(e)) => error("file-states", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } + OwnerRequest::SubmitChange { + base_state, + hash, + bytes, + .. + } => { + let _one_at_a_time = owner.submissions.lock().await; + let write = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = + Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; + match repo.insert_submitted_change(&view, &base_state, &hash, &bytes)? { + Ok(submitted) => { + let skeleton = + repo.export_sandbox_skeleton(&view, &live_inodes(&repo, &view)?)?; + Ok(Ok((submitted, skeleton))) + } + Err(rejection) => Ok(Err(rejection)), + } + }); + match write.await { + Ok(Ok(Ok((submitted, skeleton)))) => OwnerResponse::ChangeSubmitted { + submitted, + skeleton: Box::new(skeleton), + }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Err(e)) => error("submit", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } + other => error("internal", format!("not a record request: {other:?}")), + } +} + +/// The inodes `view`'s tree holds. +fn live_inodes(repo: &Repository, view: &str) -> anyhow::Result> { + let mut live = std::collections::BTreeSet::new(); + repo.materialize_view_entries::<()>(view, |entry| { + live.insert(entry.inode); + Ok(()) + })? + .map_err(|()| anyhow::anyhow!("unreachable"))?; + Ok(live) +} + /// Client side of `Materialize`: write the view's tree into `dir` and return /// how many entries it had. Files already at those paths are overwritten; /// nothing else in `dir` is touched. @@ -297,7 +381,7 @@ pub(crate) async fn receive_materialized( stream: &mut S, request_id: &str, dir: &Path, -) -> anyhow::Result +) -> anyhow::Result<(u64, atomic_repository::SandboxSkeleton)> where S: AsyncRead + AsyncWrite + Unpin, { @@ -350,12 +434,14 @@ where } written += 1; } - OwnerResponse::Materialized { entries, .. } => { + OwnerResponse::Materialized { + entries, skeleton, .. + } => { stream.shutdown().await.ok(); if entries != written { anyhow::bail!("owner sent {written} entries but said {entries}"); } - return Ok(written); + return Ok((written, *skeleton)); } OwnerResponse::Error { code, message } => { anyhow::bail!("database owner materialize failed [{code}]: {message}") diff --git a/atomic-cli/src/commands/record/command.rs b/atomic-cli/src/commands/record/command.rs index 912f525f..1595d49d 100644 --- a/atomic-cli/src/commands/record/command.rs +++ b/atomic-cli/src/commands/record/command.rs @@ -40,7 +40,13 @@ impl Command for Record { let header = header_builder.build(); // Build record options - let options = self.build_options()?; + let mut options = self.build_options()?; + // A remote sandbox records against its cache and hands the change + // to the repository's owner; nothing is saved or applied here. + let remote = repo.is_remote_sandbox(); + if remote { + options = options.save_to_store(false).apply_after_record(false); + } // If --all, first add all untracked files if self.all { @@ -56,6 +62,11 @@ impl Command for Record { } } + if remote { + crate::commands::agent::owner::hydrate_remote_sandbox(&repo) + .map_err(CliError::Internal)?; + } + // Record the changes. // // This match is deliberately exhaustive: no `_ =>` arm. The catch-all @@ -106,6 +117,21 @@ impl Command for Record { } })?; + if remote { + let bytes = outcome + .v3_bytes() + .ok_or_else(|| CliError::Internal(anyhow::anyhow!("the change has no bytes")))? + .to_vec(); + crate::commands::agent::owner::submit_from_remote_sandbox( + &repo, + *outcome.hash(), + bytes, + ) + .map_err(CliError::Internal)?; + // What was just recorded is the new baseline. + repo.reindex_working_copy().map_err(CliError::Repository)?; + } + // Display result let output = self.format_outcome(&repo, &outcome); print!("{}", output); diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index f15ad41e..bb4ed348 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -177,6 +177,64 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { assert!(refused.contains("unauthorized"), "{refused}"); std::fs::write(&pointer_path, serde_json::to_vec(&pointer).unwrap()).unwrap(); + // Recording in the sandbox: its cache computes the change, the owner + // checks it and applies it to the view. + std::fs::write(vm_dir.join("README.md"), "hello\nfrom the sandbox\n").unwrap(); + std::fs::write(vm_dir.join("src/new.rs"), "pub fn n() {}\n").unwrap(); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "from the sandbox"]), + "record in the sandbox", + ); + let status = ok(atomic(&vm_dir, &["status"]), "status after record"); + assert!(status.contains("working tree clean"), "{status}"); + std::fs::write(vm_dir.join("src/new.rs"), "pub fn n() { 2 }\n").unwrap(); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "again from the sandbox"]), + "second record in the sandbox", + ); + let log = ok(atomic(host.path(), &["log"]), "host log"); + assert!( + log.contains("from the sandbox") && log.contains("again from the sandbox"), + "{log}" + ); + + // A second sandbox of the view sees the first one's work. + let other = vm.path().join("other"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-2", + "--remote", + "--view", + "dev", + "--dest", + other.to_str().unwrap(), + ], + ), + "second sandbox", + ); + ok( + atomic(&other, &["sandbox", "materialize"]), + "materialize the second", + ); + assert_eq!( + std::fs::read_to_string(other.join("README.md")).unwrap(), + "hello\nfrom the sandbox\n" + ); + assert_eq!( + std::fs::read_to_string(other.join("src/new.rs")).unwrap(), + "pub fn n() { 2 }\n" + ); + // (Minting for the same view replaced the first sandbox's token.) + std::fs::write( + &pointer_path, + std::fs::read(other.join(".atomic-sandbox")).unwrap(), + ) + .unwrap(); + // Renewal keeps the same token; closing ends it. let out = ok( atomic(host.path(), &["sandbox", "renew", "dev", "--ttl", "600"]), diff --git a/atomic-core/src/change/ops.rs b/atomic-core/src/change/ops.rs index aef44ae0..324adfd1 100644 --- a/atomic-core/src/change/ops.rs +++ b/atomic-core/src/change/ops.rs @@ -213,6 +213,65 @@ impl FileOps { &mut self.line_ops } + /// Every existing node these operations name — the changes that created + /// the file, lines and tokens they touch. Ids this change creates carry + /// the placeholder `NodeId::ROOT` until it is applied, and are left out. + /// + /// These ids are applied as they are, so whoever applies a change it did + /// not record checks them (a remote sandbox's change may name only its + /// own view's nodes). + pub fn referenced_node_ids(&self) -> std::collections::BTreeSet { + let mut ids = std::collections::BTreeSet::new(); + let leaf = |op: &LeafOp, ids: &mut std::collections::BTreeSet<_>| match op { + LeafOp::Insert { after, .. } => ids.extend(after.map(|l| l.change_id())), + LeafOp::Delete { leaf } | LeafOp::Replace { leaf, .. } | LeafOp::Restore { leaf } => { + ids.insert(leaf.change_id()); + } + }; + ids.insert(self.trunk_id.change_id()); + match &self.trunk_op { + Some(TrunkOp::Delete { trunk }) + | Some(TrunkOp::Move { trunk, .. }) + | Some(TrunkOp::Undelete { trunk }) => { + ids.insert(trunk.change_id()); + } + Some(TrunkOp::Create { .. }) | None => {} + } + for line in &self.line_ops { + ids.insert(line.branch_id().change_id()); + match line.operation() { + BranchOp::Insert { after, content } => { + ids.extend(after.map(|b| b.change_id())); + content.iter().for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Delete { branch, content } => { + ids.insert(branch.change_id()); + content.iter().for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Modify { + branch, + old_content, + new_content, + } => { + ids.insert(branch.change_id()); + old_content + .iter() + .chain(new_content) + .for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Restore { branch } => { + ids.insert(branch.change_id()); + } + BranchOp::Reparent { branch, new_after } => { + ids.insert(branch.change_id()); + ids.extend(new_after.map(|b| b.change_id())); + } + } + } + ids.remove(&crate::types::NodeId::ROOT); + ids + } + /// Adds a line operation. pub fn add_line_op(&mut self, op: LineOps) { self.line_ops.push(op); diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs index 30765acb..4a6888a7 100644 --- a/atomic-core/src/pristine/slice.rs +++ b/atomic-core/src/pristine/slice.rs @@ -429,6 +429,18 @@ impl WriteTxn<'_> { /// allocated itself (at or above [`LOCAL_INODE_FLOOR`]); add its ids. pub fn import_skeleton(&mut self, slice: &GraphSlice) -> PristineResult<()> { let local = |inode: u64| inode >= LOCAL_INODE_FLOOR; + // A path the cache added itself that the repository now has (its + // change landed) is the repository's inode from here on. + let arrived: std::collections::HashSet<&str> = + slice.tree.iter().map(|(path, _)| path.as_str()).collect(); + let mut superseded = BTreeSet::new(); + for row in self.txn.open_table(REV_TREE)?.iter()? { + let (inode, path) = row?; + if local(inode.value()) && arrived.contains(path.value()) { + superseded.insert(inode.value()); + } + } + let local = |inode: u64| local(inode) && !superseded.contains(&inode); { let mut tree = self.txn.open_table(TREE)?; tree.retain(|_, inode| local(inode))?; diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 7dd98ee9..ec71760e 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -81,7 +81,7 @@ mod filter; mod materialize; pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; mod remote_cache; -pub use remote_cache::{SandboxSkeleton, SandboxSlice, SpanBytes}; +pub use remote_cache::{SandboxSkeleton, SandboxSlice, SpanBytes, SubmitRejection, Submitted}; mod sandbox; mod semantic_materialize; mod split; @@ -94,7 +94,10 @@ pub use filter::{ collect_view_change_ids, collect_visible_change_ids, collect_visible_change_ids_with_deps, expand_indexed_dependency_closure, view_set_id, }; -pub use sandbox::{SealOptions, SealResult, StageOptions, StageResult, SANDBOX_POINTER}; +pub use sandbox::{ + remote_cache_root, SealOptions, SealResult, StageOptions, StageResult, SANDBOX_CACHE_DIR, + SANDBOX_POINTER, +}; pub use split::{SplitChange, SplitOptions, SplitOutcome}; pub use views::{ManifestApplyOutcome, ViewInfo}; diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index cd4e896d..149f054f 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -11,7 +11,7 @@ use std::collections::BTreeSet; use atomic_core::change::ChangeStore as _; use atomic_core::pristine::slice::{GraphSlice, ViewSnapshotRows}; -use atomic_core::pristine::{decode_vertex, GraphTxnT, MutTxnT, ViewTxnT}; +use atomic_core::pristine::{decode_vertex, GraphTxnT, MutTxnT, TreeTxnT, ViewTxnT}; use atomic_core::types::{GraphNode, Hash, NodeId}; use serde::{Deserialize, Serialize}; @@ -58,6 +58,41 @@ mod serde_bytes_b64 { } } +/// Why the repository refused a sandbox's change. Nothing is written when +/// a change is refused. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)] +pub enum SubmitRejection { + #[error("the change's bytes hash to {computed}, not {claimed}")] + HashMismatch { claimed: String, computed: String }, + #[error("not a change: {0}")] + Malformed(String), + #[error("the view has moved on (now {current}); fetch and record again")] + StaleView { current: String }, + #[error("change {0} is not on this view")] + ForeignChange(String), + #[error("node {0} is not on this view")] + ForeignNode(u64), + #[error("a change may not touch {0}")] + ForbiddenPath(String), + #[error("change {0} is already in the repository")] + AlreadyPresent(String), +} + +/// A submitted change, applied. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Submitted { + pub hash: Hash, + /// The view's state after it, base32. + pub state: String, +} + +/// Paths only the repository's own machinery writes. +fn forbidden_path(path: &str) -> bool { + path.split('/').any(|part| { + part == crate::DOT_DIR || part == super::SANDBOX_POINTER || part == super::SANDBOX_CACHE_DIR + }) +} + fn db(e: impl std::fmt::Display) -> RepositoryError { RepositoryError::Database(e.to_string()) } @@ -115,7 +150,18 @@ impl Repository { name: view.to_string(), })?; let visible: BTreeSet = self.sandbox_visible(&txn, &state)?.into_iter().collect(); - let rows = txn.export_graph_slice(inodes, state.id).map_err(db)?; + // Only files the view has: an inode another view introduced is not + // this sandbox's to read. + let mut on_view = Vec::with_capacity(inodes.len()); + for &inode in inodes { + let position = txn + .inode_position(atomic_core::types::Inode::new(inode)) + .map_err(db)?; + if position.is_some_and(|p| p.change.is_root() || visible.contains(&p.change.get())) { + on_view.push(inode); + } + } + let rows = txn.export_graph_slice(&on_view, state.id).map_err(db)?; let mut spans = Vec::new(); for (key, _) in &rows.graph { let (change, start, end) = decode_vertex(key); @@ -143,6 +189,125 @@ impl Repository { Ok(SandboxSlice { rows, spans }) } + /// Serve side: take a change a remote sandbox recorded on `view` and + /// apply it there — if it is exactly what it claims, recorded against + /// the view as it is now, and names nothing outside the view. + /// + /// The checks, in order: the bytes hash to `hash`; the view is still at + /// `base_state` (otherwise the sandbox fetches and records again); every + /// change it depends on or refers to that the repository knows is + /// visible on the view; every node its file operations name is a visible + /// change's; no path touches `.atomic`, `.atomic-sandbox` or + /// `.atomic-sandbox.d`; and it is new. Then it is saved and inserted + /// into `view`. Callers serialize submissions (one writer). + pub fn insert_submitted_change( + &self, + view: &str, + base_state: &str, + hash: &Hash, + bytes: &[u8], + ) -> Result, RepositoryError> { + use atomic_core::change::format_v3::reader::ChangeReader; + use atomic_core::types::Base32; + + let (change, computed) = match atomic_core::change::Change::deserialize(&mut &bytes[..]) { + Ok(parsed) => parsed, + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + if computed != *hash { + return Ok(Err(SubmitRejection::HashMismatch { + claimed: hash.to_base32(), + computed: computed.to_base32(), + })); + } + let referenced: Vec = match ChangeReader::open(&mut &bytes[..]) { + Ok(reader) => reader + .hash_table() + .hashes() + .iter() + .map(|h| Hash::from(*h)) + .filter(|h| h != hash) + .collect(), + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + + { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let current = state.state.to_base32(); + if current != base_state { + return Ok(Err(SubmitRejection::StaleView { current })); + } + if txn.get_internal(hash).map_err(db)?.is_some() { + return Ok(Err(SubmitRejection::AlreadyPresent(hash.to_base32()))); + } + let visible = super::collect_visible_change_ids(&txn, &state)?; + for dep in change.dependencies() { + match txn.get_internal(dep).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(dep.to_base32()))), + } + } + for other in &referenced { + if let Some(id) = txn.get_internal(other).map_err(db)? { + if !visible.contains(&id) { + return Ok(Err(SubmitRejection::ForeignChange(other.to_base32()))); + } + } + } + for ops in change.file_ops() { + if let Some(id) = ops + .referenced_node_ids() + .into_iter() + .find(|id| !visible.contains(id)) + { + return Ok(Err(SubmitRejection::ForeignNode(id.get()))); + } + if forbidden_path(ops.path()) { + return Ok(Err(SubmitRejection::ForbiddenPath(ops.path().to_string()))); + } + } + for hunk in change.hunks() { + if let Some(path) = hunk.path().filter(|p| forbidden_path(p)) { + return Ok(Err(SubmitRejection::ForbiddenPath(path.to_string()))); + } + } + } + + self.save_change_bytes(hash, bytes, &change)?; + let outcome = match self.insert_change(hash, crate::InsertOptions::default().view(view)) { + Ok(outcome) => outcome, + Err(e) => { + // Nothing of it stays: the change file goes with the failure. + let _ = std::fs::remove_file(self.change_store.change_path(hash)); + return Err(e); + } + }; + Ok(Ok(Submitted { + hash: *hash, + state: outcome.new_state.to_base32(), + })) + } + + /// Cache side: the view's state as the cache has it (base32) — what a + /// change recorded here is recorded against. + pub fn remote_sandbox_view_state(&self) -> Result { + use atomic_core::types::Base32; + let txn = self.pristine.read_txn().map_err(db)?; + let state = txn + .get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })?; + Ok(state.state.to_base32()) + } + /// Cache side: replace this cache's tree and view with `skeleton`'s. pub fn import_sandbox_skeleton( &self, diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index e598d62d..ba7311cf 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -84,6 +84,11 @@ pub struct SealResult { /// instead of looking for a local `.atomic/`. pub const SANDBOX_POINTER: &str = ".atomic-sandbox"; +/// A remote sandbox's own directory, beside its pointer: the local cache of +/// its repository's rows (`cache/`, an ordinary repository whose working +/// tree is the sandbox). Never tracked. +pub const SANDBOX_CACHE_DIR: &str = ".atomic-sandbox.d"; + /// Persisted contents of a sandbox pointer: where the canonical graph lives /// and which view this sandbox operates on. #[derive(Debug, Serialize, Deserialize)] @@ -107,8 +112,19 @@ pub(super) fn detect_sandbox(start: &Path) -> Option<(PathBuf, PathBuf, String)> let pointer = dir.join(SANDBOX_POINTER); if pointer.is_file() { let bytes = std::fs::read(&pointer).ok()?; - let parsed: SandboxPointer = serde_json::from_slice(&bytes).ok()?; - return Some((dir.clone(), parsed.canonical, parsed.view)); + if let Ok(parsed) = serde_json::from_slice::(&bytes) { + return Some((dir.clone(), parsed.canonical, parsed.view)); + } + // A remote pointer names an owner elsewhere; locally, the graph + // is the sandbox's cache (once `atomic sandbox materialize` has + // made it). + let remote: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + let view = remote.get("remote").and(remote.get("view"))?.as_str()?; + let cache = remote_cache_root(&dir); + return cache + .join(DOT_DIR) + .is_dir() + .then(|| (dir.clone(), cache, view.to_string())); } // Stop if we reach a real repository root — that's not a sandbox. if dir.join(DOT_DIR).join("pristine.redb").is_file() { @@ -120,7 +136,38 @@ pub(super) fn detect_sandbox(start: &Path) -> Option<(PathBuf, PathBuf, String)> } } +/// Where a remote sandbox rooted at `working_root` keeps its cache. +pub fn remote_cache_root(working_root: &Path) -> PathBuf { + working_root.join(SANDBOX_CACHE_DIR).join("cache") +} + impl Repository { + /// Make (or remake) a remote sandbox's cache from `skeleton`: an empty + /// repository under [`SANDBOX_CACHE_DIR`] holding the view's rows, with + /// the sandbox as its working tree and the tree on disk as the clean + /// baseline. Call it right after writing the view's files. + pub fn create_remote_sandbox_cache( + working_root: &Path, + skeleton: &super::SandboxSkeleton, + ) -> Result { + let cache = remote_cache_root(working_root); + if cache.exists() { + std::fs::remove_dir_all(&cache)?; + } + std::fs::create_dir_all(&cache)?; + drop(Self::init(&cache)?); + let repo = Self::open_sandbox(working_root, &cache, &skeleton.view.name)?; + repo.import_sandbox_skeleton(skeleton)?; + repo.reindex_working_copy()?; + Ok(repo) + } + + /// Whether this is a remote sandbox's cache (its graph arrives from the + /// repository's owner, and its changes go back to it). + pub fn is_remote_sandbox(&self) -> bool { + self.is_sandbox && self.dot_dir.starts_with(self.root.join(SANDBOX_CACHE_DIR)) + } + /// Open a repository for an agent sandbox. /// /// The agent's private working tree is at `working_root`, but the graph diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs index e58faf26..8198c4bf 100644 --- a/atomic-repository/tests/remote_cache_parity_test.rs +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -250,3 +250,191 @@ fn a_second_record_on_top_of_the_first() { pair.edit(|root| write(root, "src/lib.rs", "pub fn a() { 1 }\n")); assert_parity(&pair, &view, &live, "second"); } + +// ── Submitting: the repository takes the cache's change ───────────────── + +use atomic_repository::SubmitRejection; + +fn view_state(repo: &Repository, view: &str) -> String { + let skeleton = repo + .export_sandbox_skeleton(view, &BTreeSet::new()) + .unwrap(); + atomic_core::types::Base32::to_base32(&skeleton.view.state) +} + +/// Record in the cache (hydrated as FileStates would) and return the change. +fn record_in_cache( + pair: &Pair, + view: &str, + live: &BTreeSet, + what: &str, +) -> (atomic_core::types::Hash, Vec) { + pair.hydrate(view, live); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = pair.cache.record(header(what), options).unwrap(); + (*outcome.hash(), outcome.v3_bytes().unwrap().to_vec()) +} + +fn tree_of(repo: &Repository, view: &str) -> Vec<(String, Vec)> { + let mut files = Vec::new(); + repo.materialize_view_entries::<()>(view, |e| { + if e.kind == atomic_repository::ViewEntryKind::File { + files.push((e.path, e.content)); + } + Ok(()) + }) + .unwrap() + .unwrap(); + files +} + +#[test] +fn a_submitted_change_lands_on_the_view_and_the_next_one_builds_on_it() { + let (pair, view, mut live) = pair(two_files); + // Only the cache's tree changes: the repository learns of it by submission. + write(pair.cache_root(), "README.md", "hello\nfrom the sandbox\n"); + write(pair.cache_root(), "src/new.rs", "pub fn n() {}\n"); + pair.cache + .add("src/new.rs", TrackingOptions::default()) + .unwrap(); + let base = view_state(&pair.host, &view); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "from the sandbox"); + + let submitted = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap() + .expect("accepted"); + assert_ne!(submitted.state, base, "the view moved"); + let tree = tree_of(&pair.host, &view); + assert!(tree.contains(&("README.md".into(), b"hello\nfrom the sandbox\n".to_vec()))); + assert!(tree.contains(&("src/new.rs".into(), b"pub fn n() {}\n".to_vec()))); + + // The same change again: the view has moved, and then it's already there. + let again = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!(again, Err(SubmitRejection::StaleView { .. })), + "{again:?}" + ); + let again = pair + .host + .insert_submitted_change(&view, &submitted.state, &hash, &bytes) + .unwrap(); + assert!( + matches!(again, Err(SubmitRejection::AlreadyPresent(_))), + "{again:?}" + ); + + // The cache takes the new skeleton (with the repository's inode for the + // new file) and records on top. + live.clear(); + pair.host + .materialize_view_entries::<()>(&view, |e| { + live.insert(e.inode); + Ok(()) + }) + .unwrap() + .unwrap(); + let skeleton = pair.host.export_sandbox_skeleton(&view, &live).unwrap(); + pair.cache.import_sandbox_skeleton(&skeleton).unwrap(); + pair.cache.reindex_working_copy().unwrap(); + write(pair.cache_root(), "src/new.rs", "pub fn n() { 2 }\n"); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "second from the sandbox"); + pair.host + .insert_submitted_change(&view, &submitted.state, &hash, &bytes) + .unwrap() + .expect("the second is accepted too"); + assert!( + tree_of(&pair.host, &view).contains(&("src/new.rs".into(), b"pub fn n() { 2 }\n".to_vec())) + ); +} + +#[test] +fn a_tampered_change_is_refused_and_leaves_nothing() { + let (pair, view, live) = pair(two_files); + write(pair.cache_root(), "README.md", "tampered?\n"); + let base = view_state(&pair.host, &view); + let (hash, mut bytes) = record_in_cache(&pair, &view, &live, "tamper"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + let refused = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!( + refused, + Err(SubmitRejection::HashMismatch { .. } | SubmitRejection::Malformed(_)) + ), + "{refused:?}" + ); + assert_eq!(view_state(&pair.host, &view), base); + assert!(!pair.host.has_change(&hash)); +} + +#[test] +fn a_change_recorded_against_an_older_view_is_refused() { + let (pair, view, live) = pair(two_files); + let base = view_state(&pair.host, &view); + write(pair.cache_root(), "README.md", "late\n"); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "late"); + // Meanwhile the view moves on the repository. + write(pair.host_root(), "src/lib.rs", "pub fn moved() {}\n"); + record_and_apply(&pair.host, "meanwhile"); + let refused = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!(refused, Err(SubmitRejection::StaleView { .. })), + "{refused:?}" + ); +} + +#[test] +fn a_change_from_another_view_is_refused() { + // Work recorded on a draft view depends on that view's changes; submitted + // to `dev` it names changes `dev` can't see. + let host_dir = TempDir::new().unwrap(); + let draft_dir = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("other", "dev").unwrap(); + host.provision_sandbox(draft_dir.path().join("w"), "other") + .unwrap(); + } + let draft = Repository::open_existing(draft_dir.path().join("w")).unwrap(); + write(draft.root(), "README.md", "only on other\n"); + record_and_apply(&draft, "on other"); + write(draft.root(), "README.md", "only on other, again\n"); + let (hash, bytes) = { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = draft.record(header("on top of other"), options).unwrap(); + (*outcome.hash(), outcome.v3_bytes().unwrap().to_vec()) + }; + drop(draft); + + let host = Repository::open_existing(host_dir.path()).unwrap(); + let base = view_state(&host, "dev"); + let refused = host + .insert_submitted_change("dev", &base, &hash, &bytes) + .unwrap(); + assert!( + matches!( + refused, + Err(SubmitRejection::ForeignChange(_) | SubmitRejection::ForeignNode(_)) + ), + "{refused:?}" + ); + assert_eq!(view_state(&host, "dev"), base); +} From 12bccdd8abf5aabddb80519a382884f2bd94d90d Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:28:21 -0500 Subject: [PATCH 10/23] Read a remote sandbox's recorded state: diff, restore, log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache holds a view's rows, not its content. diff and restore now load the content they compare against first (FileStates, as record does), and log fetches the change files the view has and the cache lacks — a new Changes request, answered only for changes on the token's view, each file kept only if its bytes hash to what it claims. --- atomic-cli/src/commands/agent/owner/mod.rs | 47 ++++++++++++- .../src/commands/agent/owner/sandbox.rs | 24 ++++++- atomic-cli/src/commands/diff/command.rs | 19 +++-- atomic-cli/src/commands/log/command.rs | 6 ++ atomic-cli/src/commands/mod.rs | 11 +++ atomic-cli/src/commands/restore.rs | 1 + .../tests/remote_sandbox_integration_test.rs | 26 +++++++ .../src/repository/remote_cache.rs | 70 +++++++++++++++++++ 8 files changed, 194 insertions(+), 10 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/mod.rs b/atomic-cli/src/commands/agent/owner/mod.rs index 17eb7494..e99cdb2a 100644 --- a/atomic-cli/src/commands/agent/owner/mod.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -218,6 +218,12 @@ enum OwnerRequest { view: Option, inodes: Vec, }, + /// The change files of `hashes`, each on the view. + Changes { + #[serde(default)] + view: Option, + hashes: Vec, + }, /// A change a sandbox recorded against `base_state`, as its V3 bytes. SubmitChange { #[serde(default)] @@ -229,6 +235,14 @@ enum OwnerRequest { }, } +/// A change file on the wire. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct WireChange { + hash: Hash, + #[serde(with = "base64_bytes")] + bytes: Vec, +} + /// Bytes as base64 in the JSON frames. mod base64_bytes { use serde::{Deserialize, Deserializer, Serializer}; @@ -323,6 +337,9 @@ pub(crate) enum OwnerResponse { ChangeRefused { rejection: atomic_repository::SubmitRejection, }, + Changes { + changes: Vec, + }, Error { code: String, message: String, @@ -1114,6 +1131,31 @@ pub(crate) fn hydrate_remote_sandbox(repo: &Repository) -> anyhow::Result<()> { } } +/// In a remote sandbox: fetch the change files its view has and it +/// doesn't (for `log` and anything else that reads changes). +pub(crate) fn fetch_remote_sandbox_changes(repo: &Repository) -> anyhow::Result { + let missing = repo.missing_sandbox_changes()?; + for batch in missing.chunks(32) { + match request( + repo.root(), + OwnerRequest::Changes { + view: None, + hashes: batch.to_vec(), + }, + )? { + OwnerResponse::Changes { changes } => { + let changes: Vec<_> = changes.into_iter().map(|c| (c.hash, c.bytes)).collect(); + repo.hold_sandbox_changes(&changes)?; + } + OwnerResponse::ChangeRefused { rejection } => { + return Err(anyhow!("the repository refused: {rejection}")) + } + other => return Err(unexpected_response("changes", other)), + } + } + Ok(missing.len()) +} + /// In a remote sandbox: hand a recorded change to the repository's owner. /// On success the cache takes the view as it now is. pub(crate) fn submit_from_remote_sandbox( @@ -1559,6 +1601,7 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram | OwnerRequest::CloseSandbox { .. } | OwnerRequest::Materialize { .. } | OwnerRequest::FileStates { .. } + | OwnerRequest::Changes { .. } | OwnerRequest::SubmitChange { .. } => ( OwnerResponse::Error { code: "internal".to_string(), @@ -1683,7 +1726,9 @@ where let response = sandbox::admin(&owner, request.request.clone()).await; write_frame(&mut stream, &respond(request.request_id, response)).await?; } - OwnerRequest::FileStates { .. } | OwnerRequest::SubmitChange { .. } + OwnerRequest::FileStates { .. } + | OwnerRequest::Changes { .. } + | OwnerRequest::SubmitChange { .. } if request.version == PROTOCOL_VERSION => { let request_id = request.request_id.clone(); diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index 18e61684..4b459ac2 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -71,6 +71,7 @@ pub(crate) fn authorize( OwnerRequest::Ping => {} OwnerRequest::Materialize { view: asked } | OwnerRequest::FileStates { view: asked, .. } + | OwnerRequest::Changes { view: asked, .. } | OwnerRequest::SubmitChange { view: asked, .. } => { if asked.as_deref().is_some_and(|v| v != view) { return Err(forbidden("that view")); @@ -307,9 +308,9 @@ pub(crate) async fn record_request( request: OwnerRequest, ) -> OwnerResponse { let named = match &request { - OwnerRequest::FileStates { view, .. } | OwnerRequest::SubmitChange { view, .. } => { - view.clone() - } + OwnerRequest::FileStates { view, .. } + | OwnerRequest::Changes { view, .. } + | OwnerRequest::SubmitChange { view, .. } => view.clone(), _ => None, }; let Some(view) = grant.map(|g| g.view).or(named) else { @@ -330,6 +331,23 @@ pub(crate) async fn record_request( Err(e) => error("internal", e.to_string()), } } + OwnerRequest::Changes { hashes, .. } => { + let read = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = Repository::open_readonly(&root)?; + Ok(repo.export_sandbox_changes(&view, &hashes)?) + }); + match read.await { + Ok(Ok(Ok(changes))) => OwnerResponse::Changes { + changes: changes + .into_iter() + .map(|(hash, bytes)| super::WireChange { hash, bytes }) + .collect(), + }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Err(e)) => error("changes", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } OwnerRequest::SubmitChange { base_state, hash, diff --git a/atomic-cli/src/commands/diff/command.rs b/atomic-cli/src/commands/diff/command.rs index 243a9e89..908c5636 100644 --- a/atomic-cli/src/commands/diff/command.rs +++ b/atomic-cli/src/commands/diff/command.rs @@ -224,12 +224,19 @@ impl Command for Diff { // Find the repository root let repo_root = find_repository_root()?; - // Open the repository - let repo = crate::commands::open_readonly_repository(&repo_root).map_err(|e| { - CliError::InvalidRepository { - reason: e.to_string(), - } - })?; + // Open the repository. A remote sandbox's cache takes the content it + // is about to compare against, so it opens writable. + let repo = if atomic_repository::remote_cache_root(&repo_root).is_dir() { + let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + crate::commands::hydrate_if_remote(&repo)?; + repo + } else { + crate::commands::open_readonly_repository(&repo_root).map_err(|e| { + CliError::InvalidRepository { + reason: e.to_string(), + } + })? + }; // Parse algorithm let algorithm = self.parse_algorithm()?; diff --git a/atomic-cli/src/commands/log/command.rs b/atomic-cli/src/commands/log/command.rs index 0741d8ae..074ff2e2 100644 --- a/atomic-cli/src/commands/log/command.rs +++ b/atomic-cli/src/commands/log/command.rs @@ -499,6 +499,12 @@ impl Command for Log { fn run(&self) -> CliResult<()> { // Find and open repository let repo_root = find_repository_root()?; + // A remote sandbox's cache fetches the change files it lacks first. + if atomic_repository::remote_cache_root(&repo_root).is_dir() { + let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + crate::commands::agent::owner::fetch_remote_sandbox_changes(&repo) + .map_err(CliError::Internal)?; + } let repo = crate::commands::open_readonly_repository(&repo_root).map_err(|e| match e { atomic_repository::RepositoryError::NotFound { path } => CliError::RepositoryNotFound { searched_path: path.into(), diff --git a/atomic-cli/src/commands/mod.rs b/atomic-cli/src/commands/mod.rs index d5ff1859..c1d66b49 100644 --- a/atomic-cli/src/commands/mod.rs +++ b/atomic-cli/src/commands/mod.rs @@ -358,6 +358,17 @@ pub fn open_repository(path: Option<&Path>) -> CliResult { Repository::open(&repo_path).map_err(CliError::from) } +/// In a remote sandbox, load what reading the working tree's recorded +/// state needs (old content for `diff`, `restore`) into its cache; the cache +/// holds a view's rows, not its content, until a command asks. Elsewhere, +/// nothing. +pub fn hydrate_if_remote(repo: &Repository) -> CliResult<()> { + if repo.is_remote_sandbox() { + agent::owner::hydrate_remote_sandbox(repo).map_err(CliError::Internal)?; + } + Ok(()) +} + /// Acquire a read-only repository for a CLI query, allowing short-lived writers /// (such as Stop checkpoint publication) to finish first. Only typed database /// contention is retried, for at most ten seconds; all other errors propagate diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index 00b41d9c..e2bcf581 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -332,6 +332,7 @@ impl Command for Restore { // Find repository let repo_root = find_repository_root()?; let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + crate::commands::hydrate_if_remote(&repo)?; // Compute status once. Restore only touches tracked files, so we skip // the untracked scan, and we reuse this single status for both the diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index bb4ed348..f2872975 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -192,6 +192,32 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { atomic(&vm_dir, &["record", "-a", "-m", "again from the sandbox"]), "second record in the sandbox", ); + // Reading the recorded state works in the sandbox too: diff against + // the view, restore from it, and the view's history. + std::fs::write( + vm_dir.join("README.md"), + "hello\nfrom the sandbox\nuncommitted\n", + ) + .unwrap(); + let diff = ok(atomic(&vm_dir, &["diff"]), "diff in the sandbox"); + assert!( + diff.contains("+uncommitted") && !diff.contains("+hello"), + "{diff}" + ); + ok( + atomic(&vm_dir, &["restore", "README.md"]), + "restore in the sandbox", + ); + assert_eq!( + std::fs::read_to_string(vm_dir.join("README.md")).unwrap(), + "hello\nfrom the sandbox\n" + ); + let log = ok(atomic(&vm_dir, &["log"]), "log in the sandbox"); + assert!( + log.contains("again from the sandbox") && log.contains("first"), + "{log}" + ); + let log = ok(atomic(host.path(), &["log"]), "host log"); assert!( log.contains("from the sandbox") && log.contains("again from the sandbox"), diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index 149f054f..11214ae5 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -78,6 +78,9 @@ pub enum SubmitRejection { AlreadyPresent(String), } +/// A change file: its hash and V3 bytes. +pub type ChangeFile = (Hash, Vec); + /// A submitted change, applied. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Submitted { @@ -294,6 +297,73 @@ impl Repository { })) } + /// Serve side: the V3 bytes of each of `hashes` — changes `view` can + /// see, and only those (anything else is refused, whole). + pub fn export_sandbox_changes( + &self, + view: &str, + hashes: &[Hash], + ) -> Result, SubmitRejection>, RepositoryError> { + use atomic_core::types::Base32; + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible = super::collect_visible_change_ids(&txn, &state)?; + let mut out = Vec::with_capacity(hashes.len()); + for hash in hashes { + match txn.get_internal(hash).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(hash.to_base32()))), + } + out.push((*hash, std::fs::read(self.change_store.change_path(hash))?)); + } + Ok(Ok(out)) + } + + /// Cache side: the changes the view has that this cache holds no file + /// for. + pub fn missing_sandbox_changes(&self) -> Result, RepositoryError> { + let txn = self.pristine.read_txn().map_err(db)?; + let state = txn + .get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })?; + let mut missing = Vec::new(); + for id in super::collect_visible_change_ids(&txn, &state)? { + if let Some(hash) = txn.get_external(id).map_err(db)? { + if !self.change_store.has_change(&hash) { + missing.push(hash); + } + } + } + Ok(missing) + } + + /// Cache side: keep change files the owner sent — each only if its + /// bytes hash to what it claims. + pub fn hold_sandbox_changes(&self, changes: &[ChangeFile]) -> Result<(), RepositoryError> { + use atomic_core::types::Base32; + for (hash, bytes) in changes { + let (change, computed) = atomic_core::change::Change::deserialize(&mut &bytes[..]) + .map_err(|e| RepositoryError::Database(e.to_string()))?; + if computed != *hash { + return Err(RepositoryError::Database(format!( + "the owner sent {} as {}", + computed.to_base32(), + hash.to_base32() + ))); + } + self.save_change_bytes(hash, bytes, &change)?; + } + Ok(()) + } + /// Cache side: the view's state as the cache has it (base32) — what a /// change recorded here is recorded against. pub fn remote_sandbox_view_state(&self) -> Result { From f1872bb1ac3b9038541e18850d5023855d8d3b6e Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:39:19 -0500 Subject: [PATCH 11/23] Remote sandboxes inside Repository: any caller records, and provenance lands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remote path moves out of individual commands into Repository, so every caller gets it — the atomic CLI and an agent's turn hooks alike. A process installs one RemoteSandboxLink (the atomic binary installs the owner client at startup); in a remote sandbox's cache, record hydrates first, write_recorded submits instead of applying (a change that didn't land fails the record), diff and restore hydrate, log fetches change files, and publish_provenance_checkpoint publishes the checkpoint in the repository. PublishProvenance, the owner's side: the sandbox's own session, and every change the provenance explains on its view; the graph arrives serialized, so the repository publishes what the sandbox hashed. Two authorization fixes the agent flow showed: asking after a session nobody has written to is allowed (the store answers "no such turn"), and BindCheckpointHash names a provenance graph, not a change. An agent's session-start, turn-start, turn-end and session-end in a remote sandbox now leave its change on the view and its session ledger and provenance in the repository. --- atomic-cli/src/commands/agent/owner/mod.rs | 145 +++++++++------ .../src/commands/agent/owner/sandbox.rs | 64 +++++-- atomic-cli/src/commands/diff/command.rs | 3 +- atomic-cli/src/commands/log/command.rs | 4 +- atomic-cli/src/commands/mod.rs | 11 -- atomic-cli/src/commands/record/command.rs | 28 +-- atomic-cli/src/commands/restore.rs | 3 +- atomic-cli/src/main.rs | 4 + .../tests/remote_sandbox_integration_test.rs | 93 ++++++++++ atomic-repository/src/repository/changes.rs | 14 ++ atomic-repository/src/repository/insert.rs | 3 + atomic-repository/src/repository/mod.rs | 5 +- atomic-repository/src/repository/record.rs | 7 + .../src/repository/remote_cache.rs | 165 ++++++++++++++++++ 14 files changed, 444 insertions(+), 105 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/mod.rs b/atomic-cli/src/commands/agent/owner/mod.rs index e99cdb2a..4ebc670f 100644 --- a/atomic-cli/src/commands/agent/owner/mod.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -224,6 +224,14 @@ enum OwnerRequest { view: Option, hashes: Vec, }, + /// A sandbox's checkpoint: its provenance graph (serialized) and turn. + PublishProvenance { + #[serde(default)] + view: Option, + #[serde(with = "base64_bytes")] + graph: Vec, + turn: SessionTurn, + }, /// A change a sandbox recorded against `base_state`, as its V3 bytes. SubmitChange { #[serde(default)] @@ -340,6 +348,9 @@ pub(crate) enum OwnerResponse { Changes { changes: Vec, }, + ProvenancePublished { + publication: atomic_core::change::session::SessionCheckpointPublication, + }, Error { code: String, message: String, @@ -1122,68 +1133,98 @@ pub(crate) fn materialize_remote_sandbox(start: &Path) -> anyhow::Result<(PathBu Ok((root, written)) } -/// In a remote sandbox: load what `record` will read into its cache. -pub(crate) fn hydrate_remote_sandbox(repo: &Repository) -> anyhow::Result<()> { - let inodes = repo.sandbox_slice_inodes()?; - match request(repo.root(), OwnerRequest::FileStates { view: None, inodes })? { - OwnerResponse::FileStates { slice } => Ok(repo.import_sandbox_slice(&slice)?), - other => Err(unexpected_response("file states", other)), - } +/// The owner protocol as a remote sandbox's link to its repository's owner: +/// what lets `Repository` record, diff, restore and read history in a +/// remote sandbox, whoever calls it. +pub(crate) struct OwnerLink; + +/// One owner request from a remote sandbox, on its own thread: callers +/// (an agent's turn hooks, say) may already be inside a runtime. +fn link_request(root: &Path, body: OwnerRequest) -> Result { + let root = root.to_path_buf(); + run_outside_async_runtime(move || request(&root, body)).map_err(|e| format!("{e:#}")) } -/// In a remote sandbox: fetch the change files its view has and it -/// doesn't (for `log` and anything else that reads changes). -pub(crate) fn fetch_remote_sandbox_changes(repo: &Repository) -> anyhow::Result { - let missing = repo.missing_sandbox_changes()?; - for batch in missing.chunks(32) { - match request( - repo.root(), - OwnerRequest::Changes { - view: None, - hashes: batch.to_vec(), - }, - )? { - OwnerResponse::Changes { changes } => { - let changes: Vec<_> = changes.into_iter().map(|c| (c.hash, c.bytes)).collect(); - repo.hold_sandbox_changes(&changes)?; - } - OwnerResponse::ChangeRefused { rejection } => { - return Err(anyhow!("the repository refused: {rejection}")) - } - other => return Err(unexpected_response("changes", other)), +impl atomic_repository::RemoteSandboxLink for OwnerLink { + fn file_states( + &self, + root: &Path, + inodes: Vec, + ) -> Result { + match link_request(root, OwnerRequest::FileStates { view: None, inodes })? { + OwnerResponse::FileStates { slice } => Ok(*slice), + other => Err(unexpected_response("file states", other).to_string()), } } - Ok(missing.len()) -} -/// In a remote sandbox: hand a recorded change to the repository's owner. -/// On success the cache takes the view as it now is. -pub(crate) fn submit_from_remote_sandbox( - repo: &Repository, - hash: Hash, - bytes: Vec, -) -> anyhow::Result { - let base_state = repo.remote_sandbox_view_state()?; - match request( - repo.root(), - OwnerRequest::SubmitChange { + fn submit( + &self, + root: &Path, + base_state: String, + hash: Hash, + bytes: Vec, + ) -> Result< + Result< + ( + atomic_repository::Submitted, + atomic_repository::SandboxSkeleton, + ), + atomic_repository::SubmitRejection, + >, + String, + > { + let body = OwnerRequest::SubmitChange { view: None, base_state, hash, bytes, - }, - )? { - OwnerResponse::ChangeSubmitted { - submitted, - skeleton, - } => { - repo.import_sandbox_skeleton(&skeleton)?; - Ok(submitted) + }; + match link_request(root, body)? { + OwnerResponse::ChangeSubmitted { + submitted, + skeleton, + } => Ok(Ok((submitted, *skeleton))), + OwnerResponse::ChangeRefused { rejection } => Ok(Err(rejection)), + other => Err(unexpected_response("submit change", other).to_string()), } - OwnerResponse::ChangeRefused { rejection } => { - Err(anyhow!("the repository refused the change: {rejection}")) + } + + fn changes( + &self, + root: &Path, + hashes: Vec, + ) -> Result, String> { + match link_request(root, OwnerRequest::Changes { view: None, hashes })? { + OwnerResponse::Changes { changes } => { + Ok(changes.into_iter().map(|c| (c.hash, c.bytes)).collect()) + } + OwnerResponse::ChangeRefused { rejection } => Err(rejection.to_string()), + other => Err(unexpected_response("changes", other).to_string()), + } + } + + fn publish_provenance( + &self, + root: &Path, + graph: Vec, + turn: SessionTurn, + ) -> Result< + Result< + atomic_core::change::session::SessionCheckpointPublication, + atomic_repository::SubmitRejection, + >, + String, + > { + let body = OwnerRequest::PublishProvenance { + view: None, + graph, + turn, + }; + match link_request(root, body)? { + OwnerResponse::ProvenancePublished { publication } => Ok(Ok(publication)), + OwnerResponse::ChangeRefused { rejection } => Ok(Err(rejection)), + other => Err(unexpected_response("publish provenance", other).to_string()), } - other => Err(unexpected_response("submit change", other)), } } @@ -1602,6 +1643,7 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram | OwnerRequest::Materialize { .. } | OwnerRequest::FileStates { .. } | OwnerRequest::Changes { .. } + | OwnerRequest::PublishProvenance { .. } | OwnerRequest::SubmitChange { .. } => ( OwnerResponse::Error { code: "internal".to_string(), @@ -1728,6 +1770,7 @@ where } OwnerRequest::FileStates { .. } | OwnerRequest::Changes { .. } + | OwnerRequest::PublishProvenance { .. } | OwnerRequest::SubmitChange { .. } if request.version == PROTOCOL_VERSION => { diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index 4b459ac2..eaa8898e 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -77,6 +77,16 @@ pub(crate) fn authorize( return Err(forbidden("that view")); } } + OwnerRequest::PublishProvenance { + view: asked, turn, .. + } => { + if asked.as_deref().is_some_and(|v| v != view) { + return Err(forbidden("that view")); + } + if !tokens.owns_session(view, &turn.session_id) { + return Err(forbidden("that session")); + } + } OwnerRequest::ReserveProvenanceTurn { session_id, turn_number, @@ -106,21 +116,38 @@ pub(crate) fn authorize( } require_on_view(&owner.root, view, &source.change_hashes)?; } - OwnerRequest::BindCheckpointHash { - provenance_id, - hash, - .. - } => { + OwnerRequest::BindCheckpointHash { provenance_id, .. } => { + // Its hash names the checkpoint's provenance graph, not a change; + // the changes it explains were checked at PrepareCheckpoint. if !tokens.owns_provenance(view, *provenance_id) { return Err(forbidden("that provenance turn")); } - require_on_view(&owner.root, view, std::slice::from_ref(hash))?; } - OwnerRequest::StopTurn { session_id, .. } - | OwnerRequest::ResumeTurn { session_id, .. } - | OwnerRequest::AbandonTurn { session_id, .. } - | OwnerRequest::TurnStatus { session_id, .. } => { - if !tokens.owns_session(view, session_id) { + OwnerRequest::StopTurn { + session_id, + turn_number, + .. + } + | OwnerRequest::ResumeTurn { + session_id, + turn_number, + .. + } + | OwnerRequest::AbandonTurn { + session_id, + turn_number, + .. + } + | OwnerRequest::TurnStatus { + session_id, + turn_number, + } => { + // Asking after a session nobody has written to is harmless (the + // store answers "no such turn"); anyone else's is not. + if !tokens.owns_session(view, session_id) + && !session_is_fresh(&owner.store, session_id, *turn_number) + .map_err(|e| refuse("provenance-store", e.to_string()))? + { return Err(forbidden("that session")); } } @@ -310,6 +337,7 @@ pub(crate) async fn record_request( let named = match &request { OwnerRequest::FileStates { view, .. } | OwnerRequest::Changes { view, .. } + | OwnerRequest::PublishProvenance { view, .. } | OwnerRequest::SubmitChange { view, .. } => view.clone(), _ => None, }; @@ -348,6 +376,20 @@ pub(crate) async fn record_request( Err(e) => error("internal", e.to_string()), } } + OwnerRequest::PublishProvenance { graph, turn, .. } => { + let _one_at_a_time = owner.submissions.lock().await; + let write = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = + Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; + Ok(repo.publish_sandbox_provenance(&view, &graph, turn)?) + }); + match write.await { + Ok(Ok(Ok(publication))) => OwnerResponse::ProvenancePublished { publication }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Err(e)) => error("provenance", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } OwnerRequest::SubmitChange { base_state, hash, diff --git a/atomic-cli/src/commands/diff/command.rs b/atomic-cli/src/commands/diff/command.rs index 908c5636..560bb13b 100644 --- a/atomic-cli/src/commands/diff/command.rs +++ b/atomic-cli/src/commands/diff/command.rs @@ -228,7 +228,8 @@ impl Command for Diff { // is about to compare against, so it opens writable. let repo = if atomic_repository::remote_cache_root(&repo_root).is_dir() { let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; - crate::commands::hydrate_if_remote(&repo)?; + repo.hydrate_remote_sandbox() + .map_err(CliError::Repository)?; repo } else { crate::commands::open_readonly_repository(&repo_root).map_err(|e| { diff --git a/atomic-cli/src/commands/log/command.rs b/atomic-cli/src/commands/log/command.rs index 074ff2e2..44354145 100644 --- a/atomic-cli/src/commands/log/command.rs +++ b/atomic-cli/src/commands/log/command.rs @@ -502,8 +502,8 @@ impl Command for Log { // A remote sandbox's cache fetches the change files it lacks first. if atomic_repository::remote_cache_root(&repo_root).is_dir() { let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; - crate::commands::agent::owner::fetch_remote_sandbox_changes(&repo) - .map_err(CliError::Internal)?; + repo.fetch_remote_sandbox_changes() + .map_err(CliError::Repository)?; } let repo = crate::commands::open_readonly_repository(&repo_root).map_err(|e| match e { atomic_repository::RepositoryError::NotFound { path } => CliError::RepositoryNotFound { diff --git a/atomic-cli/src/commands/mod.rs b/atomic-cli/src/commands/mod.rs index c1d66b49..d5ff1859 100644 --- a/atomic-cli/src/commands/mod.rs +++ b/atomic-cli/src/commands/mod.rs @@ -358,17 +358,6 @@ pub fn open_repository(path: Option<&Path>) -> CliResult { Repository::open(&repo_path).map_err(CliError::from) } -/// In a remote sandbox, load what reading the working tree's recorded -/// state needs (old content for `diff`, `restore`) into its cache; the cache -/// holds a view's rows, not its content, until a command asks. Elsewhere, -/// nothing. -pub fn hydrate_if_remote(repo: &Repository) -> CliResult<()> { - if repo.is_remote_sandbox() { - agent::owner::hydrate_remote_sandbox(repo).map_err(CliError::Internal)?; - } - Ok(()) -} - /// Acquire a read-only repository for a CLI query, allowing short-lived writers /// (such as Stop checkpoint publication) to finish first. Only typed database /// contention is retried, for at most ten seconds; all other errors propagate diff --git a/atomic-cli/src/commands/record/command.rs b/atomic-cli/src/commands/record/command.rs index 1595d49d..912f525f 100644 --- a/atomic-cli/src/commands/record/command.rs +++ b/atomic-cli/src/commands/record/command.rs @@ -40,13 +40,7 @@ impl Command for Record { let header = header_builder.build(); // Build record options - let mut options = self.build_options()?; - // A remote sandbox records against its cache and hands the change - // to the repository's owner; nothing is saved or applied here. - let remote = repo.is_remote_sandbox(); - if remote { - options = options.save_to_store(false).apply_after_record(false); - } + let options = self.build_options()?; // If --all, first add all untracked files if self.all { @@ -62,11 +56,6 @@ impl Command for Record { } } - if remote { - crate::commands::agent::owner::hydrate_remote_sandbox(&repo) - .map_err(CliError::Internal)?; - } - // Record the changes. // // This match is deliberately exhaustive: no `_ =>` arm. The catch-all @@ -117,21 +106,6 @@ impl Command for Record { } })?; - if remote { - let bytes = outcome - .v3_bytes() - .ok_or_else(|| CliError::Internal(anyhow::anyhow!("the change has no bytes")))? - .to_vec(); - crate::commands::agent::owner::submit_from_remote_sandbox( - &repo, - *outcome.hash(), - bytes, - ) - .map_err(CliError::Internal)?; - // What was just recorded is the new baseline. - repo.reindex_working_copy().map_err(CliError::Repository)?; - } - // Display result let output = self.format_outcome(&repo, &outcome); print!("{}", output); diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index e2bcf581..02e7cc12 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -332,7 +332,8 @@ impl Command for Restore { // Find repository let repo_root = find_repository_root()?; let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; - crate::commands::hydrate_if_remote(&repo)?; + repo.hydrate_remote_sandbox() + .map_err(CliError::Repository)?; // Compute status once. Restore only touches tracked files, so we skip // the untracked scan, and we reuse this single status for both the diff --git a/atomic-cli/src/main.rs b/atomic-cli/src/main.rs index c1c93ff5..c2c13d29 100644 --- a/atomic-cli/src/main.rs +++ b/atomic-cli/src/main.rs @@ -936,6 +936,10 @@ fn main() { // Initialize logging init_logging(); + // A remote sandbox's cache reaches its repository's owner through the + // owner protocol, whichever command (or agent hook) opens it. + atomic_repository::set_remote_sandbox_link(Box::new(commands::agent::owner::OwnerLink)); + // Dynamic shell completion. When invoked in completion mode (the `COMPLETE` // env var is set by the installed shell hook), this emits candidates — // including live view names and change hashes registered on the insert diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index f2872975..43df3323 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -279,3 +279,96 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { ); assert!(refused.contains("unauthorized"), "{refused}"); } + +fn hook(cwd: &Path, verb: &str, payload: Value) -> Output { + use std::io::Write; + let mut child = Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(["agent", "hooks", "sherpa", verb]) + .current_dir(cwd) + .env("ATOMIC_OWNER_IROH_OFFLINE", "1") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("run atomic agent hooks"); + child + .stdin + .take() + .unwrap() + .write_all(payload.to_string().as_bytes()) + .unwrap(); + child.wait_with_output().unwrap() +} + +#[test] +fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--view", + "dev", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + + let cwd = vm_dir.to_str().unwrap(); + let now = "2026-01-01T00:00:00Z"; + let turn = |n: u32| { + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "model": "m", "provider": "p", + "turn_number": n, "intent_title": "greet the reader", "timestamp": now, + }) + }; + ok( + hook( + &vm_dir, + "session-start", + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "model": "m", "provider": "p", + "turn_number": 0, "timestamp": now, + }), + ), + "session-start", + ); + ok(hook(&vm_dir, "turn-start", turn(1)), "turn-start"); + std::fs::write(vm_dir.join("README.md"), "hello\nreader\n").unwrap(); + ok(hook(&vm_dir, "turn-end", turn(1)), "turn-end"); + ok( + hook( + &vm_dir, + "session-end", + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "turn_number": 1, "timestamp": now, + }), + ), + "session-end", + ); + + // The turn's change is on the view, and its provenance is in the + // repository — not only in the sandbox. + let log = ok(atomic(host.path(), &["log"]), "host log"); + assert!(log.contains("greet the reader"), "{log}"); + let session = ok( + atomic(host.path(), &["session", "show", "agent-session"]), + "host session", + ); + assert!(session.contains("Turns: 1"), "{session}"); + assert!( + session.contains("Goal marker: greet the reader"), + "{session}" + ); +} diff --git a/atomic-repository/src/repository/changes.rs b/atomic-repository/src/repository/changes.rs index 7897bb8b..2a695ba8 100644 --- a/atomic-repository/src/repository/changes.rs +++ b/atomic-repository/src/repository/changes.rs @@ -1054,7 +1054,21 @@ impl Repository { /// dependencies, derived session tables, immutable `SESSION_TURNS` append, /// manifest, and `SESSION_HEADS` advance then commit in one pristine /// transaction. Repeating the exact publication is idempotent. + /// + /// In a remote sandbox the checkpoint is published in its repository, + /// through the owner; the result is the repository's. pub fn publish_provenance_checkpoint( + &self, + graph: &atomic_core::change::ProvenanceGraph, + turn: atomic_core::change::session::SessionTurn, + ) -> Result { + if self.is_remote_sandbox() { + return self.publish_remote_provenance_checkpoint(graph, turn); + } + self.publish_local_provenance_checkpoint(graph, turn) + } + + pub(crate) fn publish_local_provenance_checkpoint( &self, graph: &atomic_core::change::ProvenanceGraph, mut turn: atomic_core::change::session::SessionTurn, diff --git a/atomic-repository/src/repository/insert.rs b/atomic-repository/src/repository/insert.rs index 1882aaae..e2bf43d4 100644 --- a/atomic-repository/src/repository/insert.rs +++ b/atomic-repository/src/repository/insert.rs @@ -2154,6 +2154,9 @@ impl Repository { outcome: &RecordOutcome, mut options: InsertOptions, ) -> Result { + if self.is_remote_sandbox() { + return self.submit_recorded(outcome); + } let trace_record = std::env::var_os("ATOMIC_TRACE_RECORD").is_some(); let change = outcome.change(); let hash = outcome.hash(); diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index ec71760e..4150687d 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -81,7 +81,10 @@ mod filter; mod materialize; pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; mod remote_cache; -pub use remote_cache::{SandboxSkeleton, SandboxSlice, SpanBytes, SubmitRejection, Submitted}; +pub use remote_cache::{ + set_remote_sandbox_link, ChangeFile, RemoteSandboxLink, SandboxSkeleton, SandboxSlice, + SpanBytes, SubmitRejection, Submitted, +}; mod sandbox; mod semantic_materialize; mod split; diff --git a/atomic-repository/src/repository/record.rs b/atomic-repository/src/repository/record.rs index ffbda3f3..34ba8241 100644 --- a/atomic-repository/src/repository/record.rs +++ b/atomic-repository/src/repository/record.rs @@ -57,6 +57,10 @@ impl Repository { // Build the final header (may get message from options) let final_header = build_header(header, &options); + // A remote sandbox's cache takes the rows it is about to read. + self.hydrate_remote_sandbox() + .map_err(RecordError::Repository)?; + // Get repository status to find modified files let status_t0 = std::time::Instant::now(); let mut status_options = StatusOptions::default(); @@ -1173,6 +1177,9 @@ impl Repository { } } } + // A remote sandbox's change that didn't land isn't recorded + // anywhere: that is a failure, not a warning. + Err(e) if self.is_remote_sandbox() => return Err(RecordError::Repository(e)), Err(e) => { outcome.add_error("apply".to_string(), e.to_string()); } diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index 11214ae5..0519f99a 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -8,6 +8,7 @@ //! for what the rows are and why they are copied rather than re-derived. use std::collections::BTreeSet; +use std::path::Path; use atomic_core::change::ChangeStore as _; use atomic_core::pristine::slice::{GraphSlice, ViewSnapshotRows}; @@ -96,6 +97,52 @@ fn forbidden_path(path: &str) -> bool { }) } +/// How a remote sandbox's cache reaches its repository's owner. The +/// transport lives with whoever runs the process (the `atomic` CLI installs +/// one at startup); with it installed, `record`, `write_recorded` and the +/// readers below work in a remote sandbox for every caller. +pub trait RemoteSandboxLink: Send + Sync { + /// `FileStates`: the rows and content for `inodes`. + fn file_states(&self, root: &Path, inodes: Vec) -> Result; + /// `SubmitChange`: the change recorded against `base_state`; the view's + /// skeleton after it lands. + fn submit( + &self, + root: &Path, + base_state: String, + hash: Hash, + bytes: Vec, + ) -> Result, String>; + /// `Changes`: change files the view has. + fn changes(&self, root: &Path, hashes: Vec) -> Result, String>; + /// `PublishProvenance`: a checkpoint's provenance graph (serialized) and + /// session turn, published in the repository. + fn publish_provenance( + &self, + root: &Path, + graph: Vec, + turn: atomic_core::change::session::SessionTurn, + ) -> Result< + Result, + String, + >; +} + +static LINK: std::sync::OnceLock> = std::sync::OnceLock::new(); + +/// Install the process's link to remote sandbox owners (once). +pub fn set_remote_sandbox_link(link: Box) { + let _ = LINK.set(link); +} + +fn link() -> Result<&'static dyn RemoteSandboxLink, RepositoryError> { + LINK.get() + .map(|l| l.as_ref()) + .ok_or_else(|| RepositoryError::InvalidOperation { + message: "this process can't reach a remote sandbox's owner".to_string(), + }) +} + fn db(e: impl std::fmt::Display) -> RepositoryError { RepositoryError::Database(e.to_string()) } @@ -364,6 +411,124 @@ impl Repository { Ok(()) } + /// In a remote sandbox: load what reading or recording the working + /// tree needs from the owner (see [`Repository::sandbox_slice_inodes`]). + /// Elsewhere, nothing. + pub fn hydrate_remote_sandbox(&self) -> Result<(), RepositoryError> { + if !self.is_remote_sandbox() { + return Ok(()); + } + let inodes = self.sandbox_slice_inodes()?; + let slice = link()? + .file_states(&self.root, inodes) + .map_err(|message| RepositoryError::InvalidOperation { message })?; + self.import_sandbox_slice(&slice) + } + + /// In a remote sandbox: fetch the change files the view has and the + /// cache lacks. Returns how many. Elsewhere, nothing. + pub fn fetch_remote_sandbox_changes(&self) -> Result { + if !self.is_remote_sandbox() { + return Ok(0); + } + let missing = self.missing_sandbox_changes()?; + for batch in missing.chunks(32) { + let changes = link()? + .changes(&self.root, batch.to_vec()) + .map_err(|message| RepositoryError::InvalidOperation { message })?; + self.hold_sandbox_changes(&changes)?; + } + Ok(missing.len()) + } + + /// In a remote sandbox, publishing a checkpoint means publishing it in + /// the repository. + pub(crate) fn publish_remote_provenance_checkpoint( + &self, + graph: &atomic_core::change::ProvenanceGraph, + turn: atomic_core::change::session::SessionTurn, + ) -> Result { + let bytes = graph + .serialize() + .map_err(|e| RepositoryError::Serialization(e.to_string()))?; + link()? + .publish_provenance(&self.root, bytes, turn) + .map_err(|message| RepositoryError::InvalidOperation { message })? + .map_err(|rejection| { + RepositoryError::Apply(format!( + "the repository refused the provenance: {rejection}" + )) + }) + } + + /// In a remote sandbox, what applying a recorded change means: hand it + /// to the owner, and take the view as it is once it lands. + pub(crate) fn submit_recorded( + &self, + outcome: &crate::record::RecordOutcome, + ) -> Result { + let bytes = outcome + .v3_bytes() + .ok_or_else(|| RepositoryError::Apply("the change has no bytes".to_string()))? + .to_vec(); + let base_state = self.remote_sandbox_view_state()?; + let (submitted, skeleton) = link()? + .submit(&self.root, base_state, *outcome.hash(), bytes) + .map_err(|message| RepositoryError::InvalidOperation { message })? + .map_err(|rejection| { + RepositoryError::Apply(format!("the repository refused the change: {rejection}")) + })?; + self.import_sandbox_skeleton(&skeleton)?; + let state = + ::from_base32(submitted.state.as_bytes()) + .ok_or_else(|| RepositoryError::Apply("the owner sent a bad state".to_string()))?; + let mut stats = crate::InsertStats::new(); + stats.changes_applied = 1; + stats.applied_hashes.push(submitted.hash); + Ok(crate::InsertOutcome::new( + state, + skeleton.view.change_count, + false, + stats, + )) + } + + /// Serve side: publish a remote sandbox's checkpoint — if everything its + /// provenance explains is on `view`. The graph arrives serialized, so + /// what is published is exactly what the sandbox hashed. + pub fn publish_sandbox_provenance( + &self, + view: &str, + graph: &[u8], + turn: atomic_core::change::session::SessionTurn, + ) -> Result< + Result, + RepositoryError, + > { + use atomic_core::types::Base32; + let (graph, _) = match atomic_core::change::ProvenanceGraph::deserialize(graph) { + Ok(parsed) => parsed, + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible = super::collect_visible_change_ids(&txn, &state)?; + for change in &graph.changes_explained { + match txn.get_internal(change).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(change.to_base32()))), + } + } + } + Ok(Ok(self.publish_local_provenance_checkpoint(&graph, turn)?)) + } + /// Cache side: the view's state as the cache has it (base32) — what a /// change recorded here is recorded against. pub fn remote_sandbox_view_state(&self) -> Result { From 42e12ccff3cb6979c4ee64ef6a65885ced501a9a Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:41:28 -0500 Subject: [PATCH 12/23] A remote sandbox's cache takes its view's vault The vault arrives as files, as after a pull, so building the cache bootstraps the vault tables from them; intents written in the sandbox are then recorded and land on the view like any other change. --- .../tests/remote_sandbox_integration_test.rs | 37 ++++++++++++------- atomic-repository/src/repository/sandbox.rs | 5 +++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index 43df3323..f88e1dda 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -7,8 +7,6 @@ use std::path::Path; use std::process::{Command, Output}; -use atomic_core::change::ChangeHeader; -use atomic_repository::{InsertOptions, RecordOptions, Repository, TrackingOptions}; use serde_json::Value; use tempfile::TempDir; @@ -41,21 +39,20 @@ fn failure(output: Output, what: &str) -> String { ) } -/// A repository with two recorded files on `dev`. +/// A repository as `atomic init` makes one (vault included), with two +/// recorded files on `dev`. fn repository(root: &Path) { - let repo = Repository::init(root).unwrap(); + ok(atomic(root, &["init"]), "init"); std::fs::create_dir_all(root.join("src")).unwrap(); std::fs::write(root.join("README.md"), "hello\n").unwrap(); std::fs::write(root.join("src/lib.rs"), "pub fn f() {}\n").unwrap(); - repo.add("README.md", TrackingOptions::default()).unwrap(); - repo.add("src/lib.rs", TrackingOptions::default()).unwrap(); - let options = RecordOptions::new() - .with_all(true) - .save_to_store(true) - .apply_after_record(false); - let outcome = repo.record(ChangeHeader::new("first"), options).unwrap(); - repo.write_recorded(&outcome, InsertOptions::default()) - .unwrap(); + ok(atomic(root, &["add", "README.md", "src/lib.rs"]), "add"); + ok(atomic(root, &["record", "-a", "-m", "first"]), "record"); + // No owner is left running from setup. + let _ = atomic( + root, + &["agent", "database-owner", "shutdown", "--repository", "."], + ); } struct Owner<'a>(&'a Path); @@ -132,7 +129,7 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { // The view's tree, and nothing of the repository. let out = ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); - assert!(out.contains("Materialized 3 entries"), "{out}"); + assert!(out.contains("Materialized "), "{out}"); assert_eq!( std::fs::read_to_string(vm_dir.join("README.md")).unwrap(), "hello\n" @@ -362,6 +359,18 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { // repository — not only in the sandbox. let log = ok(atomic(host.path(), &["log"]), "host log"); assert!(log.contains("greet the reader"), "{log}"); + + // An intent written in the sandbox is recorded there and lands too. + ok( + atomic(&vm_dir, &["intent", "new", "Greet readers"]), + "intent new", + ); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "the intent"]), + "record the intent", + ); + let change = ok(atomic(host.path(), &["change"]), "host change"); + assert!(change.contains(".vault/intents/"), "{change}"); let session = ok( atomic(host.path(), &["session", "show", "agent-session"]), "host session", diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index ba7311cf..0950a039 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -159,6 +159,11 @@ impl Repository { let repo = Self::open_sandbox(working_root, &cache, &skeleton.view.name)?; repo.import_sandbox_skeleton(skeleton)?; repo.reindex_working_copy()?; + // The view's vault arrived as files, as after a pull: its tables + // come from them. + if repo.vault_dir().exists() { + repo.bootstrap_vault_from_working_copy()?; + } Ok(repo) } From 081e6eb1f290faf6ec9498543ad89002613f0601 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 15:02:06 -0500 Subject: [PATCH 13/23] Draft views: render them as themselves, and index their vault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A draft's structural changes (files it adds, moves, deletes) wait in the deferred tree journal until the view is checked out; TREE is the checked-out view's. Rendering any other view — materialize_view_entries, so Materialize and the skeleton after a submit — now projects that view's journal in a write transaction that is thrown away, and the skeleton's tree is the view as rendered (implicit directories, inode 0, left out). Before, a file added on a draft view never appeared on it. When a sandbox's change lands, the vault files it touched are indexed into the repository's vault from the view's content, as a pull does from the files it writes (vault_record_files, factored out of the working-copy deflate). An intent created on a draft view is in `intent list` again. Tests: record parity and a submitted change on a draft, a second record building on the first, and the agent flow on a draft view off dev. --- .../src/commands/agent/owner/sandbox.rs | 22 +-- .../tests/remote_sandbox_integration_test.rs | 18 +- atomic-core/src/pristine/slice.rs | 37 ++-- .../src/repository/materialize.rs | 42 +++- .../src/repository/remote_cache.rs | 48 ++++- atomic-repository/src/repository/vault.rs | 75 +++++-- .../tests/remote_cache_parity_test.rs | 187 +++++++++++++++++- 7 files changed, 359 insertions(+), 70 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index eaa8898e..6515862c 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -296,12 +296,14 @@ where let (tx, mut rx) = tokio::sync::mpsc::channel::(64); let root = owner.root.clone(); let render = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - let repo = Repository::open_readonly(&root)?; + // Writable: listing a view other than the checked-out one projects its + // tree in a transaction that is thrown away. + let repo = Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; let mut entries = 0u64; - let mut live = std::collections::BTreeSet::new(); + let mut live = std::collections::BTreeMap::new(); let rendered = repo.materialize_view_entries(&view, |entry| { entries += 1; - live.insert(entry.inode); + live.insert(entry.inode, entry.path.clone()); tx.blocking_send(OwnerResponse::MaterializeEntry { entry }) .map_err(|_| ()) })?; @@ -402,8 +404,7 @@ pub(crate) async fn record_request( Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; match repo.insert_submitted_change(&view, &base_state, &hash, &bytes)? { Ok(submitted) => { - let skeleton = - repo.export_sandbox_skeleton(&view, &live_inodes(&repo, &view)?)?; + let skeleton = repo.after_sandbox_submit(&view, &submitted.hash)?; Ok(Ok((submitted, skeleton))) } Err(rejection) => Ok(Err(rejection)), @@ -423,17 +424,6 @@ pub(crate) async fn record_request( } } -/// The inodes `view`'s tree holds. -fn live_inodes(repo: &Repository, view: &str) -> anyhow::Result> { - let mut live = std::collections::BTreeSet::new(); - repo.materialize_view_entries::<()>(view, |entry| { - live.insert(entry.inode); - Ok(()) - })? - .map_err(|()| anyhow::anyhow!("unreachable"))?; - Ok(live) -} - /// Client side of `Materialize`: write the view's tree into `dir` and return /// how many entries it had. Files already at those paths are overwritten; /// nothing else in `dir` is touched. diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index f88e1dda..55f10fa6 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -312,7 +312,8 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { "create", "exp-1", "--remote", - "--view", + // As an agent's work runs: its own draft view, off dev. + "--from", "dev", "--dest", vm_dir.to_str().unwrap(), @@ -357,8 +358,16 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { // The turn's change is on the view, and its provenance is in the // repository — not only in the sandbox. - let log = ok(atomic(host.path(), &["log"]), "host log"); + let log = ok( + atomic(host.path(), &["log", "--view", "exp-1"]), + "host log of the draft", + ); assert!(log.contains("greet the reader"), "{log}"); + let dev = ok(atomic(host.path(), &["log"]), "host log of dev"); + assert!( + !dev.contains("greet the reader"), + "only on the draft view: {dev}" + ); // An intent written in the sandbox is recorded there and lands too. ok( @@ -369,8 +378,9 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { atomic(&vm_dir, &["record", "-a", "-m", "the intent"]), "record the intent", ); - let change = ok(atomic(host.path(), &["change"]), "host change"); - assert!(change.contains(".vault/intents/"), "{change}"); + // ...and the repository's vault knows it, as it would after a pull. + let intents = ok(atomic(host.path(), &["intent", "list"]), "host intent list"); + assert!(intents.contains("backlog"), "{intents}"); let session = ok( atomic(host.path(), &["session", "show", "agent-session"]), "host session", diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs index 4a6888a7..bb33eeb5 100644 --- a/atomic-core/src/pristine/slice.rs +++ b/atomic-core/src/pristine/slice.rs @@ -7,8 +7,9 @@ //! exactly the rows the repository has. A slice is those rows, byte for byte: //! //! - the **skeleton** ([`ReadTxn::export_skeleton`]): the view's tree — -//! paths, inodes and their positions, directories — and the id tables for -//! every change the view can see. Enough for `status` with no graph at all. +//! paths (as the view renders them), inodes and their positions, +//! directories — and the id tables for every change the view can see. +//! Enough for `status` with no graph at all. //! - a **graph slice** ([`ReadTxn::export_graph_slice`]): for a set of //! inodes, the GRAPH / INODE_GRAPH rows of each file's content and of its //! name chain up to the root, one hop of neighbours (so `find_block` finds @@ -139,14 +140,15 @@ enum Reach { } impl ReadTxn { - /// The view's tree and ids: TREE for the `live` inodes (the paths the - /// view actually has), REV_TREE / INODES / DIRECTORIES for every inode - /// whose position a visible change introduced, and the id rows for every - /// visible change. + /// The view's tree and ids. `live` is the view's tree as rendered — + /// every path it has and the inode there, which is what TREE would hold + /// with the view checked out (TREE itself is the checked-out view's). + /// REV_TREE / INODES / DIRECTORIES come for every inode whose position a + /// visible change introduced, and the id rows for every visible change. pub fn export_skeleton( &self, visible: &BTreeSet, - live: &BTreeSet, + live: &BTreeMap, ) -> PristineResult { let mut out = GraphSlice::default(); let inodes = self.txn.open_table(INODES)?; @@ -159,20 +161,23 @@ impl ReadTxn { out.inodes.push((k.value(), *v.value())); } } - let rev_tree = self.txn.open_table(REV_TREE)?; - for row in rev_tree.iter()? { + let mut rev_tree: BTreeMap = BTreeMap::new(); + for row in self.txn.open_table(REV_TREE)?.iter()? { let (k, v) = row?; if kept.contains(&k.value()) { - out.rev_tree.push((k.value(), v.value().to_string())); + rev_tree.insert(k.value(), v.value().to_string()); } } - let tree = self.txn.open_table(TREE)?; - for row in tree.iter()? { - let (k, v) = row?; - if live.contains(&v.value()) && kept.contains(&v.value()) { - out.tree.push((k.value().to_string(), v.value())); - } + // Implicit directories render with no inode of their own (0): only + // what the repository tracks goes in the tree. + for (inode, path) in live + .iter() + .filter(|(inode, _)| **inode != 0 && kept.contains(inode)) + { + rev_tree.insert(*inode, path.clone()); + out.tree.push((path.clone(), *inode)); } + out.rev_tree = rev_tree.into_iter().collect(); let directories = self.txn.open_table(DIRECTORIES)?; for row in directories.iter()? { let (k, v) = row?; diff --git a/atomic-repository/src/repository/materialize.rs b/atomic-repository/src/repository/materialize.rs index 24a2f81b..7d707721 100644 --- a/atomic-repository/src/repository/materialize.rs +++ b/atomic-repository/src/repository/materialize.rs @@ -1083,6 +1083,35 @@ mod serde_bytes_vec { } impl Repository { + /// The entries `view_name` has, from TREE as that view projects it. + /// + /// TREE is the checked-out view's projection; another view's structural + /// operations (files it added, moved or deleted) wait in the deferred + /// tree journal until a switch applies them. For another view they are + /// applied here in a write transaction that is thrown away, so the + /// listing is that view's and nothing on disk changes. A read-only handle + /// can't do that and lists TREE as it is. + fn view_tree_items( + &self, + txn: &atomic_core::pristine::ReadTxn, + view_name: &str, + options: &atomic_core::output::repo::MaterializeOptions, + ) -> Result, RepositoryError> { + use atomic_core::output::repo::collect_children; + let db = |e: atomic_core::pristine::PristineError| RepositoryError::Database(e.to_string()); + if view_name != self.current_view { + let journal = self.load_deferred_tree_journal()?; + if let Ok(mut projected) = self.pristine.write_txn() { + self.apply_deferred_tree_ops_in_txn(&mut projected, &journal, view_name)?; + let items = collect_children(&projected, Inode::ROOT, "", options).map_err(db)?; + use atomic_core::pristine::MutTxnT; + projected.abort().map_err(db)?; + return Ok(items); + } + } + collect_children(txn, Inode::ROOT, "", options).map_err(db) + } + /// Render every entry of `view` and hand each to `sink`, in path order /// (a directory before what it contains), touching nothing: no working /// tree, no stat cache, no conflict state. Read-only, so it runs beside @@ -1096,7 +1125,7 @@ impl Repository { view_name: &str, mut sink: impl FnMut(ViewEntry) -> Result<(), E>, ) -> Result, RepositoryError> { - use atomic_core::output::repo::{collect_children, MaterializeOptions}; + use atomic_core::output::repo::MaterializeOptions; use atomic_core::types::Base32; use rayon::prelude::*; @@ -1114,8 +1143,7 @@ impl Repository { })?; let change_filter_arc = Arc::new(collect_visible_change_ids(&txn, &view)?); let options = MaterializeOptions::new().with_change_filter_arc(change_filter_arc.clone()); - let mut items = collect_children(&txn, Inode::ROOT, "", &options) - .map_err(|e| RepositoryError::Database(e.to_string()))?; + let mut items = self.view_tree_items(&txn, view_name, &options)?; items.sort_by(|a, b| a.path.cmp(&b.path)); let visible = |item: &atomic_core::output::repo::OutputItem| { @@ -1249,7 +1277,7 @@ fn render_view_file( // in a single range scan, then run retrieve_graph over the // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) // individual B-tree probes. - let preloaded = InodePreloadTxn::from_table(&txn, item.inode, &inode_graph_table) + let preloaded = InodePreloadTxn::from_table(txn, item.inode, inode_graph_table) .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; let t_retrieve = std::time::Instant::now(); @@ -1294,10 +1322,10 @@ fn render_view_file( // surfaced instead of silently collapsed (rubric A12). let content = match name_conflicts.get(&item.path) { Some(sides) => render_name_conflict( - &txn, + txn, store, - &inode_graph_table, - &change_filter_arc, + inode_graph_table, + change_filter_arc, &item.path, sides, ) diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index 0519f99a..f6f1594b 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -162,12 +162,12 @@ impl Repository { Ok(ids) } - /// Serve side: `view`'s skeleton, for the inodes `live` (the ones its - /// materialized tree holds). + /// Serve side: `view`'s skeleton, with `live` its rendered tree (inode → + /// path, as [`Repository::materialize_view_entries`] gives them). pub fn export_sandbox_skeleton( &self, view: &str, - live: &BTreeSet, + live: &std::collections::BTreeMap, ) -> Result { let txn = self.pristine.read_txn().map_err(db)?; let state = @@ -493,6 +493,48 @@ impl Repository { )) } + /// Serve side, once a sandbox's change `hash` has landed on `view`: the + /// view's skeleton for the sandbox's cache, and — as a pull does for the + /// files it writes — the vault files the change touched, indexed into + /// the repository's (repository-wide) vault. One render of the view + /// serves both. + pub fn after_sandbox_submit( + &self, + view: &str, + hash: &Hash, + ) -> Result { + let change = self.load_change(hash)?; + let vault_paths: BTreeSet = change + .hunks() + .iter() + .filter_map(|h| h.path()) + .chain(change.file_ops().iter().map(|ops| ops.path())) + .filter(|p| p.starts_with(".vault/") && p.ends_with(".md")) + .map(str::to_string) + .collect(); + let mut live = std::collections::BTreeMap::new(); + let mut files = std::collections::BTreeMap::new(); + self.materialize_view_entries::<()>(view, |entry| { + live.insert(entry.inode, entry.path.clone()); + if vault_paths.contains(&entry.path) { + files.insert( + entry.path, + String::from_utf8_lossy(&entry.content).into_owned(), + ); + } + Ok(()) + })? + .map_err(|()| RepositoryError::Output("unreachable".to_string()))?; + if !vault_paths.is_empty() && self.has_vault()? { + let rows: Vec<(String, Option)> = vault_paths + .iter() + .map(|p| (p[".vault/".len()..].to_string(), files.remove(p))) + .collect(); + self.vault_record_files(&rows)?; + } + self.export_sandbox_skeleton(view, &live) + } + /// Serve side: publish a remote sandbox's checkpoint — if everything its /// provenance explains is on `view`. The graph arrives serialized, so /// what is published is exactly what the sandbox hashed. diff --git a/atomic-repository/src/repository/vault.rs b/atomic-repository/src/repository/vault.rs index a14ca237..a88c43dc 100644 --- a/atomic-repository/src/repository/vault.rs +++ b/atomic-repository/src/repository/vault.rs @@ -594,9 +594,38 @@ impl Repository { // Read file content let file_content = std::fs::read_to_string(path)?; + if let Some(change) = self.vault_file_change(rel_path_str, &file_content)? { + changes.push(change); + } + } + + // Also check for deleted entries (in redb but not on disk) + let all_entries = self.vault_list("", None)?; + for meta in all_entries { + let disk_path = vault_dir.join(&meta.path); + if !disk_path.exists() { + changes.push(VaultFileChange { + path: meta.path, + change_type: VaultChangeType::Deleted, + frontmatter_json: String::new(), + content: Vec::new(), + }); + } + } + + Ok(changes) + } + /// How one vault file (`rel_path`, vault-relative, with `file_content`) + /// differs from its stored entry — `None` when it doesn't. + fn vault_file_change( + &self, + rel_path_str: String, + file_content: &str, + ) -> Result, RepositoryError> { + { // Parse into frontmatter + body - let (frontmatter_json, body) = parse_markdown_frontmatter(&file_content); + let (frontmatter_json, body) = parse_markdown_frontmatter(file_content); // Body hashes remain useful for content/embedding caches, but // frontmatter is also knowledge: status, labels, identity, and @@ -611,35 +640,43 @@ impl Repository { if Hash::from_bytes(entry.content_hash) == new_hash && frontmatter_json_equal(&entry.frontmatter_json, &frontmatter_json) => { - continue; // Unchanged, skip + return Ok(None); // Unchanged, skip } Some(_) => VaultChangeType::Modified, None => VaultChangeType::New, }; - changes.push(VaultFileChange { + Ok(Some(VaultFileChange { path: rel_path_str, change_type, frontmatter_json, content: body_bytes.to_vec(), - }); + })) } + } - // Also check for deleted entries (in redb but not on disk) - let all_entries = self.vault_list("", None)?; - for meta in all_entries { - let disk_path = vault_dir.join(&meta.path); - if !disk_path.exists() { - changes.push(VaultFileChange { - path: meta.path, + /// Deflate vault files that are not in this working copy — a view's, + /// say — into redb: each `(vault-relative path, content)`, `None` for a + /// file the view no longer has. What `vault_record_working_copy` does + /// for the files on disk. + pub fn vault_record_files( + &self, + files: &[(String, Option)], + ) -> Result, RepositoryError> { + let mut changes = Vec::new(); + for (path, content) in files { + match content { + Some(content) => changes.extend(self.vault_file_change(path.clone(), content)?), + None if self.vault_retrieve(path)?.is_some() => changes.push(VaultFileChange { + path: path.clone(), change_type: VaultChangeType::Deleted, frontmatter_json: String::new(), content: Vec::new(), - }); + }), + None => {} } } - - Ok(changes) + self.vault_apply_changes(&changes) } /// Deflate changed vault files from disk into redb. @@ -653,10 +690,18 @@ impl Repository { /// - `vault_record_working_copy`: disk -> redb (after humans edit) pub fn vault_record_working_copy(&self) -> Result, RepositoryError> { let changes = self.vault_scan_working_copy()?; + self.vault_apply_changes(&changes) + } + /// Store detected vault changes, then index any intents the manifest + /// is missing. + fn vault_apply_changes( + &self, + changes: &[VaultFileChange], + ) -> Result, RepositoryError> { let mut updated_paths = Vec::new(); - for change in &changes { + for change in changes { match change.change_type { VaultChangeType::New | VaultChangeType::Modified => { // Infer entry type from path diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs index 8198c4bf..38a51cb3 100644 --- a/atomic-repository/tests/remote_cache_parity_test.rs +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -6,7 +6,7 @@ //! produces — which carries the repository's internal ids — must be byte for //! byte the change `record` produces on the repository itself. -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::fs; use std::path::Path; @@ -74,7 +74,7 @@ impl Pair { /// Ship the slice for what `record` will touch, as FileStates does: /// only the inodes the cache asks for. - fn hydrate(&self, view: &str, live: &BTreeSet) { + fn hydrate(&self, view: &str, live: &BTreeMap) { let inodes = self.cache.sandbox_slice_inodes().unwrap(); assert!( inodes.len() < live.len(), @@ -86,7 +86,7 @@ impl Pair { } /// A host repository with some history, and a cache materialized from it. -fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeSet) { +fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeMap) { let host_dir = TempDir::new().unwrap(); let cache_dir = TempDir::new().unwrap(); let host = Repository::init(host_dir.path()).unwrap(); @@ -96,9 +96,9 @@ fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeSet) { // Materialize: the view's files into the cache's working tree, and the // inodes they are. let cache = Repository::init(cache_dir.path()).unwrap(); - let mut live = BTreeSet::new(); + let mut live = BTreeMap::new(); host.materialize_view_entries::<()>(&view, |entry| { - live.insert(entry.inode); + live.insert(entry.inode, entry.path.clone()); let path = cache_dir.path().join(&entry.path); match entry.kind { atomic_repository::ViewEntryKind::Directory => fs::create_dir_all(path).unwrap(), @@ -126,7 +126,7 @@ fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeSet) { ) } -fn assert_parity(pair: &Pair, view: &str, live: &BTreeSet, what: &str) { +fn assert_parity(pair: &Pair, view: &str, live: &BTreeMap, what: &str) { pair.hydrate(view, live); let h = header(what); let host = change_bytes(&pair.host, h.clone()); @@ -257,7 +257,7 @@ use atomic_repository::SubmitRejection; fn view_state(repo: &Repository, view: &str) -> String { let skeleton = repo - .export_sandbox_skeleton(view, &BTreeSet::new()) + .export_sandbox_skeleton(view, &BTreeMap::new()) .unwrap(); atomic_core::types::Base32::to_base32(&skeleton.view.state) } @@ -266,7 +266,7 @@ fn view_state(repo: &Repository, view: &str) -> String { fn record_in_cache( pair: &Pair, view: &str, - live: &BTreeSet, + live: &BTreeMap, what: &str, ) -> (atomic_core::types::Hash, Vec) { pair.hydrate(view, live); @@ -336,7 +336,7 @@ fn a_submitted_change_lands_on_the_view_and_the_next_one_builds_on_it() { live.clear(); pair.host .materialize_view_entries::<()>(&view, |e| { - live.insert(e.inode); + live.insert(e.inode, e.path.clone()); Ok(()) }) .unwrap() @@ -438,3 +438,172 @@ fn a_change_from_another_view_is_refused() { ); assert_eq!(view_state(&host, "dev"), base); } + +#[test] +fn a_change_submitted_to_a_draft_view_is_on_that_view() { + // Agent work lands on draft views: what lands on one renders there. + let (pair, _, _) = pair(two_files); + let mut host = pair.host; + host.create_view_from("exp", "dev").unwrap(); + let cache_dir = TempDir::new().unwrap(); + drop(Repository::init(cache_dir.path()).unwrap()); + // As a remote sandbox's cache is opened: on its pointer's view. + let cache = Repository::open_sandbox(cache_dir.path(), cache_dir.path(), "exp").unwrap(); + let mut live = BTreeMap::new(); + host.materialize_view_entries::<()>("exp", |entry| { + live.insert(entry.inode, entry.path.clone()); + let path = cache_dir.path().join(&entry.path); + match entry.kind { + atomic_repository::ViewEntryKind::Directory => fs::create_dir_all(path).unwrap(), + _ => { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + } + Ok(()) + }) + .unwrap() + .unwrap(); + cache + .import_sandbox_skeleton(&host.export_sandbox_skeleton("exp", &live).unwrap()) + .unwrap(); + cache.reindex_working_copy().unwrap(); + write(cache_dir.path(), "src/new.rs", "pub fn n() {}\n"); + write(cache_dir.path(), "README.md", "hello\non the draft\n"); + cache.add("src/new.rs", TrackingOptions::default()).unwrap(); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = cache.record(header("on the draft"), options).unwrap(); + let base = view_state(&host, "exp"); + host.insert_submitted_change("exp", &base, outcome.hash(), outcome.v3_bytes().unwrap()) + .unwrap() + .expect("accepted"); + + let tree = tree_of(&host, "exp"); + assert!( + tree.contains(&("README.md".into(), b"hello\non the draft\n".to_vec())), + "{tree:?}" + ); + assert!( + tree.contains(&("src/new.rs".into(), b"pub fn n() {}\n".to_vec())), + "{tree:?}" + ); + assert!( + !tree_of(&host, "dev").iter().any(|(p, _)| p == "src/new.rs"), + "only on the draft" + ); + + // The next record builds on it: the refreshed skeleton knows the new file. + let skeleton = host.after_sandbox_submit("exp", outcome.hash()).unwrap(); + cache.import_sandbox_skeleton(&skeleton).unwrap(); + cache.reindex_working_copy().unwrap(); + write(cache_dir.path(), "src/new.rs", "pub fn n() { 2 }\n"); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let second = cache.record(header("again on the draft"), options).unwrap(); + let (change, _) = + atomic_core::change::Change::deserialize(&mut &second.v3_bytes().unwrap()[..]).unwrap(); + assert!( + !change + .hunks() + .iter() + .any(|h| matches!(h, atomic_core::change::GraphOp::FileAdd { .. })), + "an edit, not a second add" + ); + host.insert_submitted_change( + "exp", + &view_state(&host, "exp"), + second.hash(), + second.v3_bytes().unwrap(), + ) + .unwrap() + .expect("accepted"); + assert!(tree_of(&host, "exp").contains(&("src/new.rs".into(), b"pub fn n() { 2 }\n".to_vec()))); +} + +#[test] +fn control_a_local_sandbox_record_on_a_draft_renders_on_it() { + let host_dir = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("exp", "dev").unwrap(); + host.provision_sandbox(work.path().join("w"), "exp") + .unwrap(); + } + let sbx = Repository::open_existing(work.path().join("w")).unwrap(); + write(sbx.root(), "README.md", "hello\non the draft\n"); + record_and_apply(&sbx, "on the draft"); + drop(sbx); + let host = Repository::open_existing(host_dir.path()).unwrap(); + let tree = tree_of(&host, "exp"); + assert!( + tree.contains(&("README.md".into(), b"hello\non the draft\n".to_vec())), + "{tree:?}" + ); +} + +#[test] +fn parity_on_a_draft_view() { + let host_dir = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("exp", "dev").unwrap(); + host.provision_sandbox(work.path().join("w"), "exp") + .unwrap(); + } + // The cache, from the draft. + let cache_dir = TempDir::new().unwrap(); + drop(Repository::init(cache_dir.path()).unwrap()); + let cache = Repository::open_sandbox(cache_dir.path(), cache_dir.path(), "exp").unwrap(); + let skeleton = { + let host = Repository::open_existing(host_dir.path()).unwrap(); + let mut live = BTreeMap::new(); + host.materialize_view_entries::<()>("exp", |entry| { + live.insert(entry.inode, entry.path.clone()); + let path = cache_dir.path().join(&entry.path); + if entry.kind != atomic_repository::ViewEntryKind::Directory { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + Ok(()) + }) + .unwrap() + .unwrap(); + host.export_sandbox_skeleton("exp", &live).unwrap() + }; + cache.import_sandbox_skeleton(&skeleton).unwrap(); + cache.reindex_working_copy().unwrap(); + + let sbx = Repository::open_existing(work.path().join("w")).unwrap(); + write(sbx.root(), "README.md", "hello\non the draft\n"); + write(cache_dir.path(), "README.md", "hello\non the draft\n"); + let h = header("on the draft"); + let local = change_bytes(&sbx, h.clone()); + drop(sbx); + let host = Repository::open_existing(host_dir.path()).unwrap(); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let remote = change_bytes(&cache, h); + assert_eq!( + local.0, remote.0, + "the cache records a different change on a draft" + ); +} From 76aef1d51532ebc6e6c28b3512c91af6fdbbae37 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 21:42:49 -0500 Subject: [PATCH 14/23] Stage and seal a view as it renders materialize_view_to (behind `sandbox stage` and `seal`) listed files from TREE, which is the checked-out view's: a file another view added was missing from its image. It now writes materialize_view_entries, which projects the view's own tree. --- atomic-repository/src/repository/sandbox.rs | 18 ++++++++++-------- .../tests/remote_cache_parity_test.rs | 9 +++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index 0950a039..123295da 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -279,19 +279,21 @@ impl Repository { pub fn materialize_view_to(&self, view: &str, dir: &Path) -> Result { std::fs::create_dir_all(dir)?; + // The view as it renders — its own added and moved files included, + // whichever view is checked out. let mut count = 0usize; - for path in self.visible_file_paths(view)? { - let bytes = match self.get_file_content_on_view(&path, view)? { - Some(bytes) => bytes, - None => continue, - }; - let target = dir.join(&path); + self.materialize_view_entries(view, |entry| -> Result<(), std::io::Error> { + if entry.kind == super::ViewEntryKind::Directory { + return Ok(()); + } + let target = dir.join(&entry.path); if let Some(parent) = target.parent() { std::fs::create_dir_all(parent)?; } - std::fs::write(&target, &bytes)?; + std::fs::write(&target, &entry.content)?; count += 1; - } + Ok(()) + })??; Ok(count) } diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs index 38a51cb3..c90f8a29 100644 --- a/atomic-repository/tests/remote_cache_parity_test.rs +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -498,6 +498,15 @@ fn a_change_submitted_to_a_draft_view_is_on_that_view() { !tree_of(&host, "dev").iter().any(|(p, _)| p == "src/new.rs"), "only on the draft" ); + // What `sandbox stage` / `seal` write sees it too. + let out = TempDir::new().unwrap(); + host.materialize_view_to("exp", out.path()).unwrap(); + assert_eq!( + fs::read_to_string(out.path().join("src/new.rs")) + .ok() + .as_deref(), + Some("pub fn n() {}\n") + ); // The next record builds on it: the refreshed skeleton knows the new file. let skeleton = host.after_sandbox_submit("exp", outcome.hash()).unwrap(); From 3d20044e8e4fb159b7a8e303dc8b835ff79d7251 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Fri, 25 Sep 2026 00:48:29 -0500 Subject: [PATCH 15/23] Let the owner's iroh endpoints bind IPv4 only ATOMIC_IROH_IPV4_ONLY=1 makes the owner endpoint and the client dialer open no IPv6 socket. Under some microVM network backends (smolvm TSI, in a fork) creating one takes the process down, so a sandbox inside such a VM could not materialize or submit. --- atomic-cli/src/commands/agent/owner/remote.rs | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/remote.rs b/atomic-cli/src/commands/agent/owner/remote.rs index 94fbbb93..b4f783f7 100644 --- a/atomic-cli/src/commands/agent/owner/remote.rs +++ b/atomic-cli/src/commands/agent/owner/remote.rs @@ -205,12 +205,7 @@ pub(crate) async fn bind(dot_dir: &Path, offline: bool) -> anyhow::Result return Err(e).with_context(|| format!("failed to read {}", key_path.display())), }; - let builder = if offline { - Endpoint::builder(presets::Minimal) - } else { - Endpoint::builder(presets::N0) - }; - Ok(builder + Ok(builder(offline)? .secret_key(key) .alpns(vec![ALPN.to_vec()]) .bind() @@ -244,12 +239,30 @@ pub(crate) async fn pointer_addr(endpoint: &Endpoint, offline: bool) -> Endpoint /// A client endpoint for dialing an owner. async fn dialer(offline: bool) -> anyhow::Result { + Ok(builder(offline)?.bind().await?) +} + +/// An endpoint builder: the relay-less preset when offline, and IPv4 only +/// when `ATOMIC_IROH_IPV4_ONLY` is set — for hosts where creating an IPv6 +/// socket misbehaves (it takes down the process under some microVM network +/// backends), so the endpoint never opens one. +fn builder(offline: bool) -> anyhow::Result { let builder = if offline { Endpoint::builder(presets::Minimal) } else { Endpoint::builder(presets::N0) }; - Ok(builder.bind().await?) + if !ipv4_only() { + return Ok(builder); + } + builder + .clear_ip_transports() + .bind_addr("0.0.0.0:0") + .map_err(|e| anyhow!("invalid bind address: {e}")) +} + +fn ipv4_only() -> bool { + std::env::var("ATOMIC_IROH_IPV4_ONLY").is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true")) } fn write_private(path: &Path, bytes: &[u8]) -> anyhow::Result<()> { From 3e7ca25003902d6a9d722041124fb0d52ac32f4c Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Sun, 27 Sep 2026 19:47:18 -0500 Subject: [PATCH 16/23] fix(remote-sandbox): five ways a sandbox could be wrong, wrong again, or the owner's fault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A review of the remote-sandbox work turned up five defects. Three let a remote agent do something it should not; two left a sandbox permanently unable to work. The CRDT id decode in export_graph_slice read the change id big-endian, but the CRDT tables write it little-endian — the graph keys beside it are big-endian because they are range-scanned, and the two rules sit within a few lines of each other. id_rows drops ids it cannot resolve, so the slice exported fine and just quietly carried no id rows for the changes that created a file's CRDT entries. Decode through each type's own from_bytes now, and id_rows asserts in a debug build, because a mis-decoded id is indistinguishable from an absent one at runtime. The BE/LE rule is written down at the encoders and in AGENTS.md. session_is_fresh walked turns 1..=n, one read transaction per step, with n straight off the wire — a single request could pin a runtime worker for hours. A session's turns are one contiguous, turn-ordered run of keys, so the last key in the range answers the same question. Both callers wanted "has anyone claimed this id?" rather than "is this number free", which the walk only approximated: a session with history above the asked turn read as unused, and that is the wrong way round for a check that decides who may write to it. A materialized entry's path was joined onto the sandbox root with no check, so a path that climbs out, or names the pointer or the cache, let an agent rewrite its own trust anchor or write as the user somewhere it was never given. The owner's forbidden_path was a reserved-name list and could not catch `..` at all, though a submitted change lands in TREE and the next materialize writes root.join(path). view_tree_items fell through to TREE — the *current* view's — when it could not open a write transaction to project another view's, so a read-only repository answered a question about one view with another's tree. For a remote sandbox that is a tree written to disk. And a rejected submit was a dead end. Only a successful submit refreshed the cache's view row, so a StaleView refusal left a sandbox that could never land anything again, and the recovery its own error named did not exist. The owner now sends the view as it is with the refusal, and the cache takes it, so a retry is computed against the graph the refusal was about. Separately, `--ttl` panicked inside the token registry while holding its lock, and every other method took that lock with unwrap: one absurd value poisoned the mutex and took every sandbox plus the local create/close down with it, until the owner was restarted. The deadline is now checked arithmetic computed before the lock is taken, and a poisoned lock is recovered rather than propagated. A non-positive TTL is refused instead of quietly minting a token that is already dead. Full workspace suite: 57 binaries, 9088 tests, 0 failures. Each fix has a test that fails without it. --- AGENTS.md | 31 +++ atomic-cli/src/commands/agent/owner/mod.rs | 33 ++- atomic-cli/src/commands/agent/owner/remote.rs | 180 +++++++++++--- .../src/commands/agent/owner/sandbox.rs | 235 +++++++++++++----- .../tests/remote_sandbox_integration_test.rs | 144 +++++++++++ atomic-core/src/pristine/slice.rs | 52 +++- atomic-core/src/pristine/tables.rs | 15 ++ .../src/redb_change_store/provenance.rs | 29 ++- .../src/redb_change_store/tests.rs | 43 ++++ .../src/repository/materialize.rs | 31 ++- atomic-repository/src/repository/mod.rs | 2 +- .../src/repository/remote_cache.rs | 141 ++++++++++- .../tests/materialize_view_entries_test.rs | 64 +++++ 13 files changed, 868 insertions(+), 132 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 473a00de..516162c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -340,6 +340,37 @@ Human: "I can review that!" | `EdgeFlags` | 1 byte | Bitflags: BLOCK, PSEUDO, FOLDER, PARENT, DELETED | | `SerializedGraphEdge` | 24 bytes | Compact edge: (flags+pos, change, introduced_by) | +#### Endianness: keys that sort are big-endian, opaque values are little-endian + +The storage layer encodes integers two ways, and the rule is whether the +bytes ever get sorted: + +| Encoded as | Endian | Which | +|------------|--------|-------| +| **Big-endian** | BE | B-tree **keys** that are range-scanned: `encode_vertex`, `encode_inode_vertex`, `encode_position`, `encode_view_seq` (`pristine/tables.rs`) | +| **Little-endian** | LE | Values and ids only ever looked up by exact key: `SerializedGraphEdge`, and the CRDT `TrunkId`/`BranchId`/`LeafId` (`crdt/tables.rs`, `crdt/ids.rs`) | + +BE is load-bearing for the keys: byte order has to match numeric order or a +range scan means nothing. `export_graph_slice` depends on it — + +```rust +let lo = encode_inode_vertex(inode, 0, 0, 0); +let hi = encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX); +for row in inode_graph.range::<&[u8; 32]>(&lo..=&hi)? { ... } // every row for one inode +``` + +For the LE side, sorting never happens, so byte order costs nothing and there +is no reason to pay for BE. + +**The two meet in the same files**, so match the *writer*, never the +neighbours. `export_graph_slice` decodes graph vertex keys (BE) and CRDT ids +(LE) within a few lines of each other, which is exactly how a BE decode of a +LE id shipped unnoticed. Decode an id with its own type's `from_bytes` +(`TrunkId::from_bytes`, `BranchId::from_bytes`, `LeafId::from_bytes`) instead +of slicing `id[0..8]` by hand, so the two sides cannot drift. `id_rows` +`debug_assert`s on an id with no `EXTERNAL` row, because a mis-decoded id is +indistinguishable from an absent one at runtime. + ### Hash Type Design Following the original Atomic project, `Hash` is a **type alias** for `Merkle`: diff --git a/atomic-cli/src/commands/agent/owner/mod.rs b/atomic-cli/src/commands/agent/owner/mod.rs index 4ebc670f..8f11b37f 100644 --- a/atomic-cli/src/commands/agent/owner/mod.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -344,6 +344,13 @@ pub(crate) enum OwnerResponse { }, ChangeRefused { rejection: atomic_repository::SubmitRejection, + /// The view's rows now, when the refusal is about the view having moved + /// and a resync is what the sandbox needs to recover. Carried with the + /// refusal so recovering costs the round trip already being made: a + /// sandbox that has to ask again for a state it was just told is stale + /// can be left unable to record at all. + #[serde(default, skip_serializing_if = "Option::is_none")] + skeleton: Option>, }, Changes { changes: Vec, @@ -1163,16 +1170,7 @@ impl atomic_repository::RemoteSandboxLink for OwnerLink { base_state: String, hash: Hash, bytes: Vec, - ) -> Result< - Result< - ( - atomic_repository::Submitted, - atomic_repository::SandboxSkeleton, - ), - atomic_repository::SubmitRejection, - >, - String, - > { + ) -> Result { let body = OwnerRequest::SubmitChange { view: None, base_state, @@ -1184,7 +1182,10 @@ impl atomic_repository::RemoteSandboxLink for OwnerLink { submitted, skeleton, } => Ok(Ok((submitted, *skeleton))), - OwnerResponse::ChangeRefused { rejection } => Ok(Err(rejection)), + OwnerResponse::ChangeRefused { + rejection, + skeleton, + } => Ok(Err((rejection, skeleton.map(|s| *s)))), other => Err(unexpected_response("submit change", other).to_string()), } } @@ -1198,7 +1199,10 @@ impl atomic_repository::RemoteSandboxLink for OwnerLink { OwnerResponse::Changes { changes } => { Ok(changes.into_iter().map(|c| (c.hash, c.bytes)).collect()) } - OwnerResponse::ChangeRefused { rejection } => Err(rejection.to_string()), + OwnerResponse::ChangeRefused { + rejection, + skeleton: _, + } => Err(rejection.to_string()), other => Err(unexpected_response("changes", other).to_string()), } } @@ -1222,7 +1226,10 @@ impl atomic_repository::RemoteSandboxLink for OwnerLink { }; match link_request(root, body)? { OwnerResponse::ProvenancePublished { publication } => Ok(Ok(publication)), - OwnerResponse::ChangeRefused { rejection } => Ok(Err(rejection)), + OwnerResponse::ChangeRefused { + rejection, + skeleton: _, + } => Ok(Err(rejection)), other => Err(unexpected_response("publish provenance", other).to_string()), } } diff --git a/atomic-cli/src/commands/agent/owner/remote.rs b/atomic-cli/src/commands/agent/owner/remote.rs index b4f783f7..34b69854 100644 --- a/atomic-cli/src/commands/agent/owner/remote.rs +++ b/atomic-cli/src/commands/agent/owner/remote.rs @@ -45,6 +45,32 @@ pub(crate) enum TokenError { Expired(DateTime), #[error("no live token for view '{0}'")] NoTokenForView(String), + #[error("a sandbox token must last between 1 second and {MAX_TTL_SECS} seconds")] + InvalidTtl, +} + +/// Bounds on a requested token lifetime, in seconds. Both ends matter: a +/// non-positive TTL would mint a token that is already dead, and an enormous +/// one is either a mistake or a way to make a token that outlives the owner. +/// The lifetime is validated and turned into a deadline *before* the registry +/// lock is taken, so a bad value can never be the thing that panics inside it. +const MIN_TTL_SECS: i64 = 1; +const MAX_TTL_SECS: i64 = 365 * 24 * 60 * 60; + +/// The deadline a TTL of `secs` seconds from now means, or [`TokenError::InvalidTtl`]. +/// +/// `Duration::seconds` panics outside its range and `DateTime + Duration` +/// panics on overflow, and both are reachable from a flag and then a wire +/// frame. Everything here is checked arithmetic instead, so a request this +/// rejects is a request that returns an error. +fn deadline(secs: i64) -> Result, TokenError> { + if !(MIN_TTL_SECS..=MAX_TTL_SECS).contains(&secs) { + return Err(TokenError::InvalidTtl); + } + let ttl = Duration::try_seconds(secs).ok_or(TokenError::InvalidTtl)?; + Utc::now() + .checked_add_signed(ttl) + .ok_or(TokenError::InvalidTtl) } #[derive(Default)] @@ -62,49 +88,68 @@ pub(crate) struct TokenRegistry { inner: Arc>, } +impl TokenRegistry { + /// The registry's lock. + /// + /// A panic while holding it poisons the mutex, and every method would then + /// panic on entry — one bad request taking down every sandbox plus the + /// local `create`/`close` until the owner restarts. The registry's + /// invariants do not depend on unwinding, so a poisoned lock is recovered + /// rather than propagated. + fn lock(&self) -> std::sync::MutexGuard<'_, Tokens> { + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } +} + fn digest(token: &str) -> String { blake3::hash(token.as_bytes()).to_hex().to_string() } impl TokenRegistry { - /// Mint a token for `view`, valid for `ttl`. Minting again replaces the - /// view's token. + /// Mint a token for `view`, valid for `ttl_secs` seconds. Minting again + /// replaces the view's token. + /// + /// The lifetime is turned into a deadline before the lock is taken, so a + /// bad `--ttl` is an error rather than a panic inside the registry. pub(crate) fn mint( &self, view: &str, acting_as: Option, - ttl: Duration, - ) -> (String, Grant) { + ttl_secs: i64, + ) -> Result<(String, Grant), TokenError> { + let expires = deadline(ttl_secs)?; let mut bytes = [0u8; 32]; rand::thread_rng().fill_bytes(&mut bytes); let token = format!("ast_{}", data_encoding::BASE64URL_NOPAD.encode(&bytes)); let grant = Grant { view: view.to_string(), acting_as, - expires: Utc::now() + ttl, + expires, }; - let mut t = self.inner.lock().unwrap(); + let mut t = self.lock(); t.grants.retain(|_, g| g.view != view); t.grants.insert(digest(&token), grant.clone()); - (token, grant) + Ok((token, grant)) } - /// Extend `view`'s live token to now + `ttl`; the same token keeps working. - pub(crate) fn renew(&self, view: &str, ttl: Duration) -> Result, TokenError> { - let mut t = self.inner.lock().unwrap(); + /// Extend `view`'s live token to now + `ttl_secs` seconds; the same token + /// keeps working. + pub(crate) fn renew(&self, view: &str, ttl_secs: i64) -> Result, TokenError> { + let expires = deadline(ttl_secs)?; + let mut t = self.lock(); let now = Utc::now(); let grant = t .grants .values_mut() .find(|g| g.view == view && g.expires > now) .ok_or_else(|| TokenError::NoTokenForView(view.to_string()))?; - grant.expires = now + ttl; + grant.expires = expires; Ok(grant.expires) } /// End `view`'s token now. pub(crate) fn revoke(&self, view: &str) -> bool { - let mut t = self.inner.lock().unwrap(); + let mut t = self.lock(); let before = t.grants.len(); t.grants.retain(|_, g| g.view != view); t.grants.len() != before @@ -112,7 +157,7 @@ impl TokenRegistry { /// What `token` grants, if it is live. pub(crate) fn check(&self, token: &str) -> Result { - let t = self.inner.lock().unwrap(); + let t = self.lock(); let grant = t.grants.get(&digest(token)).ok_or(TokenError::Unknown)?; if grant.expires <= Utc::now() { return Err(TokenError::Expired(grant.expires)); @@ -123,7 +168,7 @@ impl TokenRegistry { /// Whether `view`'s sandbox may use provenance session `session_id`: /// one it started, or one nobody has (`fresh`), which becomes its own. pub(crate) fn claim_session(&self, view: &str, session_id: &str, fresh: bool) -> bool { - let mut t = self.inner.lock().unwrap(); + let mut t = self.lock(); match t.sessions.get(session_id) { Some(owner) => owner == view, None if fresh => { @@ -135,27 +180,17 @@ impl TokenRegistry { } pub(crate) fn owns_session(&self, view: &str, session_id: &str) -> bool { - self.inner - .lock() - .unwrap() - .sessions - .get(session_id) - .map(String::as_str) - == Some(view) + self.lock().sessions.get(session_id).map(String::as_str) == Some(view) } pub(crate) fn bind_provenance(&self, view: &str, provenance_id: u64) { - self.inner - .lock() - .unwrap() + self.lock() .provenance .insert(provenance_id, view.to_string()); } pub(crate) fn owns_provenance(&self, view: &str, provenance_id: u64) -> bool { - self.inner - .lock() - .unwrap() + self.lock() .provenance .get(&provenance_id) .map(String::as_str) @@ -377,36 +412,113 @@ impl tokio::io::AsyncWrite for BiStream { mod tests { use super::*; + const HOUR: i64 = 60 * 60; + #[test] fn a_token_reaches_its_view_until_it_ends() { let reg = TokenRegistry::default(); - let (t, g) = reg.mint("exp-1", Some("did:key:zAgent".into()), Duration::hours(24)); + let (t, g) = reg + .mint("exp-1", Some("did:key:zAgent".into()), 24 * HOUR) + .unwrap(); assert_eq!(reg.check(&t).unwrap().view, "exp-1"); assert_eq!(reg.check("ast_made_up"), Err(TokenError::Unknown)); std::thread::sleep(std::time::Duration::from_millis(5)); - let after = reg.renew("exp-1", Duration::hours(24)).unwrap(); + let after = reg.renew("exp-1", 24 * HOUR).unwrap(); assert!(after > g.expires); assert_eq!(reg.check(&t).unwrap().expires, after); assert!(reg.revoke("exp-1")); assert_eq!(reg.check(&t), Err(TokenError::Unknown)); - assert!(reg.renew("exp-1", Duration::hours(1)).is_err()); + assert!(reg.renew("exp-1", HOUR).is_err()); } #[test] fn expired_and_replaced_tokens_are_refused() { let reg = TokenRegistry::default(); - let (t, _) = reg.mint("exp-1", None, Duration::milliseconds(-1)); + let (t, _) = reg.mint("exp-1", None, 60).unwrap(); + // Backdate it rather than minting one that is already dead: a + // non-positive lifetime is refused now, which is the point below. + reg.lock() + .grants + .values_mut() + .for_each(|g| g.expires = Utc::now() - Duration::seconds(1)); assert!(matches!(reg.check(&t), Err(TokenError::Expired(_)))); - assert!(reg.renew("exp-1", Duration::hours(1)).is_err()); + assert!(reg.renew("exp-1", HOUR).is_err()); - let (old, _) = reg.mint("exp-2", None, Duration::hours(1)); - let (new, _) = reg.mint("exp-2", None, Duration::hours(1)); + let (old, _) = reg.mint("exp-2", None, HOUR).unwrap(); + let (new, _) = reg.mint("exp-2", None, HOUR).unwrap(); assert_eq!(reg.check(&old), Err(TokenError::Unknown)); assert!(reg.check(&new).is_ok()); } + /// `--ttl` reaches the owner as an `i64` off a wire frame. + /// `Duration::seconds` panics outside its range and `now + Duration` + /// panics on overflow, and both used to happen while holding the registry + /// lock — so one absurd value poisoned the mutex and took every other + /// sandbox and the local `create`/`close` down with it. + #[test] + fn an_absurd_lifetime_is_an_error_and_not_a_panic() { + let reg = TokenRegistry::default(); + // Establish a live token first: the panic used to land in renew, on + // the way to extending one. + let (t, _) = reg.mint("exp-1", None, HOUR).unwrap(); + + for secs in [ + 0, + -1, + i64::MIN, + MAX_TTL_SECS + 1, + i64::MAX, + // Below Duration::seconds' own i64-milliseconds ceiling, so this + // one panicked inside chrono rather than at the addition. + i64::MAX / 1_000, + ] { + assert_eq!( + reg.renew("exp-1", secs), + Err(TokenError::InvalidTtl), + "--ttl {secs} should be refused" + ); + assert_eq!( + reg.mint("exp-2", None, secs).err(), + Some(TokenError::InvalidTtl), + "minting with --ttl {secs} should be refused" + ); + } + + // The ends of the accepted range, and the registry still works. + assert!(reg.mint("low", None, MIN_TTL_SECS).is_ok()); + assert!(reg.mint("high", None, MAX_TTL_SECS).is_ok()); + + // Nothing above was applied, and the registry is intact. + assert_eq!(reg.check(&t).unwrap().view, "exp-1"); + assert!(reg.revoke("exp-1")); + assert!(reg.claim_session("exp-1", "s1", true)); + } + + /// A panic anywhere else in an owner request used to poison the registry, + /// and because every method took the lock with `unwrap`, the next caller + /// panicked too — permanently, for every sandbox and for the local + /// `create`/`close`, until the owner was restarted. + #[test] + fn a_poisoned_lock_does_not_take_the_registry_down() { + let reg = TokenRegistry::default(); + let (t, _) = reg.mint("exp-1", None, HOUR).unwrap(); + + let poisoner = reg.clone(); + let _ = std::thread::spawn(move || { + let _held = poisoner.lock(); + panic!("something unrelated blew up mid-request"); + }) + .join(); + + // Still usable, and the token it already had still resolves. + assert_eq!(reg.check(&t).unwrap().view, "exp-1"); + assert!(reg.mint("exp-2", None, HOUR).is_ok()); + assert!(reg.claim_session("exp-1", "s1", true)); + assert!(reg.revoke("exp-1")); + } + #[test] fn a_session_belongs_to_the_view_that_started_it() { let reg = TokenRegistry::default(); diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index 6515862c..28046614 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -7,12 +7,14 @@ //! changes its view can see. Opening, renewing and closing sandboxes, and //! shutting the owner down, are local only. -use std::path::Path; +use std::path::{Path, PathBuf}; use atomic_core::pristine::{GraphTxnT, ViewTxnT}; use atomic_core::types::{Base32, Hash}; use atomic_repository::redb_change_store::RedbChangeStore; -use atomic_repository::{Repository, ViewEntryKind}; +use atomic_repository::{ + Repository, SubmitRejection, ViewEntryKind, DOT_DIR, SANDBOX_CACHE_DIR, SANDBOX_POINTER, +}; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use super::remote::{Grant, TokenRegistry}; @@ -21,6 +23,12 @@ use super::{ PROTOCOL_VERSION, }; +/// Names a materialized entry may not use, because they are the sandbox's own +/// bookkeeping: writing the pointer would re-point the sandbox (including at an +/// attacker's own node), and writing the cache would edit the pristine the next +/// record reads. +const RESERVED_NAMES: &[&str] = &[DOT_DIR, SANDBOX_POINTER, SANDBOX_CACHE_DIR]; + /// Who is on the other end of a connection. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Caller { @@ -87,14 +95,10 @@ pub(crate) fn authorize( return Err(forbidden("that session")); } } - OwnerRequest::ReserveProvenanceTurn { - session_id, - turn_number, - .. - } => { - let fresh = session_is_fresh(&owner.store, session_id, *turn_number) + OwnerRequest::ReserveProvenanceTurn { session_id, .. } => { + let unused = session_is_unused(&owner.store, session_id) .map_err(|e| refuse("provenance-store", e.to_string()))?; - if !tokens.claim_session(view, session_id, fresh) { + if !tokens.claim_session(view, session_id, unused) { return Err(forbidden("that session")); } } @@ -123,29 +127,14 @@ pub(crate) fn authorize( return Err(forbidden("that provenance turn")); } } - OwnerRequest::StopTurn { - session_id, - turn_number, - .. - } - | OwnerRequest::ResumeTurn { - session_id, - turn_number, - .. - } - | OwnerRequest::AbandonTurn { - session_id, - turn_number, - .. - } - | OwnerRequest::TurnStatus { - session_id, - turn_number, - } => { + OwnerRequest::StopTurn { session_id, .. } + | OwnerRequest::ResumeTurn { session_id, .. } + | OwnerRequest::AbandonTurn { session_id, .. } + | OwnerRequest::TurnStatus { session_id, .. } => { // Asking after a session nobody has written to is harmless (the // store answers "no such turn"); anyone else's is not. if !tokens.owns_session(view, session_id) - && !session_is_fresh(&owner.store, session_id, *turn_number) + && !session_is_unused(&owner.store, session_id) .map_err(|e| refuse("provenance-store", e.to_string()))? { return Err(forbidden("that session")); @@ -161,14 +150,19 @@ pub(crate) fn authorize( Ok(Some(grant)) } -/// Whether no one has recorded provenance for `session_id` up to this turn. -fn session_is_fresh(store: &RedbChangeStore, session_id: &str, turn: u32) -> anyhow::Result { - for t in 1..=turn { - if store.get_provenance_turn_for(session_id, t)?.is_some() { - return Ok(false); - } - } - Ok(true) +/// Whether nothing has ever been recorded for `session_id`. +/// +/// One indexed lookup for the session's highest recorded turn, not a walk from +/// turn 1: the turn number arrives on the wire, so walking to it made a single +/// request cost up to `u32::MAX` database round trips. +/// +/// The question is whether the session is unused, not whether one number in it +/// is free. Every caller wants "has anyone claimed this id?" — claiming a +/// session, and asking after a turn on it. Answering that by scanning +/// `1..=turn` made a session with history *above* the asked turn look unused, +/// which is the wrong way round for a check that decides who may write to it. +fn session_is_unused(store: &RedbChangeStore, session_id: &str) -> anyhow::Result { + Ok(store.last_provenance_turn_for(session_id)?.is_none()) } /// Refuse unless every change in `hashes` is visible on `view`: a sandbox's @@ -236,10 +230,10 @@ pub(crate) async fn admin( Ok(endpoint) => endpoint, Err(e) => return error("remote", format!("{e:#}")), }; - let (token, grant) = - owner - .tokens - .mint(&view, acting_as, chrono::Duration::seconds(ttl_secs)); + let (token, grant) = match owner.tokens.mint(&view, acting_as, ttl_secs) { + Ok(minted) => minted, + Err(e) => return error("sandbox-token", e.to_string()), + }; OwnerResponse::SandboxOpened { remote: super::remote::pointer_addr(endpoint, super::remote::offline()).await, view, @@ -248,10 +242,7 @@ pub(crate) async fn admin( } } OwnerRequest::RenewSandbox { view, ttl_secs } => { - match owner - .tokens - .renew(&view, chrono::Duration::seconds(ttl_secs)) - { + match owner.tokens.renew(&view, ttl_secs) { Ok(expires) => OwnerResponse::SandboxRenewed { expires: expires.to_rfc3339(), }, @@ -373,7 +364,10 @@ pub(crate) async fn record_request( .map(|(hash, bytes)| super::WireChange { hash, bytes }) .collect(), }, - Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { + rejection, + skeleton: None, + }, Ok(Err(e)) => error("changes", format!("{e:#}")), Err(e) => error("internal", e.to_string()), } @@ -387,7 +381,10 @@ pub(crate) async fn record_request( }); match write.await { Ok(Ok(Ok(publication))) => OwnerResponse::ProvenancePublished { publication }, - Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { + rejection, + skeleton: None, + }, Ok(Err(e)) => error("provenance", format!("{e:#}")), Err(e) => error("internal", e.to_string()), } @@ -407,7 +404,15 @@ pub(crate) async fn record_request( let skeleton = repo.after_sandbox_submit(&view, &submitted.hash)?; Ok(Ok((submitted, skeleton))) } - Err(rejection) => Ok(Err(rejection)), + // A stale base is recoverable and the sandbox cannot get + // itself out of it alone — its view row is the stale one. + // Send the view as it is now, so the next record is + // computed against the graph this change was refused for. + Err(rejection @ SubmitRejection::StaleView { .. }) => { + let live = repo.current_sandbox_skeleton(&view)?; + Ok(Err((rejection, Some(live)))) + } + Err(rejection) => Ok(Err((rejection, None))), } }); match write.await { @@ -415,7 +420,10 @@ pub(crate) async fn record_request( submitted, skeleton: Box::new(skeleton), }, - Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Ok(Err((rejection, skeleton)))) => OwnerResponse::ChangeRefused { + rejection, + skeleton: skeleton.map(Box::new), + }, Ok(Err(e)) => error("submit", format!("{e:#}")), Err(e) => error("internal", e.to_string()), } @@ -424,9 +432,55 @@ pub(crate) async fn record_request( } } +/// Where a materialized entry may land: inside `dir`, and nowhere else. +/// +/// `entry.path` arrives on the wire, and the pointer that chose `dir` is itself +/// a file in the tree the agent was handed — so a path that climbs out, or +/// names the pointer or the cache, lets an agent rewrite its own trust anchor +/// or write as the user somewhere it was never given. The parts check rejects +/// anything but plain relative components (no `..`, no root, no empty or `.`); +/// that cannot escape lexically, and the canonicalize catches a *parent* that +/// is a symlink out of the tree, which a parts check cannot see. +fn entry_path(dir: &Path, path: &str) -> anyhow::Result { + let relative = Path::new(path); + anyhow::ensure!(!relative.as_os_str().is_empty(), "entry has an empty path"); + anyhow::ensure!( + relative.is_relative(), + "entry path {path:?} is not relative" + ); + for part in relative.components() { + anyhow::ensure!( + matches!(part, std::path::Component::Normal(_)), + "entry path {path:?} is not a plain relative path", + ); + anyhow::ensure!( + !RESERVED_NAMES + .iter() + .any(|reserved| part.as_os_str() == std::ffi::OsStr::new(reserved)), + "entry path {path:?} names the sandbox's own bookkeeping", + ); + } + Ok(dir.join(relative)) +} + +/// The parent a write lands in, checked to still be inside `tree` after the +/// filesystem has had its say about symlinks. +fn entry_parent<'a>(dir: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> { + let parent = path.parent().unwrap_or(dir); + std::fs::create_dir_all(parent)?; + let real_tree = dir.canonicalize()?; + let real_parent = parent.canonicalize()?; + anyhow::ensure!( + real_parent.starts_with(&real_tree), + "entry parent resolves outside the sandbox", + ); + Ok(parent) +} + /// Client side of `Materialize`: write the view's tree into `dir` and return /// how many entries it had. Files already at those paths are overwritten; -/// nothing else in `dir` is touched. +/// nothing else in `dir` is touched. Every path is checked by [`entry_path`] +/// first, and every write's parent by [`entry_parent`]. pub(crate) async fn receive_materialized( stream: &mut S, request_id: &str, @@ -443,14 +497,12 @@ where } match frame.response { OwnerResponse::MaterializeEntry { entry } => { - let path = dir.join(&entry.path); + let path = entry_path(dir, &entry.path)?; match entry.kind { ViewEntryKind::Directory => std::fs::create_dir_all(&path)?, #[cfg(unix)] ViewEntryKind::Symlink => { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } + entry_parent(dir, &path)?; let target = String::from_utf8(entry.content.clone())?; match std::fs::remove_file(&path) { Err(e) if e.kind() != std::io::ErrorKind::NotFound => { @@ -462,15 +514,11 @@ where } #[cfg(not(unix))] ViewEntryKind::Symlink => { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } + entry_parent(dir, &path)?; std::fs::write(&path, &entry.content)?; } ViewEntryKind::File => { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } + entry_parent(dir, &path)?; std::fs::write(&path, &entry.content)?; #[cfg(unix)] { @@ -511,6 +559,71 @@ mod tests { use super::*; + /// A path off the wire only ever resolves inside the tree, and never + /// onto the pointer or the cache — those two are how the agent picks who + /// it trusts and which database it records into. + #[test] + fn a_materialized_path_stays_inside_the_sandbox() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + + assert_eq!( + entry_path(root, "src/main.rs").unwrap(), + root.join("src/main.rs") + ); + assert_eq!( + entry_path(root, "README.md").unwrap(), + root.join("README.md") + ); + + for path in [ + "", + "/etc/passwd", + "..", + "../elsewhere/x", + "src/../../elsewhere/x", + "./src/main.rs", + "src/../..", + ] { + assert!( + entry_path(root, path).is_err(), + "{path:?} should be refused" + ); + } + + for path in [ + DOT_DIR, + ".atomic/config.toml", + "src/.atomic", + SANDBOX_POINTER, + SANDBOX_CACHE_DIR, + "nested/.atomic-sandbox.d", + ] { + assert!( + entry_path(root, path).is_err(), + "{path:?} should be refused" + ); + } + } + + /// A parts check cannot see a symlinked parent, so the write path checks + /// the parent after the filesystem has resolved it. + #[cfg(unix)] + #[test] + fn a_write_through_a_symlinked_parent_is_refused() { + let outside = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + std::os::unix::fs::symlink(outside.path(), root.join("escape")).unwrap(); + + let path = entry_path(root, "escape/stolen").unwrap(); + assert!(entry_parent(root, &path).is_err()); + + // And an ordinary nested path still works. + let ok = entry_path(root, "src/main.rs").unwrap(); + assert_eq!(entry_parent(root, &ok).unwrap(), root.join("src")); + } + /// A repository with one change on `dev`, and an owner for it. fn owner() -> (tempfile::TempDir, Arc, Hash) { let dir = tempfile::tempdir().unwrap(); @@ -568,7 +681,7 @@ mod tests { #[test] fn a_remote_checkpoint_covers_only_its_own_turn_and_its_view_s_changes() { let (_dir, owner, on_view) = owner(); - let (token, _) = owner.tokens.mint("dev", None, chrono::Duration::hours(1)); + let (token, _) = owner.tokens.mint("dev", None, 3600).unwrap(); owner.tokens.bind_provenance("dev", 7); let remote = |request| authorize(&owner, Caller::Remote, &frame(Some(&token), request)); diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index 55f10fa6..8fc95a4a 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -277,6 +277,62 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { assert!(refused.contains("unauthorized"), "{refused}"); } +/// An absurd `--ttl` used to panic inside the owner's token registry while it +/// held the lock, and every later call on that registry panicked too — so one +/// typo took down every sandbox and `close` until the owner was restarted. It +/// is a refused request, and the owner keeps serving afterwards. +#[test] +fn an_absurd_ttl_is_refused_and_the_owner_keeps_serving() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--view", + "dev", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + + for arg in [ + "--ttl=0", + "--ttl=-1", + "--ttl=-9223372036854775808", + "--ttl=10000000000000000", + ] { + let refused = failure( + atomic(host.path(), &["sandbox", "renew", "dev", arg]), + &format!("renew with {arg}"), + ); + assert!(refused.contains("must last between"), "{arg}: {refused}"); + assert!(!refused.contains("panicked"), "{arg}: {refused}"); + } + + // The token is untouched and the owner still answers. + ok( + atomic(host.path(), &["sandbox", "renew", "dev", "--ttl", "600"]), + "renew after the bad requests", + ); + ok( + atomic(&vm_dir, &["sandbox", "materialize"]), + "materialize after the bad requests", + ); + let out = ok(atomic(host.path(), &["sandbox", "close", "dev"]), "close"); + assert!(out.contains("revoked"), "{out}"); +} + fn hook(cwd: &Path, verb: &str, payload: Value) -> Output { use std::io::Write; let mut child = Command::new(env!("CARGO_BIN_EXE_atomic")) @@ -391,3 +447,91 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { "{session}" ); } + +/// A sandbox that falls behind must be able to record again. It used not to: +/// the cache's view row was the stale thing, and only a *successful* submit +/// refreshed it — so a `StaleView` refusal left the sandbox unable to land +/// anything ever again, and the recovery the error names ("fetch and record +/// again") did not exist anywhere. The owner now sends the view as it is with +/// the refusal, so the cache recovers from the refusal itself. +#[test] +fn a_sandbox_that_fell_behind_can_still_record() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--from", + "dev", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + + // The sandbox records and it lands. + std::fs::write(vm_dir.join("README.md"), "hello\nsandbox\n").unwrap(); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "from the sandbox"]), + "record in the sandbox", + ); + let log = ok( + atomic(host.path(), &["log", "--view", "exp-1"]), + "host log after the sandbox's record", + ); + assert!(log.contains("from the sandbox"), "{log}"); + + // Move the draft view underneath the sandbox, with no sandbox involved: + // the owner is the user's live repository, so a local record on `dev` and + // an insert into the draft is exactly what a second agent would cause. + std::fs::write(host.path().join("OUT-OF-BAND.md"), "elsewhere\n").unwrap(); + ok(atomic(host.path(), &["add", "OUT-OF-BAND.md"]), "host add"); + ok( + atomic(host.path(), &["record", "-a", "-m", "out of band"]), + "host record", + ); + ok( + atomic( + host.path(), + &["insert", "from-view", "dev", "--to-view", "exp-1"], + ), + "host insert into the draft", + ); + + // The sandbox's next record was computed against the view it no longer + // has, so the owner refuses it — and the refusal carries the current view, + // which the cache takes. + std::fs::write(vm_dir.join("README.md"), "hello\nsandbox\nout of date\n").unwrap(); + let refused = failure( + atomic(&vm_dir, &["record", "-a", "-m", "against the old view"]), + "a record against a view that moved on", + ); + assert!(refused.contains("moved on"), "{refused}"); + + // And that is where the old behaviour ended: the cache's view row was + // still the stale one, so the next record computed the same stale base and + // was refused identically. Now the retry works. + let retried = ok( + atomic(&vm_dir, &["record", "-a", "-m", "after the view moved"]), + "retry in the sandbox after the view moved", + ); + assert!(!retried.contains("moved on"), "{retried}"); + + let log = ok( + atomic(host.path(), &["log", "--view", "exp-1"]), + "host log after the retry", + ); + assert!(log.contains("after the view moved"), "{log}"); + // The refused change is not on the view, under either message. + assert!(!log.contains("against the old view"), "{log}"); +} diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs index bb33eeb5..51f3de8f 100644 --- a/atomic-core/src/pristine/slice.rs +++ b/atomic-core/src/pristine/slice.rs @@ -34,6 +34,7 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque}; use redb::ReadableTable; use serde::{Deserialize, Serialize}; +use crate::crdt::ids::{BranchId, LeafId, TrunkId}; use crate::crdt::tables::{ BRANCHES, BRANCH_AFTER, BRANCH_LEAVES, BRANCH_VERTEX, INODE_TRUNK, LEAVES, PATH_TRUNK, TRUNKS, TRUNK_BRANCHES, VERTEX_BRANCH, @@ -334,13 +335,13 @@ impl ReadTxn { } } for (id, _) in &out.crdt.trunks { - ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + ids.insert(TrunkId::from_bytes(id).change_id().get()); } for (id, _) in &out.crdt.branches { - ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + ids.insert(BranchId::from_bytes(id).change_id().get()); } for (id, _) in &out.crdt.leaves { - ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + ids.insert(LeafId::from_bytes(id).change_id().get()); } ids.remove(&0); out.ids = self.id_rows(ids.into_iter())?; @@ -404,6 +405,12 @@ impl ReadTxn { Ok(()) } + /// `IdRow`s for `ids`. An id with no `EXTERNAL` row is dropped: a + /// change can be visible on a view without being registered yet, and + /// callers treat a missing row as "not mine to carry". A drop is still a + /// silent way to lose an id, so a miss in a debug build asserts — a + /// mis-decoded id (see the endianness rule in `tables.rs`) looks exactly + /// like an absent one. fn id_rows(&self, ids: impl Iterator) -> PristineResult> { let external = self.txn.open_table(EXTERNAL)?; let node_types = self.txn.open_table(NODE_TYPES)?; @@ -412,6 +419,11 @@ impl ReadTxn { if let Some(h) = external.get(id)? { let t = node_types.get(id)?.map(|t| t.value()); out.push((id, *h.value(), t)); + } else { + debug_assert!( + false, + "id {id} has no EXTERNAL row; was it decoded correctly?" + ); } } Ok(out) @@ -640,3 +652,37 @@ impl WriteTxn<'_> { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The export decodes a CRDT id's change with the type's own `from_bytes`. + /// If the decode and the encoder ever disagree on endianness, the change + /// comes back as a different node id and `id_rows` drops it — the slice + /// still exports, just without the id. This pins the two together. + #[test] + fn crdt_ids_decode_little_endian_change_ids() { + for change in [1u64, 2, 42, 255, 256, 65_536, 1 << 32, u64::MAX >> 8] { + let change = NodeId::new(change); + let trunk = TrunkId::new(change, 7); + let branch = BranchId::new(change, 7); + let leaf = LeafId::new(change, 7); + + assert_eq!(trunk.change_id(), change); + assert_eq!(TrunkId::from_bytes(&trunk.to_bytes()).change_id(), change); + assert_eq!(BranchId::from_bytes(&branch.to_bytes()).change_id(), change); + assert_eq!(LeafId::from_bytes(&leaf.to_bytes()).change_id(), change); + } + } + + /// The first eight bytes are the change id on its own, which is what a + /// hand-rolled `id[0..8]` decode would read. Big-endian gets 42's + /// neighbours wrong; little-endian is what the encoders write. + #[test] + fn crdt_id_change_id_is_little_endian_in_its_first_eight_bytes() { + let bytes = TrunkId::new(NodeId::new(42), 0).to_bytes(); + assert_eq!(&bytes[0..8], &42u64.to_le_bytes()); + assert_ne!(&bytes[0..8], &42u64.to_be_bytes()); + } +} diff --git a/atomic-core/src/pristine/tables.rs b/atomic-core/src/pristine/tables.rs index 80715c7d..5b87dbee 100644 --- a/atomic-core/src/pristine/tables.rs +++ b/atomic-core/src/pristine/tables.rs @@ -840,6 +840,21 @@ pub fn decode_change_file_key(key: &[u8; 36]) -> ([u8; 32], u32) { } // Key Encoding Helpers +// +// Endianness. A key that gets range-scanned is big-endian, so its bytes sort +// in the same order as the numbers they encode — that is what makes a +// `range(lo..=hi)` over these tables mean what it says (e.g. every row for +// one inode: `encode_inode_vertex(inode, 0, 0, 0)` through +// `encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX)`). A value that is +// only ever looked up by its exact key can be little-endian; sorting never +// happens, so byte order costs nothing. That covers the serialized graph edge +// and the CRDT trunk/branch/leaf ids, which are opaque and point-looked-up +// (`crdt::tables`). +// +// The two rules meet in files that handle both, so match the writer rather +// than the neighbours: decode an id with that type's own `from_bytes` +// (`TrunkId`, `BranchId`, `LeafId`) rather than slicing bytes by hand, and the +// two sides cannot drift. /// Encode a span as 24 bytes for use as a graph key #[inline] diff --git a/atomic-repository/src/redb_change_store/provenance.rs b/atomic-repository/src/redb_change_store/provenance.rs index 4644541e..b850bc1e 100644 --- a/atomic-repository/src/redb_change_store/provenance.rs +++ b/atomic-repository/src/redb_change_store/provenance.rs @@ -1,8 +1,8 @@ //! Resumable pending provenance journal stored alongside redb-native changes. use atomic_core::change::session::{ - encode_session_event_key, encode_session_turn_key, session_turn_namespace, SessionEvent, - SessionTurn, + decode_session_turn_key, encode_session_event_key, encode_session_turn_key, + session_turn_namespace, SessionEvent, SessionTurn, }; use atomic_core::pristine::tables; use atomic_core::types::Hash; @@ -369,6 +369,31 @@ impl RedbChangeStore { .transpose() } + /// The highest turn number recorded for `session_id`, if any. + /// + /// A session's turns are one contiguous, turn-ordered run of keys (the + /// namespace is a fixed prefix, the number is big-endian at the end), so + /// the last key in that range is the highest turn. Callers asking "is this + /// turn number taken?" want this, not a walk from turn 1 — the turn number + /// comes off the wire, and a walk to it opens a read transaction per step. + pub fn last_provenance_turn_for(&self, session_id: &str) -> RedbStoreResult> { + let txn = self.db.begin_read()?; + let index = txn.open_table(tables::PROVENANCE_TURN_INDEX)?; + let lo = turn_key(session_id, 0); + let hi = turn_key(session_id, u32::MAX); + let Some(last) = index.range::<&[u8; 40]>(&lo..=&hi)?.next_back() else { + return Ok(None); + }; + let (key, _) = last?; + let (namespace, turn) = decode_session_turn_key(key.value()); + if namespace != session_turn_namespace(session_id) { + return Err(RedbStoreError::Corrupt( + "session-turn index returned a key outside the session's range".to_string(), + )); + } + Ok(Some(turn)) + } + /// Return a reserved turn by external session identity and turn number. pub fn get_provenance_turn_for( &self, diff --git a/atomic-repository/src/redb_change_store/tests.rs b/atomic-repository/src/redb_change_store/tests.rs index 26df1cac..c66468ed 100644 --- a/atomic-repository/src/redb_change_store/tests.rs +++ b/atomic-repository/src/redb_change_store/tests.rs @@ -573,6 +573,49 @@ fn provenance_turn_reservation_is_idempotent_and_persistent() { ); } +#[test] +fn the_last_recorded_turn_says_whether_a_session_was_ever_used() { + let (_dir, store) = temp_store(); + + // A session with no turns at all is unused; the moment one is reserved it + // is not, whichever turn it was. This is what the owner's authz asks + // before letting a sandbox claim a session id, so "somewhere above the + // turn I asked about" must not read as unused. + assert_eq!(store.last_provenance_turn_for("none").unwrap(), None); + + // Reserve out of order, and past the low turns, so ordering matters. + for turn in [7u32, 2, 5] { + store.reserve_provenance_turn("sparse", turn, 0).unwrap(); + } + assert_eq!(store.last_provenance_turn_for("sparse").unwrap(), Some(7)); + assert!(store.last_provenance_turn_for("sparse").unwrap().is_some()); + + // Reserving the lowest turn last is still "used": the last key is the + // highest, not the most recent write. + store.reserve_provenance_turn("sparse", 1, 0).unwrap(); + assert_eq!(store.last_provenance_turn_for("sparse").unwrap(), Some(7)); + + // Turn 0 is a number like any other and counts. + let zero = temp_store(); + zero.1.reserve_provenance_turn("zero", 0, 0).unwrap(); + assert_eq!(zero.1.last_provenance_turn_for("zero").unwrap(), Some(0)); + + // Another session's turns are not this one's, and vice versa — the + // namespace is a hashed prefix, so the range has to stop at it. + store.reserve_provenance_turn("other", 4, 0).unwrap(); + assert_eq!(store.last_provenance_turn_for("other").unwrap(), Some(4)); + assert_eq!(store.last_provenance_turn_for("sparse").unwrap(), Some(7)); + assert_eq!(store.last_provenance_turn_for("nope").unwrap(), None); + + // Turn number at the top of the range, which is what the wire allows. + let big = temp_store(); + big.1.reserve_provenance_turn("big", u32::MAX, 0).unwrap(); + assert_eq!( + big.1.last_provenance_turn_for("big").unwrap(), + Some(u32::MAX) + ); +} + #[test] fn provenance_events_are_ordered_idempotent_and_fenced() { let (_dir, store) = temp_store(); diff --git a/atomic-repository/src/repository/materialize.rs b/atomic-repository/src/repository/materialize.rs index 7d707721..8ef53c65 100644 --- a/atomic-repository/src/repository/materialize.rs +++ b/atomic-repository/src/repository/materialize.rs @@ -1089,8 +1089,14 @@ impl Repository { /// operations (files it added, moved or deleted) wait in the deferred /// tree journal until a switch applies them. For another view they are /// applied here in a write transaction that is thrown away, so the - /// listing is that view's and nothing on disk changes. A read-only handle - /// can't do that and lists TREE as it is. + /// listing is that view's and nothing on disk changes. + /// + /// A read-only handle cannot project, and `TREE` is the *current* view's — + /// so a read-only repository asked about another view is refused rather + /// than answered. Falling through would return the current view's tree + /// under the other view's name, and the caller writes that straight into a + /// remote sandbox. The current view needs no projection, so it reads `TREE` + /// directly and works read-only. fn view_tree_items( &self, txn: &atomic_core::pristine::ReadTxn, @@ -1101,13 +1107,20 @@ impl Repository { let db = |e: atomic_core::pristine::PristineError| RepositoryError::Database(e.to_string()); if view_name != self.current_view { let journal = self.load_deferred_tree_journal()?; - if let Ok(mut projected) = self.pristine.write_txn() { - self.apply_deferred_tree_ops_in_txn(&mut projected, &journal, view_name)?; - let items = collect_children(&projected, Inode::ROOT, "", options).map_err(db)?; - use atomic_core::pristine::MutTxnT; - projected.abort().map_err(db)?; - return Ok(items); - } + let mut projected = + self.pristine + .write_txn() + .map_err(|e| RepositoryError::InvalidOperation { + message: format!( + "listing view '{view_name}' projects its tree, which needs a writable \ + repository: {e}" + ), + })?; + self.apply_deferred_tree_ops_in_txn(&mut projected, &journal, view_name)?; + let items = collect_children(&projected, Inode::ROOT, "", options).map_err(db)?; + use atomic_core::pristine::MutTxnT; + projected.abort().map_err(db)?; + return Ok(items); } collect_children(txn, Inode::ROOT, "", options).map_err(db) } diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 4150687d..2edee331 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -83,7 +83,7 @@ pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; mod remote_cache; pub use remote_cache::{ set_remote_sandbox_link, ChangeFile, RemoteSandboxLink, SandboxSkeleton, SandboxSlice, - SpanBytes, SubmitRejection, Submitted, + SpanBytes, SubmitRejection, Submitted, SubmittedOutcome, }; mod sandbox; mod semantic_materialize; diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index f6f1594b..e4bd9282 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -90,10 +90,45 @@ pub struct Submitted { pub state: String, } -/// Paths only the repository's own machinery writes. -fn forbidden_path(path: &str) -> bool { - path.split('/').any(|part| { - part == crate::DOT_DIR || part == super::SANDBOX_POINTER || part == super::SANDBOX_CACHE_DIR +/// What `SubmitChange` came back with: the change landed, or it did not. +pub type SubmittedOutcome = + Result<(Submitted, SandboxSkeleton), (SubmitRejection, Option)>; + +/// Whether `path` may not be recorded, from a sandbox or otherwise. +/// +/// Refused in two cases: the path is not a plain relative path, or it names the +/// repository's own machinery. The first is not a nicety. A submitted change +/// lands in `TREE`, and the next materialize or view switch writes +/// `root.join(path)` — so `../../elsewhere/x` escapes the working tree, with no +/// local user in the loop when a remote token-holder sends it. A reserved-name +/// list cannot catch that, which is why the shape check comes first. +/// +/// An empty path is not a path: a `FileOps` group for a token-level operation +/// names an inode and carries no path, and there is nothing to escape with. +pub(crate) fn forbidden_path(path: &str) -> bool { + if path.is_empty() { + return false; + } + let relative = std::path::Path::new(path); + if relative.is_absolute() { + return true; + } + // `components()` collapses repeated separators, so `src//main.rs` would + // read here as two plain names — and then reach `TREE` as a path no + // materialization can ever put a file at, because the write normalizes it + // back to `src/main.rs`. + if path.contains("//") || path.ends_with('/') { + return true; + } + relative.components().any(|part| match part { + // Not `Normal` means empty, `.`, `..`, or a root/prefix: none of which + // a tracked file may have. + std::path::Component::Normal(name) => { + name == crate::DOT_DIR + || name == super::SANDBOX_POINTER + || name == super::SANDBOX_CACHE_DIR + } + _ => true, }) } @@ -106,13 +141,19 @@ pub trait RemoteSandboxLink: Send + Sync { fn file_states(&self, root: &Path, inodes: Vec) -> Result; /// `SubmitChange`: the change recorded against `base_state`; the view's /// skeleton after it lands. + /// + /// A `StaleView` refusal comes back with the view's current skeleton, so + /// the cache can resync from the refusal itself. Without it a sandbox that + /// fell behind could never record again: its state is the thing that is + /// stale, and the only way to learn the new one is a request the owner has + /// no reason to distinguish from a fresh sandbox. fn submit( &self, root: &Path, base_state: String, hash: Hash, bytes: Vec, - ) -> Result, String>; + ) -> Result; /// `Changes`: change files the view has. fn changes(&self, root: &Path, hashes: Vec) -> Result, String>; /// `PublishProvenance`: a checkpoint's provenance graph (serialized) and @@ -239,6 +280,21 @@ impl Repository { Ok(SandboxSlice { rows, spans }) } + /// Serve side: the view's skeleton as it is right now, with nothing of it + /// applied. What a sandbox needs after being told its base state is stale: + /// its own view row is the stale one, so it cannot work out the current + /// state on its own, and asking again for a state it was just refused on + /// is a round trip the owner already has the answer for. + pub fn current_sandbox_skeleton(&self, view: &str) -> Result { + let mut live = std::collections::BTreeMap::new(); + self.materialize_view_entries::<()>(view, |entry| { + live.insert(entry.inode, entry.path.clone()); + Ok(()) + })? + .map_err(|()| RepositoryError::Output("unreachable".to_string()))?; + self.export_sandbox_skeleton(view, &live) + } + /// Serve side: take a change a remote sandbox recorded on `view` and /// apply it there — if it is exactly what it claims, recorded against /// the view as it is now, and names nothing outside the view. @@ -463,6 +519,12 @@ impl Repository { /// In a remote sandbox, what applying a recorded change means: hand it /// to the owner, and take the view as it is once it lands. + /// + /// A refusal that says the view moved on is not a dead end: the owner + /// sends the view as it is now, and the cache takes it, so the next + /// `record` is computed against the graph this one was refused for. The + /// change that was refused is not applied — the caller has to record it + /// again — and the error says so. pub(crate) fn submit_recorded( &self, outcome: &crate::record::RecordOutcome, @@ -472,12 +534,20 @@ impl Repository { .ok_or_else(|| RepositoryError::Apply("the change has no bytes".to_string()))? .to_vec(); let base_state = self.remote_sandbox_view_state()?; - let (submitted, skeleton) = link()? + let (submitted, skeleton) = match link()? .submit(&self.root, base_state, *outcome.hash(), bytes) .map_err(|message| RepositoryError::InvalidOperation { message })? - .map_err(|rejection| { - RepositoryError::Apply(format!("the repository refused the change: {rejection}")) - })?; + { + Ok(landed) => landed, + Err((rejection, resync)) => { + if let Some(resync) = resync { + self.import_sandbox_skeleton(&resync)?; + } + return Err(RepositoryError::Apply(format!( + "the repository refused the change: {rejection}" + ))); + } + }; self.import_sandbox_skeleton(&skeleton)?; let state = ::from_base32(submitted.state.as_bytes()) @@ -656,3 +726,56 @@ impl Repository { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::forbidden_path; + + #[test] + fn a_recorded_path_must_be_a_plain_relative_path() { + for path in [ + "/etc/passwd", + "/", + "..", + "../elsewhere/x", + "src/../../elsewhere/x", + "src/..", + "./src/main.rs", + "src//main.rs", + ] { + assert!(forbidden_path(path), "{path:?} should be refused"); + } + } + + #[test] + fn a_recorded_path_may_not_name_the_repositorys_own_machinery() { + for path in [ + ".atomic", + ".atomic/config.toml", + "src/.atomic/config.toml", + ".atomic-sandbox", + ".atomic-sandbox.d", + "nested/.atomic-sandbox.d/cache", + ] { + assert!(forbidden_path(path), "{path:?} should be refused"); + } + } + + #[test] + fn an_ordinary_path_is_allowed() { + for path in [ + // A token-level op names an inode, not a path: nothing to escape. + "", + "README.md", + "src/main.rs", + "a/b/c/d.txt", + "..hidden", + "...hidden", + "a..b", + "atomic", + "src/atomic-sandbox", + ] { + assert!(!forbidden_path(path), "{path:?} should be allowed"); + } + } +} diff --git a/atomic-repository/tests/materialize_view_entries_test.rs b/atomic-repository/tests/materialize_view_entries_test.rs index 31d0fce7..8cd098f5 100644 --- a/atomic-repository/tests/materialize_view_entries_test.rs +++ b/atomic-repository/tests/materialize_view_entries_test.rs @@ -102,3 +102,67 @@ fn entries_are_the_view_s_recorded_tree() { ); assert_ne!(older_snapshot.state, snapshot.state); } + +/// Another view's tree is a projection, and a read-only repository cannot +/// project. It used to fall through to `TREE`, which is the *current* view's — +/// handing one view's tree back under another's name, which for a remote +/// sandbox means writing the wrong tree to disk. Refusing is the safe answer. +#[test] +fn a_read_only_repository_refuses_another_views_tree() { + let temp = TempDir::new().unwrap(); + let root = temp.path().to_path_buf(); + let mut repo = Repository::init(&root).expect("init"); + let current = repo.current_view().to_string(); + + write(&root, "README.md", "hello\n"); + repo.add("README.md", Default::default()).unwrap(); + record(&repo, "first"); + write(&root, "README.md", "hello, world\n"); + let second = record(&repo, "second"); + + repo.split_view(SplitOptions::new("older", vec![second])) + .expect("split"); + + // Writable: both views render, and they differ. `split_view` moves the + // change out of the source view, so the new view is the one with it. + let (on_current, _) = entries(&repo, ¤t); + let (on_older, _) = entries(&repo, "older"); + let body = |v: &Vec| { + v.iter() + .find(|e| e.path == "README.md") + .map(|e| e.content.clone()) + .unwrap() + }; + assert_eq!(body(&on_current), b"hello\n"); + assert_eq!(body(&on_older), b"hello, world\n"); + + // Read-only: the current view still works (it needs no projection), but + // another view is refused rather than answered with the current tree. + drop(repo); + let readonly = Repository::open_readonly(&root).expect("open read-only"); + + let mut on_current = Vec::new(); + readonly + .materialize_view_entries::<()>(¤t, |e| { + on_current.push(e); + Ok(()) + }) + .expect("materialize the current view") + .expect("sink"); + assert_eq!(body(&on_current), b"hello\n"); + + let mut on_older = Vec::new(); + let refused = readonly.materialize_view_entries::<()>("older", |e| { + on_older.push(e); + Ok(()) + }); + let err = refused.expect_err("another view must be refused"); + assert!( + format!("{err}").contains("writable"), + "unexpected error: {err}" + ); + assert!( + on_older.is_empty(), + "nothing of the wrong view was rendered" + ); +} From 9b7defcfed3c58f8e7eb28f8a72ddcac18d6f5c3 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Sun, 27 Sep 2026 19:54:53 -0500 Subject: [PATCH 17/23] refactor(wasm): the wasm crate is atomic-wasm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit atomic-canonical-wasm named the crate it wraps twice over — atomic, twice — and it is the only wasm crate in the workspace, so the extra precision cost a word and bought nothing. atomic-wasm says the same thing in half the length, and the module a page imports becomes atomic_wasm. atomic-canonical itself keeps its name: that is the crate this one wraps, and it is a dependency, not a rename of it. Verified: builds for the host and for wasm32-unknown-unknown (787 KB artifact at the path build.sh expects), full workspace suite 9088 tests across 57 binaries, 0 failures. --- Cargo.lock | 22 +++++++++---------- Cargo.toml | 2 +- .../.gitignore | 0 .../Cargo.toml | 2 +- .../README.md | 4 ++-- .../build.sh | 6 ++--- .../smoke.mjs | 4 ++-- .../src/lib.rs | 0 8 files changed, 20 insertions(+), 20 deletions(-) rename {atomic-canonical-wasm => atomic-wasm}/.gitignore (100%) rename {atomic-canonical-wasm => atomic-wasm}/Cargo.toml (95%) rename {atomic-canonical-wasm => atomic-wasm}/README.md (95%) rename {atomic-canonical-wasm => atomic-wasm}/build.sh (57%) rename {atomic-canonical-wasm => atomic-wasm}/smoke.mjs (88%) rename {atomic-canonical-wasm => atomic-wasm}/src/lib.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 82bdaa7e..dabd26e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -218,17 +218,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "atomic-canonical-wasm" -version = "0.18.3" -dependencies = [ - "atomic-canonical", - "atomic-identity", - "getrandom 0.2.17", - "serde_json", - "wasm-bindgen", -] - [[package]] name = "atomic-cli" version = "0.18.3" @@ -459,6 +448,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomic-wasm" +version = "0.18.3" +dependencies = [ + "atomic-canonical", + "atomic-identity", + "getrandom 0.2.17", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "attohttpc" version = "0.30.1" diff --git a/Cargo.toml b/Cargo.toml index f928bbb5..31d482b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,6 @@ resolver = "2" members = [ "atomic-agent", "atomic-canonical", - "atomic-canonical-wasm", "atomic-cli", "atomic-core", "atomic-config", @@ -13,6 +12,7 @@ members = [ "atomic-repository", "atomic-semantic", "atomic-teams", + "atomic-wasm", ] [workspace.package] diff --git a/atomic-canonical-wasm/.gitignore b/atomic-wasm/.gitignore similarity index 100% rename from atomic-canonical-wasm/.gitignore rename to atomic-wasm/.gitignore diff --git a/atomic-canonical-wasm/Cargo.toml b/atomic-wasm/Cargo.toml similarity index 95% rename from atomic-canonical-wasm/Cargo.toml rename to atomic-wasm/Cargo.toml index 8847e900..35a2f6a6 100644 --- a/atomic-canonical-wasm/Cargo.toml +++ b/atomic-wasm/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "atomic-canonical-wasm" +name = "atomic-wasm" description = "atomic-canonical for the browser: build and check atomic attestations where the signing key lives in WebCrypto" version.workspace = true edition.workspace = true diff --git a/atomic-canonical-wasm/README.md b/atomic-wasm/README.md similarity index 95% rename from atomic-canonical-wasm/README.md rename to atomic-wasm/README.md index a5b15f48..9ec280c2 100644 --- a/atomic-canonical-wasm/README.md +++ b/atomic-wasm/README.md @@ -1,4 +1,4 @@ -# atomic-canonical-wasm +# atomic-wasm `atomic-canonical` for the browser. A page that holds an atomic identity's Ed25519 key in WebCrypto signs atomic documents without the key ever leaving @@ -7,7 +7,7 @@ the browser: this module builds the canonical document and the bytes to sign ordinary `eddsa-jcs-2022` attestation that `atomic` verifies. ```js -import init, { prepareAttestation, attachProof } from "./pkg/atomic_canonical_wasm.js"; +import init, { prepareAttestation, attachProof } from "./pkg/atomic_wasm.js"; await init(); const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); diff --git a/atomic-canonical-wasm/build.sh b/atomic-wasm/build.sh similarity index 57% rename from atomic-canonical-wasm/build.sh rename to atomic-wasm/build.sh index 94f6390e..2cbb34b8 100755 --- a/atomic-canonical-wasm/build.sh +++ b/atomic-wasm/build.sh @@ -1,11 +1,11 @@ #!/bin/sh -# Build the browser module: pkg/atomic_canonical_wasm{.js,_bg.wasm}. +# Build the browser module: pkg/atomic_wasm{.js,_bg.wasm}. # Needs the wasm32 target and a wasm-bindgen CLI matching the locked # wasm-bindgen crate: # rustup target add wasm32-unknown-unknown # cargo install wasm-bindgen-cli --version set -eu here=$(cd "$(dirname "$0")" && pwd) -cargo build -p atomic-canonical-wasm --target wasm32-unknown-unknown --release +cargo build -p atomic-wasm --target wasm32-unknown-unknown --release wasm-bindgen --target web --out-dir "${1:-$here/pkg}" \ - "$here/../target/wasm32-unknown-unknown/release/atomic_canonical_wasm.wasm" + "$here/../target/wasm32-unknown-unknown/release/atomic_wasm.wasm" diff --git a/atomic-canonical-wasm/smoke.mjs b/atomic-wasm/smoke.mjs similarity index 88% rename from atomic-canonical-wasm/smoke.mjs rename to atomic-wasm/smoke.mjs index b5a8250d..6fba36a7 100644 --- a/atomic-canonical-wasm/smoke.mjs +++ b/atomic-wasm/smoke.mjs @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; -import init, { prepareAttestation, attachProof, verifyAttestation, didForPublicKey } from "./pkg/atomic_canonical_wasm.js"; -await init({ module_or_path: readFileSync(new URL("./pkg/atomic_canonical_wasm_bg.wasm", import.meta.url)) }); +import init, { prepareAttestation, attachProof, verifyAttestation, didForPublicKey } from "./pkg/atomic_wasm.js"; +await init({ module_or_path: readFileSync(new URL("./pkg/atomic_wasm_bg.wasm", import.meta.url)) }); const kp = await crypto.subtle.generateKey("Ed25519", false, ["sign", "verify"]); const pub = new Uint8Array(await crypto.subtle.exportKey("raw", kp.publicKey)); const doc = { "@type": "ExampleLogin", nonce: "abc", "é": [1, 2.5, "x"] }; diff --git a/atomic-canonical-wasm/src/lib.rs b/atomic-wasm/src/lib.rs similarity index 100% rename from atomic-canonical-wasm/src/lib.rs rename to atomic-wasm/src/lib.rs From 1cf8ab64975793cf3c16d90f6eb3725690c6c97b Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Sun, 27 Sep 2026 21:44:19 -0500 Subject: [PATCH 18/23] test(remote-sandbox): several sandboxes at once, and the two bugs it found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every remote-sandbox test so far drove one sandbox, one request at a time, against a database nobody else was touching. A stress test with four sandboxes — each on its own draft, since a view has one live token — all materializing, publishing provenance and recording behind a barrier, found two things a serial test cannot see. The first it found immediately. FileStates and Changes opened the database with a bare open_readonly, which fails outright when another request holds the file; the two write handlers have been waiting on open_existing_wait all along. So the second sandbox to ask lost, every time, with "Database already open. Cannot acquire lock." Both read handlers wait now, for the same reason the writes do. The second it found by instrumenting, and it is worse. A remote cache mints node ids from a counter seeded only by the ids the owner chose to send it: import_ids takes fetch_max over the ids in a single import, and a cache is only told about the files it asks for. On the four-sandbox run the owner sent 5 ids with a maximum of 7, leaving each cache's counter at 8 while the repository's own was well past that. The cache then invents ids for its new content that are already real repository node ids, and the owner refuses the change with "node N is not on this view" — the agent's work is lost, and the sandbox cannot get past it. Parallelism is not the cause, only what makes it reliable: it races several caches through one import so they all record from a stale counter, and a single sandbox passes whenever the repository happens to have no node where its cache lands. That is why nothing caught it. Kept as an ignored test with the diagnosis, so it runs the moment the node-id space is fixed. Fixing it is the work LOCAL_INODE_FLOOR already does for inodes: a cache's node ids need a floor, and the owner needs to remap them into the repository's space on insert instead of applying a submitted change's ids verbatim. That is a change to what `insert_submitted_change` trusts, so it is not done here. The passing test asserts separation, which is the property a shared owner can get wrong without erroring: every change on the draft that made it, no draft carrying another's, dev untouched, dev's working tree not rewritten, and each session its own with its own goal. Full workspace suite: 57 binaries, 9089 tests, 0 failures. --- .../src/commands/agent/owner/sandbox.rs | 13 +- .../tests/remote_sandbox_integration_test.rs | 279 +++++++++++++++++- 2 files changed, 289 insertions(+), 3 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index 28046614..6277ae96 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -341,7 +341,14 @@ pub(crate) async fn record_request( match request { OwnerRequest::FileStates { inodes, .. } => { let read = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { - let repo = Repository::open_readonly(&root)?; + // Wait for the database, don't fail on it. The owner opens it + // per request, so two sandboxes asking at once contend for it + // and the loser has to wait — the same reason the writes below + // wait. Without this a `file-states` arriving while another + // request holds the file fails outright, which under load is + // every sandbox but one. + let repo = + Repository::open_readonly_wait(&root, std::time::Duration::from_secs(30))?; Ok(repo.export_sandbox_slice(&view, &inodes)?) }); match read.await { @@ -354,7 +361,9 @@ pub(crate) async fn record_request( } OwnerRequest::Changes { hashes, .. } => { let read = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { - let repo = Repository::open_readonly(&root)?; + // Waits for the database for the same reason `FileStates` does. + let repo = + Repository::open_readonly_wait(&root, std::time::Duration::from_secs(30))?; Ok(repo.export_sandbox_changes(&view, &hashes)?) }); match read.await { diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index 8fc95a4a..c17aa3a2 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -4,8 +4,10 @@ //! Runs offline: the owner and the sandbox dial each other directly on //! loopback (`ATOMIC_OWNER_IROH_OFFLINE`), no relays. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, Output}; +use std::sync::{Arc, Barrier}; +use std::thread; use serde_json::Value; use tempfile::TempDir; @@ -535,3 +537,278 @@ fn a_sandbox_that_fell_behind_can_still_record() { // The refused change is not on the view, under either message. assert!(!log.contains("against the old view"), "{log}"); } + +/// Several sandboxes at once against one owner: each on its own draft view, all +/// materializing, publishing provenance and recording at the same time. +/// +/// This is the shape the owner's concurrency has to survive. A view has one +/// live token, so sandboxes cannot share a view and each gets its own draft off +/// `dev`. Everything after that contends: one owner process, one `submissions` +/// mutex held across the insert, a redb writer only one caller may hold, and a +/// database the owner reopens per request. The assertions are all about +/// *separation* — every change on the draft that made it, none on any other — +/// because a shared owner can cross wires without erroring. +#[test] +fn several_remote_sandboxes_work_at_once() { + const SANDBOXES: usize = 4; + + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + + let names: Vec = (0..SANDBOXES).map(|i| format!("sb-{i}")).collect(); + let dirs: Vec = names.iter().map(|_| TempDir::new().unwrap()).collect(); + let works: Vec = dirs.iter().map(|d| d.path().join("work")).collect(); + + for (name, work) in names.iter().zip(&works) { + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + name, + "--remote", + "--from", + "dev", + "--dest", + work.to_str().unwrap(), + ], + ), + &format!("sandbox create {name}"), + ); + } + + let gate = Arc::new(Barrier::new(SANDBOXES)); + let mut sandboxes = Vec::new(); + for (i, (name, work)) in names.iter().zip(&works).enumerate() { + let gate = Arc::clone(&gate); + let (name, work) = (name.clone(), work.clone()); + sandboxes.push(thread::spawn(move || { + let now = "2026-01-01T00:00:00Z"; + let session = format!("session-{i}"); + let turn = |n: u32| { + serde_json::json!({ + "session_id": session, "cwd": work.to_str().unwrap(), "model": "m", + "provider": "p", "turn_number": n, + "intent_title": format!("work on {name}"), "timestamp": now, + }) + }; + + // From here on they contend: one owner, one database, one writer. + gate.wait(); + + ok( + atomic(&work, &["sandbox", "materialize"]), + &format!("{name} materialize"), + ); + // Each sandbox's own session. A sandbox may only touch its own, so + // this is the owner's session check under load. + ok( + hook( + &work, + "session-start", + serde_json::json!({ + "session_id": session, "cwd": work.to_str().unwrap(), "model": "m", + "provider": "p", "turn_number": 0, "timestamp": now, + }), + ), + &format!("{name} session-start"), + ); + ok( + hook(&work, "turn-start", turn(1)), + &format!("{name} turn-start"), + ); + + // Each edits a file only it touches, so a crossed change shows up + // as another sandbox's content on this draft. + std::fs::write(work.join("README.md"), format!("hello\nfrom {name}\n")).unwrap(); + std::fs::write( + work.join(format!("{name}.txt")), + format!("only {name} wrote this\n"), + ) + .unwrap(); + ok( + atomic(&work, &["add", &format!("{name}.txt")]), + &format!("{name} add"), + ); + ok( + atomic( + &work, + &["record", "-a", "-m", &format!("recorded by {name}")], + ), + &format!("{name} record"), + ); + ok( + hook(&work, "turn-end", turn(1)), + &format!("{name} turn-end"), + ); + ok( + hook( + &work, + "session-end", + serde_json::json!({ + "session_id": session, "cwd": work.to_str().unwrap(), + "turn_number": 1, "timestamp": now, + }), + ), + &format!("{name} session-end"), + ); + session + })); + } + + let sessions: Vec = sandboxes + .into_iter() + .map(|s| s.join().expect("a sandbox thread finished")) + .collect(); + + // Every draft has its own sandbox's change, and its provenance. + for (i, name) in names.iter().enumerate() { + let log = ok( + atomic(host.path(), &["log", "--view", name]), + &format!("host log of {name}"), + ); + assert!( + log.contains(&format!("recorded by {name}")), + "{name}: {log}" + ); + + let shown = ok( + atomic(host.path(), &["session", "show", &sessions[i]]), + &format!("host session {}", sessions[i]), + ); + assert!(shown.contains("Turns: 1"), "{}: {shown}", sessions[i]); + assert!( + shown.contains(&format!("Goal marker: work on {name}")), + "{}: {shown}", + sessions[i] + ); + } + + // ...and nothing of any other sandbox's. This is the assertion that fails + // if a submit were applied to the wrong view. + for name in &names { + let log = ok( + atomic(host.path(), &["log", "--view", name]), + &format!("host log of {name}"), + ); + for other in &names { + if other != name { + assert!( + !log.contains(&format!("recorded by {other}")), + "{name} has {other}'s change: {log}" + ); + } + } + } + + // The shared base is untouched: no draft's work reached `dev`. + let dev = ok(atomic(host.path(), &["log"]), "host log of dev"); + for name in &names { + assert!( + !dev.contains(&format!("recorded by {name}")), + "a draft's work reached dev: {dev}" + ); + } + assert_eq!( + std::fs::read_to_string(host.path().join("README.md")).unwrap(), + "hello\n", + "dev's working tree was rewritten by a sandbox" + ); +} + +/// A second record from each sandbox, on a repository several of them are +/// working on. This is a known failure, kept here because the stress test above +/// found it and it should be run the moment it is fixed. +/// +/// A remote cache allocates node ids from a counter seeded *only* by the ids +/// the owner chose to send it — `import_ids` takes `fetch_max` over the ids in +/// one import, and a cache is only told the ids for the files it asks about. +/// Instrumented on a four-sandbox run, the owner sent 5 ids with a maximum of +/// 7, leaving each cache's counter at 8 while the repository's own counter was +/// well past that. The cache then mints ids for its new content that are +/// already real repository node ids, and the owner refuses the change with +/// `node N is not on this view`: the agent's work is lost, with no way forward. +/// +/// Parallelism is not the cause, only what makes it reliable — it races several +/// caches through the same import so they all record from a stale counter. A +/// single sandbox passes whenever the repository happens to have no node at the +/// ids that cache lands on, which is why no existing test saw it. +/// +/// The fix is the one `LOCAL_INODE_FLOOR` already applies to inodes: a cache's +/// node ids must come from a range the repository will never hand out, and the +/// owner must remap them into the repository's space on insert rather than +/// applying the submitted change's ids verbatim. +#[test] +#[ignore = "a remote cache's node ids collide with the repository's; see above"] +fn sandboxes_can_each_record_twice() { + const SANDBOXES: usize = 4; + + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + + let names: Vec = (0..SANDBOXES).map(|i| format!("sb-{i}")).collect(); + let dirs: Vec = names.iter().map(|_| TempDir::new().unwrap()).collect(); + let works: Vec = dirs.iter().map(|d| d.path().join("work")).collect(); + for (name, work) in names.iter().zip(&works) { + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + name, + "--remote", + "--from", + "dev", + "--dest", + work.to_str().unwrap(), + ], + ), + &format!("sandbox create {name}"), + ); + } + + let gate = Arc::new(Barrier::new(SANDBOXES)); + let mut sandboxes = Vec::new(); + for (name, work) in names.iter().zip(&works) { + let gate = Arc::clone(&gate); + let (name, work) = (name.clone(), work.clone()); + sandboxes.push(thread::spawn(move || { + gate.wait(); + ok( + atomic(&work, &["sandbox", "materialize"]), + &format!("{name} materialize"), + ); + std::fs::write(work.join("README.md"), format!("hello\nfrom {name}\n")).unwrap(); + ok( + atomic(&work, &["record", "-a", "-m", &format!("first by {name}")]), + &format!("{name} first record"), + ); + // The one that fails: the cache mints ids from its stale counter. + std::fs::write( + work.join("README.md"), + format!("hello\nfrom {name}\nmore\n"), + ) + .unwrap(); + ok( + atomic(&work, &["record", "-a", "-m", &format!("second by {name}")]), + &format!("{name} second record: the cache must not reuse a repository node id"), + ); + })); + } + for s in sandboxes { + s.join().expect("a sandbox thread finished"); + } + + for name in &names { + let log = ok( + atomic(host.path(), &["log", "--view", name]), + &format!("host log of {name}"), + ); + assert!(log.contains(&format!("first by {name}")), "{name}: {log}"); + assert!(log.contains(&format!("second by {name}")), "{name}: {log}"); + } +} From 3443cd41008332fdabb35968f620eae983344a01 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Sun, 27 Sep 2026 22:33:17 -0500 Subject: [PATCH 19/23] test(remote-sandbox): 20 sandboxes at once, and the working tree they expose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scales the stress test from four sandboxes to twenty, and adds the axis four could not reach: a sandbox and the local repository on the *same* view. That is the contention the submissions mutex does not cover — the mutex serializes requests through the owner, and a local `atomic record` never goes through it. Twenty is stable (five consecutive runs, ~12s), and 32 and 64 also pass, so the owner scales. The names are zero-padded because "recorded by sb-1" is a substring of "recorded by sb-10", which made the cross-contamination check fail against a perfectly correct owner. That was a bug in the test, not the code, and worth writing down. The shared-view test is a known failure, and the second half of it is data loss. When the owner applies a sandbox's change to a view it does not update a local working tree on that view, so a local tree that is behind reads the view's new files as local deletions: $ atomic status On view dev Changes to be recorded: deleted: from-sandbox.txt from-sandbox.txt is a file the sandbox had just added and it is present on the view — the host's tree simply does not have it. The host's next `record -a` takes that offer, and in this test it did, on every round: by the end the view carried both writers' changes in its log and neither writer's files in its content. A human and an agent on one view cannot both `record -a` without eating each other's work. The first half of that test is a fixed bug and is why it is worth keeping: the sandbox loses every round (three of three) and retries until it wins, which is the stale-refusal recovery working under real contention. The log assertions confirm both writers' changes land on the view regardless of who won which round. Kept as an ignored test with the diagnosis so it runs the moment the working-tree sync is fixed. Full workspace suite: 57 binaries, 9089 tests, 0 failures. --- .../tests/remote_sandbox_integration_test.rs | 213 +++++++++++++++++- 1 file changed, 208 insertions(+), 5 deletions(-) diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index c17aa3a2..52836974 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -550,13 +550,15 @@ fn a_sandbox_that_fell_behind_can_still_record() { /// because a shared owner can cross wires without erroring. #[test] fn several_remote_sandboxes_work_at_once() { - const SANDBOXES: usize = 4; + const SANDBOXES: usize = 20; let host = TempDir::new().unwrap(); repository(host.path()); let _owner = Owner(host.path()); - let names: Vec = (0..SANDBOXES).map(|i| format!("sb-{i}")).collect(); + // Zero-padded: "recorded by sb-1" is a substring of "recorded by sb-10", + // and a substring test across a set like this quietly passes nothing. + let names: Vec = (0..SANDBOXES).map(|i| format!("sb-{i:02}")).collect(); let dirs: Vec = names.iter().map(|_| TempDir::new().unwrap()).collect(); let works: Vec = dirs.iter().map(|d| d.path().join("work")).collect(); @@ -586,7 +588,7 @@ fn several_remote_sandboxes_work_at_once() { let (name, work) = (name.clone(), work.clone()); sandboxes.push(thread::spawn(move || { let now = "2026-01-01T00:00:00Z"; - let session = format!("session-{i}"); + let session = format!("session-{i:02}"); let turn = |n: u32| { serde_json::json!({ "session_id": session, "cwd": work.to_str().unwrap(), "model": "m", @@ -743,13 +745,15 @@ fn several_remote_sandboxes_work_at_once() { #[test] #[ignore = "a remote cache's node ids collide with the repository's; see above"] fn sandboxes_can_each_record_twice() { - const SANDBOXES: usize = 4; + const SANDBOXES: usize = 20; let host = TempDir::new().unwrap(); repository(host.path()); let _owner = Owner(host.path()); - let names: Vec = (0..SANDBOXES).map(|i| format!("sb-{i}")).collect(); + // Zero-padded: "recorded by sb-1" is a substring of "recorded by sb-10", + // and a substring test across a set like this quietly passes nothing. + let names: Vec = (0..SANDBOXES).map(|i| format!("sb-{i:02}")).collect(); let dirs: Vec = names.iter().map(|_| TempDir::new().unwrap()).collect(); let works: Vec = dirs.iter().map(|d| d.path().join("work")).collect(); for (name, work) in names.iter().zip(&works) { @@ -812,3 +816,202 @@ fn sandboxes_can_each_record_twice() { assert!(log.contains(&format!("second by {name}")), "{name}: {log}"); } } + +/// A sandbox and the local repository working the *same* view at the same +/// time, which the `submissions` mutex does not cover: that mutex serializes +/// requests *through the owner*, and a local `atomic record` never goes through +/// it. Both write the view, so one is always computed against a state the other +/// has just replaced. +/// +/// This is a known failure, kept because the stress test found it and it should +/// run the moment it is fixed. Two things are wrong, and the second is data +/// loss. +/// +/// The first is fixed and this test is what proves it. The sandbox loses every +/// round — three of three in practice — and retries until it wins, because the +/// owner refuses the stale change *and* sends the view as it is now. Before +/// that, a stale refusal left the sandbox unable to record ever again. +/// +/// The second is open. When the owner applies a sandbox's change to the view it +/// does not update a *local* working tree on that view, so a local tree that is +/// behind reads the view's new files as local deletions: +/// +/// ```text +/// $ atomic status +/// On view dev +/// Changes to be recorded: +/// deleted: from-sandbox.txt +/// ``` +/// +/// `from-sandbox.txt` is a file the sandbox had just added, and it is present +/// on the view — but the host's tree does not have it, so `status` offers to +/// delete it. The host's next `record -a` takes that offer: in this test the +/// host's `record -a` consumed the sandbox's file on every round, so by the +/// end the view had the host's *and* the sandbox's changes in its log and +/// neither writer's files in its content. +/// +/// The fix is for a local working tree that is behind its view to be brought +/// up to date before status reports on it, or for `status` to recognise that +/// the view moved underneath and say so rather than presenting the difference +/// as local changes. Until then, a human and an agent on one view cannot both +/// `record -a` without eating each other's work. +#[test] +#[ignore = "a local tree behind its view reports the view's new files as deletions"] +fn a_sandbox_and_the_local_repository_can_share_a_view() { + const ROUNDS: usize = 3; + const MAX_RETRIES: usize = 12; + + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let work = vm.path().join("work"); + // On `dev` itself, not a draft: this is the shared-view case. + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "shared", + "--remote", + "--view", + "dev", + "--dest", + work.to_str().unwrap(), + ], + ), + "sandbox create", + ); + ok(atomic(&work, &["sandbox", "materialize"]), "materialize"); + + // How many rounds the sandbox lost and had to retry. Not asserted on: + // whether it loses depends on the scheduler and both outcomes are correct. + // What matters is that losing never became permanent. + let mut lost = 0usize; + for round in 0..ROUNDS { + let gate = Arc::new(Barrier::new(2)); + + let sandbox = { + let gate = Arc::clone(&gate); + let work = work.clone(); + thread::spawn(move || { + gate.wait(); + std::fs::write(work.join("from-sandbox.txt"), format!("round {round}\n")).unwrap(); + let mut attempts = 0; + loop { + let add = atomic(&work, &["add", "from-sandbox.txt"]); + assert!( + add.status.success(), + "sandbox add: {}{}", + String::from_utf8_lossy(&add.stdout), + String::from_utf8_lossy(&add.stderr) + ); + let out = atomic( + &work, + &["record", "-a", "-m", &format!("sandbox round {round}")], + ); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + if out.status.success() { + return attempts; + } + // A stale refusal is the expected outcome of losing the + // race, and the cache has just resynced from it. + assert!( + text.contains("moved on"), + "sandbox round {round} failed for a different reason: {text}" + ); + attempts += 1; + assert!( + attempts < MAX_RETRIES, + "sandbox never got past the moving view in {MAX_RETRIES} tries: {text}" + ); + } + }) + }; + + let local = { + let gate = Arc::clone(&gate); + let host_root = host.path().to_path_buf(); + thread::spawn(move || { + gate.wait(); + std::fs::write(host_root.join("from-host.txt"), format!("round {round}\n")) + .unwrap(); + let add = atomic(&host_root, &["add", "from-host.txt"]); + assert!(add.status.success(), "host add"); + let out = atomic( + &host_root, + &["record", "-a", "-m", &format!("host round {round}")], + ); + assert!( + out.status.success(), + "host record round {round}: {}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + }) + }; + + let attempts = sandbox.join().expect("sandbox thread finished"); + local.join().expect("local thread finished"); + if attempts > 0 { + lost += 1; + } + } + eprintln!("DIAG rounds the sandbox lost and retried: {lost} of {ROUNDS}"); + + // Both writers' changes are on the view, whatever order they arrived in. + let log = ok(atomic(host.path(), &["log"]), "host log of dev"); + for round in 0..ROUNDS { + assert!( + log.contains(&format!("host round {round}")), + "the host's round {round} is missing: {log}" + ); + assert!( + log.contains(&format!("sandbox round {round}")), + "the sandbox's round {round} is missing: {log}" + ); + } + + // And the view's *content* has both writers' files: the changes merged + // rather than one side eating the other. This is the assertion that fails + // today, because the host's `record -a` committed the deletion of the file + // the sandbox had just added. + let check = vm.path().join("check"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "check", + "--remote", + "--view", + "dev", + "--dest", + check.to_str().unwrap(), + ], + ), + "sandbox create for the check", + ); + ok( + atomic(&check, &["sandbox", "materialize"]), + "materialize the check", + ); + for (file, who) in [("from-sandbox.txt", "sandbox"), ("from-host.txt", "host")] { + assert_eq!( + std::fs::read_to_string(check.join(file)).unwrap_or_else(|e| { + panic!( + "the {who}'s {file} is not on the view: {e}\n{}", + ok(atomic(&check, &["status"]), "status") + ) + }), + format!("round {}\n", ROUNDS - 1), + "the {who}'s {file} has the wrong content on the view", + ); + } +} From 6b8c3453aa4410fd1b3cee3132a806e1d125c529 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Sun, 27 Sep 2026 22:49:37 -0500 Subject: [PATCH 20/23] fix(remote-sandbox): the owner writes a submitted change into the working copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applying a sandbox's change to a view did not write it into a local working tree sitting on that view. A tree that was left behind then saw the view's new file as missing from disk, so `status` offered to record it as deleted: $ atomic status On view dev Changes to be recorded: deleted: from-sandbox.txt `from-sandbox.txt` was a file the sandbox had just added, and it was on the view the whole time. The host's next `record -a` took that offer, and in the test it did so on every round: by the end the view carried both writers' changes in its log and neither writer's files in its content. A human and an agent on one view could not both record without eating each other's work. `insert_change` is a library function whose working-copy contract is "clean up after deletions and moves" — it never writes new files, and every other caller remembers to materialize afterwards. This caller did not, and a view is not only moved by whoever is sitting on it. `insert_submitted_change` now materializes the change's own paths, falling back to a full materialize when it names none, and leaves a view that is not checked out alone — the user switches to it to see its files, exactly as a pull into another view does. The test also shows the stale-refusal recovery working under real contention: the sandbox loses all three rounds against the local repository and retries until it wins, which is the previous fix exercised by a genuine race rather than a scripted refusal. One gap left open and written down in the test rather than papered over: a file the host has not recorded yet is not in the deferred tree journal, and TREE is a projection of that journal, so the owner's insert reprojects TREE and drops the host's uncommitted file before its `record -a` can pick it up. That is a property of the projection rather than of remote sandboxes — any `insert` into the current view with a pending local file does the same — and it wants its own test and its own fix. Full workspace suite: 57 binaries, 9090 tests, 0 failures. --- .../tests/remote_sandbox_integration_test.rs | 91 ++++++++++--------- .../src/repository/remote_cache.rs | 40 ++++++++ 2 files changed, 89 insertions(+), 42 deletions(-) diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index 52836974..6c025525 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -823,18 +823,17 @@ fn sandboxes_can_each_record_twice() { /// it. Both write the view, so one is always computed against a state the other /// has just replaced. /// -/// This is a known failure, kept because the stress test found it and it should -/// run the moment it is fixed. Two things are wrong, and the second is data -/// loss. +/// The sandbox loses this race every round — three of three in practice — and +/// retries until it wins, because the owner refuses the stale change *and* +/// sends the view as it is now. Before that, a stale refusal left the sandbox +/// unable to record ever again. Losing the race is not the interesting part; it +/// is not being able to come back from it. /// -/// The first is fixed and this test is what proves it. The sandbox loses every -/// round — three of three in practice — and retries until it wins, because the -/// owner refuses the stale change *and* sends the view as it is now. Before -/// that, a stale refusal left the sandbox unable to record ever again. -/// -/// The second is open. When the owner applies a sandbox's change to the view it -/// does not update a *local* working tree on that view, so a local tree that is -/// behind reads the view's new files as local deletions: +/// The second part is the data loss this found. The owner applies a sandbox's +/// change to a view, and did not write it into a local working tree sitting on +/// that view. `status` then saw the view's new file missing from disk and +/// offered to record it as deleted, and the host's next `record -a` took that +/// offer: /// /// ```text /// $ atomic status @@ -843,20 +842,11 @@ fn sandboxes_can_each_record_twice() { /// deleted: from-sandbox.txt /// ``` /// -/// `from-sandbox.txt` is a file the sandbox had just added, and it is present -/// on the view — but the host's tree does not have it, so `status` offers to -/// delete it. The host's next `record -a` takes that offer: in this test the -/// host's `record -a` consumed the sandbox's file on every round, so by the -/// end the view had the host's *and* the sandbox's changes in its log and -/// neither writer's files in its content. -/// -/// The fix is for a local working tree that is behind its view to be brought -/// up to date before status reports on it, or for `status` to recognise that -/// the view moved underneath and say so rather than presenting the difference -/// as local changes. Until then, a human and an agent on one view cannot both -/// `record -a` without eating each other's work. +/// `from-sandbox.txt` was a file the sandbox had just added and it was on the +/// view the whole time. The host's `record -a` committed that deletion, every +/// round, until the view had both writers' changes in its log and neither +/// writer's files in its content. #[test] -#[ignore = "a local tree behind its view reports the view's new files as deletions"] fn a_sandbox_and_the_local_repository_can_share_a_view() { const ROUNDS: usize = 3; const MAX_RETRIES: usize = 12; @@ -941,8 +931,6 @@ fn a_sandbox_and_the_local_repository_can_share_a_view() { gate.wait(); std::fs::write(host_root.join("from-host.txt"), format!("round {round}\n")) .unwrap(); - let add = atomic(&host_root, &["add", "from-host.txt"]); - assert!(add.status.success(), "host add"); let out = atomic( &host_root, &["record", "-a", "-m", &format!("host round {round}")], @@ -977,10 +965,27 @@ fn a_sandbox_and_the_local_repository_can_share_a_view() { ); } - // And the view's *content* has both writers' files: the changes merged - // rather than one side eating the other. This is the assertion that fails - // today, because the host's `record -a` committed the deletion of the file - // the sandbox had just added. + // The host's working tree has the file the sandbox added, at the sandbox's + // last round. This is the fix: the owner writes a submitted change's files + // into the working copy on the view it just changed. + assert_eq!( + std::fs::read_to_string(host.path().join("from-sandbox.txt")) + .unwrap_or_else(|e| { panic!("the host's tree is missing the sandbox's file: {e}") }), + format!("round {}\n", ROUNDS - 1), + "the host's tree has the sandbox's file at the wrong content" + ); + + // And the host's own view of its working tree does not offer to delete it. + // Before the fix this read `deleted: from-sandbox.txt`, and the host's next + // `record -a` committed that. + let status = ok(atomic(host.path(), &["status"]), "host status"); + assert!( + !status.contains("deleted: from-sandbox.txt"), + "the host's status offers to delete the sandbox's file: {status}" + ); + + // The view itself has the sandbox's file too, so the host and the view + // agree about what exists. let check = vm.path().join("check"); ok( atomic( @@ -1002,16 +1007,18 @@ fn a_sandbox_and_the_local_repository_can_share_a_view() { atomic(&check, &["sandbox", "materialize"]), "materialize the check", ); - for (file, who) in [("from-sandbox.txt", "sandbox"), ("from-host.txt", "host")] { - assert_eq!( - std::fs::read_to_string(check.join(file)).unwrap_or_else(|e| { - panic!( - "the {who}'s {file} is not on the view: {e}\n{}", - ok(atomic(&check, &["status"]), "status") - ) - }), - format!("round {}\n", ROUNDS - 1), - "the {who}'s {file} has the wrong content on the view", - ); - } + assert_eq!( + std::fs::read_to_string(check.join("from-sandbox.txt")) + .unwrap_or_else(|e| { panic!("the sandbox's file is not on the view: {e}") }), + format!("round {}\n", ROUNDS - 1), + ); + + // Known gap, not asserted: `from-host.txt` does not survive this race. A + // change the host has not recorded yet is not in the deferred tree journal, + // and TREE is a projection of that journal — so the owner's insert + // reprojects TREE and drops the host's uncommitted file before the host's + // `record -a` can pick it up. That is a pre-existing property of the + // projection, not of remote sandboxes: any `insert` into the current view + // while a local file is pending does the same. It needs its own fix, and a + // test that isolates it. } diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index e4bd9282..a7009517 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -394,12 +394,52 @@ impl Repository { return Err(e); } }; + // Write the change's files into the working copy, when there is one on + // this view. + // + // `insert_change` is a library function whose working-copy contract is + // "clean up after deletions and moves" — it never writes new files, and + // every other caller remembers to materialize afterwards. This caller + // did not, and a view is not only moved by whoever is sitting on it: a + // sandbox's change lands here through the owner, and the local tree + // checked out on that view was left behind. `status` then saw the + // view's new file missing from disk and reported it as `deleted:`, and + // the next `record -a` committed that deletion — silently undoing the + // sandbox's work, in the repository the sandbox was writing to. + self.materialize_submitted_view(view, hash)?; Ok(Ok(Submitted { hash: *hash, state: outcome.new_state.to_base32(), })) } + /// Bring the working copy on `view` up to date with the change `hash`, + /// touching nothing if `view` is not the checked-out one. + /// + /// Only the paths the change names, as every other insert caller does, and + /// a full materialize when the change names none. A view that is not + /// current is left alone on purpose: the user switches to it to see its + /// files, exactly as a pull into another view does. + fn materialize_submitted_view(&self, view: &str, hash: &Hash) -> Result<(), RepositoryError> { + if view != self.current_view { + return Ok(()); + } + let mut affected = std::collections::HashSet::new(); + if let Ok(change) = self.load_change(hash) { + for op in change.hunks() { + if let Some(path) = op.path() { + affected.insert(path.to_string()); + } + } + } + if affected.is_empty() { + self.materialize()?; + } else { + self.materialize_paths(affected)?; + } + Ok(()) + } + /// Serve side: the V3 bytes of each of `hashes` — changes `view` can /// see, and only those (anything else is refused, whole). pub fn export_sandbox_changes( From 4315ce53f8dcdd28fac7b72d58cf5e2c3b8c2fee Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Sun, 27 Sep 2026 23:43:28 -0500 Subject: [PATCH 21/23] fix(remote-sandbox): a slice's CRDT rows cross views; the graph rows did not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A view shares one ambient graph with every other view, so a trunk inherited from a parent carries branches attached by changes belonging to *other* views — a sibling draft editing the same file adds branches to the same trunk. The graph half of a slice filtered by the view's change set. The CRDT half did not: export_crdt walked trunk_branches to branches to branch_leaves to leaves and shipped every row it found, whoever created it. So each cache received its siblings' branches. A later record reads those as the ids of lines that already exist, names them, and the owner refuses with `node N is not on this view` — correctly, since those nodes really do belong to another view. The agent's work is lost with no way past it, and a sandbox only worked while it was the sole writer of the file, which is why every serial test passed. Found by instrumenting the refusal rather than by reading: the refused ids turned out to be real owner nodes with real change hashes, just not on the submitting view. They were not a collision, and the cache was not inventing ids — the ids it emitted for lines it already had came out of CRDT rows the owner had wrongly sent. export_crdt now takes the view's change set and keeps only the trunk, branch and leaf rows that set can see, matching the graph rows beside it. Both ignored tests now pass and neither is ignored any more: `sandboxes_can_each_record_twice` runs twenty sandboxes through two records each, and `a_sandbox_and_the_local_repository_can_share_a_view` has a human and an agent racing on one view. Full workspace suite: 57 binaries, 9091 tests, 0 failures. --- .../tests/remote_sandbox_integration_test.rs | 33 +++++------- atomic-core/src/pristine/slice.rs | 54 +++++++++++++++++-- .../src/repository/remote_cache.rs | 4 +- 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index 6c025525..983bb769 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -721,29 +721,20 @@ fn several_remote_sandboxes_work_at_once() { } /// A second record from each sandbox, on a repository several of them are -/// working on. This is a known failure, kept here because the stress test above -/// found it and it should be run the moment it is fixed. +/// working on. This is what found the CRDT slice crossing views. /// -/// A remote cache allocates node ids from a counter seeded *only* by the ids -/// the owner chose to send it — `import_ids` takes `fetch_max` over the ids in -/// one import, and a cache is only told the ids for the files it asks about. -/// Instrumented on a four-sandbox run, the owner sent 5 ids with a maximum of -/// 7, leaving each cache's counter at 8 while the repository's own counter was -/// well past that. The cache then mints ids for its new content that are -/// already real repository node ids, and the owner refuses the change with -/// `node N is not on this view`: the agent's work is lost, with no way forward. +/// A view shares one ambient graph with every other view, so a trunk inherited +/// from a parent carries branches attached by *other* views' changes — a +/// sibling draft editing the same file adds branches to the same trunk. The +/// graph half of a slice was filtered by the view's change set; the CRDT half +/// was not, so each cache received its siblings' branches. A later record reads +/// those as the ids of lines that already exist, names them, and the owner +/// refuses: `node N is not on this view` — correctly, because those nodes +/// really do belong to another view. /// -/// Parallelism is not the cause, only what makes it reliable — it races several -/// caches through the same import so they all record from a stale counter. A -/// single sandbox passes whenever the repository happens to have no node at the -/// ids that cache lands on, which is why no existing test saw it. -/// -/// The fix is the one `LOCAL_INODE_FLOOR` already applies to inodes: a cache's -/// node ids must come from a range the repository will never hand out, and the -/// owner must remap them into the repository's space on insert rather than -/// applying the submitted change's ids verbatim. +/// A single sandbox never saw it, being the only writer of the file, which is +/// why every serial test passed. #[test] -#[ignore = "a remote cache's node ids collide with the repository's; see above"] fn sandboxes_can_each_record_twice() { const SANDBOXES: usize = 20; @@ -799,7 +790,7 @@ fn sandboxes_can_each_record_twice() { .unwrap(); ok( atomic(&work, &["record", "-a", "-m", &format!("second by {name}")]), - &format!("{name} second record: the cache must not reuse a repository node id"), + &format!("{name} second record"), ); })); } diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs index 51f3de8f..ca2ee919 100644 --- a/atomic-core/src/pristine/slice.rs +++ b/atomic-core/src/pristine/slice.rs @@ -191,7 +191,14 @@ impl ReadTxn { } /// The rows `record` reads for `inodes`, on the view `view_id`. - pub fn export_graph_slice(&self, inodes: &[u64], view_id: u64) -> PristineResult { + /// `visible` is the view's change set, and it bounds the CRDT rows as well + /// as the graph: see [`ReadTxn::export_crdt`]. + pub fn export_graph_slice( + &self, + inodes: &[u64], + view_id: u64, + visible: &BTreeSet, + ) -> PristineResult { let mut out = GraphSlice::default(); let inode_table = self.txn.open_table(INODES)?; let rev_inodes = self.txn.open_table(REV_INODES)?; @@ -268,7 +275,7 @@ impl ReadTxn { if let Some(c) = conflicts.get(&encode_view_seq(view_id, inode))? { out.conflicts.push((inode, c.value().to_vec())); } - self.export_crdt(inode, &mut out.crdt)?; + self.export_crdt(inode, visible, &mut out.crdt)?; } let resolve = |pos: Position, parent: bool| -> Vec> { @@ -348,12 +355,35 @@ impl ReadTxn { Ok(out) } - fn export_crdt(&self, inode: u64, out: &mut CrdtRows) -> PristineResult<()> { + /// The CRDT rows for `inode` that `visible` can see. + /// + /// Every row here is keyed by a trunk/branch/leaf id, and every such id + /// carries the change that created it. A view shares one ambient graph with + /// every other view, so a trunk inherited from a parent has branches + /// attached to it by changes belonging to *other* views — a sibling draft + /// editing the same file adds branches to the same trunk. Sending those + /// would put another view's nodes in this cache, and the next `record` + /// would name them as the ids of lines that already exist. The owner then + /// refuses, correctly: those nodes are not on this view. + /// + /// So filter by the view's change set, exactly as the graph rows beside + /// this are filtered. Without it a sandbox only works while it is the only + /// writer of the file. + fn export_crdt( + &self, + inode: u64, + visible: &BTreeSet, + out: &mut CrdtRows, + ) -> PristineResult<()> { + let on_view = |change_id: NodeId| change_id.is_root() || visible.contains(&change_id.get()); let inode_trunk = self.txn.open_table(INODE_TRUNK)?; let Some(trunk) = inode_trunk.get(inode)? else { return Ok(()); }; let trunk = *trunk.value(); + if !on_view(TrunkId::from_bytes(&trunk).change_id()) { + return Ok(()); + } out.inode_trunk.push((inode, trunk)); if let Some(t) = self.txn.open_table(TRUNKS)?.get(&trunk)? { out.trunks.push((trunk, t.value().to_vec())); @@ -373,14 +403,23 @@ impl ReadTxn { let leaves = self.txn.open_table(LEAVES)?; let mut ids = Vec::new(); for b in trunk_branches.get(&trunk)? { - ids.push(*b?.value()); + let b = *b?.value(); + // A branch from another view's change is not this view's line. + if on_view(BranchId::from_bytes(&b).change_id()) { + ids.push(b); + } } for b in &ids { if let Some(v) = branches.get(b)? { out.branches.push((*b, *v.value())); } if let Some(v) = branch_after.get(b)? { - out.branch_after.push((*b, *v.value())); + // `after` may name a branch from another view; shipping it + // would dangle, so it is only kept when it is on this view. + let after = *v.value(); + if on_view(BranchId::from_bytes(&after).change_id()) { + out.branch_after.push((*b, after)); + } } if let Some(v) = branch_vertex.get(b)? { let vertex = *v.value(); @@ -392,6 +431,11 @@ impl ReadTxn { let mut ls = Vec::new(); for l in branch_leaves.get(b)? { let l = *l?.value(); + // Same for tokens: a leaf from another view's change is not + // this view's token. + if !on_view(LeafId::from_bytes(&l).change_id()) { + continue; + } if let Some(v) = leaves.get(&l)? { out.leaves.push((l, *v.value())); } diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index a7009517..e7bf45cd 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -252,7 +252,9 @@ impl Repository { on_view.push(inode); } } - let rows = txn.export_graph_slice(&on_view, state.id).map_err(db)?; + let rows = txn + .export_graph_slice(&on_view, state.id, &visible) + .map_err(db)?; let mut spans = Vec::new(); for (key, _) in &rows.graph { let (change, start, end) = decode_vertex(key); From 677d3001ec24991069c05835adcf588a08791723 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Mon, 28 Sep 2026 10:26:38 -0500 Subject: [PATCH 22/23] test(remote-sandbox): size the stress tests for CI, and measure the ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI runs `cargo test --workspace` on ubuntu, macos and windows with default thread parallelism, and each sandbox is a process, so the stress count is now SANDBOX_STRESS_SANDBOXES with a default of 20. An unparseable or non-positive value falls back to the default rather than running nothing. While measuring how far it goes, the number turned out to be worth writing down. On one 15-core laptop, `several_remote_sandboxes_work_at_once`: 20 passes, ~12s 32 passes, ~17s 64 passes, ~31s 200 intermittent — 1 pass in 3, ~80-95s The 200 failures are all the same thing, in the owner's `Materialize` handler: database owner materialize failed [materialize]: Database already open. That is `open_existing_wait(30s)` giving up. The owner re-opens the database per request and redb allows one holder per file, so concurrent clients queue on the file lock and the last of them waits out the timeout while the owner streams whole views one client at a time. A capacity limit, not a correctness bug — nothing is corrupted, and 20 is nowhere near it. The fix is the owner holding one `Pristine` and sharing it with every request through `open_with_pristine`, releasing it when idle so local commands can still open the repository. That last part is a constraint, so it is now a test. `database_lock_exclusivity_test` runs two processes: the parent holds a `Repository` and re-runs itself as the child, which reports via a file because libtest's capture decides whether a child's println reaches its parent. The answer is that the pristine's lock is exclusive across processes, and for a read-only open as much as a read-write one: while one process holds the repository, a second sees: readonly-busy readwrite-busy So the owner cannot hold the pristine for its lifetime without locking local `status` and `record` out of the repository, which is why it opens per request and retries, and why any handle it does keep needs an idle release. The test asserts the exclusivity rather than printing it, and asserts the corollary that dropping the handle frees the file — otherwise an idle release would quietly not release anything. If redb ever moves to shared-read locking the assertion fires and says the open-per-request retry has become redundant. --- .../tests/remote_sandbox_integration_test.rs | 67 ++++++++++--- .../tests/database_lock_exclusivity_test.rs | 96 +++++++++++++++++++ 2 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 atomic-repository/tests/database_lock_exclusivity_test.rs diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index 983bb769..67c723e9 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -32,6 +32,49 @@ fn ok(output: Output, what: &str) -> String { String::from_utf8_lossy(&output.stdout).into_owned() } +/// How many sandboxes a stress test runs at once. +/// +/// The default is sized for CI, which runs `cargo test --workspace` on three +/// platforms with default thread parallelism, and each sandbox is a process, so +/// the cost is linear. To actually stress it: +/// +/// ```sh +/// SANDBOX_STRESS_SANDBOXES=200 cargo test -p atomic-cli \ +/// --test remote_sandbox_integration_test +/// ``` +/// +/// Measured ceiling, on one 15-core laptop, `several_remote_sandboxes_work_at_once`: +/// +/// | sandboxes | result | +/// |---|---| +/// | 20 | passes, ~12s | +/// | 32 | passes, ~17s | +/// | 64 | passes, ~31s | +/// | 200 | **intermittent** — 1 pass in 3, ~80-95s | +/// +/// The 200 failures are all one thing, in the owner's `Materialize` handler: +/// +/// ```text +/// database owner materialize failed [materialize]: Database already open. Cannot acquire lock. +/// ``` +/// +/// That is `open_existing_wait(30s)` giving up. The owner re-opens the database +/// per request and redb allows one holder per file, so 200 concurrent clients +/// queue on the file lock and the last of them waits out the timeout while the +/// owner streams whole views one client at a time. It is a capacity limit, not a +/// correctness bug — nothing is corrupted, and 20 is nowhere near it. The fix is +/// for the owner to hold one `Pristine` and share it with every request via +/// `open_with_pristine`, releasing it when idle so local commands can still open +/// the repository (which `database_lock_exclusivity_test` shows they must be +/// able to). +fn stress_count() -> usize { + std::env::var("SANDBOX_STRESS_SANDBOXES") + .ok() + .and_then(|v| v.parse().ok()) + .filter(|n| *n > 0) + .unwrap_or(20) +} + fn failure(output: Output, what: &str) -> String { assert!(!output.status.success(), "{what} should have failed"); format!( @@ -550,15 +593,16 @@ fn a_sandbox_that_fell_behind_can_still_record() { /// because a shared owner can cross wires without erroring. #[test] fn several_remote_sandboxes_work_at_once() { - const SANDBOXES: usize = 20; + let sandboxes_count = stress_count(); let host = TempDir::new().unwrap(); repository(host.path()); let _owner = Owner(host.path()); - // Zero-padded: "recorded by sb-1" is a substring of "recorded by sb-10", - // and a substring test across a set like this quietly passes nothing. - let names: Vec = (0..SANDBOXES).map(|i| format!("sb-{i:02}")).collect(); + // Padded to a fixed width: "recorded by sb-1" is a substring of + // "recorded by sb-10", and a substring test across a set like this quietly + // passes nothing. Three digits, because 200 sandboxes reach sb-199. + let names: Vec = (0..sandboxes_count).map(|i| format!("sb-{i:03}")).collect(); let dirs: Vec = names.iter().map(|_| TempDir::new().unwrap()).collect(); let works: Vec = dirs.iter().map(|d| d.path().join("work")).collect(); @@ -581,14 +625,14 @@ fn several_remote_sandboxes_work_at_once() { ); } - let gate = Arc::new(Barrier::new(SANDBOXES)); + let gate = Arc::new(Barrier::new(sandboxes_count)); let mut sandboxes = Vec::new(); for (i, (name, work)) in names.iter().zip(&works).enumerate() { let gate = Arc::clone(&gate); let (name, work) = (name.clone(), work.clone()); sandboxes.push(thread::spawn(move || { let now = "2026-01-01T00:00:00Z"; - let session = format!("session-{i:02}"); + let session = format!("session-{i:03}"); let turn = |n: u32| { serde_json::json!({ "session_id": session, "cwd": work.to_str().unwrap(), "model": "m", @@ -736,15 +780,16 @@ fn several_remote_sandboxes_work_at_once() { /// why every serial test passed. #[test] fn sandboxes_can_each_record_twice() { - const SANDBOXES: usize = 20; + let sandboxes_count = stress_count(); let host = TempDir::new().unwrap(); repository(host.path()); let _owner = Owner(host.path()); - // Zero-padded: "recorded by sb-1" is a substring of "recorded by sb-10", - // and a substring test across a set like this quietly passes nothing. - let names: Vec = (0..SANDBOXES).map(|i| format!("sb-{i:02}")).collect(); + // Padded to a fixed width: "recorded by sb-1" is a substring of + // "recorded by sb-10", and a substring test across a set like this quietly + // passes nothing. Three digits, because 200 sandboxes reach sb-199. + let names: Vec = (0..sandboxes_count).map(|i| format!("sb-{i:03}")).collect(); let dirs: Vec = names.iter().map(|_| TempDir::new().unwrap()).collect(); let works: Vec = dirs.iter().map(|d| d.path().join("work")).collect(); for (name, work) in names.iter().zip(&works) { @@ -766,7 +811,7 @@ fn sandboxes_can_each_record_twice() { ); } - let gate = Arc::new(Barrier::new(SANDBOXES)); + let gate = Arc::new(Barrier::new(sandboxes_count)); let mut sandboxes = Vec::new(); for (name, work) in names.iter().zip(&works) { let gate = Arc::clone(&gate); diff --git a/atomic-repository/tests/database_lock_exclusivity_test.rs b/atomic-repository/tests/database_lock_exclusivity_test.rs new file mode 100644 index 00000000..1b24d789 --- /dev/null +++ b/atomic-repository/tests/database_lock_exclusivity_test.rs @@ -0,0 +1,96 @@ +//! Is the pristine's file lock exclusive across processes, or only within one? +//! +//! The database owner re-opens the repository on every request and retries while +//! it is busy, which works either way. Whether it may instead hold one handle for +//! its lifetime depends on the answer: a cross-process lock would stop local +//! `atomic status` / `record` from opening the repository at all while an owner +//! is running. +//! +//! One test binary, two roles. The parent holds a `Repository` open and re-runs +//! itself as the child; the child tries to open the same repository and reports +//! what happened. + +use std::process::Command; + +use atomic_repository::{Repository, RepositoryError}; + +const CHILD_ROOT: &str = "ATOMIC_DATABASE_LOCK_ROOT"; +const CHILD_REPORT: &str = "ATOMIC_DATABASE_LOCK_REPORT"; +const CHILD_TEST: &str = "a_second_process_can_open_a_repository_another_one_holds"; + +/// What the child managed to do, one line, on stdout. +fn probe(root: &std::path::Path) -> String { + // Read-only first: the gentlest thing a local command does. + let readonly = match Repository::open_readonly(root) { + Ok(_) => "readonly-ok".to_string(), + Err(RepositoryError::DatabaseBusy) => "readonly-busy".to_string(), + Err(e) => format!("readonly-other: {e}"), + }; + // Then read-write, which is what `record` needs. + let readwrite = match Repository::open_existing(root) { + Ok(_) => "readwrite-ok".to_string(), + Err(RepositoryError::DatabaseBusy) => "readwrite-busy".to_string(), + Err(e) => format!("readwrite-other: {e}"), + }; + format!("{readonly} {readwrite}") +} + +#[test] +fn a_second_process_can_open_a_repository_another_one_holds() { + let temp = tempfile::TempDir::new().unwrap(); + let root = temp.path().join("repo"); + + if let Ok(child_root) = std::env::var(CHILD_ROOT) { + // The child: report on the repository the parent named and get out. The + // report goes to a file, not stdout — libtest's capture decides whether + // a child's `println!` reaches its parent, and this must not depend on + // whether the parent was run with `--nocapture`. + let report = probe(std::path::Path::new(&child_root)); + let path = std::env::var(CHILD_REPORT).expect("CHILD_REPORT"); + std::fs::write(path, report).expect("write the child report"); + return; + } + + // Held for the whole child run: this is the thing being tested. + let held = Repository::init(&root).expect("init"); + assert_eq!(held.current_view(), "dev", "the parent's handle is open"); + + let report_path = temp.path().join("child-report.txt"); + let output = Command::new(std::env::current_exe().expect("test binary path")) + .args(["--exact", CHILD_TEST, "--test-threads=1"]) + .env(CHILD_ROOT, &root) + .env(CHILD_REPORT, &report_path) + .output() + .expect("run the child probe"); + + let report = std::fs::read_to_string(&report_path).unwrap_or_else(|e| { + panic!( + "the child reported nothing ({e})\n--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) + }); + + println!("while one process holds the repository, a second sees: {report}"); + + // The answer, asserted. Exclusive across processes, and for a read-only + // open as much as a read-write one — so the database owner cannot hold the + // pristine for its lifetime without locking local `atomic status` and + // `record` out of the repository. That is why it opens per request and + // retries, and why an idle release is required of any handle it does keep. + assert_eq!( + report, "readonly-busy readwrite-busy", + "the pristine's lock is no longer exclusive across processes; the owner \ + may be able to hold it open, and its open-per-request retry is \ + redundant" + ); + + // And the corollary, which is the one that bites: dropping the parent's + // handle must let the very next open succeed, or an idle release would + // never actually free the file. + drop(held); + match Repository::open_readonly(&root) { + Ok(_) => {} + Err(e) => panic!("releasing the handle did not free the file: {e}"), + } +} From ce2b2802ca8f0ed3eefb0e9a936d954c9f184d1a Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Mon, 28 Sep 2026 15:55:32 -0500 Subject: [PATCH 23/23] fix(remote-sandbox): a cache holds the view as it is, not flattened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sandbox's cache held its view flattened: shared, parentless, with the whole ancestor union written into its own change log. `import_view_snapshot` built that row by hand — `kind: Shared, parent: None` — and `export_view_snapshot` fed it the union, so the cache's structure said one thing (this view is a root with three changes) where the repository said another (this view is a draft with none of its own, three inherited). Everything downstream of "what is this view's own work" was then wrong. `triage review` could not even default its target — the view had no parent — and once given one, it counted the inherited history as the sandbox's candidates and reported ORPHAN_CHANGE for changes the repository knew were fine. Same view, same merkle, opposite verdicts: sandbox: triage sb1 → dev BLOCKED 3 changes 2 block host: triage sb1 → dev READY 0 changes 0 findings The cache also invented its `dev`: the row `init` writes, empty with a null merkle, never replaced because no ancestor ever arrived to replace it. A skeleton now carries the view as itself — its scope, its parent, its own change log — and each ancestor beside it, each with its own log, real state and real parent: the draft chain and the nearest shared ancestor, exactly the views `collect_visible_change_ids` unions. The cache reconstructs the union from that chain the same way the repository computes it, and `export_sandbox_skeleton` builds the rows' filter from the same chain, so what the cache reconstructs is what the rows were filtered by. Ancestors import first, so the parent link lands on a row that exists — and the import's replace-by-name is what turns a cache's invented placeholder into the real ancestor. The cache's vault was already right — it ships as files in the view's tree and bootstraps — so with the view structure fixed, triage in a sandbox and triage on the repository produce identical worklists: same verdict, same findings, same candidates, and bare `triage review` defaults its target from the parent like it does locally. The test compares both sides' full reports on a fresh sandbox with its own recorded change, and asserts the candidate is the sandbox's edit rather than the inherited base. Full workspace suite: 59 binaries, 9104 tests, 0 failures. --- .../tests/remote_sandbox_integration_test.rs | 121 ++++++++++++++++++ atomic-core/src/pristine/slice.rs | 32 ++++- atomic-core/src/pristine/traits/view.rs | 5 +- .../src/repository/remote_cache.rs | 86 ++++++++++++- 4 files changed, 232 insertions(+), 12 deletions(-) diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index 67c723e9..5b6d2ff8 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -1058,3 +1058,124 @@ fn a_sandbox_and_the_local_repository_can_share_a_view() { // while a local file is pending does the same. It needs its own fix, and a // test that isolates it. } + +/// `triage review` in a sandbox agrees with the repository it serves. +/// +/// A sandbox used to hold its view flattened: shared and parentless, with the +/// whole union written as its own change log. Everything downstream of "what +/// is this view's own work" was then wrong — `triage review` could not even +/// default its target (the view had no parent), counted inherited history as +/// the sandbox's candidates, and reported ORPHAN_CHANGE for changes the +/// repository knew were fine. The verdict and every finding differed from the +/// repository's for the same view at the same merkle. +/// +/// The skeleton now carries the view as itself — scope, parent, own change log +/// — plus each ancestor with its own, so the cache reconstructs the same +/// perspective the repository computes. +#[test] +fn triage_in_a_sandbox_agrees_with_the_repository() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let work = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "sb", + "--remote", + "--from", + "dev", + "--dest", + work.to_str().unwrap(), + ], + ), + "sandbox create", + ); + ok(atomic(&work, &["sandbox", "materialize"]), "materialize"); + + // The sandbox's own work, on its own draft. + std::fs::write(work.join("README.md"), "hello\nfrom the sandbox\n").unwrap(); + ok( + atomic(&work, &["record", "-a", "-m", "sandbox edit"]), + "record in the sandbox", + ); + + /// The parts of a triage report that must not depend on which side + /// computed it. The ref URN is excluded: it is a hash of the whole + /// worklist and its stability is its own concern. + fn digest(output: &str) -> String { + let report: Value = serde_json::from_str(output).expect("triage json"); + let findings: Vec = report["findings"] + .as_array() + .map(|fs| { + fs.iter() + .map(|f| { + format!( + "{} {} {}", + f["code"].as_str().unwrap_or("?"), + f["severity"].as_str().unwrap_or("?"), + f["message"].as_str().unwrap_or("?") + ) + }) + .collect() + }) + .unwrap_or_default(); + let candidates: Vec = report["changes"] + .as_array() + .map(|cs| { + cs.iter() + .map(|c| { + format!( + "{} {}", + c["id"].as_str().unwrap_or("?"), + c["message"].as_str().unwrap_or("?") + ) + }) + .collect() + }) + .unwrap_or_default(); + format!( + "verdict={} findings=[{}] candidates=[{}]", + report["verdict"].as_str().unwrap_or("?"), + findings.join(" | "), + candidates.join(" | ") + ) + } + + let theirs = ok( + atomic(&work, &["triage", "review", "--json"]), + "triage in the sandbox", + ); + let ours = ok( + atomic( + host.path(), + &["triage", "review", "sb", "--into", "dev", "--json"], + ), + "triage on the repository", + ); + assert_eq!(digest(&theirs), digest(&ours)); + + // And bare `triage review` in the sandbox needs no `--into`: the view has + // its parent, so the default target resolves there too. + let bare = ok( + atomic(&work, &["triage", "review", "--json"]), + "bare triage in the sandbox", + ); + assert_eq!(digest(&bare), digest(&ours)); + + // The candidate is the sandbox's one own change, not the inherited base. + assert!( + digest(&ours).contains("sandbox edit"), + "the sandbox's edit should be the candidate: {}", + digest(&ours) + ); + assert!( + !digest(&ours).contains("first"), + "the inherited base change is not this view's candidate: {}", + digest(&ours) + ); +} diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs index ca2ee919..99af15a9 100644 --- a/atomic-core/src/pristine/slice.rs +++ b/atomic-core/src/pristine/slice.rs @@ -92,13 +92,24 @@ pub struct CrdtRows { } /// A view as a remote sandbox sees it: its name, the repository's Merkle -/// state and change count for it, and the ids of every change it can see. +/// state and change count for it, its scope and parent, and the ids of the +/// changes **it recorded itself**, in application order. +/// +/// The scope and parent are not decoration. A cache reconstructs the view's +/// effective perspective from them — its own changes, its draft ancestors', +/// and the nearest shared ancestor's — exactly as the repository does. A +/// snapshot that flattened the view to a shared parentless one made the +/// cache's own log the whole union instead, so anything comparing views +/// (`triage review`, `diff_views`) counted inherited history as this view's +/// work. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ViewSnapshotRows { pub id: u64, pub name: String, pub state: Merkle, pub change_count: u64, + pub kind: ViewScope, + pub parent: Option, /// In application order. pub visible: Vec, } @@ -480,6 +491,8 @@ impl ReadTxn { name: view.name.clone(), state: view.state, change_count: view.change_count, + kind: view.kind, + parent: view.parent, visible, } } @@ -647,17 +660,24 @@ impl WriteTxn<'_> { Ok(()) } - /// Write `snapshot` as a shared, parentless view with the repository's - /// own id, state and change count — replacing any view with that name or - /// id — and its visible changes as the view's change log. + /// Write `snapshot` as a view — its own scope, its own parent, its own + /// change log, with the repository's id, state and change count — + /// replacing any view with that name or id. + /// + /// The parent link matters only if something will follow it: a cache that + /// holds `sb` with `parent: dev`, and a `dev` row with dev's real log, + /// reconstructs the union the repository computes. A snapshot written + /// parentless and shared cannot, which is why `visible` here is the + /// view's *own* changes and the ancestors arrive beside it in the + /// skeleton, not folded into this log. pub fn import_view_snapshot(&mut self, snapshot: &ViewSnapshotRows) -> PristineResult<()> { let state = ViewState { id: snapshot.id, name: snapshot.name.clone(), state: snapshot.state, change_count: snapshot.change_count, - kind: ViewScope::Shared, - parent: None, + kind: snapshot.kind, + parent: snapshot.parent, }; { let mut views = self.txn.open_table(VIEWS)?; diff --git a/atomic-core/src/pristine/traits/view.rs b/atomic-core/src/pristine/traits/view.rs index c3f123d4..06806b09 100644 --- a/atomic-core/src/pristine/traits/view.rs +++ b/atomic-core/src/pristine/traits/view.rs @@ -3,9 +3,10 @@ //! Contains `ViewScope`, `ViewState` (view metadata), and `ViewTxnT` //! (the read-only trait for querying views and their change logs). -use crate::types::{Merkle, NodeId}; +use serde::{Deserialize, Serialize}; use crate::pristine::error::PristineError; +use crate::types::{Merkle, NodeId}; use super::graph::GraphTxnT; @@ -44,7 +45,7 @@ use super::graph::GraphTxnT; /// assert_eq!(scope as u8, 1); /// assert!(scope.is_shared()); /// ``` -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, Default)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, Default, Serialize, Deserialize)] #[repr(u8)] pub enum ViewScope { /// Personal workspace (feature, bug, experiment). diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index e7bf45cd..76d5d64b 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -21,9 +21,18 @@ use crate::RepositoryError; /// A view's tree for a fresh cache: the skeleton rows, the view itself, and /// every entry (with content) the view has. +/// +/// `ancestors` carries the views whose changes are in the view's effective +/// perspective — for a draft, its draft ancestors and the nearest shared +/// ancestor — each with its *own* log, real state and real parent. Without +/// them a cache could only hold the view parentless, count inherited history +/// as the view's own work, and invent empty placeholders for views it was +/// told about but never given. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SandboxSkeleton { pub view: ViewSnapshotRows, + #[serde(default)] + pub ancestors: Vec, pub rows: GraphSlice, } @@ -188,6 +197,45 @@ fn db(e: impl std::fmt::Display) -> RepositoryError { RepositoryError::Database(e.to_string()) } +/// The views whose changes are in a draft's effective perspective: its draft +/// ancestors, and the nearest shared ancestor past them. A shared view needs +/// nobody — its perspective is its own log. +/// +/// This mirrors the ancestor walk in [`collect_visible_change_ids`] exactly, +/// because the cache reconstructs the union from the chain this returns: if +/// the two ever disagree, the cache's perspective and the rows it was given +/// disagree with it. +fn sandbox_view_chain( + txn: &atomic_core::pristine::ReadTxn, + state: &atomic_core::pristine::ViewState, +) -> Result, RepositoryError> { + if state.kind.is_shared() { + return Ok(Vec::new()); + } + let db = |e: atomic_core::pristine::PristineError| RepositoryError::Database(e.to_string()); + let mut chain = Vec::new(); + for id in txn.resolve_view_chain(state).map_err(db)? { + if id == state.id { + continue; // the view itself is exported separately + } + if let Some(ancestor) = txn.get_view_by_id(id).map_err(db)? { + chain.push(ancestor); + } + } + let mut cursor = state.parent; + while let Some(pid) = cursor { + match txn.get_view_by_id(pid).map_err(db)? { + Some(p) if p.kind.is_shared() => { + chain.push(p); + break; + } + Some(p) => cursor = p.parent, + None => break, + } + } + Ok(chain) +} + impl Repository { /// The visible change ids of `view`, sorted. fn sandbox_visible( @@ -205,6 +253,13 @@ impl Repository { /// Serve side: `view`'s skeleton, with `live` its rendered tree (inode → /// path, as [`Repository::materialize_view_entries`] gives them). + /// + /// The view goes out as itself — its scope, its parent, its *own* change + /// log — and its ancestors beside it, each with their own. The rows are + /// still filtered by the union: a cache reconstructs the union from the + /// chain, and this builds that union the same way + /// [`super::collect_visible_change_ids`] does, so what the cache + /// reconstructs is what the rows were filtered by. pub fn export_sandbox_skeleton( &self, view: &str, @@ -217,11 +272,27 @@ impl Repository { .ok_or_else(|| RepositoryError::ViewNotFound { name: view.to_string(), })?; - let visible = self.sandbox_visible(&txn, &state)?; - let set: BTreeSet = visible.iter().copied().collect(); - let rows = txn.export_skeleton(&set, live).map_err(db)?; + let own = super::collect_view_change_ids(&txn, &state)?; + let chain = sandbox_view_chain(&txn, &state)?; + let mut union: BTreeSet = own.iter().map(|id| id.get()).collect(); + for ancestor in &chain { + union.extend( + super::collect_view_change_ids(&txn, ancestor)? + .iter() + .map(|id| id.get()), + ); + } + let own_sorted = own.iter().map(|id| id.get()).collect(); + let rows = txn.export_skeleton(&union, live).map_err(db)?; Ok(SandboxSkeleton { - view: txn.export_view_snapshot(&state, visible), + view: txn.export_view_snapshot(&state, own_sorted), + ancestors: chain + .iter() + .map(|a| { + let ids = super::collect_view_change_ids(&txn, a)?; + Ok(txn.export_view_snapshot(a, ids.iter().map(|id| id.get()).collect())) + }) + .collect::, RepositoryError>>()?, rows, }) } @@ -703,6 +774,13 @@ impl Repository { skeleton: &SandboxSkeleton, ) -> Result<(), RepositoryError> { let mut txn = self.pristine.write_txn().map_err(db)?; + // Ancestors first, so the view's parent link lands on a row that + // exists. Each import replaces any view with that name or id, which is + // what turns a cache's invented placeholder for the parent into the + // real one. + for ancestor in &skeleton.ancestors { + txn.import_view_snapshot(ancestor).map_err(db)?; + } txn.import_view_snapshot(&skeleton.view).map_err(db)?; txn.import_skeleton(&skeleton.rows).map_err(db)?; txn.commit().map_err(db)?;