From e95b122334010ddf31f1f939d2aa5b1804c766ea Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 02:35:30 -0500 Subject: [PATCH 01/15] feat(canonical): sign attestations with a key held elsewhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `attest_value` splits into `prepare_attestation` (author, content hash and the exact bytes to sign) and `attach_proof`, so a signer that keeps its key outside this process — a browser's WebCrypto, a hardware token — produces the same `eddsa-jcs-2022` attestation the CLI does. `attest_value` is now those two halves around a local `Signer`. `atomic-canonical-wasm` exposes that to the browser: prepare, attach and verify attestations, plus the DID and canonical JSON of a document, all over JSON strings and byte arrays. `build.sh` builds it with wasm-bindgen; `smoke.mjs` signs with a non-extractable WebCrypto key, verifies, and checks tampering is caught. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 11 ++++ Cargo.toml | 1 + atomic-canonical-wasm/.gitignore | 1 + atomic-canonical-wasm/Cargo.toml | 24 +++++++ atomic-canonical-wasm/README.md | 20 ++++++ atomic-canonical-wasm/build.sh | 11 ++++ atomic-canonical-wasm/smoke.mjs | 13 ++++ atomic-canonical-wasm/src/lib.rs | 107 +++++++++++++++++++++++++++++++ atomic-canonical/src/proof.rs | 64 ++++++++++++++++-- 9 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 atomic-canonical-wasm/.gitignore create mode 100644 atomic-canonical-wasm/Cargo.toml create mode 100644 atomic-canonical-wasm/README.md create mode 100755 atomic-canonical-wasm/build.sh create mode 100644 atomic-canonical-wasm/smoke.mjs create mode 100644 atomic-canonical-wasm/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 830d8ca4..a23eac48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,6 +172,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "atomic-canonical-wasm" +version = "0.18.3" +dependencies = [ + "atomic-canonical", + "atomic-identity", + "getrandom 0.2.17", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "atomic-cli" version = "0.18.3" diff --git a/Cargo.toml b/Cargo.toml index 36780037..17e609ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "atomic-agent", "atomic-canonical", + "atomic-canonical-wasm", "atomic-cli", "atomic-core", "atomic-config", diff --git a/atomic-canonical-wasm/.gitignore b/atomic-canonical-wasm/.gitignore new file mode 100644 index 00000000..01d0a084 --- /dev/null +++ b/atomic-canonical-wasm/.gitignore @@ -0,0 +1 @@ +pkg/ diff --git a/atomic-canonical-wasm/Cargo.toml b/atomic-canonical-wasm/Cargo.toml new file mode 100644 index 00000000..8847e900 --- /dev/null +++ b/atomic-canonical-wasm/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "atomic-canonical-wasm" +description = "atomic-canonical for the browser: build and check atomic attestations where the signing key lives in WebCrypto" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +atomic-canonical = { workspace = true } +atomic-identity = { workspace = true } +serde_json = { workspace = true } +wasm-bindgen = "0.2" + +# The browser supplies randomness; nothing here draws any, but the +# dependency tree links getrandom, which refuses wasm32 without this. +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.2", features = ["js"] } diff --git a/atomic-canonical-wasm/README.md b/atomic-canonical-wasm/README.md new file mode 100644 index 00000000..a5b15f48 --- /dev/null +++ b/atomic-canonical-wasm/README.md @@ -0,0 +1,20 @@ +# atomic-canonical-wasm + +`atomic-canonical` for the browser. A page that holds an atomic identity's +Ed25519 key in WebCrypto signs atomic documents without the key ever leaving +the browser: this module builds the canonical document and the bytes to sign +(with the same code the CLI uses), WebCrypto signs them, and the result is an +ordinary `eddsa-jcs-2022` attestation that `atomic` verifies. + +```js +import init, { prepareAttestation, attachProof } from "./pkg/atomic_canonical_wasm.js"; +await init(); +const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); +const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); +const attested = JSON.parse(attachProof(prepared.document, publicKeyBytes, new Uint8Array(sig))); +``` + +Also: `verifyAttestation`, `didForPublicKey`, `publicKeyBase32`, `canonicalize`. + +Build with `./build.sh [out-dir]`; `node smoke.mjs` (Node ≥ 22, which has +WebCrypto Ed25519) signs, verifies and tamper-checks a document end to end. diff --git a/atomic-canonical-wasm/build.sh b/atomic-canonical-wasm/build.sh new file mode 100755 index 00000000..94f6390e --- /dev/null +++ b/atomic-canonical-wasm/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Build the browser module: pkg/atomic_canonical_wasm{.js,_bg.wasm}. +# Needs the wasm32 target and a wasm-bindgen CLI matching the locked +# wasm-bindgen crate: +# rustup target add wasm32-unknown-unknown +# cargo install wasm-bindgen-cli --version +set -eu +here=$(cd "$(dirname "$0")" && pwd) +cargo build -p atomic-canonical-wasm --target wasm32-unknown-unknown --release +wasm-bindgen --target web --out-dir "${1:-$here/pkg}" \ + "$here/../target/wasm32-unknown-unknown/release/atomic_canonical_wasm.wasm" diff --git a/atomic-canonical-wasm/smoke.mjs b/atomic-canonical-wasm/smoke.mjs new file mode 100644 index 00000000..ba7e6f86 --- /dev/null +++ b/atomic-canonical-wasm/smoke.mjs @@ -0,0 +1,13 @@ +import { readFileSync } from "node:fs"; +import init, { prepareAttestation, attachProof, verifyAttestation, didForPublicKey } from "./pkg/atomic_canonical_wasm.js"; +await init({ module_or_path: readFileSync(new URL("./pkg/atomic_canonical_wasm_bg.wasm", import.meta.url)) }); +const kp = await crypto.subtle.generateKey("Ed25519", false, ["sign", "verify"]); +const pub = new Uint8Array(await crypto.subtle.exportKey("raw", kp.publicKey)); +const doc = { "@type": "BasecampLogin", nonce: "abc", "é": [1, 2.5, "x"] }; +const p = prepareAttestation(JSON.stringify(doc), pub); +const sig = new Uint8Array(await crypto.subtle.sign("Ed25519", kp.privateKey, p.signingBytes)); +const attested = attachProof(p.document, pub, sig); +verifyAttestation(attested, pub); +console.log(didForPublicKey(pub)); +console.log(attested); +try { verifyAttestation(attested.replace('"abc"', '"abd"'), pub); console.log("TAMPER NOT DETECTED"); } catch (e) { console.log("tamper rejected:", e.message); } diff --git a/atomic-canonical-wasm/src/lib.rs b/atomic-canonical-wasm/src/lib.rs new file mode 100644 index 00000000..e4c2d55d --- /dev/null +++ b/atomic-canonical-wasm/src/lib.rs @@ -0,0 +1,107 @@ +//! atomic-canonical for the browser. +//! +//! A web page that holds an atomic identity's key in WebCrypto (as a +//! non-extractable Ed25519 key) signs atomic documents without the key ever +//! reaching Rust — or leaving the browser. This crate does everything *but* +//! the signing, with the same code the CLI uses, so the result is an ordinary +//! atomic attestation (`eddsa-jcs-2022`) that `atomic` verifies: +//! +//! ```js +//! const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); +//! const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); +//! const attested = attachProof(prepared.document, publicKeyBytes, new Uint8Array(sig)); +//! ``` +//! +//! Every function takes and returns JSON as strings and keys/signatures as +//! bytes, so there is no JS object model to keep in step with the Rust types. + +use atomic_canonical::{did, jcs, proof}; +use atomic_identity::keypair::PublicKey; +use atomic_identity::signing::Signature; +use wasm_bindgen::prelude::*; + +fn public_key(bytes: &[u8]) -> Result { + let bytes: &[u8; 32] = bytes + .try_into() + .map_err(|_| JsError::new("an Ed25519 public key is 32 bytes"))?; + PublicKey::from_bytes(bytes).map_err(|e| JsError::new(&e.to_string())) +} + +fn parse(json: &str) -> Result { + serde_json::from_str(json).map_err(|e| JsError::new(&format!("not JSON: {e}"))) +} + +/// A document ready to sign: the document to hand back to [`attach_proof`], +/// and the bytes the signature must cover. +#[wasm_bindgen] +pub struct Prepared { + document: String, + signing_bytes: Vec, +} + +#[wasm_bindgen] +impl Prepared { + /// The document with `attributedTo` and `contentHash` filled in (JSON). + #[wasm_bindgen(getter)] + pub fn document(&self) -> String { + self.document.clone() + } + + /// What to sign with the identity's Ed25519 key. + #[wasm_bindgen(getter, js_name = signingBytes)] + pub fn signing_bytes(&self) -> Vec { + self.signing_bytes.clone() + } +} + +/// Fill in the author and content hash of `document` (JSON) for the key +/// `public_key`, and return it with the bytes to sign. +#[wasm_bindgen(js_name = prepareAttestation)] +pub fn prepare_attestation(document: &str, public_key_bytes: &[u8]) -> Result { + let pk = public_key(public_key_bytes)?; + let prepared = proof::prepare_attestation(parse(document)?, &pk); + Ok(Prepared { + document: prepared.value.to_string(), + signing_bytes: prepared.signing_bytes, + }) +} + +/// Attach the proof for `signature` (64 bytes, over the prepared signing +/// bytes) and check it verifies. Returns the attested document (JSON). +#[wasm_bindgen(js_name = attachProof)] +pub fn attach_proof( + document: &str, + public_key_bytes: &[u8], + signature: &[u8], +) -> Result { + let pk = public_key(public_key_bytes)?; + let sig = Signature::from_slice(signature).map_err(|e| JsError::new(&e.to_string()))?; + let attested = proof::attach_proof(parse(document)?, &pk, &sig); + proof::verify_value(&attested, &pk).map_err(|e| JsError::new(&e.to_string()))?; + Ok(attested.to_string()) +} + +/// Check an attested document (JSON) against a public key. +#[wasm_bindgen(js_name = verifyAttestation)] +pub fn verify_attestation(document: &str, public_key_bytes: &[u8]) -> Result<(), JsError> { + let pk = public_key(public_key_bytes)?; + proof::verify_value(&parse(document)?, &pk).map_err(|e| JsError::new(&e.to_string())) +} + +/// The `did:atomic` identifier for a public key. +#[wasm_bindgen(js_name = didForPublicKey)] +pub fn did_for_public_key(public_key_bytes: &[u8]) -> Result { + Ok(did::did_for_public_key(&public_key(public_key_bytes)?)) +} + +/// The key's base32 form — what atomic writes as the `kid` of its tokens. +#[wasm_bindgen(js_name = publicKeyBase32)] +pub fn public_key_base32(public_key_bytes: &[u8]) -> Result { + Ok(public_key(public_key_bytes)?.to_base32()) +} + +/// RFC 8785 canonical JSON of `document` — the bytes atomic hashes and signs. +#[wasm_bindgen] +pub fn canonicalize(document: &str) -> Result { + Ok(jcs::canonicalize(&parse(document)?)) +} diff --git a/atomic-canonical/src/proof.rs b/atomic-canonical/src/proof.rs index afd3710d..ab45b012 100644 --- a/atomic-canonical/src/proof.rs +++ b/atomic-canonical/src/proof.rs @@ -110,8 +110,32 @@ pub fn substance_view(value: &Value) -> Value { /// path all typed nodes share. Fills `attributedTo` (from the identity's /// `did:atomic`) when absent, computes the content hash over `hashing_view`, /// signs `jcs(signing_view)`, and attaches the proof. Returns the value. -pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> Value { - let did = did::did_for_public_key(&identity.public_key); +/// +/// It is [`prepare_attestation`] → sign → [`attach_proof`]; a signer that +/// holds its key outside this process uses those two halves directly. +pub fn attest_value(value: Value, identity: &Identity, keypair: &KeyPair) -> Value { + let prepared = prepare_attestation(value, &identity.public_key); + let signature = Signer::new(keypair).sign(&prepared.signing_bytes); + attach_proof(prepared.value, &identity.public_key, &signature) +} + +/// A value made ready to sign: `attributedTo` and `contentHash` filled in, +/// and the exact bytes the signature must cover. +#[derive(Debug, Clone)] +pub struct PreparedAttestation { + /// The value to sign — pass it back to [`attach_proof`] unchanged. + pub value: Value, + /// `jcs(signing_view(value))`: what the Ed25519 signature covers. + pub signing_bytes: Vec, +} + +/// First half of [`attest_value`], for signers that hold the key somewhere +/// else — a browser's WebCrypto, a hardware token, a remote signing service. +/// Everything that must agree with [`verify_value`] (the author, the content +/// hash, the canonical bytes) is computed here, so the external signer only +/// ever signs bytes, and the result is an ordinary atomic attestation. +pub fn prepare_attestation(mut value: Value, public_key: &PublicKey) -> PreparedAttestation { + let did = did::did_for_public_key(public_key); if let Some(obj) = value.as_object_mut() { // Fill attributedTo only if there is no non-empty value already. @@ -121,7 +145,7 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> .map(|s| !s.is_empty()) .unwrap_or(false); if !has_author { - obj.insert(PROP_ATTRIBUTED_TO.to_string(), Value::String(did.clone())); + obj.insert(PROP_ATTRIBUTED_TO.to_string(), Value::String(did)); } } @@ -133,13 +157,24 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> } let signing_bytes = jcs::canonicalize(&signing_view(&value)).into_bytes(); - let signature = Signer::new(keypair).sign(&signing_bytes); + PreparedAttestation { + value, + signing_bytes, + } +} + +/// Second half of [`attest_value`]: attach the `eddsa-jcs-2022` proof for a +/// signature over [`PreparedAttestation::signing_bytes`] made by +/// `public_key`'s private key. Does not check the signature — run +/// [`verify_value`] on the result for that. +pub fn attach_proof(mut value: Value, public_key: &PublicKey, signature: &Signature) -> Value { + let did = did::did_for_public_key(public_key); let proof = Proof { type_: PROOF_TYPE.to_string(), cryptosuite: CRYPTOSUITE.to_string(), verification_method: did::verification_method(&did), proof_purpose: PROOF_PURPOSE.to_string(), - proof_value: encode_proof_value(&signature), + proof_value: encode_proof_value(signature), }; if let Some(obj) = value.as_object_mut() { obj.insert( @@ -281,6 +316,25 @@ mod tests { }) } + /// An external signer — handed only the prepared bytes — produces + /// exactly the attestation `attest_value` would, and it verifies. + #[test] + fn prepare_sign_attach_matches_attest_value() { + let (id, kp) = dev_identity(); + let prepared = prepare_attestation(minimal_value(), &kp.public); + let signature = Signer::new(&kp).sign(&prepared.signing_bytes); + let external = attach_proof(prepared.value, &kp.public, &signature); + + assert_eq!(external, attest_value(minimal_value(), &id, &kp)); + verify_value(&external, &kp.public).expect("externally signed value verifies"); + + // A signature over anything else does not. + let wrong = Signer::new(&kp).sign(b"not the prepared bytes"); + let prepared = prepare_attestation(minimal_value(), &kp.public); + let bad = attach_proof(prepared.value, &kp.public, &wrong); + assert!(verify_value(&bad, &kp.public).is_err()); + } + #[test] fn attest_value_then_verify_value_roundtrips() { let (id, kp) = dev_identity(); From 5c1dc776b16fa67915aa84cb24ff925f8272922a Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 03:19:34 -0500 Subject: [PATCH 02/15] feat(intent): attest with a key held elsewhere `atomic intent attest --prepare` prints the document an attestation signs and the exact bytes to sign, needing only the identity's public key. `--signed ` records an attestation a key holder produced from it: it must be signed by `--identity`'s key and attest the intent as it is now, so a signature over a stale or altered intent is refused. This lets a sandbox that holds only an agent's public identity attest as that agent, with the key kept by a signing service outside it. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-cli/src/commands/intent/attest.rs | 89 +++++++-- .../intent_attest_external_signer_test.rs | 185 ++++++++++++++++++ 2 files changed, 260 insertions(+), 14 deletions(-) create mode 100644 atomic-cli/tests/intent_attest_external_signer_test.rs diff --git a/atomic-cli/src/commands/intent/attest.rs b/atomic-cli/src/commands/intent/attest.rs index d879adc0..a374834e 100644 --- a/atomic-cli/src/commands/intent/attest.rs +++ b/atomic-cli/src/commands/intent/attest.rs @@ -4,6 +4,7 @@ use clap::Parser; use serde_json::Value; +use atomic_canonical::proof::prepare_attestation; use atomic_canonical::{lift_and_attest, validate_intent, verify}; use atomic_core::pristine::VaultEntryType; use atomic_identity::IdentityStore; @@ -28,6 +29,20 @@ pub struct IntentAttest { /// Output the attested node as JSON-LD. #[arg(long)] pub json: bool, + + /// Don't sign: print what a signer holding the identity's key elsewhere + /// (a browser, a hardware token, a remote signing service) must sign — + /// `{"document": …, "signingBytes": ""}`. Needs only the + /// identity's public key. Complete with `--signed`. + #[arg(long, conflicts_with = "signed")] + pub prepare: bool, + + /// Record an attestation signed elsewhere: a JSON file holding the + /// attested node (the `--prepare` document with its proof attached). It + /// must be signed by `--identity`'s key and attest the intent as it is + /// now. + #[arg(long, value_name = "PATH")] + pub signed: Option, } impl Command for IntentAttest { @@ -59,7 +74,7 @@ impl Command for IntentAttest { ))); } - // Resolve identity + keypair the way `atomic identity sign` does. + // Resolve the identity the way `atomic identity sign` does. let store = IdentityStore::open_default().map_err(|e| { CliError::Internal(anyhow::anyhow!("Failed to open identity store: {}", e)) })?; @@ -79,20 +94,66 @@ impl Command for IntentAttest { .to_string(), })? }; - let keypair = store.load_keypair(&identity.id, None).map_err(|e| { - CliError::Internal(anyhow::anyhow!( - "Failed to load keypair for '{}': {}", - identity.name, - e - )) - })?; - // Attest: lift + fill attributedTo (from the identity's did:atomic when - // absent) + hash + sign. - let node = lift_and_attest(&inputs.frontmatter, &inputs.body, &identity, &keypair) - .map_err(|e| CliError::InvalidArgument { - message: format!("could not attest intent: {e}"), + // Signing elsewhere, step 1: say what to sign. The same preparation + // `lift_and_attest` does (author, content hash, canonical bytes), with + // no private key involved. + if self.prepare { + let prepared = prepare_attestation(unattested.to_value(), &identity.public_key); + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "document": prepared.value, + "signingBytes": data_encoding::BASE64.encode(&prepared.signing_bytes), + })) + .unwrap() + ); + return Ok(()); + } + + let node = if let Some(path) = &self.signed { + // Signing elsewhere, step 2: the signature must be over exactly + // what `--prepare` produces for this intent now — so a signature + // over a stale or altered intent is refused, not recorded. + let text = std::fs::read_to_string(path).map_err(CliError::Io)?; + let signed: Value = + serde_json::from_str(&text).map_err(|e| CliError::InvalidArgument { + message: format!("{} is not JSON: {e}", path.display()), + })?; + let expected = prepare_attestation(unattested.to_value(), &identity.public_key).value; + let mut unsigned = signed.clone(); + if let Some(obj) = unsigned.as_object_mut() { + obj.remove("proof"); + } + if unsigned != expected { + return Err(CliError::InvalidArgument { + message: format!( + "the signed attestation is not of intent {} as it is now (re-run --prepare)", + self.id + ), + }); + } + serde_json::from_value::(signed).map_err(|e| { + CliError::InvalidArgument { + message: format!("not an attested intent: {e}"), + } + })? + } else { + let keypair = store.load_keypair(&identity.id, None).map_err(|e| { + CliError::Internal(anyhow::anyhow!( + "Failed to load keypair for '{}': {}", + identity.name, + e + )) })?; + // Attest: lift + fill attributedTo (from the identity's + // did:atomic when absent) + hash + sign. + lift_and_attest(&inputs.frontmatter, &inputs.body, &identity, &keypair).map_err( + |e| CliError::InvalidArgument { + message: format!("could not attest intent: {e}"), + }, + )? + }; // Belt-and-suspenders: re-gate the ATTESTED node — proof + attributedTo // must now satisfy the gate. @@ -107,7 +168,7 @@ impl Command for IntentAttest { // Self-check: the proof verifies against the signing key before we // write anything to disk. - verify(&node, &keypair.public).map_err(|e| CliError::InvalidArgument { + verify(&node, &identity.public_key).map_err(|e| CliError::InvalidArgument { message: format!("attested intent failed self-verification: {e}"), })?; diff --git a/atomic-cli/tests/intent_attest_external_signer_test.rs b/atomic-cli/tests/intent_attest_external_signer_test.rs new file mode 100644 index 00000000..d6871257 --- /dev/null +++ b/atomic-cli/tests/intent_attest_external_signer_test.rs @@ -0,0 +1,185 @@ +//! `atomic intent attest --prepare` / `--signed`: attesting an intent with a +//! key that is not on this machine. The identity here is public-only — as an +//! agent identity is inside a sandbox whose key lives with a signing +//! service — so plain `attest` can't sign; `--prepare` says what to sign, +//! the key holder signs it, and `--signed` checks and records the result. + +use std::path::Path; +use std::process::{Command, Output}; + +use atomic_identity::{Identity, IdentityStore, IdentityType, IdentityUsage, KeyPair}; + +fn atomic(home: &Path, cwd: &Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(args) + .current_dir(cwd) + .env("HOME", home) + .env("ATOMIC_CONFIG_DIR", home.join(".atomic")) + .output() + .unwrap() +} + +fn ok(out: &Output) -> String { + assert!( + out.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +/// A repo with one intent, a default human identity, and a public-only +/// agent identity whose key only the test holds. +fn setup() -> (tempfile::TempDir, tempfile::TempDir, String, KeyPair) { + let home = tempfile::tempdir().unwrap(); + let repo = tempfile::tempdir().unwrap(); + ok(&atomic( + home.path(), + repo.path(), + &[ + "identity", + "new", + "ada", + "--email", + "ada@example.com", + "--set-default", + ], + )); + ok(&atomic(home.path(), repo.path(), &["init"])); + ok(&atomic( + home.path(), + repo.path(), + &["intent", "new", "Add a greeting"], + )); + + let key = KeyPair::generate(); + let agent = Identity::builder("ada+bot") + .identity_type(IdentityType::Agent) + .usage(IdentityUsage::Bot) + .public_key(key.public.clone()) + .build() + .unwrap(); + IdentityStore::open(&home.path().join(".atomic").join("identities")) + .unwrap() + .save(&agent) + .unwrap(); + + let list = ok(&atomic( + home.path(), + repo.path(), + &["intent", "list", "--json"], + )); + let intents: serde_json::Value = serde_json::from_str(&list).unwrap(); + let id = intents + .as_array() + .and_then(|a| a.first()) + .and_then(|i| i.get("id")) + .and_then(|v| v.as_str()) + .expect("one intent") + .to_string(); + (home, repo, id, key) +} + +/// What the key holder does with `--prepare`'s output. +fn sign_elsewhere(prepared: &str, key: &KeyPair) -> serde_json::Value { + let prepared: serde_json::Value = serde_json::from_str(prepared).unwrap(); + let bytes = data_encoding::BASE64 + .decode(prepared["signingBytes"].as_str().unwrap().as_bytes()) + .unwrap(); + let signature = atomic_identity::signing::Signer::new(key).sign(&bytes); + atomic_canonical::proof::attach_proof(prepared["document"].clone(), &key.public, &signature) +} + +#[test] +fn an_intent_is_attested_by_a_key_held_elsewhere() { + let (home, repo, id, key) = setup(); + let (home, repo) = (home.path(), repo.path()); + + // No key here: plain attest can't sign as the agent. + let out = atomic( + home, + repo, + &["intent", "attest", &id, "--identity", "ada+bot"], + ); + assert!(!out.status.success(), "attest without a key must fail"); + + let prepared = ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--prepare", + ], + )); + let signed = sign_elsewhere(&prepared, &key); + let file = home.join("signed.json"); + std::fs::write(&file, signed.to_string()).unwrap(); + ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--signed", + file.to_str().unwrap(), + ], + )); + + let report: serde_json::Value = serde_json::from_str(&ok(&atomic( + home, + repo, + &["intent", "validate", &id, "--json"], + ))) + .unwrap(); + assert_eq!(report["conforms"], true, "{report}"); + assert_eq!( + signed["attributedTo"], + atomic_canonical::did::did_for_public_key(&key.public) + ); +} + +#[test] +fn a_signature_by_another_key_is_refused() { + let (home, repo, id, _key) = setup(); + let (home, repo) = (home.path(), repo.path()); + let prepared = ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--prepare", + ], + )); + let signed = sign_elsewhere(&prepared, &KeyPair::generate()); + let file = home.join("signed.json"); + std::fs::write(&file, signed.to_string()).unwrap(); + let out = atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--signed", + file.to_str().unwrap(), + ], + ); + assert!( + !out.status.success(), + "a signature by the wrong key must not be recorded" + ); +} From b5ef87b41b5f3db62870bde1bb947c8c075a3669 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 08:55:30 -0500 Subject: [PATCH 03/15] fix(agent): never record agent work under the human's key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an agent identity was named (option or ATOMIC_AGENT_IDENTITY) but couldn't be used — missing, not an agent, unreadable — recording fell back to the plus-tag author on the human's default key, putting a person's key on work an agent did, exactly when the caller said it was an agent's. It now records unkeyed instead, and says so in the log. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-agent/src/identity.rs | 61 +++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/atomic-agent/src/identity.rs b/atomic-agent/src/identity.rs index 0367c60c..b4f44dbd 100644 --- a/atomic-agent/src/identity.rs +++ b/atomic-agent/src/identity.rs @@ -125,7 +125,20 @@ pub fn resolve_agent_author(options: &AgentAuthorOptions<'_>) -> Author { return author; } - // Otherwise: plus-tag the human's identity. Legible, not provable. + // An agent identity was named but can't be used (missing, not an agent, + // unreadable). Never fall back to the human's key then: that would put a + // person's key on work they didn't do, exactly when the caller said it + // was an agent's. Record it unkeyed — honestly anonymous — instead. + if requested_agent_identity(options).is_some() { + log::warn!( + "agent identity {:?} is not usable; recording without a key rather than as the human", + requested_agent_identity(options) + ); + return fallback_agent_author(options); + } + + // No agent identity asked for: plus-tag the human's identity. Legible, + // not provable. match load_default_user_identity(options.identity_dir.as_deref()) { Some(user) => derive_agent_author(&user, options), None => fallback_agent_author(options), @@ -173,13 +186,19 @@ pub const AGENT_IDENTITY_ENV: &str = "ATOMIC_AGENT_IDENTITY"; /// unreadable. Recording a turn must not break because an agent identity was /// mistyped; falling back to the plus-tag author keeps the work attributed to /// *someone* and leaves a debug log explaining why it is not keyed. -fn delegated_agent_author(options: &AgentAuthorOptions<'_>) -> Option { - let name = options +/// The agent identity the caller asked for, explicitly or via +/// `ATOMIC_AGENT_IDENTITY`. +fn requested_agent_identity(options: &AgentAuthorOptions<'_>) -> Option { + options .agent_identity .clone() .or_else(|| std::env::var(AGENT_IDENTITY_ENV).ok()) .map(|n| n.trim().to_string()) - .filter(|n| !n.is_empty())?; + .filter(|n| !n.is_empty()) +} + +fn delegated_agent_author(options: &AgentAuthorOptions<'_>) -> Option { + let name = requested_agent_identity(options)?; let store = match options.identity_dir.as_deref() { Some(dir) => atomic_identity::IdentityStore::open(dir), @@ -784,6 +803,40 @@ mod tests { assert_eq!(author.name, "Claude Code"); } + /// Named but unusable, with a human default present: the work must not + /// be put on the human's key. + #[test] + fn an_unusable_agent_identity_never_falls_back_to_the_human_key() { + use atomic_identity::{Identity, IdentityStore, KeyPair}; + + let dir = TempDir::new().unwrap(); + let mut store = IdentityStore::open(dir.path()).unwrap(); + let key = KeyPair::generate(); + let human = Identity::builder("alice") + .email("alice@example.com") + .public_key(key.public.clone()) + .build() + .unwrap(); + store.save_with_keypair(&human, &key, None).unwrap(); + store.set_default(&human.id).unwrap(); + + for requested in ["nobody+here", "alice"] { + let author = resolve_agent_author(&AgentAuthorOptions { + agent_name: "claude-code", + agent_display_name: "Claude Code", + session_id: "sess1234", + identity_dir: Some(dir.path().to_path_buf()), + agent_identity: Some(requested.to_string()), + }); + assert_ne!( + author.identity.as_deref(), + Some(human.public_key_base32().as_str()), + "{requested}: agent work must not carry the human's key" + ); + assert_eq!(author.name, "Claude Code"); + } + } + /// No agent identity configured: unchanged behavior. #[test] fn no_agent_identity_means_no_keyed_path() { From 6875cd28fe19a8e11b6736f828516d123e752bf9 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 08:56:55 -0500 Subject: [PATCH 04/15] feat(token): name the server a token is for (aud) Self-signed request tokens now carry `aud`: the bare server URL they were minted for, normalised. A server that checks it refuses a token minted for somewhere else, so one leaked from one server can't be replayed at another within its five minutes. Verifiers that ignore unknown claims are unaffected. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-cli/src/commands/token.rs | 39 +++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/atomic-cli/src/commands/token.rs b/atomic-cli/src/commands/token.rs index 8ff725cb..86124cbf 100644 --- a/atomic-cli/src/commands/token.rs +++ b/atomic-cli/src/commands/token.rs @@ -11,7 +11,8 @@ //! `base64url(header).base64url(claims).base64url(signature)`: //! //! - header: `{"alg":"EdDSA","typ":"JWT","kid":""}` -//! - claims: `{ sub, iat, exp, jti }` (`sub` mirrors the `kid` public key) +//! - claims: `{ sub, aud, iat, exp, jti }` (`sub` mirrors the `kid` public key; +//! `aud` is the server the token is for) //! - signature: `Ed25519_sign(private_key, "header.claims")` //! //! # Acting on behalf of someone (agent identities) @@ -78,6 +79,12 @@ struct Claims { exp: i64, jti: String, + /// The server this token is for (RFC 7519 `aud`): the bare server URL, + /// without a trailing slash. A server that checks it refuses a token + /// minted for somewhere else, so a token leaked from one server can't + /// be replayed at another within its lifetime. + aud: String, + /// RFC 8693 actor claim — present only when an agent is acting. #[serde(skip_serializing_if = "Option::is_none")] act: Option, @@ -160,6 +167,7 @@ fn mint_token(server: &str, identity: &Identity) -> CliResult { let now = Utc::now(); let claims = Claims { sub, + aud: audience(server), iat: now.timestamp(), exp: (now + TOKEN_TTL).timestamp(), jti: Uuid::new_v4().to_string(), @@ -196,10 +204,36 @@ fn mint_token(server: &str, identity: &Identity) -> CliResult { Ok(format!("{signing_input}.{sig_b64}")) } +/// The `aud` for a server URL: scheme and host (and port), no trailing +/// slash, lowercased — so `https://Atomic.Storage/` and +/// `https://atomic.storage` are the same audience. +pub fn audience(server: &str) -> String { + server.trim().trim_end_matches('/').to_ascii_lowercase() +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_token_names_the_server_it_is_for() { + assert_eq!( + audience("https://Atomic.Storage/"), + "https://atomic.storage" + ); + let claims = Claims { + sub: "S".into(), + aud: audience("https://atomic.storage"), + iat: 0, + exp: 1, + jti: "j".into(), + act: None, + dlg: None, + }; + let v: serde_json::Value = serde_json::to_value(&claims).unwrap(); + assert_eq!(v["aud"], "https://atomic.storage"); + } + #[test] fn header_is_eddsa_jwt_with_kid() { let header = JwtHeader { @@ -218,6 +252,7 @@ mod tests { fn a_non_delegated_token_omits_the_actor_claims() { let claims = Claims { sub: "ABCDEF".to_string(), + aud: "https://a".to_string(), iat: 0, exp: 1, jti: "j".to_string(), @@ -237,6 +272,7 @@ mod tests { let agent = "AGENTKEY"; let claims = Claims { sub: human.to_string(), + aud: "https://a".to_string(), iat: 0, exp: 1, jti: "j".to_string(), @@ -277,6 +313,7 @@ mod tests { let now = Utc::now(); let claims = Claims { sub: public_key_b32.clone(), + aud: audience("https://atomic.storage"), iat: now.timestamp(), exp: (now + TOKEN_TTL).timestamp(), jti: Uuid::new_v4().to_string(), From e3cb25ab25614f44f808ff37b0ce6d14a8f63f2d Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 13:15:45 -0500 Subject: [PATCH 05/15] feat(repository): render a view's tree without touching disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Repository::materialize_view_entries` hands each entry of a view — path, inode, kind, mode, bytes, content hash, conflict-marker line — to a sink, in path order with directories first, plus the view's Merkle state. Read-only: no working tree, stat cache or conflict state is written, so it runs beside other readers. It serves a remote sandbox its tree and baseline. The per-file rendering is now one function, `render_view_file`, shared with `materialize_parallel`, which keeps writing to disk as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 1 + atomic-repository/Cargo.toml | 1 + .../src/repository/materialize.rs | 390 +++++++++++++----- atomic-repository/src/repository/mod.rs | 1 + .../tests/materialize_view_entries_test.rs | 104 +++++ 5 files changed, 404 insertions(+), 93 deletions(-) create mode 100644 atomic-repository/tests/materialize_view_entries_test.rs diff --git a/Cargo.lock b/Cargo.lock index a23eac48..988f7e56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,6 +345,7 @@ dependencies = [ "atomic-identity", "atomic-semantic", "chrono", + "data-encoding", "dirs", "flate2", "fs2", diff --git a/atomic-repository/Cargo.toml b/atomic-repository/Cargo.toml index 397db592..96680d45 100644 --- a/atomic-repository/Cargo.toml +++ b/atomic-repository/Cargo.toml @@ -16,6 +16,7 @@ atomic-identity = { workspace = true } atomic-semantic = { workspace = true } serde = { workspace = true } +data-encoding = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } postcard = { workspace = true } diff --git a/atomic-repository/src/repository/materialize.rs b/atomic-repository/src/repository/materialize.rs index 3a2916a1..24a2f81b 100644 --- a/atomic-repository/src/repository/materialize.rs +++ b/atomic-repository/src/repository/materialize.rs @@ -545,7 +545,6 @@ impl Repository { use atomic_core::output::repo::{ collect_children, FileOutputOptions, MaterializeOptions, OutputItem, }; - use atomic_core::output::RetrieveOptions; use rayon::prelude::*; use std::collections::HashSet as StdHashSet; @@ -725,82 +724,17 @@ impl Repository { .map(|item| { let file_start = std::time::Instant::now(); - // Build retrieve options with the shared change filter - let retrieve_opts = - RetrieveOptions::default().with_change_filter_arc(change_filter_arc.clone()); - - // Inline the output pipeline so we can trace each phase. - use atomic_core::output::repo::{ - output_graph_content_resolved, resolve_conflicts_semantically, - }; - use atomic_core::output::{compute_order, retrieve_graph, Writer}; - use atomic_core::pristine::InodePreloadTxn; - - // Pre-load ALL edges for this file's inode from INODE_GRAPH - // in a single range scan, then run retrieve_graph over the - // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) - // individual B-tree probes. - let preloaded = InodePreloadTxn::from_table(&txn, item.inode, &inode_graph_table) - .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; - - let t_retrieve = std::time::Instant::now(); - let retrieve_result = retrieve_graph(&preloaded, item.position, retrieve_opts) - .map_err(|e| format!("{}: retrieve: {:?}", item.path, e))?; - - if retrieve_result.graph.is_empty() { - return Ok(None); - } - - let vertices = retrieve_result.graph.len_vertices(); - let edges = retrieve_result.edges_traversed; - let retrieve_ms = t_retrieve.elapsed(); - - let t_order = std::time::Instant::now(); - let mut graph = retrieve_result.graph; - let order = compute_order(&mut graph); - let order_ms = t_order.elapsed(); - - let t_content = std::time::Instant::now(); - let resolved = resolve_conflicts_semantically(&preloaded, store, &graph, &order); - let buffer = Vec::with_capacity(graph.total_bytes()); - let mut writer = Writer::new(buffer); - let hash_fn = |node_id: NodeId| -> Option { - if node_id.is_root() { - return None; - } - preloaded.get_external(node_id).ok().flatten() - }; - output_graph_content_resolved( + let Some(content) = render_view_file( + &txn, store, - hash_fn, - &graph, - &order, - &mut writer, - &resolved, - ) - .map_err(|e| format!("{}: content: {:?}", item.path, e))?; - let content = writer.into_inner(); - let content_ms = t_content.elapsed(); - - if content.is_empty() { + &inode_graph_table, + &change_filter_arc, + &name_conflicts, + item, + trace_mat.then_some(file_start), + )? + else { return Ok(None); - } - - // Name-conflict override (rare): when ≥ 2 inodes are alive at - // this path on the view, replace the single-inode content with - // a marker-wrapped rendering of every side so the conflict is - // surfaced instead of silently collapsed (rubric A12). - let content = match name_conflicts.get(&item.path) { - Some(sides) => render_name_conflict( - &txn, - store, - &inode_graph_table, - &change_filter_arc, - &item.path, - sides, - ) - .unwrap_or(content), - None => content, }; // Detect conflict markers in the materialized bytes. This is @@ -860,24 +794,6 @@ impl Repository { std::fs::write(&abs_path, &content) .map_err(|e| format!("{}: write: {}", item.path, e))?; - if trace_mat { - let elapsed = file_start.elapsed(); - if elapsed > std::time::Duration::from_millis(50) { - eprintln!( - "[materialize] SLOW {} bytes={} vertices={} edges={} \ - retrieve={:?} order={:?} content={:?} total={:?}", - item.path, - bytes_written, - vertices, - edges, - retrieve_ms, - order_ms, - content_ms, - elapsed, - ); - } - } - Ok(Some(( item.path.clone(), bytes_written, @@ -1114,6 +1030,294 @@ impl Repository { } } +/// One entry of a view's tree, rendered in memory by +/// [`Repository::materialize_view_entries`] — enough for a client to write +/// the working tree and a baseline index (`status` against it) without the +/// repository. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ViewEntry { + pub path: String, + pub inode: u64, + pub kind: ViewEntryKind, + /// Unix permission bits as recorded. + pub mode: u16, + /// The file's bytes as a checkout of the view would write them (empty + /// for directories). + #[serde(with = "serde_bytes_vec")] + pub content: Vec, + /// `Hash::of(content)` — the baseline a client compares against. + pub hash: Hash, + /// 1-based line of the first conflict marker, if the file is conflicted. + pub conflict_marker_line: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ViewEntryKind { + File, + Directory, + Symlink, +} + +/// What a view looked like when it was rendered. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ViewSnapshot { + pub view: String, + /// The view's Merkle state, base32: a change recorded against this + /// snapshot applies only while the view is still here. + pub state: String, + pub change_count: u64, +} + +mod serde_bytes_vec { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(v: &[u8], s: S) -> Result { + s.serialize_str(&data_encoding::BASE64.encode(v)) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + data_encoding::BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } +} + +impl Repository { + /// Render every entry of `view` and hand each to `sink`, in path order + /// (a directory before what it contains), touching nothing: no working + /// tree, no stat cache, no conflict state. Read-only, so it runs beside + /// other readers — this is what serves a remote sandbox its tree. + /// + /// Files are rendered with the same kernel as + /// [`Repository::materialize_parallel`], in parallel chunks, so memory + /// holds one chunk of contents at a time. + pub fn materialize_view_entries( + &self, + view_name: &str, + mut sink: impl FnMut(ViewEntry) -> Result<(), E>, + ) -> Result, RepositoryError> { + use atomic_core::output::repo::{collect_children, MaterializeOptions}; + use atomic_core::types::Base32; + use rayon::prelude::*; + + const CHUNK: usize = 256; + + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let change_filter_arc = Arc::new(collect_visible_change_ids(&txn, &view)?); + let options = MaterializeOptions::new().with_change_filter_arc(change_filter_arc.clone()); + let mut items = collect_children(&txn, Inode::ROOT, "", &options) + .map_err(|e| RepositoryError::Database(e.to_string()))?; + items.sort_by(|a, b| a.path.cmp(&b.path)); + + let visible = |item: &atomic_core::output::repo::OutputItem| { + item.position.change.is_root() || change_filter_arc.contains(&item.position.change) + }; + let files: Vec<&atomic_core::output::repo::OutputItem> = items + .iter() + .filter(|i| !i.is_directory && visible(i)) + .collect(); + let name_conflicts = collect_name_conflicts( + &txn, + &self.change_store, + &files.iter().map(|i| i.path.as_str()).collect(), + &change_filter_arc, + )?; + for id in change_filter_arc.iter().filter(|id| !id.is_root()) { + if let Ok(Some(hash)) = txn.get_external(*id) { + let _ = self.change_store.load_change(&hash); + } + } + let inode_graph_table = txn + .open_inode_graph_table() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + + // Directories that hold at least one visible file, before their files. + let file_paths: std::collections::HashSet<&str> = + files.iter().map(|i| i.path.as_str()).collect(); + let mut dirs: Vec<&atomic_core::output::repo::OutputItem> = items + .iter() + .filter(|i| i.is_directory) + .filter(|d| { + let prefix = format!("{}/", d.path); + file_paths.iter().any(|p| p.starts_with(&prefix)) + }) + .collect(); + dirs.sort_by(|a, b| a.path.cmp(&b.path)); + for d in dirs { + let entry = ViewEntry { + path: d.path.clone(), + inode: d.inode.get(), + kind: ViewEntryKind::Directory, + mode: d.metadata.permissions, + content: Vec::new(), + hash: Hash::of(&[]), + conflict_marker_line: None, + }; + if let Err(e) = sink(entry) { + return Ok(Err(e)); + } + } + + let store = &self.change_store; + for chunk in files.chunks(CHUNK) { + let rendered: Vec, String>> = chunk + .par_iter() + .map(|item| { + let content = render_view_file( + &txn, + store, + &inode_graph_table, + &change_filter_arc, + &name_conflicts, + item, + None, + )?; + Ok(content.map(|content| ViewEntry { + path: item.path.clone(), + inode: item.inode.get(), + kind: if item.metadata.is_symlink { + ViewEntryKind::Symlink + } else { + ViewEntryKind::File + }, + mode: item.metadata.permissions, + conflict_marker_line: first_conflict_marker_line(&content), + hash: Hash::of(&content), + content, + })) + }) + .collect(); + for r in rendered { + match r { + Ok(Some(entry)) => { + if let Err(e) = sink(entry) { + return Ok(Err(e)); + } + } + Ok(None) => {} + Err(e) => return Err(RepositoryError::Output(e)), + } + } + } + + Ok(Ok(ViewSnapshot { + view: view.name.clone(), + state: view.state.to_base32(), + change_count: view.change_count, + })) + } +} + +/// Render one file of a view in memory: retrieve its graph under the view's +/// change filter, order it, resolve conflicts, and write the bytes a +/// checkout would produce — with a name conflict rendered as markers. The +/// one rendering kernel behind [`Repository::materialize_parallel`] (which +/// writes the result to disk) and [`Repository::materialize_view_entries`] +/// (which hands it to a caller, touching nothing). `Ok(None)`: the file has +/// no content on this view. +fn render_view_file( + txn: &atomic_core::pristine::ReadTxn, + store: &C, + inode_graph_table: &redb::ReadOnlyMultimapTable<&'static [u8; 32], &'static [u8; 24]>, + change_filter_arc: &Arc>, + name_conflicts: &NameConflicts, + item: &atomic_core::output::repo::OutputItem, + trace_from: Option, +) -> Result>, String> { + use atomic_core::output::RetrieveOptions; + // Build retrieve options with the shared change filter + let retrieve_opts = + RetrieveOptions::default().with_change_filter_arc(change_filter_arc.clone()); + + // Inline the output pipeline so we can trace each phase. + use atomic_core::output::repo::{ + output_graph_content_resolved, resolve_conflicts_semantically, + }; + use atomic_core::output::{compute_order, retrieve_graph, Writer}; + use atomic_core::pristine::InodePreloadTxn; + + // Pre-load ALL edges for this file's inode from INODE_GRAPH + // in a single range scan, then run retrieve_graph over the + // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) + // individual B-tree probes. + let preloaded = InodePreloadTxn::from_table(&txn, item.inode, &inode_graph_table) + .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; + + let t_retrieve = std::time::Instant::now(); + let retrieve_result = retrieve_graph(&preloaded, item.position, retrieve_opts) + .map_err(|e| format!("{}: retrieve: {:?}", item.path, e))?; + + if retrieve_result.graph.is_empty() { + return Ok(None); + } + + let vertices = retrieve_result.graph.len_vertices(); + let edges = retrieve_result.edges_traversed; + let retrieve_ms = t_retrieve.elapsed(); + + let t_order = std::time::Instant::now(); + let mut graph = retrieve_result.graph; + let order = compute_order(&mut graph); + let order_ms = t_order.elapsed(); + + let t_content = std::time::Instant::now(); + let resolved = resolve_conflicts_semantically(&preloaded, store, &graph, &order); + let buffer = Vec::with_capacity(graph.total_bytes()); + let mut writer = Writer::new(buffer); + let hash_fn = |node_id: NodeId| -> Option { + if node_id.is_root() { + return None; + } + preloaded.get_external(node_id).ok().flatten() + }; + output_graph_content_resolved(store, hash_fn, &graph, &order, &mut writer, &resolved) + .map_err(|e| format!("{}: content: {:?}", item.path, e))?; + let content = writer.into_inner(); + let content_ms = t_content.elapsed(); + + if content.is_empty() { + return Ok(None); + } + + // Name-conflict override (rare): when ≥ 2 inodes are alive at + // this path on the view, replace the single-inode content with + // a marker-wrapped rendering of every side so the conflict is + // surfaced instead of silently collapsed (rubric A12). + let content = match name_conflicts.get(&item.path) { + Some(sides) => render_name_conflict( + &txn, + store, + &inode_graph_table, + &change_filter_arc, + &item.path, + sides, + ) + .unwrap_or(content), + None => content, + }; + + if let Some(file_start) = trace_from { + let elapsed = file_start.elapsed(); + if elapsed > std::time::Duration::from_millis(50) { + eprintln!( + "[materialize] SLOW {} vertices={vertices} edges={edges} retrieve={retrieve_ms:?} \ + order={order_ms:?} content={content_ms:?} total={elapsed:?}", + item.path, + ); + } + } + Ok(Some(content)) +} + #[cfg(test)] mod conflict_marker_tests { use super::first_conflict_marker_line; diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 4c98e8fe..ec11bfea 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -79,6 +79,7 @@ use crate::RepositoryError; mod deferred_tree; mod filter; mod materialize; +pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; mod sandbox; mod semantic_materialize; mod split; diff --git a/atomic-repository/tests/materialize_view_entries_test.rs b/atomic-repository/tests/materialize_view_entries_test.rs new file mode 100644 index 00000000..31d0fce7 --- /dev/null +++ b/atomic-repository/tests/materialize_view_entries_test.rs @@ -0,0 +1,104 @@ +//! `Repository::materialize_view_entries`: a view's tree, rendered in memory +//! for a remote sandbox — the same bytes a checkout writes, per view, and +//! nothing touched on disk. + +use std::fs; +use std::path::Path; + +use atomic_core::change::{Author, ChangeHeader}; +use atomic_core::types::Hash; +use atomic_repository::{RecordOptions, Repository, SplitOptions, ViewEntry, ViewEntryKind}; +use tempfile::TempDir; + +fn write(repo_path: &Path, name: &str, content: &str) { + let path = repo_path.join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); +} + +fn record(repo: &Repository, message: &str) -> Hash { + let header = ChangeHeader::builder() + .message(message) + .author(Author::new("Test", Some("test@example.com"))) + .build(); + *repo + .record(header, RecordOptions::default()) + .expect("record") + .hash() +} + +fn entries(repo: &Repository, view: &str) -> (Vec, atomic_repository::ViewSnapshot) { + let mut out = Vec::new(); + let snapshot = repo + .materialize_view_entries::<()>(view, |e| { + out.push(e); + Ok(()) + }) + .expect("materialize") + .expect("sink"); + (out, snapshot) +} + +#[test] +fn entries_are_the_view_s_recorded_tree() { + let temp = TempDir::new().unwrap(); + let root = temp.path().to_path_buf(); + let repo = Repository::init(&root).expect("init"); + let view = repo.current_view().to_string(); + + write(&root, "README.md", "hello\n"); + write(&root, "src/main.rs", "fn main() {}\n"); + repo.add("README.md", Default::default()).unwrap(); + repo.add("src/main.rs", Default::default()).unwrap(); + record(&repo, "first"); + write(&root, "README.md", "hello, world\n"); + let second = record(&repo, "second"); + // Present on disk, never recorded: not part of the view. + write(&root, "scratch.txt", "not recorded\n"); + + let before: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + let (list, snapshot) = entries(&repo, &view); + let after: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + assert_eq!( + before.len(), + after.len(), + "nothing written to the working tree" + ); + + let paths: Vec<&str> = list.iter().map(|e| e.path.as_str()).collect(); + assert_eq!( + paths, + vec!["src", "README.md", "src/main.rs"], + "directories first, then files in order" + ); + let readme = list.iter().find(|e| e.path == "README.md").unwrap(); + assert_eq!(readme.kind, ViewEntryKind::File); + assert_eq!(readme.content, b"hello, world\n"); + assert_eq!(readme.hash, Hash::of(b"hello, world\n")); + assert_eq!(readme.conflict_marker_line, None); + assert!(readme.inode > 0); + assert_eq!(snapshot.view, view); + assert_eq!(snapshot.change_count, 2); + assert!(!snapshot.state.is_empty()); + + // Another view without the second change renders its own content. + let mut repo = repo; + repo.split_view(SplitOptions::new("older", vec![second])) + .expect("split"); + let (older_on_source, older_snapshot) = entries(&repo, &view); + let readme = older_on_source + .iter() + .find(|e| e.path == "README.md") + .unwrap(); + assert_eq!( + readme.content, b"hello\n", + "the source view no longer has the second change" + ); + assert_ne!(older_snapshot.state, snapshot.state); +} From 83f319fd5122ca9860166b569e9840a0ed92ac25 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 13:22:01 -0500 Subject: [PATCH 06/15] Never treat a sandbox's pointer as untracked `.atomic-sandbox` showed up as untracked inside a sandbox, so `record --all` would record it; a remote pointer carries a token. Ignore it, and the sandbox's local cache `.atomic-sandbox.d`, like `.atomic`. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-repository/src/repository/sandbox.rs | 19 +++++++++++++++++++ atomic-repository/src/status.rs | 11 +++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index adb5cebf..e598d62d 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -525,4 +525,23 @@ mod tests { "the canonical graph must not be cloned into the sandbox" ); } + + #[test] + fn a_sandbox_never_sees_its_pointer_as_untracked() { + let dir = tempdir().unwrap(); + let repo = repo_with_recorded_file(&dir.path().join("repo"), "hello.txt", b"hi\n"); + let sandbox = dir.path().join("agent-1"); + repo.provision_sandbox(&sandbox, "dev").unwrap(); + std::fs::write(sandbox.join("new.txt"), b"new\n").unwrap(); + drop(repo); + + let opened = Repository::open_existing(&sandbox).unwrap(); + let status = opened.status(Default::default()).unwrap(); + let untracked: Vec<_> = status.untracked().map(|e| e.path().to_path_buf()).collect(); + assert_eq!( + untracked, + vec![PathBuf::from("new.txt")], + "the pointer can carry a credential; `record --all` must never pick it up" + ); + } } diff --git a/atomic-repository/src/status.rs b/atomic-repository/src/status.rs index 6c470d28..0cdbf9dd 100644 --- a/atomic-repository/src/status.rs +++ b/atomic-repository/src/status.rs @@ -89,8 +89,10 @@ use crate::ignore::IgnoreRules; // Constants -/// Patterns that are always ignored (internal directories) -const ALWAYS_IGNORED: &[&str] = &[".atomic", ".git"]; +/// Patterns that are always ignored: internal directories, and a sandbox's +/// pointer and local cache — the pointer can carry a credential, and neither +/// belongs in history. +const ALWAYS_IGNORED: &[&str] = &[".atomic", ".git", ".atomic-sandbox", ".atomic-sandbox.d"]; // Error Types @@ -1255,10 +1257,15 @@ mod tests { assert!(is_always_ignored(Path::new("src/.atomic/test"))); assert!(is_always_ignored(Path::new(".git"))); assert!(is_always_ignored(Path::new(".git/objects"))); + assert!(is_always_ignored(Path::new(".atomic-sandbox"))); + assert!(is_always_ignored(Path::new( + ".atomic-sandbox.d/pristine.redb" + ))); assert!(!is_always_ignored(Path::new("src"))); assert!(!is_always_ignored(Path::new("src/main.rs"))); assert!(!is_always_ignored(Path::new("atomic"))); + assert!(!is_always_ignored(Path::new(".atomic-sandbox-notes"))); } #[test] From 902ca4f8b16f9e1a00253fa9b435121f68a23061 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 13:59:49 -0500 Subject: [PATCH 07/15] Reach a repository's owner from a remote sandbox, over the same protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remote sandboxes use the database owner's protocol rather than a second one: iroh is another transport beside the local socket, answered by the same handler. Each remote frame carries a view-scoped sandbox token (in memory only, hashed); a remote caller gets Ping, the provenance requests and Materialize, each checked against its token — its own view, the sessions and turns it started, checkpoints only for changes its view can see. Opening, renewing and closing sandboxes, and shutdown, stay local. The client picks its route from .atomic-sandbox, so provenance from a remote sandbox goes through the same OwnerJournalSink as a local hook. `atomic sandbox create --remote` mints the token and writes the pointer (the owner's iroh address, the view, the token; 0600); `materialize` writes the view's tree inside the sandbox; `renew` and `close` manage the token. The owner's iroh key persists in .atomic so pointers survive a restart. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 2091 ++++++++++++++++- Cargo.toml | 1 + atomic-cli/Cargo.toml | 2 + atomic-cli/src/commands/agent/mod.rs | 2 +- .../commands/agent/{owner.rs => owner/mod.rs} | 327 ++- atomic-cli/src/commands/agent/owner/remote.rs | 410 ++++ .../src/commands/agent/owner/sandbox.rs | 471 ++++ atomic-cli/src/commands/sandbox.rs | 140 +- .../tests/remote_sandbox_integration_test.rs | 197 ++ 9 files changed, 3552 insertions(+), 89 deletions(-) rename atomic-cli/src/commands/agent/{owner.rs => owner/mod.rs} (86%) create mode 100644 atomic-cli/src/commands/agent/owner/remote.rs create mode 100644 atomic-cli/src/commands/agent/owner/sandbox.rs create mode 100644 atomic-cli/tests/remote_sandbox_integration_test.rs diff --git a/Cargo.lock b/Cargo.lock index 988f7e56..a2715754 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -127,6 +162,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async_io_stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" +dependencies = [ + "futures 0.3.32", + "pharos", + "rustc_version", +] + [[package]] name = "atomic-agent" version = "0.18.3" @@ -148,7 +194,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 1.0.69", "tokio", @@ -211,10 +257,12 @@ dependencies = [ "fs2", "git2", "indicatif", + "iroh", "log", "open", + "rand 0.8.7", "rayon", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serial_test", @@ -278,7 +326,7 @@ dependencies = [ "chrono", "data-encoding", "dirs", - "ed25519-dalek", + "ed25519-dalek 2.2.0", "quickcheck", "quickcheck_macros", "rand 0.8.7", @@ -322,7 +370,7 @@ dependencies = [ "chrono", "env_logger", "log", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "tempfile", @@ -356,10 +404,10 @@ dependencies = [ "rayon", "redb 4.2.0", "reflink-copy", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "syntext", "tar", "tempfile", @@ -408,18 +456,53 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64 0.22.1", + "http", + "log", + "url", +] + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -464,6 +547,24 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bs58" version = "0.5.1" @@ -522,6 +623,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -530,9 +637,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" @@ -559,6 +666,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -603,7 +720,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -612,6 +729,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cobs" version = "0.3.0" @@ -627,6 +750,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compression-codecs" version = "0.4.38" @@ -663,12 +796,37 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cordyceps" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9ab7e0ca1d179628fa0172b2b97203c7fa0cd81be2448bd446fb9559ca9261" +dependencies = [ + "loom", + "tracing", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -763,6 +921,34 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -772,9 +958,27 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", - "fiat-crypto", + "digest 0.10.7", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto 0.3.0", + "rand_core 0.10.1", "rustc_version", + "serde", "subtle", "zeroize", ] @@ -787,14 +991,34 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "data-encoding-macro" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6a127ecbb3c4632e1525380e04c0c3fcf8dcb44d32a79ea290d8a36906edcd8" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "c54e03a951783e8b327515db3f2a2fd0e3bed362a96b066f341ce66ed49b4ead" +dependencies = [ + "data-encoding", + "syn 2.0.118", +] [[package]] name = "defmt" @@ -815,7 +1039,7 @@ dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -833,10 +1057,50 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "pem-rfc7468", "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.118", + "unicode-xid", +] + [[package]] name = "dialoguer" version = "0.11.0" @@ -850,14 +1114,30 @@ dependencies = [ "zeroize", ] +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", ] [[package]] @@ -881,6 +1161,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -889,7 +1181,18 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "dlopen2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2c5bd4158e66d1e215c49b837e11d62f3267b30c92f1d171c4d3105e3dc4d4" +dependencies = [ + "libc", + "once_cell", + "winapi", ] [[package]] @@ -898,9 +1201,20 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "serde", - "signature", + "signature 2.2.0", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8 0.11.0", + "serdect", + "signature 3.0.0", ] [[package]] @@ -909,10 +1223,26 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ - "curve25519-dalek", - "ed25519", + "curve25519-dalek 4.1.3", + "ed25519 2.2.3", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" +dependencies = [ + "curve25519-dalek 5.0.0", + "ed25519 3.0.0", + "rand_core 0.10.1", "serde", - "sha2", + "sha2 0.11.0", + "signature 3.0.0", "subtle", "zeroize", ] @@ -950,6 +1280,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-assoc" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0590c4a94da3372e83493b956755a6e2266830b6e4e3b101afe66e3f39477b91" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "env_filter" version = "2.0.0" @@ -1019,6 +1360,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "filetime" version = "0.2.29" @@ -1051,6 +1398,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1106,6 +1459,19 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.1", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -1139,6 +1505,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.32" @@ -1147,7 +1526,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1182,16 +1561,31 @@ dependencies = [ ] [[package]] -name = "generic-array" -version = "0.14.7" +name = "generator" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "54ade96dc9003043bce7c035c85a9df5a858bfb2039c5a2e6fdf00f324f6c551" dependencies = [ - "typenum", - "version_check", -] - -[[package]] + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] name = "getrandom" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1230,6 +1624,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "git2" version = "0.19.0" @@ -1258,6 +1662,18 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.15" @@ -1300,6 +1716,9 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" @@ -1320,7 +1739,7 @@ dependencies = [ "hash32", "rustc_version", "serde", - "spin", + "spin 0.9.8", "stable_deref_trait", ] @@ -1369,6 +1788,21 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -1383,6 +1817,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1428,7 +1863,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -1440,7 +1875,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", - "system-configuration", + "system-configuration 0.7.0", "tokio", "tower-service", "tracing", @@ -1553,6 +1988,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "identity-hash" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfdd7caa900436d8f13b2346fe10257e0c05c1f1f9e351f4f5d57c03bd5f45da" + [[package]] name = "idna" version = "1.1.0" @@ -1574,6 +2015,26 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "igd-next" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7238d487a9aff61f81b5ab41c0a841532a115a398b5fa92a2fadd0885e2581" +dependencies = [ + "attohttpc", + "bytes", + "futures 0.3.32", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "rand 0.10.2", + "tokio", + "url", + "xmltree", +] + [[package]] name = "ignore" version = "0.4.28" @@ -1613,12 +2074,197 @@ dependencies = [ "web-time", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "iroh" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f8d1cfffc83efe39a1031aab423ce09cb8048071baba550508931e9a81ce46" +dependencies = [ + "backon", + "blake3", + "bytes", + "cfg_aliases", + "ctutils", + "data-encoding", + "derive_more", + "ed25519-dalek 3.0.0", + "futures-util", + "getrandom 0.4.3", + "http", + "ipnet", + "iroh-base", + "iroh-dns", + "iroh-metrics", + "iroh-relay", + "n0-error", + "n0-future", + "n0-watcher", + "netwatch", + "noq", + "noq-proto", + "noq-udp", + "papaya", + "pin-project", + "portable-atomic", + "portmapper", + "rand 0.10.2", + "reqwest 0.13.5", + "rustc-hash", + "rustls", + "rustls-pki-types", + "serde", + "smallvec", + "strum", + "time", + "tokio", + "tokio-stream", + "tokio-util 0.7.18", + "tracing", + "url", + "wasm-bindgen-futures", +] + +[[package]] +name = "iroh-base" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd1efd1aaecd68d4d1b0727a30c5811f43fb822843e48f65f6e5db3b7256cc9" +dependencies = [ + "curve25519-dalek 5.0.0", + "data-encoding", + "data-encoding-macro", + "derive_more", + "ed25519-dalek 3.0.0", + "getrandom 0.4.3", + "n0-error", + "rand 0.10.2", + "serde", + "url", + "zeroize", +] + +[[package]] +name = "iroh-dns" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80276761b56e904e26ea71d0863beef0bb8d5ab6b639cbd1338e4b615209e3a1" +dependencies = [ + "arc-swap", + "cfg_aliases", + "derive_more", + "iroh-base", + "n0-dns-resolver", + "n0-error", + "n0-future", + "portable-atomic", + "rand 0.10.2", + "rustls", + "simple-dns", + "strum", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "iroh-metrics" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291065721ad7c477b972e581bbc528df031dc8eb5e39fe1ff3300ae5dfb157ef" +dependencies = [ + "iroh-metrics-derive", + "itoa", + "n0-error", + "portable-atomic", + "ryu", + "serde", + "tracing", +] + +[[package]] +name = "iroh-metrics-derive" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae5f0c4405d1fbc9fb16ff422ca40620e93dc36c30ecaba0c2aee3992b7bd48" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "iroh-relay" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beb2294a9749d6a25fd7cd8bcf0fccd932f20716d4f967d85d3f23c7135ae6a2" +dependencies = [ + "blake3", + "bytes", + "cfg_aliases", + "data-encoding", + "derive_more", + "getrandom 0.4.3", + "http", + "http-body-util", + "hyper", + "hyper-util", + "iroh-base", + "iroh-dns", + "iroh-metrics", + "lru", + "n0-error", + "n0-future", + "noq", + "noq-proto", + "num_enum", + "pin-project", + "postcard", + "rand 0.10.2", + "reqwest 0.13.5", + "rustls", + "rustls-pki-types", + "serde", + "serde_bytes", + "strum", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", + "tokio-websockets", + "tracing", + "url", + "webpki-roots", + "ws_stream_wasm", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -1681,7 +2327,81 @@ checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.118", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", ] [[package]] @@ -1705,6 +2425,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.186" @@ -1798,18 +2524,55 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "mac-addr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3d25b0e0b648a86960ac23b7ad4abb9717601dec6f66c165f5b037f3f03065f" + [[package]] name = "maplit" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1852,6 +2615,84 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "n0-dns-resolver" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535c99c9a786bee714155d2ee4fc2477405c2c43536e1bc6a51de09a0d1448f7" +dependencies = [ + "derive_more", + "ipconfig", + "jni 0.22.4", + "lru", + "n0-error", + "n0-future", + "ndk-context", + "rand 0.10.2", + "reqwest 0.13.5", + "rustc-hash", + "rustls", + "simple-dns", + "system-configuration 0.8.0", + "tokio", + "tokio-rustls", + "tracing", + "webpki-roots", +] + +[[package]] +name = "n0-error" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c59ea174675ce6bc196878851e5049e11b8b1f9af3ba87e4d6df8d828bb001" +dependencies = [ + "n0-error-macros", + "spez", +] + +[[package]] +name = "n0-error-macros" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4412346410d6616f3668c40e53c298e5320c7b856f7a960e5914e442601be79f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "n0-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ab99dfb861450e68853d34ae665243a88b8c493d01ba957321a1e9b2312bbe" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util 0.7.18", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "n0-watcher" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc618745ad0b7414b149d0517ad8b5573b2fb4d4e2717add3d2446ce1fdd826" +dependencies = [ + "derive_more", + "n0-error", + "n0-future", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -1869,6 +2710,257 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "netdev" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096c44b66a9b09b99b4dd36b1c295ff3a492039ccecaf55466bd6ddcdba59968" +dependencies = [ + "block2", + "dispatch2", + "dlopen2", + "ipnet", + "jni 0.21.1", + "libc", + "mac-addr", + "ndk-context", + "netlink-packet-core 0.8.2", + "netlink-packet-route 0.31.0", + "netlink-sys 0.8.8", + "objc2", + "objc2-core-foundation", + "objc2-core-wlan", + "objc2-foundation", + "objc2-system-configuration", + "once_cell", + "plist", + "windows-sys 0.61.2", +] + +[[package]] +name = "netdev" +version = "0.46.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c52c2584961c68f5a6e3356797344061f818c93b8acc9cd6d7e284795e563e" +dependencies = [ + "block2", + "dispatch2", + "dlopen2", + "ipnet", + "jni 0.21.1", + "libc", + "mac-addr", + "ndk-context", + "netlink-packet-core 0.9.0", + "netlink-packet-route 0.33.0", + "netlink-sys 0.9.0", + "objc2-core-foundation", + "objc2-system-configuration", + "once_cell", + "plist", + "windows-sys 0.61.2", +] + +[[package]] +name = "netlink-packet-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b897d7bd4f0af82e68d40d0344cf37e97f9c97ddf74a098de3e4da05e96ca395" +dependencies = [ + "paste", +] + +[[package]] +name = "netlink-packet-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6e2daf9a8c2e21302714706860a0e6be02e03d17294ecc66b354ea7d4059dd" + +[[package]] +name = "netlink-packet-route" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2288fcb784eb3defd5fb16f4c4160d5f477de192eac730f43e1d11c24d9a007" +dependencies = [ + "bitflags 2.13.0", + "libc", + "log", + "netlink-packet-core 0.8.2", +] + +[[package]] +name = "netlink-packet-route" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59d48ffc8b73a506e1ff0878528c72668f2bfbc3d875b6be890a96be5d0d5576" +dependencies = [ + "bitflags 2.13.0", + "libc", + "log", + "netlink-packet-core 0.9.0", + "zerocopy", +] + +[[package]] +name = "netlink-proto" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93af8261786086024cd5e96e0a991dd65ced07bbf7c233a487bbc96b971d5539" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "log", + "netlink-packet-core 0.8.2", + "netlink-sys 0.8.8", + "thiserror 2.0.18", +] + +[[package]] +name = "netlink-sys" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6c30ed10fa69cc491d491b85cc971f6bdeb8e7367b7cde2ee6cc878d583fae" +dependencies = [ + "bytes", + "futures-util", + "libc", + "log", + "tokio", +] + +[[package]] +name = "netlink-sys" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c99d38e00b420df49e940fe0826e667c3dad82ac68eb4c2bbf754c1c14a83a10" +dependencies = [ + "bytes", + "libc", + "log", +] + +[[package]] +name = "netwatch" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39da9cad5f23aa43401f09497d3b280e51b5feba115d9ffbf38ca35d6ff95e96" +dependencies = [ + "atomic-waker", + "bytes", + "cfg_aliases", + "derive_more", + "ipnet", + "js-sys", + "libc", + "n0-error", + "n0-future", + "n0-watcher", + "netdev 0.45.1", + "netdev 0.46.3", + "netlink-packet-core 0.8.2", + "netlink-packet-route 0.31.0", + "netlink-proto", + "netlink-sys 0.8.8", + "noq-udp", + "objc2-core-foundation", + "objc2-system-configuration", + "pin-project-lite", + "serde", + "socket2", + "time", + "tokio", + "tokio-util 0.7.18", + "tracing", + "web-sys", + "windows", + "windows-result", + "wmi", +] + +[[package]] +name = "noq" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b78be567e796cfa74bb9bdc4117790af2d505eb887019cf8803244353eb09d89" +dependencies = [ + "bytes", + "cfg_aliases", + "derive_more", + "noq-proto", + "noq-udp", + "pin-project-lite", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "web-time", +] + +[[package]] +name = "noq-proto" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c1e5b6fe668491eca022f745a0a9402585626c73a7b839b3424ace15d6a9c8f" +dependencies = [ + "aes-gcm", + "bytes", + "derive_more", + "enum-assoc", + "getrandom 0.4.3", + "identity-hash", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "sorted-index-buffer", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "noq-udp" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc50afea61aff926e150a36c31f06094608848e114a3fe667d76ec233163b1c" +dependencies = [ + "cfg_aliases", + "libc", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-traits" version = "0.2.19" @@ -1878,12 +2970,124 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "number_prefix" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.0", + "block2", + "dispatch2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-core-wlan" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e34919aba0d701380d911702455038a8a3587467fe0141d6a71501e7ffe48" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-security", + "objc2-security-foundation", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-security" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe137109bd1e8b5a99390f77a7d8b2961dafc1a1c5db8f2e60329ad6d895a" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-security-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef76382e9cedd18123099f17638715cc3d81dba3637d4c0d39ab69df2ef345a5" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "bitflags 2.13.0", + "dispatch2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-security", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1896,6 +3100,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "open" version = "5.4.0" @@ -1928,7 +3138,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1969,7 +3179,23 @@ dependencies = [ name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "papaya" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da2442474a9404698c42509b8967f437249dbc7b50493e83020333d3943ec0ae" +dependencies = [ + "equivalent", + "seize", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -1994,12 +3220,57 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures 0.3.32", + "rustc_version", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2012,8 +3283,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.0", ] [[package]] @@ -2022,11 +3303,39 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plist" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896bade328c13f7042a297ea5ac5b0951f6cf989dea5f32c2fd98da398195cb" +dependencies = [ + "base64 0.23.1", + "indexmap", + "quick-xml", + "serde", + "time", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +dependencies = [ + "serde", +] [[package]] name = "portable-atomic-util" @@ -2037,6 +3346,35 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "portmapper" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca97242f016e090a25330613bc2382aab65eb22f9149dbe84d8f8e711d49a530" +dependencies = [ + "base64 0.22.1", + "bytes", + "derive_more", + "hyper-util", + "igd-next", + "iroh-metrics", + "libc", + "n0-error", + "n0-future", + "netwatch", + "num_enum", + "rand 0.10.2", + "serde", + "smallvec", + "socket2", + "time", + "tokio", + "tokio-util 0.7.18", + "tower-layer", + "tracing", + "url", +] + [[package]] name = "postcard" version = "1.1.3" @@ -2047,9 +3385,21 @@ dependencies = [ "embedded-io 0.4.0", "embedded-io 0.6.1", "heapless", + "postcard-derive", "serde", ] +[[package]] +name = "postcard-derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0232bd009a197ceec9cc881ba46f727fcd8060a2d8d6a9dde7a69030a6fe2bb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -2059,6 +3409,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2068,6 +3424,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.15+spec-1.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -2077,6 +3442,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", +] + [[package]] name = "quickcheck" version = "1.1.0" @@ -2096,7 +3470,7 @@ checksum = "a9a28b8493dd664c8b171dd944da82d933f7d456b829bfb236738e1fe06c5ba4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2366,7 +3740,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -2402,11 +3776,48 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -2479,6 +3890,7 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -2487,6 +3899,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.15.0" @@ -2497,6 +3921,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -2538,6 +3989,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -2567,12 +4024,24 @@ dependencies = [ "libc", ] +[[package]] +name = "seize" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" + [[package]] name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + [[package]] name = "serde" version = "1.0.228" @@ -2624,7 +4093,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2662,6 +4131,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "serial_test" version = "3.5.0" @@ -2684,9 +4163,15 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -2695,7 +4180,27 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", ] [[package]] @@ -2729,18 +4234,52 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "simple-dns" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b6f884fa9a8d48101774bfbd3aeb81e968dd22cffd19a372da69f183db22c1a" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "slab" version = "0.4.12" @@ -2763,6 +4302,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sorted-index-buffer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea06cc588e43c632923a55450401b8f25e628131571d4e1baea1bdfdb2b5ed06" + +[[package]] +name = "spez" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87e960f4dca2788eeb86bbdde8dd246be8948790b7618d656e68f9b720a86e8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "spin" version = "0.9.8" @@ -2772,6 +4328,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -2779,7 +4341,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der 0.8.2", ] [[package]] @@ -2795,10 +4367,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" [[package]] -name = "strsim" -version = "0.11.1" +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "subtle" @@ -2817,6 +4410,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -2834,7 +4438,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2870,6 +4474,17 @@ dependencies = [ "system-configuration-sys", ] +[[package]] +name = "system-configuration" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501336eb7ba9e417300a6a0fa985721065467aa83a6dcf0422a8e43e4c0328fa" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.10.1", + "system-configuration-sys", +] + [[package]] name = "system-configuration-sys" version = "0.6.0" @@ -2940,7 +4555,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2951,7 +4566,47 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "js-sys", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", ] [[package]] @@ -3004,7 +4659,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3027,6 +4682,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util 0.7.18", +] + [[package]] name = "tokio-util" version = "0.6.10" @@ -3052,10 +4719,34 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] +[[package]] +name = "tokio-websockets" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52efb639344a7c6adb8e62c6f3d2c19c001ff1b79a5041ba1c6ed42e19c6aa5" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-sink", + "getrandom 0.4.3", + "http", + "httparse", + "rand 0.10.2", + "ring", + "rustls-pki-types", + "sha1_smol", + "simdutf8", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", +] + [[package]] name = "toml" version = "0.8.23" @@ -3064,8 +4755,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", ] [[package]] @@ -3077,6 +4768,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -3086,9 +4786,30 @@ dependencies = [ "indexmap", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "toml_write", - "winnow", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", ] [[package]] @@ -3153,10 +4874,23 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -3164,6 +4898,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -3294,12 +5058,34 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -3316,6 +5102,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -3348,6 +5135,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3436,7 +5229,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -3462,6 +5255,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "watchman_client" version = "0.9.0" @@ -3500,6 +5306,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.8" @@ -3509,6 +5324,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -3593,7 +5414,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3604,7 +5425,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3652,6 +5473,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -3688,6 +5518,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -3728,6 +5573,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3740,6 +5591,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3752,6 +5609,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3770,6 +5633,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3782,6 +5651,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3794,6 +5669,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3806,6 +5687,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3827,18 +5714,61 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wmi" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c81b85c57a57500e56669586496bf2abd5cf082b9d32995251185d105208b64" +dependencies = [ + "chrono", + "futures 0.3.32", + "log", + "serde", + "thiserror 2.0.18", + "windows", + "windows-core", +] + [[package]] name = "writeable" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "ws_stream_wasm" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc" +dependencies = [ + "async_io_stream", + "futures 0.3.32", + "js-sys", + "log", + "pharos", + "rustc_version", + "send_wrapper", + "thiserror 2.0.18", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "xattr" version = "1.6.1" @@ -3849,6 +5779,21 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + +[[package]] +name = "xmltree" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" +dependencies = [ + "xml-rs", +] + [[package]] name = "xxhash-rust" version = "0.8.16" @@ -3874,7 +5819,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -3895,7 +5840,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3915,7 +5860,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -3924,6 +5869,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "zerotrie" @@ -3955,7 +5914,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 17e609ad..f928bbb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ bs58 = "0.5" blake3 = "1.5" ed25519-dalek = { version = "2.1", features = ["serde"] } rand = "0.8" +iroh = "1" # Compression zstd = "0.13" diff --git a/atomic-cli/Cargo.toml b/atomic-cli/Cargo.toml index a60709a4..2f58f426 100644 --- a/atomic-cli/Cargo.toml +++ b/atomic-cli/Cargo.toml @@ -65,6 +65,8 @@ data-encoding = "2.6" # Hashing (source-content digest for canonical intent sidecars) blake3 = { workspace = true } +iroh = { workspace = true } +rand = { workspace = true } # Open URLs in default browser (for device auth flow) open = "5" diff --git a/atomic-cli/src/commands/agent/mod.rs b/atomic-cli/src/commands/agent/mod.rs index 67fa5202..5f1055f5 100644 --- a/atomic-cli/src/commands/agent/mod.rs +++ b/atomic-cli/src/commands/agent/mod.rs @@ -43,7 +43,7 @@ mod enable; mod explain; mod hooks; mod lifecycle; -mod owner; +pub(crate) mod owner; mod status; use clap::{Args, Subcommand}; diff --git a/atomic-cli/src/commands/agent/owner.rs b/atomic-cli/src/commands/agent/owner/mod.rs similarity index 86% rename from atomic-cli/src/commands/agent/owner.rs rename to atomic-cli/src/commands/agent/owner/mod.rs index 537a0592..84a95bf1 100644 --- a/atomic-cli/src/commands/agent/owner.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -3,6 +3,14 @@ //! redb permits one writable process to open a database. This service elects //! that process with an OS-backed file lock and exposes a small, versioned local //! protocol so short-lived agent hooks never open `changes.redb` themselves. +//! +//! The same protocol reaches remote sandboxes over iroh (see [`remote`] and +//! [`sandbox`]): a sandbox on another machine records provenance and reads its +//! view through the owner of the repository it belongs to, with every request +//! checked against its sandbox token. + +mod remote; +mod sandbox; use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; @@ -34,6 +42,9 @@ use uuid::Uuid; use crate::commands::Command; use crate::error::CliResult; +use self::remote::{RemotePointer, TokenRegistry}; +use self::sandbox::Caller; + const PROTOCOL_VERSION: u16 = 1; // Local IPC only; a single oversized envelope (e.g. a huge recovered reasoning // block) must still fit one frame even after client-side chunking. @@ -106,6 +117,9 @@ struct ReserveArgs { struct RequestFrame { version: u16, request_id: String, + /// A remote sandbox's token; local callers send none. + #[serde(default, skip_serializing_if = "Option::is_none")] + token: Option, request: OwnerRequest, } @@ -175,6 +189,28 @@ enum OwnerRequest { turn_number: u32, }, Shutdown, + /// Local only: mint a sandbox token for `view`, reachable over iroh. + OpenSandbox { + view: String, + #[serde(default)] + acting_as: Option, + ttl_secs: i64, + }, + /// Local only: extend `view`'s sandbox token. + RenewSandbox { + view: String, + ttl_secs: i64, + }, + /// Local only: end `view`'s sandbox token now. + CloseSandbox { + view: String, + }, + /// A view's tree: `MaterializeEntry` frames, then `Materialized`. A + /// remote caller gets its token's view; a local one names it. + Materialize { + #[serde(default)] + view: Option, + }, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -197,6 +233,8 @@ pub(crate) enum OwnerResponse { // Additive v1 capability: old clients ignore it, old owners omit it. #[serde(default)] frozen_envelope_paging: bool, + #[serde(default)] + remote_sandboxes: bool, }, ProvenanceTurn { turn: StoredProvenanceTurn, @@ -221,6 +259,25 @@ pub(crate) enum OwnerResponse { ShuttingDown { pid: u32, }, + SandboxOpened { + remote: iroh::EndpointAddr, + view: String, + token: String, + expires: String, + }, + SandboxRenewed { + expires: String, + }, + SandboxClosed { + revoked: bool, + }, + MaterializeEntry { + entry: atomic_repository::ViewEntry, + }, + Materialized { + snapshot: atomic_repository::ViewSnapshot, + entries: u64, + }, Error { code: String, message: String, @@ -784,6 +841,10 @@ fn runtime() -> anyhow::Result { /// Start the repository owner or reconnect when another process won election. pub(crate) fn start_or_reconnect(repository: &Path) -> anyhow::Result { + // A remote sandbox's owner runs elsewhere; there is nothing to start. + if remote::find_remote_pointer(repository).is_some() { + return request(repository, OwnerRequest::Ping); + } if let Ok(response) = request(repository, OwnerRequest::Ping) { return Ok(response); } @@ -860,16 +921,29 @@ where .map_err(|_| anyhow!("database owner client thread panicked"))? } -/// Send one request to an already-running owner. +/// Send one request to an already-running owner: the local one, or — in a +/// remote sandbox — the one its pointer names, with its token. fn request(repository: &Path, request_body: OwnerRequest) -> anyhow::Result { - let dot_dir = Repository::canonical_dot_dir(repository)?; - let endpoint = endpoint_name(&dot_dir); + let remote = remote::find_remote_pointer(repository).map(|(_, pointer)| pointer); let frame = RequestFrame { version: PROTOCOL_VERSION, request_id: Uuid::new_v4().to_string(), + token: remote.as_ref().map(|pointer| pointer.token.clone()), request: request_body, }; - let response = runtime()?.block_on(exchange(&endpoint, &frame))?; + let response = match &remote { + Some(pointer) => runtime()?.block_on(async { + let mut stream = remote::BiStream::dial(pointer.remote.clone()).await?; + write_frame(&mut stream, &frame).await?; + let response = read_frame::(&mut stream).await; + stream.close().await; + response + })?, + None => { + let dot_dir = Repository::canonical_dot_dir(repository)?; + runtime()?.block_on(exchange(&endpoint_name(&dot_dir), &frame))? + } + }; if response.version != PROTOCOL_VERSION { return Err(anyhow!( "database owner protocol mismatch: client {}, server {}", @@ -892,6 +966,98 @@ fn request(repository: &Path, request_body: OwnerRequest) -> anyhow::Result, + ttl_secs: i64, +) -> anyhow::Result { + start_or_reconnect(repository)?; + match request( + repository, + OwnerRequest::OpenSandbox { + view: view.to_string(), + acting_as, + ttl_secs, + }, + )? { + OwnerResponse::SandboxOpened { + remote, + view, + token, + expires, + } => Ok(OpenedSandbox { + pointer: serde_json::to_value(RemotePointer { + remote, + view, + token, + })?, + expires, + }), + other => Err(unexpected_response("open sandbox", other)), + } +} + +/// Extend `view`'s sandbox token; returns the new expiry. +pub(crate) fn renew_sandbox( + repository: &Path, + view: &str, + ttl_secs: i64, +) -> anyhow::Result { + match request( + repository, + OwnerRequest::RenewSandbox { + view: view.to_string(), + ttl_secs, + }, + )? { + OwnerResponse::SandboxRenewed { expires } => Ok(expires), + other => Err(unexpected_response("renew sandbox", other)), + } +} + +/// End `view`'s sandbox token; whether there was one. +pub(crate) fn close_sandbox(repository: &Path, view: &str) -> anyhow::Result { + match request( + repository, + OwnerRequest::CloseSandbox { + view: view.to_string(), + }, + )? { + OwnerResponse::SandboxClosed { revoked } => Ok(revoked), + other => Err(unexpected_response("close sandbox", other)), + } +} + +/// In a remote sandbox: write its view's tree into the sandbox root. +/// Returns the root and the number of entries written. +pub(crate) fn materialize_remote_sandbox(start: &Path) -> anyhow::Result<(PathBuf, u64)> { + let (root, pointer) = remote::find_remote_pointer(start) + .ok_or_else(|| anyhow!("{} is not in a remote sandbox", start.display()))?; + let frame = RequestFrame { + version: PROTOCOL_VERSION, + request_id: Uuid::new_v4().to_string(), + token: Some(pointer.token.clone()), + request: OwnerRequest::Materialize { view: None }, + }; + let written = runtime()?.block_on(async { + let mut stream = remote::BiStream::dial(pointer.remote.clone()).await?; + write_frame(&mut stream, &frame).await?; + let written = sandbox::receive_materialized(&mut stream, &frame.request_id, &root).await; + stream.close().await; + written + })?; + Ok((root, written)) +} + fn serve(repository: &Path) -> CliResult<()> { let dot_dir = Repository::canonical_dot_dir(repository)?; let owner_lock = acquire_owner_lock(&dot_dir)?; @@ -901,7 +1067,8 @@ fn serve(repository: &Path) -> CliResult<()> { .with_context(|| format!("failed to open {}", store_path.display()))?, ); let endpoint = endpoint_name(&dot_dir); - runtime()?.block_on(run_server(&endpoint, store))?; + let owner = OwnerState::new(store, dot_dir.clone()); + runtime()?.block_on(run_server(&endpoint, owner))?; FileExt::unlock(&owner_lock).context("failed to release database-owner lock")?; Ok(()) } @@ -965,6 +1132,7 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram OwnerResponse::Pong { pid: std::process::id(), frozen_envelope_paging: true, + remote_sandboxes: true, }, false, ), @@ -1298,6 +1466,17 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram }, true, ), + // Answered by `handle_connection`, which needs the whole owner. + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + | OwnerRequest::Materialize { .. } => ( + OwnerResponse::Error { + code: "internal".to_string(), + message: "sandbox requests need the owner's connection handler".to_string(), + }, + false, + ), }; ( @@ -1310,22 +1489,119 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram ) } -async fn handle_connection( - mut stream: S, +/// Everything a running owner holds. +pub(crate) struct OwnerState { store: Arc, + /// The repository root. + root: PathBuf, + dot_dir: PathBuf, + tokens: TokenRegistry, + /// Bound on the first `OpenSandbox`; accepts remote callers from then on. + remote: tokio::sync::OnceCell, shutdown: Arc, +} + +impl OwnerState { + fn new(store: Arc, dot_dir: PathBuf) -> Arc { + let root = dot_dir + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| dot_dir.clone()); + Arc::new(Self { + store, + root, + dot_dir, + tokens: TokenRegistry::default(), + remote: tokio::sync::OnceCell::new(), + shutdown: Arc::new(Notify::new()), + }) + } + + /// The owner's iroh endpoint, bound (and accepting) on first use. + async fn remote_endpoint(self: &Arc) -> anyhow::Result<&iroh::Endpoint> { + self.remote + .get_or_try_init(|| async { + let endpoint = remote::bind(&self.dot_dir, remote::offline()).await?; + spawn_accept_remote(endpoint.clone(), Arc::clone(self)); + Ok(endpoint) + }) + .await + } +} + +/// A plain function between binding and accepting: the accept loop answers +/// `OpenSandbox`, which binds, so the two futures can't contain each other. +fn spawn_accept_remote(endpoint: iroh::Endpoint, owner: Arc) { + tokio::spawn(accept_remote(endpoint, owner)); +} + +/// Answer remote callers: each bi-stream is one request, as on the socket. +async fn accept_remote(endpoint: iroh::Endpoint, owner: Arc) { + while let Some(incoming) = endpoint.accept().await { + let owner = Arc::clone(&owner); + tokio::spawn(async move { + let Ok(connection) = incoming.await else { + return; + }; + while let Some(stream) = remote::BiStream::accept(&connection).await { + let owner = Arc::clone(&owner); + tokio::spawn(async move { + if let Err(error) = handle_connection(stream, owner, Caller::Remote).await { + log::warn!("database-owner remote request failed: {error}"); + } + }); + } + }); + } +} + +async fn handle_connection( + mut stream: S, + owner: Arc, + caller: Caller, ) -> anyhow::Result<()> where S: AsyncRead + AsyncWrite + Unpin, { let request: RequestFrame = read_frame(&mut stream).await?; - let (response, should_shutdown) = handle_request(&store, request); - write_frame(&mut stream, &response).await?; + let respond = |request_id: String, response| ResponseFrame { + version: PROTOCOL_VERSION, + request_id, + response, + }; + let grant = match sandbox::authorize(&owner, caller, &request) { + Ok(grant) => grant, + Err(refusal) => { + write_frame(&mut stream, &respond(request.request_id, *refusal)).await?; + stream.shutdown().await?; + return Ok(()); + } + }; + let mut should_shutdown = false; + match &request.request { + OwnerRequest::Materialize { .. } if request.version == PROTOCOL_VERSION => { + sandbox::materialize(&owner, grant, request, &mut stream).await?; + } + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + if request.version == PROTOCOL_VERSION => + { + let response = sandbox::admin(&owner, request.request.clone()).await; + write_frame(&mut stream, &respond(request.request_id, response)).await?; + } + _ => { + let (response, shutdown) = handle_request(&owner.store, request); + sandbox::note_response(&owner.tokens, grant.as_ref(), &response.response); + write_frame(&mut stream, &response).await?; + should_shutdown = shutdown; + } + } stream.shutdown().await?; if should_shutdown { // `notify_one` retains a permit if the accept loop is between polls, // preventing a shutdown request from being acknowledged but lost. - shutdown.notify_one(); + owner.shutdown.notify_one(); } Ok(()) } @@ -1459,7 +1735,7 @@ async fn connect_owner_pipe( } #[cfg(unix)] -async fn run_server(endpoint: &str, store: Arc) -> anyhow::Result<()> { +async fn run_server(endpoint: &str, owner: Arc) -> anyhow::Result<()> { use tokio::net::UnixListener; let path = Path::new(endpoint); @@ -1469,16 +1745,15 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } let listener = UnixListener::bind(path) .with_context(|| format!("failed to bind database owner at {endpoint}"))?; - let shutdown = Arc::new(Notify::new()); + let shutdown = Arc::clone(&owner.shutdown); loop { tokio::select! { accepted = listener.accept() => { let (stream, _) = accepted?; - let store = Arc::clone(&store); - let shutdown = Arc::clone(&shutdown); + let owner = Arc::clone(&owner); tokio::spawn(async move { - if let Err(error) = handle_connection(stream, store, shutdown).await { + if let Err(error) = handle_connection(stream, owner, Caller::Local).await { log::warn!("database-owner connection failed: {error}"); } }); @@ -1488,6 +1763,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } drop(listener); + if let Some(remote) = owner.remote.get() { + remote.close().await; + } if path.exists() { std::fs::remove_file(path) .with_context(|| format!("failed to remove endpoint {endpoint}"))?; @@ -1496,10 +1774,10 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } #[cfg(windows)] -async fn run_server(endpoint: &str, store: Arc) -> anyhow::Result<()> { +async fn run_server(endpoint: &str, owner: Arc) -> anyhow::Result<()> { use tokio::net::windows::named_pipe::ServerOptions; - let shutdown = Arc::new(Notify::new()); + let shutdown = Arc::clone(&owner.shutdown); let mut server = ServerOptions::new() .first_pipe_instance(true) .create(endpoint) @@ -1515,10 +1793,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu .create(endpoint) .with_context(|| format!("failed to create database owner pipe {endpoint}"))?; let connected = std::mem::replace(&mut server, next); - let store = Arc::clone(&store); - let shutdown = Arc::clone(&shutdown); + let owner = Arc::clone(&owner); tokio::spawn(async move { - if let Err(error) = handle_connection(connected, store, shutdown).await { + if let Err(error) = handle_connection(connected, owner, Caller::Local).await { log::warn!("database-owner connection failed: {error}"); } }); @@ -1526,6 +1803,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu () = shutdown.notified() => break, } } + if let Some(remote) = owner.remote.get() { + remote.close().await; + } Ok(()) } @@ -1698,6 +1978,7 @@ mod tests { let request = RequestFrame { version: PROTOCOL_VERSION, request_id: request_id.clone(), + token: None, request: OwnerRequest::LoadFrozenEnvelopesPage { provenance_id: turn.provenance_id.get(), attempt_generation: attempt.attempt_generation, @@ -1737,7 +2018,8 @@ mod tests { .contains("database-owner shutdown")); assert!(require_frozen_paging(OwnerResponse::Pong { pid: 42, - frozen_envelope_paging: true + frozen_envelope_paging: true, + remote_sandboxes: false, }) .is_ok()); } @@ -1765,6 +2047,7 @@ mod tests { RequestFrame { version: PROTOCOL_VERSION, request_id: "retry".into(), + token: None, request: OwnerRequest::PrepareCheckpoint { provenance_id: turn.provenance_id.get(), expected_generation: generation, @@ -1812,6 +2095,7 @@ mod tests { let expected = RequestFrame { version: PROTOCOL_VERSION, request_id: "request-1".to_string(), + token: None, request: OwnerRequest::Ping, }; let sent = expected.clone(); @@ -1845,6 +2129,7 @@ mod tests { let frame = RequestFrame { version: PROTOCOL_VERSION, request_id: "stale-request".to_string(), + token: None, request: OwnerRequest::AppendProvenanceEnvelopes { provenance_id: running.provenance_id.get(), expected_generation: running.generation, diff --git a/atomic-cli/src/commands/agent/owner/remote.rs b/atomic-cli/src/commands/agent/owner/remote.rs new file mode 100644 index 00000000..94fbbb93 --- /dev/null +++ b/atomic-cli/src/commands/agent/owner/remote.rs @@ -0,0 +1,410 @@ +//! The owner's remote transport: the same protocol over iroh, for sandboxes +//! on another machine. +//! +//! The local socket trusts its caller (the filesystem guards it). A remote +//! caller is trusted only as far as its sandbox token: every frame carries +//! it, and it reaches exactly one view, optionally as one identity, until it +//! expires. Tokens live **in memory only** — they end when the owner exits — +//! and only their hashes are kept. +//! +//! Provenance a remote sandbox records is bound to its token the same way: a +//! session it starts is its own, and it can touch no other. + +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use anyhow::{anyhow, Context}; +use chrono::{DateTime, Duration, Utc}; +use iroh::endpoint::{presets, Connection, RecvStream, SendStream}; +use iroh::{Endpoint, EndpointAddr, SecretKey}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; + +/// ALPN for the owner protocol over iroh. +pub(crate) const ALPN: &[u8] = b"atomic-owner/1"; + +/// The owner's iroh key, kept beside its lock so its address survives a +/// restart and existing pointers keep working. +const IROH_KEY_FILE: &str = "owner-iroh.key"; + +/// What a sandbox token grants. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Grant { + pub(crate) view: String, + /// The identity the sandbox's work is attributed to (e.g. an agent's). + pub(crate) acting_as: Option, + pub(crate) expires: DateTime, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub(crate) enum TokenError { + #[error("unknown or revoked sandbox token")] + Unknown, + #[error("sandbox token expired at {0}")] + Expired(DateTime), + #[error("no live token for view '{0}'")] + NoTokenForView(String), +} + +#[derive(Default)] +struct Tokens { + /// Token digest → grant. A view has at most one live token. + grants: HashMap, + /// Provenance sessions and turns each view's sandbox started. + sessions: HashMap, + provenance: HashMap, +} + +/// In-memory, view-scoped sandbox tokens. Cheap to clone (shared). +#[derive(Clone, Default)] +pub(crate) struct TokenRegistry { + inner: Arc>, +} + +fn digest(token: &str) -> String { + blake3::hash(token.as_bytes()).to_hex().to_string() +} + +impl TokenRegistry { + /// Mint a token for `view`, valid for `ttl`. Minting again replaces the + /// view's token. + pub(crate) fn mint( + &self, + view: &str, + acting_as: Option, + ttl: Duration, + ) -> (String, Grant) { + let mut bytes = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut bytes); + let token = format!("ast_{}", data_encoding::BASE64URL_NOPAD.encode(&bytes)); + let grant = Grant { + view: view.to_string(), + acting_as, + expires: Utc::now() + ttl, + }; + let mut t = self.inner.lock().unwrap(); + t.grants.retain(|_, g| g.view != view); + t.grants.insert(digest(&token), grant.clone()); + (token, grant) + } + + /// Extend `view`'s live token to now + `ttl`; the same token keeps working. + pub(crate) fn renew(&self, view: &str, ttl: Duration) -> Result, TokenError> { + let mut t = self.inner.lock().unwrap(); + let now = Utc::now(); + let grant = t + .grants + .values_mut() + .find(|g| g.view == view && g.expires > now) + .ok_or_else(|| TokenError::NoTokenForView(view.to_string()))?; + grant.expires = now + ttl; + Ok(grant.expires) + } + + /// End `view`'s token now. + pub(crate) fn revoke(&self, view: &str) -> bool { + let mut t = self.inner.lock().unwrap(); + let before = t.grants.len(); + t.grants.retain(|_, g| g.view != view); + t.grants.len() != before + } + + /// What `token` grants, if it is live. + pub(crate) fn check(&self, token: &str) -> Result { + let t = self.inner.lock().unwrap(); + let grant = t.grants.get(&digest(token)).ok_or(TokenError::Unknown)?; + if grant.expires <= Utc::now() { + return Err(TokenError::Expired(grant.expires)); + } + Ok(grant.clone()) + } + + /// Whether `view`'s sandbox may use provenance session `session_id`: + /// one it started, or one nobody has (`fresh`), which becomes its own. + pub(crate) fn claim_session(&self, view: &str, session_id: &str, fresh: bool) -> bool { + let mut t = self.inner.lock().unwrap(); + match t.sessions.get(session_id) { + Some(owner) => owner == view, + None if fresh => { + t.sessions.insert(session_id.to_string(), view.to_string()); + true + } + None => false, + } + } + + pub(crate) fn owns_session(&self, view: &str, session_id: &str) -> bool { + self.inner + .lock() + .unwrap() + .sessions + .get(session_id) + .map(String::as_str) + == Some(view) + } + + pub(crate) fn bind_provenance(&self, view: &str, provenance_id: u64) { + self.inner + .lock() + .unwrap() + .provenance + .insert(provenance_id, view.to_string()); + } + + pub(crate) fn owns_provenance(&self, view: &str, provenance_id: u64) -> bool { + self.inner + .lock() + .unwrap() + .provenance + .get(&provenance_id) + .map(String::as_str) + == Some(view) + } +} + +/// A remote sandbox's `.atomic-sandbox`: where its repository's owner is, +/// the view it works on, and its token. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct RemotePointer { + pub(crate) remote: EndpointAddr, + pub(crate) view: String, + pub(crate) token: String, +} + +/// Walk up from `start` to the nearest `.atomic-sandbox`; if it names a +/// remote owner, return the sandbox root and the pointer. +pub(crate) fn find_remote_pointer(start: &Path) -> Option<(std::path::PathBuf, RemotePointer)> { + let mut dir = std::path::absolute(start).ok()?; + loop { + let path = dir.join(atomic_repository::SANDBOX_POINTER); + if path.is_file() { + let bytes = std::fs::read(&path).ok()?; + return serde_json::from_slice(&bytes).ok().map(|p| (dir, p)); + } + if dir.join(".atomic").is_dir() || !dir.pop() { + return None; + } + } +} + +/// The owner's iroh endpoint, with a key persisted in `dot_dir`. +pub(crate) async fn bind(dot_dir: &Path, offline: bool) -> anyhow::Result { + let key_path = dot_dir.join(IROH_KEY_FILE); + let key = match std::fs::read(&key_path) { + Ok(bytes) => { + let bytes: [u8; 32] = bytes + .try_into() + .map_err(|_| anyhow!("{} is not a 32-byte key", key_path.display()))?; + SecretKey::from_bytes(&bytes) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let key = SecretKey::generate(); + write_private(&key_path, &key.to_bytes())?; + key + } + Err(e) => return Err(e).with_context(|| format!("failed to read {}", key_path.display())), + }; + let builder = if offline { + Endpoint::builder(presets::Minimal) + } else { + Endpoint::builder(presets::N0) + }; + Ok(builder + .secret_key(key) + .alpns(vec![ALPN.to_vec()]) + .bind() + .await?) +} + +/// The address to put in a pointer: once online, what iroh knows (relay and +/// direct addresses); offline, the bound sockets — unspecified ones as +/// loopback, which is all an offline owner can promise. +pub(crate) async fn pointer_addr(endpoint: &Endpoint, offline: bool) -> EndpointAddr { + if !offline { + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), endpoint.online()).await; + } + let addr = endpoint.addr(); + if addr.ip_addrs().next().is_some() || !offline { + return addr; + } + endpoint + .bound_sockets() + .into_iter() + .fold(addr, |addr, mut socket| { + if socket.ip().is_unspecified() { + socket.set_ip(match socket { + std::net::SocketAddr::V4(_) => std::net::Ipv4Addr::LOCALHOST.into(), + std::net::SocketAddr::V6(_) => std::net::Ipv6Addr::LOCALHOST.into(), + }); + } + addr.with_ip_addr(socket) + }) +} + +/// A client endpoint for dialing an owner. +async fn dialer(offline: bool) -> anyhow::Result { + let builder = if offline { + Endpoint::builder(presets::Minimal) + } else { + Endpoint::builder(presets::N0) + }; + Ok(builder.bind().await?) +} + +fn write_private(path: &Path, bytes: &[u8]) -> anyhow::Result<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(path) + .with_context(|| format!("failed to create {}", path.display()))?; + file.write_all(bytes)?; + Ok(()) +} + +/// Whether owners and clients skip iroh's relays and address lookup, and +/// dial by the direct addresses a pointer carries (tests, air-gapped hosts). +pub(crate) fn offline() -> bool { + std::env::var_os("ATOMIC_OWNER_IROH_OFFLINE").is_some() +} + +/// One iroh bi-stream as `AsyncRead + AsyncWrite`: one request, as on the +/// local socket. +pub(crate) struct BiStream { + send: SendStream, + recv: RecvStream, + _connection: Connection, + /// A dialing client's own endpoint, which must outlive the stream. + dialer: Option, +} + +impl BiStream { + /// Dial the owner at `addr` and open one request stream. + pub(crate) async fn dial(addr: EndpointAddr) -> anyhow::Result { + let endpoint = dialer(offline()).await?; + let id = addr.id; + let connection = endpoint + .connect(addr, ALPN) + .await + .with_context(|| format!("database owner {id} is not reachable"))?; + let (send, recv) = connection.open_bi().await?; + Ok(Self { + send, + recv, + _connection: connection, + dialer: Some(endpoint), + }) + } + + /// Done with the request: close the dialing endpoint gracefully. + pub(crate) async fn close(self) { + if let Some(endpoint) = self.dialer { + endpoint.close().await; + } + } + + pub(crate) async fn accept(connection: &Connection) -> Option { + let (send, recv) = connection.accept_bi().await.ok()?; + Some(Self { + send, + recv, + _connection: connection.clone(), + dialer: None, + }) + } +} + +impl tokio::io::AsyncRead for BiStream { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.recv).poll_read(cx, buf) + } +} + +impl tokio::io::AsyncWrite for BiStream { + fn poll_write( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_write(cx, buf) + .map_err(std::io::Error::other) + } + + fn poll_flush( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_flush(cx) + .map_err(std::io::Error::other) + } + + fn poll_shutdown( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_shutdown(cx) + .map_err(std::io::Error::other) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_token_reaches_its_view_until_it_ends() { + let reg = TokenRegistry::default(); + let (t, g) = reg.mint("exp-1", Some("did:key:zAgent".into()), Duration::hours(24)); + assert_eq!(reg.check(&t).unwrap().view, "exp-1"); + assert_eq!(reg.check("ast_made_up"), Err(TokenError::Unknown)); + + std::thread::sleep(std::time::Duration::from_millis(5)); + let after = reg.renew("exp-1", Duration::hours(24)).unwrap(); + assert!(after > g.expires); + assert_eq!(reg.check(&t).unwrap().expires, after); + + assert!(reg.revoke("exp-1")); + assert_eq!(reg.check(&t), Err(TokenError::Unknown)); + assert!(reg.renew("exp-1", Duration::hours(1)).is_err()); + } + + #[test] + fn expired_and_replaced_tokens_are_refused() { + let reg = TokenRegistry::default(); + let (t, _) = reg.mint("exp-1", None, Duration::milliseconds(-1)); + assert!(matches!(reg.check(&t), Err(TokenError::Expired(_)))); + assert!(reg.renew("exp-1", Duration::hours(1)).is_err()); + + let (old, _) = reg.mint("exp-2", None, Duration::hours(1)); + let (new, _) = reg.mint("exp-2", None, Duration::hours(1)); + assert_eq!(reg.check(&old), Err(TokenError::Unknown)); + assert!(reg.check(&new).is_ok()); + } + + #[test] + fn a_session_belongs_to_the_view_that_started_it() { + let reg = TokenRegistry::default(); + assert!(reg.claim_session("exp-1", "s1", true)); + assert!(reg.claim_session("exp-1", "s1", false), "its own, again"); + assert!(!reg.claim_session("exp-2", "s1", true), "another view's"); + assert!( + !reg.claim_session("exp-2", "s-local", false), + "an existing session it didn't start" + ); + assert!(reg.owns_session("exp-1", "s1")); + assert!(!reg.owns_session("exp-2", "s1")); + } +} diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs new file mode 100644 index 00000000..33ad6345 --- /dev/null +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -0,0 +1,471 @@ +//! Remote sandboxes on the owner protocol: who may ask for what, the +//! requests that open and close a sandbox, and materializing its view. +//! +//! A local caller is trusted as it always was. A remote caller (iroh) gets +//! the provenance requests and `Materialize`, each checked against its +//! token: its own view, its own sessions and turns, and checkpoints only for +//! changes its view can see. Opening, renewing and closing sandboxes, and +//! shutting the owner down, are local only. + +use std::path::Path; + +use atomic_core::pristine::{GraphTxnT, ViewTxnT}; +use atomic_core::types::{Base32, Hash}; +use atomic_repository::redb_change_store::RedbChangeStore; +use atomic_repository::{Repository, ViewEntryKind}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; + +use super::remote::{Grant, TokenRegistry}; +use super::{ + read_frame, write_frame, OwnerRequest, OwnerResponse, OwnerState, RequestFrame, ResponseFrame, + PROTOCOL_VERSION, +}; + +/// Who is on the other end of a connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Caller { + /// The local socket or pipe. + Local, + /// iroh: trusted only as far as the frame's token. + Remote, +} + +fn error(code: &str, message: impl Into) -> OwnerResponse { + OwnerResponse::Error { + code: code.to_string(), + message: message.into(), + } +} + +fn refuse(code: &str, message: impl Into) -> Box { + Box::new(error(code, message)) +} + +/// The grant `frame` acts under (`None` for a local caller), or the +/// refusal to send instead. +pub(crate) fn authorize( + owner: &OwnerState, + caller: Caller, + frame: &RequestFrame, +) -> Result, Box> { + if caller == Caller::Local { + return Ok(None); + } + let token = frame + .token + .as_deref() + .ok_or_else(|| refuse("unauthorized", "a remote request needs a sandbox token"))?; + let grant = owner + .tokens + .check(token) + .map_err(|e| refuse("unauthorized", e.to_string()))?; + let view = grant.view.as_str(); + let tokens = &owner.tokens; + let forbidden = |what: &str| { + refuse( + "forbidden", + format!("{what} is not this sandbox's (view '{view}')"), + ) + }; + match &frame.request { + OwnerRequest::Ping => {} + OwnerRequest::Materialize { view: asked } => { + if asked.as_deref().is_some_and(|v| v != view) { + return Err(forbidden("that view")); + } + } + OwnerRequest::ReserveProvenanceTurn { + session_id, + turn_number, + .. + } => { + let fresh = session_is_fresh(&owner.store, session_id, *turn_number) + .map_err(|e| refuse("provenance-store", e.to_string()))?; + if !tokens.claim_session(view, session_id, fresh) { + return Err(forbidden("that session")); + } + } + OwnerRequest::AppendProvenanceEnvelopes { provenance_id, .. } + | OwnerRequest::LoadFrozenEnvelopes { provenance_id } + | OwnerRequest::LoadFrozenEnvelopesPage { provenance_id, .. } + | OwnerRequest::AcknowledgeCheckpoint { provenance_id, .. } => { + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + } + OwnerRequest::PrepareCheckpoint { + provenance_id, + source, + .. + } => { + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + require_on_view(&owner.root, view, &source.change_hashes)?; + } + OwnerRequest::BindCheckpointHash { + provenance_id, + hash, + .. + } => { + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + require_on_view(&owner.root, view, std::slice::from_ref(hash))?; + } + OwnerRequest::StopTurn { session_id, .. } + | OwnerRequest::ResumeTurn { session_id, .. } + | OwnerRequest::AbandonTurn { session_id, .. } + | OwnerRequest::TurnStatus { session_id, .. } => { + if !tokens.owns_session(view, session_id) { + return Err(forbidden("that session")); + } + } + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + | OwnerRequest::Shutdown => { + return Err(refuse("forbidden", "only a local caller may do that")); + } + } + Ok(Some(grant)) +} + +/// Whether no one has recorded provenance for `session_id` up to this turn. +fn session_is_fresh(store: &RedbChangeStore, session_id: &str, turn: u32) -> anyhow::Result { + for t in 1..=turn { + if store.get_provenance_turn_for(session_id, t)?.is_some() { + return Ok(false); + } + } + Ok(true) +} + +/// Refuse unless every change in `hashes` is visible on `view`: a sandbox's +/// provenance can only describe its own view's work. +fn require_on_view(root: &Path, view: &str, hashes: &[Hash]) -> Result<(), Box> { + if hashes.is_empty() { + return Ok(()); + } + let check = || -> anyhow::Result> { + let repo = Repository::open_readonly(root)?; + let txn = repo.pristine().read_txn()?; + let state = txn + .get_view(view)? + .ok_or_else(|| anyhow::anyhow!("view '{view}' no longer exists"))?; + let visible = atomic_repository::collect_visible_change_ids(&txn, &state)?; + for hash in hashes { + match txn.get_internal(hash)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Some(*hash)), + } + } + Ok(None) + }; + match check() { + Ok(None) => Ok(()), + Ok(Some(hash)) => Err(refuse( + "forbidden", + format!("change {} is not on view '{view}'", hash.to_base32()), + )), + Err(e) => Err(refuse("repository", e.to_string())), + } +} + +/// After a remote reservation succeeds, the turn is the sandbox's. +pub(crate) fn note_response( + tokens: &TokenRegistry, + grant: Option<&Grant>, + response: &OwnerResponse, +) { + if let (Some(grant), OwnerResponse::ProvenanceTurn { turn, .. }) = (grant, response) { + tokens.bind_provenance(&grant.view, turn.provenance_id.get()); + } +} + +/// `OpenSandbox`, `RenewSandbox`, `CloseSandbox` (local callers only). +pub(crate) async fn admin( + owner: &std::sync::Arc, + request: OwnerRequest, +) -> OwnerResponse { + match request { + OwnerRequest::OpenSandbox { + view, + acting_as, + ttl_secs, + } => { + match Repository::open_readonly(&owner.root) + .map_err(anyhow::Error::from) + .and_then(|repo| Ok(repo.pristine().read_txn()?.get_view(&view)?)) + { + Ok(Some(_)) => {} + Ok(None) => return error("view", format!("no view '{view}'")), + Err(e) => return error("repository", e.to_string()), + } + let endpoint = match owner.remote_endpoint().await { + Ok(endpoint) => endpoint, + Err(e) => return error("remote", format!("{e:#}")), + }; + let (token, grant) = + owner + .tokens + .mint(&view, acting_as, chrono::Duration::seconds(ttl_secs)); + OwnerResponse::SandboxOpened { + remote: super::remote::pointer_addr(endpoint, super::remote::offline()).await, + view, + token, + expires: grant.expires.to_rfc3339(), + } + } + OwnerRequest::RenewSandbox { view, ttl_secs } => { + match owner + .tokens + .renew(&view, chrono::Duration::seconds(ttl_secs)) + { + Ok(expires) => OwnerResponse::SandboxRenewed { + expires: expires.to_rfc3339(), + }, + Err(e) => error("sandbox-token", e.to_string()), + } + } + OwnerRequest::CloseSandbox { view } => OwnerResponse::SandboxClosed { + revoked: owner.tokens.revoke(&view), + }, + other => error("internal", format!("not a sandbox request: {other:?}")), + } +} + +/// `Materialize`: every entry of the view as its own frame, in path order, +/// then `Materialized`. The one request answered with more than one frame. +pub(crate) async fn materialize( + owner: &OwnerState, + grant: Option, + frame: RequestFrame, + stream: &mut S, +) -> anyhow::Result<()> +where + S: AsyncWrite + Unpin, +{ + let request_id = frame.request_id; + let respond = |response| ResponseFrame { + version: PROTOCOL_VERSION, + request_id: request_id.clone(), + response, + }; + let view = match (grant, frame.request) { + (Some(grant), _) => grant.view, + (None, OwnerRequest::Materialize { view: Some(view) }) => view, + (None, _) => { + return write_frame( + stream, + &respond(error("view", "name the view to materialize")), + ) + .await; + } + }; + let (tx, mut rx) = tokio::sync::mpsc::channel::(64); + let root = owner.root.clone(); + let render = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let repo = Repository::open_readonly(&root)?; + let mut entries = 0u64; + let rendered = repo.materialize_view_entries(&view, |entry| { + entries += 1; + tx.blocking_send(OwnerResponse::MaterializeEntry { entry }) + .map_err(|_| ()) + })?; + match rendered { + Ok(snapshot) => { + let _ = tx.blocking_send(OwnerResponse::Materialized { snapshot, entries }); + Ok(()) + } + Err(()) => Err(anyhow::anyhow!("the client went away")), + } + }); + while let Some(response) = rx.recv().await { + write_frame(stream, &respond(response)).await?; + } + match render.await? { + Ok(()) => Ok(()), + Err(e) => write_frame(stream, &respond(error("materialize", format!("{e:#}")))).await, + } +} + +/// Client side of `Materialize`: write the view's tree into `dir` and return +/// how many entries it had. Files already at those paths are overwritten; +/// nothing else in `dir` is touched. +pub(crate) async fn receive_materialized( + stream: &mut S, + request_id: &str, + dir: &Path, +) -> anyhow::Result +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut written = 0u64; + loop { + let frame: ResponseFrame = read_frame(stream).await?; + if frame.request_id != request_id { + anyhow::bail!("database owner response request id mismatch"); + } + match frame.response { + OwnerResponse::MaterializeEntry { entry } => { + let path = dir.join(&entry.path); + match entry.kind { + ViewEntryKind::Directory => std::fs::create_dir_all(&path)?, + #[cfg(unix)] + ViewEntryKind::Symlink => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let target = String::from_utf8(entry.content.clone())?; + match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(e.into()) + } + _ => {} + } + std::os::unix::fs::symlink(target, &path)?; + } + #[cfg(not(unix))] + ViewEntryKind::Symlink => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, &entry.content)?; + } + ViewEntryKind::File => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, &entry.content)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + &path, + std::fs::Permissions::from_mode(entry.mode as u32), + )?; + } + } + } + written += 1; + } + OwnerResponse::Materialized { entries, .. } => { + stream.shutdown().await.ok(); + if entries != written { + anyhow::bail!("owner sent {written} entries but said {entries}"); + } + return Ok(written); + } + OwnerResponse::Error { code, message } => { + anyhow::bail!("database owner materialize failed [{code}]: {message}") + } + other => anyhow::bail!("unexpected materialize response: {other:?}"), + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use atomic_core::change::ChangeHeader; + use atomic_repository::redb_change_store::ProvenanceCheckpointSource; + use atomic_repository::{InsertOptions, RecordOptions, TrackingOptions}; + + use super::*; + + /// A repository with one change on `dev`, and an owner for it. + fn owner() -> (tempfile::TempDir, Arc, Hash) { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("repo"); + let repo = Repository::init(&root).unwrap(); + std::fs::write(root.join("a.txt"), "a\n").unwrap(); + repo.add("a.txt", TrackingOptions::default()).unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(false); + let outcome = repo.record(ChangeHeader::new("a"), options).unwrap(); + repo.write_recorded(&outcome, InsertOptions::default()) + .unwrap(); + let hash = *outcome.hash(); + drop(repo); + let store = Arc::new(RedbChangeStore::open(dir.path().join("journal.redb")).unwrap()); + (dir, OwnerState::new(store, root.join(".atomic")), hash) + } + + fn frame(token: Option<&str>, request: OwnerRequest) -> RequestFrame { + RequestFrame { + version: PROTOCOL_VERSION, + request_id: "r".into(), + token: token.map(str::to_string), + request, + } + } + + fn prepare(provenance_id: u64, change_hashes: Vec) -> OwnerRequest { + OwnerRequest::PrepareCheckpoint { + provenance_id, + expected_generation: 1, + source: ProvenanceCheckpointSource { + agent_name: "a".into(), + agent_display_name: "A".into(), + agent_vendor: "v".into(), + change_hashes, + previous_provenance: None, + plan_id: None, + ledger_turn_number: 1, + }, + reuse_frozen_changes: false, + now: 0, + } + } + + fn refusal(result: Result, Box>) -> String { + match result.map_err(|e| *e) { + Err(OwnerResponse::Error { code, .. }) => code, + other => panic!("expected a refusal, got {other:?}"), + } + } + + #[test] + fn a_remote_checkpoint_covers_only_its_own_turn_and_its_view_s_changes() { + let (_dir, owner, on_view) = owner(); + let (token, _) = owner.tokens.mint("dev", None, chrono::Duration::hours(1)); + owner.tokens.bind_provenance("dev", 7); + let remote = |request| authorize(&owner, Caller::Remote, &frame(Some(&token), request)); + + assert!(remote(prepare(7, vec![on_view])).unwrap().is_some()); + assert_eq!( + refusal(remote(prepare(8, vec![on_view]))), + "forbidden", + "not its turn" + ); + let elsewhere = Hash::of(b"a change on no view"); + assert_eq!( + refusal(remote(prepare(7, vec![elsewhere]))), + "forbidden", + "not its view's change" + ); + + assert_eq!( + refusal(authorize( + &owner, + Caller::Remote, + &frame(None, OwnerRequest::Ping) + )), + "unauthorized" + ); + assert!( + authorize( + &owner, + Caller::Local, + &frame(None, prepare(8, vec![elsewhere])) + ) + .unwrap() + .is_none(), + "a local caller is trusted as before" + ); + } +} diff --git a/atomic-cli/src/commands/sandbox.rs b/atomic-cli/src/commands/sandbox.rs index 8c91949d..1797e161 100644 --- a/atomic-cli/src/commands/sandbox.rs +++ b/atomic-cli/src/commands/sandbox.rs @@ -14,7 +14,16 @@ //! //! ```text //! atomic sandbox create [--dest ] [--view ] +//! atomic sandbox create --remote [--dest ] [--acting-as ] [--ttl ] +//! atomic sandbox materialize +//! atomic sandbox renew [--ttl ] +//! atomic sandbox close //! ``` +//! +//! A **remote** sandbox is for another machine (a VM, say): it holds only a +//! pointer — the repository owner's iroh address, its view, and a token that +//! reaches that view alone — and talks to the owner over the same protocol +//! local hooks use. `materialize`, run inside it, writes the view's tree. use std::path::PathBuf; @@ -22,6 +31,7 @@ use clap::{Parser, Subcommand}; use atomic_repository::{Repository, SealOptions, StageOptions}; +use crate::commands::agent::owner; use crate::commands::{find_repository_root, Command}; use crate::error::{CliError, CliResult}; @@ -50,6 +60,15 @@ pub enum SandboxCommands { /// Produces a single-layer deployable image of the full merged state — /// "run this exact version" anywhere an OCI runtime is available. Seal(Seal), + + /// Inside a remote sandbox: write its view's tree from the owner. + Materialize(Materialize), + + /// Extend a remote sandbox's token. + Renew(Renew), + + /// End a remote sandbox's token now. + Close(Close), } impl Command for Sandbox { @@ -58,6 +77,9 @@ impl Command for Sandbox { SandboxCommands::Create(cmd) => cmd.run(), SandboxCommands::Stage(cmd) => cmd.run(), SandboxCommands::Seal(cmd) => cmd.run(), + SandboxCommands::Materialize(cmd) => cmd.run(), + SandboxCommands::Renew(cmd) => cmd.run(), + SandboxCommands::Close(cmd) => cmd.run(), } } } @@ -93,8 +115,26 @@ pub struct Create { /// view — isolating each agent's history as well as its files. #[arg(long, value_name = "VIEW")] pub from: Option, + + /// Make a remote sandbox: no working tree here, only a pointer (written + /// into `--dest`, or printed) for a machine that reaches this + /// repository's owner over iroh. + #[arg(long)] + pub remote: bool, + + /// The identity the remote sandbox's work is attributed to. + #[arg(long, value_name = "DID", requires = "remote")] + pub acting_as: Option, + + /// How long the remote sandbox's token lasts, in seconds (renew it with + /// `atomic sandbox renew`). + #[arg(long, value_name = "SECS", default_value_t = DEFAULT_TTL_SECS, requires = "remote")] + pub ttl: i64, } +/// Two hours: long enough to outlast a renewal missed or two. +const DEFAULT_TTL_SECS: i64 = 2 * 60 * 60; + impl Create { fn default_dest(repo_root: &std::path::Path, name: &str) -> PathBuf { let repo_name = repo_root @@ -116,7 +156,7 @@ impl Command for Create { .clone() .unwrap_or_else(|| Self::default_dest(&root, &self.name)); - if dest.exists() { + if !self.remote && dest.exists() { return Err(CliError::InvalidArgument { message: format!("destination already exists: {}", dest.display()), }); @@ -138,6 +178,11 @@ impl Command for Create { (v, false) }; + if self.remote { + drop(repo); + return self.create_remote(&root, &view); + } + let count = repo .provision_sandbox(&dest, &view) .map_err(CliError::Repository)?; @@ -162,6 +207,99 @@ impl Command for Create { } } +impl Create { + fn create_remote(&self, root: &std::path::Path, view: &str) -> CliResult<()> { + let opened = owner::open_sandbox(root, view, self.acting_as.clone(), self.ttl) + .map_err(CliError::Internal)?; + let pointer = serde_json::to_vec_pretty(&opened.pointer).map_err(anyhow::Error::from)?; + match &self.dest { + Some(dest) => { + std::fs::create_dir_all(dest).map_err(anyhow::Error::from)?; + let path = dest.join(atomic_repository::SANDBOX_POINTER); + write_private(&path, &pointer)?; + println!("Remote sandbox '{}' created", self.name); + println!(" Pointer: {}", path.display()); + println!(" View: {view}"); + println!(" Expires: {}", opened.expires); + println!(" Next: run `atomic sandbox materialize` there"); + } + None => println!("{}", String::from_utf8_lossy(&pointer)), + } + Ok(()) + } +} + +/// The pointer carries a token: readable by its owner only. +fn write_private(path: &std::path::Path, bytes: &[u8]) -> CliResult<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path).map_err(anyhow::Error::from)?; + file.write_all(bytes).map_err(anyhow::Error::from)?; + Ok(()) +} + +/// Write a remote sandbox's view from its repository's owner. +#[derive(Parser, Debug)] +pub struct Materialize {} + +impl Command for Materialize { + fn run(&self) -> CliResult<()> { + let cwd = std::env::current_dir().map_err(anyhow::Error::from)?; + let (root, entries) = + owner::materialize_remote_sandbox(&cwd).map_err(CliError::Internal)?; + println!("Materialized {entries} entries into {}", root.display()); + Ok(()) + } +} + +/// Extend a remote sandbox's token. +#[derive(Parser, Debug)] +pub struct Renew { + /// The sandbox's view. + #[arg(value_name = "VIEW")] + pub view: String, + + /// New lifetime from now, in seconds. + #[arg(long, value_name = "SECS", default_value_t = DEFAULT_TTL_SECS)] + pub ttl: i64, +} + +impl Command for Renew { + fn run(&self) -> CliResult<()> { + let root = find_repository_root()?; + let expires = + owner::renew_sandbox(&root, &self.view, self.ttl).map_err(CliError::Internal)?; + println!("Sandbox token for '{}' now expires {expires}", self.view); + Ok(()) + } +} + +/// End a remote sandbox's token now. +#[derive(Parser, Debug)] +pub struct Close { + /// The sandbox's view. + #[arg(value_name = "VIEW")] + pub view: String, +} + +impl Command for Close { + fn run(&self) -> CliResult<()> { + let root = find_repository_root()?; + if owner::close_sandbox(&root, &self.view).map_err(CliError::Internal)? { + println!("Sandbox token for '{}' revoked", self.view); + } else { + println!("No live sandbox token for '{}'", self.view); + } + Ok(()) + } +} + /// Stage a view as a layered OCI image (base + delta). #[derive(Parser, Debug)] pub struct Stage { diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs new file mode 100644 index 00000000..f15ad41e --- /dev/null +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -0,0 +1,197 @@ +//! A remote sandbox talks to its repository's database owner over iroh — +//! the same protocol, with every request checked against its token. +//! +//! Runs offline: the owner and the sandbox dial each other directly on +//! loopback (`ATOMIC_OWNER_IROH_OFFLINE`), no relays. + +use std::path::Path; +use std::process::{Command, Output}; + +use atomic_core::change::ChangeHeader; +use atomic_repository::{InsertOptions, RecordOptions, Repository, TrackingOptions}; +use serde_json::Value; +use tempfile::TempDir; + +fn atomic(cwd: &Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(args) + .arg("--no-color") + .current_dir(cwd) + .env("ATOMIC_OWNER_IROH_OFFLINE", "1") + .output() + .expect("run atomic") +} + +fn ok(output: Output, what: &str) -> String { + assert!( + output.status.success(), + "{what} failed: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn failure(output: Output, what: &str) -> String { + assert!(!output.status.success(), "{what} should have failed"); + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// A repository with two recorded files on `dev`. +fn repository(root: &Path) { + let repo = Repository::init(root).unwrap(); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("README.md"), "hello\n").unwrap(); + std::fs::write(root.join("src/lib.rs"), "pub fn f() {}\n").unwrap(); + repo.add("README.md", TrackingOptions::default()).unwrap(); + repo.add("src/lib.rs", TrackingOptions::default()).unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(false); + let outcome = repo.record(ChangeHeader::new("first"), options).unwrap(); + repo.write_recorded(&outcome, InsertOptions::default()) + .unwrap(); +} + +struct Owner<'a>(&'a Path); + +impl Drop for Owner<'_> { + fn drop(&mut self) { + let _ = atomic( + self.0, + &["agent", "database-owner", "shutdown", "--repository", "."], + ); + } +} + +fn reserve(cwd: &Path, session: &str, turn: &str) -> Output { + atomic( + cwd, + &[ + "agent", + "database-owner", + "reserve", + "--repository", + ".", + "--session-id", + session, + "--turn", + turn, + "--now", + "1700000000", + "--json", + ], + ) +} + +#[test] +fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + + let out = ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--view", + "dev", + "--acting-as", + "did:key:zAgent", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + assert!(out.contains("Remote sandbox 'exp-1' created"), "{out}"); + let pointer_path = vm_dir.join(".atomic-sandbox"); + let pointer: Value = serde_json::from_slice(&std::fs::read(&pointer_path).unwrap()).unwrap(); + assert_eq!(pointer["view"], "dev"); + assert!(pointer["token"].as_str().unwrap().starts_with("ast_")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&pointer_path) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o077, 0, "the pointer holds a token: owner-only"); + } + + // The view's tree, and nothing of the repository. + let out = ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + assert!(out.contains("Materialized 3 entries"), "{out}"); + assert_eq!( + std::fs::read_to_string(vm_dir.join("README.md")).unwrap(), + "hello\n" + ); + assert_eq!( + std::fs::read_to_string(vm_dir.join("src/lib.rs")).unwrap(), + "pub fn f() {}\n" + ); + assert!(!vm_dir.join(".atomic").exists()); + + // Provenance goes over the same protocol, bound to the sandbox's view: + // its own sessions, never the host's. + let reserved: Value = + serde_json::from_str(&ok(reserve(&vm_dir, "vm-session", "1"), "remote reserve")).unwrap(); + assert_eq!(reserved["committed"], true); + ok(reserve(host.path(), "host-session", "1"), "local reserve"); + let refused = failure( + reserve(&vm_dir, "host-session", "2"), + "reserving the host's session", + ); + assert!(refused.contains("forbidden"), "{refused}"); + ok( + reserve(&vm_dir, "vm-session", "2"), + "its own session's next turn", + ); + + // Only a local caller administers sandboxes or the owner. + let refused = failure( + atomic( + &vm_dir, + &["agent", "database-owner", "shutdown", "--repository", "."], + ), + "remote shutdown", + ); + assert!(refused.contains("forbidden"), "{refused}"); + + // A tampered token reaches nothing. + let mut forged = pointer.clone(); + forged["token"] = Value::String("ast_forged".into()); + std::fs::write(&pointer_path, serde_json::to_vec(&forged).unwrap()).unwrap(); + let refused = failure(atomic(&vm_dir, &["sandbox", "materialize"]), "forged token"); + assert!(refused.contains("unauthorized"), "{refused}"); + std::fs::write(&pointer_path, serde_json::to_vec(&pointer).unwrap()).unwrap(); + + // Renewal keeps the same token; closing ends it. + let out = ok( + atomic(host.path(), &["sandbox", "renew", "dev", "--ttl", "600"]), + "renew", + ); + assert!(out.contains("now expires"), "{out}"); + ok( + atomic(&vm_dir, &["sandbox", "materialize"]), + "materialize after renew", + ); + let out = ok(atomic(host.path(), &["sandbox", "close", "dev"]), "close"); + assert!(out.contains("revoked"), "{out}"); + let refused = failure( + atomic(&vm_dir, &["sandbox", "materialize"]), + "closed sandbox", + ); + assert!(refused.contains("unauthorized"), "{refused}"); +} From 9e8d77c31924614c9a45a17a95e92ed0c4222378 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:10:56 -0500 Subject: [PATCH 08/15] Record in a remote sandbox's cache exactly as on the repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A change carries the repository's internal node ids and inode numbers, so a sandbox that records without the repository must read the repository's own rows. atomic-core's pristine::slice exports and imports them byte for byte: a skeleton (the view's tree, inodes and positions, directories, ids of every visible change) and, per record, a graph slice for the inodes it touches (each file's content and name chain, one hop of neighbours so find_block resolves as it does on the repository, its CRDT rows and conflicts). The view is imported flattened, with the repository's own Merkle state. Inodes the cache adds itself start at 2^62, clear of the repository's. The change store can hold content spans without their change files, and consults them first. Repository::{export,import}_sandbox_{skeleton,slice} wrap both sides; sandbox_slice_inodes is what a cache asks for — changed files and the directories on the way to changed or new paths. status: a tracked file missing from disk is "already deleted" only when the graph holds its vertex; a cache has the tree before the graph. The parity test records the same edit on a repository and on a cache built from it — modify, deletions spanning changes, delete, add (existing and new directory), move, move and edit, and a second record after the first lands — and requires identical hashes and bytes. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-core/src/pristine/mod.rs | 1 + atomic-core/src/pristine/slice.rs | 625 ++++++++++++++++++ atomic-core/src/pristine/txn/mod.rs | 1 + atomic-repository/src/changestore/mod.rs | 47 ++ atomic-repository/src/repository/mod.rs | 2 + .../src/repository/remote_cache.rs | 216 ++++++ atomic-repository/src/repository/status.rs | 9 +- .../tests/remote_cache_parity_test.rs | 252 +++++++ 8 files changed, 1152 insertions(+), 1 deletion(-) create mode 100644 atomic-core/src/pristine/slice.rs create mode 100644 atomic-repository/src/repository/remote_cache.rs create mode 100644 atomic-repository/tests/remote_cache_parity_test.rs diff --git a/atomic-core/src/pristine/mod.rs b/atomic-core/src/pristine/mod.rs index a013e1cf..eb52a072 100644 --- a/atomic-core/src/pristine/mod.rs +++ b/atomic-core/src/pristine/mod.rs @@ -150,6 +150,7 @@ mod error; mod inode_graph; pub mod ontology; +pub mod slice; pub(crate) mod span_index; pub mod tables; mod traits; diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs new file mode 100644 index 00000000..30765acb --- /dev/null +++ b/atomic-core/src/pristine/slice.rs @@ -0,0 +1,625 @@ +//! Graph slices: a repository's own pristine rows, copied for a remote +//! sandbox's local cache. +//! +//! A remote sandbox never holds its repository, but `record` has to read the +//! graph to compute a change — and the change it computes carries the +//! repository's internal node ids and inode numbers, so the sandbox must read +//! exactly the rows the repository has. A slice is those rows, byte for byte: +//! +//! - the **skeleton** ([`ReadTxn::export_skeleton`]): the view's tree — +//! paths, inodes and their positions, directories — and the id tables for +//! every change the view can see. Enough for `status` with no graph at all. +//! - a **graph slice** ([`ReadTxn::export_graph_slice`]): for a set of +//! inodes, the GRAPH / INODE_GRAPH rows of each file's content and of its +//! name chain up to the root, one hop of neighbours (so `find_block` finds +//! what it would find on the repository), the file's CRDT rows, and its +//! recorded conflicts on the view. +//! +//! [`WriteTxn::import_skeleton`] and [`WriteTxn::import_graph_slice`] write +//! them into a fresh pristine; [`WriteTxn::import_view_snapshot`] writes the +//! view itself, flattened and with the repository's own Merkle state. +//! +//! Rows are copied, never re-derived, so a read on the cache returns what the +//! same read returns on the repository for every row the slice covers. +//! +//! Edges are not filtered by view: a vertex's key exists only while it has an +//! edge, and `find_block` must see the same vertices the repository sees. +//! Reads that matter filter by view themselves. Content is another matter — +//! the slice carries none; span bytes travel separately, and only for changes +//! the view can see. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use redb::ReadableTable; +use serde::{Deserialize, Serialize}; + +use crate::crdt::tables::{ + BRANCHES, BRANCH_AFTER, BRANCH_LEAVES, BRANCH_VERTEX, INODE_TRUNK, LEAVES, PATH_TRUNK, TRUNKS, + TRUNK_BRANCHES, VERTEX_BRANCH, +}; +use crate::pristine::error::PristineResult; +use crate::pristine::tables::*; +use crate::pristine::traits::{GraphTxnT, ViewScope, ViewState}; +use crate::pristine::txn::{ReadTxn, WriteTxn}; +use crate::types::{ChangePosition, EdgeFlags, GraphNode, Merkle, NodeId, Position}; + +/// Inodes a remote sandbox allocates for itself (`atomic add`) start here, +/// far above any the repository hands out, so the two never collide. +pub const LOCAL_INODE_FLOOR: u64 = 1 << 62; + +/// An EXTERNAL / INTERNAL / NODE_TYPES row: `(id, hash, node type)`. +pub type IdRow = (u64, [u8; 32], Option); + +/// Raw pristine rows. Every field is one table's rows, key and value exactly +/// as stored. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphSlice { + /// EXTERNAL / INTERNAL / NODE_TYPES: `(id, hash, node type)`. + pub ids: Vec, + /// TREE: `path → inode`. + pub tree: Vec<(String, u64)>, + /// REV_TREE: `inode → path`. + pub rev_tree: Vec<(u64, String)>, + /// INODES (REV_INODES is its inverse): `inode → position`. + pub inodes: Vec<(u64, [u8; 16])>, + /// DIRECTORIES: `inode → flags`. + pub directories: Vec<(u64, u8)>, + /// GRAPH: `vertex → edges`. + pub graph: Vec<([u8; 24], Vec<[u8; 24]>)>, + /// INODE_GRAPH: `(inode, vertex) → edges`. + pub inode_graph: Vec<([u8; 32], Vec<[u8; 24]>)>, + pub crdt: CrdtRows, + /// CONFLICTS for the view: `inode → serialized conflicts`. Keyed by inode + /// alone; the view id is the importer's. + pub conflicts: Vec<(u64, Vec)>, +} + +/// A file's CRDT rows (see `crdt::tables`). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct CrdtRows { + pub trunks: Vec<([u8; 12], Vec)>, + pub inode_trunk: Vec<(u64, [u8; 12])>, + pub path_trunk: Vec<(String, [u8; 12])>, + pub trunk_branches: Vec<([u8; 12], Vec<[u8; 12]>)>, + pub branches: Vec<([u8; 12], [u8; 24])>, + pub branch_after: Vec<([u8; 12], [u8; 12])>, + pub branch_vertex: Vec<([u8; 12], [u8; 24])>, + pub vertex_branch: Vec<([u8; 24], [u8; 12])>, + pub branch_leaves: Vec<([u8; 12], Vec<[u8; 12]>)>, + pub leaves: Vec<([u8; 12], [u8; 22])>, +} + +/// A view as a remote sandbox sees it: its name, the repository's Merkle +/// state and change count for it, and the ids of every change it can see. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ViewSnapshotRows { + pub id: u64, + pub name: String, + pub state: Merkle, + pub change_count: u64, + /// In application order. + pub visible: Vec, +} + +fn change_of_edge(edge: &[u8; 24]) -> (u64, u64) { + ( + u64::from_le_bytes(edge[8..16].try_into().unwrap()), + u64::from_le_bytes(edge[16..24].try_into().unwrap()), + ) +} + +fn edge_flags(edge: &[u8; 24]) -> EdgeFlags { + let flag_and_pos = u64::from_le_bytes(edge[0..8].try_into().unwrap()); + EdgeFlags::from_bits_truncate((flag_and_pos >> 56) as u8) +} + +fn edge_dest(edge: &[u8; 24]) -> Position { + let flag_and_pos = u64::from_le_bytes(edge[0..8].try_into().unwrap()); + let change = u64::from_le_bytes(edge[8..16].try_into().unwrap()); + Position::new( + NodeId::new(change), + ChangePosition::new(flag_and_pos & ((1 << 56) - 1)), + ) +} + +fn vertex_key(node: GraphNode) -> [u8; 24] { + encode_vertex(node.change.get(), node.start.get(), node.end.get()) +} + +/// How far a traversal expands from a vertex. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Reach { + /// Its row only. + Row, + /// Its row and its name chain towards the root (a name or a directory). + Up, + /// Its row, and every edge onward except into another inode's content + /// (a file's content). + File, +} + +impl ReadTxn { + /// The view's tree and ids: TREE for the `live` inodes (the paths the + /// view actually has), REV_TREE / INODES / DIRECTORIES for every inode + /// whose position a visible change introduced, and the id rows for every + /// visible change. + pub fn export_skeleton( + &self, + visible: &BTreeSet, + live: &BTreeSet, + ) -> PristineResult { + let mut out = GraphSlice::default(); + let inodes = self.txn.open_table(INODES)?; + let mut kept = BTreeSet::new(); + for row in inodes.iter()? { + let (k, v) = row?; + let (change, _) = decode_position(v.value()); + if change == 0 || visible.contains(&change) { + kept.insert(k.value()); + out.inodes.push((k.value(), *v.value())); + } + } + let rev_tree = self.txn.open_table(REV_TREE)?; + for row in rev_tree.iter()? { + let (k, v) = row?; + if kept.contains(&k.value()) { + out.rev_tree.push((k.value(), v.value().to_string())); + } + } + let tree = self.txn.open_table(TREE)?; + for row in tree.iter()? { + let (k, v) = row?; + if live.contains(&v.value()) && kept.contains(&v.value()) { + out.tree.push((k.value().to_string(), v.value())); + } + } + let directories = self.txn.open_table(DIRECTORIES)?; + for row in directories.iter()? { + let (k, v) = row?; + if kept.contains(&k.value()) { + out.directories.push((k.value(), v.value())); + } + } + out.ids = self.id_rows(visible.iter().copied())?; + Ok(out) + } + + /// The rows `record` reads for `inodes`, on the view `view_id`. + pub fn export_graph_slice(&self, inodes: &[u64], view_id: u64) -> PristineResult { + let mut out = GraphSlice::default(); + let inode_table = self.txn.open_table(INODES)?; + let rev_inodes = self.txn.open_table(REV_INODES)?; + let graph = self.txn.open_multimap_table(GRAPH)?; + let inode_graph = self.txn.open_multimap_table(INODE_GRAPH)?; + let conflicts = self.txn.open_table(CONFLICTS)?; + + let edges_of = |key: &[u8; 24]| -> PristineResult> { + let mut edges = Vec::new(); + for e in graph.get(key)? { + edges.push(*e?.value()); + } + Ok(edges) + }; + let is_inode_vertex = |node: GraphNode| -> PristineResult { + Ok(node.start == node.end + && rev_inodes + .get(&encode_position(node.change.get(), node.start.get()))? + .is_some()) + }; + + // Each vertex once, at the widest reach any path gives it. + let mut reach: BTreeMap<[u8; 24], Reach> = BTreeMap::new(); + let mut queue: VecDeque<(GraphNode, Reach)> = VecDeque::new(); + let enqueue = |node: GraphNode, + r: Reach, + reach: &mut BTreeMap<[u8; 24], Reach>, + queue: &mut VecDeque<(GraphNode, Reach)>| { + if node.is_root() { + return; + } + let key = vertex_key(node); + if reach.get(&key).is_some_and(|have| *have >= r) { + return; + } + reach.insert(key, r); + queue.push_back((node, r)); + }; + + for &inode in inodes { + let Some(pos) = inode_table.get(inode)? else { + continue; + }; + let (change, p) = decode_position(pos.value()); + let seed = GraphNode { + change: NodeId::new(change), + start: ChangePosition::new(p), + end: ChangePosition::new(p), + }; + enqueue(seed, Reach::File, &mut reach, &mut queue); + // INODE_GRAPH may hold content the forward walk doesn't reach. + let lo = encode_inode_vertex(inode, 0, 0, 0); + let hi = encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX); + for row in inode_graph.range::<&[u8; 32]>(&lo..=&hi)? { + let (k, values) = row?; + let key = *k.value(); + let mut edges = Vec::new(); + for e in values { + edges.push(*e?.value()); + } + let (_, c, s, e) = decode_inode_vertex(&key); + out.inode_graph.push((key, edges)); + enqueue( + GraphNode { + change: NodeId::new(c), + start: ChangePosition::new(s), + end: ChangePosition::new(e), + }, + Reach::File, + &mut reach, + &mut queue, + ); + } + if let Some(c) = conflicts.get(&encode_view_seq(view_id, inode))? { + out.conflicts.push((inode, c.value().to_vec())); + } + self.export_crdt(inode, &mut out.crdt)?; + } + + let resolve = |pos: Position, parent: bool| -> Vec> { + let mut found = Vec::new(); + if parent { + if let Ok(n) = self.find_block_end(pos) { + found.push(n); + } + if pos.pos.get() > 0 { + if let Ok(n) = self.find_block(Position::new( + pos.change, + ChangePosition::new(pos.pos.get() - 1), + )) { + found.push(n); + } + } + } else if let Ok(n) = self.find_block(pos) { + found.push(n); + } + found + }; + + while let Some((node, r)) = queue.pop_front() { + if r == Reach::Row { + continue; + } + for edge in edges_of(&vertex_key(node))? { + let flags = edge_flags(&edge); + let parent = flags.contains(EdgeFlags::PARENT); + let folder = flags.contains(EdgeFlags::FOLDER); + for next in resolve(edge_dest(&edge), parent) { + let onward = match r { + Reach::File if parent && folder => Reach::Up, + Reach::File if folder => Reach::Row, + Reach::File if is_inode_vertex(next)? => Reach::Row, + Reach::File => Reach::File, + Reach::Up if parent && folder => Reach::Up, + _ => Reach::Row, + }; + enqueue(next, onward, &mut reach, &mut queue); + } + } + } + + let mut ids = BTreeSet::new(); + for key in reach.keys() { + let edges = edges_of(key)?; + if edges.is_empty() { + continue; + } + ids.insert(decode_vertex(key).0); + for e in &edges { + let (c, by) = change_of_edge(e); + ids.insert(c); + ids.insert(by); + } + out.graph.push((*key, edges)); + } + for (_, edges) in &out.inode_graph { + for e in edges { + let (c, by) = change_of_edge(e); + ids.insert(c); + ids.insert(by); + } + } + for (id, _) in &out.crdt.trunks { + ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + } + for (id, _) in &out.crdt.branches { + ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + } + for (id, _) in &out.crdt.leaves { + ids.insert(u64::from_be_bytes(id[0..8].try_into().unwrap())); + } + ids.remove(&0); + out.ids = self.id_rows(ids.into_iter())?; + Ok(out) + } + + fn export_crdt(&self, inode: u64, out: &mut CrdtRows) -> PristineResult<()> { + let inode_trunk = self.txn.open_table(INODE_TRUNK)?; + let Some(trunk) = inode_trunk.get(inode)? else { + return Ok(()); + }; + let trunk = *trunk.value(); + out.inode_trunk.push((inode, trunk)); + if let Some(t) = self.txn.open_table(TRUNKS)?.get(&trunk)? { + out.trunks.push((trunk, t.value().to_vec())); + } + for row in self.txn.open_table(PATH_TRUNK)?.iter()? { + let (k, v) = row?; + if *v.value() == trunk { + out.path_trunk.push((k.value().to_string(), trunk)); + } + } + let trunk_branches = self.txn.open_multimap_table(TRUNK_BRANCHES)?; + let branches = self.txn.open_table(BRANCHES)?; + let branch_after = self.txn.open_table(BRANCH_AFTER)?; + let branch_vertex = self.txn.open_table(BRANCH_VERTEX)?; + let vertex_branch = self.txn.open_table(VERTEX_BRANCH)?; + let branch_leaves = self.txn.open_multimap_table(BRANCH_LEAVES)?; + let leaves = self.txn.open_table(LEAVES)?; + let mut ids = Vec::new(); + for b in trunk_branches.get(&trunk)? { + ids.push(*b?.value()); + } + for b in &ids { + if let Some(v) = branches.get(b)? { + out.branches.push((*b, *v.value())); + } + if let Some(v) = branch_after.get(b)? { + out.branch_after.push((*b, *v.value())); + } + if let Some(v) = branch_vertex.get(b)? { + let vertex = *v.value(); + out.branch_vertex.push((*b, vertex)); + if let Some(back) = vertex_branch.get(&vertex)? { + out.vertex_branch.push((vertex, *back.value())); + } + } + let mut ls = Vec::new(); + for l in branch_leaves.get(b)? { + let l = *l?.value(); + if let Some(v) = leaves.get(&l)? { + out.leaves.push((l, *v.value())); + } + ls.push(l); + } + if !ls.is_empty() { + out.branch_leaves.push((*b, ls)); + } + } + out.trunk_branches.push((trunk, ids)); + Ok(()) + } + + fn id_rows(&self, ids: impl Iterator) -> PristineResult> { + let external = self.txn.open_table(EXTERNAL)?; + let node_types = self.txn.open_table(NODE_TYPES)?; + let mut out = Vec::new(); + for id in ids { + if let Some(h) = external.get(id)? { + let t = node_types.get(id)?.map(|t| t.value()); + out.push((id, *h.value(), t)); + } + } + Ok(out) + } + + /// The view's snapshot: its own state, and every change it can see. + pub fn export_view_snapshot(&self, view: &ViewState, visible: Vec) -> ViewSnapshotRows { + ViewSnapshotRows { + id: view.id, + name: view.name.clone(), + state: view.state, + change_count: view.change_count, + visible, + } + } +} + +impl WriteTxn<'_> { + /// Replace the tree tables with `slice`'s, keeping inodes this cache + /// allocated itself (at or above [`LOCAL_INODE_FLOOR`]); add its ids. + pub fn import_skeleton(&mut self, slice: &GraphSlice) -> PristineResult<()> { + let local = |inode: u64| inode >= LOCAL_INODE_FLOOR; + { + let mut tree = self.txn.open_table(TREE)?; + tree.retain(|_, inode| local(inode))?; + for (path, inode) in &slice.tree { + tree.insert(path.as_str(), *inode)?; + } + } + { + let mut rev_tree = self.txn.open_table(REV_TREE)?; + rev_tree.retain(|inode, _| local(inode))?; + for (inode, path) in &slice.rev_tree { + rev_tree.insert(*inode, path.as_str())?; + } + } + { + let mut inodes = self.txn.open_table(INODES)?; + inodes.retain(|inode, _| local(inode))?; + let mut rev = self.txn.open_table(REV_INODES)?; + rev.retain(|_, inode| local(inode))?; + for (inode, pos) in &slice.inodes { + inodes.insert(*inode, pos)?; + rev.insert(pos, *inode)?; + } + } + { + let mut dirs = self.txn.open_table(DIRECTORIES)?; + dirs.retain(|inode, _| local(inode))?; + for (inode, flags) in &slice.directories { + dirs.insert(*inode, *flags)?; + } + } + self.import_ids(&slice.ids)?; + // From here on this cache's own inodes (`atomic add`) come from above + // the floor, clear of every inode the repository has or will hand out. + self.next_inode + .fetch_max(LOCAL_INODE_FLOOR, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + /// Drop every graph, CRDT and conflict row, then write `slice`'s. + /// Conflicts are written for `view_id`. + pub fn import_graph_slice(&mut self, slice: &GraphSlice, view_id: u64) -> PristineResult<()> { + self.txn.delete_multimap_table(GRAPH)?; + self.txn.delete_multimap_table(INODE_GRAPH)?; + self.txn.delete_table(CONFLICTS)?; + self.txn.delete_table(TRUNKS)?; + self.txn.delete_table(INODE_TRUNK)?; + self.txn.delete_table(PATH_TRUNK)?; + self.txn.delete_multimap_table(TRUNK_BRANCHES)?; + self.txn.delete_table(BRANCHES)?; + self.txn.delete_table(BRANCH_AFTER)?; + self.txn.delete_table(BRANCH_VERTEX)?; + self.txn.delete_table(VERTEX_BRANCH)?; + self.txn.delete_multimap_table(BRANCH_LEAVES)?; + self.txn.delete_table(LEAVES)?; + + { + let mut t = self.txn.open_multimap_table(GRAPH)?; + for (k, edges) in &slice.graph { + for e in edges { + t.insert(k, e)?; + } + } + } + { + let mut t = self.txn.open_multimap_table(INODE_GRAPH)?; + for (k, edges) in &slice.inode_graph { + for e in edges { + t.insert(k, e)?; + } + } + } + { + let mut t = self.txn.open_table(CONFLICTS)?; + for (inode, c) in &slice.conflicts { + t.insert(&encode_view_seq(view_id, *inode), c.as_slice())?; + } + } + let c = &slice.crdt; + { + let mut t = self.txn.open_table(TRUNKS)?; + for (k, v) in &c.trunks { + t.insert(k, v.as_slice())?; + } + let mut t = self.txn.open_table(INODE_TRUNK)?; + for (k, v) in &c.inode_trunk { + t.insert(*k, v)?; + } + let mut t = self.txn.open_table(PATH_TRUNK)?; + for (k, v) in &c.path_trunk { + t.insert(k.as_str(), v)?; + } + let mut t = self.txn.open_multimap_table(TRUNK_BRANCHES)?; + for (k, vs) in &c.trunk_branches { + for v in vs { + t.insert(k, v)?; + } + } + let mut t = self.txn.open_table(BRANCHES)?; + for (k, v) in &c.branches { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(BRANCH_AFTER)?; + for (k, v) in &c.branch_after { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(BRANCH_VERTEX)?; + for (k, v) in &c.branch_vertex { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(VERTEX_BRANCH)?; + for (k, v) in &c.vertex_branch { + t.insert(k, v)?; + } + let mut t = self.txn.open_multimap_table(BRANCH_LEAVES)?; + for (k, vs) in &c.branch_leaves { + for v in vs { + t.insert(k, v)?; + } + } + let mut t = self.txn.open_table(LEAVES)?; + for (k, v) in &c.leaves { + t.insert(k, v)?; + } + } + self.import_ids(&slice.ids) + } + + fn import_ids(&mut self, ids: &[IdRow]) -> PristineResult<()> { + let mut external = self.txn.open_table(EXTERNAL)?; + let mut internal = self.txn.open_table(INTERNAL)?; + let mut types = self.txn.open_table(NODE_TYPES)?; + for (id, hash, t) in ids { + external.insert(*id, hash)?; + internal.insert(hash, *id)?; + if let Some(t) = t { + types.insert(*id, *t)?; + } + } + let max = ids.iter().map(|(id, _, _)| *id).max().unwrap_or(0); + self.next_node_id + .fetch_max(max + 1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + /// Write `snapshot` as a shared, parentless view with the repository's + /// own id, state and change count — replacing any view with that name or + /// id — and its visible changes as the view's change log. + pub fn import_view_snapshot(&mut self, snapshot: &ViewSnapshotRows) -> PristineResult<()> { + let state = ViewState { + id: snapshot.id, + name: snapshot.name.clone(), + state: snapshot.state, + change_count: snapshot.change_count, + kind: ViewScope::Shared, + parent: None, + }; + { + let mut views = self.txn.open_table(VIEWS)?; + let mut stale = Vec::new(); + for row in views.iter()? { + let (k, v) = row?; + let existing = crate::pristine::txn::deserialize_view_state(v.value())?; + if existing.id == snapshot.id || k.value() == snapshot.name { + stale.push(k.value().to_string()); + } + } + for name in stale { + views.remove(name.as_str())?; + } + let bytes = crate::pristine::txn::serialize_view_state(&state); + views.insert(snapshot.name.as_str(), bytes.as_slice())?; + } + let lo = encode_view_seq(snapshot.id, 0); + let hi = encode_view_seq(snapshot.id, u64::MAX); + { + let mut log = self.txn.open_table(VIEW_CHANGES)?; + log.retain_in::<&[u8; 16], _>(&lo..=&hi, |_, _| false)?; + for (seq, change) in snapshot.visible.iter().enumerate() { + log.insert(&encode_view_seq(snapshot.id, seq as u64), *change)?; + } + } + { + let mut rev = self.txn.open_table(REV_VIEW_CHANGES)?; + rev.retain_in::<&[u8; 16], _>(&lo..=&hi, |_, _| false)?; + for (seq, change) in snapshot.visible.iter().enumerate() { + rev.insert(&encode_view_seq(snapshot.id, *change), seq as u64)?; + } + } + self.next_view_id + .fetch_max(snapshot.id + 1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } +} diff --git a/atomic-core/src/pristine/txn/mod.rs b/atomic-core/src/pristine/txn/mod.rs index 178a2570..bcc62b8c 100644 --- a/atomic-core/src/pristine/txn/mod.rs +++ b/atomic-core/src/pristine/txn/mod.rs @@ -14,6 +14,7 @@ mod read; mod write; pub use helpers::AdjIterator; +pub(crate) use helpers::{deserialize_view_state, serialize_view_state}; pub use pristine::Pristine; pub use read::CachedGraphTxn; pub use read::InodePreloadTxn; diff --git a/atomic-repository/src/changestore/mod.rs b/atomic-repository/src/changestore/mod.rs index 22cc916f..973caf1e 100644 --- a/atomic-repository/src/changestore/mod.rs +++ b/atomic-repository/src/changestore/mod.rs @@ -121,6 +121,32 @@ pub struct ChangeStore { /// We use `RwLock` for thread-safe interior mutability, allowing /// concurrent read access while ensuring exclusive write access. pub(crate) cache: RwLock>, + + /// Content spans held without their change files — a remote sandbox's + /// cache has the bytes of the vertices it was sent, never whole changes. + /// Consulted before the change files. + pub(crate) spans: RwLock, +} + +/// Content bytes by `(change, start)`, each covering `start..end`. +#[derive(Debug, Default)] +pub struct SpanOverlay { + spans: std::collections::BTreeMap<(Hash, u64), (u64, Vec)>, +} + +impl SpanOverlay { + /// The bytes of `start..end` in `hash`'s content, if one held span covers them. + fn copy(&self, hash: &Hash, start: usize, end: usize, buf: &mut [u8]) -> Option { + let (&(h, s), (e, bytes)) = self.spans.range(..=(*hash, start as u64)).next_back()?; + if h != *hash || (end as u64) > *e { + return None; + } + let from = start - s as usize; + let len = end - start; + buf.get_mut(..len)? + .copy_from_slice(bytes.get(from..from + len)?); + Some(len) + } } impl std::fmt::Debug for ChangeStore { @@ -189,6 +215,7 @@ impl ChangeStore { Ok(Self { changes_dir, cache: RwLock::new(LruCache::new(cache_capacity)), + spans: RwLock::new(SpanOverlay::default()), }) } @@ -403,6 +430,21 @@ impl ChangeStore { Ok(change) } + /// Hold `bytes` as `hash`'s content at `start..start + bytes.len()`. + pub fn hold_span(&self, hash: Hash, start: u64, bytes: Vec) { + if let Ok(mut spans) = self.spans.write() { + let end = start + bytes.len() as u64; + spans.spans.insert((hash, start), (end, bytes)); + } + } + + /// Drop every held span. + pub fn clear_spans(&self) { + if let Ok(mut spans) = self.spans.write() { + spans.spans.clear(); + } + } + /// Copy a content span from a change without cloning the full `Change`. /// /// Graph output calls this for every vertex it materializes. Using @@ -416,6 +458,11 @@ impl ChangeStore { end: usize, buf: &mut [u8], ) -> ChangeStoreResult { + if let Ok(spans) = self.spans.read() { + if let Some(n) = spans.copy(hash, start, end, buf) { + return Ok(n); + } + } // Fast path: shared read lock — multiple threads can read concurrently. // peek() doesn't update LRU order, which is an acceptable trade-off // to avoid serializing all readers on a write lock. diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index ec11bfea..7dd98ee9 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -80,6 +80,8 @@ mod deferred_tree; mod filter; mod materialize; pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; +mod remote_cache; +pub use remote_cache::{SandboxSkeleton, SandboxSlice, SpanBytes}; mod sandbox; mod semantic_materialize; mod split; diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs new file mode 100644 index 00000000..cd4e896d --- /dev/null +++ b/atomic-repository/src/repository/remote_cache.rs @@ -0,0 +1,216 @@ +//! A remote sandbox's local cache: the repository's own rows for one view, +//! enough for `status` and `record` without the repository. +//! +//! The serving repository exports ([`Repository::export_sandbox_skeleton`], +//! [`Repository::export_sandbox_slice`]); the sandbox's cache imports +//! ([`Repository::import_sandbox_skeleton`], +//! [`Repository::import_sandbox_slice`]). See `atomic_core::pristine::slice` +//! for what the rows are and why they are copied rather than re-derived. + +use std::collections::BTreeSet; + +use atomic_core::change::ChangeStore as _; +use atomic_core::pristine::slice::{GraphSlice, ViewSnapshotRows}; +use atomic_core::pristine::{decode_vertex, GraphTxnT, MutTxnT, ViewTxnT}; +use atomic_core::types::{GraphNode, Hash, NodeId}; +use serde::{Deserialize, Serialize}; + +use super::Repository; +use crate::RepositoryError; + +/// A view's tree for a fresh cache: the skeleton rows, the view itself, and +/// every entry (with content) the view has. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SandboxSkeleton { + pub view: ViewSnapshotRows, + pub rows: GraphSlice, +} + +/// The rows and content `record` reads for some paths. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SandboxSlice { + pub rows: GraphSlice, + pub spans: Vec, +} + +/// Content bytes of one vertex: `change`'s content at `start..start + len`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpanBytes { + pub change: Hash, + pub start: u64, + #[serde(with = "serde_bytes_b64")] + pub bytes: Vec, +} + +mod serde_bytes_b64 { + use data_encoding::BASE64; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(bytes: &[u8], s: S) -> Result { + s.serialize_str(&BASE64.encode(bytes)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } +} + +fn db(e: impl std::fmt::Display) -> RepositoryError { + RepositoryError::Database(e.to_string()) +} + +impl Repository { + /// The visible change ids of `view`, sorted. + fn sandbox_visible( + &self, + txn: &atomic_core::pristine::ReadTxn, + view: &atomic_core::pristine::ViewState, + ) -> Result, RepositoryError> { + let mut ids: Vec = super::collect_visible_change_ids(txn, view)? + .into_iter() + .map(|id| id.get()) + .collect(); + ids.sort_unstable(); + Ok(ids) + } + + /// Serve side: `view`'s skeleton, for the inodes `live` (the ones its + /// materialized tree holds). + pub fn export_sandbox_skeleton( + &self, + view: &str, + live: &BTreeSet, + ) -> Result { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible = self.sandbox_visible(&txn, &state)?; + let set: BTreeSet = visible.iter().copied().collect(); + let rows = txn.export_skeleton(&set, live).map_err(db)?; + Ok(SandboxSkeleton { + view: txn.export_view_snapshot(&state, visible), + rows, + }) + } + + /// Serve side: what `record` on `view` reads for `inodes`, with the + /// content of every vertex a visible change introduced. + pub fn export_sandbox_slice( + &self, + view: &str, + inodes: &[u64], + ) -> Result { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible: BTreeSet = self.sandbox_visible(&txn, &state)?.into_iter().collect(); + let rows = txn.export_graph_slice(inodes, state.id).map_err(db)?; + let mut spans = Vec::new(); + for (key, _) in &rows.graph { + let (change, start, end) = decode_vertex(key); + if start >= end || !visible.contains(&change) { + continue; + } + let Some(hash) = txn.get_external(NodeId::new(change)).map_err(db)? else { + continue; + }; + let node = GraphNode { + change: NodeId::new(change), + start: atomic_core::types::ChangePosition::new(start), + end: atomic_core::types::ChangePosition::new(end), + }; + let mut bytes = vec![0u8; (end - start) as usize]; + self.change_store + .get_contents(|_| Some(hash), node, &mut bytes) + .map_err(db)?; + spans.push(SpanBytes { + change: hash, + start, + bytes, + }); + } + Ok(SandboxSlice { rows, spans }) + } + + /// Cache side: replace this cache's tree and view with `skeleton`'s. + pub fn import_sandbox_skeleton( + &self, + skeleton: &SandboxSkeleton, + ) -> Result<(), RepositoryError> { + let mut txn = self.pristine.write_txn().map_err(db)?; + txn.import_view_snapshot(&skeleton.view).map_err(db)?; + txn.import_skeleton(&skeleton.rows).map_err(db)?; + txn.commit().map_err(db)?; + Ok(()) + } + + /// Cache side: the inodes `record` will read — each tracked file that + /// differs from its baseline, and every tracked directory on the way to + /// any changed or new path (a new file's parent is where it is added). + /// After [`Repository::reindex_working_copy`] on a fresh tree, a clean + /// file costs nothing here. + pub fn sandbox_slice_inodes(&self) -> Result, RepositoryError> { + use crate::status::{FileStatus, StatusOptions}; + use atomic_core::pristine::slice::LOCAL_INODE_FLOOR; + + let status = self.status(StatusOptions::default())?; + let mut out = BTreeSet::new(); + let keep = |inode: atomic_core::types::Inode, out: &mut BTreeSet| { + if inode.get() < LOCAL_INODE_FLOOR { + out.insert(inode.get()); + } + }; + for entry in status.entries() { + if entry.status() == FileStatus::Clean { + continue; + } + if let Some(inode) = entry.inode() { + keep(inode, &mut out); + } else if let Some(inode) = self.get_file_inode(entry.path())? { + keep(inode, &mut out); + } + let mut dir = entry.path().parent(); + while let Some(d) = dir.filter(|d| !d.as_os_str().is_empty()) { + if let Some(inode) = self.get_file_inode(d)? { + keep(inode, &mut out); + } + dir = d.parent(); + } + } + Ok(out.into_iter().collect()) + } + + /// Cache side: replace this cache's graph rows and held content with + /// `slice`'s. + pub fn import_sandbox_slice(&self, slice: &SandboxSlice) -> Result<(), RepositoryError> { + let view_id = { + let txn = self.pristine.read_txn().map_err(db)?; + txn.get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })? + .id + }; + let mut txn = self.pristine.write_txn().map_err(db)?; + txn.import_graph_slice(&slice.rows, view_id).map_err(db)?; + txn.commit().map_err(db)?; + self.change_store.clear_spans(); + for span in &slice.spans { + self.change_store + .hold_span(span.change, span.start, span.bytes.clone()); + } + Ok(()) + } +} diff --git a/atomic-repository/src/repository/status.rs b/atomic-repository/src/repository/status.rs index af1dfabe..c694263c 100644 --- a/atomic-repository/src/repository/status.rs +++ b/atomic-repository/src/repository/status.rs @@ -285,11 +285,18 @@ impl Repository { // materialize uses) to ask: does this file have // content from this view's perspective? If not, the // deletion is already recorded. + // + // Only a graph that holds the file can say so: a remote + // sandbox's cache has its tree before any of its graph, + // and there no vertex means "ask", not "deleted". if has_graph { if let Some(inode_val) = inode { if let Ok(Some(position)) = txn.inode_position(inode_val) { if let Some(ref ids) = current_view_change_ids { - if !is_file_alive_via_retrieval(&txn, inode_val, position, ids) + if txn.has_vertex(position.inode_node()).unwrap_or(false) + && !is_file_alive_via_retrieval( + &txn, inode_val, position, ids, + ) { // Deletion already recorded — skip found_on_disk.insert(path.clone()); diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs new file mode 100644 index 00000000..e58faf26 --- /dev/null +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -0,0 +1,252 @@ +//! A remote sandbox's cache records the same change the repository would. +//! +//! The cache holds only the repository's own rows for the view (the +//! skeleton) and for the paths being recorded (a slice), plus the content +//! bytes those need. `record` runs unchanged against it, and the change it +//! produces — which carries the repository's internal ids — must be byte for +//! byte the change `record` produces on the repository itself. + +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; + +use atomic_core::change::{Author, ChangeHeader}; +use atomic_repository::{InsertOptions, RecordOptions, Repository, TrackingOptions}; +use tempfile::TempDir; + +fn write(root: &Path, rel: &str, content: &str) { + let path = root.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); +} + +fn header(message: &str) -> ChangeHeader { + ChangeHeader::builder() + .message(message) + .author(Author::new("Test", Some("test@example.com"))) + .build() +} + +fn record_and_apply(repo: &Repository, message: &str) { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(false); + let outcome = repo.record(header(message), options).unwrap(); + repo.write_recorded(&outcome, InsertOptions::default()) + .unwrap(); +} + +/// Record without saving or applying: just the change. +fn change_bytes(repo: &Repository, header: ChangeHeader) -> (String, Vec) { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = repo + .record(header, options) + .unwrap_or_else(|e| panic!("record in {}: {e}", repo.root().display())); + ( + atomic_core::types::Base32::to_base32(outcome.hash()), + outcome.v3_bytes().expect("v3 bytes").to_vec(), + ) +} + +struct Pair { + _dirs: (TempDir, TempDir), + host: Repository, + cache: Repository, +} + +impl Pair { + fn host_root(&self) -> &Path { + self.host.root() + } + fn cache_root(&self) -> &Path { + self.cache.root() + } + + /// Apply the same edit to both working trees. + fn edit(&self, f: impl Fn(&Path)) { + f(self.host_root()); + f(self.cache_root()); + } + + /// Ship the slice for what `record` will touch, as FileStates does: + /// only the inodes the cache asks for. + fn hydrate(&self, view: &str, live: &BTreeSet) { + let inodes = self.cache.sandbox_slice_inodes().unwrap(); + assert!( + inodes.len() < live.len(), + "the cache asks for what changed, not everything: {inodes:?} of {live:?}" + ); + let slice = self.host.export_sandbox_slice(view, &inodes).unwrap(); + self.cache.import_sandbox_slice(&slice).unwrap(); + } +} + +/// A host repository with some history, and a cache materialized from it. +fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeSet) { + let host_dir = TempDir::new().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let host = Repository::init(host_dir.path()).unwrap(); + history(&host); + let view = host.current_view().to_string(); + + // Materialize: the view's files into the cache's working tree, and the + // inodes they are. + let cache = Repository::init(cache_dir.path()).unwrap(); + let mut live = BTreeSet::new(); + host.materialize_view_entries::<()>(&view, |entry| { + live.insert(entry.inode); + let path = cache_dir.path().join(&entry.path); + match entry.kind { + atomic_repository::ViewEntryKind::Directory => fs::create_dir_all(path).unwrap(), + _ => { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + } + Ok(()) + }) + .unwrap() + .unwrap(); + let skeleton = host.export_sandbox_skeleton(&view, &live).unwrap(); + cache.import_sandbox_skeleton(&skeleton).unwrap(); + // The baseline: what was just written is clean. + cache.reindex_working_copy().unwrap(); + ( + Pair { + _dirs: (host_dir, cache_dir), + host, + cache, + }, + view, + live, + ) +} + +fn assert_parity(pair: &Pair, view: &str, live: &BTreeSet, what: &str) { + pair.hydrate(view, live); + let h = header(what); + let host = change_bytes(&pair.host, h.clone()); + let cache = change_bytes(&pair.cache, h); + assert_eq!( + host.0, cache.0, + "{what}: the cache records a different change" + ); + assert_eq!(host.1, cache.1, "{what}: same hash, different bytes"); +} + +fn two_files(repo: &Repository) { + let root = repo.root().to_path_buf(); + write(&root, "README.md", "hello\nworld\n"); + write(&root, "src/lib.rs", "pub fn a() {}\npub fn b() {}\n"); + repo.add("README.md", TrackingOptions::default()).unwrap(); + repo.add("src/lib.rs", TrackingOptions::default()).unwrap(); + record_and_apply(repo, "first"); + write(&root, "README.md", "hello\nthere\nworld\n"); + record_and_apply(repo, "second"); +} + +#[test] +fn modifying_a_file() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "hello\nthere\nbig\nworld\n")); + assert_parity(&pair, &view, &live, "modify"); +} + +#[test] +fn modifying_a_file_changed_by_several_changes() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "world\n")); + assert_parity(&pair, &view, &live, "delete lines from two changes"); +} + +#[test] +fn deleting_a_file() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| fs::remove_file(root.join("src/lib.rs")).unwrap()); + assert_parity(&pair, &view, &live, "delete"); +} + +#[test] +fn adding_a_file_to_an_existing_directory() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "src/new.rs", "pub fn c() {}\n")); + for repo in [&pair.host, &pair.cache] { + repo.add("src/new.rs", TrackingOptions::default()).unwrap(); + } + assert_parity(&pair, &view, &live, "add"); +} + +#[test] +fn adding_a_file_in_a_new_directory() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "docs/guide.md", "# guide\n")); + for repo in [&pair.host, &pair.cache] { + repo.add("docs/guide.md", TrackingOptions::default()) + .unwrap(); + } + assert_parity(&pair, &view, &live, "add in new dir"); +} + +#[test] +fn without_the_slice_the_cache_cannot_record_the_same_change() { + let (pair, _view, _live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "hello\nthere\nbig\nworld\n")); + let h = header("no slice"); + let host = change_bytes(&pair.host, h.clone()); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + match pair.cache.record(h, options) { + Err(_) => {} + Ok(outcome) => assert_ne!( + atomic_core::types::Base32::to_base32(outcome.hash()), + host.0, + "the graph rows are what make the change right" + ), + } + assert!(host.1.len() > 200, "a real change: {} bytes", host.1.len()); +} + +#[test] +fn moving_a_file() { + // As `atomic mv` does: the rename on disk, detected by content. + let (pair, view, live) = pair(two_files); + for repo in [&pair.host, &pair.cache] { + fs::rename( + repo.root().join("src/lib.rs"), + repo.root().join("src/core.rs"), + ) + .unwrap(); + } + assert_parity(&pair, &view, &live, "move"); +} + +#[test] +fn moving_and_editing_a_file() { + let (pair, view, live) = pair(two_files); + for repo in [&pair.host, &pair.cache] { + fs::rename(repo.root().join("README.md"), repo.root().join("READ.md")).unwrap(); + write(repo.root(), "READ.md", "hello\nthere\nworld\nagain\n"); + } + assert_parity(&pair, &view, &live, "move and edit"); +} + +#[test] +fn a_second_record_on_top_of_the_first() { + // After the first change lands on the repository, the cache takes a fresh + // skeleton and slice and records the next one identically. + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "src/lib.rs", "pub fn a() {}\n")); + assert_parity(&pair, &view, &live, "first"); + record_and_apply(&pair.host, "first"); + let skeleton = pair.host.export_sandbox_skeleton(&view, &live).unwrap(); + pair.cache.import_sandbox_skeleton(&skeleton).unwrap(); + pair.cache.reindex_working_copy().unwrap(); + pair.edit(|root| write(root, "src/lib.rs", "pub fn a() { 1 }\n")); + assert_parity(&pair, &view, &live, "second"); +} From ae0537a4b8d51298433b09f615dfe23ee8275393 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:23:38 -0500 Subject: [PATCH 09/15] Record in a remote sandbox and land the change through the owner Two more requests on the owner protocol, checked against the sandbox's token like the rest: FileStates hands the cache the repository's rows and content for the inodes record will read (only inodes on the view), and SubmitChange takes the V3 bytes the cache recorded. The owner applies a submitted change only if it is what it claims and names nothing outside the view: the bytes hash to the claimed hash; the view is still at the state it was recorded against; every change it depends on or refers to that the repository knows is visible on the view; every node its file operations name is a visible change's (FileOps::referenced_node_ids); no path touches .atomic, .atomic-sandbox or .atomic-sandbox.d; and it is new. Submissions go in one at a time. Refusals write nothing. In the sandbox, materialize also builds the cache: an ordinary repository under .atomic-sandbox.d/cache whose working tree is the sandbox, so a remote pointer opens it and status, log and diff work locally. record hydrates first, records without saving or applying, submits, then takes the view's new skeleton and marks the tree clean. When a change lands, a path the sandbox added gets the repository's inode. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-cli/src/commands/agent/owner/mod.rs | 106 +++++++++- .../src/commands/agent/owner/sandbox.rs | 96 ++++++++- atomic-cli/src/commands/record/command.rs | 28 ++- .../tests/remote_sandbox_integration_test.rs | 58 ++++++ atomic-core/src/change/ops.rs | 59 ++++++ atomic-core/src/pristine/slice.rs | 12 ++ atomic-repository/src/repository/mod.rs | 7 +- .../src/repository/remote_cache.rs | 169 +++++++++++++++- atomic-repository/src/repository/sandbox.rs | 51 ++++- .../tests/remote_cache_parity_test.rs | 188 ++++++++++++++++++ 10 files changed, 759 insertions(+), 15 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/mod.rs b/atomic-cli/src/commands/agent/owner/mod.rs index 84a95bf1..17eb7494 100644 --- a/atomic-cli/src/commands/agent/owner/mod.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -211,6 +211,38 @@ enum OwnerRequest { #[serde(default)] view: Option, }, + /// The repository's rows (and content) `record` reads for `inodes` — + /// what a sandbox's cache loads before recording. + FileStates { + #[serde(default)] + view: Option, + inodes: Vec, + }, + /// A change a sandbox recorded against `base_state`, as its V3 bytes. + SubmitChange { + #[serde(default)] + view: Option, + base_state: String, + hash: Hash, + #[serde(with = "base64_bytes")] + bytes: Vec, + }, +} + +/// Bytes as base64 in the JSON frames. +mod base64_bytes { + use serde::{Deserialize, Deserializer, Serializer}; + + pub(super) fn serialize(bytes: &[u8], s: S) -> Result { + s.serialize_str(&data_encoding::BASE64.encode(bytes)) + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + data_encoding::BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -277,6 +309,19 @@ pub(crate) enum OwnerResponse { Materialized { snapshot: atomic_repository::ViewSnapshot, entries: u64, + /// The view's rows, for the sandbox's cache. + skeleton: Box, + }, + FileStates { + slice: Box, + }, + ChangeSubmitted { + submitted: atomic_repository::Submitted, + /// The view's rows now, for the sandbox's cache. + skeleton: Box, + }, + ChangeRefused { + rejection: atomic_repository::SubmitRejection, }, Error { code: String, @@ -1037,8 +1082,8 @@ pub(crate) fn close_sandbox(repository: &Path, view: &str) -> anyhow::Result anyhow::Result<(PathBuf, u64)> { let (root, pointer) = remote::find_remote_pointer(start) .ok_or_else(|| anyhow!("{} is not in a remote sandbox", start.display()))?; @@ -1055,9 +1100,51 @@ pub(crate) fn materialize_remote_sandbox(start: &Path) -> anyhow::Result<(PathBu stream.close().await; written })?; + let (written, skeleton) = written; + Repository::create_remote_sandbox_cache(&root, &skeleton)?; Ok((root, written)) } +/// In a remote sandbox: load what `record` will read into its cache. +pub(crate) fn hydrate_remote_sandbox(repo: &Repository) -> anyhow::Result<()> { + let inodes = repo.sandbox_slice_inodes()?; + match request(repo.root(), OwnerRequest::FileStates { view: None, inodes })? { + OwnerResponse::FileStates { slice } => Ok(repo.import_sandbox_slice(&slice)?), + other => Err(unexpected_response("file states", other)), + } +} + +/// In a remote sandbox: hand a recorded change to the repository's owner. +/// On success the cache takes the view as it now is. +pub(crate) fn submit_from_remote_sandbox( + repo: &Repository, + hash: Hash, + bytes: Vec, +) -> anyhow::Result { + let base_state = repo.remote_sandbox_view_state()?; + match request( + repo.root(), + OwnerRequest::SubmitChange { + view: None, + base_state, + hash, + bytes, + }, + )? { + OwnerResponse::ChangeSubmitted { + submitted, + skeleton, + } => { + repo.import_sandbox_skeleton(&skeleton)?; + Ok(submitted) + } + OwnerResponse::ChangeRefused { rejection } => { + Err(anyhow!("the repository refused the change: {rejection}")) + } + other => Err(unexpected_response("submit change", other)), + } +} + fn serve(repository: &Path) -> CliResult<()> { let dot_dir = Repository::canonical_dot_dir(repository)?; let owner_lock = acquire_owner_lock(&dot_dir)?; @@ -1470,7 +1557,9 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram OwnerRequest::OpenSandbox { .. } | OwnerRequest::RenewSandbox { .. } | OwnerRequest::CloseSandbox { .. } - | OwnerRequest::Materialize { .. } => ( + | OwnerRequest::Materialize { .. } + | OwnerRequest::FileStates { .. } + | OwnerRequest::SubmitChange { .. } => ( OwnerResponse::Error { code: "internal".to_string(), message: "sandbox requests need the owner's connection handler".to_string(), @@ -1498,6 +1587,9 @@ pub(crate) struct OwnerState { tokens: TokenRegistry, /// Bound on the first `OpenSandbox`; accepts remote callers from then on. remote: tokio::sync::OnceCell, + /// Submitted changes go in one at a time: each is checked against the + /// view's state as it is when it's applied. + submissions: tokio::sync::Mutex<()>, shutdown: Arc, } @@ -1513,6 +1605,7 @@ impl OwnerState { dot_dir, tokens: TokenRegistry::default(), remote: tokio::sync::OnceCell::new(), + submissions: tokio::sync::Mutex::new(()), shutdown: Arc::new(Notify::new()), }) } @@ -1590,6 +1683,13 @@ where let response = sandbox::admin(&owner, request.request.clone()).await; write_frame(&mut stream, &respond(request.request_id, response)).await?; } + OwnerRequest::FileStates { .. } | OwnerRequest::SubmitChange { .. } + if request.version == PROTOCOL_VERSION => + { + let request_id = request.request_id.clone(); + let response = sandbox::record_request(&owner, grant, request.request).await; + write_frame(&mut stream, &respond(request_id, response)).await?; + } _ => { let (response, shutdown) = handle_request(&owner.store, request); sandbox::note_response(&owner.tokens, grant.as_ref(), &response.response); diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index 33ad6345..18e61684 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -69,7 +69,9 @@ pub(crate) fn authorize( }; match &frame.request { OwnerRequest::Ping => {} - OwnerRequest::Materialize { view: asked } => { + OwnerRequest::Materialize { view: asked } + | OwnerRequest::FileStates { view: asked, .. } + | OwnerRequest::SubmitChange { view: asked, .. } => { if asked.as_deref().is_some_and(|v| v != view) { return Err(forbidden("that view")); } @@ -268,14 +270,21 @@ where let render = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { let repo = Repository::open_readonly(&root)?; let mut entries = 0u64; + let mut live = std::collections::BTreeSet::new(); let rendered = repo.materialize_view_entries(&view, |entry| { entries += 1; + live.insert(entry.inode); tx.blocking_send(OwnerResponse::MaterializeEntry { entry }) .map_err(|_| ()) })?; match rendered { Ok(snapshot) => { - let _ = tx.blocking_send(OwnerResponse::Materialized { snapshot, entries }); + let skeleton = Box::new(repo.export_sandbox_skeleton(&view, &live)?); + let _ = tx.blocking_send(OwnerResponse::Materialized { + snapshot, + entries, + skeleton, + }); Ok(()) } Err(()) => Err(anyhow::anyhow!("the client went away")), @@ -290,6 +299,81 @@ where } } +/// `FileStates` and `SubmitChange`: the view is the token's (remote) or +/// the one named (local). +pub(crate) async fn record_request( + owner: &OwnerState, + grant: Option, + request: OwnerRequest, +) -> OwnerResponse { + let named = match &request { + OwnerRequest::FileStates { view, .. } | OwnerRequest::SubmitChange { view, .. } => { + view.clone() + } + _ => None, + }; + let Some(view) = grant.map(|g| g.view).or(named) else { + return error("view", "name the view"); + }; + let root = owner.root.clone(); + match request { + OwnerRequest::FileStates { inodes, .. } => { + let read = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = Repository::open_readonly(&root)?; + Ok(repo.export_sandbox_slice(&view, &inodes)?) + }); + match read.await { + Ok(Ok(slice)) => OwnerResponse::FileStates { + slice: Box::new(slice), + }, + Ok(Err(e)) => error("file-states", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } + OwnerRequest::SubmitChange { + base_state, + hash, + bytes, + .. + } => { + let _one_at_a_time = owner.submissions.lock().await; + let write = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = + Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; + match repo.insert_submitted_change(&view, &base_state, &hash, &bytes)? { + Ok(submitted) => { + let skeleton = + repo.export_sandbox_skeleton(&view, &live_inodes(&repo, &view)?)?; + Ok(Ok((submitted, skeleton))) + } + Err(rejection) => Ok(Err(rejection)), + } + }); + match write.await { + Ok(Ok(Ok((submitted, skeleton)))) => OwnerResponse::ChangeSubmitted { + submitted, + skeleton: Box::new(skeleton), + }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Err(e)) => error("submit", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } + other => error("internal", format!("not a record request: {other:?}")), + } +} + +/// The inodes `view`'s tree holds. +fn live_inodes(repo: &Repository, view: &str) -> anyhow::Result> { + let mut live = std::collections::BTreeSet::new(); + repo.materialize_view_entries::<()>(view, |entry| { + live.insert(entry.inode); + Ok(()) + })? + .map_err(|()| anyhow::anyhow!("unreachable"))?; + Ok(live) +} + /// Client side of `Materialize`: write the view's tree into `dir` and return /// how many entries it had. Files already at those paths are overwritten; /// nothing else in `dir` is touched. @@ -297,7 +381,7 @@ pub(crate) async fn receive_materialized( stream: &mut S, request_id: &str, dir: &Path, -) -> anyhow::Result +) -> anyhow::Result<(u64, atomic_repository::SandboxSkeleton)> where S: AsyncRead + AsyncWrite + Unpin, { @@ -350,12 +434,14 @@ where } written += 1; } - OwnerResponse::Materialized { entries, .. } => { + OwnerResponse::Materialized { + entries, skeleton, .. + } => { stream.shutdown().await.ok(); if entries != written { anyhow::bail!("owner sent {written} entries but said {entries}"); } - return Ok(written); + return Ok((written, *skeleton)); } OwnerResponse::Error { code, message } => { anyhow::bail!("database owner materialize failed [{code}]: {message}") diff --git a/atomic-cli/src/commands/record/command.rs b/atomic-cli/src/commands/record/command.rs index 912f525f..1595d49d 100644 --- a/atomic-cli/src/commands/record/command.rs +++ b/atomic-cli/src/commands/record/command.rs @@ -40,7 +40,13 @@ impl Command for Record { let header = header_builder.build(); // Build record options - let options = self.build_options()?; + let mut options = self.build_options()?; + // A remote sandbox records against its cache and hands the change + // to the repository's owner; nothing is saved or applied here. + let remote = repo.is_remote_sandbox(); + if remote { + options = options.save_to_store(false).apply_after_record(false); + } // If --all, first add all untracked files if self.all { @@ -56,6 +62,11 @@ impl Command for Record { } } + if remote { + crate::commands::agent::owner::hydrate_remote_sandbox(&repo) + .map_err(CliError::Internal)?; + } + // Record the changes. // // This match is deliberately exhaustive: no `_ =>` arm. The catch-all @@ -106,6 +117,21 @@ impl Command for Record { } })?; + if remote { + let bytes = outcome + .v3_bytes() + .ok_or_else(|| CliError::Internal(anyhow::anyhow!("the change has no bytes")))? + .to_vec(); + crate::commands::agent::owner::submit_from_remote_sandbox( + &repo, + *outcome.hash(), + bytes, + ) + .map_err(CliError::Internal)?; + // What was just recorded is the new baseline. + repo.reindex_working_copy().map_err(CliError::Repository)?; + } + // Display result let output = self.format_outcome(&repo, &outcome); print!("{}", output); diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index f15ad41e..bb4ed348 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -177,6 +177,64 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { assert!(refused.contains("unauthorized"), "{refused}"); std::fs::write(&pointer_path, serde_json::to_vec(&pointer).unwrap()).unwrap(); + // Recording in the sandbox: its cache computes the change, the owner + // checks it and applies it to the view. + std::fs::write(vm_dir.join("README.md"), "hello\nfrom the sandbox\n").unwrap(); + std::fs::write(vm_dir.join("src/new.rs"), "pub fn n() {}\n").unwrap(); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "from the sandbox"]), + "record in the sandbox", + ); + let status = ok(atomic(&vm_dir, &["status"]), "status after record"); + assert!(status.contains("working tree clean"), "{status}"); + std::fs::write(vm_dir.join("src/new.rs"), "pub fn n() { 2 }\n").unwrap(); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "again from the sandbox"]), + "second record in the sandbox", + ); + let log = ok(atomic(host.path(), &["log"]), "host log"); + assert!( + log.contains("from the sandbox") && log.contains("again from the sandbox"), + "{log}" + ); + + // A second sandbox of the view sees the first one's work. + let other = vm.path().join("other"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-2", + "--remote", + "--view", + "dev", + "--dest", + other.to_str().unwrap(), + ], + ), + "second sandbox", + ); + ok( + atomic(&other, &["sandbox", "materialize"]), + "materialize the second", + ); + assert_eq!( + std::fs::read_to_string(other.join("README.md")).unwrap(), + "hello\nfrom the sandbox\n" + ); + assert_eq!( + std::fs::read_to_string(other.join("src/new.rs")).unwrap(), + "pub fn n() { 2 }\n" + ); + // (Minting for the same view replaced the first sandbox's token.) + std::fs::write( + &pointer_path, + std::fs::read(other.join(".atomic-sandbox")).unwrap(), + ) + .unwrap(); + // Renewal keeps the same token; closing ends it. let out = ok( atomic(host.path(), &["sandbox", "renew", "dev", "--ttl", "600"]), diff --git a/atomic-core/src/change/ops.rs b/atomic-core/src/change/ops.rs index aef44ae0..324adfd1 100644 --- a/atomic-core/src/change/ops.rs +++ b/atomic-core/src/change/ops.rs @@ -213,6 +213,65 @@ impl FileOps { &mut self.line_ops } + /// Every existing node these operations name — the changes that created + /// the file, lines and tokens they touch. Ids this change creates carry + /// the placeholder `NodeId::ROOT` until it is applied, and are left out. + /// + /// These ids are applied as they are, so whoever applies a change it did + /// not record checks them (a remote sandbox's change may name only its + /// own view's nodes). + pub fn referenced_node_ids(&self) -> std::collections::BTreeSet { + let mut ids = std::collections::BTreeSet::new(); + let leaf = |op: &LeafOp, ids: &mut std::collections::BTreeSet<_>| match op { + LeafOp::Insert { after, .. } => ids.extend(after.map(|l| l.change_id())), + LeafOp::Delete { leaf } | LeafOp::Replace { leaf, .. } | LeafOp::Restore { leaf } => { + ids.insert(leaf.change_id()); + } + }; + ids.insert(self.trunk_id.change_id()); + match &self.trunk_op { + Some(TrunkOp::Delete { trunk }) + | Some(TrunkOp::Move { trunk, .. }) + | Some(TrunkOp::Undelete { trunk }) => { + ids.insert(trunk.change_id()); + } + Some(TrunkOp::Create { .. }) | None => {} + } + for line in &self.line_ops { + ids.insert(line.branch_id().change_id()); + match line.operation() { + BranchOp::Insert { after, content } => { + ids.extend(after.map(|b| b.change_id())); + content.iter().for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Delete { branch, content } => { + ids.insert(branch.change_id()); + content.iter().for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Modify { + branch, + old_content, + new_content, + } => { + ids.insert(branch.change_id()); + old_content + .iter() + .chain(new_content) + .for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Restore { branch } => { + ids.insert(branch.change_id()); + } + BranchOp::Reparent { branch, new_after } => { + ids.insert(branch.change_id()); + ids.extend(new_after.map(|b| b.change_id())); + } + } + } + ids.remove(&crate::types::NodeId::ROOT); + ids + } + /// Adds a line operation. pub fn add_line_op(&mut self, op: LineOps) { self.line_ops.push(op); diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs index 30765acb..4a6888a7 100644 --- a/atomic-core/src/pristine/slice.rs +++ b/atomic-core/src/pristine/slice.rs @@ -429,6 +429,18 @@ impl WriteTxn<'_> { /// allocated itself (at or above [`LOCAL_INODE_FLOOR`]); add its ids. pub fn import_skeleton(&mut self, slice: &GraphSlice) -> PristineResult<()> { let local = |inode: u64| inode >= LOCAL_INODE_FLOOR; + // A path the cache added itself that the repository now has (its + // change landed) is the repository's inode from here on. + let arrived: std::collections::HashSet<&str> = + slice.tree.iter().map(|(path, _)| path.as_str()).collect(); + let mut superseded = BTreeSet::new(); + for row in self.txn.open_table(REV_TREE)?.iter()? { + let (inode, path) = row?; + if local(inode.value()) && arrived.contains(path.value()) { + superseded.insert(inode.value()); + } + } + let local = |inode: u64| local(inode) && !superseded.contains(&inode); { let mut tree = self.txn.open_table(TREE)?; tree.retain(|_, inode| local(inode))?; diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 7dd98ee9..ec71760e 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -81,7 +81,7 @@ mod filter; mod materialize; pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; mod remote_cache; -pub use remote_cache::{SandboxSkeleton, SandboxSlice, SpanBytes}; +pub use remote_cache::{SandboxSkeleton, SandboxSlice, SpanBytes, SubmitRejection, Submitted}; mod sandbox; mod semantic_materialize; mod split; @@ -94,7 +94,10 @@ pub use filter::{ collect_view_change_ids, collect_visible_change_ids, collect_visible_change_ids_with_deps, expand_indexed_dependency_closure, view_set_id, }; -pub use sandbox::{SealOptions, SealResult, StageOptions, StageResult, SANDBOX_POINTER}; +pub use sandbox::{ + remote_cache_root, SealOptions, SealResult, StageOptions, StageResult, SANDBOX_CACHE_DIR, + SANDBOX_POINTER, +}; pub use split::{SplitChange, SplitOptions, SplitOutcome}; pub use views::{ManifestApplyOutcome, ViewInfo}; diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index cd4e896d..149f054f 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -11,7 +11,7 @@ use std::collections::BTreeSet; use atomic_core::change::ChangeStore as _; use atomic_core::pristine::slice::{GraphSlice, ViewSnapshotRows}; -use atomic_core::pristine::{decode_vertex, GraphTxnT, MutTxnT, ViewTxnT}; +use atomic_core::pristine::{decode_vertex, GraphTxnT, MutTxnT, TreeTxnT, ViewTxnT}; use atomic_core::types::{GraphNode, Hash, NodeId}; use serde::{Deserialize, Serialize}; @@ -58,6 +58,41 @@ mod serde_bytes_b64 { } } +/// Why the repository refused a sandbox's change. Nothing is written when +/// a change is refused. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)] +pub enum SubmitRejection { + #[error("the change's bytes hash to {computed}, not {claimed}")] + HashMismatch { claimed: String, computed: String }, + #[error("not a change: {0}")] + Malformed(String), + #[error("the view has moved on (now {current}); fetch and record again")] + StaleView { current: String }, + #[error("change {0} is not on this view")] + ForeignChange(String), + #[error("node {0} is not on this view")] + ForeignNode(u64), + #[error("a change may not touch {0}")] + ForbiddenPath(String), + #[error("change {0} is already in the repository")] + AlreadyPresent(String), +} + +/// A submitted change, applied. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Submitted { + pub hash: Hash, + /// The view's state after it, base32. + pub state: String, +} + +/// Paths only the repository's own machinery writes. +fn forbidden_path(path: &str) -> bool { + path.split('/').any(|part| { + part == crate::DOT_DIR || part == super::SANDBOX_POINTER || part == super::SANDBOX_CACHE_DIR + }) +} + fn db(e: impl std::fmt::Display) -> RepositoryError { RepositoryError::Database(e.to_string()) } @@ -115,7 +150,18 @@ impl Repository { name: view.to_string(), })?; let visible: BTreeSet = self.sandbox_visible(&txn, &state)?.into_iter().collect(); - let rows = txn.export_graph_slice(inodes, state.id).map_err(db)?; + // Only files the view has: an inode another view introduced is not + // this sandbox's to read. + let mut on_view = Vec::with_capacity(inodes.len()); + for &inode in inodes { + let position = txn + .inode_position(atomic_core::types::Inode::new(inode)) + .map_err(db)?; + if position.is_some_and(|p| p.change.is_root() || visible.contains(&p.change.get())) { + on_view.push(inode); + } + } + let rows = txn.export_graph_slice(&on_view, state.id).map_err(db)?; let mut spans = Vec::new(); for (key, _) in &rows.graph { let (change, start, end) = decode_vertex(key); @@ -143,6 +189,125 @@ impl Repository { Ok(SandboxSlice { rows, spans }) } + /// Serve side: take a change a remote sandbox recorded on `view` and + /// apply it there — if it is exactly what it claims, recorded against + /// the view as it is now, and names nothing outside the view. + /// + /// The checks, in order: the bytes hash to `hash`; the view is still at + /// `base_state` (otherwise the sandbox fetches and records again); every + /// change it depends on or refers to that the repository knows is + /// visible on the view; every node its file operations name is a visible + /// change's; no path touches `.atomic`, `.atomic-sandbox` or + /// `.atomic-sandbox.d`; and it is new. Then it is saved and inserted + /// into `view`. Callers serialize submissions (one writer). + pub fn insert_submitted_change( + &self, + view: &str, + base_state: &str, + hash: &Hash, + bytes: &[u8], + ) -> Result, RepositoryError> { + use atomic_core::change::format_v3::reader::ChangeReader; + use atomic_core::types::Base32; + + let (change, computed) = match atomic_core::change::Change::deserialize(&mut &bytes[..]) { + Ok(parsed) => parsed, + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + if computed != *hash { + return Ok(Err(SubmitRejection::HashMismatch { + claimed: hash.to_base32(), + computed: computed.to_base32(), + })); + } + let referenced: Vec = match ChangeReader::open(&mut &bytes[..]) { + Ok(reader) => reader + .hash_table() + .hashes() + .iter() + .map(|h| Hash::from(*h)) + .filter(|h| h != hash) + .collect(), + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + + { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let current = state.state.to_base32(); + if current != base_state { + return Ok(Err(SubmitRejection::StaleView { current })); + } + if txn.get_internal(hash).map_err(db)?.is_some() { + return Ok(Err(SubmitRejection::AlreadyPresent(hash.to_base32()))); + } + let visible = super::collect_visible_change_ids(&txn, &state)?; + for dep in change.dependencies() { + match txn.get_internal(dep).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(dep.to_base32()))), + } + } + for other in &referenced { + if let Some(id) = txn.get_internal(other).map_err(db)? { + if !visible.contains(&id) { + return Ok(Err(SubmitRejection::ForeignChange(other.to_base32()))); + } + } + } + for ops in change.file_ops() { + if let Some(id) = ops + .referenced_node_ids() + .into_iter() + .find(|id| !visible.contains(id)) + { + return Ok(Err(SubmitRejection::ForeignNode(id.get()))); + } + if forbidden_path(ops.path()) { + return Ok(Err(SubmitRejection::ForbiddenPath(ops.path().to_string()))); + } + } + for hunk in change.hunks() { + if let Some(path) = hunk.path().filter(|p| forbidden_path(p)) { + return Ok(Err(SubmitRejection::ForbiddenPath(path.to_string()))); + } + } + } + + self.save_change_bytes(hash, bytes, &change)?; + let outcome = match self.insert_change(hash, crate::InsertOptions::default().view(view)) { + Ok(outcome) => outcome, + Err(e) => { + // Nothing of it stays: the change file goes with the failure. + let _ = std::fs::remove_file(self.change_store.change_path(hash)); + return Err(e); + } + }; + Ok(Ok(Submitted { + hash: *hash, + state: outcome.new_state.to_base32(), + })) + } + + /// Cache side: the view's state as the cache has it (base32) — what a + /// change recorded here is recorded against. + pub fn remote_sandbox_view_state(&self) -> Result { + use atomic_core::types::Base32; + let txn = self.pristine.read_txn().map_err(db)?; + let state = txn + .get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })?; + Ok(state.state.to_base32()) + } + /// Cache side: replace this cache's tree and view with `skeleton`'s. pub fn import_sandbox_skeleton( &self, diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index e598d62d..ba7311cf 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -84,6 +84,11 @@ pub struct SealResult { /// instead of looking for a local `.atomic/`. pub const SANDBOX_POINTER: &str = ".atomic-sandbox"; +/// A remote sandbox's own directory, beside its pointer: the local cache of +/// its repository's rows (`cache/`, an ordinary repository whose working +/// tree is the sandbox). Never tracked. +pub const SANDBOX_CACHE_DIR: &str = ".atomic-sandbox.d"; + /// Persisted contents of a sandbox pointer: where the canonical graph lives /// and which view this sandbox operates on. #[derive(Debug, Serialize, Deserialize)] @@ -107,8 +112,19 @@ pub(super) fn detect_sandbox(start: &Path) -> Option<(PathBuf, PathBuf, String)> let pointer = dir.join(SANDBOX_POINTER); if pointer.is_file() { let bytes = std::fs::read(&pointer).ok()?; - let parsed: SandboxPointer = serde_json::from_slice(&bytes).ok()?; - return Some((dir.clone(), parsed.canonical, parsed.view)); + if let Ok(parsed) = serde_json::from_slice::(&bytes) { + return Some((dir.clone(), parsed.canonical, parsed.view)); + } + // A remote pointer names an owner elsewhere; locally, the graph + // is the sandbox's cache (once `atomic sandbox materialize` has + // made it). + let remote: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + let view = remote.get("remote").and(remote.get("view"))?.as_str()?; + let cache = remote_cache_root(&dir); + return cache + .join(DOT_DIR) + .is_dir() + .then(|| (dir.clone(), cache, view.to_string())); } // Stop if we reach a real repository root — that's not a sandbox. if dir.join(DOT_DIR).join("pristine.redb").is_file() { @@ -120,7 +136,38 @@ pub(super) fn detect_sandbox(start: &Path) -> Option<(PathBuf, PathBuf, String)> } } +/// Where a remote sandbox rooted at `working_root` keeps its cache. +pub fn remote_cache_root(working_root: &Path) -> PathBuf { + working_root.join(SANDBOX_CACHE_DIR).join("cache") +} + impl Repository { + /// Make (or remake) a remote sandbox's cache from `skeleton`: an empty + /// repository under [`SANDBOX_CACHE_DIR`] holding the view's rows, with + /// the sandbox as its working tree and the tree on disk as the clean + /// baseline. Call it right after writing the view's files. + pub fn create_remote_sandbox_cache( + working_root: &Path, + skeleton: &super::SandboxSkeleton, + ) -> Result { + let cache = remote_cache_root(working_root); + if cache.exists() { + std::fs::remove_dir_all(&cache)?; + } + std::fs::create_dir_all(&cache)?; + drop(Self::init(&cache)?); + let repo = Self::open_sandbox(working_root, &cache, &skeleton.view.name)?; + repo.import_sandbox_skeleton(skeleton)?; + repo.reindex_working_copy()?; + Ok(repo) + } + + /// Whether this is a remote sandbox's cache (its graph arrives from the + /// repository's owner, and its changes go back to it). + pub fn is_remote_sandbox(&self) -> bool { + self.is_sandbox && self.dot_dir.starts_with(self.root.join(SANDBOX_CACHE_DIR)) + } + /// Open a repository for an agent sandbox. /// /// The agent's private working tree is at `working_root`, but the graph diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs index e58faf26..8198c4bf 100644 --- a/atomic-repository/tests/remote_cache_parity_test.rs +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -250,3 +250,191 @@ fn a_second_record_on_top_of_the_first() { pair.edit(|root| write(root, "src/lib.rs", "pub fn a() { 1 }\n")); assert_parity(&pair, &view, &live, "second"); } + +// ── Submitting: the repository takes the cache's change ───────────────── + +use atomic_repository::SubmitRejection; + +fn view_state(repo: &Repository, view: &str) -> String { + let skeleton = repo + .export_sandbox_skeleton(view, &BTreeSet::new()) + .unwrap(); + atomic_core::types::Base32::to_base32(&skeleton.view.state) +} + +/// Record in the cache (hydrated as FileStates would) and return the change. +fn record_in_cache( + pair: &Pair, + view: &str, + live: &BTreeSet, + what: &str, +) -> (atomic_core::types::Hash, Vec) { + pair.hydrate(view, live); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = pair.cache.record(header(what), options).unwrap(); + (*outcome.hash(), outcome.v3_bytes().unwrap().to_vec()) +} + +fn tree_of(repo: &Repository, view: &str) -> Vec<(String, Vec)> { + let mut files = Vec::new(); + repo.materialize_view_entries::<()>(view, |e| { + if e.kind == atomic_repository::ViewEntryKind::File { + files.push((e.path, e.content)); + } + Ok(()) + }) + .unwrap() + .unwrap(); + files +} + +#[test] +fn a_submitted_change_lands_on_the_view_and_the_next_one_builds_on_it() { + let (pair, view, mut live) = pair(two_files); + // Only the cache's tree changes: the repository learns of it by submission. + write(pair.cache_root(), "README.md", "hello\nfrom the sandbox\n"); + write(pair.cache_root(), "src/new.rs", "pub fn n() {}\n"); + pair.cache + .add("src/new.rs", TrackingOptions::default()) + .unwrap(); + let base = view_state(&pair.host, &view); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "from the sandbox"); + + let submitted = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap() + .expect("accepted"); + assert_ne!(submitted.state, base, "the view moved"); + let tree = tree_of(&pair.host, &view); + assert!(tree.contains(&("README.md".into(), b"hello\nfrom the sandbox\n".to_vec()))); + assert!(tree.contains(&("src/new.rs".into(), b"pub fn n() {}\n".to_vec()))); + + // The same change again: the view has moved, and then it's already there. + let again = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!(again, Err(SubmitRejection::StaleView { .. })), + "{again:?}" + ); + let again = pair + .host + .insert_submitted_change(&view, &submitted.state, &hash, &bytes) + .unwrap(); + assert!( + matches!(again, Err(SubmitRejection::AlreadyPresent(_))), + "{again:?}" + ); + + // The cache takes the new skeleton (with the repository's inode for the + // new file) and records on top. + live.clear(); + pair.host + .materialize_view_entries::<()>(&view, |e| { + live.insert(e.inode); + Ok(()) + }) + .unwrap() + .unwrap(); + let skeleton = pair.host.export_sandbox_skeleton(&view, &live).unwrap(); + pair.cache.import_sandbox_skeleton(&skeleton).unwrap(); + pair.cache.reindex_working_copy().unwrap(); + write(pair.cache_root(), "src/new.rs", "pub fn n() { 2 }\n"); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "second from the sandbox"); + pair.host + .insert_submitted_change(&view, &submitted.state, &hash, &bytes) + .unwrap() + .expect("the second is accepted too"); + assert!( + tree_of(&pair.host, &view).contains(&("src/new.rs".into(), b"pub fn n() { 2 }\n".to_vec())) + ); +} + +#[test] +fn a_tampered_change_is_refused_and_leaves_nothing() { + let (pair, view, live) = pair(two_files); + write(pair.cache_root(), "README.md", "tampered?\n"); + let base = view_state(&pair.host, &view); + let (hash, mut bytes) = record_in_cache(&pair, &view, &live, "tamper"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + let refused = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!( + refused, + Err(SubmitRejection::HashMismatch { .. } | SubmitRejection::Malformed(_)) + ), + "{refused:?}" + ); + assert_eq!(view_state(&pair.host, &view), base); + assert!(!pair.host.has_change(&hash)); +} + +#[test] +fn a_change_recorded_against_an_older_view_is_refused() { + let (pair, view, live) = pair(two_files); + let base = view_state(&pair.host, &view); + write(pair.cache_root(), "README.md", "late\n"); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "late"); + // Meanwhile the view moves on the repository. + write(pair.host_root(), "src/lib.rs", "pub fn moved() {}\n"); + record_and_apply(&pair.host, "meanwhile"); + let refused = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!(refused, Err(SubmitRejection::StaleView { .. })), + "{refused:?}" + ); +} + +#[test] +fn a_change_from_another_view_is_refused() { + // Work recorded on a draft view depends on that view's changes; submitted + // to `dev` it names changes `dev` can't see. + let host_dir = TempDir::new().unwrap(); + let draft_dir = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("other", "dev").unwrap(); + host.provision_sandbox(draft_dir.path().join("w"), "other") + .unwrap(); + } + let draft = Repository::open_existing(draft_dir.path().join("w")).unwrap(); + write(draft.root(), "README.md", "only on other\n"); + record_and_apply(&draft, "on other"); + write(draft.root(), "README.md", "only on other, again\n"); + let (hash, bytes) = { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = draft.record(header("on top of other"), options).unwrap(); + (*outcome.hash(), outcome.v3_bytes().unwrap().to_vec()) + }; + drop(draft); + + let host = Repository::open_existing(host_dir.path()).unwrap(); + let base = view_state(&host, "dev"); + let refused = host + .insert_submitted_change("dev", &base, &hash, &bytes) + .unwrap(); + assert!( + matches!( + refused, + Err(SubmitRejection::ForeignChange(_) | SubmitRejection::ForeignNode(_)) + ), + "{refused:?}" + ); + assert_eq!(view_state(&host, "dev"), base); +} From 93e8af36ab0f2a9c3f2a8da7f24ce25c73bf9a89 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:28:21 -0500 Subject: [PATCH 10/15] Read a remote sandbox's recorded state: diff, restore, log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache holds a view's rows, not its content. diff and restore now load the content they compare against first (FileStates, as record does), and log fetches the change files the view has and the cache lacks — a new Changes request, answered only for changes on the token's view, each file kept only if its bytes hash to what it claims. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-cli/src/commands/agent/owner/mod.rs | 47 ++++++++++++- .../src/commands/agent/owner/sandbox.rs | 24 ++++++- atomic-cli/src/commands/diff/command.rs | 19 +++-- atomic-cli/src/commands/log/command.rs | 6 ++ atomic-cli/src/commands/mod.rs | 11 +++ atomic-cli/src/commands/restore.rs | 1 + .../tests/remote_sandbox_integration_test.rs | 26 +++++++ .../src/repository/remote_cache.rs | 70 +++++++++++++++++++ 8 files changed, 194 insertions(+), 10 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/mod.rs b/atomic-cli/src/commands/agent/owner/mod.rs index 17eb7494..e99cdb2a 100644 --- a/atomic-cli/src/commands/agent/owner/mod.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -218,6 +218,12 @@ enum OwnerRequest { view: Option, inodes: Vec, }, + /// The change files of `hashes`, each on the view. + Changes { + #[serde(default)] + view: Option, + hashes: Vec, + }, /// A change a sandbox recorded against `base_state`, as its V3 bytes. SubmitChange { #[serde(default)] @@ -229,6 +235,14 @@ enum OwnerRequest { }, } +/// A change file on the wire. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct WireChange { + hash: Hash, + #[serde(with = "base64_bytes")] + bytes: Vec, +} + /// Bytes as base64 in the JSON frames. mod base64_bytes { use serde::{Deserialize, Deserializer, Serializer}; @@ -323,6 +337,9 @@ pub(crate) enum OwnerResponse { ChangeRefused { rejection: atomic_repository::SubmitRejection, }, + Changes { + changes: Vec, + }, Error { code: String, message: String, @@ -1114,6 +1131,31 @@ pub(crate) fn hydrate_remote_sandbox(repo: &Repository) -> anyhow::Result<()> { } } +/// In a remote sandbox: fetch the change files its view has and it +/// doesn't (for `log` and anything else that reads changes). +pub(crate) fn fetch_remote_sandbox_changes(repo: &Repository) -> anyhow::Result { + let missing = repo.missing_sandbox_changes()?; + for batch in missing.chunks(32) { + match request( + repo.root(), + OwnerRequest::Changes { + view: None, + hashes: batch.to_vec(), + }, + )? { + OwnerResponse::Changes { changes } => { + let changes: Vec<_> = changes.into_iter().map(|c| (c.hash, c.bytes)).collect(); + repo.hold_sandbox_changes(&changes)?; + } + OwnerResponse::ChangeRefused { rejection } => { + return Err(anyhow!("the repository refused: {rejection}")) + } + other => return Err(unexpected_response("changes", other)), + } + } + Ok(missing.len()) +} + /// In a remote sandbox: hand a recorded change to the repository's owner. /// On success the cache takes the view as it now is. pub(crate) fn submit_from_remote_sandbox( @@ -1559,6 +1601,7 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram | OwnerRequest::CloseSandbox { .. } | OwnerRequest::Materialize { .. } | OwnerRequest::FileStates { .. } + | OwnerRequest::Changes { .. } | OwnerRequest::SubmitChange { .. } => ( OwnerResponse::Error { code: "internal".to_string(), @@ -1683,7 +1726,9 @@ where let response = sandbox::admin(&owner, request.request.clone()).await; write_frame(&mut stream, &respond(request.request_id, response)).await?; } - OwnerRequest::FileStates { .. } | OwnerRequest::SubmitChange { .. } + OwnerRequest::FileStates { .. } + | OwnerRequest::Changes { .. } + | OwnerRequest::SubmitChange { .. } if request.version == PROTOCOL_VERSION => { let request_id = request.request_id.clone(); diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index 18e61684..4b459ac2 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -71,6 +71,7 @@ pub(crate) fn authorize( OwnerRequest::Ping => {} OwnerRequest::Materialize { view: asked } | OwnerRequest::FileStates { view: asked, .. } + | OwnerRequest::Changes { view: asked, .. } | OwnerRequest::SubmitChange { view: asked, .. } => { if asked.as_deref().is_some_and(|v| v != view) { return Err(forbidden("that view")); @@ -307,9 +308,9 @@ pub(crate) async fn record_request( request: OwnerRequest, ) -> OwnerResponse { let named = match &request { - OwnerRequest::FileStates { view, .. } | OwnerRequest::SubmitChange { view, .. } => { - view.clone() - } + OwnerRequest::FileStates { view, .. } + | OwnerRequest::Changes { view, .. } + | OwnerRequest::SubmitChange { view, .. } => view.clone(), _ => None, }; let Some(view) = grant.map(|g| g.view).or(named) else { @@ -330,6 +331,23 @@ pub(crate) async fn record_request( Err(e) => error("internal", e.to_string()), } } + OwnerRequest::Changes { hashes, .. } => { + let read = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = Repository::open_readonly(&root)?; + Ok(repo.export_sandbox_changes(&view, &hashes)?) + }); + match read.await { + Ok(Ok(Ok(changes))) => OwnerResponse::Changes { + changes: changes + .into_iter() + .map(|(hash, bytes)| super::WireChange { hash, bytes }) + .collect(), + }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Err(e)) => error("changes", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } OwnerRequest::SubmitChange { base_state, hash, diff --git a/atomic-cli/src/commands/diff/command.rs b/atomic-cli/src/commands/diff/command.rs index 243a9e89..908c5636 100644 --- a/atomic-cli/src/commands/diff/command.rs +++ b/atomic-cli/src/commands/diff/command.rs @@ -224,12 +224,19 @@ impl Command for Diff { // Find the repository root let repo_root = find_repository_root()?; - // Open the repository - let repo = crate::commands::open_readonly_repository(&repo_root).map_err(|e| { - CliError::InvalidRepository { - reason: e.to_string(), - } - })?; + // Open the repository. A remote sandbox's cache takes the content it + // is about to compare against, so it opens writable. + let repo = if atomic_repository::remote_cache_root(&repo_root).is_dir() { + let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + crate::commands::hydrate_if_remote(&repo)?; + repo + } else { + crate::commands::open_readonly_repository(&repo_root).map_err(|e| { + CliError::InvalidRepository { + reason: e.to_string(), + } + })? + }; // Parse algorithm let algorithm = self.parse_algorithm()?; diff --git a/atomic-cli/src/commands/log/command.rs b/atomic-cli/src/commands/log/command.rs index 0741d8ae..074ff2e2 100644 --- a/atomic-cli/src/commands/log/command.rs +++ b/atomic-cli/src/commands/log/command.rs @@ -499,6 +499,12 @@ impl Command for Log { fn run(&self) -> CliResult<()> { // Find and open repository let repo_root = find_repository_root()?; + // A remote sandbox's cache fetches the change files it lacks first. + if atomic_repository::remote_cache_root(&repo_root).is_dir() { + let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + crate::commands::agent::owner::fetch_remote_sandbox_changes(&repo) + .map_err(CliError::Internal)?; + } let repo = crate::commands::open_readonly_repository(&repo_root).map_err(|e| match e { atomic_repository::RepositoryError::NotFound { path } => CliError::RepositoryNotFound { searched_path: path.into(), diff --git a/atomic-cli/src/commands/mod.rs b/atomic-cli/src/commands/mod.rs index d5ff1859..c1d66b49 100644 --- a/atomic-cli/src/commands/mod.rs +++ b/atomic-cli/src/commands/mod.rs @@ -358,6 +358,17 @@ pub fn open_repository(path: Option<&Path>) -> CliResult { Repository::open(&repo_path).map_err(CliError::from) } +/// In a remote sandbox, load what reading the working tree's recorded +/// state needs (old content for `diff`, `restore`) into its cache; the cache +/// holds a view's rows, not its content, until a command asks. Elsewhere, +/// nothing. +pub fn hydrate_if_remote(repo: &Repository) -> CliResult<()> { + if repo.is_remote_sandbox() { + agent::owner::hydrate_remote_sandbox(repo).map_err(CliError::Internal)?; + } + Ok(()) +} + /// Acquire a read-only repository for a CLI query, allowing short-lived writers /// (such as Stop checkpoint publication) to finish first. Only typed database /// contention is retried, for at most ten seconds; all other errors propagate diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index 00b41d9c..e2bcf581 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -332,6 +332,7 @@ impl Command for Restore { // Find repository let repo_root = find_repository_root()?; let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + crate::commands::hydrate_if_remote(&repo)?; // Compute status once. Restore only touches tracked files, so we skip // the untracked scan, and we reuse this single status for both the diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index bb4ed348..f2872975 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -192,6 +192,32 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { atomic(&vm_dir, &["record", "-a", "-m", "again from the sandbox"]), "second record in the sandbox", ); + // Reading the recorded state works in the sandbox too: diff against + // the view, restore from it, and the view's history. + std::fs::write( + vm_dir.join("README.md"), + "hello\nfrom the sandbox\nuncommitted\n", + ) + .unwrap(); + let diff = ok(atomic(&vm_dir, &["diff"]), "diff in the sandbox"); + assert!( + diff.contains("+uncommitted") && !diff.contains("+hello"), + "{diff}" + ); + ok( + atomic(&vm_dir, &["restore", "README.md"]), + "restore in the sandbox", + ); + assert_eq!( + std::fs::read_to_string(vm_dir.join("README.md")).unwrap(), + "hello\nfrom the sandbox\n" + ); + let log = ok(atomic(&vm_dir, &["log"]), "log in the sandbox"); + assert!( + log.contains("again from the sandbox") && log.contains("first"), + "{log}" + ); + let log = ok(atomic(host.path(), &["log"]), "host log"); assert!( log.contains("from the sandbox") && log.contains("again from the sandbox"), diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index 149f054f..11214ae5 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -78,6 +78,9 @@ pub enum SubmitRejection { AlreadyPresent(String), } +/// A change file: its hash and V3 bytes. +pub type ChangeFile = (Hash, Vec); + /// A submitted change, applied. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Submitted { @@ -294,6 +297,73 @@ impl Repository { })) } + /// Serve side: the V3 bytes of each of `hashes` — changes `view` can + /// see, and only those (anything else is refused, whole). + pub fn export_sandbox_changes( + &self, + view: &str, + hashes: &[Hash], + ) -> Result, SubmitRejection>, RepositoryError> { + use atomic_core::types::Base32; + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible = super::collect_visible_change_ids(&txn, &state)?; + let mut out = Vec::with_capacity(hashes.len()); + for hash in hashes { + match txn.get_internal(hash).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(hash.to_base32()))), + } + out.push((*hash, std::fs::read(self.change_store.change_path(hash))?)); + } + Ok(Ok(out)) + } + + /// Cache side: the changes the view has that this cache holds no file + /// for. + pub fn missing_sandbox_changes(&self) -> Result, RepositoryError> { + let txn = self.pristine.read_txn().map_err(db)?; + let state = txn + .get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })?; + let mut missing = Vec::new(); + for id in super::collect_visible_change_ids(&txn, &state)? { + if let Some(hash) = txn.get_external(id).map_err(db)? { + if !self.change_store.has_change(&hash) { + missing.push(hash); + } + } + } + Ok(missing) + } + + /// Cache side: keep change files the owner sent — each only if its + /// bytes hash to what it claims. + pub fn hold_sandbox_changes(&self, changes: &[ChangeFile]) -> Result<(), RepositoryError> { + use atomic_core::types::Base32; + for (hash, bytes) in changes { + let (change, computed) = atomic_core::change::Change::deserialize(&mut &bytes[..]) + .map_err(|e| RepositoryError::Database(e.to_string()))?; + if computed != *hash { + return Err(RepositoryError::Database(format!( + "the owner sent {} as {}", + computed.to_base32(), + hash.to_base32() + ))); + } + self.save_change_bytes(hash, bytes, &change)?; + } + Ok(()) + } + /// Cache side: the view's state as the cache has it (base32) — what a /// change recorded here is recorded against. pub fn remote_sandbox_view_state(&self) -> Result { From b9a63e3427a3e251d3568188b552a11ea85a8865 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:39:19 -0500 Subject: [PATCH 11/15] Remote sandboxes inside Repository: any caller records, and provenance lands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remote path moves out of individual commands into Repository, so every caller gets it — the atomic CLI and an agent's turn hooks alike. A process installs one RemoteSandboxLink (the atomic binary installs the owner client at startup); in a remote sandbox's cache, record hydrates first, write_recorded submits instead of applying (a change that didn't land fails the record), diff and restore hydrate, log fetches change files, and publish_provenance_checkpoint publishes the checkpoint in the repository. PublishProvenance, the owner's side: the sandbox's own session, and every change the provenance explains on its view; the graph arrives serialized, so the repository publishes what the sandbox hashed. Two authorization fixes the agent flow showed: asking after a session nobody has written to is allowed (the store answers "no such turn"), and BindCheckpointHash names a provenance graph, not a change. An agent's session-start, turn-start, turn-end and session-end in a remote sandbox now leave its change on the view and its session ledger and provenance in the repository. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-cli/src/commands/agent/owner/mod.rs | 145 +++++++++------ .../src/commands/agent/owner/sandbox.rs | 64 +++++-- atomic-cli/src/commands/diff/command.rs | 3 +- atomic-cli/src/commands/log/command.rs | 4 +- atomic-cli/src/commands/mod.rs | 11 -- atomic-cli/src/commands/record/command.rs | 28 +-- atomic-cli/src/commands/restore.rs | 3 +- atomic-cli/src/main.rs | 4 + .../tests/remote_sandbox_integration_test.rs | 93 ++++++++++ atomic-repository/src/repository/changes.rs | 14 ++ atomic-repository/src/repository/insert.rs | 3 + atomic-repository/src/repository/mod.rs | 5 +- atomic-repository/src/repository/record.rs | 7 + .../src/repository/remote_cache.rs | 165 ++++++++++++++++++ 14 files changed, 444 insertions(+), 105 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/mod.rs b/atomic-cli/src/commands/agent/owner/mod.rs index e99cdb2a..4ebc670f 100644 --- a/atomic-cli/src/commands/agent/owner/mod.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -224,6 +224,14 @@ enum OwnerRequest { view: Option, hashes: Vec, }, + /// A sandbox's checkpoint: its provenance graph (serialized) and turn. + PublishProvenance { + #[serde(default)] + view: Option, + #[serde(with = "base64_bytes")] + graph: Vec, + turn: SessionTurn, + }, /// A change a sandbox recorded against `base_state`, as its V3 bytes. SubmitChange { #[serde(default)] @@ -340,6 +348,9 @@ pub(crate) enum OwnerResponse { Changes { changes: Vec, }, + ProvenancePublished { + publication: atomic_core::change::session::SessionCheckpointPublication, + }, Error { code: String, message: String, @@ -1122,68 +1133,98 @@ pub(crate) fn materialize_remote_sandbox(start: &Path) -> anyhow::Result<(PathBu Ok((root, written)) } -/// In a remote sandbox: load what `record` will read into its cache. -pub(crate) fn hydrate_remote_sandbox(repo: &Repository) -> anyhow::Result<()> { - let inodes = repo.sandbox_slice_inodes()?; - match request(repo.root(), OwnerRequest::FileStates { view: None, inodes })? { - OwnerResponse::FileStates { slice } => Ok(repo.import_sandbox_slice(&slice)?), - other => Err(unexpected_response("file states", other)), - } +/// The owner protocol as a remote sandbox's link to its repository's owner: +/// what lets `Repository` record, diff, restore and read history in a +/// remote sandbox, whoever calls it. +pub(crate) struct OwnerLink; + +/// One owner request from a remote sandbox, on its own thread: callers +/// (an agent's turn hooks, say) may already be inside a runtime. +fn link_request(root: &Path, body: OwnerRequest) -> Result { + let root = root.to_path_buf(); + run_outside_async_runtime(move || request(&root, body)).map_err(|e| format!("{e:#}")) } -/// In a remote sandbox: fetch the change files its view has and it -/// doesn't (for `log` and anything else that reads changes). -pub(crate) fn fetch_remote_sandbox_changes(repo: &Repository) -> anyhow::Result { - let missing = repo.missing_sandbox_changes()?; - for batch in missing.chunks(32) { - match request( - repo.root(), - OwnerRequest::Changes { - view: None, - hashes: batch.to_vec(), - }, - )? { - OwnerResponse::Changes { changes } => { - let changes: Vec<_> = changes.into_iter().map(|c| (c.hash, c.bytes)).collect(); - repo.hold_sandbox_changes(&changes)?; - } - OwnerResponse::ChangeRefused { rejection } => { - return Err(anyhow!("the repository refused: {rejection}")) - } - other => return Err(unexpected_response("changes", other)), +impl atomic_repository::RemoteSandboxLink for OwnerLink { + fn file_states( + &self, + root: &Path, + inodes: Vec, + ) -> Result { + match link_request(root, OwnerRequest::FileStates { view: None, inodes })? { + OwnerResponse::FileStates { slice } => Ok(*slice), + other => Err(unexpected_response("file states", other).to_string()), } } - Ok(missing.len()) -} -/// In a remote sandbox: hand a recorded change to the repository's owner. -/// On success the cache takes the view as it now is. -pub(crate) fn submit_from_remote_sandbox( - repo: &Repository, - hash: Hash, - bytes: Vec, -) -> anyhow::Result { - let base_state = repo.remote_sandbox_view_state()?; - match request( - repo.root(), - OwnerRequest::SubmitChange { + fn submit( + &self, + root: &Path, + base_state: String, + hash: Hash, + bytes: Vec, + ) -> Result< + Result< + ( + atomic_repository::Submitted, + atomic_repository::SandboxSkeleton, + ), + atomic_repository::SubmitRejection, + >, + String, + > { + let body = OwnerRequest::SubmitChange { view: None, base_state, hash, bytes, - }, - )? { - OwnerResponse::ChangeSubmitted { - submitted, - skeleton, - } => { - repo.import_sandbox_skeleton(&skeleton)?; - Ok(submitted) + }; + match link_request(root, body)? { + OwnerResponse::ChangeSubmitted { + submitted, + skeleton, + } => Ok(Ok((submitted, *skeleton))), + OwnerResponse::ChangeRefused { rejection } => Ok(Err(rejection)), + other => Err(unexpected_response("submit change", other).to_string()), } - OwnerResponse::ChangeRefused { rejection } => { - Err(anyhow!("the repository refused the change: {rejection}")) + } + + fn changes( + &self, + root: &Path, + hashes: Vec, + ) -> Result, String> { + match link_request(root, OwnerRequest::Changes { view: None, hashes })? { + OwnerResponse::Changes { changes } => { + Ok(changes.into_iter().map(|c| (c.hash, c.bytes)).collect()) + } + OwnerResponse::ChangeRefused { rejection } => Err(rejection.to_string()), + other => Err(unexpected_response("changes", other).to_string()), + } + } + + fn publish_provenance( + &self, + root: &Path, + graph: Vec, + turn: SessionTurn, + ) -> Result< + Result< + atomic_core::change::session::SessionCheckpointPublication, + atomic_repository::SubmitRejection, + >, + String, + > { + let body = OwnerRequest::PublishProvenance { + view: None, + graph, + turn, + }; + match link_request(root, body)? { + OwnerResponse::ProvenancePublished { publication } => Ok(Ok(publication)), + OwnerResponse::ChangeRefused { rejection } => Ok(Err(rejection)), + other => Err(unexpected_response("publish provenance", other).to_string()), } - other => Err(unexpected_response("submit change", other)), } } @@ -1602,6 +1643,7 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram | OwnerRequest::Materialize { .. } | OwnerRequest::FileStates { .. } | OwnerRequest::Changes { .. } + | OwnerRequest::PublishProvenance { .. } | OwnerRequest::SubmitChange { .. } => ( OwnerResponse::Error { code: "internal".to_string(), @@ -1728,6 +1770,7 @@ where } OwnerRequest::FileStates { .. } | OwnerRequest::Changes { .. } + | OwnerRequest::PublishProvenance { .. } | OwnerRequest::SubmitChange { .. } if request.version == PROTOCOL_VERSION => { diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index 4b459ac2..eaa8898e 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -77,6 +77,16 @@ pub(crate) fn authorize( return Err(forbidden("that view")); } } + OwnerRequest::PublishProvenance { + view: asked, turn, .. + } => { + if asked.as_deref().is_some_and(|v| v != view) { + return Err(forbidden("that view")); + } + if !tokens.owns_session(view, &turn.session_id) { + return Err(forbidden("that session")); + } + } OwnerRequest::ReserveProvenanceTurn { session_id, turn_number, @@ -106,21 +116,38 @@ pub(crate) fn authorize( } require_on_view(&owner.root, view, &source.change_hashes)?; } - OwnerRequest::BindCheckpointHash { - provenance_id, - hash, - .. - } => { + OwnerRequest::BindCheckpointHash { provenance_id, .. } => { + // Its hash names the checkpoint's provenance graph, not a change; + // the changes it explains were checked at PrepareCheckpoint. if !tokens.owns_provenance(view, *provenance_id) { return Err(forbidden("that provenance turn")); } - require_on_view(&owner.root, view, std::slice::from_ref(hash))?; } - OwnerRequest::StopTurn { session_id, .. } - | OwnerRequest::ResumeTurn { session_id, .. } - | OwnerRequest::AbandonTurn { session_id, .. } - | OwnerRequest::TurnStatus { session_id, .. } => { - if !tokens.owns_session(view, session_id) { + OwnerRequest::StopTurn { + session_id, + turn_number, + .. + } + | OwnerRequest::ResumeTurn { + session_id, + turn_number, + .. + } + | OwnerRequest::AbandonTurn { + session_id, + turn_number, + .. + } + | OwnerRequest::TurnStatus { + session_id, + turn_number, + } => { + // Asking after a session nobody has written to is harmless (the + // store answers "no such turn"); anyone else's is not. + if !tokens.owns_session(view, session_id) + && !session_is_fresh(&owner.store, session_id, *turn_number) + .map_err(|e| refuse("provenance-store", e.to_string()))? + { return Err(forbidden("that session")); } } @@ -310,6 +337,7 @@ pub(crate) async fn record_request( let named = match &request { OwnerRequest::FileStates { view, .. } | OwnerRequest::Changes { view, .. } + | OwnerRequest::PublishProvenance { view, .. } | OwnerRequest::SubmitChange { view, .. } => view.clone(), _ => None, }; @@ -348,6 +376,20 @@ pub(crate) async fn record_request( Err(e) => error("internal", e.to_string()), } } + OwnerRequest::PublishProvenance { graph, turn, .. } => { + let _one_at_a_time = owner.submissions.lock().await; + let write = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = + Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; + Ok(repo.publish_sandbox_provenance(&view, &graph, turn)?) + }); + match write.await { + Ok(Ok(Ok(publication))) => OwnerResponse::ProvenancePublished { publication }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { rejection }, + Ok(Err(e)) => error("provenance", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } OwnerRequest::SubmitChange { base_state, hash, diff --git a/atomic-cli/src/commands/diff/command.rs b/atomic-cli/src/commands/diff/command.rs index 908c5636..560bb13b 100644 --- a/atomic-cli/src/commands/diff/command.rs +++ b/atomic-cli/src/commands/diff/command.rs @@ -228,7 +228,8 @@ impl Command for Diff { // is about to compare against, so it opens writable. let repo = if atomic_repository::remote_cache_root(&repo_root).is_dir() { let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; - crate::commands::hydrate_if_remote(&repo)?; + repo.hydrate_remote_sandbox() + .map_err(CliError::Repository)?; repo } else { crate::commands::open_readonly_repository(&repo_root).map_err(|e| { diff --git a/atomic-cli/src/commands/log/command.rs b/atomic-cli/src/commands/log/command.rs index 074ff2e2..44354145 100644 --- a/atomic-cli/src/commands/log/command.rs +++ b/atomic-cli/src/commands/log/command.rs @@ -502,8 +502,8 @@ impl Command for Log { // A remote sandbox's cache fetches the change files it lacks first. if atomic_repository::remote_cache_root(&repo_root).is_dir() { let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; - crate::commands::agent::owner::fetch_remote_sandbox_changes(&repo) - .map_err(CliError::Internal)?; + repo.fetch_remote_sandbox_changes() + .map_err(CliError::Repository)?; } let repo = crate::commands::open_readonly_repository(&repo_root).map_err(|e| match e { atomic_repository::RepositoryError::NotFound { path } => CliError::RepositoryNotFound { diff --git a/atomic-cli/src/commands/mod.rs b/atomic-cli/src/commands/mod.rs index c1d66b49..d5ff1859 100644 --- a/atomic-cli/src/commands/mod.rs +++ b/atomic-cli/src/commands/mod.rs @@ -358,17 +358,6 @@ pub fn open_repository(path: Option<&Path>) -> CliResult { Repository::open(&repo_path).map_err(CliError::from) } -/// In a remote sandbox, load what reading the working tree's recorded -/// state needs (old content for `diff`, `restore`) into its cache; the cache -/// holds a view's rows, not its content, until a command asks. Elsewhere, -/// nothing. -pub fn hydrate_if_remote(repo: &Repository) -> CliResult<()> { - if repo.is_remote_sandbox() { - agent::owner::hydrate_remote_sandbox(repo).map_err(CliError::Internal)?; - } - Ok(()) -} - /// Acquire a read-only repository for a CLI query, allowing short-lived writers /// (such as Stop checkpoint publication) to finish first. Only typed database /// contention is retried, for at most ten seconds; all other errors propagate diff --git a/atomic-cli/src/commands/record/command.rs b/atomic-cli/src/commands/record/command.rs index 1595d49d..912f525f 100644 --- a/atomic-cli/src/commands/record/command.rs +++ b/atomic-cli/src/commands/record/command.rs @@ -40,13 +40,7 @@ impl Command for Record { let header = header_builder.build(); // Build record options - let mut options = self.build_options()?; - // A remote sandbox records against its cache and hands the change - // to the repository's owner; nothing is saved or applied here. - let remote = repo.is_remote_sandbox(); - if remote { - options = options.save_to_store(false).apply_after_record(false); - } + let options = self.build_options()?; // If --all, first add all untracked files if self.all { @@ -62,11 +56,6 @@ impl Command for Record { } } - if remote { - crate::commands::agent::owner::hydrate_remote_sandbox(&repo) - .map_err(CliError::Internal)?; - } - // Record the changes. // // This match is deliberately exhaustive: no `_ =>` arm. The catch-all @@ -117,21 +106,6 @@ impl Command for Record { } })?; - if remote { - let bytes = outcome - .v3_bytes() - .ok_or_else(|| CliError::Internal(anyhow::anyhow!("the change has no bytes")))? - .to_vec(); - crate::commands::agent::owner::submit_from_remote_sandbox( - &repo, - *outcome.hash(), - bytes, - ) - .map_err(CliError::Internal)?; - // What was just recorded is the new baseline. - repo.reindex_working_copy().map_err(CliError::Repository)?; - } - // Display result let output = self.format_outcome(&repo, &outcome); print!("{}", output); diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index e2bcf581..02e7cc12 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -332,7 +332,8 @@ impl Command for Restore { // Find repository let repo_root = find_repository_root()?; let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; - crate::commands::hydrate_if_remote(&repo)?; + repo.hydrate_remote_sandbox() + .map_err(CliError::Repository)?; // Compute status once. Restore only touches tracked files, so we skip // the untracked scan, and we reuse this single status for both the diff --git a/atomic-cli/src/main.rs b/atomic-cli/src/main.rs index c1c93ff5..c2c13d29 100644 --- a/atomic-cli/src/main.rs +++ b/atomic-cli/src/main.rs @@ -936,6 +936,10 @@ fn main() { // Initialize logging init_logging(); + // A remote sandbox's cache reaches its repository's owner through the + // owner protocol, whichever command (or agent hook) opens it. + atomic_repository::set_remote_sandbox_link(Box::new(commands::agent::owner::OwnerLink)); + // Dynamic shell completion. When invoked in completion mode (the `COMPLETE` // env var is set by the installed shell hook), this emits candidates — // including live view names and change hashes registered on the insert diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index f2872975..43df3323 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -279,3 +279,96 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { ); assert!(refused.contains("unauthorized"), "{refused}"); } + +fn hook(cwd: &Path, verb: &str, payload: Value) -> Output { + use std::io::Write; + let mut child = Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(["agent", "hooks", "sherpa", verb]) + .current_dir(cwd) + .env("ATOMIC_OWNER_IROH_OFFLINE", "1") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("run atomic agent hooks"); + child + .stdin + .take() + .unwrap() + .write_all(payload.to_string().as_bytes()) + .unwrap(); + child.wait_with_output().unwrap() +} + +#[test] +fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--view", + "dev", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + + let cwd = vm_dir.to_str().unwrap(); + let now = "2026-01-01T00:00:00Z"; + let turn = |n: u32| { + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "model": "m", "provider": "p", + "turn_number": n, "intent_title": "greet the reader", "timestamp": now, + }) + }; + ok( + hook( + &vm_dir, + "session-start", + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "model": "m", "provider": "p", + "turn_number": 0, "timestamp": now, + }), + ), + "session-start", + ); + ok(hook(&vm_dir, "turn-start", turn(1)), "turn-start"); + std::fs::write(vm_dir.join("README.md"), "hello\nreader\n").unwrap(); + ok(hook(&vm_dir, "turn-end", turn(1)), "turn-end"); + ok( + hook( + &vm_dir, + "session-end", + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "turn_number": 1, "timestamp": now, + }), + ), + "session-end", + ); + + // The turn's change is on the view, and its provenance is in the + // repository — not only in the sandbox. + let log = ok(atomic(host.path(), &["log"]), "host log"); + assert!(log.contains("greet the reader"), "{log}"); + let session = ok( + atomic(host.path(), &["session", "show", "agent-session"]), + "host session", + ); + assert!(session.contains("Turns: 1"), "{session}"); + assert!( + session.contains("Goal marker: greet the reader"), + "{session}" + ); +} diff --git a/atomic-repository/src/repository/changes.rs b/atomic-repository/src/repository/changes.rs index 7897bb8b..2a695ba8 100644 --- a/atomic-repository/src/repository/changes.rs +++ b/atomic-repository/src/repository/changes.rs @@ -1054,7 +1054,21 @@ impl Repository { /// dependencies, derived session tables, immutable `SESSION_TURNS` append, /// manifest, and `SESSION_HEADS` advance then commit in one pristine /// transaction. Repeating the exact publication is idempotent. + /// + /// In a remote sandbox the checkpoint is published in its repository, + /// through the owner; the result is the repository's. pub fn publish_provenance_checkpoint( + &self, + graph: &atomic_core::change::ProvenanceGraph, + turn: atomic_core::change::session::SessionTurn, + ) -> Result { + if self.is_remote_sandbox() { + return self.publish_remote_provenance_checkpoint(graph, turn); + } + self.publish_local_provenance_checkpoint(graph, turn) + } + + pub(crate) fn publish_local_provenance_checkpoint( &self, graph: &atomic_core::change::ProvenanceGraph, mut turn: atomic_core::change::session::SessionTurn, diff --git a/atomic-repository/src/repository/insert.rs b/atomic-repository/src/repository/insert.rs index 1882aaae..e2bf43d4 100644 --- a/atomic-repository/src/repository/insert.rs +++ b/atomic-repository/src/repository/insert.rs @@ -2154,6 +2154,9 @@ impl Repository { outcome: &RecordOutcome, mut options: InsertOptions, ) -> Result { + if self.is_remote_sandbox() { + return self.submit_recorded(outcome); + } let trace_record = std::env::var_os("ATOMIC_TRACE_RECORD").is_some(); let change = outcome.change(); let hash = outcome.hash(); diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index ec71760e..4150687d 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -81,7 +81,10 @@ mod filter; mod materialize; pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; mod remote_cache; -pub use remote_cache::{SandboxSkeleton, SandboxSlice, SpanBytes, SubmitRejection, Submitted}; +pub use remote_cache::{ + set_remote_sandbox_link, ChangeFile, RemoteSandboxLink, SandboxSkeleton, SandboxSlice, + SpanBytes, SubmitRejection, Submitted, +}; mod sandbox; mod semantic_materialize; mod split; diff --git a/atomic-repository/src/repository/record.rs b/atomic-repository/src/repository/record.rs index ffbda3f3..34ba8241 100644 --- a/atomic-repository/src/repository/record.rs +++ b/atomic-repository/src/repository/record.rs @@ -57,6 +57,10 @@ impl Repository { // Build the final header (may get message from options) let final_header = build_header(header, &options); + // A remote sandbox's cache takes the rows it is about to read. + self.hydrate_remote_sandbox() + .map_err(RecordError::Repository)?; + // Get repository status to find modified files let status_t0 = std::time::Instant::now(); let mut status_options = StatusOptions::default(); @@ -1173,6 +1177,9 @@ impl Repository { } } } + // A remote sandbox's change that didn't land isn't recorded + // anywhere: that is a failure, not a warning. + Err(e) if self.is_remote_sandbox() => return Err(RecordError::Repository(e)), Err(e) => { outcome.add_error("apply".to_string(), e.to_string()); } diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index 11214ae5..0519f99a 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -8,6 +8,7 @@ //! for what the rows are and why they are copied rather than re-derived. use std::collections::BTreeSet; +use std::path::Path; use atomic_core::change::ChangeStore as _; use atomic_core::pristine::slice::{GraphSlice, ViewSnapshotRows}; @@ -96,6 +97,52 @@ fn forbidden_path(path: &str) -> bool { }) } +/// How a remote sandbox's cache reaches its repository's owner. The +/// transport lives with whoever runs the process (the `atomic` CLI installs +/// one at startup); with it installed, `record`, `write_recorded` and the +/// readers below work in a remote sandbox for every caller. +pub trait RemoteSandboxLink: Send + Sync { + /// `FileStates`: the rows and content for `inodes`. + fn file_states(&self, root: &Path, inodes: Vec) -> Result; + /// `SubmitChange`: the change recorded against `base_state`; the view's + /// skeleton after it lands. + fn submit( + &self, + root: &Path, + base_state: String, + hash: Hash, + bytes: Vec, + ) -> Result, String>; + /// `Changes`: change files the view has. + fn changes(&self, root: &Path, hashes: Vec) -> Result, String>; + /// `PublishProvenance`: a checkpoint's provenance graph (serialized) and + /// session turn, published in the repository. + fn publish_provenance( + &self, + root: &Path, + graph: Vec, + turn: atomic_core::change::session::SessionTurn, + ) -> Result< + Result, + String, + >; +} + +static LINK: std::sync::OnceLock> = std::sync::OnceLock::new(); + +/// Install the process's link to remote sandbox owners (once). +pub fn set_remote_sandbox_link(link: Box) { + let _ = LINK.set(link); +} + +fn link() -> Result<&'static dyn RemoteSandboxLink, RepositoryError> { + LINK.get() + .map(|l| l.as_ref()) + .ok_or_else(|| RepositoryError::InvalidOperation { + message: "this process can't reach a remote sandbox's owner".to_string(), + }) +} + fn db(e: impl std::fmt::Display) -> RepositoryError { RepositoryError::Database(e.to_string()) } @@ -364,6 +411,124 @@ impl Repository { Ok(()) } + /// In a remote sandbox: load what reading or recording the working + /// tree needs from the owner (see [`Repository::sandbox_slice_inodes`]). + /// Elsewhere, nothing. + pub fn hydrate_remote_sandbox(&self) -> Result<(), RepositoryError> { + if !self.is_remote_sandbox() { + return Ok(()); + } + let inodes = self.sandbox_slice_inodes()?; + let slice = link()? + .file_states(&self.root, inodes) + .map_err(|message| RepositoryError::InvalidOperation { message })?; + self.import_sandbox_slice(&slice) + } + + /// In a remote sandbox: fetch the change files the view has and the + /// cache lacks. Returns how many. Elsewhere, nothing. + pub fn fetch_remote_sandbox_changes(&self) -> Result { + if !self.is_remote_sandbox() { + return Ok(0); + } + let missing = self.missing_sandbox_changes()?; + for batch in missing.chunks(32) { + let changes = link()? + .changes(&self.root, batch.to_vec()) + .map_err(|message| RepositoryError::InvalidOperation { message })?; + self.hold_sandbox_changes(&changes)?; + } + Ok(missing.len()) + } + + /// In a remote sandbox, publishing a checkpoint means publishing it in + /// the repository. + pub(crate) fn publish_remote_provenance_checkpoint( + &self, + graph: &atomic_core::change::ProvenanceGraph, + turn: atomic_core::change::session::SessionTurn, + ) -> Result { + let bytes = graph + .serialize() + .map_err(|e| RepositoryError::Serialization(e.to_string()))?; + link()? + .publish_provenance(&self.root, bytes, turn) + .map_err(|message| RepositoryError::InvalidOperation { message })? + .map_err(|rejection| { + RepositoryError::Apply(format!( + "the repository refused the provenance: {rejection}" + )) + }) + } + + /// In a remote sandbox, what applying a recorded change means: hand it + /// to the owner, and take the view as it is once it lands. + pub(crate) fn submit_recorded( + &self, + outcome: &crate::record::RecordOutcome, + ) -> Result { + let bytes = outcome + .v3_bytes() + .ok_or_else(|| RepositoryError::Apply("the change has no bytes".to_string()))? + .to_vec(); + let base_state = self.remote_sandbox_view_state()?; + let (submitted, skeleton) = link()? + .submit(&self.root, base_state, *outcome.hash(), bytes) + .map_err(|message| RepositoryError::InvalidOperation { message })? + .map_err(|rejection| { + RepositoryError::Apply(format!("the repository refused the change: {rejection}")) + })?; + self.import_sandbox_skeleton(&skeleton)?; + let state = + ::from_base32(submitted.state.as_bytes()) + .ok_or_else(|| RepositoryError::Apply("the owner sent a bad state".to_string()))?; + let mut stats = crate::InsertStats::new(); + stats.changes_applied = 1; + stats.applied_hashes.push(submitted.hash); + Ok(crate::InsertOutcome::new( + state, + skeleton.view.change_count, + false, + stats, + )) + } + + /// Serve side: publish a remote sandbox's checkpoint — if everything its + /// provenance explains is on `view`. The graph arrives serialized, so + /// what is published is exactly what the sandbox hashed. + pub fn publish_sandbox_provenance( + &self, + view: &str, + graph: &[u8], + turn: atomic_core::change::session::SessionTurn, + ) -> Result< + Result, + RepositoryError, + > { + use atomic_core::types::Base32; + let (graph, _) = match atomic_core::change::ProvenanceGraph::deserialize(graph) { + Ok(parsed) => parsed, + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible = super::collect_visible_change_ids(&txn, &state)?; + for change in &graph.changes_explained { + match txn.get_internal(change).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(change.to_base32()))), + } + } + } + Ok(Ok(self.publish_local_provenance_checkpoint(&graph, turn)?)) + } + /// Cache side: the view's state as the cache has it (base32) — what a /// change recorded here is recorded against. pub fn remote_sandbox_view_state(&self) -> Result { From 9aa898f9f770291b00bc48733c61a8c4b56f9d28 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 14:41:28 -0500 Subject: [PATCH 12/15] A remote sandbox's cache takes its view's vault The vault arrives as files, as after a pull, so building the cache bootstraps the vault tables from them; intents written in the sandbox are then recorded and land on the view like any other change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/remote_sandbox_integration_test.rs | 37 ++++++++++++------- atomic-repository/src/repository/sandbox.rs | 5 +++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index 43df3323..f88e1dda 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -7,8 +7,6 @@ use std::path::Path; use std::process::{Command, Output}; -use atomic_core::change::ChangeHeader; -use atomic_repository::{InsertOptions, RecordOptions, Repository, TrackingOptions}; use serde_json::Value; use tempfile::TempDir; @@ -41,21 +39,20 @@ fn failure(output: Output, what: &str) -> String { ) } -/// A repository with two recorded files on `dev`. +/// A repository as `atomic init` makes one (vault included), with two +/// recorded files on `dev`. fn repository(root: &Path) { - let repo = Repository::init(root).unwrap(); + ok(atomic(root, &["init"]), "init"); std::fs::create_dir_all(root.join("src")).unwrap(); std::fs::write(root.join("README.md"), "hello\n").unwrap(); std::fs::write(root.join("src/lib.rs"), "pub fn f() {}\n").unwrap(); - repo.add("README.md", TrackingOptions::default()).unwrap(); - repo.add("src/lib.rs", TrackingOptions::default()).unwrap(); - let options = RecordOptions::new() - .with_all(true) - .save_to_store(true) - .apply_after_record(false); - let outcome = repo.record(ChangeHeader::new("first"), options).unwrap(); - repo.write_recorded(&outcome, InsertOptions::default()) - .unwrap(); + ok(atomic(root, &["add", "README.md", "src/lib.rs"]), "add"); + ok(atomic(root, &["record", "-a", "-m", "first"]), "record"); + // No owner is left running from setup. + let _ = atomic( + root, + &["agent", "database-owner", "shutdown", "--repository", "."], + ); } struct Owner<'a>(&'a Path); @@ -132,7 +129,7 @@ fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { // The view's tree, and nothing of the repository. let out = ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); - assert!(out.contains("Materialized 3 entries"), "{out}"); + assert!(out.contains("Materialized "), "{out}"); assert_eq!( std::fs::read_to_string(vm_dir.join("README.md")).unwrap(), "hello\n" @@ -362,6 +359,18 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { // repository — not only in the sandbox. let log = ok(atomic(host.path(), &["log"]), "host log"); assert!(log.contains("greet the reader"), "{log}"); + + // An intent written in the sandbox is recorded there and lands too. + ok( + atomic(&vm_dir, &["intent", "new", "Greet readers"]), + "intent new", + ); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "the intent"]), + "record the intent", + ); + let change = ok(atomic(host.path(), &["change"]), "host change"); + assert!(change.contains(".vault/intents/"), "{change}"); let session = ok( atomic(host.path(), &["session", "show", "agent-session"]), "host session", diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index ba7311cf..0950a039 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -159,6 +159,11 @@ impl Repository { let repo = Self::open_sandbox(working_root, &cache, &skeleton.view.name)?; repo.import_sandbox_skeleton(skeleton)?; repo.reindex_working_copy()?; + // The view's vault arrived as files, as after a pull: its tables + // come from them. + if repo.vault_dir().exists() { + repo.bootstrap_vault_from_working_copy()?; + } Ok(repo) } From 3001ba7224ca7d7399988db528eccd041add2d38 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 15:02:06 -0500 Subject: [PATCH 13/15] Expeditions' draft views: render them as themselves, and index their vault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A draft's structural changes (files it adds, moves, deletes) wait in the deferred tree journal until the view is checked out; TREE is the checked-out view's. Rendering any other view — materialize_view_entries, so Materialize and the skeleton after a submit — now projects that view's journal in a write transaction that is thrown away, and the skeleton's tree is the view as rendered (implicit directories, inode 0, left out). Before, a file added on an expedition's draft never appeared on it. When a sandbox's change lands, the vault files it touched are indexed into the repository's vault from the view's content, as a pull does from the files it writes (vault_record_files, factored out of the working-copy deflate). An intent created in an expedition is in `intent list` again. Tests: record parity and a submitted change on a draft, a second record building on the first, and the agent flow on a draft view off dev. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/agent/owner/sandbox.rs | 22 +-- .../tests/remote_sandbox_integration_test.rs | 18 +- atomic-core/src/pristine/slice.rs | 37 ++-- .../src/repository/materialize.rs | 42 +++- .../src/repository/remote_cache.rs | 48 ++++- atomic-repository/src/repository/vault.rs | 75 +++++-- .../tests/remote_cache_parity_test.rs | 187 +++++++++++++++++- 7 files changed, 359 insertions(+), 70 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs index eaa8898e..6515862c 100644 --- a/atomic-cli/src/commands/agent/owner/sandbox.rs +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -296,12 +296,14 @@ where let (tx, mut rx) = tokio::sync::mpsc::channel::(64); let root = owner.root.clone(); let render = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - let repo = Repository::open_readonly(&root)?; + // Writable: listing a view other than the checked-out one projects its + // tree in a transaction that is thrown away. + let repo = Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; let mut entries = 0u64; - let mut live = std::collections::BTreeSet::new(); + let mut live = std::collections::BTreeMap::new(); let rendered = repo.materialize_view_entries(&view, |entry| { entries += 1; - live.insert(entry.inode); + live.insert(entry.inode, entry.path.clone()); tx.blocking_send(OwnerResponse::MaterializeEntry { entry }) .map_err(|_| ()) })?; @@ -402,8 +404,7 @@ pub(crate) async fn record_request( Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; match repo.insert_submitted_change(&view, &base_state, &hash, &bytes)? { Ok(submitted) => { - let skeleton = - repo.export_sandbox_skeleton(&view, &live_inodes(&repo, &view)?)?; + let skeleton = repo.after_sandbox_submit(&view, &submitted.hash)?; Ok(Ok((submitted, skeleton))) } Err(rejection) => Ok(Err(rejection)), @@ -423,17 +424,6 @@ pub(crate) async fn record_request( } } -/// The inodes `view`'s tree holds. -fn live_inodes(repo: &Repository, view: &str) -> anyhow::Result> { - let mut live = std::collections::BTreeSet::new(); - repo.materialize_view_entries::<()>(view, |entry| { - live.insert(entry.inode); - Ok(()) - })? - .map_err(|()| anyhow::anyhow!("unreachable"))?; - Ok(live) -} - /// Client side of `Materialize`: write the view's tree into `dir` and return /// how many entries it had. Files already at those paths are overwritten; /// nothing else in `dir` is touched. diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs index f88e1dda..c4e26b70 100644 --- a/atomic-cli/tests/remote_sandbox_integration_test.rs +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -312,7 +312,8 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { "create", "exp-1", "--remote", - "--view", + // As basecamp runs an expedition: its own draft view, off dev. + "--from", "dev", "--dest", vm_dir.to_str().unwrap(), @@ -357,8 +358,16 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { // The turn's change is on the view, and its provenance is in the // repository — not only in the sandbox. - let log = ok(atomic(host.path(), &["log"]), "host log"); + let log = ok( + atomic(host.path(), &["log", "--view", "exp-1"]), + "host log of the draft", + ); assert!(log.contains("greet the reader"), "{log}"); + let dev = ok(atomic(host.path(), &["log"]), "host log of dev"); + assert!( + !dev.contains("greet the reader"), + "only on the expedition's view: {dev}" + ); // An intent written in the sandbox is recorded there and lands too. ok( @@ -369,8 +378,9 @@ fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { atomic(&vm_dir, &["record", "-a", "-m", "the intent"]), "record the intent", ); - let change = ok(atomic(host.path(), &["change"]), "host change"); - assert!(change.contains(".vault/intents/"), "{change}"); + // ...and the repository's vault knows it, as it would after a pull. + let intents = ok(atomic(host.path(), &["intent", "list"]), "host intent list"); + assert!(intents.contains("backlog"), "{intents}"); let session = ok( atomic(host.path(), &["session", "show", "agent-session"]), "host session", diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs index 4a6888a7..bb33eeb5 100644 --- a/atomic-core/src/pristine/slice.rs +++ b/atomic-core/src/pristine/slice.rs @@ -7,8 +7,9 @@ //! exactly the rows the repository has. A slice is those rows, byte for byte: //! //! - the **skeleton** ([`ReadTxn::export_skeleton`]): the view's tree — -//! paths, inodes and their positions, directories — and the id tables for -//! every change the view can see. Enough for `status` with no graph at all. +//! paths (as the view renders them), inodes and their positions, +//! directories — and the id tables for every change the view can see. +//! Enough for `status` with no graph at all. //! - a **graph slice** ([`ReadTxn::export_graph_slice`]): for a set of //! inodes, the GRAPH / INODE_GRAPH rows of each file's content and of its //! name chain up to the root, one hop of neighbours (so `find_block` finds @@ -139,14 +140,15 @@ enum Reach { } impl ReadTxn { - /// The view's tree and ids: TREE for the `live` inodes (the paths the - /// view actually has), REV_TREE / INODES / DIRECTORIES for every inode - /// whose position a visible change introduced, and the id rows for every - /// visible change. + /// The view's tree and ids. `live` is the view's tree as rendered — + /// every path it has and the inode there, which is what TREE would hold + /// with the view checked out (TREE itself is the checked-out view's). + /// REV_TREE / INODES / DIRECTORIES come for every inode whose position a + /// visible change introduced, and the id rows for every visible change. pub fn export_skeleton( &self, visible: &BTreeSet, - live: &BTreeSet, + live: &BTreeMap, ) -> PristineResult { let mut out = GraphSlice::default(); let inodes = self.txn.open_table(INODES)?; @@ -159,20 +161,23 @@ impl ReadTxn { out.inodes.push((k.value(), *v.value())); } } - let rev_tree = self.txn.open_table(REV_TREE)?; - for row in rev_tree.iter()? { + let mut rev_tree: BTreeMap = BTreeMap::new(); + for row in self.txn.open_table(REV_TREE)?.iter()? { let (k, v) = row?; if kept.contains(&k.value()) { - out.rev_tree.push((k.value(), v.value().to_string())); + rev_tree.insert(k.value(), v.value().to_string()); } } - let tree = self.txn.open_table(TREE)?; - for row in tree.iter()? { - let (k, v) = row?; - if live.contains(&v.value()) && kept.contains(&v.value()) { - out.tree.push((k.value().to_string(), v.value())); - } + // Implicit directories render with no inode of their own (0): only + // what the repository tracks goes in the tree. + for (inode, path) in live + .iter() + .filter(|(inode, _)| **inode != 0 && kept.contains(inode)) + { + rev_tree.insert(*inode, path.clone()); + out.tree.push((path.clone(), *inode)); } + out.rev_tree = rev_tree.into_iter().collect(); let directories = self.txn.open_table(DIRECTORIES)?; for row in directories.iter()? { let (k, v) = row?; diff --git a/atomic-repository/src/repository/materialize.rs b/atomic-repository/src/repository/materialize.rs index 24a2f81b..7d707721 100644 --- a/atomic-repository/src/repository/materialize.rs +++ b/atomic-repository/src/repository/materialize.rs @@ -1083,6 +1083,35 @@ mod serde_bytes_vec { } impl Repository { + /// The entries `view_name` has, from TREE as that view projects it. + /// + /// TREE is the checked-out view's projection; another view's structural + /// operations (files it added, moved or deleted) wait in the deferred + /// tree journal until a switch applies them. For another view they are + /// applied here in a write transaction that is thrown away, so the + /// listing is that view's and nothing on disk changes. A read-only handle + /// can't do that and lists TREE as it is. + fn view_tree_items( + &self, + txn: &atomic_core::pristine::ReadTxn, + view_name: &str, + options: &atomic_core::output::repo::MaterializeOptions, + ) -> Result, RepositoryError> { + use atomic_core::output::repo::collect_children; + let db = |e: atomic_core::pristine::PristineError| RepositoryError::Database(e.to_string()); + if view_name != self.current_view { + let journal = self.load_deferred_tree_journal()?; + if let Ok(mut projected) = self.pristine.write_txn() { + self.apply_deferred_tree_ops_in_txn(&mut projected, &journal, view_name)?; + let items = collect_children(&projected, Inode::ROOT, "", options).map_err(db)?; + use atomic_core::pristine::MutTxnT; + projected.abort().map_err(db)?; + return Ok(items); + } + } + collect_children(txn, Inode::ROOT, "", options).map_err(db) + } + /// Render every entry of `view` and hand each to `sink`, in path order /// (a directory before what it contains), touching nothing: no working /// tree, no stat cache, no conflict state. Read-only, so it runs beside @@ -1096,7 +1125,7 @@ impl Repository { view_name: &str, mut sink: impl FnMut(ViewEntry) -> Result<(), E>, ) -> Result, RepositoryError> { - use atomic_core::output::repo::{collect_children, MaterializeOptions}; + use atomic_core::output::repo::MaterializeOptions; use atomic_core::types::Base32; use rayon::prelude::*; @@ -1114,8 +1143,7 @@ impl Repository { })?; let change_filter_arc = Arc::new(collect_visible_change_ids(&txn, &view)?); let options = MaterializeOptions::new().with_change_filter_arc(change_filter_arc.clone()); - let mut items = collect_children(&txn, Inode::ROOT, "", &options) - .map_err(|e| RepositoryError::Database(e.to_string()))?; + let mut items = self.view_tree_items(&txn, view_name, &options)?; items.sort_by(|a, b| a.path.cmp(&b.path)); let visible = |item: &atomic_core::output::repo::OutputItem| { @@ -1249,7 +1277,7 @@ fn render_view_file( // in a single range scan, then run retrieve_graph over the // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) // individual B-tree probes. - let preloaded = InodePreloadTxn::from_table(&txn, item.inode, &inode_graph_table) + let preloaded = InodePreloadTxn::from_table(txn, item.inode, inode_graph_table) .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; let t_retrieve = std::time::Instant::now(); @@ -1294,10 +1322,10 @@ fn render_view_file( // surfaced instead of silently collapsed (rubric A12). let content = match name_conflicts.get(&item.path) { Some(sides) => render_name_conflict( - &txn, + txn, store, - &inode_graph_table, - &change_filter_arc, + inode_graph_table, + change_filter_arc, &item.path, sides, ) diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs index 0519f99a..f6f1594b 100644 --- a/atomic-repository/src/repository/remote_cache.rs +++ b/atomic-repository/src/repository/remote_cache.rs @@ -162,12 +162,12 @@ impl Repository { Ok(ids) } - /// Serve side: `view`'s skeleton, for the inodes `live` (the ones its - /// materialized tree holds). + /// Serve side: `view`'s skeleton, with `live` its rendered tree (inode → + /// path, as [`Repository::materialize_view_entries`] gives them). pub fn export_sandbox_skeleton( &self, view: &str, - live: &BTreeSet, + live: &std::collections::BTreeMap, ) -> Result { let txn = self.pristine.read_txn().map_err(db)?; let state = @@ -493,6 +493,48 @@ impl Repository { )) } + /// Serve side, once a sandbox's change `hash` has landed on `view`: the + /// view's skeleton for the sandbox's cache, and — as a pull does for the + /// files it writes — the vault files the change touched, indexed into + /// the repository's (repository-wide) vault. One render of the view + /// serves both. + pub fn after_sandbox_submit( + &self, + view: &str, + hash: &Hash, + ) -> Result { + let change = self.load_change(hash)?; + let vault_paths: BTreeSet = change + .hunks() + .iter() + .filter_map(|h| h.path()) + .chain(change.file_ops().iter().map(|ops| ops.path())) + .filter(|p| p.starts_with(".vault/") && p.ends_with(".md")) + .map(str::to_string) + .collect(); + let mut live = std::collections::BTreeMap::new(); + let mut files = std::collections::BTreeMap::new(); + self.materialize_view_entries::<()>(view, |entry| { + live.insert(entry.inode, entry.path.clone()); + if vault_paths.contains(&entry.path) { + files.insert( + entry.path, + String::from_utf8_lossy(&entry.content).into_owned(), + ); + } + Ok(()) + })? + .map_err(|()| RepositoryError::Output("unreachable".to_string()))?; + if !vault_paths.is_empty() && self.has_vault()? { + let rows: Vec<(String, Option)> = vault_paths + .iter() + .map(|p| (p[".vault/".len()..].to_string(), files.remove(p))) + .collect(); + self.vault_record_files(&rows)?; + } + self.export_sandbox_skeleton(view, &live) + } + /// Serve side: publish a remote sandbox's checkpoint — if everything its /// provenance explains is on `view`. The graph arrives serialized, so /// what is published is exactly what the sandbox hashed. diff --git a/atomic-repository/src/repository/vault.rs b/atomic-repository/src/repository/vault.rs index a14ca237..a88c43dc 100644 --- a/atomic-repository/src/repository/vault.rs +++ b/atomic-repository/src/repository/vault.rs @@ -594,9 +594,38 @@ impl Repository { // Read file content let file_content = std::fs::read_to_string(path)?; + if let Some(change) = self.vault_file_change(rel_path_str, &file_content)? { + changes.push(change); + } + } + + // Also check for deleted entries (in redb but not on disk) + let all_entries = self.vault_list("", None)?; + for meta in all_entries { + let disk_path = vault_dir.join(&meta.path); + if !disk_path.exists() { + changes.push(VaultFileChange { + path: meta.path, + change_type: VaultChangeType::Deleted, + frontmatter_json: String::new(), + content: Vec::new(), + }); + } + } + + Ok(changes) + } + /// How one vault file (`rel_path`, vault-relative, with `file_content`) + /// differs from its stored entry — `None` when it doesn't. + fn vault_file_change( + &self, + rel_path_str: String, + file_content: &str, + ) -> Result, RepositoryError> { + { // Parse into frontmatter + body - let (frontmatter_json, body) = parse_markdown_frontmatter(&file_content); + let (frontmatter_json, body) = parse_markdown_frontmatter(file_content); // Body hashes remain useful for content/embedding caches, but // frontmatter is also knowledge: status, labels, identity, and @@ -611,35 +640,43 @@ impl Repository { if Hash::from_bytes(entry.content_hash) == new_hash && frontmatter_json_equal(&entry.frontmatter_json, &frontmatter_json) => { - continue; // Unchanged, skip + return Ok(None); // Unchanged, skip } Some(_) => VaultChangeType::Modified, None => VaultChangeType::New, }; - changes.push(VaultFileChange { + Ok(Some(VaultFileChange { path: rel_path_str, change_type, frontmatter_json, content: body_bytes.to_vec(), - }); + })) } + } - // Also check for deleted entries (in redb but not on disk) - let all_entries = self.vault_list("", None)?; - for meta in all_entries { - let disk_path = vault_dir.join(&meta.path); - if !disk_path.exists() { - changes.push(VaultFileChange { - path: meta.path, + /// Deflate vault files that are not in this working copy — a view's, + /// say — into redb: each `(vault-relative path, content)`, `None` for a + /// file the view no longer has. What `vault_record_working_copy` does + /// for the files on disk. + pub fn vault_record_files( + &self, + files: &[(String, Option)], + ) -> Result, RepositoryError> { + let mut changes = Vec::new(); + for (path, content) in files { + match content { + Some(content) => changes.extend(self.vault_file_change(path.clone(), content)?), + None if self.vault_retrieve(path)?.is_some() => changes.push(VaultFileChange { + path: path.clone(), change_type: VaultChangeType::Deleted, frontmatter_json: String::new(), content: Vec::new(), - }); + }), + None => {} } } - - Ok(changes) + self.vault_apply_changes(&changes) } /// Deflate changed vault files from disk into redb. @@ -653,10 +690,18 @@ impl Repository { /// - `vault_record_working_copy`: disk -> redb (after humans edit) pub fn vault_record_working_copy(&self) -> Result, RepositoryError> { let changes = self.vault_scan_working_copy()?; + self.vault_apply_changes(&changes) + } + /// Store detected vault changes, then index any intents the manifest + /// is missing. + fn vault_apply_changes( + &self, + changes: &[VaultFileChange], + ) -> Result, RepositoryError> { let mut updated_paths = Vec::new(); - for change in &changes { + for change in changes { match change.change_type { VaultChangeType::New | VaultChangeType::Modified => { // Infer entry type from path diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs index 8198c4bf..1c4be435 100644 --- a/atomic-repository/tests/remote_cache_parity_test.rs +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -6,7 +6,7 @@ //! produces — which carries the repository's internal ids — must be byte for //! byte the change `record` produces on the repository itself. -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::fs; use std::path::Path; @@ -74,7 +74,7 @@ impl Pair { /// Ship the slice for what `record` will touch, as FileStates does: /// only the inodes the cache asks for. - fn hydrate(&self, view: &str, live: &BTreeSet) { + fn hydrate(&self, view: &str, live: &BTreeMap) { let inodes = self.cache.sandbox_slice_inodes().unwrap(); assert!( inodes.len() < live.len(), @@ -86,7 +86,7 @@ impl Pair { } /// A host repository with some history, and a cache materialized from it. -fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeSet) { +fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeMap) { let host_dir = TempDir::new().unwrap(); let cache_dir = TempDir::new().unwrap(); let host = Repository::init(host_dir.path()).unwrap(); @@ -96,9 +96,9 @@ fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeSet) { // Materialize: the view's files into the cache's working tree, and the // inodes they are. let cache = Repository::init(cache_dir.path()).unwrap(); - let mut live = BTreeSet::new(); + let mut live = BTreeMap::new(); host.materialize_view_entries::<()>(&view, |entry| { - live.insert(entry.inode); + live.insert(entry.inode, entry.path.clone()); let path = cache_dir.path().join(&entry.path); match entry.kind { atomic_repository::ViewEntryKind::Directory => fs::create_dir_all(path).unwrap(), @@ -126,7 +126,7 @@ fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeSet) { ) } -fn assert_parity(pair: &Pair, view: &str, live: &BTreeSet, what: &str) { +fn assert_parity(pair: &Pair, view: &str, live: &BTreeMap, what: &str) { pair.hydrate(view, live); let h = header(what); let host = change_bytes(&pair.host, h.clone()); @@ -257,7 +257,7 @@ use atomic_repository::SubmitRejection; fn view_state(repo: &Repository, view: &str) -> String { let skeleton = repo - .export_sandbox_skeleton(view, &BTreeSet::new()) + .export_sandbox_skeleton(view, &BTreeMap::new()) .unwrap(); atomic_core::types::Base32::to_base32(&skeleton.view.state) } @@ -266,7 +266,7 @@ fn view_state(repo: &Repository, view: &str) -> String { fn record_in_cache( pair: &Pair, view: &str, - live: &BTreeSet, + live: &BTreeMap, what: &str, ) -> (atomic_core::types::Hash, Vec) { pair.hydrate(view, live); @@ -336,7 +336,7 @@ fn a_submitted_change_lands_on_the_view_and_the_next_one_builds_on_it() { live.clear(); pair.host .materialize_view_entries::<()>(&view, |e| { - live.insert(e.inode); + live.insert(e.inode, e.path.clone()); Ok(()) }) .unwrap() @@ -438,3 +438,172 @@ fn a_change_from_another_view_is_refused() { ); assert_eq!(view_state(&host, "dev"), base); } + +#[test] +fn a_change_submitted_to_a_draft_view_is_on_that_view() { + // Expeditions work on draft views: what lands on one renders there. + let (pair, _, _) = pair(two_files); + let mut host = pair.host; + host.create_view_from("exp", "dev").unwrap(); + let cache_dir = TempDir::new().unwrap(); + drop(Repository::init(cache_dir.path()).unwrap()); + // As a remote sandbox's cache is opened: on its pointer's view. + let cache = Repository::open_sandbox(cache_dir.path(), cache_dir.path(), "exp").unwrap(); + let mut live = BTreeMap::new(); + host.materialize_view_entries::<()>("exp", |entry| { + live.insert(entry.inode, entry.path.clone()); + let path = cache_dir.path().join(&entry.path); + match entry.kind { + atomic_repository::ViewEntryKind::Directory => fs::create_dir_all(path).unwrap(), + _ => { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + } + Ok(()) + }) + .unwrap() + .unwrap(); + cache + .import_sandbox_skeleton(&host.export_sandbox_skeleton("exp", &live).unwrap()) + .unwrap(); + cache.reindex_working_copy().unwrap(); + write(cache_dir.path(), "src/new.rs", "pub fn n() {}\n"); + write(cache_dir.path(), "README.md", "hello\non the draft\n"); + cache.add("src/new.rs", TrackingOptions::default()).unwrap(); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = cache.record(header("on the draft"), options).unwrap(); + let base = view_state(&host, "exp"); + host.insert_submitted_change("exp", &base, outcome.hash(), outcome.v3_bytes().unwrap()) + .unwrap() + .expect("accepted"); + + let tree = tree_of(&host, "exp"); + assert!( + tree.contains(&("README.md".into(), b"hello\non the draft\n".to_vec())), + "{tree:?}" + ); + assert!( + tree.contains(&("src/new.rs".into(), b"pub fn n() {}\n".to_vec())), + "{tree:?}" + ); + assert!( + !tree_of(&host, "dev").iter().any(|(p, _)| p == "src/new.rs"), + "only on the draft" + ); + + // The next record builds on it: the refreshed skeleton knows the new file. + let skeleton = host.after_sandbox_submit("exp", outcome.hash()).unwrap(); + cache.import_sandbox_skeleton(&skeleton).unwrap(); + cache.reindex_working_copy().unwrap(); + write(cache_dir.path(), "src/new.rs", "pub fn n() { 2 }\n"); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let second = cache.record(header("again on the draft"), options).unwrap(); + let (change, _) = + atomic_core::change::Change::deserialize(&mut &second.v3_bytes().unwrap()[..]).unwrap(); + assert!( + !change + .hunks() + .iter() + .any(|h| matches!(h, atomic_core::change::GraphOp::FileAdd { .. })), + "an edit, not a second add" + ); + host.insert_submitted_change( + "exp", + &view_state(&host, "exp"), + second.hash(), + second.v3_bytes().unwrap(), + ) + .unwrap() + .expect("accepted"); + assert!(tree_of(&host, "exp").contains(&("src/new.rs".into(), b"pub fn n() { 2 }\n".to_vec()))); +} + +#[test] +fn control_a_local_sandbox_record_on_a_draft_renders_on_it() { + let host_dir = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("exp", "dev").unwrap(); + host.provision_sandbox(work.path().join("w"), "exp") + .unwrap(); + } + let sbx = Repository::open_existing(work.path().join("w")).unwrap(); + write(sbx.root(), "README.md", "hello\non the draft\n"); + record_and_apply(&sbx, "on the draft"); + drop(sbx); + let host = Repository::open_existing(host_dir.path()).unwrap(); + let tree = tree_of(&host, "exp"); + assert!( + tree.contains(&("README.md".into(), b"hello\non the draft\n".to_vec())), + "{tree:?}" + ); +} + +#[test] +fn parity_on_a_draft_view() { + let host_dir = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("exp", "dev").unwrap(); + host.provision_sandbox(work.path().join("w"), "exp") + .unwrap(); + } + // The cache, from the draft. + let cache_dir = TempDir::new().unwrap(); + drop(Repository::init(cache_dir.path()).unwrap()); + let cache = Repository::open_sandbox(cache_dir.path(), cache_dir.path(), "exp").unwrap(); + let skeleton = { + let host = Repository::open_existing(host_dir.path()).unwrap(); + let mut live = BTreeMap::new(); + host.materialize_view_entries::<()>("exp", |entry| { + live.insert(entry.inode, entry.path.clone()); + let path = cache_dir.path().join(&entry.path); + if entry.kind != atomic_repository::ViewEntryKind::Directory { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + Ok(()) + }) + .unwrap() + .unwrap(); + host.export_sandbox_skeleton("exp", &live).unwrap() + }; + cache.import_sandbox_skeleton(&skeleton).unwrap(); + cache.reindex_working_copy().unwrap(); + + let sbx = Repository::open_existing(work.path().join("w")).unwrap(); + write(sbx.root(), "README.md", "hello\non the draft\n"); + write(cache_dir.path(), "README.md", "hello\non the draft\n"); + let h = header("on the draft"); + let local = change_bytes(&sbx, h.clone()); + drop(sbx); + let host = Repository::open_existing(host_dir.path()).unwrap(); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let remote = change_bytes(&cache, h); + assert_eq!( + local.0, remote.0, + "the cache records a different change on a draft" + ); +} From 555a7d28971344db02ffd38740571df4cf262f9d Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Thu, 24 Sep 2026 21:42:49 -0500 Subject: [PATCH 14/15] Stage and seal a view as it renders materialize_view_to (behind `sandbox stage` and `seal`) listed files from TREE, which is the checked-out view's: a file another view added was missing from its image. It now writes materialize_view_entries, which projects the view's own tree. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-repository/src/repository/sandbox.rs | 18 ++++++++++-------- .../tests/remote_cache_parity_test.rs | 9 +++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index 0950a039..123295da 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -279,19 +279,21 @@ impl Repository { pub fn materialize_view_to(&self, view: &str, dir: &Path) -> Result { std::fs::create_dir_all(dir)?; + // The view as it renders — its own added and moved files included, + // whichever view is checked out. let mut count = 0usize; - for path in self.visible_file_paths(view)? { - let bytes = match self.get_file_content_on_view(&path, view)? { - Some(bytes) => bytes, - None => continue, - }; - let target = dir.join(&path); + self.materialize_view_entries(view, |entry| -> Result<(), std::io::Error> { + if entry.kind == super::ViewEntryKind::Directory { + return Ok(()); + } + let target = dir.join(&entry.path); if let Some(parent) = target.parent() { std::fs::create_dir_all(parent)?; } - std::fs::write(&target, &bytes)?; + std::fs::write(&target, &entry.content)?; count += 1; - } + Ok(()) + })??; Ok(count) } diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs index 1c4be435..a54823c1 100644 --- a/atomic-repository/tests/remote_cache_parity_test.rs +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -498,6 +498,15 @@ fn a_change_submitted_to_a_draft_view_is_on_that_view() { !tree_of(&host, "dev").iter().any(|(p, _)| p == "src/new.rs"), "only on the draft" ); + // What `sandbox stage` / `seal` write sees it too. + let out = TempDir::new().unwrap(); + host.materialize_view_to("exp", out.path()).unwrap(); + assert_eq!( + fs::read_to_string(out.path().join("src/new.rs")) + .ok() + .as_deref(), + Some("pub fn n() {}\n") + ); // The next record builds on it: the refreshed skeleton knows the new file. let skeleton = host.after_sandbox_submit("exp", outcome.hash()).unwrap(); From d4c0d8dd07d540dd376cc54cf3ebecc408a6ce3d Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Fri, 25 Sep 2026 00:48:29 -0500 Subject: [PATCH 15/15] Let the owner's iroh endpoints bind IPv4 only ATOMIC_IROH_IPV4_ONLY=1 makes the owner endpoint and the client dialer open no IPv6 socket. Under some microVM network backends (smolvm TSI, in a fork) creating one takes the process down, so a sandbox inside such a VM could not materialize or submit. Co-Authored-By: Claude Opus 5.5 (1M context) --- atomic-cli/src/commands/agent/owner/remote.rs | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/atomic-cli/src/commands/agent/owner/remote.rs b/atomic-cli/src/commands/agent/owner/remote.rs index 94fbbb93..b4f783f7 100644 --- a/atomic-cli/src/commands/agent/owner/remote.rs +++ b/atomic-cli/src/commands/agent/owner/remote.rs @@ -205,12 +205,7 @@ pub(crate) async fn bind(dot_dir: &Path, offline: bool) -> anyhow::Result return Err(e).with_context(|| format!("failed to read {}", key_path.display())), }; - let builder = if offline { - Endpoint::builder(presets::Minimal) - } else { - Endpoint::builder(presets::N0) - }; - Ok(builder + Ok(builder(offline)? .secret_key(key) .alpns(vec![ALPN.to_vec()]) .bind() @@ -244,12 +239,30 @@ pub(crate) async fn pointer_addr(endpoint: &Endpoint, offline: bool) -> Endpoint /// A client endpoint for dialing an owner. async fn dialer(offline: bool) -> anyhow::Result { + Ok(builder(offline)?.bind().await?) +} + +/// An endpoint builder: the relay-less preset when offline, and IPv4 only +/// when `ATOMIC_IROH_IPV4_ONLY` is set — for hosts where creating an IPv6 +/// socket misbehaves (it takes down the process under some microVM network +/// backends), so the endpoint never opens one. +fn builder(offline: bool) -> anyhow::Result { let builder = if offline { Endpoint::builder(presets::Minimal) } else { Endpoint::builder(presets::N0) }; - Ok(builder.bind().await?) + if !ipv4_only() { + return Ok(builder); + } + builder + .clear_ip_transports() + .bind_addr("0.0.0.0:0") + .map_err(|e| anyhow!("invalid bind address: {e}")) +} + +fn ipv4_only() -> bool { + std::env::var("ATOMIC_IROH_IPV4_ONLY").is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true")) } fn write_private(path: &Path, bytes: &[u8]) -> anyhow::Result<()> {