diff --git a/AGENTS.md b/AGENTS.md index cc81a200..2184d574 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -340,6 +340,38 @@ Human: "I can review that!" | `EdgeFlags` | 1 byte | Bitflags: BLOCK, PSEUDO, FOLDER, PARENT, DELETED | | `SerializedGraphEdge` | 24 bytes | Compact edge: (flags+pos, change, introduced_by) | +#### Endianness: keys that sort are big-endian, opaque values are little-endian + +The storage layer encodes integers two ways, and the rule is whether the +bytes ever get sorted: + +| Encoded as | Endian | Which | +|------------|--------|-------| +| **Big-endian** | BE | B-tree **keys** that are range-scanned: `encode_vertex`, `encode_inode_vertex`, `encode_position`, `encode_view_seq` (`pristine/tables.rs`) | +| **Little-endian** | LE | Values and ids only ever looked up by exact key: `SerializedGraphEdge`, and the CRDT `TrunkId`/`BranchId`/`LeafId` (`crdt/tables.rs`, `crdt/ids.rs`) | + +BE is load-bearing for the keys: byte order has to match numeric order or a +range scan means nothing. The file-local INODE_GRAPH traversal depends on it — + +```rust +let lo = encode_inode_vertex(inode, 0, 0, 0); +let hi = encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX); +for row in inode_graph.range::<&[u8; 32]>(&lo..=&hi)? { ... } // every row for one inode +``` + +For the LE side, sorting never happens, so byte order costs nothing and there +is no reason to pay for BE. + +**The two meet in the same files**, so match the *writer*, never the +neighbours: a file that decodes graph vertex keys (BE) can also decode CRDT +ids (LE) within a few lines, and a hand-rolled `id[0..8]` slice reads the +wrong end of a LE id without any visible failure — the id just looks absent. +Decode an id with its own type's `from_bytes` (`TrunkId::from_bytes`, +`BranchId::from_bytes`, `LeafId::from_bytes`) instead of slicing by hand, so +the two sides cannot drift; a mis-decoded id is indistinguishable from an +absent one at runtime, so an `EXTERNAL`-row lookup for it is worth a +`debug_assert`. + ### Hash Type Design Following the original Atomic project, `Hash` is a **type alias** for `Merkle`: diff --git a/Cargo.lock b/Cargo.lock index 358b0f79..89ca000f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -455,6 +455,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomic-wasm" +version = "0.19.1" +dependencies = [ + "atomic-canonical", + "atomic-identity", + "getrandom 0.2.17", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "autocfg" version = "1.5.1" diff --git a/Cargo.toml b/Cargo.toml index d50c46c8..cfa22eba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "atomic-repository", "atomic-semantic", "atomic-teams", + "atomic-wasm", "libatomic", ] diff --git a/atomic-canonical/src/proof.rs b/atomic-canonical/src/proof.rs index afd3710d..52bf1da3 100644 --- a/atomic-canonical/src/proof.rs +++ b/atomic-canonical/src/proof.rs @@ -110,8 +110,32 @@ pub fn substance_view(value: &Value) -> Value { /// path all typed nodes share. Fills `attributedTo` (from the identity's /// `did:atomic`) when absent, computes the content hash over `hashing_view`, /// signs `jcs(signing_view)`, and attaches the proof. Returns the value. -pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> Value { - let did = did::did_for_public_key(&identity.public_key); +/// +/// It is [`prepare_attestation`] → sign → [`attach_proof`]; a signer that +/// holds its key outside this process uses those two halves directly. +pub fn attest_value(value: Value, identity: &Identity, keypair: &KeyPair) -> Value { + let prepared = prepare_attestation(value, &identity.public_key); + let signature = Signer::new(keypair).sign(&prepared.signing_bytes); + attach_proof(prepared.value, &identity.public_key, &signature) +} + +/// A value made ready to sign: `attributedTo` and `contentHash` filled in, +/// and the exact bytes the signature must cover. +#[derive(Debug, Clone)] +pub struct PreparedAttestation { + /// The value to sign — pass it back to [`attach_proof`] unchanged. + pub value: Value, + /// `jcs(signing_view(value))`: what the Ed25519 signature covers. + pub signing_bytes: Vec, +} + +/// First half of [`attest_value`], for signers that hold the key somewhere +/// else — a browser's WebCrypto, a hardware token, a remote signing service. +/// Everything that must agree with [`verify_value`] (the author, the content +/// hash, the canonical bytes) is computed here, so the external signer only +/// ever signs bytes, and the result is an ordinary atomic attestation. +pub fn prepare_attestation(mut value: Value, public_key: &PublicKey) -> PreparedAttestation { + let did = did::did_for_public_key(public_key); if let Some(obj) = value.as_object_mut() { // Fill attributedTo only if there is no non-empty value already. @@ -121,7 +145,7 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> .map(|s| !s.is_empty()) .unwrap_or(false); if !has_author { - obj.insert(PROP_ATTRIBUTED_TO.to_string(), Value::String(did.clone())); + obj.insert(PROP_ATTRIBUTED_TO.to_string(), Value::String(did)); } } @@ -133,13 +157,24 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> } let signing_bytes = jcs::canonicalize(&signing_view(&value)).into_bytes(); - let signature = Signer::new(keypair).sign(&signing_bytes); + PreparedAttestation { + value, + signing_bytes, + } +} + +/// Second half of [`attest_value`]: attach the `eddsa-jcs-2022` proof for a +/// signature over [`PreparedAttestation::signing_bytes`] made by +/// `public_key`'s private key. Does not check the signature — run +/// [`verify_value`] on the result for that. +pub fn attach_proof(mut value: Value, public_key: &PublicKey, signature: &Signature) -> Value { + let did = did::did_for_public_key(public_key); let proof = Proof { type_: PROOF_TYPE.to_string(), cryptosuite: CRYPTOSUITE.to_string(), verification_method: did::verification_method(&did), proof_purpose: PROOF_PURPOSE.to_string(), - proof_value: encode_proof_value(&signature), + proof_value: encode_proof_value(signature), }; if let Some(obj) = value.as_object_mut() { obj.insert( @@ -150,6 +185,18 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> value } +/// The signature a value's proof carries — what `--signed` hands to the +/// recording service after a key holder signed `--prepare`'s bytes elsewhere. +pub fn proof_signature(value: &Value) -> Result { + let proof: Proof = value + .as_object() + .and_then(|obj| obj.get(PROP_PROOF)) + .cloned() + .and_then(|p| serde_json::from_value(p).ok()) + .ok_or_else(|| CanonicalError::Proof("node carries no proof".into()))?; + decode_proof_value(&proof.proof_value) +} + /// Generic verify over a JSON-LD value — the same three checks as the typed /// path: (1) the content hash recomputes over `hashing_view`, (2) the signature /// verifies over `jcs(signing_view)`, (3) the proof's verificationMethod DID @@ -281,6 +328,25 @@ mod tests { }) } + /// An external signer — handed only the prepared bytes — produces + /// exactly the attestation `attest_value` would, and it verifies. + #[test] + fn prepare_sign_attach_matches_attest_value() { + let (id, kp) = dev_identity(); + let prepared = prepare_attestation(minimal_value(), &kp.public); + let signature = Signer::new(&kp).sign(&prepared.signing_bytes); + let external = attach_proof(prepared.value, &kp.public, &signature); + + assert_eq!(external, attest_value(minimal_value(), &id, &kp)); + verify_value(&external, &kp.public).expect("externally signed value verifies"); + + // A signature over anything else does not. + let wrong = Signer::new(&kp).sign(b"not the prepared bytes"); + let prepared = prepare_attestation(minimal_value(), &kp.public); + let bad = attach_proof(prepared.value, &kp.public, &wrong); + assert!(verify_value(&bad, &kp.public).is_err()); + } + #[test] fn attest_value_then_verify_value_roundtrips() { let (id, kp) = dev_identity(); diff --git a/atomic-cli/src/commands/intent/attest.rs b/atomic-cli/src/commands/intent/attest.rs index 1dd67db7..d80311cd 100644 --- a/atomic-cli/src/commands/intent/attest.rs +++ b/atomic-cli/src/commands/intent/attest.rs @@ -4,6 +4,7 @@ use clap::Parser; use serde_json::Value; +use atomic_canonical::proof::prepare_attestation; use atomic_canonical::{lift_and_attest, validate_intent, verify}; use atomic_core::pristine::VaultEntryType; use atomic_identity::IdentityStore; @@ -28,6 +29,20 @@ pub struct IntentAttest { /// Output the attested node as JSON-LD. #[arg(long)] pub json: bool, + + /// Don't sign: print what a signer holding the identity's key elsewhere + /// (a browser, a hardware token, a remote signing service) must sign — + /// `{"document": …, "signingBytes": ""}`. Needs only the + /// identity's public key. Complete with `--signed`. + #[arg(long, conflicts_with = "signed")] + pub prepare: bool, + + /// Record an attestation signed elsewhere: a JSON file holding the + /// attested node (the `--prepare` document with its proof attached). It + /// must be signed by `--identity`'s key and attest the intent as it is + /// now. + #[arg(long, value_name = "PATH")] + pub signed: Option, } impl Command for IntentAttest { @@ -63,7 +78,7 @@ impl Command for IntentAttest { ))); } - // Resolve identity + keypair the way `atomic identity sign` does. + // Resolve the identity the way `atomic identity sign` does. let store = IdentityStore::open_default().map_err(|e| { CliError::Internal(anyhow::anyhow!("Failed to open identity store: {}", e)) })?; @@ -83,20 +98,66 @@ impl Command for IntentAttest { .to_string(), })? }; - let keypair = store.load_keypair(&identity.id, None).map_err(|e| { - CliError::Internal(anyhow::anyhow!( - "Failed to load keypair for '{}': {}", - identity.name, - e - )) - })?; - // Attest: lift + fill attributedTo (from the identity's did:atomic when - // absent) + hash + sign. - let node = lift_and_attest(&inputs.frontmatter, &inputs.body, &identity, &keypair) - .map_err(|e| CliError::InvalidArgument { - message: format!("could not attest intent: {e}"), + // Signing elsewhere, step 1: say what to sign. The same preparation + // `lift_and_attest` does (author, content hash, canonical bytes), with + // no private key involved. + if self.prepare { + let prepared = prepare_attestation(unattested.to_value(), &identity.public_key); + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "document": prepared.value, + "signingBytes": data_encoding::BASE64.encode(&prepared.signing_bytes), + })) + .unwrap() + ); + return Ok(()); + } + + let node = if let Some(path) = &self.signed { + // Signing elsewhere, step 2: the signature must be over exactly + // what `--prepare` produces for this intent now — so a signature + // over a stale or altered intent is refused, not recorded. + let text = std::fs::read_to_string(path).map_err(CliError::Io)?; + let signed: Value = + serde_json::from_str(&text).map_err(|e| CliError::InvalidArgument { + message: format!("{} is not JSON: {e}", path.display()), + })?; + let expected = prepare_attestation(unattested.to_value(), &identity.public_key).value; + let mut unsigned = signed.clone(); + if let Some(obj) = unsigned.as_object_mut() { + obj.remove("proof"); + } + if unsigned != expected { + return Err(CliError::InvalidArgument { + message: format!( + "the signed attestation is not of intent {} as it is now (re-run --prepare)", + self.id + ), + }); + } + serde_json::from_value::(signed).map_err(|e| { + CliError::InvalidArgument { + message: format!("not an attested intent: {e}"), + } + })? + } else { + let keypair = store.load_keypair(&identity.id, None).map_err(|e| { + CliError::Internal(anyhow::anyhow!( + "Failed to load keypair for '{}': {}", + identity.name, + e + )) })?; + // Attest: lift + fill attributedTo (from the identity's + // did:atomic when absent) + hash + sign. + lift_and_attest(&inputs.frontmatter, &inputs.body, &identity, &keypair).map_err( + |e| CliError::InvalidArgument { + message: format!("could not attest intent: {e}"), + }, + )? + }; // Belt-and-suspenders: re-gate the ATTESTED node — proof + attributedTo // must now satisfy the gate. @@ -111,7 +172,7 @@ impl Command for IntentAttest { // Self-check: the proof verifies against the signing key before we // write anything to disk. - verify(&node, &keypair.public).map_err(|e| CliError::InvalidArgument { + verify(&node, &identity.public_key).map_err(|e| CliError::InvalidArgument { message: format!("attested intent failed self-verification: {e}"), })?; diff --git a/atomic-cli/src/commands/rpc.rs b/atomic-cli/src/commands/rpc.rs index f3a872d7..821f0712 100644 --- a/atomic-cli/src/commands/rpc.rs +++ b/atomic-cli/src/commands/rpc.rs @@ -803,14 +803,61 @@ pub fn intent_attest(args: &super::intent::attest::IntentAttest) -> CliResult CliRes let Some(session) = Service::open()? else { return Ok(false); }; - validate_entity(&session, Kind::Intent, &args.id_or_path) + validate_entity(&session, Kind::Intent, &args.id_or_path, args.json) } pub fn intent_verify(args: &super::intent::verify::IntentVerify) -> CliResult { @@ -953,7 +1000,7 @@ pub fn intent_show(args: &super::intent::show::IntentShow) -> CliResult { Ok(true) } -fn validate_entity(session: &Service, kind: Kind, id: &str) -> CliResult { +fn validate_entity(session: &Service, kind: Kind, id: &str, json: bool) -> CliResult { let request = pb::ValidateVaultEntityRequest { repository: Some(session.reference.clone()), kind: kind as i32, @@ -964,6 +1011,24 @@ fn validate_entity(session: &Service, kind: Kind, id: &str) -> CliResult { view: None, }; let response = session.validate_vault_entity(request)?; + if json { + // Same top-level shape the local body's `report_json` prints; the + // wire carries each violation as one message string, so an entry is + // its message. + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "conforms": response.valid, + "results": response + .issues + .iter() + .map(|issue| serde_json::json!({ "message": issue })) + .collect::>(), + })) + .unwrap() + ); + return Ok(true); + } if response.valid { println!("conforms: yes"); } else { @@ -1046,7 +1111,7 @@ pub fn memory_validate(args: &super::memory::validate::MemoryValidate) -> CliRes let Some(session) = Service::open()? else { return Ok(false); }; - validate_entity(&session, Kind::Memory, &args.id_or_path) + validate_entity(&session, Kind::Memory, &args.id_or_path, args.json) } pub fn memory_verify(args: &super::memory::verify::MemoryVerify) -> CliResult { diff --git a/atomic-cli/src/commands/token.rs b/atomic-cli/src/commands/token.rs index 8ff725cb..86124cbf 100644 --- a/atomic-cli/src/commands/token.rs +++ b/atomic-cli/src/commands/token.rs @@ -11,7 +11,8 @@ //! `base64url(header).base64url(claims).base64url(signature)`: //! //! - header: `{"alg":"EdDSA","typ":"JWT","kid":""}` -//! - claims: `{ sub, iat, exp, jti }` (`sub` mirrors the `kid` public key) +//! - claims: `{ sub, aud, iat, exp, jti }` (`sub` mirrors the `kid` public key; +//! `aud` is the server the token is for) //! - signature: `Ed25519_sign(private_key, "header.claims")` //! //! # Acting on behalf of someone (agent identities) @@ -78,6 +79,12 @@ struct Claims { exp: i64, jti: String, + /// The server this token is for (RFC 7519 `aud`): the bare server URL, + /// without a trailing slash. A server that checks it refuses a token + /// minted for somewhere else, so a token leaked from one server can't + /// be replayed at another within its lifetime. + aud: String, + /// RFC 8693 actor claim — present only when an agent is acting. #[serde(skip_serializing_if = "Option::is_none")] act: Option, @@ -160,6 +167,7 @@ fn mint_token(server: &str, identity: &Identity) -> CliResult { let now = Utc::now(); let claims = Claims { sub, + aud: audience(server), iat: now.timestamp(), exp: (now + TOKEN_TTL).timestamp(), jti: Uuid::new_v4().to_string(), @@ -196,10 +204,36 @@ fn mint_token(server: &str, identity: &Identity) -> CliResult { Ok(format!("{signing_input}.{sig_b64}")) } +/// The `aud` for a server URL: scheme and host (and port), no trailing +/// slash, lowercased — so `https://Atomic.Storage/` and +/// `https://atomic.storage` are the same audience. +pub fn audience(server: &str) -> String { + server.trim().trim_end_matches('/').to_ascii_lowercase() +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_token_names_the_server_it_is_for() { + assert_eq!( + audience("https://Atomic.Storage/"), + "https://atomic.storage" + ); + let claims = Claims { + sub: "S".into(), + aud: audience("https://atomic.storage"), + iat: 0, + exp: 1, + jti: "j".into(), + act: None, + dlg: None, + }; + let v: serde_json::Value = serde_json::to_value(&claims).unwrap(); + assert_eq!(v["aud"], "https://atomic.storage"); + } + #[test] fn header_is_eddsa_jwt_with_kid() { let header = JwtHeader { @@ -218,6 +252,7 @@ mod tests { fn a_non_delegated_token_omits_the_actor_claims() { let claims = Claims { sub: "ABCDEF".to_string(), + aud: "https://a".to_string(), iat: 0, exp: 1, jti: "j".to_string(), @@ -237,6 +272,7 @@ mod tests { let agent = "AGENTKEY"; let claims = Claims { sub: human.to_string(), + aud: "https://a".to_string(), iat: 0, exp: 1, jti: "j".to_string(), @@ -277,6 +313,7 @@ mod tests { let now = Utc::now(); let claims = Claims { sub: public_key_b32.clone(), + aud: audience("https://atomic.storage"), iat: now.timestamp(), exp: (now + TOKEN_TTL).timestamp(), jti: Uuid::new_v4().to_string(), diff --git a/atomic-cli/src/service/mod.rs b/atomic-cli/src/service/mod.rs index a4296065..94f4916f 100644 --- a/atomic-cli/src/service/mod.rs +++ b/atomic-cli/src/service/mod.rs @@ -944,6 +944,28 @@ impl Service { // --------------------------------------------------------------------------- impl Service { + pub fn prepare_attestation( + &self, + request: pb::PrepareAttestationRequest, + ) -> CliResult { + self.call(move |backend| async move { + match backend { + Backend::Local(state) => services_agent::AttestationImpl { state } + .prepare_attestation(Request::new(request)) + .await + .map(|response| response.into_inner()), + Backend::Reactor(channel) => { + let mut attestation = + pb::attestation_service_client::AttestationServiceClient::new(channel); + attestation + .prepare_attestation(request) + .await + .map(|response| response.into_inner()) + } + } + }) + } + pub fn record_attestation( &self, request: pb::RecordAttestationRequest, diff --git a/atomic-cli/tests/intent_attest_external_signer_test.rs b/atomic-cli/tests/intent_attest_external_signer_test.rs new file mode 100644 index 00000000..d6871257 --- /dev/null +++ b/atomic-cli/tests/intent_attest_external_signer_test.rs @@ -0,0 +1,185 @@ +//! `atomic intent attest --prepare` / `--signed`: attesting an intent with a +//! key that is not on this machine. The identity here is public-only — as an +//! agent identity is inside a sandbox whose key lives with a signing +//! service — so plain `attest` can't sign; `--prepare` says what to sign, +//! the key holder signs it, and `--signed` checks and records the result. + +use std::path::Path; +use std::process::{Command, Output}; + +use atomic_identity::{Identity, IdentityStore, IdentityType, IdentityUsage, KeyPair}; + +fn atomic(home: &Path, cwd: &Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(args) + .current_dir(cwd) + .env("HOME", home) + .env("ATOMIC_CONFIG_DIR", home.join(".atomic")) + .output() + .unwrap() +} + +fn ok(out: &Output) -> String { + assert!( + out.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +/// A repo with one intent, a default human identity, and a public-only +/// agent identity whose key only the test holds. +fn setup() -> (tempfile::TempDir, tempfile::TempDir, String, KeyPair) { + let home = tempfile::tempdir().unwrap(); + let repo = tempfile::tempdir().unwrap(); + ok(&atomic( + home.path(), + repo.path(), + &[ + "identity", + "new", + "ada", + "--email", + "ada@example.com", + "--set-default", + ], + )); + ok(&atomic(home.path(), repo.path(), &["init"])); + ok(&atomic( + home.path(), + repo.path(), + &["intent", "new", "Add a greeting"], + )); + + let key = KeyPair::generate(); + let agent = Identity::builder("ada+bot") + .identity_type(IdentityType::Agent) + .usage(IdentityUsage::Bot) + .public_key(key.public.clone()) + .build() + .unwrap(); + IdentityStore::open(&home.path().join(".atomic").join("identities")) + .unwrap() + .save(&agent) + .unwrap(); + + let list = ok(&atomic( + home.path(), + repo.path(), + &["intent", "list", "--json"], + )); + let intents: serde_json::Value = serde_json::from_str(&list).unwrap(); + let id = intents + .as_array() + .and_then(|a| a.first()) + .and_then(|i| i.get("id")) + .and_then(|v| v.as_str()) + .expect("one intent") + .to_string(); + (home, repo, id, key) +} + +/// What the key holder does with `--prepare`'s output. +fn sign_elsewhere(prepared: &str, key: &KeyPair) -> serde_json::Value { + let prepared: serde_json::Value = serde_json::from_str(prepared).unwrap(); + let bytes = data_encoding::BASE64 + .decode(prepared["signingBytes"].as_str().unwrap().as_bytes()) + .unwrap(); + let signature = atomic_identity::signing::Signer::new(key).sign(&bytes); + atomic_canonical::proof::attach_proof(prepared["document"].clone(), &key.public, &signature) +} + +#[test] +fn an_intent_is_attested_by_a_key_held_elsewhere() { + let (home, repo, id, key) = setup(); + let (home, repo) = (home.path(), repo.path()); + + // No key here: plain attest can't sign as the agent. + let out = atomic( + home, + repo, + &["intent", "attest", &id, "--identity", "ada+bot"], + ); + assert!(!out.status.success(), "attest without a key must fail"); + + let prepared = ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--prepare", + ], + )); + let signed = sign_elsewhere(&prepared, &key); + let file = home.join("signed.json"); + std::fs::write(&file, signed.to_string()).unwrap(); + ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--signed", + file.to_str().unwrap(), + ], + )); + + let report: serde_json::Value = serde_json::from_str(&ok(&atomic( + home, + repo, + &["intent", "validate", &id, "--json"], + ))) + .unwrap(); + assert_eq!(report["conforms"], true, "{report}"); + assert_eq!( + signed["attributedTo"], + atomic_canonical::did::did_for_public_key(&key.public) + ); +} + +#[test] +fn a_signature_by_another_key_is_refused() { + let (home, repo, id, _key) = setup(); + let (home, repo) = (home.path(), repo.path()); + let prepared = ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--prepare", + ], + )); + let signed = sign_elsewhere(&prepared, &KeyPair::generate()); + let file = home.join("signed.json"); + std::fs::write(&file, signed.to_string()).unwrap(); + let out = atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--signed", + file.to_str().unwrap(), + ], + ); + assert!( + !out.status.success(), + "a signature by the wrong key must not be recorded" + ); +} diff --git a/atomic-core/src/pristine/tables.rs b/atomic-core/src/pristine/tables.rs index 8dd71b80..2d43077a 100644 --- a/atomic-core/src/pristine/tables.rs +++ b/atomic-core/src/pristine/tables.rs @@ -850,6 +850,21 @@ pub fn decode_change_file_key(key: &[u8; 36]) -> ([u8; 32], u32) { } // Key Encoding Helpers +// +// Endianness. A key that gets range-scanned is big-endian, so its bytes sort +// in the same order as the numbers they encode — that is what makes a +// `range(lo..=hi)` over these tables mean what it says (e.g. every row for +// one inode: `encode_inode_vertex(inode, 0, 0, 0)` through +// `encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX)`). A value that is +// only ever looked up by its exact key can be little-endian; sorting never +// happens, so byte order costs nothing. That covers the serialized graph edge +// and the CRDT trunk/branch/leaf ids, which are opaque and point-looked-up +// (`crdt::tables`). +// +// The two rules meet in files that handle both, so match the writer rather +// than the neighbours: decode an id with that type's own `from_bytes` +// (`TrunkId`, `BranchId`, `LeafId`) rather than slicing bytes by hand, and the +// two sides cannot drift. /// Encode a span as 24 bytes for use as a graph key #[inline] diff --git a/atomic-repository/Cargo.toml b/atomic-repository/Cargo.toml index 47bd8d18..307062cc 100644 --- a/atomic-repository/Cargo.toml +++ b/atomic-repository/Cargo.toml @@ -16,6 +16,7 @@ atomic-identity = { workspace = true } atomic-semantic = { workspace = true } serde = { workspace = true } +data-encoding = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } postcard = { workspace = true } diff --git a/atomic-repository/src/repository/materialize.rs b/atomic-repository/src/repository/materialize.rs index 3a2916a1..e22b20fc 100644 --- a/atomic-repository/src/repository/materialize.rs +++ b/atomic-repository/src/repository/materialize.rs @@ -545,7 +545,6 @@ impl Repository { use atomic_core::output::repo::{ collect_children, FileOutputOptions, MaterializeOptions, OutputItem, }; - use atomic_core::output::RetrieveOptions; use rayon::prelude::*; use std::collections::HashSet as StdHashSet; @@ -725,82 +724,17 @@ impl Repository { .map(|item| { let file_start = std::time::Instant::now(); - // Build retrieve options with the shared change filter - let retrieve_opts = - RetrieveOptions::default().with_change_filter_arc(change_filter_arc.clone()); - - // Inline the output pipeline so we can trace each phase. - use atomic_core::output::repo::{ - output_graph_content_resolved, resolve_conflicts_semantically, - }; - use atomic_core::output::{compute_order, retrieve_graph, Writer}; - use atomic_core::pristine::InodePreloadTxn; - - // Pre-load ALL edges for this file's inode from INODE_GRAPH - // in a single range scan, then run retrieve_graph over the - // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) - // individual B-tree probes. - let preloaded = InodePreloadTxn::from_table(&txn, item.inode, &inode_graph_table) - .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; - - let t_retrieve = std::time::Instant::now(); - let retrieve_result = retrieve_graph(&preloaded, item.position, retrieve_opts) - .map_err(|e| format!("{}: retrieve: {:?}", item.path, e))?; - - if retrieve_result.graph.is_empty() { - return Ok(None); - } - - let vertices = retrieve_result.graph.len_vertices(); - let edges = retrieve_result.edges_traversed; - let retrieve_ms = t_retrieve.elapsed(); - - let t_order = std::time::Instant::now(); - let mut graph = retrieve_result.graph; - let order = compute_order(&mut graph); - let order_ms = t_order.elapsed(); - - let t_content = std::time::Instant::now(); - let resolved = resolve_conflicts_semantically(&preloaded, store, &graph, &order); - let buffer = Vec::with_capacity(graph.total_bytes()); - let mut writer = Writer::new(buffer); - let hash_fn = |node_id: NodeId| -> Option { - if node_id.is_root() { - return None; - } - preloaded.get_external(node_id).ok().flatten() - }; - output_graph_content_resolved( + let Some(content) = render_view_file( + &txn, store, - hash_fn, - &graph, - &order, - &mut writer, - &resolved, - ) - .map_err(|e| format!("{}: content: {:?}", item.path, e))?; - let content = writer.into_inner(); - let content_ms = t_content.elapsed(); - - if content.is_empty() { + &inode_graph_table, + &change_filter_arc, + &name_conflicts, + item, + trace_mat.then_some(file_start), + )? + else { return Ok(None); - } - - // Name-conflict override (rare): when ≥ 2 inodes are alive at - // this path on the view, replace the single-inode content with - // a marker-wrapped rendering of every side so the conflict is - // surfaced instead of silently collapsed (rubric A12). - let content = match name_conflicts.get(&item.path) { - Some(sides) => render_name_conflict( - &txn, - store, - &inode_graph_table, - &change_filter_arc, - &item.path, - sides, - ) - .unwrap_or(content), - None => content, }; // Detect conflict markers in the materialized bytes. This is @@ -860,24 +794,6 @@ impl Repository { std::fs::write(&abs_path, &content) .map_err(|e| format!("{}: write: {}", item.path, e))?; - if trace_mat { - let elapsed = file_start.elapsed(); - if elapsed > std::time::Duration::from_millis(50) { - eprintln!( - "[materialize] SLOW {} bytes={} vertices={} edges={} \ - retrieve={:?} order={:?} content={:?} total={:?}", - item.path, - bytes_written, - vertices, - edges, - retrieve_ms, - order_ms, - content_ms, - elapsed, - ); - } - } - Ok(Some(( item.path.clone(), bytes_written, @@ -1114,6 +1030,335 @@ impl Repository { } } +/// One entry of a view's tree, rendered in memory by +/// [`Repository::materialize_view_entries`] — enough for a client to write +/// the working tree and a baseline index (`status` against it) without the +/// repository. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ViewEntry { + pub path: String, + pub inode: u64, + pub kind: ViewEntryKind, + /// Unix permission bits as recorded. + pub mode: u16, + /// The file's bytes as a checkout of the view would write them (empty + /// for directories). + #[serde(with = "serde_bytes_vec")] + pub content: Vec, + /// `Hash::of(content)` — the baseline a client compares against. + pub hash: Hash, + /// 1-based line of the first conflict marker, if the file is conflicted. + pub conflict_marker_line: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ViewEntryKind { + File, + Directory, + Symlink, +} + +/// What a view looked like when it was rendered. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ViewSnapshot { + pub view: String, + /// The view's Merkle state, base32: a change recorded against this + /// snapshot applies only while the view is still here. + pub state: String, + pub change_count: u64, +} + +mod serde_bytes_vec { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(v: &[u8], s: S) -> Result { + s.serialize_str(&data_encoding::BASE64.encode(v)) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + data_encoding::BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } +} + +impl Repository { + /// The entries `view_name` has, from TREE as that view projects it. + /// + /// TREE is the checked-out view's projection; another view's structural + /// operations (files it added, moved or deleted) wait in the deferred + /// tree journal until a switch applies them. For another view they are + /// applied here in a write transaction that is thrown away, so the + /// listing is that view's and nothing on disk changes. + /// + /// A read-only handle cannot project, and `TREE` is the *current* view's — + /// so a read-only repository asked about another view is refused rather + /// than answered. Falling through would return the current view's tree + /// under the other view's name, and the caller writes that straight + /// somewhere. The current view needs no projection, so it reads `TREE` + /// directly and works read-only. + fn view_tree_items( + &self, + txn: &atomic_core::pristine::ReadTxn, + view_name: &str, + options: &atomic_core::output::repo::MaterializeOptions, + ) -> Result, RepositoryError> { + use atomic_core::output::repo::collect_children; + let db = |e: atomic_core::pristine::PristineError| RepositoryError::Database(e.to_string()); + if view_name != self.current_view { + let journal = self.load_deferred_tree_journal()?; + let mut projected = + self.pristine + .write_txn() + .map_err(|e| RepositoryError::InvalidOperation { + message: format!( + "listing view '{view_name}' projects its tree, which needs a writable \ + repository: {e}" + ), + })?; + self.apply_deferred_tree_ops_in_txn(&mut projected, &journal, view_name)?; + let items = collect_children(&projected, Inode::ROOT, "", options).map_err(db)?; + use atomic_core::pristine::MutTxnT; + projected.abort().map_err(db)?; + return Ok(items); + } + collect_children(txn, Inode::ROOT, "", options).map_err(db) + } + + /// Render every entry of `view` and hand each to `sink`, in path order + /// (a directory before what it contains), touching nothing: no working + /// tree, no stat cache, no conflict state. Read-only, so it runs beside + /// other readers — this is what serves a client a view's tree. + /// + /// Files are rendered with the same kernel as + /// [`Repository::materialize_parallel`], in parallel chunks, so memory + /// holds one chunk of contents at a time. + pub fn materialize_view_entries( + &self, + view_name: &str, + mut sink: impl FnMut(ViewEntry) -> Result<(), E>, + ) -> Result, RepositoryError> { + use atomic_core::output::repo::MaterializeOptions; + use atomic_core::types::Base32; + use rayon::prelude::*; + + const CHUNK: usize = 256; + + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let change_filter_arc = Arc::new(collect_visible_change_ids(&txn, &view)?); + let options = MaterializeOptions::new().with_change_filter_arc(change_filter_arc.clone()); + let mut items = self.view_tree_items(&txn, view_name, &options)?; + items.sort_by(|a, b| a.path.cmp(&b.path)); + + let visible = |item: &atomic_core::output::repo::OutputItem| { + item.position.change.is_root() || change_filter_arc.contains(&item.position.change) + }; + let files: Vec<&atomic_core::output::repo::OutputItem> = items + .iter() + .filter(|i| !i.is_directory && visible(i)) + .collect(); + let name_conflicts = collect_name_conflicts( + &txn, + &self.change_store, + &files.iter().map(|i| i.path.as_str()).collect(), + &change_filter_arc, + )?; + for id in change_filter_arc.iter().filter(|id| !id.is_root()) { + if let Ok(Some(hash)) = txn.get_external(*id) { + let _ = self.change_store.load_change(&hash); + } + } + let inode_graph_table = txn + .open_inode_graph_table() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + + // Directories that hold at least one visible file, before their files. + let file_paths: std::collections::HashSet<&str> = + files.iter().map(|i| i.path.as_str()).collect(); + let mut dirs: Vec<&atomic_core::output::repo::OutputItem> = items + .iter() + .filter(|i| i.is_directory) + .filter(|d| { + let prefix = format!("{}/", d.path); + file_paths.iter().any(|p| p.starts_with(&prefix)) + }) + .collect(); + dirs.sort_by(|a, b| a.path.cmp(&b.path)); + for d in dirs { + let entry = ViewEntry { + path: d.path.clone(), + inode: d.inode.get(), + kind: ViewEntryKind::Directory, + mode: d.metadata.permissions, + content: Vec::new(), + hash: Hash::of(&[]), + conflict_marker_line: None, + }; + if let Err(e) = sink(entry) { + return Ok(Err(e)); + } + } + + let store = &self.change_store; + for chunk in files.chunks(CHUNK) { + let rendered: Vec, String>> = chunk + .par_iter() + .map(|item| { + let content = render_view_file( + &txn, + store, + &inode_graph_table, + &change_filter_arc, + &name_conflicts, + item, + None, + )?; + Ok(content.map(|content| ViewEntry { + path: item.path.clone(), + inode: item.inode.get(), + kind: if item.metadata.is_symlink { + ViewEntryKind::Symlink + } else { + ViewEntryKind::File + }, + mode: item.metadata.permissions, + conflict_marker_line: first_conflict_marker_line(&content), + hash: Hash::of(&content), + content, + })) + }) + .collect(); + for r in rendered { + match r { + Ok(Some(entry)) => { + if let Err(e) = sink(entry) { + return Ok(Err(e)); + } + } + Ok(None) => {} + Err(e) => return Err(RepositoryError::Output(e)), + } + } + } + + Ok(Ok(ViewSnapshot { + view: view.name.clone(), + state: view.state.to_base32(), + change_count: view.change_count, + })) + } +} + +/// Render one file of a view in memory: retrieve its graph under the view's +/// change filter, order it, resolve conflicts, and write the bytes a +/// checkout would produce — with a name conflict rendered as markers. The +/// one rendering kernel behind [`Repository::materialize_parallel`] (which +/// writes the result to disk) and [`Repository::materialize_view_entries`] +/// (which hands it to a caller, touching nothing). `Ok(None)`: the file has +/// no content on this view. +fn render_view_file( + txn: &atomic_core::pristine::ReadTxn, + store: &C, + inode_graph_table: &redb::ReadOnlyMultimapTable<&'static [u8; 32], &'static [u8; 24]>, + change_filter_arc: &Arc>, + name_conflicts: &NameConflicts, + item: &atomic_core::output::repo::OutputItem, + trace_from: Option, +) -> Result>, String> { + use atomic_core::output::RetrieveOptions; + // Build retrieve options with the shared change filter + let retrieve_opts = + RetrieveOptions::default().with_change_filter_arc(change_filter_arc.clone()); + + // Inline the output pipeline so we can trace each phase. + use atomic_core::output::repo::{ + output_graph_content_resolved, resolve_conflicts_semantically, + }; + use atomic_core::output::{compute_order, retrieve_graph, Writer}; + use atomic_core::pristine::InodePreloadTxn; + + // Pre-load ALL edges for this file's inode from INODE_GRAPH + // in a single range scan, then run retrieve_graph over the + // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) + // individual B-tree probes. + let preloaded = InodePreloadTxn::from_table(txn, item.inode, inode_graph_table) + .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; + + let t_retrieve = std::time::Instant::now(); + let retrieve_result = retrieve_graph(&preloaded, item.position, retrieve_opts) + .map_err(|e| format!("{}: retrieve: {:?}", item.path, e))?; + + if retrieve_result.graph.is_empty() { + return Ok(None); + } + + let vertices = retrieve_result.graph.len_vertices(); + let edges = retrieve_result.edges_traversed; + let retrieve_ms = t_retrieve.elapsed(); + + let t_order = std::time::Instant::now(); + let mut graph = retrieve_result.graph; + let order = compute_order(&mut graph); + let order_ms = t_order.elapsed(); + + let t_content = std::time::Instant::now(); + let resolved = resolve_conflicts_semantically(&preloaded, store, &graph, &order); + let buffer = Vec::with_capacity(graph.total_bytes()); + let mut writer = Writer::new(buffer); + let hash_fn = |node_id: NodeId| -> Option { + if node_id.is_root() { + return None; + } + preloaded.get_external(node_id).ok().flatten() + }; + output_graph_content_resolved(store, hash_fn, &graph, &order, &mut writer, &resolved) + .map_err(|e| format!("{}: content: {:?}", item.path, e))?; + let content = writer.into_inner(); + let content_ms = t_content.elapsed(); + + if content.is_empty() { + return Ok(None); + } + + // Name-conflict override (rare): when ≥ 2 inodes are alive at + // this path on the view, replace the single-inode content with + // a marker-wrapped rendering of every side so the conflict is + // surfaced instead of silently collapsed (rubric A12). + let content = match name_conflicts.get(&item.path) { + Some(sides) => render_name_conflict( + txn, + store, + inode_graph_table, + change_filter_arc, + &item.path, + sides, + ) + .unwrap_or(content), + None => content, + }; + + if let Some(file_start) = trace_from { + let elapsed = file_start.elapsed(); + if elapsed > std::time::Duration::from_millis(50) { + eprintln!( + "[materialize] SLOW {} vertices={vertices} edges={edges} retrieve={retrieve_ms:?} \ + order={order_ms:?} content={content_ms:?} total={elapsed:?}", + item.path, + ); + } + } + Ok(Some(content)) +} + #[cfg(test)] mod conflict_marker_tests { use super::first_conflict_marker_line; diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index e7fceb2a..df154249 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -81,6 +81,7 @@ pub mod database; mod deferred_tree; mod filter; mod materialize; +pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; mod revise; mod sandbox; mod semantic_materialize; diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index 4976768f..39602f03 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -227,19 +227,21 @@ impl Repository { pub fn materialize_view_to(&self, view: &str, dir: &Path) -> Result { std::fs::create_dir_all(dir)?; + // The view as it renders — its own added and moved files included, + // whichever view is checked out. let mut count = 0usize; - for path in self.visible_file_paths(view)? { - let bytes = match self.get_file_content_on_view(&path, view)? { - Some(bytes) => bytes, - None => continue, - }; - let target = dir.join(&path); + self.materialize_view_entries(view, |entry| -> Result<(), std::io::Error> { + if entry.kind == super::ViewEntryKind::Directory { + return Ok(()); + } + let target = dir.join(&entry.path); if let Some(parent) = target.parent() { std::fs::create_dir_all(parent)?; } - std::fs::write(&target, &bytes)?; + std::fs::write(&target, &entry.content)?; count += 1; - } + Ok(()) + })??; Ok(count) } @@ -525,4 +527,23 @@ mod tests { "the canonical graph must not be cloned into the sandbox" ); } + + #[test] + fn a_sandbox_never_sees_its_pointer_as_untracked() { + let dir = tempdir().unwrap(); + let repo = repo_with_recorded_file(&dir.path().join("repo"), "hello.txt", b"hi\n"); + let sandbox = dir.path().join("agent-1"); + repo.provision_sandbox(&sandbox, "dev").unwrap(); + std::fs::write(sandbox.join("new.txt"), b"new\n").unwrap(); + drop(repo); + + let opened = Repository::open_existing(&sandbox).unwrap(); + let status = opened.status(Default::default()).unwrap(); + let untracked: Vec<_> = status.untracked().map(|e| e.path().to_path_buf()).collect(); + assert_eq!( + untracked, + vec![PathBuf::from("new.txt")], + "the pointer can carry a credential; `record --all` must never pick it up" + ); + } } diff --git a/atomic-repository/src/status.rs b/atomic-repository/src/status.rs index 6c470d28..0cdbf9dd 100644 --- a/atomic-repository/src/status.rs +++ b/atomic-repository/src/status.rs @@ -89,8 +89,10 @@ use crate::ignore::IgnoreRules; // Constants -/// Patterns that are always ignored (internal directories) -const ALWAYS_IGNORED: &[&str] = &[".atomic", ".git"]; +/// Patterns that are always ignored: internal directories, and a sandbox's +/// pointer and local cache — the pointer can carry a credential, and neither +/// belongs in history. +const ALWAYS_IGNORED: &[&str] = &[".atomic", ".git", ".atomic-sandbox", ".atomic-sandbox.d"]; // Error Types @@ -1255,10 +1257,15 @@ mod tests { assert!(is_always_ignored(Path::new("src/.atomic/test"))); assert!(is_always_ignored(Path::new(".git"))); assert!(is_always_ignored(Path::new(".git/objects"))); + assert!(is_always_ignored(Path::new(".atomic-sandbox"))); + assert!(is_always_ignored(Path::new( + ".atomic-sandbox.d/pristine.redb" + ))); assert!(!is_always_ignored(Path::new("src"))); assert!(!is_always_ignored(Path::new("src/main.rs"))); assert!(!is_always_ignored(Path::new("atomic"))); + assert!(!is_always_ignored(Path::new(".atomic-sandbox-notes"))); } #[test] diff --git a/atomic-repository/tests/database_lock_exclusivity_test.rs b/atomic-repository/tests/database_lock_exclusivity_test.rs new file mode 100644 index 00000000..1b24d789 --- /dev/null +++ b/atomic-repository/tests/database_lock_exclusivity_test.rs @@ -0,0 +1,96 @@ +//! Is the pristine's file lock exclusive across processes, or only within one? +//! +//! The database owner re-opens the repository on every request and retries while +//! it is busy, which works either way. Whether it may instead hold one handle for +//! its lifetime depends on the answer: a cross-process lock would stop local +//! `atomic status` / `record` from opening the repository at all while an owner +//! is running. +//! +//! One test binary, two roles. The parent holds a `Repository` open and re-runs +//! itself as the child; the child tries to open the same repository and reports +//! what happened. + +use std::process::Command; + +use atomic_repository::{Repository, RepositoryError}; + +const CHILD_ROOT: &str = "ATOMIC_DATABASE_LOCK_ROOT"; +const CHILD_REPORT: &str = "ATOMIC_DATABASE_LOCK_REPORT"; +const CHILD_TEST: &str = "a_second_process_can_open_a_repository_another_one_holds"; + +/// What the child managed to do, one line, on stdout. +fn probe(root: &std::path::Path) -> String { + // Read-only first: the gentlest thing a local command does. + let readonly = match Repository::open_readonly(root) { + Ok(_) => "readonly-ok".to_string(), + Err(RepositoryError::DatabaseBusy) => "readonly-busy".to_string(), + Err(e) => format!("readonly-other: {e}"), + }; + // Then read-write, which is what `record` needs. + let readwrite = match Repository::open_existing(root) { + Ok(_) => "readwrite-ok".to_string(), + Err(RepositoryError::DatabaseBusy) => "readwrite-busy".to_string(), + Err(e) => format!("readwrite-other: {e}"), + }; + format!("{readonly} {readwrite}") +} + +#[test] +fn a_second_process_can_open_a_repository_another_one_holds() { + let temp = tempfile::TempDir::new().unwrap(); + let root = temp.path().join("repo"); + + if let Ok(child_root) = std::env::var(CHILD_ROOT) { + // The child: report on the repository the parent named and get out. The + // report goes to a file, not stdout — libtest's capture decides whether + // a child's `println!` reaches its parent, and this must not depend on + // whether the parent was run with `--nocapture`. + let report = probe(std::path::Path::new(&child_root)); + let path = std::env::var(CHILD_REPORT).expect("CHILD_REPORT"); + std::fs::write(path, report).expect("write the child report"); + return; + } + + // Held for the whole child run: this is the thing being tested. + let held = Repository::init(&root).expect("init"); + assert_eq!(held.current_view(), "dev", "the parent's handle is open"); + + let report_path = temp.path().join("child-report.txt"); + let output = Command::new(std::env::current_exe().expect("test binary path")) + .args(["--exact", CHILD_TEST, "--test-threads=1"]) + .env(CHILD_ROOT, &root) + .env(CHILD_REPORT, &report_path) + .output() + .expect("run the child probe"); + + let report = std::fs::read_to_string(&report_path).unwrap_or_else(|e| { + panic!( + "the child reported nothing ({e})\n--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) + }); + + println!("while one process holds the repository, a second sees: {report}"); + + // The answer, asserted. Exclusive across processes, and for a read-only + // open as much as a read-write one — so the database owner cannot hold the + // pristine for its lifetime without locking local `atomic status` and + // `record` out of the repository. That is why it opens per request and + // retries, and why an idle release is required of any handle it does keep. + assert_eq!( + report, "readonly-busy readwrite-busy", + "the pristine's lock is no longer exclusive across processes; the owner \ + may be able to hold it open, and its open-per-request retry is \ + redundant" + ); + + // And the corollary, which is the one that bites: dropping the parent's + // handle must let the very next open succeed, or an idle release would + // never actually free the file. + drop(held); + match Repository::open_readonly(&root) { + Ok(_) => {} + Err(e) => panic!("releasing the handle did not free the file: {e}"), + } +} diff --git a/atomic-repository/tests/materialize_view_entries_test.rs b/atomic-repository/tests/materialize_view_entries_test.rs new file mode 100644 index 00000000..92327820 --- /dev/null +++ b/atomic-repository/tests/materialize_view_entries_test.rs @@ -0,0 +1,168 @@ +//! `Repository::materialize_view_entries`: a view's tree, rendered in memory +//! for a client — the same bytes a checkout writes, per view, and nothing +//! touched on disk. + +use std::fs; +use std::path::Path; + +use atomic_core::change::{Author, ChangeHeader}; +use atomic_core::types::Hash; +use atomic_repository::{RecordOptions, Repository, SplitOptions, ViewEntry, ViewEntryKind}; +use tempfile::TempDir; + +fn write(repo_path: &Path, name: &str, content: &str) { + let path = repo_path.join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); +} + +fn record(repo: &Repository, message: &str) -> Hash { + let header = ChangeHeader::builder() + .message(message) + .author(Author::new("Test", Some("test@example.com"))) + .build(); + *repo + .record(header, RecordOptions::default()) + .expect("record") + .hash() +} + +fn entries(repo: &Repository, view: &str) -> (Vec, atomic_repository::ViewSnapshot) { + let mut out = Vec::new(); + let snapshot = repo + .materialize_view_entries::<()>(view, |e| { + out.push(e); + Ok(()) + }) + .expect("materialize") + .expect("sink"); + (out, snapshot) +} + +#[test] +fn entries_are_the_view_s_recorded_tree() { + let temp = TempDir::new().unwrap(); + let root = temp.path().to_path_buf(); + let repo = Repository::init(&root).expect("init"); + let view = repo.current_view().to_string(); + + write(&root, "README.md", "hello\n"); + write(&root, "src/main.rs", "fn main() {}\n"); + repo.add("README.md", Default::default()).unwrap(); + repo.add("src/main.rs", Default::default()).unwrap(); + record(&repo, "first"); + write(&root, "README.md", "hello, world\n"); + let second = record(&repo, "second"); + // Present on disk, never recorded: not part of the view. + write(&root, "scratch.txt", "not recorded\n"); + + let before: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + let (list, snapshot) = entries(&repo, &view); + let after: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + assert_eq!( + before.len(), + after.len(), + "nothing written to the working tree" + ); + + let paths: Vec<&str> = list.iter().map(|e| e.path.as_str()).collect(); + assert_eq!( + paths, + vec!["src", "README.md", "src/main.rs"], + "directories first, then files in order" + ); + let readme = list.iter().find(|e| e.path == "README.md").unwrap(); + assert_eq!(readme.kind, ViewEntryKind::File); + assert_eq!(readme.content, b"hello, world\n"); + assert_eq!(readme.hash, Hash::of(b"hello, world\n")); + assert_eq!(readme.conflict_marker_line, None); + assert!(readme.inode > 0); + assert_eq!(snapshot.view, view); + assert_eq!(snapshot.change_count, 2); + assert!(!snapshot.state.is_empty()); + + // Another view without the second change renders its own content. + let mut repo = repo; + repo.split_view(SplitOptions::new("older", vec![second])) + .expect("split"); + let (older_on_source, older_snapshot) = entries(&repo, &view); + let readme = older_on_source + .iter() + .find(|e| e.path == "README.md") + .unwrap(); + assert_eq!( + readme.content, b"hello\n", + "the source view no longer has the second change" + ); + assert_ne!(older_snapshot.state, snapshot.state); +} + +/// Another view's tree is a projection, and a read-only repository cannot +/// project. It used to fall through to `TREE`, which is the *current* view's — +/// handing one view's tree back under another's name, which means writing +/// the wrong tree to disk. Refusing is the safe answer. +#[test] +fn a_read_only_repository_refuses_another_views_tree() { + let temp = TempDir::new().unwrap(); + let root = temp.path().to_path_buf(); + let mut repo = Repository::init(&root).expect("init"); + let current = repo.current_view().to_string(); + + write(&root, "README.md", "hello\n"); + repo.add("README.md", Default::default()).unwrap(); + record(&repo, "first"); + write(&root, "README.md", "hello, world\n"); + let second = record(&repo, "second"); + + repo.split_view(SplitOptions::new("older", vec![second])) + .expect("split"); + + // Writable: both views render, and they differ. `split_view` moves the + // change out of the source view, so the new view is the one with it. + let (on_current, _) = entries(&repo, ¤t); + let (on_older, _) = entries(&repo, "older"); + let body = |v: &Vec| { + v.iter() + .find(|e| e.path == "README.md") + .map(|e| e.content.clone()) + .unwrap() + }; + assert_eq!(body(&on_current), b"hello\n"); + assert_eq!(body(&on_older), b"hello, world\n"); + + // Read-only: the current view still works (it needs no projection), but + // another view is refused rather than answered with the current tree. + drop(repo); + let readonly = Repository::open_readonly(&root).expect("open read-only"); + + let mut on_current = Vec::new(); + readonly + .materialize_view_entries::<()>(¤t, |e| { + on_current.push(e); + Ok(()) + }) + .expect("materialize the current view") + .expect("sink"); + assert_eq!(body(&on_current), b"hello\n"); + + let mut on_older = Vec::new(); + let refused = readonly.materialize_view_entries::<()>("older", |e| { + on_older.push(e); + Ok(()) + }); + let err = refused.expect_err("another view must be refused"); + assert!( + format!("{err}").contains("writable"), + "unexpected error: {err}" + ); + assert!( + on_older.is_empty(), + "nothing of the wrong view was rendered" + ); +} diff --git a/atomic-wasm/.gitignore b/atomic-wasm/.gitignore new file mode 100644 index 00000000..01d0a084 --- /dev/null +++ b/atomic-wasm/.gitignore @@ -0,0 +1 @@ +pkg/ diff --git a/atomic-wasm/Cargo.toml b/atomic-wasm/Cargo.toml new file mode 100644 index 00000000..35a2f6a6 --- /dev/null +++ b/atomic-wasm/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "atomic-wasm" +description = "atomic-canonical for the browser: build and check atomic attestations where the signing key lives in WebCrypto" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +atomic-canonical = { workspace = true } +atomic-identity = { workspace = true } +serde_json = { workspace = true } +wasm-bindgen = "0.2" + +# The browser supplies randomness; nothing here draws any, but the +# dependency tree links getrandom, which refuses wasm32 without this. +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.2", features = ["js"] } diff --git a/atomic-wasm/README.md b/atomic-wasm/README.md new file mode 100644 index 00000000..9ec280c2 --- /dev/null +++ b/atomic-wasm/README.md @@ -0,0 +1,20 @@ +# atomic-wasm + +`atomic-canonical` for the browser. A page that holds an atomic identity's +Ed25519 key in WebCrypto signs atomic documents without the key ever leaving +the browser: this module builds the canonical document and the bytes to sign +(with the same code the CLI uses), WebCrypto signs them, and the result is an +ordinary `eddsa-jcs-2022` attestation that `atomic` verifies. + +```js +import init, { prepareAttestation, attachProof } from "./pkg/atomic_wasm.js"; +await init(); +const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); +const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); +const attested = JSON.parse(attachProof(prepared.document, publicKeyBytes, new Uint8Array(sig))); +``` + +Also: `verifyAttestation`, `didForPublicKey`, `publicKeyBase32`, `canonicalize`. + +Build with `./build.sh [out-dir]`; `node smoke.mjs` (Node ≥ 22, which has +WebCrypto Ed25519) signs, verifies and tamper-checks a document end to end. diff --git a/atomic-wasm/build.sh b/atomic-wasm/build.sh new file mode 100755 index 00000000..2cbb34b8 --- /dev/null +++ b/atomic-wasm/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Build the browser module: pkg/atomic_wasm{.js,_bg.wasm}. +# Needs the wasm32 target and a wasm-bindgen CLI matching the locked +# wasm-bindgen crate: +# rustup target add wasm32-unknown-unknown +# cargo install wasm-bindgen-cli --version +set -eu +here=$(cd "$(dirname "$0")" && pwd) +cargo build -p atomic-wasm --target wasm32-unknown-unknown --release +wasm-bindgen --target web --out-dir "${1:-$here/pkg}" \ + "$here/../target/wasm32-unknown-unknown/release/atomic_wasm.wasm" diff --git a/atomic-wasm/smoke.mjs b/atomic-wasm/smoke.mjs new file mode 100644 index 00000000..6fba36a7 --- /dev/null +++ b/atomic-wasm/smoke.mjs @@ -0,0 +1,13 @@ +import { readFileSync } from "node:fs"; +import init, { prepareAttestation, attachProof, verifyAttestation, didForPublicKey } from "./pkg/atomic_wasm.js"; +await init({ module_or_path: readFileSync(new URL("./pkg/atomic_wasm_bg.wasm", import.meta.url)) }); +const kp = await crypto.subtle.generateKey("Ed25519", false, ["sign", "verify"]); +const pub = new Uint8Array(await crypto.subtle.exportKey("raw", kp.publicKey)); +const doc = { "@type": "ExampleLogin", nonce: "abc", "é": [1, 2.5, "x"] }; +const p = prepareAttestation(JSON.stringify(doc), pub); +const sig = new Uint8Array(await crypto.subtle.sign("Ed25519", kp.privateKey, p.signingBytes)); +const attested = attachProof(p.document, pub, sig); +verifyAttestation(attested, pub); +console.log(didForPublicKey(pub)); +console.log(attested); +try { verifyAttestation(attested.replace('"abc"', '"abd"'), pub); console.log("TAMPER NOT DETECTED"); } catch (e) { console.log("tamper rejected:", e.message); } diff --git a/atomic-wasm/src/lib.rs b/atomic-wasm/src/lib.rs new file mode 100644 index 00000000..e4c2d55d --- /dev/null +++ b/atomic-wasm/src/lib.rs @@ -0,0 +1,107 @@ +//! atomic-canonical for the browser. +//! +//! A web page that holds an atomic identity's key in WebCrypto (as a +//! non-extractable Ed25519 key) signs atomic documents without the key ever +//! reaching Rust — or leaving the browser. This crate does everything *but* +//! the signing, with the same code the CLI uses, so the result is an ordinary +//! atomic attestation (`eddsa-jcs-2022`) that `atomic` verifies: +//! +//! ```js +//! const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); +//! const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); +//! const attested = attachProof(prepared.document, publicKeyBytes, new Uint8Array(sig)); +//! ``` +//! +//! Every function takes and returns JSON as strings and keys/signatures as +//! bytes, so there is no JS object model to keep in step with the Rust types. + +use atomic_canonical::{did, jcs, proof}; +use atomic_identity::keypair::PublicKey; +use atomic_identity::signing::Signature; +use wasm_bindgen::prelude::*; + +fn public_key(bytes: &[u8]) -> Result { + let bytes: &[u8; 32] = bytes + .try_into() + .map_err(|_| JsError::new("an Ed25519 public key is 32 bytes"))?; + PublicKey::from_bytes(bytes).map_err(|e| JsError::new(&e.to_string())) +} + +fn parse(json: &str) -> Result { + serde_json::from_str(json).map_err(|e| JsError::new(&format!("not JSON: {e}"))) +} + +/// A document ready to sign: the document to hand back to [`attach_proof`], +/// and the bytes the signature must cover. +#[wasm_bindgen] +pub struct Prepared { + document: String, + signing_bytes: Vec, +} + +#[wasm_bindgen] +impl Prepared { + /// The document with `attributedTo` and `contentHash` filled in (JSON). + #[wasm_bindgen(getter)] + pub fn document(&self) -> String { + self.document.clone() + } + + /// What to sign with the identity's Ed25519 key. + #[wasm_bindgen(getter, js_name = signingBytes)] + pub fn signing_bytes(&self) -> Vec { + self.signing_bytes.clone() + } +} + +/// Fill in the author and content hash of `document` (JSON) for the key +/// `public_key`, and return it with the bytes to sign. +#[wasm_bindgen(js_name = prepareAttestation)] +pub fn prepare_attestation(document: &str, public_key_bytes: &[u8]) -> Result { + let pk = public_key(public_key_bytes)?; + let prepared = proof::prepare_attestation(parse(document)?, &pk); + Ok(Prepared { + document: prepared.value.to_string(), + signing_bytes: prepared.signing_bytes, + }) +} + +/// Attach the proof for `signature` (64 bytes, over the prepared signing +/// bytes) and check it verifies. Returns the attested document (JSON). +#[wasm_bindgen(js_name = attachProof)] +pub fn attach_proof( + document: &str, + public_key_bytes: &[u8], + signature: &[u8], +) -> Result { + let pk = public_key(public_key_bytes)?; + let sig = Signature::from_slice(signature).map_err(|e| JsError::new(&e.to_string()))?; + let attested = proof::attach_proof(parse(document)?, &pk, &sig); + proof::verify_value(&attested, &pk).map_err(|e| JsError::new(&e.to_string()))?; + Ok(attested.to_string()) +} + +/// Check an attested document (JSON) against a public key. +#[wasm_bindgen(js_name = verifyAttestation)] +pub fn verify_attestation(document: &str, public_key_bytes: &[u8]) -> Result<(), JsError> { + let pk = public_key(public_key_bytes)?; + proof::verify_value(&parse(document)?, &pk).map_err(|e| JsError::new(&e.to_string())) +} + +/// The `did:atomic` identifier for a public key. +#[wasm_bindgen(js_name = didForPublicKey)] +pub fn did_for_public_key(public_key_bytes: &[u8]) -> Result { + Ok(did::did_for_public_key(&public_key(public_key_bytes)?)) +} + +/// The key's base32 form — what atomic writes as the `kid` of its tokens. +#[wasm_bindgen(js_name = publicKeyBase32)] +pub fn public_key_base32(public_key_bytes: &[u8]) -> Result { + Ok(public_key(public_key_bytes)?.to_base32()) +} + +/// RFC 8785 canonical JSON of `document` — the bytes atomic hashes and signs. +#[wasm_bindgen] +pub fn canonicalize(document: &str) -> Result { + Ok(jcs::canonicalize(&parse(document)?)) +} diff --git a/libatomic/src/daemon/services_agent.rs b/libatomic/src/daemon/services_agent.rs index ffb35d3a..5e8fbef8 100644 --- a/libatomic/src/daemon/services_agent.rs +++ b/libatomic/src/daemon/services_agent.rs @@ -16,12 +16,14 @@ use atomic_canonical::did::did_for_public_key; use atomic_canonical::gate::{validate_intent, validate_memory}; use atomic_canonical::lift::lift_intent; use atomic_canonical::memory::lift_memory; +use atomic_canonical::proof; use atomic_canonical::{ lift_and_attest, lift_and_attest_memory, verify, verify_memory, CanonicalNode, MemoryNode, }; use atomic_core::change::attestation::Attestation; use atomic_core::pristine::VaultEntryType; use atomic_core::types::Base32; +use atomic_identity::signing::Signature; use atomic_identity::{Identity, IdentityStore}; use atomic_repository::IntentCreateOptions; use atomic_repository::IntentUpdateOptions; @@ -2531,6 +2533,45 @@ fn load_identity(name: Option<&str>) -> Result { Ok(IdentityMaterial { identity, keypair }) } +/// Load an identity for VERIFICATION only — the public key, never a secret +/// key. The external-signing flow (`--prepare`/`--signed`) must work for an +/// identity whose key is held elsewhere, so it may not touch the key store. +/// The name is required: an unbound caller cannot borrow the daemon's +/// default human key by signing as nobody (CONTRACT "Sandbox and vault +/// writes": unbound grants cannot use daemon default human keys). +fn load_identity_public(name: &str) -> Result { + let store = IdentityStore::open_default() + .map_err(|error| domain_status(ErrorCode::Internal, format!("identity store: {error}")))?; + if name.is_empty() { + return Err(domain_status( + ErrorCode::InvalidArgument, + "an externally-signed attestation names the identity it is signed by", + )); + } + store + .load_by_name(name) + .map_err(|error| domain_status(ErrorCode::NotFound, format!("identity '{name}': {error}"))) +} + +/// The pre-attest gate the CLI's `attest` body applies: refuse violations +/// signing cannot fill (everything except `proof` and `attributedTo`), so a +/// signer never signs a node recording would refuse anyway. +fn refuse_unfillable(node: &CanonicalNode) -> Result<(), Status> { + let report = validate_intent(node); + let blocking: Vec<_> = report + .results + .iter() + .filter(|v| !matches!(v.path.as_deref(), Some("proof") | Some("attributedTo"))) + .collect(); + if !blocking.is_empty() { + return Err(domain_status( + ErrorCode::PreconditionFailed, + format!("the intent does not conform: {report}"), + )); + } + Ok(()) +} + /// blake3 source hash of (frontmatter + body) — the attestation freshness /// anchor, mirroring the CLI bridges. fn source_content_hash(frontmatter: &Map, body: &str) -> String { @@ -2555,11 +2596,61 @@ fn write_all(path: &std::path::Path, bytes: &[u8]) -> Result<(), Status> { impl attestation_service_server::AttestationService for AttestationImpl { async fn prepare_attestation( &self, - _request: Request, + request: Request, ) -> Result, Status> { - Err(Status::unimplemented( - "PrepareAttestation (external signing) lands with its slice", - )) + let request = request.into_inner(); + let target = request.target.clone().ok_or_else(|| { + domain_status(ErrorCode::InvalidArgument, "attestation target required") + })?; + let target_kind = target.kind.ok_or_else(|| { + domain_status( + ErrorCode::InvalidArgument, + "attestation target kind required", + ) + })?; + let handle = + self.state + .resolve(request.repository.as_ref().ok_or_else(|| { + domain_status(ErrorCode::InvalidArgument, "repository required") + })?)?; + self.state.log_rpc("PrepareAttestation", Some(&handle)); + let gate_handle = handle.clone(); + let _gate = gate_handle.exclusive().await; + let identity_name = request.identity_did.clone(); + + let response = tokio::task::spawn_blocking(move || match target_kind { + TargetKind::IntentId(intent_id) => { + let repo = handle.repository()?; + let entry = repo + .vault_intent_show(&intent_id) + .map_err(repository_error)?; + let frontmatter = parse_frontmatter(&entry)?; + let body = body_of(&entry); + // No key here: everything that must agree with verification — + // the author, the content hash, the canonical bytes — is + // computed now, so the external signer only ever signs bytes. + let identity = load_identity_public(&identity_name)?; + let node = lift_intent(&frontmatter, &body).map_err(|error| { + domain_status(ErrorCode::InvalidArgument, format!("attest: {error}")) + })?; + refuse_unfillable(&node)?; + let prepared = proof::prepare_attestation(node.to_value(), &identity.public_key); + let document = serde_json::to_string(&prepared.value) + .map_err(|e| Status::internal(e.to_string()))?; + Ok::<_, Status>(PrepareAttestationResponse { + document, + signing_bytes: prepared.signing_bytes, + snapshot: None, + }) + } + other => Err(domain_status( + ErrorCode::InvalidArgument, + format!("attestation target {other:?} lands with its slice"), + )), + }) + .await + .map_err(|e| Status::internal(e.to_string()))??; + Ok(Response::new(response)) } async fn record_attestation( @@ -2582,6 +2673,7 @@ impl attestation_service_server::AttestationService for AttestationImpl { let _gate = gate_handle.exclusive().await; let identity_name = request.identity_did.clone(); let meta = request.meta.clone(); + let caller_signature = request.signature.clone(); let response = tokio::task::spawn_blocking(move || match target_kind { TargetKind::IntentId(intent_id) => { @@ -2591,16 +2683,62 @@ impl attestation_service_server::AttestationService for AttestationImpl { .map_err(repository_error)?; let frontmatter = parse_frontmatter(&entry)?; let body = body_of(&entry); - let material = load_identity(if identity_name.is_empty() { - None + let (node, public_key): (CanonicalNode, _) = if caller_signature.is_empty() { + // Sign here, with this machine's key — the plain `attest`. + let material = load_identity(if identity_name.is_empty() { + None + } else { + Some(&identity_name) + })?; + ( + lift_and_attest(&frontmatter, &body, &material.identity, &material.keypair) + .map_err(|error| { + domain_status( + ErrorCode::InvalidArgument, + format!("attest: {error}"), + ) + })?, + material.identity.public_key.clone(), + ) } else { - Some(&identity_name) - })?; - let node: CanonicalNode = - lift_and_attest(&frontmatter, &body, &material.identity, &material.keypair) - .map_err(|error| { - domain_status(ErrorCode::InvalidArgument, format!("attest: {error}")) + // The key was held elsewhere: verify the caller's + // signature against the target AS IT IS NOW (a stale or + // altered intent hashes differently, so an old signature + // cannot land), and attach exactly the proof it earned. + // No secret key is loaded. + let identity = load_identity_public(&identity_name)?; + let unattested = lift_intent(&frontmatter, &body).map_err(|error| { + domain_status(ErrorCode::InvalidArgument, format!("attest: {error}")) + })?; + refuse_unfillable(&unattested)?; + let prepared = + proof::prepare_attestation(unattested.to_value(), &identity.public_key); + let signature = Signature::from_slice(&caller_signature).map_err(|error| { + domain_status(ErrorCode::InvalidArgument, format!("signature: {error}")) + })?; + signature + .verify(&prepared.signing_bytes, &identity.public_key) + .map_err(|_| { + domain_status( + ErrorCode::PreconditionFailed, + format!( + "the signature does not verify against intent {intent_id} \ + as it is now (re-run --prepare)", + ), + ) })?; + ( + serde_json::from_value(proof::attach_proof( + prepared.value, + &identity.public_key, + &signature, + )) + .map_err(|e| { + domain_status(ErrorCode::Internal, format!("attested intent: {e}")) + })?, + identity.public_key.clone(), + ) + }; let report = validate_intent(&node); if !report.conforms { return Err(domain_status( @@ -2608,7 +2746,7 @@ impl attestation_service_server::AttestationService for AttestationImpl { format!("attested intent does not conform: {}", report), )); } - verify(&node, &material.keypair.public).map_err(|error| { + verify(&node, &public_key).map_err(|error| { domain_status(ErrorCode::Internal, format!("self-check failed: {error}")) })?; let normalized = normalized_id(&repo, &intent_id);