diff --git a/AGENTS.md b/AGENTS.md index 473a00de..516162c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -340,6 +340,37 @@ Human: "I can review that!" | `EdgeFlags` | 1 byte | Bitflags: BLOCK, PSEUDO, FOLDER, PARENT, DELETED | | `SerializedGraphEdge` | 24 bytes | Compact edge: (flags+pos, change, introduced_by) | +#### Endianness: keys that sort are big-endian, opaque values are little-endian + +The storage layer encodes integers two ways, and the rule is whether the +bytes ever get sorted: + +| Encoded as | Endian | Which | +|------------|--------|-------| +| **Big-endian** | BE | B-tree **keys** that are range-scanned: `encode_vertex`, `encode_inode_vertex`, `encode_position`, `encode_view_seq` (`pristine/tables.rs`) | +| **Little-endian** | LE | Values and ids only ever looked up by exact key: `SerializedGraphEdge`, and the CRDT `TrunkId`/`BranchId`/`LeafId` (`crdt/tables.rs`, `crdt/ids.rs`) | + +BE is load-bearing for the keys: byte order has to match numeric order or a +range scan means nothing. `export_graph_slice` depends on it — + +```rust +let lo = encode_inode_vertex(inode, 0, 0, 0); +let hi = encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX); +for row in inode_graph.range::<&[u8; 32]>(&lo..=&hi)? { ... } // every row for one inode +``` + +For the LE side, sorting never happens, so byte order costs nothing and there +is no reason to pay for BE. + +**The two meet in the same files**, so match the *writer*, never the +neighbours. `export_graph_slice` decodes graph vertex keys (BE) and CRDT ids +(LE) within a few lines of each other, which is exactly how a BE decode of a +LE id shipped unnoticed. Decode an id with its own type's `from_bytes` +(`TrunkId::from_bytes`, `BranchId::from_bytes`, `LeafId::from_bytes`) instead +of slicing `id[0..8]` by hand, so the two sides cannot drift. `id_rows` +`debug_assert`s on an id with no `EXTERNAL` row, because a mis-decoded id is +indistinguishable from an absent one at runtime. + ### Hash Type Design Following the original Atomic project, `Hash` is a **type alias** for `Merkle`: diff --git a/Cargo.lock b/Cargo.lock index be8ff24c..3558ef13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -127,6 +162,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async_io_stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" +dependencies = [ + "futures 0.3.32", + "pharos", + "rustc_version", +] + [[package]] name = "atomic-agent" version = "0.18.3" @@ -148,7 +194,7 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 1.0.69", "tokio", @@ -201,10 +247,12 @@ dependencies = [ "fs2", "git2", "indicatif", + "iroh", "log", "open", + "rand 0.8.7", "rayon", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serial_test", @@ -239,7 +287,7 @@ dependencies = [ "blake3", "chrono", "data-encoding", - "ed25519-dalek", + "ed25519-dalek 2.2.0", "fastcdc", "log", "parking_lot", @@ -270,7 +318,7 @@ dependencies = [ "chrono", "data-encoding", "dirs", - "ed25519-dalek", + "ed25519-dalek 2.2.0", "quickcheck", "quickcheck_macros", "rand 0.8.7", @@ -314,7 +362,7 @@ dependencies = [ "chrono", "env_logger", "log", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "tempfile", @@ -339,7 +387,7 @@ dependencies = [ "chrono", "data-encoding", "dirs", - "ed25519-dalek", + "ed25519-dalek 2.2.0", "flate2", "fs2", "ignore", @@ -349,10 +397,10 @@ dependencies = [ "rayon", "redb 4.2.0", "reflink-copy", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "syntext", "tar", "tempfile", @@ -401,18 +449,64 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomic-wasm" +version = "0.18.3" +dependencies = [ + "atomic-canonical", + "atomic-identity", + "getrandom 0.2.17", + "serde_json", + "wasm-bindgen", +] + +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64 0.22.1", + "http", + "log", + "url", +] + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -457,6 +551,24 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bs58" version = "0.5.1" @@ -515,6 +627,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" @@ -523,9 +641,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" @@ -552,6 +670,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -596,7 +724,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -605,6 +733,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cobs" version = "0.3.0" @@ -620,6 +754,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compression-codecs" version = "0.4.38" @@ -656,12 +800,37 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "constant_time_eq" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cordyceps" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9ab7e0ca1d179628fa0172b2b97203c7fa0cd81be2448bd446fb9559ca9261" +dependencies = [ + "loom", + "tracing", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -756,6 +925,34 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -765,9 +962,27 @@ dependencies = [ "cfg-if", "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", - "fiat-crypto", + "digest 0.10.7", + "fiat-crypto 0.2.9", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto 0.3.0", + "rand_core 0.10.1", "rustc_version", + "serde", "subtle", "zeroize", ] @@ -780,14 +995,34 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "data-encoding-macro" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6a127ecbb3c4632e1525380e04c0c3fcf8dcb44d32a79ea290d8a36906edcd8" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "c54e03a951783e8b327515db3f2a2fd0e3bed362a96b066f341ce66ed49b4ead" +dependencies = [ + "data-encoding", + "syn 2.0.118", +] [[package]] name = "defmt" @@ -808,7 +1043,7 @@ dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -826,10 +1061,50 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "pem-rfc7468", "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.118", + "unicode-xid", +] + [[package]] name = "dialoguer" version = "0.11.0" @@ -843,14 +1118,30 @@ dependencies = [ "zeroize", ] +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", ] [[package]] @@ -874,6 +1165,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -882,7 +1185,18 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "dlopen2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2c5bd4158e66d1e215c49b837e11d62f3267b30c92f1d171c4d3105e3dc4d4" +dependencies = [ + "libc", + "once_cell", + "winapi", ] [[package]] @@ -891,9 +1205,20 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "serde", - "signature", + "signature 2.2.0", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8 0.11.0", + "serdect", + "signature 3.0.0", ] [[package]] @@ -902,10 +1227,26 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ - "curve25519-dalek", - "ed25519", + "curve25519-dalek 4.1.3", + "ed25519 2.2.3", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ebaa1a2bf1290ab3bfe5a7b771d050ebffab2711c19a81691c683a5144a25de" +dependencies = [ + "curve25519-dalek 5.0.0", + "ed25519 3.0.0", + "rand_core 0.10.1", "serde", - "sha2", + "sha2 0.11.0", + "signature 3.0.0", "subtle", "zeroize", ] @@ -943,6 +1284,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-assoc" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0590c4a94da3372e83493b956755a6e2266830b6e4e3b101afe66e3f39477b91" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "env_filter" version = "2.0.0" @@ -1012,6 +1364,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "filetime" version = "0.2.29" @@ -1044,6 +1402,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1099,6 +1463,19 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.1", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -1133,16 +1510,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] -name = "futures-macro" -version = "0.3.32" +name = "futures-lite" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" dependencies = [ - "proc-macro2", - "quote", - "syn", -] - + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -1174,6 +1564,21 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ade96dc9003043bce7c035c85a9df5a858bfb2039c5a2e6fdf00f324f6c551" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1223,6 +1628,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "git2" version = "0.19.0" @@ -1251,6 +1666,18 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.15" @@ -1293,6 +1720,9 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] [[package]] name = "hashlink" @@ -1313,7 +1743,7 @@ dependencies = [ "hash32", "rustc_version", "serde", - "spin", + "spin 0.9.8", "stable_deref_trait", ] @@ -1362,6 +1792,21 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -1376,6 +1821,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -1421,7 +1867,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -1433,7 +1879,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", - "system-configuration", + "system-configuration 0.7.0", "tokio", "tower-service", "tracing", @@ -1546,6 +1992,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "identity-hash" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfdd7caa900436d8f13b2346fe10257e0c05c1f1f9e351f4f5d57c03bd5f45da" + [[package]] name = "idna" version = "1.1.0" @@ -1567,6 +2019,26 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "igd-next" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7238d487a9aff61f81b5ab41c0a841532a115a398b5fa92a2fadd0885e2581" +dependencies = [ + "attohttpc", + "bytes", + "futures 0.3.32", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "rand 0.10.2", + "tokio", + "url", + "xmltree", +] + [[package]] name = "ignore" version = "0.4.28" @@ -1606,12 +2078,197 @@ dependencies = [ "web-time", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "iroh" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f8d1cfffc83efe39a1031aab423ce09cb8048071baba550508931e9a81ce46" +dependencies = [ + "backon", + "blake3", + "bytes", + "cfg_aliases", + "ctutils", + "data-encoding", + "derive_more", + "ed25519-dalek 3.0.0", + "futures-util", + "getrandom 0.4.3", + "http", + "ipnet", + "iroh-base", + "iroh-dns", + "iroh-metrics", + "iroh-relay", + "n0-error", + "n0-future", + "n0-watcher", + "netwatch", + "noq", + "noq-proto", + "noq-udp", + "papaya", + "pin-project", + "portable-atomic", + "portmapper", + "rand 0.10.2", + "reqwest 0.13.5", + "rustc-hash", + "rustls", + "rustls-pki-types", + "serde", + "smallvec", + "strum", + "time", + "tokio", + "tokio-stream", + "tokio-util 0.7.18", + "tracing", + "url", + "wasm-bindgen-futures", +] + +[[package]] +name = "iroh-base" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffd1efd1aaecd68d4d1b0727a30c5811f43fb822843e48f65f6e5db3b7256cc9" +dependencies = [ + "curve25519-dalek 5.0.0", + "data-encoding", + "data-encoding-macro", + "derive_more", + "ed25519-dalek 3.0.0", + "getrandom 0.4.3", + "n0-error", + "rand 0.10.2", + "serde", + "url", + "zeroize", +] + +[[package]] +name = "iroh-dns" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80276761b56e904e26ea71d0863beef0bb8d5ab6b639cbd1338e4b615209e3a1" +dependencies = [ + "arc-swap", + "cfg_aliases", + "derive_more", + "iroh-base", + "n0-dns-resolver", + "n0-error", + "n0-future", + "portable-atomic", + "rand 0.10.2", + "rustls", + "simple-dns", + "strum", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "iroh-metrics" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291065721ad7c477b972e581bbc528df031dc8eb5e39fe1ff3300ae5dfb157ef" +dependencies = [ + "iroh-metrics-derive", + "itoa", + "n0-error", + "portable-atomic", + "ryu", + "serde", + "tracing", +] + +[[package]] +name = "iroh-metrics-derive" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae5f0c4405d1fbc9fb16ff422ca40620e93dc36c30ecaba0c2aee3992b7bd48" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "iroh-relay" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beb2294a9749d6a25fd7cd8bcf0fccd932f20716d4f967d85d3f23c7135ae6a2" +dependencies = [ + "blake3", + "bytes", + "cfg_aliases", + "data-encoding", + "derive_more", + "getrandom 0.4.3", + "http", + "http-body-util", + "hyper", + "hyper-util", + "iroh-base", + "iroh-dns", + "iroh-metrics", + "lru", + "n0-error", + "n0-future", + "noq", + "noq-proto", + "num_enum", + "pin-project", + "postcard", + "rand 0.10.2", + "reqwest 0.13.5", + "rustls", + "rustls-pki-types", + "serde", + "serde_bytes", + "strum", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", + "tokio-websockets", + "tracing", + "url", + "webpki-roots", + "ws_stream_wasm", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -1674,7 +2331,81 @@ checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.118", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", ] [[package]] @@ -1698,6 +2429,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.186" @@ -1791,6 +2528,28 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1798,11 +2557,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] -name = "maplit" -version = "1.0.2" +name = "mac-addr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3d25b0e0b648a86960ac23b7ad4abb9717601dec6f66c165f5b037f3f03065f" + +[[package]] +name = "maplit" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1845,6 +2619,84 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "n0-dns-resolver" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535c99c9a786bee714155d2ee4fc2477405c2c43536e1bc6a51de09a0d1448f7" +dependencies = [ + "derive_more", + "ipconfig", + "jni 0.22.4", + "lru", + "n0-error", + "n0-future", + "ndk-context", + "rand 0.10.2", + "reqwest 0.13.5", + "rustc-hash", + "rustls", + "simple-dns", + "system-configuration 0.8.0", + "tokio", + "tokio-rustls", + "tracing", + "webpki-roots", +] + +[[package]] +name = "n0-error" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c59ea174675ce6bc196878851e5049e11b8b1f9af3ba87e4d6df8d828bb001" +dependencies = [ + "n0-error-macros", + "spez", +] + +[[package]] +name = "n0-error-macros" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4412346410d6616f3668c40e53c298e5320c7b856f7a960e5914e442601be79f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "n0-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ab99dfb861450e68853d34ae665243a88b8c493d01ba957321a1e9b2312bbe" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util 0.7.18", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "n0-watcher" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc618745ad0b7414b149d0517ad8b5573b2fb4d4e2717add3d2446ce1fdd826" +dependencies = [ + "derive_more", + "n0-error", + "n0-future", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -1862,6 +2714,257 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "netdev" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096c44b66a9b09b99b4dd36b1c295ff3a492039ccecaf55466bd6ddcdba59968" +dependencies = [ + "block2", + "dispatch2", + "dlopen2", + "ipnet", + "jni 0.21.1", + "libc", + "mac-addr", + "ndk-context", + "netlink-packet-core 0.8.2", + "netlink-packet-route 0.31.0", + "netlink-sys 0.8.8", + "objc2", + "objc2-core-foundation", + "objc2-core-wlan", + "objc2-foundation", + "objc2-system-configuration", + "once_cell", + "plist", + "windows-sys 0.61.2", +] + +[[package]] +name = "netdev" +version = "0.46.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c52c2584961c68f5a6e3356797344061f818c93b8acc9cd6d7e284795e563e" +dependencies = [ + "block2", + "dispatch2", + "dlopen2", + "ipnet", + "jni 0.21.1", + "libc", + "mac-addr", + "ndk-context", + "netlink-packet-core 0.9.0", + "netlink-packet-route 0.33.0", + "netlink-sys 0.9.0", + "objc2-core-foundation", + "objc2-system-configuration", + "once_cell", + "plist", + "windows-sys 0.61.2", +] + +[[package]] +name = "netlink-packet-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b897d7bd4f0af82e68d40d0344cf37e97f9c97ddf74a098de3e4da05e96ca395" +dependencies = [ + "paste", +] + +[[package]] +name = "netlink-packet-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6e2daf9a8c2e21302714706860a0e6be02e03d17294ecc66b354ea7d4059dd" + +[[package]] +name = "netlink-packet-route" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2288fcb784eb3defd5fb16f4c4160d5f477de192eac730f43e1d11c24d9a007" +dependencies = [ + "bitflags 2.13.0", + "libc", + "log", + "netlink-packet-core 0.8.2", +] + +[[package]] +name = "netlink-packet-route" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59d48ffc8b73a506e1ff0878528c72668f2bfbc3d875b6be890a96be5d0d5576" +dependencies = [ + "bitflags 2.13.0", + "libc", + "log", + "netlink-packet-core 0.9.0", + "zerocopy", +] + +[[package]] +name = "netlink-proto" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93af8261786086024cd5e96e0a991dd65ced07bbf7c233a487bbc96b971d5539" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "log", + "netlink-packet-core 0.8.2", + "netlink-sys 0.8.8", + "thiserror 2.0.18", +] + +[[package]] +name = "netlink-sys" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6c30ed10fa69cc491d491b85cc971f6bdeb8e7367b7cde2ee6cc878d583fae" +dependencies = [ + "bytes", + "futures-util", + "libc", + "log", + "tokio", +] + +[[package]] +name = "netlink-sys" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c99d38e00b420df49e940fe0826e667c3dad82ac68eb4c2bbf754c1c14a83a10" +dependencies = [ + "bytes", + "libc", + "log", +] + +[[package]] +name = "netwatch" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39da9cad5f23aa43401f09497d3b280e51b5feba115d9ffbf38ca35d6ff95e96" +dependencies = [ + "atomic-waker", + "bytes", + "cfg_aliases", + "derive_more", + "ipnet", + "js-sys", + "libc", + "n0-error", + "n0-future", + "n0-watcher", + "netdev 0.45.1", + "netdev 0.46.3", + "netlink-packet-core 0.8.2", + "netlink-packet-route 0.31.0", + "netlink-proto", + "netlink-sys 0.8.8", + "noq-udp", + "objc2-core-foundation", + "objc2-system-configuration", + "pin-project-lite", + "serde", + "socket2", + "time", + "tokio", + "tokio-util 0.7.18", + "tracing", + "web-sys", + "windows", + "windows-result", + "wmi", +] + +[[package]] +name = "noq" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b78be567e796cfa74bb9bdc4117790af2d505eb887019cf8803244353eb09d89" +dependencies = [ + "bytes", + "cfg_aliases", + "derive_more", + "noq-proto", + "noq-udp", + "pin-project-lite", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "web-time", +] + +[[package]] +name = "noq-proto" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c1e5b6fe668491eca022f745a0a9402585626c73a7b839b3424ace15d6a9c8f" +dependencies = [ + "aes-gcm", + "bytes", + "derive_more", + "enum-assoc", + "getrandom 0.4.3", + "identity-hash", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "sorted-index-buffer", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "noq-udp" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc50afea61aff926e150a36c31f06094608848e114a3fe667d76ec233163b1c" +dependencies = [ + "cfg_aliases", + "libc", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-traits" version = "0.2.19" @@ -1871,12 +2974,124 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "number_prefix" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.0", + "block2", + "dispatch2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-core-wlan" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e34919aba0d701380d911702455038a8a3587467fe0141d6a71501e7ffe48" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-security", + "objc2-security-foundation", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-security" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe137109bd1e8b5a99390f77a7d8b2961dafc1a1c5db8f2e60329ad6d895a" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-security-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef76382e9cedd18123099f17638715cc3d81dba3637d4c0d39ab69df2ef345a5" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "bitflags 2.13.0", + "dispatch2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-security", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1889,6 +3104,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "open" version = "5.4.0" @@ -1921,7 +3142,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1962,7 +3183,23 @@ dependencies = [ name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "papaya" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da2442474a9404698c42509b8967f437249dbc7b50493e83020333d3943ec0ae" +dependencies = [ + "equivalent", + "seize", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -1987,12 +3224,57 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures 0.3.32", + "rustc_version", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2005,8 +3287,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.0", ] [[package]] @@ -2015,11 +3307,39 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plist" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896bade328c13f7042a297ea5ac5b0951f6cf989dea5f32c2fd98da398195cb" +dependencies = [ + "base64 0.23.1", + "indexmap", + "quick-xml", + "serde", + "time", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +dependencies = [ + "serde", +] [[package]] name = "portable-atomic-util" @@ -2030,6 +3350,35 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "portmapper" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca97242f016e090a25330613bc2382aab65eb22f9149dbe84d8f8e711d49a530" +dependencies = [ + "base64 0.22.1", + "bytes", + "derive_more", + "hyper-util", + "igd-next", + "iroh-metrics", + "libc", + "n0-error", + "n0-future", + "netwatch", + "num_enum", + "rand 0.10.2", + "serde", + "smallvec", + "socket2", + "time", + "tokio", + "tokio-util 0.7.18", + "tower-layer", + "tracing", + "url", +] + [[package]] name = "postcard" version = "1.1.3" @@ -2040,9 +3389,21 @@ dependencies = [ "embedded-io 0.4.0", "embedded-io 0.6.1", "heapless", + "postcard-derive", "serde", ] +[[package]] +name = "postcard-derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0232bd009a197ceec9cc881ba46f727fcd8060a2d8d6a9dde7a69030a6fe2bb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -2052,6 +3413,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2061,6 +3428,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.15+spec-1.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -2070,6 +3446,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", +] + [[package]] name = "quickcheck" version = "1.1.0" @@ -2089,7 +3474,7 @@ checksum = "a9a28b8493dd664c8b171dd944da82d933f7d456b829bfb236738e1fe06c5ba4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2359,7 +3744,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -2395,11 +3780,48 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -2472,6 +3894,7 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -2480,6 +3903,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.15.0" @@ -2490,6 +3925,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -2537,6 +3999,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -2566,12 +4034,24 @@ dependencies = [ "libc", ] +[[package]] +name = "seize" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" + [[package]] name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + [[package]] name = "serde" version = "1.0.228" @@ -2623,7 +4103,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2672,6 +4152,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "serial_test" version = "3.5.0" @@ -2694,9 +4184,15 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -2705,7 +4201,27 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", ] [[package]] @@ -2739,18 +4255,52 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "simple-dns" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b6f884fa9a8d48101774bfbd3aeb81e968dd22cffd19a372da69f183db22c1a" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "slab" version = "0.4.12" @@ -2773,6 +4323,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sorted-index-buffer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea06cc588e43c632923a55450401b8f25e628131571d4e1baea1bdfdb2b5ed06" + +[[package]] +name = "spez" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87e960f4dca2788eeb86bbdde8dd246be8948790b7618d656e68f9b720a86e8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "spin" version = "0.9.8" @@ -2782,6 +4349,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -2789,7 +4362,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der 0.8.2", ] [[package]] @@ -2805,10 +4388,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" [[package]] -name = "strsim" -version = "0.11.1" +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "subtle" @@ -2827,6 +4431,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -2844,7 +4459,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2880,6 +4495,17 @@ dependencies = [ "system-configuration-sys", ] +[[package]] +name = "system-configuration" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501336eb7ba9e417300a6a0fa985721065467aa83a6dcf0422a8e43e4c0328fa" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.10.1", + "system-configuration-sys", +] + [[package]] name = "system-configuration-sys" version = "0.6.0" @@ -2950,7 +4576,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2961,7 +4587,47 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "js-sys", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", ] [[package]] @@ -3014,7 +4680,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3037,6 +4703,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util 0.7.18", +] + [[package]] name = "tokio-util" version = "0.6.10" @@ -3062,10 +4740,34 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] +[[package]] +name = "tokio-websockets" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52efb639344a7c6adb8e62c6f3d2c19c001ff1b79a5041ba1c6ed42e19c6aa5" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-sink", + "getrandom 0.4.3", + "http", + "httparse", + "rand 0.10.2", + "ring", + "rustls-pki-types", + "sha1_smol", + "simdutf8", + "tokio", + "tokio-rustls", + "tokio-util 0.7.18", +] + [[package]] name = "toml" version = "0.8.23" @@ -3074,8 +4776,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", ] [[package]] @@ -3087,6 +4789,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -3096,9 +4807,30 @@ dependencies = [ "indexmap", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "toml_write", - "winnow", + "winnow 0.7.15", +] + +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", ] [[package]] @@ -3163,10 +4895,23 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -3174,6 +4919,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -3304,12 +5079,34 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -3326,6 +5123,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -3358,6 +5156,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3446,7 +5250,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -3472,6 +5276,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "watchman_client" version = "0.9.0" @@ -3510,6 +5327,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.8" @@ -3519,6 +5345,12 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -3603,7 +5435,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3614,7 +5446,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3662,6 +5494,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -3698,6 +5539,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -3738,6 +5594,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3750,6 +5612,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3762,6 +5630,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3780,6 +5654,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3792,6 +5672,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3804,6 +5690,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3816,6 +5708,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3837,18 +5735,61 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wmi" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c81b85c57a57500e56669586496bf2abd5cf082b9d32995251185d105208b64" +dependencies = [ + "chrono", + "futures 0.3.32", + "log", + "serde", + "thiserror 2.0.18", + "windows", + "windows-core", +] + [[package]] name = "writeable" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "ws_stream_wasm" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc" +dependencies = [ + "async_io_stream", + "futures 0.3.32", + "js-sys", + "log", + "pharos", + "rustc_version", + "send_wrapper", + "thiserror 2.0.18", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "xattr" version = "1.6.1" @@ -3859,6 +5800,21 @@ dependencies = [ "rustix", ] +[[package]] +name = "xml-rs" +version = "0.8.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" + +[[package]] +name = "xmltree" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" +dependencies = [ + "xml-rs", +] + [[package]] name = "xxhash-rust" version = "0.8.16" @@ -3884,7 +5840,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -3905,7 +5861,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3925,7 +5881,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -3934,6 +5890,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "zerotrie" @@ -3965,7 +5935,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 36780037..31d482b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "atomic-repository", "atomic-semantic", "atomic-teams", + "atomic-wasm", ] [workspace.package] @@ -54,6 +55,7 @@ bs58 = "0.5" blake3 = "1.5" ed25519-dalek = { version = "2.1", features = ["serde"] } rand = "0.8" +iroh = "1" # Compression zstd = "0.13" diff --git a/atomic-canonical/src/proof.rs b/atomic-canonical/src/proof.rs index afd3710d..ab45b012 100644 --- a/atomic-canonical/src/proof.rs +++ b/atomic-canonical/src/proof.rs @@ -110,8 +110,32 @@ pub fn substance_view(value: &Value) -> Value { /// path all typed nodes share. Fills `attributedTo` (from the identity's /// `did:atomic`) when absent, computes the content hash over `hashing_view`, /// signs `jcs(signing_view)`, and attaches the proof. Returns the value. -pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> Value { - let did = did::did_for_public_key(&identity.public_key); +/// +/// It is [`prepare_attestation`] → sign → [`attach_proof`]; a signer that +/// holds its key outside this process uses those two halves directly. +pub fn attest_value(value: Value, identity: &Identity, keypair: &KeyPair) -> Value { + let prepared = prepare_attestation(value, &identity.public_key); + let signature = Signer::new(keypair).sign(&prepared.signing_bytes); + attach_proof(prepared.value, &identity.public_key, &signature) +} + +/// A value made ready to sign: `attributedTo` and `contentHash` filled in, +/// and the exact bytes the signature must cover. +#[derive(Debug, Clone)] +pub struct PreparedAttestation { + /// The value to sign — pass it back to [`attach_proof`] unchanged. + pub value: Value, + /// `jcs(signing_view(value))`: what the Ed25519 signature covers. + pub signing_bytes: Vec, +} + +/// First half of [`attest_value`], for signers that hold the key somewhere +/// else — a browser's WebCrypto, a hardware token, a remote signing service. +/// Everything that must agree with [`verify_value`] (the author, the content +/// hash, the canonical bytes) is computed here, so the external signer only +/// ever signs bytes, and the result is an ordinary atomic attestation. +pub fn prepare_attestation(mut value: Value, public_key: &PublicKey) -> PreparedAttestation { + let did = did::did_for_public_key(public_key); if let Some(obj) = value.as_object_mut() { // Fill attributedTo only if there is no non-empty value already. @@ -121,7 +145,7 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> .map(|s| !s.is_empty()) .unwrap_or(false); if !has_author { - obj.insert(PROP_ATTRIBUTED_TO.to_string(), Value::String(did.clone())); + obj.insert(PROP_ATTRIBUTED_TO.to_string(), Value::String(did)); } } @@ -133,13 +157,24 @@ pub fn attest_value(mut value: Value, identity: &Identity, keypair: &KeyPair) -> } let signing_bytes = jcs::canonicalize(&signing_view(&value)).into_bytes(); - let signature = Signer::new(keypair).sign(&signing_bytes); + PreparedAttestation { + value, + signing_bytes, + } +} + +/// Second half of [`attest_value`]: attach the `eddsa-jcs-2022` proof for a +/// signature over [`PreparedAttestation::signing_bytes`] made by +/// `public_key`'s private key. Does not check the signature — run +/// [`verify_value`] on the result for that. +pub fn attach_proof(mut value: Value, public_key: &PublicKey, signature: &Signature) -> Value { + let did = did::did_for_public_key(public_key); let proof = Proof { type_: PROOF_TYPE.to_string(), cryptosuite: CRYPTOSUITE.to_string(), verification_method: did::verification_method(&did), proof_purpose: PROOF_PURPOSE.to_string(), - proof_value: encode_proof_value(&signature), + proof_value: encode_proof_value(signature), }; if let Some(obj) = value.as_object_mut() { obj.insert( @@ -281,6 +316,25 @@ mod tests { }) } + /// An external signer — handed only the prepared bytes — produces + /// exactly the attestation `attest_value` would, and it verifies. + #[test] + fn prepare_sign_attach_matches_attest_value() { + let (id, kp) = dev_identity(); + let prepared = prepare_attestation(minimal_value(), &kp.public); + let signature = Signer::new(&kp).sign(&prepared.signing_bytes); + let external = attach_proof(prepared.value, &kp.public, &signature); + + assert_eq!(external, attest_value(minimal_value(), &id, &kp)); + verify_value(&external, &kp.public).expect("externally signed value verifies"); + + // A signature over anything else does not. + let wrong = Signer::new(&kp).sign(b"not the prepared bytes"); + let prepared = prepare_attestation(minimal_value(), &kp.public); + let bad = attach_proof(prepared.value, &kp.public, &wrong); + assert!(verify_value(&bad, &kp.public).is_err()); + } + #[test] fn attest_value_then_verify_value_roundtrips() { let (id, kp) = dev_identity(); diff --git a/atomic-cli/Cargo.toml b/atomic-cli/Cargo.toml index a60709a4..2f58f426 100644 --- a/atomic-cli/Cargo.toml +++ b/atomic-cli/Cargo.toml @@ -65,6 +65,8 @@ data-encoding = "2.6" # Hashing (source-content digest for canonical intent sidecars) blake3 = { workspace = true } +iroh = { workspace = true } +rand = { workspace = true } # Open URLs in default browser (for device auth flow) open = "5" diff --git a/atomic-cli/src/commands/agent/mod.rs b/atomic-cli/src/commands/agent/mod.rs index dc4df1ea..b9871d91 100644 --- a/atomic-cli/src/commands/agent/mod.rs +++ b/atomic-cli/src/commands/agent/mod.rs @@ -44,7 +44,7 @@ mod explain; mod hooks; mod identity; mod lifecycle; -mod owner; +pub(crate) mod owner; mod status; use clap::{Args, Subcommand}; diff --git a/atomic-cli/src/commands/agent/owner.rs b/atomic-cli/src/commands/agent/owner/mod.rs similarity index 79% rename from atomic-cli/src/commands/agent/owner.rs rename to atomic-cli/src/commands/agent/owner/mod.rs index 537a0592..8f11b37f 100644 --- a/atomic-cli/src/commands/agent/owner.rs +++ b/atomic-cli/src/commands/agent/owner/mod.rs @@ -3,6 +3,14 @@ //! redb permits one writable process to open a database. This service elects //! that process with an OS-backed file lock and exposes a small, versioned local //! protocol so short-lived agent hooks never open `changes.redb` themselves. +//! +//! The same protocol reaches remote sandboxes over iroh (see [`remote`] and +//! [`sandbox`]): a sandbox on another machine records provenance and reads its +//! view through the owner of the repository it belongs to, with every request +//! checked against its sandbox token. + +mod remote; +mod sandbox; use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; @@ -34,6 +42,9 @@ use uuid::Uuid; use crate::commands::Command; use crate::error::CliResult; +use self::remote::{RemotePointer, TokenRegistry}; +use self::sandbox::Caller; + const PROTOCOL_VERSION: u16 = 1; // Local IPC only; a single oversized envelope (e.g. a huge recovered reasoning // block) must still fit one frame even after client-side chunking. @@ -106,6 +117,9 @@ struct ReserveArgs { struct RequestFrame { version: u16, request_id: String, + /// A remote sandbox's token; local callers send none. + #[serde(default, skip_serializing_if = "Option::is_none")] + token: Option, request: OwnerRequest, } @@ -175,6 +189,82 @@ enum OwnerRequest { turn_number: u32, }, Shutdown, + /// Local only: mint a sandbox token for `view`, reachable over iroh. + OpenSandbox { + view: String, + #[serde(default)] + acting_as: Option, + ttl_secs: i64, + }, + /// Local only: extend `view`'s sandbox token. + RenewSandbox { + view: String, + ttl_secs: i64, + }, + /// Local only: end `view`'s sandbox token now. + CloseSandbox { + view: String, + }, + /// A view's tree: `MaterializeEntry` frames, then `Materialized`. A + /// remote caller gets its token's view; a local one names it. + Materialize { + #[serde(default)] + view: Option, + }, + /// The repository's rows (and content) `record` reads for `inodes` — + /// what a sandbox's cache loads before recording. + FileStates { + #[serde(default)] + view: Option, + inodes: Vec, + }, + /// The change files of `hashes`, each on the view. + Changes { + #[serde(default)] + view: Option, + hashes: Vec, + }, + /// A sandbox's checkpoint: its provenance graph (serialized) and turn. + PublishProvenance { + #[serde(default)] + view: Option, + #[serde(with = "base64_bytes")] + graph: Vec, + turn: SessionTurn, + }, + /// A change a sandbox recorded against `base_state`, as its V3 bytes. + SubmitChange { + #[serde(default)] + view: Option, + base_state: String, + hash: Hash, + #[serde(with = "base64_bytes")] + bytes: Vec, + }, +} + +/// A change file on the wire. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct WireChange { + hash: Hash, + #[serde(with = "base64_bytes")] + bytes: Vec, +} + +/// Bytes as base64 in the JSON frames. +mod base64_bytes { + use serde::{Deserialize, Deserializer, Serializer}; + + pub(super) fn serialize(bytes: &[u8], s: S) -> Result { + s.serialize_str(&data_encoding::BASE64.encode(bytes)) + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + data_encoding::BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -197,6 +287,8 @@ pub(crate) enum OwnerResponse { // Additive v1 capability: old clients ignore it, old owners omit it. #[serde(default)] frozen_envelope_paging: bool, + #[serde(default)] + remote_sandboxes: bool, }, ProvenanceTurn { turn: StoredProvenanceTurn, @@ -221,6 +313,51 @@ pub(crate) enum OwnerResponse { ShuttingDown { pid: u32, }, + SandboxOpened { + remote: iroh::EndpointAddr, + view: String, + token: String, + expires: String, + }, + SandboxRenewed { + expires: String, + }, + SandboxClosed { + revoked: bool, + }, + MaterializeEntry { + entry: atomic_repository::ViewEntry, + }, + Materialized { + snapshot: atomic_repository::ViewSnapshot, + entries: u64, + /// The view's rows, for the sandbox's cache. + skeleton: Box, + }, + FileStates { + slice: Box, + }, + ChangeSubmitted { + submitted: atomic_repository::Submitted, + /// The view's rows now, for the sandbox's cache. + skeleton: Box, + }, + ChangeRefused { + rejection: atomic_repository::SubmitRejection, + /// The view's rows now, when the refusal is about the view having moved + /// and a resync is what the sandbox needs to recover. Carried with the + /// refusal so recovering costs the round trip already being made: a + /// sandbox that has to ask again for a state it was just told is stale + /// can be left unable to record at all. + #[serde(default, skip_serializing_if = "Option::is_none")] + skeleton: Option>, + }, + Changes { + changes: Vec, + }, + ProvenancePublished { + publication: atomic_core::change::session::SessionCheckpointPublication, + }, Error { code: String, message: String, @@ -784,6 +921,10 @@ fn runtime() -> anyhow::Result { /// Start the repository owner or reconnect when another process won election. pub(crate) fn start_or_reconnect(repository: &Path) -> anyhow::Result { + // A remote sandbox's owner runs elsewhere; there is nothing to start. + if remote::find_remote_pointer(repository).is_some() { + return request(repository, OwnerRequest::Ping); + } if let Ok(response) = request(repository, OwnerRequest::Ping) { return Ok(response); } @@ -860,16 +1001,29 @@ where .map_err(|_| anyhow!("database owner client thread panicked"))? } -/// Send one request to an already-running owner. +/// Send one request to an already-running owner: the local one, or — in a +/// remote sandbox — the one its pointer names, with its token. fn request(repository: &Path, request_body: OwnerRequest) -> anyhow::Result { - let dot_dir = Repository::canonical_dot_dir(repository)?; - let endpoint = endpoint_name(&dot_dir); + let remote = remote::find_remote_pointer(repository).map(|(_, pointer)| pointer); let frame = RequestFrame { version: PROTOCOL_VERSION, request_id: Uuid::new_v4().to_string(), + token: remote.as_ref().map(|pointer| pointer.token.clone()), request: request_body, }; - let response = runtime()?.block_on(exchange(&endpoint, &frame))?; + let response = match &remote { + Some(pointer) => runtime()?.block_on(async { + let mut stream = remote::BiStream::dial(pointer.remote.clone()).await?; + write_frame(&mut stream, &frame).await?; + let response = read_frame::(&mut stream).await; + stream.close().await; + response + })?, + None => { + let dot_dir = Repository::canonical_dot_dir(repository)?; + runtime()?.block_on(exchange(&endpoint_name(&dot_dir), &frame))? + } + }; if response.version != PROTOCOL_VERSION { return Err(anyhow!( "database owner protocol mismatch: client {}, server {}", @@ -892,6 +1046,195 @@ fn request(repository: &Path, request_body: OwnerRequest) -> anyhow::Result, + ttl_secs: i64, +) -> anyhow::Result { + start_or_reconnect(repository)?; + match request( + repository, + OwnerRequest::OpenSandbox { + view: view.to_string(), + acting_as, + ttl_secs, + }, + )? { + OwnerResponse::SandboxOpened { + remote, + view, + token, + expires, + } => Ok(OpenedSandbox { + pointer: serde_json::to_value(RemotePointer { + remote, + view, + token, + })?, + expires, + }), + other => Err(unexpected_response("open sandbox", other)), + } +} + +/// Extend `view`'s sandbox token; returns the new expiry. +pub(crate) fn renew_sandbox( + repository: &Path, + view: &str, + ttl_secs: i64, +) -> anyhow::Result { + match request( + repository, + OwnerRequest::RenewSandbox { + view: view.to_string(), + ttl_secs, + }, + )? { + OwnerResponse::SandboxRenewed { expires } => Ok(expires), + other => Err(unexpected_response("renew sandbox", other)), + } +} + +/// End `view`'s sandbox token; whether there was one. +pub(crate) fn close_sandbox(repository: &Path, view: &str) -> anyhow::Result { + match request( + repository, + OwnerRequest::CloseSandbox { + view: view.to_string(), + }, + )? { + OwnerResponse::SandboxClosed { revoked } => Ok(revoked), + other => Err(unexpected_response("close sandbox", other)), + } +} + +/// In a remote sandbox: write its view's tree into the sandbox root and +/// make its cache. Returns the root and the number of entries written. +pub(crate) fn materialize_remote_sandbox(start: &Path) -> anyhow::Result<(PathBuf, u64)> { + let (root, pointer) = remote::find_remote_pointer(start) + .ok_or_else(|| anyhow!("{} is not in a remote sandbox", start.display()))?; + let frame = RequestFrame { + version: PROTOCOL_VERSION, + request_id: Uuid::new_v4().to_string(), + token: Some(pointer.token.clone()), + request: OwnerRequest::Materialize { view: None }, + }; + let written = runtime()?.block_on(async { + let mut stream = remote::BiStream::dial(pointer.remote.clone()).await?; + write_frame(&mut stream, &frame).await?; + let written = sandbox::receive_materialized(&mut stream, &frame.request_id, &root).await; + stream.close().await; + written + })?; + let (written, skeleton) = written; + Repository::create_remote_sandbox_cache(&root, &skeleton)?; + Ok((root, written)) +} + +/// The owner protocol as a remote sandbox's link to its repository's owner: +/// what lets `Repository` record, diff, restore and read history in a +/// remote sandbox, whoever calls it. +pub(crate) struct OwnerLink; + +/// One owner request from a remote sandbox, on its own thread: callers +/// (an agent's turn hooks, say) may already be inside a runtime. +fn link_request(root: &Path, body: OwnerRequest) -> Result { + let root = root.to_path_buf(); + run_outside_async_runtime(move || request(&root, body)).map_err(|e| format!("{e:#}")) +} + +impl atomic_repository::RemoteSandboxLink for OwnerLink { + fn file_states( + &self, + root: &Path, + inodes: Vec, + ) -> Result { + match link_request(root, OwnerRequest::FileStates { view: None, inodes })? { + OwnerResponse::FileStates { slice } => Ok(*slice), + other => Err(unexpected_response("file states", other).to_string()), + } + } + + fn submit( + &self, + root: &Path, + base_state: String, + hash: Hash, + bytes: Vec, + ) -> Result { + let body = OwnerRequest::SubmitChange { + view: None, + base_state, + hash, + bytes, + }; + match link_request(root, body)? { + OwnerResponse::ChangeSubmitted { + submitted, + skeleton, + } => Ok(Ok((submitted, *skeleton))), + OwnerResponse::ChangeRefused { + rejection, + skeleton, + } => Ok(Err((rejection, skeleton.map(|s| *s)))), + other => Err(unexpected_response("submit change", other).to_string()), + } + } + + fn changes( + &self, + root: &Path, + hashes: Vec, + ) -> Result, String> { + match link_request(root, OwnerRequest::Changes { view: None, hashes })? { + OwnerResponse::Changes { changes } => { + Ok(changes.into_iter().map(|c| (c.hash, c.bytes)).collect()) + } + OwnerResponse::ChangeRefused { + rejection, + skeleton: _, + } => Err(rejection.to_string()), + other => Err(unexpected_response("changes", other).to_string()), + } + } + + fn publish_provenance( + &self, + root: &Path, + graph: Vec, + turn: SessionTurn, + ) -> Result< + Result< + atomic_core::change::session::SessionCheckpointPublication, + atomic_repository::SubmitRejection, + >, + String, + > { + let body = OwnerRequest::PublishProvenance { + view: None, + graph, + turn, + }; + match link_request(root, body)? { + OwnerResponse::ProvenancePublished { publication } => Ok(Ok(publication)), + OwnerResponse::ChangeRefused { + rejection, + skeleton: _, + } => Ok(Err(rejection)), + other => Err(unexpected_response("publish provenance", other).to_string()), + } + } +} + fn serve(repository: &Path) -> CliResult<()> { let dot_dir = Repository::canonical_dot_dir(repository)?; let owner_lock = acquire_owner_lock(&dot_dir)?; @@ -901,7 +1244,8 @@ fn serve(repository: &Path) -> CliResult<()> { .with_context(|| format!("failed to open {}", store_path.display()))?, ); let endpoint = endpoint_name(&dot_dir); - runtime()?.block_on(run_server(&endpoint, store))?; + let owner = OwnerState::new(store, dot_dir.clone()); + runtime()?.block_on(run_server(&endpoint, owner))?; FileExt::unlock(&owner_lock).context("failed to release database-owner lock")?; Ok(()) } @@ -965,6 +1309,7 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram OwnerResponse::Pong { pid: std::process::id(), frozen_envelope_paging: true, + remote_sandboxes: true, }, false, ), @@ -1298,6 +1643,21 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram }, true, ), + // Answered by `handle_connection`, which needs the whole owner. + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + | OwnerRequest::Materialize { .. } + | OwnerRequest::FileStates { .. } + | OwnerRequest::Changes { .. } + | OwnerRequest::PublishProvenance { .. } + | OwnerRequest::SubmitChange { .. } => ( + OwnerResponse::Error { + code: "internal".to_string(), + message: "sandbox requests need the owner's connection handler".to_string(), + }, + false, + ), }; ( @@ -1310,22 +1670,133 @@ fn handle_request(store: &RedbChangeStore, frame: RequestFrame) -> (ResponseFram ) } -async fn handle_connection( - mut stream: S, +/// Everything a running owner holds. +pub(crate) struct OwnerState { store: Arc, + /// The repository root. + root: PathBuf, + dot_dir: PathBuf, + tokens: TokenRegistry, + /// Bound on the first `OpenSandbox`; accepts remote callers from then on. + remote: tokio::sync::OnceCell, + /// Submitted changes go in one at a time: each is checked against the + /// view's state as it is when it's applied. + submissions: tokio::sync::Mutex<()>, shutdown: Arc, +} + +impl OwnerState { + fn new(store: Arc, dot_dir: PathBuf) -> Arc { + let root = dot_dir + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| dot_dir.clone()); + Arc::new(Self { + store, + root, + dot_dir, + tokens: TokenRegistry::default(), + remote: tokio::sync::OnceCell::new(), + submissions: tokio::sync::Mutex::new(()), + shutdown: Arc::new(Notify::new()), + }) + } + + /// The owner's iroh endpoint, bound (and accepting) on first use. + async fn remote_endpoint(self: &Arc) -> anyhow::Result<&iroh::Endpoint> { + self.remote + .get_or_try_init(|| async { + let endpoint = remote::bind(&self.dot_dir, remote::offline()).await?; + spawn_accept_remote(endpoint.clone(), Arc::clone(self)); + Ok(endpoint) + }) + .await + } +} + +/// A plain function between binding and accepting: the accept loop answers +/// `OpenSandbox`, which binds, so the two futures can't contain each other. +fn spawn_accept_remote(endpoint: iroh::Endpoint, owner: Arc) { + tokio::spawn(accept_remote(endpoint, owner)); +} + +/// Answer remote callers: each bi-stream is one request, as on the socket. +async fn accept_remote(endpoint: iroh::Endpoint, owner: Arc) { + while let Some(incoming) = endpoint.accept().await { + let owner = Arc::clone(&owner); + tokio::spawn(async move { + let Ok(connection) = incoming.await else { + return; + }; + while let Some(stream) = remote::BiStream::accept(&connection).await { + let owner = Arc::clone(&owner); + tokio::spawn(async move { + if let Err(error) = handle_connection(stream, owner, Caller::Remote).await { + log::warn!("database-owner remote request failed: {error}"); + } + }); + } + }); + } +} + +async fn handle_connection( + mut stream: S, + owner: Arc, + caller: Caller, ) -> anyhow::Result<()> where S: AsyncRead + AsyncWrite + Unpin, { let request: RequestFrame = read_frame(&mut stream).await?; - let (response, should_shutdown) = handle_request(&store, request); - write_frame(&mut stream, &response).await?; + let respond = |request_id: String, response| ResponseFrame { + version: PROTOCOL_VERSION, + request_id, + response, + }; + let grant = match sandbox::authorize(&owner, caller, &request) { + Ok(grant) => grant, + Err(refusal) => { + write_frame(&mut stream, &respond(request.request_id, *refusal)).await?; + stream.shutdown().await?; + return Ok(()); + } + }; + let mut should_shutdown = false; + match &request.request { + OwnerRequest::Materialize { .. } if request.version == PROTOCOL_VERSION => { + sandbox::materialize(&owner, grant, request, &mut stream).await?; + } + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + if request.version == PROTOCOL_VERSION => + { + let response = sandbox::admin(&owner, request.request.clone()).await; + write_frame(&mut stream, &respond(request.request_id, response)).await?; + } + OwnerRequest::FileStates { .. } + | OwnerRequest::Changes { .. } + | OwnerRequest::PublishProvenance { .. } + | OwnerRequest::SubmitChange { .. } + if request.version == PROTOCOL_VERSION => + { + let request_id = request.request_id.clone(); + let response = sandbox::record_request(&owner, grant, request.request).await; + write_frame(&mut stream, &respond(request_id, response)).await?; + } + _ => { + let (response, shutdown) = handle_request(&owner.store, request); + sandbox::note_response(&owner.tokens, grant.as_ref(), &response.response); + write_frame(&mut stream, &response).await?; + should_shutdown = shutdown; + } + } stream.shutdown().await?; if should_shutdown { // `notify_one` retains a permit if the accept loop is between polls, // preventing a shutdown request from being acknowledged but lost. - shutdown.notify_one(); + owner.shutdown.notify_one(); } Ok(()) } @@ -1459,7 +1930,7 @@ async fn connect_owner_pipe( } #[cfg(unix)] -async fn run_server(endpoint: &str, store: Arc) -> anyhow::Result<()> { +async fn run_server(endpoint: &str, owner: Arc) -> anyhow::Result<()> { use tokio::net::UnixListener; let path = Path::new(endpoint); @@ -1469,16 +1940,15 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } let listener = UnixListener::bind(path) .with_context(|| format!("failed to bind database owner at {endpoint}"))?; - let shutdown = Arc::new(Notify::new()); + let shutdown = Arc::clone(&owner.shutdown); loop { tokio::select! { accepted = listener.accept() => { let (stream, _) = accepted?; - let store = Arc::clone(&store); - let shutdown = Arc::clone(&shutdown); + let owner = Arc::clone(&owner); tokio::spawn(async move { - if let Err(error) = handle_connection(stream, store, shutdown).await { + if let Err(error) = handle_connection(stream, owner, Caller::Local).await { log::warn!("database-owner connection failed: {error}"); } }); @@ -1488,6 +1958,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } drop(listener); + if let Some(remote) = owner.remote.get() { + remote.close().await; + } if path.exists() { std::fs::remove_file(path) .with_context(|| format!("failed to remove endpoint {endpoint}"))?; @@ -1496,10 +1969,10 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu } #[cfg(windows)] -async fn run_server(endpoint: &str, store: Arc) -> anyhow::Result<()> { +async fn run_server(endpoint: &str, owner: Arc) -> anyhow::Result<()> { use tokio::net::windows::named_pipe::ServerOptions; - let shutdown = Arc::new(Notify::new()); + let shutdown = Arc::clone(&owner.shutdown); let mut server = ServerOptions::new() .first_pipe_instance(true) .create(endpoint) @@ -1515,10 +1988,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu .create(endpoint) .with_context(|| format!("failed to create database owner pipe {endpoint}"))?; let connected = std::mem::replace(&mut server, next); - let store = Arc::clone(&store); - let shutdown = Arc::clone(&shutdown); + let owner = Arc::clone(&owner); tokio::spawn(async move { - if let Err(error) = handle_connection(connected, store, shutdown).await { + if let Err(error) = handle_connection(connected, owner, Caller::Local).await { log::warn!("database-owner connection failed: {error}"); } }); @@ -1526,6 +1998,9 @@ async fn run_server(endpoint: &str, store: Arc) -> anyhow::Resu () = shutdown.notified() => break, } } + if let Some(remote) = owner.remote.get() { + remote.close().await; + } Ok(()) } @@ -1698,6 +2173,7 @@ mod tests { let request = RequestFrame { version: PROTOCOL_VERSION, request_id: request_id.clone(), + token: None, request: OwnerRequest::LoadFrozenEnvelopesPage { provenance_id: turn.provenance_id.get(), attempt_generation: attempt.attempt_generation, @@ -1737,7 +2213,8 @@ mod tests { .contains("database-owner shutdown")); assert!(require_frozen_paging(OwnerResponse::Pong { pid: 42, - frozen_envelope_paging: true + frozen_envelope_paging: true, + remote_sandboxes: false, }) .is_ok()); } @@ -1765,6 +2242,7 @@ mod tests { RequestFrame { version: PROTOCOL_VERSION, request_id: "retry".into(), + token: None, request: OwnerRequest::PrepareCheckpoint { provenance_id: turn.provenance_id.get(), expected_generation: generation, @@ -1812,6 +2290,7 @@ mod tests { let expected = RequestFrame { version: PROTOCOL_VERSION, request_id: "request-1".to_string(), + token: None, request: OwnerRequest::Ping, }; let sent = expected.clone(); @@ -1845,6 +2324,7 @@ mod tests { let frame = RequestFrame { version: PROTOCOL_VERSION, request_id: "stale-request".to_string(), + token: None, request: OwnerRequest::AppendProvenanceEnvelopes { provenance_id: running.provenance_id.get(), expected_generation: running.generation, diff --git a/atomic-cli/src/commands/agent/owner/remote.rs b/atomic-cli/src/commands/agent/owner/remote.rs new file mode 100644 index 00000000..34b69854 --- /dev/null +++ b/atomic-cli/src/commands/agent/owner/remote.rs @@ -0,0 +1,535 @@ +//! The owner's remote transport: the same protocol over iroh, for sandboxes +//! on another machine. +//! +//! The local socket trusts its caller (the filesystem guards it). A remote +//! caller is trusted only as far as its sandbox token: every frame carries +//! it, and it reaches exactly one view, optionally as one identity, until it +//! expires. Tokens live **in memory only** — they end when the owner exits — +//! and only their hashes are kept. +//! +//! Provenance a remote sandbox records is bound to its token the same way: a +//! session it starts is its own, and it can touch no other. + +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use anyhow::{anyhow, Context}; +use chrono::{DateTime, Duration, Utc}; +use iroh::endpoint::{presets, Connection, RecvStream, SendStream}; +use iroh::{Endpoint, EndpointAddr, SecretKey}; +use rand::RngCore; +use serde::{Deserialize, Serialize}; + +/// ALPN for the owner protocol over iroh. +pub(crate) const ALPN: &[u8] = b"atomic-owner/1"; + +/// The owner's iroh key, kept beside its lock so its address survives a +/// restart and existing pointers keep working. +const IROH_KEY_FILE: &str = "owner-iroh.key"; + +/// What a sandbox token grants. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Grant { + pub(crate) view: String, + /// The identity the sandbox's work is attributed to (e.g. an agent's). + pub(crate) acting_as: Option, + pub(crate) expires: DateTime, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub(crate) enum TokenError { + #[error("unknown or revoked sandbox token")] + Unknown, + #[error("sandbox token expired at {0}")] + Expired(DateTime), + #[error("no live token for view '{0}'")] + NoTokenForView(String), + #[error("a sandbox token must last between 1 second and {MAX_TTL_SECS} seconds")] + InvalidTtl, +} + +/// Bounds on a requested token lifetime, in seconds. Both ends matter: a +/// non-positive TTL would mint a token that is already dead, and an enormous +/// one is either a mistake or a way to make a token that outlives the owner. +/// The lifetime is validated and turned into a deadline *before* the registry +/// lock is taken, so a bad value can never be the thing that panics inside it. +const MIN_TTL_SECS: i64 = 1; +const MAX_TTL_SECS: i64 = 365 * 24 * 60 * 60; + +/// The deadline a TTL of `secs` seconds from now means, or [`TokenError::InvalidTtl`]. +/// +/// `Duration::seconds` panics outside its range and `DateTime + Duration` +/// panics on overflow, and both are reachable from a flag and then a wire +/// frame. Everything here is checked arithmetic instead, so a request this +/// rejects is a request that returns an error. +fn deadline(secs: i64) -> Result, TokenError> { + if !(MIN_TTL_SECS..=MAX_TTL_SECS).contains(&secs) { + return Err(TokenError::InvalidTtl); + } + let ttl = Duration::try_seconds(secs).ok_or(TokenError::InvalidTtl)?; + Utc::now() + .checked_add_signed(ttl) + .ok_or(TokenError::InvalidTtl) +} + +#[derive(Default)] +struct Tokens { + /// Token digest → grant. A view has at most one live token. + grants: HashMap, + /// Provenance sessions and turns each view's sandbox started. + sessions: HashMap, + provenance: HashMap, +} + +/// In-memory, view-scoped sandbox tokens. Cheap to clone (shared). +#[derive(Clone, Default)] +pub(crate) struct TokenRegistry { + inner: Arc>, +} + +impl TokenRegistry { + /// The registry's lock. + /// + /// A panic while holding it poisons the mutex, and every method would then + /// panic on entry — one bad request taking down every sandbox plus the + /// local `create`/`close` until the owner restarts. The registry's + /// invariants do not depend on unwinding, so a poisoned lock is recovered + /// rather than propagated. + fn lock(&self) -> std::sync::MutexGuard<'_, Tokens> { + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } +} + +fn digest(token: &str) -> String { + blake3::hash(token.as_bytes()).to_hex().to_string() +} + +impl TokenRegistry { + /// Mint a token for `view`, valid for `ttl_secs` seconds. Minting again + /// replaces the view's token. + /// + /// The lifetime is turned into a deadline before the lock is taken, so a + /// bad `--ttl` is an error rather than a panic inside the registry. + pub(crate) fn mint( + &self, + view: &str, + acting_as: Option, + ttl_secs: i64, + ) -> Result<(String, Grant), TokenError> { + let expires = deadline(ttl_secs)?; + let mut bytes = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut bytes); + let token = format!("ast_{}", data_encoding::BASE64URL_NOPAD.encode(&bytes)); + let grant = Grant { + view: view.to_string(), + acting_as, + expires, + }; + let mut t = self.lock(); + t.grants.retain(|_, g| g.view != view); + t.grants.insert(digest(&token), grant.clone()); + Ok((token, grant)) + } + + /// Extend `view`'s live token to now + `ttl_secs` seconds; the same token + /// keeps working. + pub(crate) fn renew(&self, view: &str, ttl_secs: i64) -> Result, TokenError> { + let expires = deadline(ttl_secs)?; + let mut t = self.lock(); + let now = Utc::now(); + let grant = t + .grants + .values_mut() + .find(|g| g.view == view && g.expires > now) + .ok_or_else(|| TokenError::NoTokenForView(view.to_string()))?; + grant.expires = expires; + Ok(grant.expires) + } + + /// End `view`'s token now. + pub(crate) fn revoke(&self, view: &str) -> bool { + let mut t = self.lock(); + let before = t.grants.len(); + t.grants.retain(|_, g| g.view != view); + t.grants.len() != before + } + + /// What `token` grants, if it is live. + pub(crate) fn check(&self, token: &str) -> Result { + let t = self.lock(); + let grant = t.grants.get(&digest(token)).ok_or(TokenError::Unknown)?; + if grant.expires <= Utc::now() { + return Err(TokenError::Expired(grant.expires)); + } + Ok(grant.clone()) + } + + /// Whether `view`'s sandbox may use provenance session `session_id`: + /// one it started, or one nobody has (`fresh`), which becomes its own. + pub(crate) fn claim_session(&self, view: &str, session_id: &str, fresh: bool) -> bool { + let mut t = self.lock(); + match t.sessions.get(session_id) { + Some(owner) => owner == view, + None if fresh => { + t.sessions.insert(session_id.to_string(), view.to_string()); + true + } + None => false, + } + } + + pub(crate) fn owns_session(&self, view: &str, session_id: &str) -> bool { + self.lock().sessions.get(session_id).map(String::as_str) == Some(view) + } + + pub(crate) fn bind_provenance(&self, view: &str, provenance_id: u64) { + self.lock() + .provenance + .insert(provenance_id, view.to_string()); + } + + pub(crate) fn owns_provenance(&self, view: &str, provenance_id: u64) -> bool { + self.lock() + .provenance + .get(&provenance_id) + .map(String::as_str) + == Some(view) + } +} + +/// A remote sandbox's `.atomic-sandbox`: where its repository's owner is, +/// the view it works on, and its token. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct RemotePointer { + pub(crate) remote: EndpointAddr, + pub(crate) view: String, + pub(crate) token: String, +} + +/// Walk up from `start` to the nearest `.atomic-sandbox`; if it names a +/// remote owner, return the sandbox root and the pointer. +pub(crate) fn find_remote_pointer(start: &Path) -> Option<(std::path::PathBuf, RemotePointer)> { + let mut dir = std::path::absolute(start).ok()?; + loop { + let path = dir.join(atomic_repository::SANDBOX_POINTER); + if path.is_file() { + let bytes = std::fs::read(&path).ok()?; + return serde_json::from_slice(&bytes).ok().map(|p| (dir, p)); + } + if dir.join(".atomic").is_dir() || !dir.pop() { + return None; + } + } +} + +/// The owner's iroh endpoint, with a key persisted in `dot_dir`. +pub(crate) async fn bind(dot_dir: &Path, offline: bool) -> anyhow::Result { + let key_path = dot_dir.join(IROH_KEY_FILE); + let key = match std::fs::read(&key_path) { + Ok(bytes) => { + let bytes: [u8; 32] = bytes + .try_into() + .map_err(|_| anyhow!("{} is not a 32-byte key", key_path.display()))?; + SecretKey::from_bytes(&bytes) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let key = SecretKey::generate(); + write_private(&key_path, &key.to_bytes())?; + key + } + Err(e) => return Err(e).with_context(|| format!("failed to read {}", key_path.display())), + }; + Ok(builder(offline)? + .secret_key(key) + .alpns(vec![ALPN.to_vec()]) + .bind() + .await?) +} + +/// The address to put in a pointer: once online, what iroh knows (relay and +/// direct addresses); offline, the bound sockets — unspecified ones as +/// loopback, which is all an offline owner can promise. +pub(crate) async fn pointer_addr(endpoint: &Endpoint, offline: bool) -> EndpointAddr { + if !offline { + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), endpoint.online()).await; + } + let addr = endpoint.addr(); + if addr.ip_addrs().next().is_some() || !offline { + return addr; + } + endpoint + .bound_sockets() + .into_iter() + .fold(addr, |addr, mut socket| { + if socket.ip().is_unspecified() { + socket.set_ip(match socket { + std::net::SocketAddr::V4(_) => std::net::Ipv4Addr::LOCALHOST.into(), + std::net::SocketAddr::V6(_) => std::net::Ipv6Addr::LOCALHOST.into(), + }); + } + addr.with_ip_addr(socket) + }) +} + +/// A client endpoint for dialing an owner. +async fn dialer(offline: bool) -> anyhow::Result { + Ok(builder(offline)?.bind().await?) +} + +/// An endpoint builder: the relay-less preset when offline, and IPv4 only +/// when `ATOMIC_IROH_IPV4_ONLY` is set — for hosts where creating an IPv6 +/// socket misbehaves (it takes down the process under some microVM network +/// backends), so the endpoint never opens one. +fn builder(offline: bool) -> anyhow::Result { + let builder = if offline { + Endpoint::builder(presets::Minimal) + } else { + Endpoint::builder(presets::N0) + }; + if !ipv4_only() { + return Ok(builder); + } + builder + .clear_ip_transports() + .bind_addr("0.0.0.0:0") + .map_err(|e| anyhow!("invalid bind address: {e}")) +} + +fn ipv4_only() -> bool { + std::env::var("ATOMIC_IROH_IPV4_ONLY").is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true")) +} + +fn write_private(path: &Path, bytes: &[u8]) -> anyhow::Result<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options + .open(path) + .with_context(|| format!("failed to create {}", path.display()))?; + file.write_all(bytes)?; + Ok(()) +} + +/// Whether owners and clients skip iroh's relays and address lookup, and +/// dial by the direct addresses a pointer carries (tests, air-gapped hosts). +pub(crate) fn offline() -> bool { + std::env::var_os("ATOMIC_OWNER_IROH_OFFLINE").is_some() +} + +/// One iroh bi-stream as `AsyncRead + AsyncWrite`: one request, as on the +/// local socket. +pub(crate) struct BiStream { + send: SendStream, + recv: RecvStream, + _connection: Connection, + /// A dialing client's own endpoint, which must outlive the stream. + dialer: Option, +} + +impl BiStream { + /// Dial the owner at `addr` and open one request stream. + pub(crate) async fn dial(addr: EndpointAddr) -> anyhow::Result { + let endpoint = dialer(offline()).await?; + let id = addr.id; + let connection = endpoint + .connect(addr, ALPN) + .await + .with_context(|| format!("database owner {id} is not reachable"))?; + let (send, recv) = connection.open_bi().await?; + Ok(Self { + send, + recv, + _connection: connection, + dialer: Some(endpoint), + }) + } + + /// Done with the request: close the dialing endpoint gracefully. + pub(crate) async fn close(self) { + if let Some(endpoint) = self.dialer { + endpoint.close().await; + } + } + + pub(crate) async fn accept(connection: &Connection) -> Option { + let (send, recv) = connection.accept_bi().await.ok()?; + Some(Self { + send, + recv, + _connection: connection.clone(), + dialer: None, + }) + } +} + +impl tokio::io::AsyncRead for BiStream { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.recv).poll_read(cx, buf) + } +} + +impl tokio::io::AsyncWrite for BiStream { + fn poll_write( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_write(cx, buf) + .map_err(std::io::Error::other) + } + + fn poll_flush( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_flush(cx) + .map_err(std::io::Error::other) + } + + fn poll_shutdown( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.send) + .poll_shutdown(cx) + .map_err(std::io::Error::other) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const HOUR: i64 = 60 * 60; + + #[test] + fn a_token_reaches_its_view_until_it_ends() { + let reg = TokenRegistry::default(); + let (t, g) = reg + .mint("exp-1", Some("did:key:zAgent".into()), 24 * HOUR) + .unwrap(); + assert_eq!(reg.check(&t).unwrap().view, "exp-1"); + assert_eq!(reg.check("ast_made_up"), Err(TokenError::Unknown)); + + std::thread::sleep(std::time::Duration::from_millis(5)); + let after = reg.renew("exp-1", 24 * HOUR).unwrap(); + assert!(after > g.expires); + assert_eq!(reg.check(&t).unwrap().expires, after); + + assert!(reg.revoke("exp-1")); + assert_eq!(reg.check(&t), Err(TokenError::Unknown)); + assert!(reg.renew("exp-1", HOUR).is_err()); + } + + #[test] + fn expired_and_replaced_tokens_are_refused() { + let reg = TokenRegistry::default(); + let (t, _) = reg.mint("exp-1", None, 60).unwrap(); + // Backdate it rather than minting one that is already dead: a + // non-positive lifetime is refused now, which is the point below. + reg.lock() + .grants + .values_mut() + .for_each(|g| g.expires = Utc::now() - Duration::seconds(1)); + assert!(matches!(reg.check(&t), Err(TokenError::Expired(_)))); + assert!(reg.renew("exp-1", HOUR).is_err()); + + let (old, _) = reg.mint("exp-2", None, HOUR).unwrap(); + let (new, _) = reg.mint("exp-2", None, HOUR).unwrap(); + assert_eq!(reg.check(&old), Err(TokenError::Unknown)); + assert!(reg.check(&new).is_ok()); + } + + /// `--ttl` reaches the owner as an `i64` off a wire frame. + /// `Duration::seconds` panics outside its range and `now + Duration` + /// panics on overflow, and both used to happen while holding the registry + /// lock — so one absurd value poisoned the mutex and took every other + /// sandbox and the local `create`/`close` down with it. + #[test] + fn an_absurd_lifetime_is_an_error_and_not_a_panic() { + let reg = TokenRegistry::default(); + // Establish a live token first: the panic used to land in renew, on + // the way to extending one. + let (t, _) = reg.mint("exp-1", None, HOUR).unwrap(); + + for secs in [ + 0, + -1, + i64::MIN, + MAX_TTL_SECS + 1, + i64::MAX, + // Below Duration::seconds' own i64-milliseconds ceiling, so this + // one panicked inside chrono rather than at the addition. + i64::MAX / 1_000, + ] { + assert_eq!( + reg.renew("exp-1", secs), + Err(TokenError::InvalidTtl), + "--ttl {secs} should be refused" + ); + assert_eq!( + reg.mint("exp-2", None, secs).err(), + Some(TokenError::InvalidTtl), + "minting with --ttl {secs} should be refused" + ); + } + + // The ends of the accepted range, and the registry still works. + assert!(reg.mint("low", None, MIN_TTL_SECS).is_ok()); + assert!(reg.mint("high", None, MAX_TTL_SECS).is_ok()); + + // Nothing above was applied, and the registry is intact. + assert_eq!(reg.check(&t).unwrap().view, "exp-1"); + assert!(reg.revoke("exp-1")); + assert!(reg.claim_session("exp-1", "s1", true)); + } + + /// A panic anywhere else in an owner request used to poison the registry, + /// and because every method took the lock with `unwrap`, the next caller + /// panicked too — permanently, for every sandbox and for the local + /// `create`/`close`, until the owner was restarted. + #[test] + fn a_poisoned_lock_does_not_take_the_registry_down() { + let reg = TokenRegistry::default(); + let (t, _) = reg.mint("exp-1", None, HOUR).unwrap(); + + let poisoner = reg.clone(); + let _ = std::thread::spawn(move || { + let _held = poisoner.lock(); + panic!("something unrelated blew up mid-request"); + }) + .join(); + + // Still usable, and the token it already had still resolves. + assert_eq!(reg.check(&t).unwrap().view, "exp-1"); + assert!(reg.mint("exp-2", None, HOUR).is_ok()); + assert!(reg.claim_session("exp-1", "s1", true)); + assert!(reg.revoke("exp-1")); + } + + #[test] + fn a_session_belongs_to_the_view_that_started_it() { + let reg = TokenRegistry::default(); + assert!(reg.claim_session("exp-1", "s1", true)); + assert!(reg.claim_session("exp-1", "s1", false), "its own, again"); + assert!(!reg.claim_session("exp-2", "s1", true), "another view's"); + assert!( + !reg.claim_session("exp-2", "s-local", false), + "an existing session it didn't start" + ); + assert!(reg.owns_session("exp-1", "s1")); + assert!(!reg.owns_session("exp-2", "s1")); + } +} diff --git a/atomic-cli/src/commands/agent/owner/sandbox.rs b/atomic-cli/src/commands/agent/owner/sandbox.rs new file mode 100644 index 00000000..6277ae96 --- /dev/null +++ b/atomic-cli/src/commands/agent/owner/sandbox.rs @@ -0,0 +1,729 @@ +//! Remote sandboxes on the owner protocol: who may ask for what, the +//! requests that open and close a sandbox, and materializing its view. +//! +//! A local caller is trusted as it always was. A remote caller (iroh) gets +//! the provenance requests and `Materialize`, each checked against its +//! token: its own view, its own sessions and turns, and checkpoints only for +//! changes its view can see. Opening, renewing and closing sandboxes, and +//! shutting the owner down, are local only. + +use std::path::{Path, PathBuf}; + +use atomic_core::pristine::{GraphTxnT, ViewTxnT}; +use atomic_core::types::{Base32, Hash}; +use atomic_repository::redb_change_store::RedbChangeStore; +use atomic_repository::{ + Repository, SubmitRejection, ViewEntryKind, DOT_DIR, SANDBOX_CACHE_DIR, SANDBOX_POINTER, +}; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; + +use super::remote::{Grant, TokenRegistry}; +use super::{ + read_frame, write_frame, OwnerRequest, OwnerResponse, OwnerState, RequestFrame, ResponseFrame, + PROTOCOL_VERSION, +}; + +/// Names a materialized entry may not use, because they are the sandbox's own +/// bookkeeping: writing the pointer would re-point the sandbox (including at an +/// attacker's own node), and writing the cache would edit the pristine the next +/// record reads. +const RESERVED_NAMES: &[&str] = &[DOT_DIR, SANDBOX_POINTER, SANDBOX_CACHE_DIR]; + +/// Who is on the other end of a connection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Caller { + /// The local socket or pipe. + Local, + /// iroh: trusted only as far as the frame's token. + Remote, +} + +fn error(code: &str, message: impl Into) -> OwnerResponse { + OwnerResponse::Error { + code: code.to_string(), + message: message.into(), + } +} + +fn refuse(code: &str, message: impl Into) -> Box { + Box::new(error(code, message)) +} + +/// The grant `frame` acts under (`None` for a local caller), or the +/// refusal to send instead. +pub(crate) fn authorize( + owner: &OwnerState, + caller: Caller, + frame: &RequestFrame, +) -> Result, Box> { + if caller == Caller::Local { + return Ok(None); + } + let token = frame + .token + .as_deref() + .ok_or_else(|| refuse("unauthorized", "a remote request needs a sandbox token"))?; + let grant = owner + .tokens + .check(token) + .map_err(|e| refuse("unauthorized", e.to_string()))?; + let view = grant.view.as_str(); + let tokens = &owner.tokens; + let forbidden = |what: &str| { + refuse( + "forbidden", + format!("{what} is not this sandbox's (view '{view}')"), + ) + }; + match &frame.request { + OwnerRequest::Ping => {} + OwnerRequest::Materialize { view: asked } + | OwnerRequest::FileStates { view: asked, .. } + | OwnerRequest::Changes { view: asked, .. } + | OwnerRequest::SubmitChange { view: asked, .. } => { + if asked.as_deref().is_some_and(|v| v != view) { + return Err(forbidden("that view")); + } + } + OwnerRequest::PublishProvenance { + view: asked, turn, .. + } => { + if asked.as_deref().is_some_and(|v| v != view) { + return Err(forbidden("that view")); + } + if !tokens.owns_session(view, &turn.session_id) { + return Err(forbidden("that session")); + } + } + OwnerRequest::ReserveProvenanceTurn { session_id, .. } => { + let unused = session_is_unused(&owner.store, session_id) + .map_err(|e| refuse("provenance-store", e.to_string()))?; + if !tokens.claim_session(view, session_id, unused) { + return Err(forbidden("that session")); + } + } + OwnerRequest::AppendProvenanceEnvelopes { provenance_id, .. } + | OwnerRequest::LoadFrozenEnvelopes { provenance_id } + | OwnerRequest::LoadFrozenEnvelopesPage { provenance_id, .. } + | OwnerRequest::AcknowledgeCheckpoint { provenance_id, .. } => { + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + } + OwnerRequest::PrepareCheckpoint { + provenance_id, + source, + .. + } => { + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + require_on_view(&owner.root, view, &source.change_hashes)?; + } + OwnerRequest::BindCheckpointHash { provenance_id, .. } => { + // Its hash names the checkpoint's provenance graph, not a change; + // the changes it explains were checked at PrepareCheckpoint. + if !tokens.owns_provenance(view, *provenance_id) { + return Err(forbidden("that provenance turn")); + } + } + OwnerRequest::StopTurn { session_id, .. } + | OwnerRequest::ResumeTurn { session_id, .. } + | OwnerRequest::AbandonTurn { session_id, .. } + | OwnerRequest::TurnStatus { session_id, .. } => { + // Asking after a session nobody has written to is harmless (the + // store answers "no such turn"); anyone else's is not. + if !tokens.owns_session(view, session_id) + && !session_is_unused(&owner.store, session_id) + .map_err(|e| refuse("provenance-store", e.to_string()))? + { + return Err(forbidden("that session")); + } + } + OwnerRequest::OpenSandbox { .. } + | OwnerRequest::RenewSandbox { .. } + | OwnerRequest::CloseSandbox { .. } + | OwnerRequest::Shutdown => { + return Err(refuse("forbidden", "only a local caller may do that")); + } + } + Ok(Some(grant)) +} + +/// Whether nothing has ever been recorded for `session_id`. +/// +/// One indexed lookup for the session's highest recorded turn, not a walk from +/// turn 1: the turn number arrives on the wire, so walking to it made a single +/// request cost up to `u32::MAX` database round trips. +/// +/// The question is whether the session is unused, not whether one number in it +/// is free. Every caller wants "has anyone claimed this id?" — claiming a +/// session, and asking after a turn on it. Answering that by scanning +/// `1..=turn` made a session with history *above* the asked turn look unused, +/// which is the wrong way round for a check that decides who may write to it. +fn session_is_unused(store: &RedbChangeStore, session_id: &str) -> anyhow::Result { + Ok(store.last_provenance_turn_for(session_id)?.is_none()) +} + +/// Refuse unless every change in `hashes` is visible on `view`: a sandbox's +/// provenance can only describe its own view's work. +fn require_on_view(root: &Path, view: &str, hashes: &[Hash]) -> Result<(), Box> { + if hashes.is_empty() { + return Ok(()); + } + let check = || -> anyhow::Result> { + let repo = Repository::open_readonly(root)?; + let txn = repo.pristine().read_txn()?; + let state = txn + .get_view(view)? + .ok_or_else(|| anyhow::anyhow!("view '{view}' no longer exists"))?; + let visible = atomic_repository::collect_visible_change_ids(&txn, &state)?; + for hash in hashes { + match txn.get_internal(hash)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Some(*hash)), + } + } + Ok(None) + }; + match check() { + Ok(None) => Ok(()), + Ok(Some(hash)) => Err(refuse( + "forbidden", + format!("change {} is not on view '{view}'", hash.to_base32()), + )), + Err(e) => Err(refuse("repository", e.to_string())), + } +} + +/// After a remote reservation succeeds, the turn is the sandbox's. +pub(crate) fn note_response( + tokens: &TokenRegistry, + grant: Option<&Grant>, + response: &OwnerResponse, +) { + if let (Some(grant), OwnerResponse::ProvenanceTurn { turn, .. }) = (grant, response) { + tokens.bind_provenance(&grant.view, turn.provenance_id.get()); + } +} + +/// `OpenSandbox`, `RenewSandbox`, `CloseSandbox` (local callers only). +pub(crate) async fn admin( + owner: &std::sync::Arc, + request: OwnerRequest, +) -> OwnerResponse { + match request { + OwnerRequest::OpenSandbox { + view, + acting_as, + ttl_secs, + } => { + match Repository::open_readonly(&owner.root) + .map_err(anyhow::Error::from) + .and_then(|repo| Ok(repo.pristine().read_txn()?.get_view(&view)?)) + { + Ok(Some(_)) => {} + Ok(None) => return error("view", format!("no view '{view}'")), + Err(e) => return error("repository", e.to_string()), + } + let endpoint = match owner.remote_endpoint().await { + Ok(endpoint) => endpoint, + Err(e) => return error("remote", format!("{e:#}")), + }; + let (token, grant) = match owner.tokens.mint(&view, acting_as, ttl_secs) { + Ok(minted) => minted, + Err(e) => return error("sandbox-token", e.to_string()), + }; + OwnerResponse::SandboxOpened { + remote: super::remote::pointer_addr(endpoint, super::remote::offline()).await, + view, + token, + expires: grant.expires.to_rfc3339(), + } + } + OwnerRequest::RenewSandbox { view, ttl_secs } => { + match owner.tokens.renew(&view, ttl_secs) { + Ok(expires) => OwnerResponse::SandboxRenewed { + expires: expires.to_rfc3339(), + }, + Err(e) => error("sandbox-token", e.to_string()), + } + } + OwnerRequest::CloseSandbox { view } => OwnerResponse::SandboxClosed { + revoked: owner.tokens.revoke(&view), + }, + other => error("internal", format!("not a sandbox request: {other:?}")), + } +} + +/// `Materialize`: every entry of the view as its own frame, in path order, +/// then `Materialized`. The one request answered with more than one frame. +pub(crate) async fn materialize( + owner: &OwnerState, + grant: Option, + frame: RequestFrame, + stream: &mut S, +) -> anyhow::Result<()> +where + S: AsyncWrite + Unpin, +{ + let request_id = frame.request_id; + let respond = |response| ResponseFrame { + version: PROTOCOL_VERSION, + request_id: request_id.clone(), + response, + }; + let view = match (grant, frame.request) { + (Some(grant), _) => grant.view, + (None, OwnerRequest::Materialize { view: Some(view) }) => view, + (None, _) => { + return write_frame( + stream, + &respond(error("view", "name the view to materialize")), + ) + .await; + } + }; + let (tx, mut rx) = tokio::sync::mpsc::channel::(64); + let root = owner.root.clone(); + let render = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + // Writable: listing a view other than the checked-out one projects its + // tree in a transaction that is thrown away. + let repo = Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; + let mut entries = 0u64; + let mut live = std::collections::BTreeMap::new(); + let rendered = repo.materialize_view_entries(&view, |entry| { + entries += 1; + live.insert(entry.inode, entry.path.clone()); + tx.blocking_send(OwnerResponse::MaterializeEntry { entry }) + .map_err(|_| ()) + })?; + match rendered { + Ok(snapshot) => { + let skeleton = Box::new(repo.export_sandbox_skeleton(&view, &live)?); + let _ = tx.blocking_send(OwnerResponse::Materialized { + snapshot, + entries, + skeleton, + }); + Ok(()) + } + Err(()) => Err(anyhow::anyhow!("the client went away")), + } + }); + while let Some(response) = rx.recv().await { + write_frame(stream, &respond(response)).await?; + } + match render.await? { + Ok(()) => Ok(()), + Err(e) => write_frame(stream, &respond(error("materialize", format!("{e:#}")))).await, + } +} + +/// `FileStates` and `SubmitChange`: the view is the token's (remote) or +/// the one named (local). +pub(crate) async fn record_request( + owner: &OwnerState, + grant: Option, + request: OwnerRequest, +) -> OwnerResponse { + let named = match &request { + OwnerRequest::FileStates { view, .. } + | OwnerRequest::Changes { view, .. } + | OwnerRequest::PublishProvenance { view, .. } + | OwnerRequest::SubmitChange { view, .. } => view.clone(), + _ => None, + }; + let Some(view) = grant.map(|g| g.view).or(named) else { + return error("view", "name the view"); + }; + let root = owner.root.clone(); + match request { + OwnerRequest::FileStates { inodes, .. } => { + let read = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + // Wait for the database, don't fail on it. The owner opens it + // per request, so two sandboxes asking at once contend for it + // and the loser has to wait — the same reason the writes below + // wait. Without this a `file-states` arriving while another + // request holds the file fails outright, which under load is + // every sandbox but one. + let repo = + Repository::open_readonly_wait(&root, std::time::Duration::from_secs(30))?; + Ok(repo.export_sandbox_slice(&view, &inodes)?) + }); + match read.await { + Ok(Ok(slice)) => OwnerResponse::FileStates { + slice: Box::new(slice), + }, + Ok(Err(e)) => error("file-states", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } + OwnerRequest::Changes { hashes, .. } => { + let read = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + // Waits for the database for the same reason `FileStates` does. + let repo = + Repository::open_readonly_wait(&root, std::time::Duration::from_secs(30))?; + Ok(repo.export_sandbox_changes(&view, &hashes)?) + }); + match read.await { + Ok(Ok(Ok(changes))) => OwnerResponse::Changes { + changes: changes + .into_iter() + .map(|(hash, bytes)| super::WireChange { hash, bytes }) + .collect(), + }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { + rejection, + skeleton: None, + }, + Ok(Err(e)) => error("changes", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } + OwnerRequest::PublishProvenance { graph, turn, .. } => { + let _one_at_a_time = owner.submissions.lock().await; + let write = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = + Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; + Ok(repo.publish_sandbox_provenance(&view, &graph, turn)?) + }); + match write.await { + Ok(Ok(Ok(publication))) => OwnerResponse::ProvenancePublished { publication }, + Ok(Ok(Err(rejection))) => OwnerResponse::ChangeRefused { + rejection, + skeleton: None, + }, + Ok(Err(e)) => error("provenance", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } + OwnerRequest::SubmitChange { + base_state, + hash, + bytes, + .. + } => { + let _one_at_a_time = owner.submissions.lock().await; + let write = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + let repo = + Repository::open_existing_wait(&root, std::time::Duration::from_secs(30))?; + match repo.insert_submitted_change(&view, &base_state, &hash, &bytes)? { + Ok(submitted) => { + let skeleton = repo.after_sandbox_submit(&view, &submitted.hash)?; + Ok(Ok((submitted, skeleton))) + } + // A stale base is recoverable and the sandbox cannot get + // itself out of it alone — its view row is the stale one. + // Send the view as it is now, so the next record is + // computed against the graph this change was refused for. + Err(rejection @ SubmitRejection::StaleView { .. }) => { + let live = repo.current_sandbox_skeleton(&view)?; + Ok(Err((rejection, Some(live)))) + } + Err(rejection) => Ok(Err((rejection, None))), + } + }); + match write.await { + Ok(Ok(Ok((submitted, skeleton)))) => OwnerResponse::ChangeSubmitted { + submitted, + skeleton: Box::new(skeleton), + }, + Ok(Ok(Err((rejection, skeleton)))) => OwnerResponse::ChangeRefused { + rejection, + skeleton: skeleton.map(Box::new), + }, + Ok(Err(e)) => error("submit", format!("{e:#}")), + Err(e) => error("internal", e.to_string()), + } + } + other => error("internal", format!("not a record request: {other:?}")), + } +} + +/// Where a materialized entry may land: inside `dir`, and nowhere else. +/// +/// `entry.path` arrives on the wire, and the pointer that chose `dir` is itself +/// a file in the tree the agent was handed — so a path that climbs out, or +/// names the pointer or the cache, lets an agent rewrite its own trust anchor +/// or write as the user somewhere it was never given. The parts check rejects +/// anything but plain relative components (no `..`, no root, no empty or `.`); +/// that cannot escape lexically, and the canonicalize catches a *parent* that +/// is a symlink out of the tree, which a parts check cannot see. +fn entry_path(dir: &Path, path: &str) -> anyhow::Result { + let relative = Path::new(path); + anyhow::ensure!(!relative.as_os_str().is_empty(), "entry has an empty path"); + anyhow::ensure!( + relative.is_relative(), + "entry path {path:?} is not relative" + ); + for part in relative.components() { + anyhow::ensure!( + matches!(part, std::path::Component::Normal(_)), + "entry path {path:?} is not a plain relative path", + ); + anyhow::ensure!( + !RESERVED_NAMES + .iter() + .any(|reserved| part.as_os_str() == std::ffi::OsStr::new(reserved)), + "entry path {path:?} names the sandbox's own bookkeeping", + ); + } + Ok(dir.join(relative)) +} + +/// The parent a write lands in, checked to still be inside `tree` after the +/// filesystem has had its say about symlinks. +fn entry_parent<'a>(dir: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> { + let parent = path.parent().unwrap_or(dir); + std::fs::create_dir_all(parent)?; + let real_tree = dir.canonicalize()?; + let real_parent = parent.canonicalize()?; + anyhow::ensure!( + real_parent.starts_with(&real_tree), + "entry parent resolves outside the sandbox", + ); + Ok(parent) +} + +/// Client side of `Materialize`: write the view's tree into `dir` and return +/// how many entries it had. Files already at those paths are overwritten; +/// nothing else in `dir` is touched. Every path is checked by [`entry_path`] +/// first, and every write's parent by [`entry_parent`]. +pub(crate) async fn receive_materialized( + stream: &mut S, + request_id: &str, + dir: &Path, +) -> anyhow::Result<(u64, atomic_repository::SandboxSkeleton)> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut written = 0u64; + loop { + let frame: ResponseFrame = read_frame(stream).await?; + if frame.request_id != request_id { + anyhow::bail!("database owner response request id mismatch"); + } + match frame.response { + OwnerResponse::MaterializeEntry { entry } => { + let path = entry_path(dir, &entry.path)?; + match entry.kind { + ViewEntryKind::Directory => std::fs::create_dir_all(&path)?, + #[cfg(unix)] + ViewEntryKind::Symlink => { + entry_parent(dir, &path)?; + let target = String::from_utf8(entry.content.clone())?; + match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + return Err(e.into()) + } + _ => {} + } + std::os::unix::fs::symlink(target, &path)?; + } + #[cfg(not(unix))] + ViewEntryKind::Symlink => { + entry_parent(dir, &path)?; + std::fs::write(&path, &entry.content)?; + } + ViewEntryKind::File => { + entry_parent(dir, &path)?; + std::fs::write(&path, &entry.content)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + &path, + std::fs::Permissions::from_mode(entry.mode as u32), + )?; + } + } + } + written += 1; + } + OwnerResponse::Materialized { + entries, skeleton, .. + } => { + stream.shutdown().await.ok(); + if entries != written { + anyhow::bail!("owner sent {written} entries but said {entries}"); + } + return Ok((written, *skeleton)); + } + OwnerResponse::Error { code, message } => { + anyhow::bail!("database owner materialize failed [{code}]: {message}") + } + other => anyhow::bail!("unexpected materialize response: {other:?}"), + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use atomic_core::change::ChangeHeader; + use atomic_repository::redb_change_store::ProvenanceCheckpointSource; + use atomic_repository::{InsertOptions, RecordOptions, TrackingOptions}; + + use super::*; + + /// A path off the wire only ever resolves inside the tree, and never + /// onto the pointer or the cache — those two are how the agent picks who + /// it trusts and which database it records into. + #[test] + fn a_materialized_path_stays_inside_the_sandbox() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + + assert_eq!( + entry_path(root, "src/main.rs").unwrap(), + root.join("src/main.rs") + ); + assert_eq!( + entry_path(root, "README.md").unwrap(), + root.join("README.md") + ); + + for path in [ + "", + "/etc/passwd", + "..", + "../elsewhere/x", + "src/../../elsewhere/x", + "./src/main.rs", + "src/../..", + ] { + assert!( + entry_path(root, path).is_err(), + "{path:?} should be refused" + ); + } + + for path in [ + DOT_DIR, + ".atomic/config.toml", + "src/.atomic", + SANDBOX_POINTER, + SANDBOX_CACHE_DIR, + "nested/.atomic-sandbox.d", + ] { + assert!( + entry_path(root, path).is_err(), + "{path:?} should be refused" + ); + } + } + + /// A parts check cannot see a symlinked parent, so the write path checks + /// the parent after the filesystem has resolved it. + #[cfg(unix)] + #[test] + fn a_write_through_a_symlinked_parent_is_refused() { + let outside = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + std::os::unix::fs::symlink(outside.path(), root.join("escape")).unwrap(); + + let path = entry_path(root, "escape/stolen").unwrap(); + assert!(entry_parent(root, &path).is_err()); + + // And an ordinary nested path still works. + let ok = entry_path(root, "src/main.rs").unwrap(); + assert_eq!(entry_parent(root, &ok).unwrap(), root.join("src")); + } + + /// A repository with one change on `dev`, and an owner for it. + fn owner() -> (tempfile::TempDir, Arc, Hash) { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("repo"); + let repo = Repository::init(&root).unwrap(); + std::fs::write(root.join("a.txt"), "a\n").unwrap(); + repo.add("a.txt", TrackingOptions::default()).unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(false); + let outcome = repo.record(ChangeHeader::new("a"), options).unwrap(); + repo.write_recorded(&outcome, InsertOptions::default()) + .unwrap(); + let hash = *outcome.hash(); + drop(repo); + let store = Arc::new(RedbChangeStore::open(dir.path().join("journal.redb")).unwrap()); + (dir, OwnerState::new(store, root.join(".atomic")), hash) + } + + fn frame(token: Option<&str>, request: OwnerRequest) -> RequestFrame { + RequestFrame { + version: PROTOCOL_VERSION, + request_id: "r".into(), + token: token.map(str::to_string), + request, + } + } + + fn prepare(provenance_id: u64, change_hashes: Vec) -> OwnerRequest { + OwnerRequest::PrepareCheckpoint { + provenance_id, + expected_generation: 1, + source: ProvenanceCheckpointSource { + agent_name: "a".into(), + agent_display_name: "A".into(), + agent_vendor: "v".into(), + change_hashes, + previous_provenance: None, + plan_id: None, + ledger_turn_number: 1, + }, + reuse_frozen_changes: false, + now: 0, + } + } + + fn refusal(result: Result, Box>) -> String { + match result.map_err(|e| *e) { + Err(OwnerResponse::Error { code, .. }) => code, + other => panic!("expected a refusal, got {other:?}"), + } + } + + #[test] + fn a_remote_checkpoint_covers_only_its_own_turn_and_its_view_s_changes() { + let (_dir, owner, on_view) = owner(); + let (token, _) = owner.tokens.mint("dev", None, 3600).unwrap(); + owner.tokens.bind_provenance("dev", 7); + let remote = |request| authorize(&owner, Caller::Remote, &frame(Some(&token), request)); + + assert!(remote(prepare(7, vec![on_view])).unwrap().is_some()); + assert_eq!( + refusal(remote(prepare(8, vec![on_view]))), + "forbidden", + "not its turn" + ); + let elsewhere = Hash::of(b"a change on no view"); + assert_eq!( + refusal(remote(prepare(7, vec![elsewhere]))), + "forbidden", + "not its view's change" + ); + + assert_eq!( + refusal(authorize( + &owner, + Caller::Remote, + &frame(None, OwnerRequest::Ping) + )), + "unauthorized" + ); + assert!( + authorize( + &owner, + Caller::Local, + &frame(None, prepare(8, vec![elsewhere])) + ) + .unwrap() + .is_none(), + "a local caller is trusted as before" + ); + } +} diff --git a/atomic-cli/src/commands/diff/command.rs b/atomic-cli/src/commands/diff/command.rs index 5a67eed7..a10dec0c 100644 --- a/atomic-cli/src/commands/diff/command.rs +++ b/atomic-cli/src/commands/diff/command.rs @@ -241,12 +241,20 @@ impl Command for Diff { // Find the repository root let repo_root = find_repository_root()?; - // Open the repository - let repo = crate::commands::open_readonly_repository(&repo_root).map_err(|e| { - CliError::InvalidRepository { - reason: e.to_string(), - } - })?; + // Open the repository. A remote sandbox's cache takes the content it + // is about to compare against, so it opens writable. + let repo = if atomic_repository::remote_cache_root(&repo_root).is_dir() { + let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + repo.hydrate_remote_sandbox() + .map_err(CliError::Repository)?; + repo + } else { + crate::commands::open_readonly_repository(&repo_root).map_err(|e| { + CliError::InvalidRepository { + reason: e.to_string(), + } + })? + }; // Parse algorithm let algorithm = self.parse_algorithm()?; diff --git a/atomic-cli/src/commands/intent/attest.rs b/atomic-cli/src/commands/intent/attest.rs index d879adc0..a374834e 100644 --- a/atomic-cli/src/commands/intent/attest.rs +++ b/atomic-cli/src/commands/intent/attest.rs @@ -4,6 +4,7 @@ use clap::Parser; use serde_json::Value; +use atomic_canonical::proof::prepare_attestation; use atomic_canonical::{lift_and_attest, validate_intent, verify}; use atomic_core::pristine::VaultEntryType; use atomic_identity::IdentityStore; @@ -28,6 +29,20 @@ pub struct IntentAttest { /// Output the attested node as JSON-LD. #[arg(long)] pub json: bool, + + /// Don't sign: print what a signer holding the identity's key elsewhere + /// (a browser, a hardware token, a remote signing service) must sign — + /// `{"document": …, "signingBytes": ""}`. Needs only the + /// identity's public key. Complete with `--signed`. + #[arg(long, conflicts_with = "signed")] + pub prepare: bool, + + /// Record an attestation signed elsewhere: a JSON file holding the + /// attested node (the `--prepare` document with its proof attached). It + /// must be signed by `--identity`'s key and attest the intent as it is + /// now. + #[arg(long, value_name = "PATH")] + pub signed: Option, } impl Command for IntentAttest { @@ -59,7 +74,7 @@ impl Command for IntentAttest { ))); } - // Resolve identity + keypair the way `atomic identity sign` does. + // Resolve the identity the way `atomic identity sign` does. let store = IdentityStore::open_default().map_err(|e| { CliError::Internal(anyhow::anyhow!("Failed to open identity store: {}", e)) })?; @@ -79,20 +94,66 @@ impl Command for IntentAttest { .to_string(), })? }; - let keypair = store.load_keypair(&identity.id, None).map_err(|e| { - CliError::Internal(anyhow::anyhow!( - "Failed to load keypair for '{}': {}", - identity.name, - e - )) - })?; - // Attest: lift + fill attributedTo (from the identity's did:atomic when - // absent) + hash + sign. - let node = lift_and_attest(&inputs.frontmatter, &inputs.body, &identity, &keypair) - .map_err(|e| CliError::InvalidArgument { - message: format!("could not attest intent: {e}"), + // Signing elsewhere, step 1: say what to sign. The same preparation + // `lift_and_attest` does (author, content hash, canonical bytes), with + // no private key involved. + if self.prepare { + let prepared = prepare_attestation(unattested.to_value(), &identity.public_key); + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "document": prepared.value, + "signingBytes": data_encoding::BASE64.encode(&prepared.signing_bytes), + })) + .unwrap() + ); + return Ok(()); + } + + let node = if let Some(path) = &self.signed { + // Signing elsewhere, step 2: the signature must be over exactly + // what `--prepare` produces for this intent now — so a signature + // over a stale or altered intent is refused, not recorded. + let text = std::fs::read_to_string(path).map_err(CliError::Io)?; + let signed: Value = + serde_json::from_str(&text).map_err(|e| CliError::InvalidArgument { + message: format!("{} is not JSON: {e}", path.display()), + })?; + let expected = prepare_attestation(unattested.to_value(), &identity.public_key).value; + let mut unsigned = signed.clone(); + if let Some(obj) = unsigned.as_object_mut() { + obj.remove("proof"); + } + if unsigned != expected { + return Err(CliError::InvalidArgument { + message: format!( + "the signed attestation is not of intent {} as it is now (re-run --prepare)", + self.id + ), + }); + } + serde_json::from_value::(signed).map_err(|e| { + CliError::InvalidArgument { + message: format!("not an attested intent: {e}"), + } + })? + } else { + let keypair = store.load_keypair(&identity.id, None).map_err(|e| { + CliError::Internal(anyhow::anyhow!( + "Failed to load keypair for '{}': {}", + identity.name, + e + )) })?; + // Attest: lift + fill attributedTo (from the identity's + // did:atomic when absent) + hash + sign. + lift_and_attest(&inputs.frontmatter, &inputs.body, &identity, &keypair).map_err( + |e| CliError::InvalidArgument { + message: format!("could not attest intent: {e}"), + }, + )? + }; // Belt-and-suspenders: re-gate the ATTESTED node — proof + attributedTo // must now satisfy the gate. @@ -107,7 +168,7 @@ impl Command for IntentAttest { // Self-check: the proof verifies against the signing key before we // write anything to disk. - verify(&node, &keypair.public).map_err(|e| CliError::InvalidArgument { + verify(&node, &identity.public_key).map_err(|e| CliError::InvalidArgument { message: format!("attested intent failed self-verification: {e}"), })?; diff --git a/atomic-cli/src/commands/log/command.rs b/atomic-cli/src/commands/log/command.rs index 0741d8ae..44354145 100644 --- a/atomic-cli/src/commands/log/command.rs +++ b/atomic-cli/src/commands/log/command.rs @@ -499,6 +499,12 @@ impl Command for Log { fn run(&self) -> CliResult<()> { // Find and open repository let repo_root = find_repository_root()?; + // A remote sandbox's cache fetches the change files it lacks first. + if atomic_repository::remote_cache_root(&repo_root).is_dir() { + let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + repo.fetch_remote_sandbox_changes() + .map_err(CliError::Repository)?; + } let repo = crate::commands::open_readonly_repository(&repo_root).map_err(|e| match e { atomic_repository::RepositoryError::NotFound { path } => CliError::RepositoryNotFound { searched_path: path.into(), diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index 00b41d9c..02e7cc12 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -332,6 +332,8 @@ impl Command for Restore { // Find repository let repo_root = find_repository_root()?; let repo = Repository::open(&repo_root).map_err(CliError::Repository)?; + repo.hydrate_remote_sandbox() + .map_err(CliError::Repository)?; // Compute status once. Restore only touches tracked files, so we skip // the untracked scan, and we reuse this single status for both the diff --git a/atomic-cli/src/commands/sandbox.rs b/atomic-cli/src/commands/sandbox.rs index 8c91949d..1797e161 100644 --- a/atomic-cli/src/commands/sandbox.rs +++ b/atomic-cli/src/commands/sandbox.rs @@ -14,7 +14,16 @@ //! //! ```text //! atomic sandbox create [--dest ] [--view ] +//! atomic sandbox create --remote [--dest ] [--acting-as ] [--ttl ] +//! atomic sandbox materialize +//! atomic sandbox renew [--ttl ] +//! atomic sandbox close //! ``` +//! +//! A **remote** sandbox is for another machine (a VM, say): it holds only a +//! pointer — the repository owner's iroh address, its view, and a token that +//! reaches that view alone — and talks to the owner over the same protocol +//! local hooks use. `materialize`, run inside it, writes the view's tree. use std::path::PathBuf; @@ -22,6 +31,7 @@ use clap::{Parser, Subcommand}; use atomic_repository::{Repository, SealOptions, StageOptions}; +use crate::commands::agent::owner; use crate::commands::{find_repository_root, Command}; use crate::error::{CliError, CliResult}; @@ -50,6 +60,15 @@ pub enum SandboxCommands { /// Produces a single-layer deployable image of the full merged state — /// "run this exact version" anywhere an OCI runtime is available. Seal(Seal), + + /// Inside a remote sandbox: write its view's tree from the owner. + Materialize(Materialize), + + /// Extend a remote sandbox's token. + Renew(Renew), + + /// End a remote sandbox's token now. + Close(Close), } impl Command for Sandbox { @@ -58,6 +77,9 @@ impl Command for Sandbox { SandboxCommands::Create(cmd) => cmd.run(), SandboxCommands::Stage(cmd) => cmd.run(), SandboxCommands::Seal(cmd) => cmd.run(), + SandboxCommands::Materialize(cmd) => cmd.run(), + SandboxCommands::Renew(cmd) => cmd.run(), + SandboxCommands::Close(cmd) => cmd.run(), } } } @@ -93,8 +115,26 @@ pub struct Create { /// view — isolating each agent's history as well as its files. #[arg(long, value_name = "VIEW")] pub from: Option, + + /// Make a remote sandbox: no working tree here, only a pointer (written + /// into `--dest`, or printed) for a machine that reaches this + /// repository's owner over iroh. + #[arg(long)] + pub remote: bool, + + /// The identity the remote sandbox's work is attributed to. + #[arg(long, value_name = "DID", requires = "remote")] + pub acting_as: Option, + + /// How long the remote sandbox's token lasts, in seconds (renew it with + /// `atomic sandbox renew`). + #[arg(long, value_name = "SECS", default_value_t = DEFAULT_TTL_SECS, requires = "remote")] + pub ttl: i64, } +/// Two hours: long enough to outlast a renewal missed or two. +const DEFAULT_TTL_SECS: i64 = 2 * 60 * 60; + impl Create { fn default_dest(repo_root: &std::path::Path, name: &str) -> PathBuf { let repo_name = repo_root @@ -116,7 +156,7 @@ impl Command for Create { .clone() .unwrap_or_else(|| Self::default_dest(&root, &self.name)); - if dest.exists() { + if !self.remote && dest.exists() { return Err(CliError::InvalidArgument { message: format!("destination already exists: {}", dest.display()), }); @@ -138,6 +178,11 @@ impl Command for Create { (v, false) }; + if self.remote { + drop(repo); + return self.create_remote(&root, &view); + } + let count = repo .provision_sandbox(&dest, &view) .map_err(CliError::Repository)?; @@ -162,6 +207,99 @@ impl Command for Create { } } +impl Create { + fn create_remote(&self, root: &std::path::Path, view: &str) -> CliResult<()> { + let opened = owner::open_sandbox(root, view, self.acting_as.clone(), self.ttl) + .map_err(CliError::Internal)?; + let pointer = serde_json::to_vec_pretty(&opened.pointer).map_err(anyhow::Error::from)?; + match &self.dest { + Some(dest) => { + std::fs::create_dir_all(dest).map_err(anyhow::Error::from)?; + let path = dest.join(atomic_repository::SANDBOX_POINTER); + write_private(&path, &pointer)?; + println!("Remote sandbox '{}' created", self.name); + println!(" Pointer: {}", path.display()); + println!(" View: {view}"); + println!(" Expires: {}", opened.expires); + println!(" Next: run `atomic sandbox materialize` there"); + } + None => println!("{}", String::from_utf8_lossy(&pointer)), + } + Ok(()) + } +} + +/// The pointer carries a token: readable by its owner only. +fn write_private(path: &std::path::Path, bytes: &[u8]) -> CliResult<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path).map_err(anyhow::Error::from)?; + file.write_all(bytes).map_err(anyhow::Error::from)?; + Ok(()) +} + +/// Write a remote sandbox's view from its repository's owner. +#[derive(Parser, Debug)] +pub struct Materialize {} + +impl Command for Materialize { + fn run(&self) -> CliResult<()> { + let cwd = std::env::current_dir().map_err(anyhow::Error::from)?; + let (root, entries) = + owner::materialize_remote_sandbox(&cwd).map_err(CliError::Internal)?; + println!("Materialized {entries} entries into {}", root.display()); + Ok(()) + } +} + +/// Extend a remote sandbox's token. +#[derive(Parser, Debug)] +pub struct Renew { + /// The sandbox's view. + #[arg(value_name = "VIEW")] + pub view: String, + + /// New lifetime from now, in seconds. + #[arg(long, value_name = "SECS", default_value_t = DEFAULT_TTL_SECS)] + pub ttl: i64, +} + +impl Command for Renew { + fn run(&self) -> CliResult<()> { + let root = find_repository_root()?; + let expires = + owner::renew_sandbox(&root, &self.view, self.ttl).map_err(CliError::Internal)?; + println!("Sandbox token for '{}' now expires {expires}", self.view); + Ok(()) + } +} + +/// End a remote sandbox's token now. +#[derive(Parser, Debug)] +pub struct Close { + /// The sandbox's view. + #[arg(value_name = "VIEW")] + pub view: String, +} + +impl Command for Close { + fn run(&self) -> CliResult<()> { + let root = find_repository_root()?; + if owner::close_sandbox(&root, &self.view).map_err(CliError::Internal)? { + println!("Sandbox token for '{}' revoked", self.view); + } else { + println!("No live sandbox token for '{}'", self.view); + } + Ok(()) + } +} + /// Stage a view as a layered OCI image (base + delta). #[derive(Parser, Debug)] pub struct Stage { diff --git a/atomic-cli/src/commands/token.rs b/atomic-cli/src/commands/token.rs index 8ff725cb..86124cbf 100644 --- a/atomic-cli/src/commands/token.rs +++ b/atomic-cli/src/commands/token.rs @@ -11,7 +11,8 @@ //! `base64url(header).base64url(claims).base64url(signature)`: //! //! - header: `{"alg":"EdDSA","typ":"JWT","kid":""}` -//! - claims: `{ sub, iat, exp, jti }` (`sub` mirrors the `kid` public key) +//! - claims: `{ sub, aud, iat, exp, jti }` (`sub` mirrors the `kid` public key; +//! `aud` is the server the token is for) //! - signature: `Ed25519_sign(private_key, "header.claims")` //! //! # Acting on behalf of someone (agent identities) @@ -78,6 +79,12 @@ struct Claims { exp: i64, jti: String, + /// The server this token is for (RFC 7519 `aud`): the bare server URL, + /// without a trailing slash. A server that checks it refuses a token + /// minted for somewhere else, so a token leaked from one server can't + /// be replayed at another within its lifetime. + aud: String, + /// RFC 8693 actor claim — present only when an agent is acting. #[serde(skip_serializing_if = "Option::is_none")] act: Option, @@ -160,6 +167,7 @@ fn mint_token(server: &str, identity: &Identity) -> CliResult { let now = Utc::now(); let claims = Claims { sub, + aud: audience(server), iat: now.timestamp(), exp: (now + TOKEN_TTL).timestamp(), jti: Uuid::new_v4().to_string(), @@ -196,10 +204,36 @@ fn mint_token(server: &str, identity: &Identity) -> CliResult { Ok(format!("{signing_input}.{sig_b64}")) } +/// The `aud` for a server URL: scheme and host (and port), no trailing +/// slash, lowercased — so `https://Atomic.Storage/` and +/// `https://atomic.storage` are the same audience. +pub fn audience(server: &str) -> String { + server.trim().trim_end_matches('/').to_ascii_lowercase() +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn a_token_names_the_server_it_is_for() { + assert_eq!( + audience("https://Atomic.Storage/"), + "https://atomic.storage" + ); + let claims = Claims { + sub: "S".into(), + aud: audience("https://atomic.storage"), + iat: 0, + exp: 1, + jti: "j".into(), + act: None, + dlg: None, + }; + let v: serde_json::Value = serde_json::to_value(&claims).unwrap(); + assert_eq!(v["aud"], "https://atomic.storage"); + } + #[test] fn header_is_eddsa_jwt_with_kid() { let header = JwtHeader { @@ -218,6 +252,7 @@ mod tests { fn a_non_delegated_token_omits_the_actor_claims() { let claims = Claims { sub: "ABCDEF".to_string(), + aud: "https://a".to_string(), iat: 0, exp: 1, jti: "j".to_string(), @@ -237,6 +272,7 @@ mod tests { let agent = "AGENTKEY"; let claims = Claims { sub: human.to_string(), + aud: "https://a".to_string(), iat: 0, exp: 1, jti: "j".to_string(), @@ -277,6 +313,7 @@ mod tests { let now = Utc::now(); let claims = Claims { sub: public_key_b32.clone(), + aud: audience("https://atomic.storage"), iat: now.timestamp(), exp: (now + TOKEN_TTL).timestamp(), jti: Uuid::new_v4().to_string(), diff --git a/atomic-cli/src/main.rs b/atomic-cli/src/main.rs index 1cd1e275..1e6cea48 100644 --- a/atomic-cli/src/main.rs +++ b/atomic-cli/src/main.rs @@ -926,6 +926,10 @@ fn main() { // Initialize logging init_logging(); + // A remote sandbox's cache reaches its repository's owner through the + // owner protocol, whichever command (or agent hook) opens it. + atomic_repository::set_remote_sandbox_link(Box::new(commands::agent::owner::OwnerLink)); + // Dynamic shell completion. When invoked in completion mode (the `COMPLETE` // env var is set by the installed shell hook), this emits candidates — // including live view names and change hashes registered on the insert diff --git a/atomic-cli/tests/intent_attest_external_signer_test.rs b/atomic-cli/tests/intent_attest_external_signer_test.rs new file mode 100644 index 00000000..d6871257 --- /dev/null +++ b/atomic-cli/tests/intent_attest_external_signer_test.rs @@ -0,0 +1,185 @@ +//! `atomic intent attest --prepare` / `--signed`: attesting an intent with a +//! key that is not on this machine. The identity here is public-only — as an +//! agent identity is inside a sandbox whose key lives with a signing +//! service — so plain `attest` can't sign; `--prepare` says what to sign, +//! the key holder signs it, and `--signed` checks and records the result. + +use std::path::Path; +use std::process::{Command, Output}; + +use atomic_identity::{Identity, IdentityStore, IdentityType, IdentityUsage, KeyPair}; + +fn atomic(home: &Path, cwd: &Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(args) + .current_dir(cwd) + .env("HOME", home) + .env("ATOMIC_CONFIG_DIR", home.join(".atomic")) + .output() + .unwrap() +} + +fn ok(out: &Output) -> String { + assert!( + out.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +/// A repo with one intent, a default human identity, and a public-only +/// agent identity whose key only the test holds. +fn setup() -> (tempfile::TempDir, tempfile::TempDir, String, KeyPair) { + let home = tempfile::tempdir().unwrap(); + let repo = tempfile::tempdir().unwrap(); + ok(&atomic( + home.path(), + repo.path(), + &[ + "identity", + "new", + "ada", + "--email", + "ada@example.com", + "--set-default", + ], + )); + ok(&atomic(home.path(), repo.path(), &["init"])); + ok(&atomic( + home.path(), + repo.path(), + &["intent", "new", "Add a greeting"], + )); + + let key = KeyPair::generate(); + let agent = Identity::builder("ada+bot") + .identity_type(IdentityType::Agent) + .usage(IdentityUsage::Bot) + .public_key(key.public.clone()) + .build() + .unwrap(); + IdentityStore::open(&home.path().join(".atomic").join("identities")) + .unwrap() + .save(&agent) + .unwrap(); + + let list = ok(&atomic( + home.path(), + repo.path(), + &["intent", "list", "--json"], + )); + let intents: serde_json::Value = serde_json::from_str(&list).unwrap(); + let id = intents + .as_array() + .and_then(|a| a.first()) + .and_then(|i| i.get("id")) + .and_then(|v| v.as_str()) + .expect("one intent") + .to_string(); + (home, repo, id, key) +} + +/// What the key holder does with `--prepare`'s output. +fn sign_elsewhere(prepared: &str, key: &KeyPair) -> serde_json::Value { + let prepared: serde_json::Value = serde_json::from_str(prepared).unwrap(); + let bytes = data_encoding::BASE64 + .decode(prepared["signingBytes"].as_str().unwrap().as_bytes()) + .unwrap(); + let signature = atomic_identity::signing::Signer::new(key).sign(&bytes); + atomic_canonical::proof::attach_proof(prepared["document"].clone(), &key.public, &signature) +} + +#[test] +fn an_intent_is_attested_by_a_key_held_elsewhere() { + let (home, repo, id, key) = setup(); + let (home, repo) = (home.path(), repo.path()); + + // No key here: plain attest can't sign as the agent. + let out = atomic( + home, + repo, + &["intent", "attest", &id, "--identity", "ada+bot"], + ); + assert!(!out.status.success(), "attest without a key must fail"); + + let prepared = ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--prepare", + ], + )); + let signed = sign_elsewhere(&prepared, &key); + let file = home.join("signed.json"); + std::fs::write(&file, signed.to_string()).unwrap(); + ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--signed", + file.to_str().unwrap(), + ], + )); + + let report: serde_json::Value = serde_json::from_str(&ok(&atomic( + home, + repo, + &["intent", "validate", &id, "--json"], + ))) + .unwrap(); + assert_eq!(report["conforms"], true, "{report}"); + assert_eq!( + signed["attributedTo"], + atomic_canonical::did::did_for_public_key(&key.public) + ); +} + +#[test] +fn a_signature_by_another_key_is_refused() { + let (home, repo, id, _key) = setup(); + let (home, repo) = (home.path(), repo.path()); + let prepared = ok(&atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--prepare", + ], + )); + let signed = sign_elsewhere(&prepared, &KeyPair::generate()); + let file = home.join("signed.json"); + std::fs::write(&file, signed.to_string()).unwrap(); + let out = atomic( + home, + repo, + &[ + "intent", + "attest", + &id, + "--identity", + "ada+bot", + "--signed", + file.to_str().unwrap(), + ], + ); + assert!( + !out.status.success(), + "a signature by the wrong key must not be recorded" + ); +} diff --git a/atomic-cli/tests/remote_sandbox_integration_test.rs b/atomic-cli/tests/remote_sandbox_integration_test.rs new file mode 100644 index 00000000..5b6d2ff8 --- /dev/null +++ b/atomic-cli/tests/remote_sandbox_integration_test.rs @@ -0,0 +1,1181 @@ +//! A remote sandbox talks to its repository's database owner over iroh — +//! the same protocol, with every request checked against its token. +//! +//! Runs offline: the owner and the sandbox dial each other directly on +//! loopback (`ATOMIC_OWNER_IROH_OFFLINE`), no relays. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::{Arc, Barrier}; +use std::thread; + +use serde_json::Value; +use tempfile::TempDir; + +fn atomic(cwd: &Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(args) + .arg("--no-color") + .current_dir(cwd) + .env("ATOMIC_OWNER_IROH_OFFLINE", "1") + .output() + .expect("run atomic") +} + +fn ok(output: Output, what: &str) -> String { + assert!( + output.status.success(), + "{what} failed: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +/// How many sandboxes a stress test runs at once. +/// +/// The default is sized for CI, which runs `cargo test --workspace` on three +/// platforms with default thread parallelism, and each sandbox is a process, so +/// the cost is linear. To actually stress it: +/// +/// ```sh +/// SANDBOX_STRESS_SANDBOXES=200 cargo test -p atomic-cli \ +/// --test remote_sandbox_integration_test +/// ``` +/// +/// Measured ceiling, on one 15-core laptop, `several_remote_sandboxes_work_at_once`: +/// +/// | sandboxes | result | +/// |---|---| +/// | 20 | passes, ~12s | +/// | 32 | passes, ~17s | +/// | 64 | passes, ~31s | +/// | 200 | **intermittent** — 1 pass in 3, ~80-95s | +/// +/// The 200 failures are all one thing, in the owner's `Materialize` handler: +/// +/// ```text +/// database owner materialize failed [materialize]: Database already open. Cannot acquire lock. +/// ``` +/// +/// That is `open_existing_wait(30s)` giving up. The owner re-opens the database +/// per request and redb allows one holder per file, so 200 concurrent clients +/// queue on the file lock and the last of them waits out the timeout while the +/// owner streams whole views one client at a time. It is a capacity limit, not a +/// correctness bug — nothing is corrupted, and 20 is nowhere near it. The fix is +/// for the owner to hold one `Pristine` and share it with every request via +/// `open_with_pristine`, releasing it when idle so local commands can still open +/// the repository (which `database_lock_exclusivity_test` shows they must be +/// able to). +fn stress_count() -> usize { + std::env::var("SANDBOX_STRESS_SANDBOXES") + .ok() + .and_then(|v| v.parse().ok()) + .filter(|n| *n > 0) + .unwrap_or(20) +} + +fn failure(output: Output, what: &str) -> String { + assert!(!output.status.success(), "{what} should have failed"); + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// A repository as `atomic init` makes one (vault included), with two +/// recorded files on `dev`. +fn repository(root: &Path) { + ok(atomic(root, &["init"]), "init"); + std::fs::create_dir_all(root.join("src")).unwrap(); + std::fs::write(root.join("README.md"), "hello\n").unwrap(); + std::fs::write(root.join("src/lib.rs"), "pub fn f() {}\n").unwrap(); + ok(atomic(root, &["add", "README.md", "src/lib.rs"]), "add"); + ok(atomic(root, &["record", "-a", "-m", "first"]), "record"); + // No owner is left running from setup. + let _ = atomic( + root, + &["agent", "database-owner", "shutdown", "--repository", "."], + ); +} + +struct Owner<'a>(&'a Path); + +impl Drop for Owner<'_> { + fn drop(&mut self) { + let _ = atomic( + self.0, + &["agent", "database-owner", "shutdown", "--repository", "."], + ); + } +} + +fn reserve(cwd: &Path, session: &str, turn: &str) -> Output { + atomic( + cwd, + &[ + "agent", + "database-owner", + "reserve", + "--repository", + ".", + "--session-id", + session, + "--turn", + turn, + "--now", + "1700000000", + "--json", + ], + ) +} + +#[test] +fn a_remote_sandbox_reaches_its_view_through_the_owner_and_nothing_else() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + + let out = ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--view", + "dev", + "--acting-as", + "did:key:zAgent", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + assert!(out.contains("Remote sandbox 'exp-1' created"), "{out}"); + let pointer_path = vm_dir.join(".atomic-sandbox"); + let pointer: Value = serde_json::from_slice(&std::fs::read(&pointer_path).unwrap()).unwrap(); + assert_eq!(pointer["view"], "dev"); + assert!(pointer["token"].as_str().unwrap().starts_with("ast_")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&pointer_path) + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o077, 0, "the pointer holds a token: owner-only"); + } + + // The view's tree, and nothing of the repository. + let out = ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + assert!(out.contains("Materialized "), "{out}"); + assert_eq!( + std::fs::read_to_string(vm_dir.join("README.md")).unwrap(), + "hello\n" + ); + assert_eq!( + std::fs::read_to_string(vm_dir.join("src/lib.rs")).unwrap(), + "pub fn f() {}\n" + ); + assert!(!vm_dir.join(".atomic").exists()); + + // Provenance goes over the same protocol, bound to the sandbox's view: + // its own sessions, never the host's. + let reserved: Value = + serde_json::from_str(&ok(reserve(&vm_dir, "vm-session", "1"), "remote reserve")).unwrap(); + assert_eq!(reserved["committed"], true); + ok(reserve(host.path(), "host-session", "1"), "local reserve"); + let refused = failure( + reserve(&vm_dir, "host-session", "2"), + "reserving the host's session", + ); + assert!(refused.contains("forbidden"), "{refused}"); + ok( + reserve(&vm_dir, "vm-session", "2"), + "its own session's next turn", + ); + + // Only a local caller administers sandboxes or the owner. + let refused = failure( + atomic( + &vm_dir, + &["agent", "database-owner", "shutdown", "--repository", "."], + ), + "remote shutdown", + ); + assert!(refused.contains("forbidden"), "{refused}"); + + // A tampered token reaches nothing. + let mut forged = pointer.clone(); + forged["token"] = Value::String("ast_forged".into()); + std::fs::write(&pointer_path, serde_json::to_vec(&forged).unwrap()).unwrap(); + let refused = failure(atomic(&vm_dir, &["sandbox", "materialize"]), "forged token"); + assert!(refused.contains("unauthorized"), "{refused}"); + std::fs::write(&pointer_path, serde_json::to_vec(&pointer).unwrap()).unwrap(); + + // Recording in the sandbox: its cache computes the change, the owner + // checks it and applies it to the view. + std::fs::write(vm_dir.join("README.md"), "hello\nfrom the sandbox\n").unwrap(); + std::fs::write(vm_dir.join("src/new.rs"), "pub fn n() {}\n").unwrap(); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "from the sandbox"]), + "record in the sandbox", + ); + let status = ok(atomic(&vm_dir, &["status"]), "status after record"); + assert!(status.contains("working tree clean"), "{status}"); + std::fs::write(vm_dir.join("src/new.rs"), "pub fn n() { 2 }\n").unwrap(); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "again from the sandbox"]), + "second record in the sandbox", + ); + // Reading the recorded state works in the sandbox too: diff against + // the view, restore from it, and the view's history. + std::fs::write( + vm_dir.join("README.md"), + "hello\nfrom the sandbox\nuncommitted\n", + ) + .unwrap(); + let diff = ok(atomic(&vm_dir, &["diff"]), "diff in the sandbox"); + assert!( + diff.contains("+uncommitted") && !diff.contains("+hello"), + "{diff}" + ); + ok( + atomic(&vm_dir, &["restore", "README.md"]), + "restore in the sandbox", + ); + assert_eq!( + std::fs::read_to_string(vm_dir.join("README.md")).unwrap(), + "hello\nfrom the sandbox\n" + ); + let log = ok(atomic(&vm_dir, &["log"]), "log in the sandbox"); + assert!( + log.contains("again from the sandbox") && log.contains("first"), + "{log}" + ); + + let log = ok(atomic(host.path(), &["log"]), "host log"); + assert!( + log.contains("from the sandbox") && log.contains("again from the sandbox"), + "{log}" + ); + + // A second sandbox of the view sees the first one's work. + let other = vm.path().join("other"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-2", + "--remote", + "--view", + "dev", + "--dest", + other.to_str().unwrap(), + ], + ), + "second sandbox", + ); + ok( + atomic(&other, &["sandbox", "materialize"]), + "materialize the second", + ); + assert_eq!( + std::fs::read_to_string(other.join("README.md")).unwrap(), + "hello\nfrom the sandbox\n" + ); + assert_eq!( + std::fs::read_to_string(other.join("src/new.rs")).unwrap(), + "pub fn n() { 2 }\n" + ); + // (Minting for the same view replaced the first sandbox's token.) + std::fs::write( + &pointer_path, + std::fs::read(other.join(".atomic-sandbox")).unwrap(), + ) + .unwrap(); + + // Renewal keeps the same token; closing ends it. + let out = ok( + atomic(host.path(), &["sandbox", "renew", "dev", "--ttl", "600"]), + "renew", + ); + assert!(out.contains("now expires"), "{out}"); + ok( + atomic(&vm_dir, &["sandbox", "materialize"]), + "materialize after renew", + ); + let out = ok(atomic(host.path(), &["sandbox", "close", "dev"]), "close"); + assert!(out.contains("revoked"), "{out}"); + let refused = failure( + atomic(&vm_dir, &["sandbox", "materialize"]), + "closed sandbox", + ); + assert!(refused.contains("unauthorized"), "{refused}"); +} + +/// An absurd `--ttl` used to panic inside the owner's token registry while it +/// held the lock, and every later call on that registry panicked too — so one +/// typo took down every sandbox and `close` until the owner was restarted. It +/// is a refused request, and the owner keeps serving afterwards. +#[test] +fn an_absurd_ttl_is_refused_and_the_owner_keeps_serving() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--view", + "dev", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + + for arg in [ + "--ttl=0", + "--ttl=-1", + "--ttl=-9223372036854775808", + "--ttl=10000000000000000", + ] { + let refused = failure( + atomic(host.path(), &["sandbox", "renew", "dev", arg]), + &format!("renew with {arg}"), + ); + assert!(refused.contains("must last between"), "{arg}: {refused}"); + assert!(!refused.contains("panicked"), "{arg}: {refused}"); + } + + // The token is untouched and the owner still answers. + ok( + atomic(host.path(), &["sandbox", "renew", "dev", "--ttl", "600"]), + "renew after the bad requests", + ); + ok( + atomic(&vm_dir, &["sandbox", "materialize"]), + "materialize after the bad requests", + ); + let out = ok(atomic(host.path(), &["sandbox", "close", "dev"]), "close"); + assert!(out.contains("revoked"), "{out}"); +} + +fn hook(cwd: &Path, verb: &str, payload: Value) -> Output { + use std::io::Write; + let mut child = Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(["agent", "hooks", "sherpa", verb]) + .current_dir(cwd) + .env("ATOMIC_OWNER_IROH_OFFLINE", "1") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("run atomic agent hooks"); + child + .stdin + .take() + .unwrap() + .write_all(payload.to_string().as_bytes()) + .unwrap(); + child.wait_with_output().unwrap() +} + +#[test] +fn an_agent_turn_in_a_remote_sandbox_lands_with_its_provenance() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + // As an agent's work runs: its own draft view, off dev. + "--from", + "dev", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + + let cwd = vm_dir.to_str().unwrap(); + let now = "2026-01-01T00:00:00Z"; + let turn = |n: u32| { + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "model": "m", "provider": "p", + "turn_number": n, "intent_title": "greet the reader", "timestamp": now, + }) + }; + ok( + hook( + &vm_dir, + "session-start", + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "model": "m", "provider": "p", + "turn_number": 0, "timestamp": now, + }), + ), + "session-start", + ); + ok(hook(&vm_dir, "turn-start", turn(1)), "turn-start"); + std::fs::write(vm_dir.join("README.md"), "hello\nreader\n").unwrap(); + ok(hook(&vm_dir, "turn-end", turn(1)), "turn-end"); + ok( + hook( + &vm_dir, + "session-end", + serde_json::json!({ + "session_id": "agent-session", "cwd": cwd, "turn_number": 1, "timestamp": now, + }), + ), + "session-end", + ); + + // The turn's change is on the view, and its provenance is in the + // repository — not only in the sandbox. + let log = ok( + atomic(host.path(), &["log", "--view", "exp-1"]), + "host log of the draft", + ); + assert!(log.contains("greet the reader"), "{log}"); + let dev = ok(atomic(host.path(), &["log"]), "host log of dev"); + assert!( + !dev.contains("greet the reader"), + "only on the draft view: {dev}" + ); + + // An intent written in the sandbox is recorded there and lands too. + ok( + atomic(&vm_dir, &["intent", "new", "Greet readers"]), + "intent new", + ); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "the intent"]), + "record the intent", + ); + // ...and the repository's vault knows it, as it would after a pull. + let intents = ok(atomic(host.path(), &["intent", "list"]), "host intent list"); + assert!(intents.contains("backlog"), "{intents}"); + let session = ok( + atomic(host.path(), &["session", "show", "agent-session"]), + "host session", + ); + assert!(session.contains("Turns: 1"), "{session}"); + assert!( + session.contains("Goal marker: greet the reader"), + "{session}" + ); +} + +/// A sandbox that falls behind must be able to record again. It used not to: +/// the cache's view row was the stale thing, and only a *successful* submit +/// refreshed it — so a `StaleView` refusal left the sandbox unable to land +/// anything ever again, and the recovery the error names ("fetch and record +/// again") did not exist anywhere. The owner now sends the view as it is with +/// the refusal, so the cache recovers from the refusal itself. +#[test] +fn a_sandbox_that_fell_behind_can_still_record() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let vm_dir = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "exp-1", + "--remote", + "--from", + "dev", + "--dest", + vm_dir.to_str().unwrap(), + ], + ), + "sandbox create --remote", + ); + ok(atomic(&vm_dir, &["sandbox", "materialize"]), "materialize"); + + // The sandbox records and it lands. + std::fs::write(vm_dir.join("README.md"), "hello\nsandbox\n").unwrap(); + ok( + atomic(&vm_dir, &["record", "-a", "-m", "from the sandbox"]), + "record in the sandbox", + ); + let log = ok( + atomic(host.path(), &["log", "--view", "exp-1"]), + "host log after the sandbox's record", + ); + assert!(log.contains("from the sandbox"), "{log}"); + + // Move the draft view underneath the sandbox, with no sandbox involved: + // the owner is the user's live repository, so a local record on `dev` and + // an insert into the draft is exactly what a second agent would cause. + std::fs::write(host.path().join("OUT-OF-BAND.md"), "elsewhere\n").unwrap(); + ok(atomic(host.path(), &["add", "OUT-OF-BAND.md"]), "host add"); + ok( + atomic(host.path(), &["record", "-a", "-m", "out of band"]), + "host record", + ); + ok( + atomic( + host.path(), + &["insert", "from-view", "dev", "--to-view", "exp-1"], + ), + "host insert into the draft", + ); + + // The sandbox's next record was computed against the view it no longer + // has, so the owner refuses it — and the refusal carries the current view, + // which the cache takes. + std::fs::write(vm_dir.join("README.md"), "hello\nsandbox\nout of date\n").unwrap(); + let refused = failure( + atomic(&vm_dir, &["record", "-a", "-m", "against the old view"]), + "a record against a view that moved on", + ); + assert!(refused.contains("moved on"), "{refused}"); + + // And that is where the old behaviour ended: the cache's view row was + // still the stale one, so the next record computed the same stale base and + // was refused identically. Now the retry works. + let retried = ok( + atomic(&vm_dir, &["record", "-a", "-m", "after the view moved"]), + "retry in the sandbox after the view moved", + ); + assert!(!retried.contains("moved on"), "{retried}"); + + let log = ok( + atomic(host.path(), &["log", "--view", "exp-1"]), + "host log after the retry", + ); + assert!(log.contains("after the view moved"), "{log}"); + // The refused change is not on the view, under either message. + assert!(!log.contains("against the old view"), "{log}"); +} + +/// Several sandboxes at once against one owner: each on its own draft view, all +/// materializing, publishing provenance and recording at the same time. +/// +/// This is the shape the owner's concurrency has to survive. A view has one +/// live token, so sandboxes cannot share a view and each gets its own draft off +/// `dev`. Everything after that contends: one owner process, one `submissions` +/// mutex held across the insert, a redb writer only one caller may hold, and a +/// database the owner reopens per request. The assertions are all about +/// *separation* — every change on the draft that made it, none on any other — +/// because a shared owner can cross wires without erroring. +#[test] +fn several_remote_sandboxes_work_at_once() { + let sandboxes_count = stress_count(); + + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + + // Padded to a fixed width: "recorded by sb-1" is a substring of + // "recorded by sb-10", and a substring test across a set like this quietly + // passes nothing. Three digits, because 200 sandboxes reach sb-199. + let names: Vec = (0..sandboxes_count).map(|i| format!("sb-{i:03}")).collect(); + let dirs: Vec = names.iter().map(|_| TempDir::new().unwrap()).collect(); + let works: Vec = dirs.iter().map(|d| d.path().join("work")).collect(); + + for (name, work) in names.iter().zip(&works) { + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + name, + "--remote", + "--from", + "dev", + "--dest", + work.to_str().unwrap(), + ], + ), + &format!("sandbox create {name}"), + ); + } + + let gate = Arc::new(Barrier::new(sandboxes_count)); + let mut sandboxes = Vec::new(); + for (i, (name, work)) in names.iter().zip(&works).enumerate() { + let gate = Arc::clone(&gate); + let (name, work) = (name.clone(), work.clone()); + sandboxes.push(thread::spawn(move || { + let now = "2026-01-01T00:00:00Z"; + let session = format!("session-{i:03}"); + let turn = |n: u32| { + serde_json::json!({ + "session_id": session, "cwd": work.to_str().unwrap(), "model": "m", + "provider": "p", "turn_number": n, + "intent_title": format!("work on {name}"), "timestamp": now, + }) + }; + + // From here on they contend: one owner, one database, one writer. + gate.wait(); + + ok( + atomic(&work, &["sandbox", "materialize"]), + &format!("{name} materialize"), + ); + // Each sandbox's own session. A sandbox may only touch its own, so + // this is the owner's session check under load. + ok( + hook( + &work, + "session-start", + serde_json::json!({ + "session_id": session, "cwd": work.to_str().unwrap(), "model": "m", + "provider": "p", "turn_number": 0, "timestamp": now, + }), + ), + &format!("{name} session-start"), + ); + ok( + hook(&work, "turn-start", turn(1)), + &format!("{name} turn-start"), + ); + + // Each edits a file only it touches, so a crossed change shows up + // as another sandbox's content on this draft. + std::fs::write(work.join("README.md"), format!("hello\nfrom {name}\n")).unwrap(); + std::fs::write( + work.join(format!("{name}.txt")), + format!("only {name} wrote this\n"), + ) + .unwrap(); + ok( + atomic(&work, &["add", &format!("{name}.txt")]), + &format!("{name} add"), + ); + ok( + atomic( + &work, + &["record", "-a", "-m", &format!("recorded by {name}")], + ), + &format!("{name} record"), + ); + ok( + hook(&work, "turn-end", turn(1)), + &format!("{name} turn-end"), + ); + ok( + hook( + &work, + "session-end", + serde_json::json!({ + "session_id": session, "cwd": work.to_str().unwrap(), + "turn_number": 1, "timestamp": now, + }), + ), + &format!("{name} session-end"), + ); + session + })); + } + + let sessions: Vec = sandboxes + .into_iter() + .map(|s| s.join().expect("a sandbox thread finished")) + .collect(); + + // Every draft has its own sandbox's change, and its provenance. + for (i, name) in names.iter().enumerate() { + let log = ok( + atomic(host.path(), &["log", "--view", name]), + &format!("host log of {name}"), + ); + assert!( + log.contains(&format!("recorded by {name}")), + "{name}: {log}" + ); + + let shown = ok( + atomic(host.path(), &["session", "show", &sessions[i]]), + &format!("host session {}", sessions[i]), + ); + assert!(shown.contains("Turns: 1"), "{}: {shown}", sessions[i]); + assert!( + shown.contains(&format!("Goal marker: work on {name}")), + "{}: {shown}", + sessions[i] + ); + } + + // ...and nothing of any other sandbox's. This is the assertion that fails + // if a submit were applied to the wrong view. + for name in &names { + let log = ok( + atomic(host.path(), &["log", "--view", name]), + &format!("host log of {name}"), + ); + for other in &names { + if other != name { + assert!( + !log.contains(&format!("recorded by {other}")), + "{name} has {other}'s change: {log}" + ); + } + } + } + + // The shared base is untouched: no draft's work reached `dev`. + let dev = ok(atomic(host.path(), &["log"]), "host log of dev"); + for name in &names { + assert!( + !dev.contains(&format!("recorded by {name}")), + "a draft's work reached dev: {dev}" + ); + } + assert_eq!( + std::fs::read_to_string(host.path().join("README.md")).unwrap(), + "hello\n", + "dev's working tree was rewritten by a sandbox" + ); +} + +/// A second record from each sandbox, on a repository several of them are +/// working on. This is what found the CRDT slice crossing views. +/// +/// A view shares one ambient graph with every other view, so a trunk inherited +/// from a parent carries branches attached by *other* views' changes — a +/// sibling draft editing the same file adds branches to the same trunk. The +/// graph half of a slice was filtered by the view's change set; the CRDT half +/// was not, so each cache received its siblings' branches. A later record reads +/// those as the ids of lines that already exist, names them, and the owner +/// refuses: `node N is not on this view` — correctly, because those nodes +/// really do belong to another view. +/// +/// A single sandbox never saw it, being the only writer of the file, which is +/// why every serial test passed. +#[test] +fn sandboxes_can_each_record_twice() { + let sandboxes_count = stress_count(); + + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + + // Padded to a fixed width: "recorded by sb-1" is a substring of + // "recorded by sb-10", and a substring test across a set like this quietly + // passes nothing. Three digits, because 200 sandboxes reach sb-199. + let names: Vec = (0..sandboxes_count).map(|i| format!("sb-{i:03}")).collect(); + let dirs: Vec = names.iter().map(|_| TempDir::new().unwrap()).collect(); + let works: Vec = dirs.iter().map(|d| d.path().join("work")).collect(); + for (name, work) in names.iter().zip(&works) { + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + name, + "--remote", + "--from", + "dev", + "--dest", + work.to_str().unwrap(), + ], + ), + &format!("sandbox create {name}"), + ); + } + + let gate = Arc::new(Barrier::new(sandboxes_count)); + let mut sandboxes = Vec::new(); + for (name, work) in names.iter().zip(&works) { + let gate = Arc::clone(&gate); + let (name, work) = (name.clone(), work.clone()); + sandboxes.push(thread::spawn(move || { + gate.wait(); + ok( + atomic(&work, &["sandbox", "materialize"]), + &format!("{name} materialize"), + ); + std::fs::write(work.join("README.md"), format!("hello\nfrom {name}\n")).unwrap(); + ok( + atomic(&work, &["record", "-a", "-m", &format!("first by {name}")]), + &format!("{name} first record"), + ); + // The one that fails: the cache mints ids from its stale counter. + std::fs::write( + work.join("README.md"), + format!("hello\nfrom {name}\nmore\n"), + ) + .unwrap(); + ok( + atomic(&work, &["record", "-a", "-m", &format!("second by {name}")]), + &format!("{name} second record"), + ); + })); + } + for s in sandboxes { + s.join().expect("a sandbox thread finished"); + } + + for name in &names { + let log = ok( + atomic(host.path(), &["log", "--view", name]), + &format!("host log of {name}"), + ); + assert!(log.contains(&format!("first by {name}")), "{name}: {log}"); + assert!(log.contains(&format!("second by {name}")), "{name}: {log}"); + } +} + +/// A sandbox and the local repository working the *same* view at the same +/// time, which the `submissions` mutex does not cover: that mutex serializes +/// requests *through the owner*, and a local `atomic record` never goes through +/// it. Both write the view, so one is always computed against a state the other +/// has just replaced. +/// +/// The sandbox loses this race every round — three of three in practice — and +/// retries until it wins, because the owner refuses the stale change *and* +/// sends the view as it is now. Before that, a stale refusal left the sandbox +/// unable to record ever again. Losing the race is not the interesting part; it +/// is not being able to come back from it. +/// +/// The second part is the data loss this found. The owner applies a sandbox's +/// change to a view, and did not write it into a local working tree sitting on +/// that view. `status` then saw the view's new file missing from disk and +/// offered to record it as deleted, and the host's next `record -a` took that +/// offer: +/// +/// ```text +/// $ atomic status +/// On view dev +/// Changes to be recorded: +/// deleted: from-sandbox.txt +/// ``` +/// +/// `from-sandbox.txt` was a file the sandbox had just added and it was on the +/// view the whole time. The host's `record -a` committed that deletion, every +/// round, until the view had both writers' changes in its log and neither +/// writer's files in its content. +#[test] +fn a_sandbox_and_the_local_repository_can_share_a_view() { + const ROUNDS: usize = 3; + const MAX_RETRIES: usize = 12; + + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let work = vm.path().join("work"); + // On `dev` itself, not a draft: this is the shared-view case. + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "shared", + "--remote", + "--view", + "dev", + "--dest", + work.to_str().unwrap(), + ], + ), + "sandbox create", + ); + ok(atomic(&work, &["sandbox", "materialize"]), "materialize"); + + // How many rounds the sandbox lost and had to retry. Not asserted on: + // whether it loses depends on the scheduler and both outcomes are correct. + // What matters is that losing never became permanent. + let mut lost = 0usize; + for round in 0..ROUNDS { + let gate = Arc::new(Barrier::new(2)); + + let sandbox = { + let gate = Arc::clone(&gate); + let work = work.clone(); + thread::spawn(move || { + gate.wait(); + std::fs::write(work.join("from-sandbox.txt"), format!("round {round}\n")).unwrap(); + let mut attempts = 0; + loop { + let add = atomic(&work, &["add", "from-sandbox.txt"]); + assert!( + add.status.success(), + "sandbox add: {}{}", + String::from_utf8_lossy(&add.stdout), + String::from_utf8_lossy(&add.stderr) + ); + let out = atomic( + &work, + &["record", "-a", "-m", &format!("sandbox round {round}")], + ); + let text = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + if out.status.success() { + return attempts; + } + // A stale refusal is the expected outcome of losing the + // race, and the cache has just resynced from it. + assert!( + text.contains("moved on"), + "sandbox round {round} failed for a different reason: {text}" + ); + attempts += 1; + assert!( + attempts < MAX_RETRIES, + "sandbox never got past the moving view in {MAX_RETRIES} tries: {text}" + ); + } + }) + }; + + let local = { + let gate = Arc::clone(&gate); + let host_root = host.path().to_path_buf(); + thread::spawn(move || { + gate.wait(); + std::fs::write(host_root.join("from-host.txt"), format!("round {round}\n")) + .unwrap(); + let out = atomic( + &host_root, + &["record", "-a", "-m", &format!("host round {round}")], + ); + assert!( + out.status.success(), + "host record round {round}: {}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + }) + }; + + let attempts = sandbox.join().expect("sandbox thread finished"); + local.join().expect("local thread finished"); + if attempts > 0 { + lost += 1; + } + } + eprintln!("DIAG rounds the sandbox lost and retried: {lost} of {ROUNDS}"); + + // Both writers' changes are on the view, whatever order they arrived in. + let log = ok(atomic(host.path(), &["log"]), "host log of dev"); + for round in 0..ROUNDS { + assert!( + log.contains(&format!("host round {round}")), + "the host's round {round} is missing: {log}" + ); + assert!( + log.contains(&format!("sandbox round {round}")), + "the sandbox's round {round} is missing: {log}" + ); + } + + // The host's working tree has the file the sandbox added, at the sandbox's + // last round. This is the fix: the owner writes a submitted change's files + // into the working copy on the view it just changed. + assert_eq!( + std::fs::read_to_string(host.path().join("from-sandbox.txt")) + .unwrap_or_else(|e| { panic!("the host's tree is missing the sandbox's file: {e}") }), + format!("round {}\n", ROUNDS - 1), + "the host's tree has the sandbox's file at the wrong content" + ); + + // And the host's own view of its working tree does not offer to delete it. + // Before the fix this read `deleted: from-sandbox.txt`, and the host's next + // `record -a` committed that. + let status = ok(atomic(host.path(), &["status"]), "host status"); + assert!( + !status.contains("deleted: from-sandbox.txt"), + "the host's status offers to delete the sandbox's file: {status}" + ); + + // The view itself has the sandbox's file too, so the host and the view + // agree about what exists. + let check = vm.path().join("check"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "check", + "--remote", + "--view", + "dev", + "--dest", + check.to_str().unwrap(), + ], + ), + "sandbox create for the check", + ); + ok( + atomic(&check, &["sandbox", "materialize"]), + "materialize the check", + ); + assert_eq!( + std::fs::read_to_string(check.join("from-sandbox.txt")) + .unwrap_or_else(|e| { panic!("the sandbox's file is not on the view: {e}") }), + format!("round {}\n", ROUNDS - 1), + ); + + // Known gap, not asserted: `from-host.txt` does not survive this race. A + // change the host has not recorded yet is not in the deferred tree journal, + // and TREE is a projection of that journal — so the owner's insert + // reprojects TREE and drops the host's uncommitted file before the host's + // `record -a` can pick it up. That is a pre-existing property of the + // projection, not of remote sandboxes: any `insert` into the current view + // while a local file is pending does the same. It needs its own fix, and a + // test that isolates it. +} + +/// `triage review` in a sandbox agrees with the repository it serves. +/// +/// A sandbox used to hold its view flattened: shared and parentless, with the +/// whole union written as its own change log. Everything downstream of "what +/// is this view's own work" was then wrong — `triage review` could not even +/// default its target (the view had no parent), counted inherited history as +/// the sandbox's candidates, and reported ORPHAN_CHANGE for changes the +/// repository knew were fine. The verdict and every finding differed from the +/// repository's for the same view at the same merkle. +/// +/// The skeleton now carries the view as itself — scope, parent, own change log +/// — plus each ancestor with its own, so the cache reconstructs the same +/// perspective the repository computes. +#[test] +fn triage_in_a_sandbox_agrees_with_the_repository() { + let host = TempDir::new().unwrap(); + repository(host.path()); + let _owner = Owner(host.path()); + let vm = TempDir::new().unwrap(); + let work = vm.path().join("work"); + ok( + atomic( + host.path(), + &[ + "sandbox", + "create", + "sb", + "--remote", + "--from", + "dev", + "--dest", + work.to_str().unwrap(), + ], + ), + "sandbox create", + ); + ok(atomic(&work, &["sandbox", "materialize"]), "materialize"); + + // The sandbox's own work, on its own draft. + std::fs::write(work.join("README.md"), "hello\nfrom the sandbox\n").unwrap(); + ok( + atomic(&work, &["record", "-a", "-m", "sandbox edit"]), + "record in the sandbox", + ); + + /// The parts of a triage report that must not depend on which side + /// computed it. The ref URN is excluded: it is a hash of the whole + /// worklist and its stability is its own concern. + fn digest(output: &str) -> String { + let report: Value = serde_json::from_str(output).expect("triage json"); + let findings: Vec = report["findings"] + .as_array() + .map(|fs| { + fs.iter() + .map(|f| { + format!( + "{} {} {}", + f["code"].as_str().unwrap_or("?"), + f["severity"].as_str().unwrap_or("?"), + f["message"].as_str().unwrap_or("?") + ) + }) + .collect() + }) + .unwrap_or_default(); + let candidates: Vec = report["changes"] + .as_array() + .map(|cs| { + cs.iter() + .map(|c| { + format!( + "{} {}", + c["id"].as_str().unwrap_or("?"), + c["message"].as_str().unwrap_or("?") + ) + }) + .collect() + }) + .unwrap_or_default(); + format!( + "verdict={} findings=[{}] candidates=[{}]", + report["verdict"].as_str().unwrap_or("?"), + findings.join(" | "), + candidates.join(" | ") + ) + } + + let theirs = ok( + atomic(&work, &["triage", "review", "--json"]), + "triage in the sandbox", + ); + let ours = ok( + atomic( + host.path(), + &["triage", "review", "sb", "--into", "dev", "--json"], + ), + "triage on the repository", + ); + assert_eq!(digest(&theirs), digest(&ours)); + + // And bare `triage review` in the sandbox needs no `--into`: the view has + // its parent, so the default target resolves there too. + let bare = ok( + atomic(&work, &["triage", "review", "--json"]), + "bare triage in the sandbox", + ); + assert_eq!(digest(&bare), digest(&ours)); + + // The candidate is the sandbox's one own change, not the inherited base. + assert!( + digest(&ours).contains("sandbox edit"), + "the sandbox's edit should be the candidate: {}", + digest(&ours) + ); + assert!( + !digest(&ours).contains("first"), + "the inherited base change is not this view's candidate: {}", + digest(&ours) + ); +} diff --git a/atomic-core/src/change/ops.rs b/atomic-core/src/change/ops.rs index aef44ae0..324adfd1 100644 --- a/atomic-core/src/change/ops.rs +++ b/atomic-core/src/change/ops.rs @@ -213,6 +213,65 @@ impl FileOps { &mut self.line_ops } + /// Every existing node these operations name — the changes that created + /// the file, lines and tokens they touch. Ids this change creates carry + /// the placeholder `NodeId::ROOT` until it is applied, and are left out. + /// + /// These ids are applied as they are, so whoever applies a change it did + /// not record checks them (a remote sandbox's change may name only its + /// own view's nodes). + pub fn referenced_node_ids(&self) -> std::collections::BTreeSet { + let mut ids = std::collections::BTreeSet::new(); + let leaf = |op: &LeafOp, ids: &mut std::collections::BTreeSet<_>| match op { + LeafOp::Insert { after, .. } => ids.extend(after.map(|l| l.change_id())), + LeafOp::Delete { leaf } | LeafOp::Replace { leaf, .. } | LeafOp::Restore { leaf } => { + ids.insert(leaf.change_id()); + } + }; + ids.insert(self.trunk_id.change_id()); + match &self.trunk_op { + Some(TrunkOp::Delete { trunk }) + | Some(TrunkOp::Move { trunk, .. }) + | Some(TrunkOp::Undelete { trunk }) => { + ids.insert(trunk.change_id()); + } + Some(TrunkOp::Create { .. }) | None => {} + } + for line in &self.line_ops { + ids.insert(line.branch_id().change_id()); + match line.operation() { + BranchOp::Insert { after, content } => { + ids.extend(after.map(|b| b.change_id())); + content.iter().for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Delete { branch, content } => { + ids.insert(branch.change_id()); + content.iter().for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Modify { + branch, + old_content, + new_content, + } => { + ids.insert(branch.change_id()); + old_content + .iter() + .chain(new_content) + .for_each(|op| leaf(op, &mut ids)); + } + BranchOp::Restore { branch } => { + ids.insert(branch.change_id()); + } + BranchOp::Reparent { branch, new_after } => { + ids.insert(branch.change_id()); + ids.extend(new_after.map(|b| b.change_id())); + } + } + } + ids.remove(&crate::types::NodeId::ROOT); + ids + } + /// Adds a line operation. pub fn add_line_op(&mut self, op: LineOps) { self.line_ops.push(op); diff --git a/atomic-core/src/pristine/mod.rs b/atomic-core/src/pristine/mod.rs index a013e1cf..eb52a072 100644 --- a/atomic-core/src/pristine/mod.rs +++ b/atomic-core/src/pristine/mod.rs @@ -150,6 +150,7 @@ mod error; mod inode_graph; pub mod ontology; +pub mod slice; pub(crate) mod span_index; pub mod tables; mod traits; diff --git a/atomic-core/src/pristine/slice.rs b/atomic-core/src/pristine/slice.rs new file mode 100644 index 00000000..99af15a9 --- /dev/null +++ b/atomic-core/src/pristine/slice.rs @@ -0,0 +1,752 @@ +//! Graph slices: a repository's own pristine rows, copied for a remote +//! sandbox's local cache. +//! +//! A remote sandbox never holds its repository, but `record` has to read the +//! graph to compute a change — and the change it computes carries the +//! repository's internal node ids and inode numbers, so the sandbox must read +//! exactly the rows the repository has. A slice is those rows, byte for byte: +//! +//! - the **skeleton** ([`ReadTxn::export_skeleton`]): the view's tree — +//! paths (as the view renders them), inodes and their positions, +//! directories — and the id tables for every change the view can see. +//! Enough for `status` with no graph at all. +//! - a **graph slice** ([`ReadTxn::export_graph_slice`]): for a set of +//! inodes, the GRAPH / INODE_GRAPH rows of each file's content and of its +//! name chain up to the root, one hop of neighbours (so `find_block` finds +//! what it would find on the repository), the file's CRDT rows, and its +//! recorded conflicts on the view. +//! +//! [`WriteTxn::import_skeleton`] and [`WriteTxn::import_graph_slice`] write +//! them into a fresh pristine; [`WriteTxn::import_view_snapshot`] writes the +//! view itself, flattened and with the repository's own Merkle state. +//! +//! Rows are copied, never re-derived, so a read on the cache returns what the +//! same read returns on the repository for every row the slice covers. +//! +//! Edges are not filtered by view: a vertex's key exists only while it has an +//! edge, and `find_block` must see the same vertices the repository sees. +//! Reads that matter filter by view themselves. Content is another matter — +//! the slice carries none; span bytes travel separately, and only for changes +//! the view can see. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use redb::ReadableTable; +use serde::{Deserialize, Serialize}; + +use crate::crdt::ids::{BranchId, LeafId, TrunkId}; +use crate::crdt::tables::{ + BRANCHES, BRANCH_AFTER, BRANCH_LEAVES, BRANCH_VERTEX, INODE_TRUNK, LEAVES, PATH_TRUNK, TRUNKS, + TRUNK_BRANCHES, VERTEX_BRANCH, +}; +use crate::pristine::error::PristineResult; +use crate::pristine::tables::*; +use crate::pristine::traits::{GraphTxnT, ViewScope, ViewState}; +use crate::pristine::txn::{ReadTxn, WriteTxn}; +use crate::types::{ChangePosition, EdgeFlags, GraphNode, Merkle, NodeId, Position}; + +/// Inodes a remote sandbox allocates for itself (`atomic add`) start here, +/// far above any the repository hands out, so the two never collide. +pub const LOCAL_INODE_FLOOR: u64 = 1 << 62; + +/// An EXTERNAL / INTERNAL / NODE_TYPES row: `(id, hash, node type)`. +pub type IdRow = (u64, [u8; 32], Option); + +/// Raw pristine rows. Every field is one table's rows, key and value exactly +/// as stored. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct GraphSlice { + /// EXTERNAL / INTERNAL / NODE_TYPES: `(id, hash, node type)`. + pub ids: Vec, + /// TREE: `path → inode`. + pub tree: Vec<(String, u64)>, + /// REV_TREE: `inode → path`. + pub rev_tree: Vec<(u64, String)>, + /// INODES (REV_INODES is its inverse): `inode → position`. + pub inodes: Vec<(u64, [u8; 16])>, + /// DIRECTORIES: `inode → flags`. + pub directories: Vec<(u64, u8)>, + /// GRAPH: `vertex → edges`. + pub graph: Vec<([u8; 24], Vec<[u8; 24]>)>, + /// INODE_GRAPH: `(inode, vertex) → edges`. + pub inode_graph: Vec<([u8; 32], Vec<[u8; 24]>)>, + pub crdt: CrdtRows, + /// CONFLICTS for the view: `inode → serialized conflicts`. Keyed by inode + /// alone; the view id is the importer's. + pub conflicts: Vec<(u64, Vec)>, +} + +/// A file's CRDT rows (see `crdt::tables`). +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct CrdtRows { + pub trunks: Vec<([u8; 12], Vec)>, + pub inode_trunk: Vec<(u64, [u8; 12])>, + pub path_trunk: Vec<(String, [u8; 12])>, + pub trunk_branches: Vec<([u8; 12], Vec<[u8; 12]>)>, + pub branches: Vec<([u8; 12], [u8; 24])>, + pub branch_after: Vec<([u8; 12], [u8; 12])>, + pub branch_vertex: Vec<([u8; 12], [u8; 24])>, + pub vertex_branch: Vec<([u8; 24], [u8; 12])>, + pub branch_leaves: Vec<([u8; 12], Vec<[u8; 12]>)>, + pub leaves: Vec<([u8; 12], [u8; 22])>, +} + +/// A view as a remote sandbox sees it: its name, the repository's Merkle +/// state and change count for it, its scope and parent, and the ids of the +/// changes **it recorded itself**, in application order. +/// +/// The scope and parent are not decoration. A cache reconstructs the view's +/// effective perspective from them — its own changes, its draft ancestors', +/// and the nearest shared ancestor's — exactly as the repository does. A +/// snapshot that flattened the view to a shared parentless one made the +/// cache's own log the whole union instead, so anything comparing views +/// (`triage review`, `diff_views`) counted inherited history as this view's +/// work. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ViewSnapshotRows { + pub id: u64, + pub name: String, + pub state: Merkle, + pub change_count: u64, + pub kind: ViewScope, + pub parent: Option, + /// In application order. + pub visible: Vec, +} + +fn change_of_edge(edge: &[u8; 24]) -> (u64, u64) { + ( + u64::from_le_bytes(edge[8..16].try_into().unwrap()), + u64::from_le_bytes(edge[16..24].try_into().unwrap()), + ) +} + +fn edge_flags(edge: &[u8; 24]) -> EdgeFlags { + let flag_and_pos = u64::from_le_bytes(edge[0..8].try_into().unwrap()); + EdgeFlags::from_bits_truncate((flag_and_pos >> 56) as u8) +} + +fn edge_dest(edge: &[u8; 24]) -> Position { + let flag_and_pos = u64::from_le_bytes(edge[0..8].try_into().unwrap()); + let change = u64::from_le_bytes(edge[8..16].try_into().unwrap()); + Position::new( + NodeId::new(change), + ChangePosition::new(flag_and_pos & ((1 << 56) - 1)), + ) +} + +fn vertex_key(node: GraphNode) -> [u8; 24] { + encode_vertex(node.change.get(), node.start.get(), node.end.get()) +} + +/// How far a traversal expands from a vertex. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Reach { + /// Its row only. + Row, + /// Its row and its name chain towards the root (a name or a directory). + Up, + /// Its row, and every edge onward except into another inode's content + /// (a file's content). + File, +} + +impl ReadTxn { + /// The view's tree and ids. `live` is the view's tree as rendered — + /// every path it has and the inode there, which is what TREE would hold + /// with the view checked out (TREE itself is the checked-out view's). + /// REV_TREE / INODES / DIRECTORIES come for every inode whose position a + /// visible change introduced, and the id rows for every visible change. + pub fn export_skeleton( + &self, + visible: &BTreeSet, + live: &BTreeMap, + ) -> PristineResult { + let mut out = GraphSlice::default(); + let inodes = self.txn.open_table(INODES)?; + let mut kept = BTreeSet::new(); + for row in inodes.iter()? { + let (k, v) = row?; + let (change, _) = decode_position(v.value()); + if change == 0 || visible.contains(&change) { + kept.insert(k.value()); + out.inodes.push((k.value(), *v.value())); + } + } + let mut rev_tree: BTreeMap = BTreeMap::new(); + for row in self.txn.open_table(REV_TREE)?.iter()? { + let (k, v) = row?; + if kept.contains(&k.value()) { + rev_tree.insert(k.value(), v.value().to_string()); + } + } + // Implicit directories render with no inode of their own (0): only + // what the repository tracks goes in the tree. + for (inode, path) in live + .iter() + .filter(|(inode, _)| **inode != 0 && kept.contains(inode)) + { + rev_tree.insert(*inode, path.clone()); + out.tree.push((path.clone(), *inode)); + } + out.rev_tree = rev_tree.into_iter().collect(); + let directories = self.txn.open_table(DIRECTORIES)?; + for row in directories.iter()? { + let (k, v) = row?; + if kept.contains(&k.value()) { + out.directories.push((k.value(), v.value())); + } + } + out.ids = self.id_rows(visible.iter().copied())?; + Ok(out) + } + + /// The rows `record` reads for `inodes`, on the view `view_id`. + /// `visible` is the view's change set, and it bounds the CRDT rows as well + /// as the graph: see [`ReadTxn::export_crdt`]. + pub fn export_graph_slice( + &self, + inodes: &[u64], + view_id: u64, + visible: &BTreeSet, + ) -> PristineResult { + let mut out = GraphSlice::default(); + let inode_table = self.txn.open_table(INODES)?; + let rev_inodes = self.txn.open_table(REV_INODES)?; + let graph = self.txn.open_multimap_table(GRAPH)?; + let inode_graph = self.txn.open_multimap_table(INODE_GRAPH)?; + let conflicts = self.txn.open_table(CONFLICTS)?; + + let edges_of = |key: &[u8; 24]| -> PristineResult> { + let mut edges = Vec::new(); + for e in graph.get(key)? { + edges.push(*e?.value()); + } + Ok(edges) + }; + let is_inode_vertex = |node: GraphNode| -> PristineResult { + Ok(node.start == node.end + && rev_inodes + .get(&encode_position(node.change.get(), node.start.get()))? + .is_some()) + }; + + // Each vertex once, at the widest reach any path gives it. + let mut reach: BTreeMap<[u8; 24], Reach> = BTreeMap::new(); + let mut queue: VecDeque<(GraphNode, Reach)> = VecDeque::new(); + let enqueue = |node: GraphNode, + r: Reach, + reach: &mut BTreeMap<[u8; 24], Reach>, + queue: &mut VecDeque<(GraphNode, Reach)>| { + if node.is_root() { + return; + } + let key = vertex_key(node); + if reach.get(&key).is_some_and(|have| *have >= r) { + return; + } + reach.insert(key, r); + queue.push_back((node, r)); + }; + + for &inode in inodes { + let Some(pos) = inode_table.get(inode)? else { + continue; + }; + let (change, p) = decode_position(pos.value()); + let seed = GraphNode { + change: NodeId::new(change), + start: ChangePosition::new(p), + end: ChangePosition::new(p), + }; + enqueue(seed, Reach::File, &mut reach, &mut queue); + // INODE_GRAPH may hold content the forward walk doesn't reach. + let lo = encode_inode_vertex(inode, 0, 0, 0); + let hi = encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX); + for row in inode_graph.range::<&[u8; 32]>(&lo..=&hi)? { + let (k, values) = row?; + let key = *k.value(); + let mut edges = Vec::new(); + for e in values { + edges.push(*e?.value()); + } + let (_, c, s, e) = decode_inode_vertex(&key); + out.inode_graph.push((key, edges)); + enqueue( + GraphNode { + change: NodeId::new(c), + start: ChangePosition::new(s), + end: ChangePosition::new(e), + }, + Reach::File, + &mut reach, + &mut queue, + ); + } + if let Some(c) = conflicts.get(&encode_view_seq(view_id, inode))? { + out.conflicts.push((inode, c.value().to_vec())); + } + self.export_crdt(inode, visible, &mut out.crdt)?; + } + + let resolve = |pos: Position, parent: bool| -> Vec> { + let mut found = Vec::new(); + if parent { + if let Ok(n) = self.find_block_end(pos) { + found.push(n); + } + if pos.pos.get() > 0 { + if let Ok(n) = self.find_block(Position::new( + pos.change, + ChangePosition::new(pos.pos.get() - 1), + )) { + found.push(n); + } + } + } else if let Ok(n) = self.find_block(pos) { + found.push(n); + } + found + }; + + while let Some((node, r)) = queue.pop_front() { + if r == Reach::Row { + continue; + } + for edge in edges_of(&vertex_key(node))? { + let flags = edge_flags(&edge); + let parent = flags.contains(EdgeFlags::PARENT); + let folder = flags.contains(EdgeFlags::FOLDER); + for next in resolve(edge_dest(&edge), parent) { + let onward = match r { + Reach::File if parent && folder => Reach::Up, + Reach::File if folder => Reach::Row, + Reach::File if is_inode_vertex(next)? => Reach::Row, + Reach::File => Reach::File, + Reach::Up if parent && folder => Reach::Up, + _ => Reach::Row, + }; + enqueue(next, onward, &mut reach, &mut queue); + } + } + } + + let mut ids = BTreeSet::new(); + for key in reach.keys() { + let edges = edges_of(key)?; + if edges.is_empty() { + continue; + } + ids.insert(decode_vertex(key).0); + for e in &edges { + let (c, by) = change_of_edge(e); + ids.insert(c); + ids.insert(by); + } + out.graph.push((*key, edges)); + } + for (_, edges) in &out.inode_graph { + for e in edges { + let (c, by) = change_of_edge(e); + ids.insert(c); + ids.insert(by); + } + } + for (id, _) in &out.crdt.trunks { + ids.insert(TrunkId::from_bytes(id).change_id().get()); + } + for (id, _) in &out.crdt.branches { + ids.insert(BranchId::from_bytes(id).change_id().get()); + } + for (id, _) in &out.crdt.leaves { + ids.insert(LeafId::from_bytes(id).change_id().get()); + } + ids.remove(&0); + out.ids = self.id_rows(ids.into_iter())?; + Ok(out) + } + + /// The CRDT rows for `inode` that `visible` can see. + /// + /// Every row here is keyed by a trunk/branch/leaf id, and every such id + /// carries the change that created it. A view shares one ambient graph with + /// every other view, so a trunk inherited from a parent has branches + /// attached to it by changes belonging to *other* views — a sibling draft + /// editing the same file adds branches to the same trunk. Sending those + /// would put another view's nodes in this cache, and the next `record` + /// would name them as the ids of lines that already exist. The owner then + /// refuses, correctly: those nodes are not on this view. + /// + /// So filter by the view's change set, exactly as the graph rows beside + /// this are filtered. Without it a sandbox only works while it is the only + /// writer of the file. + fn export_crdt( + &self, + inode: u64, + visible: &BTreeSet, + out: &mut CrdtRows, + ) -> PristineResult<()> { + let on_view = |change_id: NodeId| change_id.is_root() || visible.contains(&change_id.get()); + let inode_trunk = self.txn.open_table(INODE_TRUNK)?; + let Some(trunk) = inode_trunk.get(inode)? else { + return Ok(()); + }; + let trunk = *trunk.value(); + if !on_view(TrunkId::from_bytes(&trunk).change_id()) { + return Ok(()); + } + out.inode_trunk.push((inode, trunk)); + if let Some(t) = self.txn.open_table(TRUNKS)?.get(&trunk)? { + out.trunks.push((trunk, t.value().to_vec())); + } + for row in self.txn.open_table(PATH_TRUNK)?.iter()? { + let (k, v) = row?; + if *v.value() == trunk { + out.path_trunk.push((k.value().to_string(), trunk)); + } + } + let trunk_branches = self.txn.open_multimap_table(TRUNK_BRANCHES)?; + let branches = self.txn.open_table(BRANCHES)?; + let branch_after = self.txn.open_table(BRANCH_AFTER)?; + let branch_vertex = self.txn.open_table(BRANCH_VERTEX)?; + let vertex_branch = self.txn.open_table(VERTEX_BRANCH)?; + let branch_leaves = self.txn.open_multimap_table(BRANCH_LEAVES)?; + let leaves = self.txn.open_table(LEAVES)?; + let mut ids = Vec::new(); + for b in trunk_branches.get(&trunk)? { + let b = *b?.value(); + // A branch from another view's change is not this view's line. + if on_view(BranchId::from_bytes(&b).change_id()) { + ids.push(b); + } + } + for b in &ids { + if let Some(v) = branches.get(b)? { + out.branches.push((*b, *v.value())); + } + if let Some(v) = branch_after.get(b)? { + // `after` may name a branch from another view; shipping it + // would dangle, so it is only kept when it is on this view. + let after = *v.value(); + if on_view(BranchId::from_bytes(&after).change_id()) { + out.branch_after.push((*b, after)); + } + } + if let Some(v) = branch_vertex.get(b)? { + let vertex = *v.value(); + out.branch_vertex.push((*b, vertex)); + if let Some(back) = vertex_branch.get(&vertex)? { + out.vertex_branch.push((vertex, *back.value())); + } + } + let mut ls = Vec::new(); + for l in branch_leaves.get(b)? { + let l = *l?.value(); + // Same for tokens: a leaf from another view's change is not + // this view's token. + if !on_view(LeafId::from_bytes(&l).change_id()) { + continue; + } + if let Some(v) = leaves.get(&l)? { + out.leaves.push((l, *v.value())); + } + ls.push(l); + } + if !ls.is_empty() { + out.branch_leaves.push((*b, ls)); + } + } + out.trunk_branches.push((trunk, ids)); + Ok(()) + } + + /// `IdRow`s for `ids`. An id with no `EXTERNAL` row is dropped: a + /// change can be visible on a view without being registered yet, and + /// callers treat a missing row as "not mine to carry". A drop is still a + /// silent way to lose an id, so a miss in a debug build asserts — a + /// mis-decoded id (see the endianness rule in `tables.rs`) looks exactly + /// like an absent one. + fn id_rows(&self, ids: impl Iterator) -> PristineResult> { + let external = self.txn.open_table(EXTERNAL)?; + let node_types = self.txn.open_table(NODE_TYPES)?; + let mut out = Vec::new(); + for id in ids { + if let Some(h) = external.get(id)? { + let t = node_types.get(id)?.map(|t| t.value()); + out.push((id, *h.value(), t)); + } else { + debug_assert!( + false, + "id {id} has no EXTERNAL row; was it decoded correctly?" + ); + } + } + Ok(out) + } + + /// The view's snapshot: its own state, and every change it can see. + pub fn export_view_snapshot(&self, view: &ViewState, visible: Vec) -> ViewSnapshotRows { + ViewSnapshotRows { + id: view.id, + name: view.name.clone(), + state: view.state, + change_count: view.change_count, + kind: view.kind, + parent: view.parent, + visible, + } + } +} + +impl WriteTxn<'_> { + /// Replace the tree tables with `slice`'s, keeping inodes this cache + /// allocated itself (at or above [`LOCAL_INODE_FLOOR`]); add its ids. + pub fn import_skeleton(&mut self, slice: &GraphSlice) -> PristineResult<()> { + let local = |inode: u64| inode >= LOCAL_INODE_FLOOR; + // A path the cache added itself that the repository now has (its + // change landed) is the repository's inode from here on. + let arrived: std::collections::HashSet<&str> = + slice.tree.iter().map(|(path, _)| path.as_str()).collect(); + let mut superseded = BTreeSet::new(); + for row in self.txn.open_table(REV_TREE)?.iter()? { + let (inode, path) = row?; + if local(inode.value()) && arrived.contains(path.value()) { + superseded.insert(inode.value()); + } + } + let local = |inode: u64| local(inode) && !superseded.contains(&inode); + { + let mut tree = self.txn.open_table(TREE)?; + tree.retain(|_, inode| local(inode))?; + for (path, inode) in &slice.tree { + tree.insert(path.as_str(), *inode)?; + } + } + { + let mut rev_tree = self.txn.open_table(REV_TREE)?; + rev_tree.retain(|inode, _| local(inode))?; + for (inode, path) in &slice.rev_tree { + rev_tree.insert(*inode, path.as_str())?; + } + } + { + let mut inodes = self.txn.open_table(INODES)?; + inodes.retain(|inode, _| local(inode))?; + let mut rev = self.txn.open_table(REV_INODES)?; + rev.retain(|_, inode| local(inode))?; + for (inode, pos) in &slice.inodes { + inodes.insert(*inode, pos)?; + rev.insert(pos, *inode)?; + } + } + { + let mut dirs = self.txn.open_table(DIRECTORIES)?; + dirs.retain(|inode, _| local(inode))?; + for (inode, flags) in &slice.directories { + dirs.insert(*inode, *flags)?; + } + } + self.import_ids(&slice.ids)?; + // From here on this cache's own inodes (`atomic add`) come from above + // the floor, clear of every inode the repository has or will hand out. + self.next_inode + .fetch_max(LOCAL_INODE_FLOOR, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + /// Drop every graph, CRDT and conflict row, then write `slice`'s. + /// Conflicts are written for `view_id`. + pub fn import_graph_slice(&mut self, slice: &GraphSlice, view_id: u64) -> PristineResult<()> { + self.txn.delete_multimap_table(GRAPH)?; + self.txn.delete_multimap_table(INODE_GRAPH)?; + self.txn.delete_table(CONFLICTS)?; + self.txn.delete_table(TRUNKS)?; + self.txn.delete_table(INODE_TRUNK)?; + self.txn.delete_table(PATH_TRUNK)?; + self.txn.delete_multimap_table(TRUNK_BRANCHES)?; + self.txn.delete_table(BRANCHES)?; + self.txn.delete_table(BRANCH_AFTER)?; + self.txn.delete_table(BRANCH_VERTEX)?; + self.txn.delete_table(VERTEX_BRANCH)?; + self.txn.delete_multimap_table(BRANCH_LEAVES)?; + self.txn.delete_table(LEAVES)?; + + { + let mut t = self.txn.open_multimap_table(GRAPH)?; + for (k, edges) in &slice.graph { + for e in edges { + t.insert(k, e)?; + } + } + } + { + let mut t = self.txn.open_multimap_table(INODE_GRAPH)?; + for (k, edges) in &slice.inode_graph { + for e in edges { + t.insert(k, e)?; + } + } + } + { + let mut t = self.txn.open_table(CONFLICTS)?; + for (inode, c) in &slice.conflicts { + t.insert(&encode_view_seq(view_id, *inode), c.as_slice())?; + } + } + let c = &slice.crdt; + { + let mut t = self.txn.open_table(TRUNKS)?; + for (k, v) in &c.trunks { + t.insert(k, v.as_slice())?; + } + let mut t = self.txn.open_table(INODE_TRUNK)?; + for (k, v) in &c.inode_trunk { + t.insert(*k, v)?; + } + let mut t = self.txn.open_table(PATH_TRUNK)?; + for (k, v) in &c.path_trunk { + t.insert(k.as_str(), v)?; + } + let mut t = self.txn.open_multimap_table(TRUNK_BRANCHES)?; + for (k, vs) in &c.trunk_branches { + for v in vs { + t.insert(k, v)?; + } + } + let mut t = self.txn.open_table(BRANCHES)?; + for (k, v) in &c.branches { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(BRANCH_AFTER)?; + for (k, v) in &c.branch_after { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(BRANCH_VERTEX)?; + for (k, v) in &c.branch_vertex { + t.insert(k, v)?; + } + let mut t = self.txn.open_table(VERTEX_BRANCH)?; + for (k, v) in &c.vertex_branch { + t.insert(k, v)?; + } + let mut t = self.txn.open_multimap_table(BRANCH_LEAVES)?; + for (k, vs) in &c.branch_leaves { + for v in vs { + t.insert(k, v)?; + } + } + let mut t = self.txn.open_table(LEAVES)?; + for (k, v) in &c.leaves { + t.insert(k, v)?; + } + } + self.import_ids(&slice.ids) + } + + fn import_ids(&mut self, ids: &[IdRow]) -> PristineResult<()> { + let mut external = self.txn.open_table(EXTERNAL)?; + let mut internal = self.txn.open_table(INTERNAL)?; + let mut types = self.txn.open_table(NODE_TYPES)?; + for (id, hash, t) in ids { + external.insert(*id, hash)?; + internal.insert(hash, *id)?; + if let Some(t) = t { + types.insert(*id, *t)?; + } + } + let max = ids.iter().map(|(id, _, _)| *id).max().unwrap_or(0); + self.next_node_id + .fetch_max(max + 1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + /// Write `snapshot` as a view — its own scope, its own parent, its own + /// change log, with the repository's id, state and change count — + /// replacing any view with that name or id. + /// + /// The parent link matters only if something will follow it: a cache that + /// holds `sb` with `parent: dev`, and a `dev` row with dev's real log, + /// reconstructs the union the repository computes. A snapshot written + /// parentless and shared cannot, which is why `visible` here is the + /// view's *own* changes and the ancestors arrive beside it in the + /// skeleton, not folded into this log. + pub fn import_view_snapshot(&mut self, snapshot: &ViewSnapshotRows) -> PristineResult<()> { + let state = ViewState { + id: snapshot.id, + name: snapshot.name.clone(), + state: snapshot.state, + change_count: snapshot.change_count, + kind: snapshot.kind, + parent: snapshot.parent, + }; + { + let mut views = self.txn.open_table(VIEWS)?; + let mut stale = Vec::new(); + for row in views.iter()? { + let (k, v) = row?; + let existing = crate::pristine::txn::deserialize_view_state(v.value())?; + if existing.id == snapshot.id || k.value() == snapshot.name { + stale.push(k.value().to_string()); + } + } + for name in stale { + views.remove(name.as_str())?; + } + let bytes = crate::pristine::txn::serialize_view_state(&state); + views.insert(snapshot.name.as_str(), bytes.as_slice())?; + } + let lo = encode_view_seq(snapshot.id, 0); + let hi = encode_view_seq(snapshot.id, u64::MAX); + { + let mut log = self.txn.open_table(VIEW_CHANGES)?; + log.retain_in::<&[u8; 16], _>(&lo..=&hi, |_, _| false)?; + for (seq, change) in snapshot.visible.iter().enumerate() { + log.insert(&encode_view_seq(snapshot.id, seq as u64), *change)?; + } + } + { + let mut rev = self.txn.open_table(REV_VIEW_CHANGES)?; + rev.retain_in::<&[u8; 16], _>(&lo..=&hi, |_, _| false)?; + for (seq, change) in snapshot.visible.iter().enumerate() { + rev.insert(&encode_view_seq(snapshot.id, *change), seq as u64)?; + } + } + self.next_view_id + .fetch_max(snapshot.id + 1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The export decodes a CRDT id's change with the type's own `from_bytes`. + /// If the decode and the encoder ever disagree on endianness, the change + /// comes back as a different node id and `id_rows` drops it — the slice + /// still exports, just without the id. This pins the two together. + #[test] + fn crdt_ids_decode_little_endian_change_ids() { + for change in [1u64, 2, 42, 255, 256, 65_536, 1 << 32, u64::MAX >> 8] { + let change = NodeId::new(change); + let trunk = TrunkId::new(change, 7); + let branch = BranchId::new(change, 7); + let leaf = LeafId::new(change, 7); + + assert_eq!(trunk.change_id(), change); + assert_eq!(TrunkId::from_bytes(&trunk.to_bytes()).change_id(), change); + assert_eq!(BranchId::from_bytes(&branch.to_bytes()).change_id(), change); + assert_eq!(LeafId::from_bytes(&leaf.to_bytes()).change_id(), change); + } + } + + /// The first eight bytes are the change id on its own, which is what a + /// hand-rolled `id[0..8]` decode would read. Big-endian gets 42's + /// neighbours wrong; little-endian is what the encoders write. + #[test] + fn crdt_id_change_id_is_little_endian_in_its_first_eight_bytes() { + let bytes = TrunkId::new(NodeId::new(42), 0).to_bytes(); + assert_eq!(&bytes[0..8], &42u64.to_le_bytes()); + assert_ne!(&bytes[0..8], &42u64.to_be_bytes()); + } +} diff --git a/atomic-core/src/pristine/tables.rs b/atomic-core/src/pristine/tables.rs index 80715c7d..5b87dbee 100644 --- a/atomic-core/src/pristine/tables.rs +++ b/atomic-core/src/pristine/tables.rs @@ -840,6 +840,21 @@ pub fn decode_change_file_key(key: &[u8; 36]) -> ([u8; 32], u32) { } // Key Encoding Helpers +// +// Endianness. A key that gets range-scanned is big-endian, so its bytes sort +// in the same order as the numbers they encode — that is what makes a +// `range(lo..=hi)` over these tables mean what it says (e.g. every row for +// one inode: `encode_inode_vertex(inode, 0, 0, 0)` through +// `encode_inode_vertex(inode, u64::MAX, u64::MAX, u64::MAX)`). A value that is +// only ever looked up by its exact key can be little-endian; sorting never +// happens, so byte order costs nothing. That covers the serialized graph edge +// and the CRDT trunk/branch/leaf ids, which are opaque and point-looked-up +// (`crdt::tables`). +// +// The two rules meet in files that handle both, so match the writer rather +// than the neighbours: decode an id with that type's own `from_bytes` +// (`TrunkId`, `BranchId`, `LeafId`) rather than slicing bytes by hand, and the +// two sides cannot drift. /// Encode a span as 24 bytes for use as a graph key #[inline] diff --git a/atomic-core/src/pristine/traits/view.rs b/atomic-core/src/pristine/traits/view.rs index c3f123d4..06806b09 100644 --- a/atomic-core/src/pristine/traits/view.rs +++ b/atomic-core/src/pristine/traits/view.rs @@ -3,9 +3,10 @@ //! Contains `ViewScope`, `ViewState` (view metadata), and `ViewTxnT` //! (the read-only trait for querying views and their change logs). -use crate::types::{Merkle, NodeId}; +use serde::{Deserialize, Serialize}; use crate::pristine::error::PristineError; +use crate::types::{Merkle, NodeId}; use super::graph::GraphTxnT; @@ -44,7 +45,7 @@ use super::graph::GraphTxnT; /// assert_eq!(scope as u8, 1); /// assert!(scope.is_shared()); /// ``` -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, Default)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, Default, Serialize, Deserialize)] #[repr(u8)] pub enum ViewScope { /// Personal workspace (feature, bug, experiment). diff --git a/atomic-core/src/pristine/txn/mod.rs b/atomic-core/src/pristine/txn/mod.rs index 178a2570..bcc62b8c 100644 --- a/atomic-core/src/pristine/txn/mod.rs +++ b/atomic-core/src/pristine/txn/mod.rs @@ -14,6 +14,7 @@ mod read; mod write; pub use helpers::AdjIterator; +pub(crate) use helpers::{deserialize_view_state, serialize_view_state}; pub use pristine::Pristine; pub use read::CachedGraphTxn; pub use read::InodePreloadTxn; diff --git a/atomic-repository/Cargo.toml b/atomic-repository/Cargo.toml index 47bd8d18..307062cc 100644 --- a/atomic-repository/Cargo.toml +++ b/atomic-repository/Cargo.toml @@ -16,6 +16,7 @@ atomic-identity = { workspace = true } atomic-semantic = { workspace = true } serde = { workspace = true } +data-encoding = { workspace = true } serde_json = { workspace = true } toml = { workspace = true } postcard = { workspace = true } diff --git a/atomic-repository/src/changestore/mod.rs b/atomic-repository/src/changestore/mod.rs index 22cc916f..973caf1e 100644 --- a/atomic-repository/src/changestore/mod.rs +++ b/atomic-repository/src/changestore/mod.rs @@ -121,6 +121,32 @@ pub struct ChangeStore { /// We use `RwLock` for thread-safe interior mutability, allowing /// concurrent read access while ensuring exclusive write access. pub(crate) cache: RwLock>, + + /// Content spans held without their change files — a remote sandbox's + /// cache has the bytes of the vertices it was sent, never whole changes. + /// Consulted before the change files. + pub(crate) spans: RwLock, +} + +/// Content bytes by `(change, start)`, each covering `start..end`. +#[derive(Debug, Default)] +pub struct SpanOverlay { + spans: std::collections::BTreeMap<(Hash, u64), (u64, Vec)>, +} + +impl SpanOverlay { + /// The bytes of `start..end` in `hash`'s content, if one held span covers them. + fn copy(&self, hash: &Hash, start: usize, end: usize, buf: &mut [u8]) -> Option { + let (&(h, s), (e, bytes)) = self.spans.range(..=(*hash, start as u64)).next_back()?; + if h != *hash || (end as u64) > *e { + return None; + } + let from = start - s as usize; + let len = end - start; + buf.get_mut(..len)? + .copy_from_slice(bytes.get(from..from + len)?); + Some(len) + } } impl std::fmt::Debug for ChangeStore { @@ -189,6 +215,7 @@ impl ChangeStore { Ok(Self { changes_dir, cache: RwLock::new(LruCache::new(cache_capacity)), + spans: RwLock::new(SpanOverlay::default()), }) } @@ -403,6 +430,21 @@ impl ChangeStore { Ok(change) } + /// Hold `bytes` as `hash`'s content at `start..start + bytes.len()`. + pub fn hold_span(&self, hash: Hash, start: u64, bytes: Vec) { + if let Ok(mut spans) = self.spans.write() { + let end = start + bytes.len() as u64; + spans.spans.insert((hash, start), (end, bytes)); + } + } + + /// Drop every held span. + pub fn clear_spans(&self) { + if let Ok(mut spans) = self.spans.write() { + spans.spans.clear(); + } + } + /// Copy a content span from a change without cloning the full `Change`. /// /// Graph output calls this for every vertex it materializes. Using @@ -416,6 +458,11 @@ impl ChangeStore { end: usize, buf: &mut [u8], ) -> ChangeStoreResult { + if let Ok(spans) = self.spans.read() { + if let Some(n) = spans.copy(hash, start, end, buf) { + return Ok(n); + } + } // Fast path: shared read lock — multiple threads can read concurrently. // peek() doesn't update LRU order, which is an acceptable trade-off // to avoid serializing all readers on a write lock. diff --git a/atomic-repository/src/redb_change_store/provenance.rs b/atomic-repository/src/redb_change_store/provenance.rs index 4644541e..b850bc1e 100644 --- a/atomic-repository/src/redb_change_store/provenance.rs +++ b/atomic-repository/src/redb_change_store/provenance.rs @@ -1,8 +1,8 @@ //! Resumable pending provenance journal stored alongside redb-native changes. use atomic_core::change::session::{ - encode_session_event_key, encode_session_turn_key, session_turn_namespace, SessionEvent, - SessionTurn, + decode_session_turn_key, encode_session_event_key, encode_session_turn_key, + session_turn_namespace, SessionEvent, SessionTurn, }; use atomic_core::pristine::tables; use atomic_core::types::Hash; @@ -369,6 +369,31 @@ impl RedbChangeStore { .transpose() } + /// The highest turn number recorded for `session_id`, if any. + /// + /// A session's turns are one contiguous, turn-ordered run of keys (the + /// namespace is a fixed prefix, the number is big-endian at the end), so + /// the last key in that range is the highest turn. Callers asking "is this + /// turn number taken?" want this, not a walk from turn 1 — the turn number + /// comes off the wire, and a walk to it opens a read transaction per step. + pub fn last_provenance_turn_for(&self, session_id: &str) -> RedbStoreResult> { + let txn = self.db.begin_read()?; + let index = txn.open_table(tables::PROVENANCE_TURN_INDEX)?; + let lo = turn_key(session_id, 0); + let hi = turn_key(session_id, u32::MAX); + let Some(last) = index.range::<&[u8; 40]>(&lo..=&hi)?.next_back() else { + return Ok(None); + }; + let (key, _) = last?; + let (namespace, turn) = decode_session_turn_key(key.value()); + if namespace != session_turn_namespace(session_id) { + return Err(RedbStoreError::Corrupt( + "session-turn index returned a key outside the session's range".to_string(), + )); + } + Ok(Some(turn)) + } + /// Return a reserved turn by external session identity and turn number. pub fn get_provenance_turn_for( &self, diff --git a/atomic-repository/src/redb_change_store/tests.rs b/atomic-repository/src/redb_change_store/tests.rs index 26df1cac..c66468ed 100644 --- a/atomic-repository/src/redb_change_store/tests.rs +++ b/atomic-repository/src/redb_change_store/tests.rs @@ -573,6 +573,49 @@ fn provenance_turn_reservation_is_idempotent_and_persistent() { ); } +#[test] +fn the_last_recorded_turn_says_whether_a_session_was_ever_used() { + let (_dir, store) = temp_store(); + + // A session with no turns at all is unused; the moment one is reserved it + // is not, whichever turn it was. This is what the owner's authz asks + // before letting a sandbox claim a session id, so "somewhere above the + // turn I asked about" must not read as unused. + assert_eq!(store.last_provenance_turn_for("none").unwrap(), None); + + // Reserve out of order, and past the low turns, so ordering matters. + for turn in [7u32, 2, 5] { + store.reserve_provenance_turn("sparse", turn, 0).unwrap(); + } + assert_eq!(store.last_provenance_turn_for("sparse").unwrap(), Some(7)); + assert!(store.last_provenance_turn_for("sparse").unwrap().is_some()); + + // Reserving the lowest turn last is still "used": the last key is the + // highest, not the most recent write. + store.reserve_provenance_turn("sparse", 1, 0).unwrap(); + assert_eq!(store.last_provenance_turn_for("sparse").unwrap(), Some(7)); + + // Turn 0 is a number like any other and counts. + let zero = temp_store(); + zero.1.reserve_provenance_turn("zero", 0, 0).unwrap(); + assert_eq!(zero.1.last_provenance_turn_for("zero").unwrap(), Some(0)); + + // Another session's turns are not this one's, and vice versa — the + // namespace is a hashed prefix, so the range has to stop at it. + store.reserve_provenance_turn("other", 4, 0).unwrap(); + assert_eq!(store.last_provenance_turn_for("other").unwrap(), Some(4)); + assert_eq!(store.last_provenance_turn_for("sparse").unwrap(), Some(7)); + assert_eq!(store.last_provenance_turn_for("nope").unwrap(), None); + + // Turn number at the top of the range, which is what the wire allows. + let big = temp_store(); + big.1.reserve_provenance_turn("big", u32::MAX, 0).unwrap(); + assert_eq!( + big.1.last_provenance_turn_for("big").unwrap(), + Some(u32::MAX) + ); +} + #[test] fn provenance_events_are_ordered_idempotent_and_fenced() { let (_dir, store) = temp_store(); diff --git a/atomic-repository/src/repository/changes.rs b/atomic-repository/src/repository/changes.rs index 7897bb8b..2a695ba8 100644 --- a/atomic-repository/src/repository/changes.rs +++ b/atomic-repository/src/repository/changes.rs @@ -1054,7 +1054,21 @@ impl Repository { /// dependencies, derived session tables, immutable `SESSION_TURNS` append, /// manifest, and `SESSION_HEADS` advance then commit in one pristine /// transaction. Repeating the exact publication is idempotent. + /// + /// In a remote sandbox the checkpoint is published in its repository, + /// through the owner; the result is the repository's. pub fn publish_provenance_checkpoint( + &self, + graph: &atomic_core::change::ProvenanceGraph, + turn: atomic_core::change::session::SessionTurn, + ) -> Result { + if self.is_remote_sandbox() { + return self.publish_remote_provenance_checkpoint(graph, turn); + } + self.publish_local_provenance_checkpoint(graph, turn) + } + + pub(crate) fn publish_local_provenance_checkpoint( &self, graph: &atomic_core::change::ProvenanceGraph, mut turn: atomic_core::change::session::SessionTurn, diff --git a/atomic-repository/src/repository/insert.rs b/atomic-repository/src/repository/insert.rs index 1882aaae..e2bf43d4 100644 --- a/atomic-repository/src/repository/insert.rs +++ b/atomic-repository/src/repository/insert.rs @@ -2154,6 +2154,9 @@ impl Repository { outcome: &RecordOutcome, mut options: InsertOptions, ) -> Result { + if self.is_remote_sandbox() { + return self.submit_recorded(outcome); + } let trace_record = std::env::var_os("ATOMIC_TRACE_RECORD").is_some(); let change = outcome.change(); let hash = outcome.hash(); diff --git a/atomic-repository/src/repository/materialize.rs b/atomic-repository/src/repository/materialize.rs index 3a2916a1..8ef53c65 100644 --- a/atomic-repository/src/repository/materialize.rs +++ b/atomic-repository/src/repository/materialize.rs @@ -545,7 +545,6 @@ impl Repository { use atomic_core::output::repo::{ collect_children, FileOutputOptions, MaterializeOptions, OutputItem, }; - use atomic_core::output::RetrieveOptions; use rayon::prelude::*; use std::collections::HashSet as StdHashSet; @@ -725,82 +724,17 @@ impl Repository { .map(|item| { let file_start = std::time::Instant::now(); - // Build retrieve options with the shared change filter - let retrieve_opts = - RetrieveOptions::default().with_change_filter_arc(change_filter_arc.clone()); - - // Inline the output pipeline so we can trace each phase. - use atomic_core::output::repo::{ - output_graph_content_resolved, resolve_conflicts_semantically, - }; - use atomic_core::output::{compute_order, retrieve_graph, Writer}; - use atomic_core::pristine::InodePreloadTxn; - - // Pre-load ALL edges for this file's inode from INODE_GRAPH - // in a single range scan, then run retrieve_graph over the - // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) - // individual B-tree probes. - let preloaded = InodePreloadTxn::from_table(&txn, item.inode, &inode_graph_table) - .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; - - let t_retrieve = std::time::Instant::now(); - let retrieve_result = retrieve_graph(&preloaded, item.position, retrieve_opts) - .map_err(|e| format!("{}: retrieve: {:?}", item.path, e))?; - - if retrieve_result.graph.is_empty() { - return Ok(None); - } - - let vertices = retrieve_result.graph.len_vertices(); - let edges = retrieve_result.edges_traversed; - let retrieve_ms = t_retrieve.elapsed(); - - let t_order = std::time::Instant::now(); - let mut graph = retrieve_result.graph; - let order = compute_order(&mut graph); - let order_ms = t_order.elapsed(); - - let t_content = std::time::Instant::now(); - let resolved = resolve_conflicts_semantically(&preloaded, store, &graph, &order); - let buffer = Vec::with_capacity(graph.total_bytes()); - let mut writer = Writer::new(buffer); - let hash_fn = |node_id: NodeId| -> Option { - if node_id.is_root() { - return None; - } - preloaded.get_external(node_id).ok().flatten() - }; - output_graph_content_resolved( + let Some(content) = render_view_file( + &txn, store, - hash_fn, - &graph, - &order, - &mut writer, - &resolved, - ) - .map_err(|e| format!("{}: content: {:?}", item.path, e))?; - let content = writer.into_inner(); - let content_ms = t_content.elapsed(); - - if content.is_empty() { + &inode_graph_table, + &change_filter_arc, + &name_conflicts, + item, + trace_mat.then_some(file_start), + )? + else { return Ok(None); - } - - // Name-conflict override (rare): when ≥ 2 inodes are alive at - // this path on the view, replace the single-inode content with - // a marker-wrapped rendering of every side so the conflict is - // surfaced instead of silently collapsed (rubric A12). - let content = match name_conflicts.get(&item.path) { - Some(sides) => render_name_conflict( - &txn, - store, - &inode_graph_table, - &change_filter_arc, - &item.path, - sides, - ) - .unwrap_or(content), - None => content, }; // Detect conflict markers in the materialized bytes. This is @@ -860,24 +794,6 @@ impl Repository { std::fs::write(&abs_path, &content) .map_err(|e| format!("{}: write: {}", item.path, e))?; - if trace_mat { - let elapsed = file_start.elapsed(); - if elapsed > std::time::Duration::from_millis(50) { - eprintln!( - "[materialize] SLOW {} bytes={} vertices={} edges={} \ - retrieve={:?} order={:?} content={:?} total={:?}", - item.path, - bytes_written, - vertices, - edges, - retrieve_ms, - order_ms, - content_ms, - elapsed, - ); - } - } - Ok(Some(( item.path.clone(), bytes_written, @@ -1114,6 +1030,335 @@ impl Repository { } } +/// One entry of a view's tree, rendered in memory by +/// [`Repository::materialize_view_entries`] — enough for a client to write +/// the working tree and a baseline index (`status` against it) without the +/// repository. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ViewEntry { + pub path: String, + pub inode: u64, + pub kind: ViewEntryKind, + /// Unix permission bits as recorded. + pub mode: u16, + /// The file's bytes as a checkout of the view would write them (empty + /// for directories). + #[serde(with = "serde_bytes_vec")] + pub content: Vec, + /// `Hash::of(content)` — the baseline a client compares against. + pub hash: Hash, + /// 1-based line of the first conflict marker, if the file is conflicted. + pub conflict_marker_line: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ViewEntryKind { + File, + Directory, + Symlink, +} + +/// What a view looked like when it was rendered. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ViewSnapshot { + pub view: String, + /// The view's Merkle state, base32: a change recorded against this + /// snapshot applies only while the view is still here. + pub state: String, + pub change_count: u64, +} + +mod serde_bytes_vec { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(v: &[u8], s: S) -> Result { + s.serialize_str(&data_encoding::BASE64.encode(v)) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + data_encoding::BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } +} + +impl Repository { + /// The entries `view_name` has, from TREE as that view projects it. + /// + /// TREE is the checked-out view's projection; another view's structural + /// operations (files it added, moved or deleted) wait in the deferred + /// tree journal until a switch applies them. For another view they are + /// applied here in a write transaction that is thrown away, so the + /// listing is that view's and nothing on disk changes. + /// + /// A read-only handle cannot project, and `TREE` is the *current* view's — + /// so a read-only repository asked about another view is refused rather + /// than answered. Falling through would return the current view's tree + /// under the other view's name, and the caller writes that straight into a + /// remote sandbox. The current view needs no projection, so it reads `TREE` + /// directly and works read-only. + fn view_tree_items( + &self, + txn: &atomic_core::pristine::ReadTxn, + view_name: &str, + options: &atomic_core::output::repo::MaterializeOptions, + ) -> Result, RepositoryError> { + use atomic_core::output::repo::collect_children; + let db = |e: atomic_core::pristine::PristineError| RepositoryError::Database(e.to_string()); + if view_name != self.current_view { + let journal = self.load_deferred_tree_journal()?; + let mut projected = + self.pristine + .write_txn() + .map_err(|e| RepositoryError::InvalidOperation { + message: format!( + "listing view '{view_name}' projects its tree, which needs a writable \ + repository: {e}" + ), + })?; + self.apply_deferred_tree_ops_in_txn(&mut projected, &journal, view_name)?; + let items = collect_children(&projected, Inode::ROOT, "", options).map_err(db)?; + use atomic_core::pristine::MutTxnT; + projected.abort().map_err(db)?; + return Ok(items); + } + collect_children(txn, Inode::ROOT, "", options).map_err(db) + } + + /// Render every entry of `view` and hand each to `sink`, in path order + /// (a directory before what it contains), touching nothing: no working + /// tree, no stat cache, no conflict state. Read-only, so it runs beside + /// other readers — this is what serves a remote sandbox its tree. + /// + /// Files are rendered with the same kernel as + /// [`Repository::materialize_parallel`], in parallel chunks, so memory + /// holds one chunk of contents at a time. + pub fn materialize_view_entries( + &self, + view_name: &str, + mut sink: impl FnMut(ViewEntry) -> Result<(), E>, + ) -> Result, RepositoryError> { + use atomic_core::output::repo::MaterializeOptions; + use atomic_core::types::Base32; + use rayon::prelude::*; + + const CHUNK: usize = 256; + + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let change_filter_arc = Arc::new(collect_visible_change_ids(&txn, &view)?); + let options = MaterializeOptions::new().with_change_filter_arc(change_filter_arc.clone()); + let mut items = self.view_tree_items(&txn, view_name, &options)?; + items.sort_by(|a, b| a.path.cmp(&b.path)); + + let visible = |item: &atomic_core::output::repo::OutputItem| { + item.position.change.is_root() || change_filter_arc.contains(&item.position.change) + }; + let files: Vec<&atomic_core::output::repo::OutputItem> = items + .iter() + .filter(|i| !i.is_directory && visible(i)) + .collect(); + let name_conflicts = collect_name_conflicts( + &txn, + &self.change_store, + &files.iter().map(|i| i.path.as_str()).collect(), + &change_filter_arc, + )?; + for id in change_filter_arc.iter().filter(|id| !id.is_root()) { + if let Ok(Some(hash)) = txn.get_external(*id) { + let _ = self.change_store.load_change(&hash); + } + } + let inode_graph_table = txn + .open_inode_graph_table() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + + // Directories that hold at least one visible file, before their files. + let file_paths: std::collections::HashSet<&str> = + files.iter().map(|i| i.path.as_str()).collect(); + let mut dirs: Vec<&atomic_core::output::repo::OutputItem> = items + .iter() + .filter(|i| i.is_directory) + .filter(|d| { + let prefix = format!("{}/", d.path); + file_paths.iter().any(|p| p.starts_with(&prefix)) + }) + .collect(); + dirs.sort_by(|a, b| a.path.cmp(&b.path)); + for d in dirs { + let entry = ViewEntry { + path: d.path.clone(), + inode: d.inode.get(), + kind: ViewEntryKind::Directory, + mode: d.metadata.permissions, + content: Vec::new(), + hash: Hash::of(&[]), + conflict_marker_line: None, + }; + if let Err(e) = sink(entry) { + return Ok(Err(e)); + } + } + + let store = &self.change_store; + for chunk in files.chunks(CHUNK) { + let rendered: Vec, String>> = chunk + .par_iter() + .map(|item| { + let content = render_view_file( + &txn, + store, + &inode_graph_table, + &change_filter_arc, + &name_conflicts, + item, + None, + )?; + Ok(content.map(|content| ViewEntry { + path: item.path.clone(), + inode: item.inode.get(), + kind: if item.metadata.is_symlink { + ViewEntryKind::Symlink + } else { + ViewEntryKind::File + }, + mode: item.metadata.permissions, + conflict_marker_line: first_conflict_marker_line(&content), + hash: Hash::of(&content), + content, + })) + }) + .collect(); + for r in rendered { + match r { + Ok(Some(entry)) => { + if let Err(e) = sink(entry) { + return Ok(Err(e)); + } + } + Ok(None) => {} + Err(e) => return Err(RepositoryError::Output(e)), + } + } + } + + Ok(Ok(ViewSnapshot { + view: view.name.clone(), + state: view.state.to_base32(), + change_count: view.change_count, + })) + } +} + +/// Render one file of a view in memory: retrieve its graph under the view's +/// change filter, order it, resolve conflicts, and write the bytes a +/// checkout would produce — with a name conflict rendered as markers. The +/// one rendering kernel behind [`Repository::materialize_parallel`] (which +/// writes the result to disk) and [`Repository::materialize_view_entries`] +/// (which hands it to a caller, touching nothing). `Ok(None)`: the file has +/// no content on this view. +fn render_view_file( + txn: &atomic_core::pristine::ReadTxn, + store: &C, + inode_graph_table: &redb::ReadOnlyMultimapTable<&'static [u8; 32], &'static [u8; 24]>, + change_filter_arc: &Arc>, + name_conflicts: &NameConflicts, + item: &atomic_core::output::repo::OutputItem, + trace_from: Option, +) -> Result>, String> { + use atomic_core::output::RetrieveOptions; + // Build retrieve options with the shared change filter + let retrieve_opts = + RetrieveOptions::default().with_change_filter_arc(change_filter_arc.clone()); + + // Inline the output pipeline so we can trace each phase. + use atomic_core::output::repo::{ + output_graph_content_resolved, resolve_conflicts_semantically, + }; + use atomic_core::output::{compute_order, retrieve_graph, Writer}; + use atomic_core::pristine::InodePreloadTxn; + + // Pre-load ALL edges for this file's inode from INODE_GRAPH + // in a single range scan, then run retrieve_graph over the + // in-memory HashMap. O(M) scan + O(1) lookups vs O(V×log N) + // individual B-tree probes. + let preloaded = InodePreloadTxn::from_table(txn, item.inode, inode_graph_table) + .map_err(|e| format!("{}: preload: {:?}", item.path, e))?; + + let t_retrieve = std::time::Instant::now(); + let retrieve_result = retrieve_graph(&preloaded, item.position, retrieve_opts) + .map_err(|e| format!("{}: retrieve: {:?}", item.path, e))?; + + if retrieve_result.graph.is_empty() { + return Ok(None); + } + + let vertices = retrieve_result.graph.len_vertices(); + let edges = retrieve_result.edges_traversed; + let retrieve_ms = t_retrieve.elapsed(); + + let t_order = std::time::Instant::now(); + let mut graph = retrieve_result.graph; + let order = compute_order(&mut graph); + let order_ms = t_order.elapsed(); + + let t_content = std::time::Instant::now(); + let resolved = resolve_conflicts_semantically(&preloaded, store, &graph, &order); + let buffer = Vec::with_capacity(graph.total_bytes()); + let mut writer = Writer::new(buffer); + let hash_fn = |node_id: NodeId| -> Option { + if node_id.is_root() { + return None; + } + preloaded.get_external(node_id).ok().flatten() + }; + output_graph_content_resolved(store, hash_fn, &graph, &order, &mut writer, &resolved) + .map_err(|e| format!("{}: content: {:?}", item.path, e))?; + let content = writer.into_inner(); + let content_ms = t_content.elapsed(); + + if content.is_empty() { + return Ok(None); + } + + // Name-conflict override (rare): when ≥ 2 inodes are alive at + // this path on the view, replace the single-inode content with + // a marker-wrapped rendering of every side so the conflict is + // surfaced instead of silently collapsed (rubric A12). + let content = match name_conflicts.get(&item.path) { + Some(sides) => render_name_conflict( + txn, + store, + inode_graph_table, + change_filter_arc, + &item.path, + sides, + ) + .unwrap_or(content), + None => content, + }; + + if let Some(file_start) = trace_from { + let elapsed = file_start.elapsed(); + if elapsed > std::time::Duration::from_millis(50) { + eprintln!( + "[materialize] SLOW {} vertices={vertices} edges={edges} retrieve={retrieve_ms:?} \ + order={order_ms:?} content={content_ms:?} total={elapsed:?}", + item.path, + ); + } + } + Ok(Some(content)) +} + #[cfg(test)] mod conflict_marker_tests { use super::first_conflict_marker_line; diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 4c98e8fe..2edee331 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -79,6 +79,12 @@ use crate::RepositoryError; mod deferred_tree; mod filter; mod materialize; +pub use materialize::{ViewEntry, ViewEntryKind, ViewSnapshot}; +mod remote_cache; +pub use remote_cache::{ + set_remote_sandbox_link, ChangeFile, RemoteSandboxLink, SandboxSkeleton, SandboxSlice, + SpanBytes, SubmitRejection, Submitted, SubmittedOutcome, +}; mod sandbox; mod semantic_materialize; mod split; @@ -91,7 +97,10 @@ pub use filter::{ collect_view_change_ids, collect_visible_change_ids, collect_visible_change_ids_with_deps, expand_indexed_dependency_closure, view_set_id, }; -pub use sandbox::{SealOptions, SealResult, StageOptions, StageResult, SANDBOX_POINTER}; +pub use sandbox::{ + remote_cache_root, SealOptions, SealResult, StageOptions, StageResult, SANDBOX_CACHE_DIR, + SANDBOX_POINTER, +}; pub use split::{SplitChange, SplitOptions, SplitOutcome}; pub use views::{ManifestApplyOutcome, ViewInfo}; diff --git a/atomic-repository/src/repository/record.rs b/atomic-repository/src/repository/record.rs index 31050715..ffc74ab9 100644 --- a/atomic-repository/src/repository/record.rs +++ b/atomic-repository/src/repository/record.rs @@ -57,6 +57,10 @@ impl Repository { // Build the final header (may get message from options) let final_header = build_header(header, &options); + // A remote sandbox's cache takes the rows it is about to read. + self.hydrate_remote_sandbox() + .map_err(RecordError::Repository)?; + // Get repository status to find modified files let status_t0 = std::time::Instant::now(); let mut status_options = StatusOptions::default(); @@ -1206,6 +1210,9 @@ impl Repository { } } } + // A remote sandbox's change that didn't land isn't recorded + // anywhere: that is a failure, not a warning. + Err(e) if self.is_remote_sandbox() => return Err(RecordError::Repository(e)), Err(e) => { outcome.add_error("apply".to_string(), e.to_string()); } diff --git a/atomic-repository/src/repository/remote_cache.rs b/atomic-repository/src/repository/remote_cache.rs new file mode 100644 index 00000000..76d5d64b --- /dev/null +++ b/atomic-repository/src/repository/remote_cache.rs @@ -0,0 +1,901 @@ +//! A remote sandbox's local cache: the repository's own rows for one view, +//! enough for `status` and `record` without the repository. +//! +//! The serving repository exports ([`Repository::export_sandbox_skeleton`], +//! [`Repository::export_sandbox_slice`]); the sandbox's cache imports +//! ([`Repository::import_sandbox_skeleton`], +//! [`Repository::import_sandbox_slice`]). See `atomic_core::pristine::slice` +//! for what the rows are and why they are copied rather than re-derived. + +use std::collections::BTreeSet; +use std::path::Path; + +use atomic_core::change::ChangeStore as _; +use atomic_core::pristine::slice::{GraphSlice, ViewSnapshotRows}; +use atomic_core::pristine::{decode_vertex, GraphTxnT, MutTxnT, TreeTxnT, ViewTxnT}; +use atomic_core::types::{GraphNode, Hash, NodeId}; +use serde::{Deserialize, Serialize}; + +use super::Repository; +use crate::RepositoryError; + +/// A view's tree for a fresh cache: the skeleton rows, the view itself, and +/// every entry (with content) the view has. +/// +/// `ancestors` carries the views whose changes are in the view's effective +/// perspective — for a draft, its draft ancestors and the nearest shared +/// ancestor — each with its *own* log, real state and real parent. Without +/// them a cache could only hold the view parentless, count inherited history +/// as the view's own work, and invent empty placeholders for views it was +/// told about but never given. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SandboxSkeleton { + pub view: ViewSnapshotRows, + #[serde(default)] + pub ancestors: Vec, + pub rows: GraphSlice, +} + +/// The rows and content `record` reads for some paths. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SandboxSlice { + pub rows: GraphSlice, + pub spans: Vec, +} + +/// Content bytes of one vertex: `change`'s content at `start..start + len`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SpanBytes { + pub change: Hash, + pub start: u64, + #[serde(with = "serde_bytes_b64")] + pub bytes: Vec, +} + +mod serde_bytes_b64 { + use data_encoding::BASE64; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(bytes: &[u8], s: S) -> Result { + s.serialize_str(&BASE64.encode(bytes)) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + let s = String::deserialize(d)?; + BASE64 + .decode(s.as_bytes()) + .map_err(serde::de::Error::custom) + } +} + +/// Why the repository refused a sandbox's change. Nothing is written when +/// a change is refused. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)] +pub enum SubmitRejection { + #[error("the change's bytes hash to {computed}, not {claimed}")] + HashMismatch { claimed: String, computed: String }, + #[error("not a change: {0}")] + Malformed(String), + #[error("the view has moved on (now {current}); fetch and record again")] + StaleView { current: String }, + #[error("change {0} is not on this view")] + ForeignChange(String), + #[error("node {0} is not on this view")] + ForeignNode(u64), + #[error("a change may not touch {0}")] + ForbiddenPath(String), + #[error("change {0} is already in the repository")] + AlreadyPresent(String), +} + +/// A change file: its hash and V3 bytes. +pub type ChangeFile = (Hash, Vec); + +/// A submitted change, applied. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Submitted { + pub hash: Hash, + /// The view's state after it, base32. + pub state: String, +} + +/// What `SubmitChange` came back with: the change landed, or it did not. +pub type SubmittedOutcome = + Result<(Submitted, SandboxSkeleton), (SubmitRejection, Option)>; + +/// Whether `path` may not be recorded, from a sandbox or otherwise. +/// +/// Refused in two cases: the path is not a plain relative path, or it names the +/// repository's own machinery. The first is not a nicety. A submitted change +/// lands in `TREE`, and the next materialize or view switch writes +/// `root.join(path)` — so `../../elsewhere/x` escapes the working tree, with no +/// local user in the loop when a remote token-holder sends it. A reserved-name +/// list cannot catch that, which is why the shape check comes first. +/// +/// An empty path is not a path: a `FileOps` group for a token-level operation +/// names an inode and carries no path, and there is nothing to escape with. +pub(crate) fn forbidden_path(path: &str) -> bool { + if path.is_empty() { + return false; + } + let relative = std::path::Path::new(path); + if relative.is_absolute() { + return true; + } + // `components()` collapses repeated separators, so `src//main.rs` would + // read here as two plain names — and then reach `TREE` as a path no + // materialization can ever put a file at, because the write normalizes it + // back to `src/main.rs`. + if path.contains("//") || path.ends_with('/') { + return true; + } + relative.components().any(|part| match part { + // Not `Normal` means empty, `.`, `..`, or a root/prefix: none of which + // a tracked file may have. + std::path::Component::Normal(name) => { + name == crate::DOT_DIR + || name == super::SANDBOX_POINTER + || name == super::SANDBOX_CACHE_DIR + } + _ => true, + }) +} + +/// How a remote sandbox's cache reaches its repository's owner. The +/// transport lives with whoever runs the process (the `atomic` CLI installs +/// one at startup); with it installed, `record`, `write_recorded` and the +/// readers below work in a remote sandbox for every caller. +pub trait RemoteSandboxLink: Send + Sync { + /// `FileStates`: the rows and content for `inodes`. + fn file_states(&self, root: &Path, inodes: Vec) -> Result; + /// `SubmitChange`: the change recorded against `base_state`; the view's + /// skeleton after it lands. + /// + /// A `StaleView` refusal comes back with the view's current skeleton, so + /// the cache can resync from the refusal itself. Without it a sandbox that + /// fell behind could never record again: its state is the thing that is + /// stale, and the only way to learn the new one is a request the owner has + /// no reason to distinguish from a fresh sandbox. + fn submit( + &self, + root: &Path, + base_state: String, + hash: Hash, + bytes: Vec, + ) -> Result; + /// `Changes`: change files the view has. + fn changes(&self, root: &Path, hashes: Vec) -> Result, String>; + /// `PublishProvenance`: a checkpoint's provenance graph (serialized) and + /// session turn, published in the repository. + fn publish_provenance( + &self, + root: &Path, + graph: Vec, + turn: atomic_core::change::session::SessionTurn, + ) -> Result< + Result, + String, + >; +} + +static LINK: std::sync::OnceLock> = std::sync::OnceLock::new(); + +/// Install the process's link to remote sandbox owners (once). +pub fn set_remote_sandbox_link(link: Box) { + let _ = LINK.set(link); +} + +fn link() -> Result<&'static dyn RemoteSandboxLink, RepositoryError> { + LINK.get() + .map(|l| l.as_ref()) + .ok_or_else(|| RepositoryError::InvalidOperation { + message: "this process can't reach a remote sandbox's owner".to_string(), + }) +} + +fn db(e: impl std::fmt::Display) -> RepositoryError { + RepositoryError::Database(e.to_string()) +} + +/// The views whose changes are in a draft's effective perspective: its draft +/// ancestors, and the nearest shared ancestor past them. A shared view needs +/// nobody — its perspective is its own log. +/// +/// This mirrors the ancestor walk in [`collect_visible_change_ids`] exactly, +/// because the cache reconstructs the union from the chain this returns: if +/// the two ever disagree, the cache's perspective and the rows it was given +/// disagree with it. +fn sandbox_view_chain( + txn: &atomic_core::pristine::ReadTxn, + state: &atomic_core::pristine::ViewState, +) -> Result, RepositoryError> { + if state.kind.is_shared() { + return Ok(Vec::new()); + } + let db = |e: atomic_core::pristine::PristineError| RepositoryError::Database(e.to_string()); + let mut chain = Vec::new(); + for id in txn.resolve_view_chain(state).map_err(db)? { + if id == state.id { + continue; // the view itself is exported separately + } + if let Some(ancestor) = txn.get_view_by_id(id).map_err(db)? { + chain.push(ancestor); + } + } + let mut cursor = state.parent; + while let Some(pid) = cursor { + match txn.get_view_by_id(pid).map_err(db)? { + Some(p) if p.kind.is_shared() => { + chain.push(p); + break; + } + Some(p) => cursor = p.parent, + None => break, + } + } + Ok(chain) +} + +impl Repository { + /// The visible change ids of `view`, sorted. + fn sandbox_visible( + &self, + txn: &atomic_core::pristine::ReadTxn, + view: &atomic_core::pristine::ViewState, + ) -> Result, RepositoryError> { + let mut ids: Vec = super::collect_visible_change_ids(txn, view)? + .into_iter() + .map(|id| id.get()) + .collect(); + ids.sort_unstable(); + Ok(ids) + } + + /// Serve side: `view`'s skeleton, with `live` its rendered tree (inode → + /// path, as [`Repository::materialize_view_entries`] gives them). + /// + /// The view goes out as itself — its scope, its parent, its *own* change + /// log — and its ancestors beside it, each with their own. The rows are + /// still filtered by the union: a cache reconstructs the union from the + /// chain, and this builds that union the same way + /// [`super::collect_visible_change_ids`] does, so what the cache + /// reconstructs is what the rows were filtered by. + pub fn export_sandbox_skeleton( + &self, + view: &str, + live: &std::collections::BTreeMap, + ) -> Result { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let own = super::collect_view_change_ids(&txn, &state)?; + let chain = sandbox_view_chain(&txn, &state)?; + let mut union: BTreeSet = own.iter().map(|id| id.get()).collect(); + for ancestor in &chain { + union.extend( + super::collect_view_change_ids(&txn, ancestor)? + .iter() + .map(|id| id.get()), + ); + } + let own_sorted = own.iter().map(|id| id.get()).collect(); + let rows = txn.export_skeleton(&union, live).map_err(db)?; + Ok(SandboxSkeleton { + view: txn.export_view_snapshot(&state, own_sorted), + ancestors: chain + .iter() + .map(|a| { + let ids = super::collect_view_change_ids(&txn, a)?; + Ok(txn.export_view_snapshot(a, ids.iter().map(|id| id.get()).collect())) + }) + .collect::, RepositoryError>>()?, + rows, + }) + } + + /// Serve side: what `record` on `view` reads for `inodes`, with the + /// content of every vertex a visible change introduced. + pub fn export_sandbox_slice( + &self, + view: &str, + inodes: &[u64], + ) -> Result { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible: BTreeSet = self.sandbox_visible(&txn, &state)?.into_iter().collect(); + // Only files the view has: an inode another view introduced is not + // this sandbox's to read. + let mut on_view = Vec::with_capacity(inodes.len()); + for &inode in inodes { + let position = txn + .inode_position(atomic_core::types::Inode::new(inode)) + .map_err(db)?; + if position.is_some_and(|p| p.change.is_root() || visible.contains(&p.change.get())) { + on_view.push(inode); + } + } + let rows = txn + .export_graph_slice(&on_view, state.id, &visible) + .map_err(db)?; + let mut spans = Vec::new(); + for (key, _) in &rows.graph { + let (change, start, end) = decode_vertex(key); + if start >= end || !visible.contains(&change) { + continue; + } + let Some(hash) = txn.get_external(NodeId::new(change)).map_err(db)? else { + continue; + }; + let node = GraphNode { + change: NodeId::new(change), + start: atomic_core::types::ChangePosition::new(start), + end: atomic_core::types::ChangePosition::new(end), + }; + let mut bytes = vec![0u8; (end - start) as usize]; + self.change_store + .get_contents(|_| Some(hash), node, &mut bytes) + .map_err(db)?; + spans.push(SpanBytes { + change: hash, + start, + bytes, + }); + } + Ok(SandboxSlice { rows, spans }) + } + + /// Serve side: the view's skeleton as it is right now, with nothing of it + /// applied. What a sandbox needs after being told its base state is stale: + /// its own view row is the stale one, so it cannot work out the current + /// state on its own, and asking again for a state it was just refused on + /// is a round trip the owner already has the answer for. + pub fn current_sandbox_skeleton(&self, view: &str) -> Result { + let mut live = std::collections::BTreeMap::new(); + self.materialize_view_entries::<()>(view, |entry| { + live.insert(entry.inode, entry.path.clone()); + Ok(()) + })? + .map_err(|()| RepositoryError::Output("unreachable".to_string()))?; + self.export_sandbox_skeleton(view, &live) + } + + /// Serve side: take a change a remote sandbox recorded on `view` and + /// apply it there — if it is exactly what it claims, recorded against + /// the view as it is now, and names nothing outside the view. + /// + /// The checks, in order: the bytes hash to `hash`; the view is still at + /// `base_state` (otherwise the sandbox fetches and records again); every + /// change it depends on or refers to that the repository knows is + /// visible on the view; every node its file operations name is a visible + /// change's; no path touches `.atomic`, `.atomic-sandbox` or + /// `.atomic-sandbox.d`; and it is new. Then it is saved and inserted + /// into `view`. Callers serialize submissions (one writer). + pub fn insert_submitted_change( + &self, + view: &str, + base_state: &str, + hash: &Hash, + bytes: &[u8], + ) -> Result, RepositoryError> { + use atomic_core::change::format_v3::reader::ChangeReader; + use atomic_core::types::Base32; + + let (change, computed) = match atomic_core::change::Change::deserialize(&mut &bytes[..]) { + Ok(parsed) => parsed, + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + if computed != *hash { + return Ok(Err(SubmitRejection::HashMismatch { + claimed: hash.to_base32(), + computed: computed.to_base32(), + })); + } + let referenced: Vec = match ChangeReader::open(&mut &bytes[..]) { + Ok(reader) => reader + .hash_table() + .hashes() + .iter() + .map(|h| Hash::from(*h)) + .filter(|h| h != hash) + .collect(), + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + + { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let current = state.state.to_base32(); + if current != base_state { + return Ok(Err(SubmitRejection::StaleView { current })); + } + if txn.get_internal(hash).map_err(db)?.is_some() { + return Ok(Err(SubmitRejection::AlreadyPresent(hash.to_base32()))); + } + let visible = super::collect_visible_change_ids(&txn, &state)?; + for dep in change.dependencies() { + match txn.get_internal(dep).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(dep.to_base32()))), + } + } + for other in &referenced { + if let Some(id) = txn.get_internal(other).map_err(db)? { + if !visible.contains(&id) { + return Ok(Err(SubmitRejection::ForeignChange(other.to_base32()))); + } + } + } + for ops in change.file_ops() { + if let Some(id) = ops + .referenced_node_ids() + .into_iter() + .find(|id| !visible.contains(id)) + { + return Ok(Err(SubmitRejection::ForeignNode(id.get()))); + } + if forbidden_path(ops.path()) { + return Ok(Err(SubmitRejection::ForbiddenPath(ops.path().to_string()))); + } + } + for hunk in change.hunks() { + if let Some(path) = hunk.path().filter(|p| forbidden_path(p)) { + return Ok(Err(SubmitRejection::ForbiddenPath(path.to_string()))); + } + } + } + + self.save_change_bytes(hash, bytes, &change)?; + let outcome = match self.insert_change(hash, crate::InsertOptions::default().view(view)) { + Ok(outcome) => outcome, + Err(e) => { + // Nothing of it stays: the change file goes with the failure. + let _ = std::fs::remove_file(self.change_store.change_path(hash)); + return Err(e); + } + }; + // Write the change's files into the working copy, when there is one on + // this view. + // + // `insert_change` is a library function whose working-copy contract is + // "clean up after deletions and moves" — it never writes new files, and + // every other caller remembers to materialize afterwards. This caller + // did not, and a view is not only moved by whoever is sitting on it: a + // sandbox's change lands here through the owner, and the local tree + // checked out on that view was left behind. `status` then saw the + // view's new file missing from disk and reported it as `deleted:`, and + // the next `record -a` committed that deletion — silently undoing the + // sandbox's work, in the repository the sandbox was writing to. + self.materialize_submitted_view(view, hash)?; + Ok(Ok(Submitted { + hash: *hash, + state: outcome.new_state.to_base32(), + })) + } + + /// Bring the working copy on `view` up to date with the change `hash`, + /// touching nothing if `view` is not the checked-out one. + /// + /// Only the paths the change names, as every other insert caller does, and + /// a full materialize when the change names none. A view that is not + /// current is left alone on purpose: the user switches to it to see its + /// files, exactly as a pull into another view does. + fn materialize_submitted_view(&self, view: &str, hash: &Hash) -> Result<(), RepositoryError> { + if view != self.current_view { + return Ok(()); + } + let mut affected = std::collections::HashSet::new(); + if let Ok(change) = self.load_change(hash) { + for op in change.hunks() { + if let Some(path) = op.path() { + affected.insert(path.to_string()); + } + } + } + if affected.is_empty() { + self.materialize()?; + } else { + self.materialize_paths(affected)?; + } + Ok(()) + } + + /// Serve side: the V3 bytes of each of `hashes` — changes `view` can + /// see, and only those (anything else is refused, whole). + pub fn export_sandbox_changes( + &self, + view: &str, + hashes: &[Hash], + ) -> Result, SubmitRejection>, RepositoryError> { + use atomic_core::types::Base32; + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible = super::collect_visible_change_ids(&txn, &state)?; + let mut out = Vec::with_capacity(hashes.len()); + for hash in hashes { + match txn.get_internal(hash).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(hash.to_base32()))), + } + out.push((*hash, std::fs::read(self.change_store.change_path(hash))?)); + } + Ok(Ok(out)) + } + + /// Cache side: the changes the view has that this cache holds no file + /// for. + pub fn missing_sandbox_changes(&self) -> Result, RepositoryError> { + let txn = self.pristine.read_txn().map_err(db)?; + let state = txn + .get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })?; + let mut missing = Vec::new(); + for id in super::collect_visible_change_ids(&txn, &state)? { + if let Some(hash) = txn.get_external(id).map_err(db)? { + if !self.change_store.has_change(&hash) { + missing.push(hash); + } + } + } + Ok(missing) + } + + /// Cache side: keep change files the owner sent — each only if its + /// bytes hash to what it claims. + pub fn hold_sandbox_changes(&self, changes: &[ChangeFile]) -> Result<(), RepositoryError> { + use atomic_core::types::Base32; + for (hash, bytes) in changes { + let (change, computed) = atomic_core::change::Change::deserialize(&mut &bytes[..]) + .map_err(|e| RepositoryError::Database(e.to_string()))?; + if computed != *hash { + return Err(RepositoryError::Database(format!( + "the owner sent {} as {}", + computed.to_base32(), + hash.to_base32() + ))); + } + self.save_change_bytes(hash, bytes, &change)?; + } + Ok(()) + } + + /// In a remote sandbox: load what reading or recording the working + /// tree needs from the owner (see [`Repository::sandbox_slice_inodes`]). + /// Elsewhere, nothing. + pub fn hydrate_remote_sandbox(&self) -> Result<(), RepositoryError> { + if !self.is_remote_sandbox() { + return Ok(()); + } + let inodes = self.sandbox_slice_inodes()?; + let slice = link()? + .file_states(&self.root, inodes) + .map_err(|message| RepositoryError::InvalidOperation { message })?; + self.import_sandbox_slice(&slice) + } + + /// In a remote sandbox: fetch the change files the view has and the + /// cache lacks. Returns how many. Elsewhere, nothing. + pub fn fetch_remote_sandbox_changes(&self) -> Result { + if !self.is_remote_sandbox() { + return Ok(0); + } + let missing = self.missing_sandbox_changes()?; + for batch in missing.chunks(32) { + let changes = link()? + .changes(&self.root, batch.to_vec()) + .map_err(|message| RepositoryError::InvalidOperation { message })?; + self.hold_sandbox_changes(&changes)?; + } + Ok(missing.len()) + } + + /// In a remote sandbox, publishing a checkpoint means publishing it in + /// the repository. + pub(crate) fn publish_remote_provenance_checkpoint( + &self, + graph: &atomic_core::change::ProvenanceGraph, + turn: atomic_core::change::session::SessionTurn, + ) -> Result { + let bytes = graph + .serialize() + .map_err(|e| RepositoryError::Serialization(e.to_string()))?; + link()? + .publish_provenance(&self.root, bytes, turn) + .map_err(|message| RepositoryError::InvalidOperation { message })? + .map_err(|rejection| { + RepositoryError::Apply(format!( + "the repository refused the provenance: {rejection}" + )) + }) + } + + /// In a remote sandbox, what applying a recorded change means: hand it + /// to the owner, and take the view as it is once it lands. + /// + /// A refusal that says the view moved on is not a dead end: the owner + /// sends the view as it is now, and the cache takes it, so the next + /// `record` is computed against the graph this one was refused for. The + /// change that was refused is not applied — the caller has to record it + /// again — and the error says so. + pub(crate) fn submit_recorded( + &self, + outcome: &crate::record::RecordOutcome, + ) -> Result { + let bytes = outcome + .v3_bytes() + .ok_or_else(|| RepositoryError::Apply("the change has no bytes".to_string()))? + .to_vec(); + let base_state = self.remote_sandbox_view_state()?; + let (submitted, skeleton) = match link()? + .submit(&self.root, base_state, *outcome.hash(), bytes) + .map_err(|message| RepositoryError::InvalidOperation { message })? + { + Ok(landed) => landed, + Err((rejection, resync)) => { + if let Some(resync) = resync { + self.import_sandbox_skeleton(&resync)?; + } + return Err(RepositoryError::Apply(format!( + "the repository refused the change: {rejection}" + ))); + } + }; + self.import_sandbox_skeleton(&skeleton)?; + let state = + ::from_base32(submitted.state.as_bytes()) + .ok_or_else(|| RepositoryError::Apply("the owner sent a bad state".to_string()))?; + let mut stats = crate::InsertStats::new(); + stats.changes_applied = 1; + stats.applied_hashes.push(submitted.hash); + Ok(crate::InsertOutcome::new( + state, + skeleton.view.change_count, + false, + stats, + )) + } + + /// Serve side, once a sandbox's change `hash` has landed on `view`: the + /// view's skeleton for the sandbox's cache, and — as a pull does for the + /// files it writes — the vault files the change touched, indexed into + /// the repository's (repository-wide) vault. One render of the view + /// serves both. + pub fn after_sandbox_submit( + &self, + view: &str, + hash: &Hash, + ) -> Result { + let change = self.load_change(hash)?; + let vault_paths: BTreeSet = change + .hunks() + .iter() + .filter_map(|h| h.path()) + .chain(change.file_ops().iter().map(|ops| ops.path())) + .filter(|p| p.starts_with(".vault/") && p.ends_with(".md")) + .map(str::to_string) + .collect(); + let mut live = std::collections::BTreeMap::new(); + let mut files = std::collections::BTreeMap::new(); + self.materialize_view_entries::<()>(view, |entry| { + live.insert(entry.inode, entry.path.clone()); + if vault_paths.contains(&entry.path) { + files.insert( + entry.path, + String::from_utf8_lossy(&entry.content).into_owned(), + ); + } + Ok(()) + })? + .map_err(|()| RepositoryError::Output("unreachable".to_string()))?; + if !vault_paths.is_empty() && self.has_vault()? { + let rows: Vec<(String, Option)> = vault_paths + .iter() + .map(|p| (p[".vault/".len()..].to_string(), files.remove(p))) + .collect(); + self.vault_record_files(&rows)?; + } + self.export_sandbox_skeleton(view, &live) + } + + /// Serve side: publish a remote sandbox's checkpoint — if everything its + /// provenance explains is on `view`. The graph arrives serialized, so + /// what is published is exactly what the sandbox hashed. + pub fn publish_sandbox_provenance( + &self, + view: &str, + graph: &[u8], + turn: atomic_core::change::session::SessionTurn, + ) -> Result< + Result, + RepositoryError, + > { + use atomic_core::types::Base32; + let (graph, _) = match atomic_core::change::ProvenanceGraph::deserialize(graph) { + Ok(parsed) => parsed, + Err(e) => return Ok(Err(SubmitRejection::Malformed(e.to_string()))), + }; + { + let txn = self.pristine.read_txn().map_err(db)?; + let state = + txn.get_view(view) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view.to_string(), + })?; + let visible = super::collect_visible_change_ids(&txn, &state)?; + for change in &graph.changes_explained { + match txn.get_internal(change).map_err(db)? { + Some(id) if visible.contains(&id) => {} + _ => return Ok(Err(SubmitRejection::ForeignChange(change.to_base32()))), + } + } + } + Ok(Ok(self.publish_local_provenance_checkpoint(&graph, turn)?)) + } + + /// Cache side: the view's state as the cache has it (base32) — what a + /// change recorded here is recorded against. + pub fn remote_sandbox_view_state(&self) -> Result { + use atomic_core::types::Base32; + let txn = self.pristine.read_txn().map_err(db)?; + let state = txn + .get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })?; + Ok(state.state.to_base32()) + } + + /// Cache side: replace this cache's tree and view with `skeleton`'s. + pub fn import_sandbox_skeleton( + &self, + skeleton: &SandboxSkeleton, + ) -> Result<(), RepositoryError> { + let mut txn = self.pristine.write_txn().map_err(db)?; + // Ancestors first, so the view's parent link lands on a row that + // exists. Each import replaces any view with that name or id, which is + // what turns a cache's invented placeholder for the parent into the + // real one. + for ancestor in &skeleton.ancestors { + txn.import_view_snapshot(ancestor).map_err(db)?; + } + txn.import_view_snapshot(&skeleton.view).map_err(db)?; + txn.import_skeleton(&skeleton.rows).map_err(db)?; + txn.commit().map_err(db)?; + Ok(()) + } + + /// Cache side: the inodes `record` will read — each tracked file that + /// differs from its baseline, and every tracked directory on the way to + /// any changed or new path (a new file's parent is where it is added). + /// After [`Repository::reindex_working_copy`] on a fresh tree, a clean + /// file costs nothing here. + pub fn sandbox_slice_inodes(&self) -> Result, RepositoryError> { + use crate::status::{FileStatus, StatusOptions}; + use atomic_core::pristine::slice::LOCAL_INODE_FLOOR; + + let status = self.status(StatusOptions::default())?; + let mut out = BTreeSet::new(); + let keep = |inode: atomic_core::types::Inode, out: &mut BTreeSet| { + if inode.get() < LOCAL_INODE_FLOOR { + out.insert(inode.get()); + } + }; + for entry in status.entries() { + if entry.status() == FileStatus::Clean { + continue; + } + if let Some(inode) = entry.inode() { + keep(inode, &mut out); + } else if let Some(inode) = self.get_file_inode(entry.path())? { + keep(inode, &mut out); + } + let mut dir = entry.path().parent(); + while let Some(d) = dir.filter(|d| !d.as_os_str().is_empty()) { + if let Some(inode) = self.get_file_inode(d)? { + keep(inode, &mut out); + } + dir = d.parent(); + } + } + Ok(out.into_iter().collect()) + } + + /// Cache side: replace this cache's graph rows and held content with + /// `slice`'s. + pub fn import_sandbox_slice(&self, slice: &SandboxSlice) -> Result<(), RepositoryError> { + let view_id = { + let txn = self.pristine.read_txn().map_err(db)?; + txn.get_view(self.current_view()) + .map_err(db)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: self.current_view().to_string(), + })? + .id + }; + let mut txn = self.pristine.write_txn().map_err(db)?; + txn.import_graph_slice(&slice.rows, view_id).map_err(db)?; + txn.commit().map_err(db)?; + self.change_store.clear_spans(); + for span in &slice.spans { + self.change_store + .hold_span(span.change, span.start, span.bytes.clone()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::forbidden_path; + + #[test] + fn a_recorded_path_must_be_a_plain_relative_path() { + for path in [ + "/etc/passwd", + "/", + "..", + "../elsewhere/x", + "src/../../elsewhere/x", + "src/..", + "./src/main.rs", + "src//main.rs", + ] { + assert!(forbidden_path(path), "{path:?} should be refused"); + } + } + + #[test] + fn a_recorded_path_may_not_name_the_repositorys_own_machinery() { + for path in [ + ".atomic", + ".atomic/config.toml", + "src/.atomic/config.toml", + ".atomic-sandbox", + ".atomic-sandbox.d", + "nested/.atomic-sandbox.d/cache", + ] { + assert!(forbidden_path(path), "{path:?} should be refused"); + } + } + + #[test] + fn an_ordinary_path_is_allowed() { + for path in [ + // A token-level op names an inode, not a path: nothing to escape. + "", + "README.md", + "src/main.rs", + "a/b/c/d.txt", + "..hidden", + "...hidden", + "a..b", + "atomic", + "src/atomic-sandbox", + ] { + assert!(!forbidden_path(path), "{path:?} should be allowed"); + } + } +} diff --git a/atomic-repository/src/repository/sandbox.rs b/atomic-repository/src/repository/sandbox.rs index adb5cebf..123295da 100644 --- a/atomic-repository/src/repository/sandbox.rs +++ b/atomic-repository/src/repository/sandbox.rs @@ -84,6 +84,11 @@ pub struct SealResult { /// instead of looking for a local `.atomic/`. pub const SANDBOX_POINTER: &str = ".atomic-sandbox"; +/// A remote sandbox's own directory, beside its pointer: the local cache of +/// its repository's rows (`cache/`, an ordinary repository whose working +/// tree is the sandbox). Never tracked. +pub const SANDBOX_CACHE_DIR: &str = ".atomic-sandbox.d"; + /// Persisted contents of a sandbox pointer: where the canonical graph lives /// and which view this sandbox operates on. #[derive(Debug, Serialize, Deserialize)] @@ -107,8 +112,19 @@ pub(super) fn detect_sandbox(start: &Path) -> Option<(PathBuf, PathBuf, String)> let pointer = dir.join(SANDBOX_POINTER); if pointer.is_file() { let bytes = std::fs::read(&pointer).ok()?; - let parsed: SandboxPointer = serde_json::from_slice(&bytes).ok()?; - return Some((dir.clone(), parsed.canonical, parsed.view)); + if let Ok(parsed) = serde_json::from_slice::(&bytes) { + return Some((dir.clone(), parsed.canonical, parsed.view)); + } + // A remote pointer names an owner elsewhere; locally, the graph + // is the sandbox's cache (once `atomic sandbox materialize` has + // made it). + let remote: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + let view = remote.get("remote").and(remote.get("view"))?.as_str()?; + let cache = remote_cache_root(&dir); + return cache + .join(DOT_DIR) + .is_dir() + .then(|| (dir.clone(), cache, view.to_string())); } // Stop if we reach a real repository root — that's not a sandbox. if dir.join(DOT_DIR).join("pristine.redb").is_file() { @@ -120,7 +136,43 @@ pub(super) fn detect_sandbox(start: &Path) -> Option<(PathBuf, PathBuf, String)> } } +/// Where a remote sandbox rooted at `working_root` keeps its cache. +pub fn remote_cache_root(working_root: &Path) -> PathBuf { + working_root.join(SANDBOX_CACHE_DIR).join("cache") +} + impl Repository { + /// Make (or remake) a remote sandbox's cache from `skeleton`: an empty + /// repository under [`SANDBOX_CACHE_DIR`] holding the view's rows, with + /// the sandbox as its working tree and the tree on disk as the clean + /// baseline. Call it right after writing the view's files. + pub fn create_remote_sandbox_cache( + working_root: &Path, + skeleton: &super::SandboxSkeleton, + ) -> Result { + let cache = remote_cache_root(working_root); + if cache.exists() { + std::fs::remove_dir_all(&cache)?; + } + std::fs::create_dir_all(&cache)?; + drop(Self::init(&cache)?); + let repo = Self::open_sandbox(working_root, &cache, &skeleton.view.name)?; + repo.import_sandbox_skeleton(skeleton)?; + repo.reindex_working_copy()?; + // The view's vault arrived as files, as after a pull: its tables + // come from them. + if repo.vault_dir().exists() { + repo.bootstrap_vault_from_working_copy()?; + } + Ok(repo) + } + + /// Whether this is a remote sandbox's cache (its graph arrives from the + /// repository's owner, and its changes go back to it). + pub fn is_remote_sandbox(&self) -> bool { + self.is_sandbox && self.dot_dir.starts_with(self.root.join(SANDBOX_CACHE_DIR)) + } + /// Open a repository for an agent sandbox. /// /// The agent's private working tree is at `working_root`, but the graph @@ -227,19 +279,21 @@ impl Repository { pub fn materialize_view_to(&self, view: &str, dir: &Path) -> Result { std::fs::create_dir_all(dir)?; + // The view as it renders — its own added and moved files included, + // whichever view is checked out. let mut count = 0usize; - for path in self.visible_file_paths(view)? { - let bytes = match self.get_file_content_on_view(&path, view)? { - Some(bytes) => bytes, - None => continue, - }; - let target = dir.join(&path); + self.materialize_view_entries(view, |entry| -> Result<(), std::io::Error> { + if entry.kind == super::ViewEntryKind::Directory { + return Ok(()); + } + let target = dir.join(&entry.path); if let Some(parent) = target.parent() { std::fs::create_dir_all(parent)?; } - std::fs::write(&target, &bytes)?; + std::fs::write(&target, &entry.content)?; count += 1; - } + Ok(()) + })??; Ok(count) } @@ -525,4 +579,23 @@ mod tests { "the canonical graph must not be cloned into the sandbox" ); } + + #[test] + fn a_sandbox_never_sees_its_pointer_as_untracked() { + let dir = tempdir().unwrap(); + let repo = repo_with_recorded_file(&dir.path().join("repo"), "hello.txt", b"hi\n"); + let sandbox = dir.path().join("agent-1"); + repo.provision_sandbox(&sandbox, "dev").unwrap(); + std::fs::write(sandbox.join("new.txt"), b"new\n").unwrap(); + drop(repo); + + let opened = Repository::open_existing(&sandbox).unwrap(); + let status = opened.status(Default::default()).unwrap(); + let untracked: Vec<_> = status.untracked().map(|e| e.path().to_path_buf()).collect(); + assert_eq!( + untracked, + vec![PathBuf::from("new.txt")], + "the pointer can carry a credential; `record --all` must never pick it up" + ); + } } diff --git a/atomic-repository/src/repository/status.rs b/atomic-repository/src/repository/status.rs index af1dfabe..c694263c 100644 --- a/atomic-repository/src/repository/status.rs +++ b/atomic-repository/src/repository/status.rs @@ -285,11 +285,18 @@ impl Repository { // materialize uses) to ask: does this file have // content from this view's perspective? If not, the // deletion is already recorded. + // + // Only a graph that holds the file can say so: a remote + // sandbox's cache has its tree before any of its graph, + // and there no vertex means "ask", not "deleted". if has_graph { if let Some(inode_val) = inode { if let Ok(Some(position)) = txn.inode_position(inode_val) { if let Some(ref ids) = current_view_change_ids { - if !is_file_alive_via_retrieval(&txn, inode_val, position, ids) + if txn.has_vertex(position.inode_node()).unwrap_or(false) + && !is_file_alive_via_retrieval( + &txn, inode_val, position, ids, + ) { // Deletion already recorded — skip found_on_disk.insert(path.clone()); diff --git a/atomic-repository/src/repository/vault.rs b/atomic-repository/src/repository/vault.rs index a14ca237..a88c43dc 100644 --- a/atomic-repository/src/repository/vault.rs +++ b/atomic-repository/src/repository/vault.rs @@ -594,9 +594,38 @@ impl Repository { // Read file content let file_content = std::fs::read_to_string(path)?; + if let Some(change) = self.vault_file_change(rel_path_str, &file_content)? { + changes.push(change); + } + } + + // Also check for deleted entries (in redb but not on disk) + let all_entries = self.vault_list("", None)?; + for meta in all_entries { + let disk_path = vault_dir.join(&meta.path); + if !disk_path.exists() { + changes.push(VaultFileChange { + path: meta.path, + change_type: VaultChangeType::Deleted, + frontmatter_json: String::new(), + content: Vec::new(), + }); + } + } + + Ok(changes) + } + /// How one vault file (`rel_path`, vault-relative, with `file_content`) + /// differs from its stored entry — `None` when it doesn't. + fn vault_file_change( + &self, + rel_path_str: String, + file_content: &str, + ) -> Result, RepositoryError> { + { // Parse into frontmatter + body - let (frontmatter_json, body) = parse_markdown_frontmatter(&file_content); + let (frontmatter_json, body) = parse_markdown_frontmatter(file_content); // Body hashes remain useful for content/embedding caches, but // frontmatter is also knowledge: status, labels, identity, and @@ -611,35 +640,43 @@ impl Repository { if Hash::from_bytes(entry.content_hash) == new_hash && frontmatter_json_equal(&entry.frontmatter_json, &frontmatter_json) => { - continue; // Unchanged, skip + return Ok(None); // Unchanged, skip } Some(_) => VaultChangeType::Modified, None => VaultChangeType::New, }; - changes.push(VaultFileChange { + Ok(Some(VaultFileChange { path: rel_path_str, change_type, frontmatter_json, content: body_bytes.to_vec(), - }); + })) } + } - // Also check for deleted entries (in redb but not on disk) - let all_entries = self.vault_list("", None)?; - for meta in all_entries { - let disk_path = vault_dir.join(&meta.path); - if !disk_path.exists() { - changes.push(VaultFileChange { - path: meta.path, + /// Deflate vault files that are not in this working copy — a view's, + /// say — into redb: each `(vault-relative path, content)`, `None` for a + /// file the view no longer has. What `vault_record_working_copy` does + /// for the files on disk. + pub fn vault_record_files( + &self, + files: &[(String, Option)], + ) -> Result, RepositoryError> { + let mut changes = Vec::new(); + for (path, content) in files { + match content { + Some(content) => changes.extend(self.vault_file_change(path.clone(), content)?), + None if self.vault_retrieve(path)?.is_some() => changes.push(VaultFileChange { + path: path.clone(), change_type: VaultChangeType::Deleted, frontmatter_json: String::new(), content: Vec::new(), - }); + }), + None => {} } } - - Ok(changes) + self.vault_apply_changes(&changes) } /// Deflate changed vault files from disk into redb. @@ -653,10 +690,18 @@ impl Repository { /// - `vault_record_working_copy`: disk -> redb (after humans edit) pub fn vault_record_working_copy(&self) -> Result, RepositoryError> { let changes = self.vault_scan_working_copy()?; + self.vault_apply_changes(&changes) + } + /// Store detected vault changes, then index any intents the manifest + /// is missing. + fn vault_apply_changes( + &self, + changes: &[VaultFileChange], + ) -> Result, RepositoryError> { let mut updated_paths = Vec::new(); - for change in &changes { + for change in changes { match change.change_type { VaultChangeType::New | VaultChangeType::Modified => { // Infer entry type from path diff --git a/atomic-repository/src/status.rs b/atomic-repository/src/status.rs index 6c470d28..0cdbf9dd 100644 --- a/atomic-repository/src/status.rs +++ b/atomic-repository/src/status.rs @@ -89,8 +89,10 @@ use crate::ignore::IgnoreRules; // Constants -/// Patterns that are always ignored (internal directories) -const ALWAYS_IGNORED: &[&str] = &[".atomic", ".git"]; +/// Patterns that are always ignored: internal directories, and a sandbox's +/// pointer and local cache — the pointer can carry a credential, and neither +/// belongs in history. +const ALWAYS_IGNORED: &[&str] = &[".atomic", ".git", ".atomic-sandbox", ".atomic-sandbox.d"]; // Error Types @@ -1255,10 +1257,15 @@ mod tests { assert!(is_always_ignored(Path::new("src/.atomic/test"))); assert!(is_always_ignored(Path::new(".git"))); assert!(is_always_ignored(Path::new(".git/objects"))); + assert!(is_always_ignored(Path::new(".atomic-sandbox"))); + assert!(is_always_ignored(Path::new( + ".atomic-sandbox.d/pristine.redb" + ))); assert!(!is_always_ignored(Path::new("src"))); assert!(!is_always_ignored(Path::new("src/main.rs"))); assert!(!is_always_ignored(Path::new("atomic"))); + assert!(!is_always_ignored(Path::new(".atomic-sandbox-notes"))); } #[test] diff --git a/atomic-repository/tests/database_lock_exclusivity_test.rs b/atomic-repository/tests/database_lock_exclusivity_test.rs new file mode 100644 index 00000000..1b24d789 --- /dev/null +++ b/atomic-repository/tests/database_lock_exclusivity_test.rs @@ -0,0 +1,96 @@ +//! Is the pristine's file lock exclusive across processes, or only within one? +//! +//! The database owner re-opens the repository on every request and retries while +//! it is busy, which works either way. Whether it may instead hold one handle for +//! its lifetime depends on the answer: a cross-process lock would stop local +//! `atomic status` / `record` from opening the repository at all while an owner +//! is running. +//! +//! One test binary, two roles. The parent holds a `Repository` open and re-runs +//! itself as the child; the child tries to open the same repository and reports +//! what happened. + +use std::process::Command; + +use atomic_repository::{Repository, RepositoryError}; + +const CHILD_ROOT: &str = "ATOMIC_DATABASE_LOCK_ROOT"; +const CHILD_REPORT: &str = "ATOMIC_DATABASE_LOCK_REPORT"; +const CHILD_TEST: &str = "a_second_process_can_open_a_repository_another_one_holds"; + +/// What the child managed to do, one line, on stdout. +fn probe(root: &std::path::Path) -> String { + // Read-only first: the gentlest thing a local command does. + let readonly = match Repository::open_readonly(root) { + Ok(_) => "readonly-ok".to_string(), + Err(RepositoryError::DatabaseBusy) => "readonly-busy".to_string(), + Err(e) => format!("readonly-other: {e}"), + }; + // Then read-write, which is what `record` needs. + let readwrite = match Repository::open_existing(root) { + Ok(_) => "readwrite-ok".to_string(), + Err(RepositoryError::DatabaseBusy) => "readwrite-busy".to_string(), + Err(e) => format!("readwrite-other: {e}"), + }; + format!("{readonly} {readwrite}") +} + +#[test] +fn a_second_process_can_open_a_repository_another_one_holds() { + let temp = tempfile::TempDir::new().unwrap(); + let root = temp.path().join("repo"); + + if let Ok(child_root) = std::env::var(CHILD_ROOT) { + // The child: report on the repository the parent named and get out. The + // report goes to a file, not stdout — libtest's capture decides whether + // a child's `println!` reaches its parent, and this must not depend on + // whether the parent was run with `--nocapture`. + let report = probe(std::path::Path::new(&child_root)); + let path = std::env::var(CHILD_REPORT).expect("CHILD_REPORT"); + std::fs::write(path, report).expect("write the child report"); + return; + } + + // Held for the whole child run: this is the thing being tested. + let held = Repository::init(&root).expect("init"); + assert_eq!(held.current_view(), "dev", "the parent's handle is open"); + + let report_path = temp.path().join("child-report.txt"); + let output = Command::new(std::env::current_exe().expect("test binary path")) + .args(["--exact", CHILD_TEST, "--test-threads=1"]) + .env(CHILD_ROOT, &root) + .env(CHILD_REPORT, &report_path) + .output() + .expect("run the child probe"); + + let report = std::fs::read_to_string(&report_path).unwrap_or_else(|e| { + panic!( + "the child reported nothing ({e})\n--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) + }); + + println!("while one process holds the repository, a second sees: {report}"); + + // The answer, asserted. Exclusive across processes, and for a read-only + // open as much as a read-write one — so the database owner cannot hold the + // pristine for its lifetime without locking local `atomic status` and + // `record` out of the repository. That is why it opens per request and + // retries, and why an idle release is required of any handle it does keep. + assert_eq!( + report, "readonly-busy readwrite-busy", + "the pristine's lock is no longer exclusive across processes; the owner \ + may be able to hold it open, and its open-per-request retry is \ + redundant" + ); + + // And the corollary, which is the one that bites: dropping the parent's + // handle must let the very next open succeed, or an idle release would + // never actually free the file. + drop(held); + match Repository::open_readonly(&root) { + Ok(_) => {} + Err(e) => panic!("releasing the handle did not free the file: {e}"), + } +} diff --git a/atomic-repository/tests/materialize_view_entries_test.rs b/atomic-repository/tests/materialize_view_entries_test.rs new file mode 100644 index 00000000..8cd098f5 --- /dev/null +++ b/atomic-repository/tests/materialize_view_entries_test.rs @@ -0,0 +1,168 @@ +//! `Repository::materialize_view_entries`: a view's tree, rendered in memory +//! for a remote sandbox — the same bytes a checkout writes, per view, and +//! nothing touched on disk. + +use std::fs; +use std::path::Path; + +use atomic_core::change::{Author, ChangeHeader}; +use atomic_core::types::Hash; +use atomic_repository::{RecordOptions, Repository, SplitOptions, ViewEntry, ViewEntryKind}; +use tempfile::TempDir; + +fn write(repo_path: &Path, name: &str, content: &str) { + let path = repo_path.join(name); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); +} + +fn record(repo: &Repository, message: &str) -> Hash { + let header = ChangeHeader::builder() + .message(message) + .author(Author::new("Test", Some("test@example.com"))) + .build(); + *repo + .record(header, RecordOptions::default()) + .expect("record") + .hash() +} + +fn entries(repo: &Repository, view: &str) -> (Vec, atomic_repository::ViewSnapshot) { + let mut out = Vec::new(); + let snapshot = repo + .materialize_view_entries::<()>(view, |e| { + out.push(e); + Ok(()) + }) + .expect("materialize") + .expect("sink"); + (out, snapshot) +} + +#[test] +fn entries_are_the_view_s_recorded_tree() { + let temp = TempDir::new().unwrap(); + let root = temp.path().to_path_buf(); + let repo = Repository::init(&root).expect("init"); + let view = repo.current_view().to_string(); + + write(&root, "README.md", "hello\n"); + write(&root, "src/main.rs", "fn main() {}\n"); + repo.add("README.md", Default::default()).unwrap(); + repo.add("src/main.rs", Default::default()).unwrap(); + record(&repo, "first"); + write(&root, "README.md", "hello, world\n"); + let second = record(&repo, "second"); + // Present on disk, never recorded: not part of the view. + write(&root, "scratch.txt", "not recorded\n"); + + let before: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + let (list, snapshot) = entries(&repo, &view); + let after: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + assert_eq!( + before.len(), + after.len(), + "nothing written to the working tree" + ); + + let paths: Vec<&str> = list.iter().map(|e| e.path.as_str()).collect(); + assert_eq!( + paths, + vec!["src", "README.md", "src/main.rs"], + "directories first, then files in order" + ); + let readme = list.iter().find(|e| e.path == "README.md").unwrap(); + assert_eq!(readme.kind, ViewEntryKind::File); + assert_eq!(readme.content, b"hello, world\n"); + assert_eq!(readme.hash, Hash::of(b"hello, world\n")); + assert_eq!(readme.conflict_marker_line, None); + assert!(readme.inode > 0); + assert_eq!(snapshot.view, view); + assert_eq!(snapshot.change_count, 2); + assert!(!snapshot.state.is_empty()); + + // Another view without the second change renders its own content. + let mut repo = repo; + repo.split_view(SplitOptions::new("older", vec![second])) + .expect("split"); + let (older_on_source, older_snapshot) = entries(&repo, &view); + let readme = older_on_source + .iter() + .find(|e| e.path == "README.md") + .unwrap(); + assert_eq!( + readme.content, b"hello\n", + "the source view no longer has the second change" + ); + assert_ne!(older_snapshot.state, snapshot.state); +} + +/// Another view's tree is a projection, and a read-only repository cannot +/// project. It used to fall through to `TREE`, which is the *current* view's — +/// handing one view's tree back under another's name, which for a remote +/// sandbox means writing the wrong tree to disk. Refusing is the safe answer. +#[test] +fn a_read_only_repository_refuses_another_views_tree() { + let temp = TempDir::new().unwrap(); + let root = temp.path().to_path_buf(); + let mut repo = Repository::init(&root).expect("init"); + let current = repo.current_view().to_string(); + + write(&root, "README.md", "hello\n"); + repo.add("README.md", Default::default()).unwrap(); + record(&repo, "first"); + write(&root, "README.md", "hello, world\n"); + let second = record(&repo, "second"); + + repo.split_view(SplitOptions::new("older", vec![second])) + .expect("split"); + + // Writable: both views render, and they differ. `split_view` moves the + // change out of the source view, so the new view is the one with it. + let (on_current, _) = entries(&repo, ¤t); + let (on_older, _) = entries(&repo, "older"); + let body = |v: &Vec| { + v.iter() + .find(|e| e.path == "README.md") + .map(|e| e.content.clone()) + .unwrap() + }; + assert_eq!(body(&on_current), b"hello\n"); + assert_eq!(body(&on_older), b"hello, world\n"); + + // Read-only: the current view still works (it needs no projection), but + // another view is refused rather than answered with the current tree. + drop(repo); + let readonly = Repository::open_readonly(&root).expect("open read-only"); + + let mut on_current = Vec::new(); + readonly + .materialize_view_entries::<()>(¤t, |e| { + on_current.push(e); + Ok(()) + }) + .expect("materialize the current view") + .expect("sink"); + assert_eq!(body(&on_current), b"hello\n"); + + let mut on_older = Vec::new(); + let refused = readonly.materialize_view_entries::<()>("older", |e| { + on_older.push(e); + Ok(()) + }); + let err = refused.expect_err("another view must be refused"); + assert!( + format!("{err}").contains("writable"), + "unexpected error: {err}" + ); + assert!( + on_older.is_empty(), + "nothing of the wrong view was rendered" + ); +} diff --git a/atomic-repository/tests/remote_cache_parity_test.rs b/atomic-repository/tests/remote_cache_parity_test.rs new file mode 100644 index 00000000..c90f8a29 --- /dev/null +++ b/atomic-repository/tests/remote_cache_parity_test.rs @@ -0,0 +1,618 @@ +//! A remote sandbox's cache records the same change the repository would. +//! +//! The cache holds only the repository's own rows for the view (the +//! skeleton) and for the paths being recorded (a slice), plus the content +//! bytes those need. `record` runs unchanged against it, and the change it +//! produces — which carries the repository's internal ids — must be byte for +//! byte the change `record` produces on the repository itself. + +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; + +use atomic_core::change::{Author, ChangeHeader}; +use atomic_repository::{InsertOptions, RecordOptions, Repository, TrackingOptions}; +use tempfile::TempDir; + +fn write(root: &Path, rel: &str, content: &str) { + let path = root.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); +} + +fn header(message: &str) -> ChangeHeader { + ChangeHeader::builder() + .message(message) + .author(Author::new("Test", Some("test@example.com"))) + .build() +} + +fn record_and_apply(repo: &Repository, message: &str) { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(false); + let outcome = repo.record(header(message), options).unwrap(); + repo.write_recorded(&outcome, InsertOptions::default()) + .unwrap(); +} + +/// Record without saving or applying: just the change. +fn change_bytes(repo: &Repository, header: ChangeHeader) -> (String, Vec) { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = repo + .record(header, options) + .unwrap_or_else(|e| panic!("record in {}: {e}", repo.root().display())); + ( + atomic_core::types::Base32::to_base32(outcome.hash()), + outcome.v3_bytes().expect("v3 bytes").to_vec(), + ) +} + +struct Pair { + _dirs: (TempDir, TempDir), + host: Repository, + cache: Repository, +} + +impl Pair { + fn host_root(&self) -> &Path { + self.host.root() + } + fn cache_root(&self) -> &Path { + self.cache.root() + } + + /// Apply the same edit to both working trees. + fn edit(&self, f: impl Fn(&Path)) { + f(self.host_root()); + f(self.cache_root()); + } + + /// Ship the slice for what `record` will touch, as FileStates does: + /// only the inodes the cache asks for. + fn hydrate(&self, view: &str, live: &BTreeMap) { + let inodes = self.cache.sandbox_slice_inodes().unwrap(); + assert!( + inodes.len() < live.len(), + "the cache asks for what changed, not everything: {inodes:?} of {live:?}" + ); + let slice = self.host.export_sandbox_slice(view, &inodes).unwrap(); + self.cache.import_sandbox_slice(&slice).unwrap(); + } +} + +/// A host repository with some history, and a cache materialized from it. +fn pair(history: impl Fn(&Repository)) -> (Pair, String, BTreeMap) { + let host_dir = TempDir::new().unwrap(); + let cache_dir = TempDir::new().unwrap(); + let host = Repository::init(host_dir.path()).unwrap(); + history(&host); + let view = host.current_view().to_string(); + + // Materialize: the view's files into the cache's working tree, and the + // inodes they are. + let cache = Repository::init(cache_dir.path()).unwrap(); + let mut live = BTreeMap::new(); + host.materialize_view_entries::<()>(&view, |entry| { + live.insert(entry.inode, entry.path.clone()); + let path = cache_dir.path().join(&entry.path); + match entry.kind { + atomic_repository::ViewEntryKind::Directory => fs::create_dir_all(path).unwrap(), + _ => { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + } + Ok(()) + }) + .unwrap() + .unwrap(); + let skeleton = host.export_sandbox_skeleton(&view, &live).unwrap(); + cache.import_sandbox_skeleton(&skeleton).unwrap(); + // The baseline: what was just written is clean. + cache.reindex_working_copy().unwrap(); + ( + Pair { + _dirs: (host_dir, cache_dir), + host, + cache, + }, + view, + live, + ) +} + +fn assert_parity(pair: &Pair, view: &str, live: &BTreeMap, what: &str) { + pair.hydrate(view, live); + let h = header(what); + let host = change_bytes(&pair.host, h.clone()); + let cache = change_bytes(&pair.cache, h); + assert_eq!( + host.0, cache.0, + "{what}: the cache records a different change" + ); + assert_eq!(host.1, cache.1, "{what}: same hash, different bytes"); +} + +fn two_files(repo: &Repository) { + let root = repo.root().to_path_buf(); + write(&root, "README.md", "hello\nworld\n"); + write(&root, "src/lib.rs", "pub fn a() {}\npub fn b() {}\n"); + repo.add("README.md", TrackingOptions::default()).unwrap(); + repo.add("src/lib.rs", TrackingOptions::default()).unwrap(); + record_and_apply(repo, "first"); + write(&root, "README.md", "hello\nthere\nworld\n"); + record_and_apply(repo, "second"); +} + +#[test] +fn modifying_a_file() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "hello\nthere\nbig\nworld\n")); + assert_parity(&pair, &view, &live, "modify"); +} + +#[test] +fn modifying_a_file_changed_by_several_changes() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "world\n")); + assert_parity(&pair, &view, &live, "delete lines from two changes"); +} + +#[test] +fn deleting_a_file() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| fs::remove_file(root.join("src/lib.rs")).unwrap()); + assert_parity(&pair, &view, &live, "delete"); +} + +#[test] +fn adding_a_file_to_an_existing_directory() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "src/new.rs", "pub fn c() {}\n")); + for repo in [&pair.host, &pair.cache] { + repo.add("src/new.rs", TrackingOptions::default()).unwrap(); + } + assert_parity(&pair, &view, &live, "add"); +} + +#[test] +fn adding_a_file_in_a_new_directory() { + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "docs/guide.md", "# guide\n")); + for repo in [&pair.host, &pair.cache] { + repo.add("docs/guide.md", TrackingOptions::default()) + .unwrap(); + } + assert_parity(&pair, &view, &live, "add in new dir"); +} + +#[test] +fn without_the_slice_the_cache_cannot_record_the_same_change() { + let (pair, _view, _live) = pair(two_files); + pair.edit(|root| write(root, "README.md", "hello\nthere\nbig\nworld\n")); + let h = header("no slice"); + let host = change_bytes(&pair.host, h.clone()); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + match pair.cache.record(h, options) { + Err(_) => {} + Ok(outcome) => assert_ne!( + atomic_core::types::Base32::to_base32(outcome.hash()), + host.0, + "the graph rows are what make the change right" + ), + } + assert!(host.1.len() > 200, "a real change: {} bytes", host.1.len()); +} + +#[test] +fn moving_a_file() { + // As `atomic mv` does: the rename on disk, detected by content. + let (pair, view, live) = pair(two_files); + for repo in [&pair.host, &pair.cache] { + fs::rename( + repo.root().join("src/lib.rs"), + repo.root().join("src/core.rs"), + ) + .unwrap(); + } + assert_parity(&pair, &view, &live, "move"); +} + +#[test] +fn moving_and_editing_a_file() { + let (pair, view, live) = pair(two_files); + for repo in [&pair.host, &pair.cache] { + fs::rename(repo.root().join("README.md"), repo.root().join("READ.md")).unwrap(); + write(repo.root(), "READ.md", "hello\nthere\nworld\nagain\n"); + } + assert_parity(&pair, &view, &live, "move and edit"); +} + +#[test] +fn a_second_record_on_top_of_the_first() { + // After the first change lands on the repository, the cache takes a fresh + // skeleton and slice and records the next one identically. + let (pair, view, live) = pair(two_files); + pair.edit(|root| write(root, "src/lib.rs", "pub fn a() {}\n")); + assert_parity(&pair, &view, &live, "first"); + record_and_apply(&pair.host, "first"); + let skeleton = pair.host.export_sandbox_skeleton(&view, &live).unwrap(); + pair.cache.import_sandbox_skeleton(&skeleton).unwrap(); + pair.cache.reindex_working_copy().unwrap(); + pair.edit(|root| write(root, "src/lib.rs", "pub fn a() { 1 }\n")); + assert_parity(&pair, &view, &live, "second"); +} + +// ── Submitting: the repository takes the cache's change ───────────────── + +use atomic_repository::SubmitRejection; + +fn view_state(repo: &Repository, view: &str) -> String { + let skeleton = repo + .export_sandbox_skeleton(view, &BTreeMap::new()) + .unwrap(); + atomic_core::types::Base32::to_base32(&skeleton.view.state) +} + +/// Record in the cache (hydrated as FileStates would) and return the change. +fn record_in_cache( + pair: &Pair, + view: &str, + live: &BTreeMap, + what: &str, +) -> (atomic_core::types::Hash, Vec) { + pair.hydrate(view, live); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = pair.cache.record(header(what), options).unwrap(); + (*outcome.hash(), outcome.v3_bytes().unwrap().to_vec()) +} + +fn tree_of(repo: &Repository, view: &str) -> Vec<(String, Vec)> { + let mut files = Vec::new(); + repo.materialize_view_entries::<()>(view, |e| { + if e.kind == atomic_repository::ViewEntryKind::File { + files.push((e.path, e.content)); + } + Ok(()) + }) + .unwrap() + .unwrap(); + files +} + +#[test] +fn a_submitted_change_lands_on_the_view_and_the_next_one_builds_on_it() { + let (pair, view, mut live) = pair(two_files); + // Only the cache's tree changes: the repository learns of it by submission. + write(pair.cache_root(), "README.md", "hello\nfrom the sandbox\n"); + write(pair.cache_root(), "src/new.rs", "pub fn n() {}\n"); + pair.cache + .add("src/new.rs", TrackingOptions::default()) + .unwrap(); + let base = view_state(&pair.host, &view); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "from the sandbox"); + + let submitted = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap() + .expect("accepted"); + assert_ne!(submitted.state, base, "the view moved"); + let tree = tree_of(&pair.host, &view); + assert!(tree.contains(&("README.md".into(), b"hello\nfrom the sandbox\n".to_vec()))); + assert!(tree.contains(&("src/new.rs".into(), b"pub fn n() {}\n".to_vec()))); + + // The same change again: the view has moved, and then it's already there. + let again = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!(again, Err(SubmitRejection::StaleView { .. })), + "{again:?}" + ); + let again = pair + .host + .insert_submitted_change(&view, &submitted.state, &hash, &bytes) + .unwrap(); + assert!( + matches!(again, Err(SubmitRejection::AlreadyPresent(_))), + "{again:?}" + ); + + // The cache takes the new skeleton (with the repository's inode for the + // new file) and records on top. + live.clear(); + pair.host + .materialize_view_entries::<()>(&view, |e| { + live.insert(e.inode, e.path.clone()); + Ok(()) + }) + .unwrap() + .unwrap(); + let skeleton = pair.host.export_sandbox_skeleton(&view, &live).unwrap(); + pair.cache.import_sandbox_skeleton(&skeleton).unwrap(); + pair.cache.reindex_working_copy().unwrap(); + write(pair.cache_root(), "src/new.rs", "pub fn n() { 2 }\n"); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "second from the sandbox"); + pair.host + .insert_submitted_change(&view, &submitted.state, &hash, &bytes) + .unwrap() + .expect("the second is accepted too"); + assert!( + tree_of(&pair.host, &view).contains(&("src/new.rs".into(), b"pub fn n() { 2 }\n".to_vec())) + ); +} + +#[test] +fn a_tampered_change_is_refused_and_leaves_nothing() { + let (pair, view, live) = pair(two_files); + write(pair.cache_root(), "README.md", "tampered?\n"); + let base = view_state(&pair.host, &view); + let (hash, mut bytes) = record_in_cache(&pair, &view, &live, "tamper"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + let refused = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!( + refused, + Err(SubmitRejection::HashMismatch { .. } | SubmitRejection::Malformed(_)) + ), + "{refused:?}" + ); + assert_eq!(view_state(&pair.host, &view), base); + assert!(!pair.host.has_change(&hash)); +} + +#[test] +fn a_change_recorded_against_an_older_view_is_refused() { + let (pair, view, live) = pair(two_files); + let base = view_state(&pair.host, &view); + write(pair.cache_root(), "README.md", "late\n"); + let (hash, bytes) = record_in_cache(&pair, &view, &live, "late"); + // Meanwhile the view moves on the repository. + write(pair.host_root(), "src/lib.rs", "pub fn moved() {}\n"); + record_and_apply(&pair.host, "meanwhile"); + let refused = pair + .host + .insert_submitted_change(&view, &base, &hash, &bytes) + .unwrap(); + assert!( + matches!(refused, Err(SubmitRejection::StaleView { .. })), + "{refused:?}" + ); +} + +#[test] +fn a_change_from_another_view_is_refused() { + // Work recorded on a draft view depends on that view's changes; submitted + // to `dev` it names changes `dev` can't see. + let host_dir = TempDir::new().unwrap(); + let draft_dir = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("other", "dev").unwrap(); + host.provision_sandbox(draft_dir.path().join("w"), "other") + .unwrap(); + } + let draft = Repository::open_existing(draft_dir.path().join("w")).unwrap(); + write(draft.root(), "README.md", "only on other\n"); + record_and_apply(&draft, "on other"); + write(draft.root(), "README.md", "only on other, again\n"); + let (hash, bytes) = { + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = draft.record(header("on top of other"), options).unwrap(); + (*outcome.hash(), outcome.v3_bytes().unwrap().to_vec()) + }; + drop(draft); + + let host = Repository::open_existing(host_dir.path()).unwrap(); + let base = view_state(&host, "dev"); + let refused = host + .insert_submitted_change("dev", &base, &hash, &bytes) + .unwrap(); + assert!( + matches!( + refused, + Err(SubmitRejection::ForeignChange(_) | SubmitRejection::ForeignNode(_)) + ), + "{refused:?}" + ); + assert_eq!(view_state(&host, "dev"), base); +} + +#[test] +fn a_change_submitted_to_a_draft_view_is_on_that_view() { + // Agent work lands on draft views: what lands on one renders there. + let (pair, _, _) = pair(two_files); + let mut host = pair.host; + host.create_view_from("exp", "dev").unwrap(); + let cache_dir = TempDir::new().unwrap(); + drop(Repository::init(cache_dir.path()).unwrap()); + // As a remote sandbox's cache is opened: on its pointer's view. + let cache = Repository::open_sandbox(cache_dir.path(), cache_dir.path(), "exp").unwrap(); + let mut live = BTreeMap::new(); + host.materialize_view_entries::<()>("exp", |entry| { + live.insert(entry.inode, entry.path.clone()); + let path = cache_dir.path().join(&entry.path); + match entry.kind { + atomic_repository::ViewEntryKind::Directory => fs::create_dir_all(path).unwrap(), + _ => { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + } + Ok(()) + }) + .unwrap() + .unwrap(); + cache + .import_sandbox_skeleton(&host.export_sandbox_skeleton("exp", &live).unwrap()) + .unwrap(); + cache.reindex_working_copy().unwrap(); + write(cache_dir.path(), "src/new.rs", "pub fn n() {}\n"); + write(cache_dir.path(), "README.md", "hello\non the draft\n"); + cache.add("src/new.rs", TrackingOptions::default()).unwrap(); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let outcome = cache.record(header("on the draft"), options).unwrap(); + let base = view_state(&host, "exp"); + host.insert_submitted_change("exp", &base, outcome.hash(), outcome.v3_bytes().unwrap()) + .unwrap() + .expect("accepted"); + + let tree = tree_of(&host, "exp"); + assert!( + tree.contains(&("README.md".into(), b"hello\non the draft\n".to_vec())), + "{tree:?}" + ); + assert!( + tree.contains(&("src/new.rs".into(), b"pub fn n() {}\n".to_vec())), + "{tree:?}" + ); + assert!( + !tree_of(&host, "dev").iter().any(|(p, _)| p == "src/new.rs"), + "only on the draft" + ); + // What `sandbox stage` / `seal` write sees it too. + let out = TempDir::new().unwrap(); + host.materialize_view_to("exp", out.path()).unwrap(); + assert_eq!( + fs::read_to_string(out.path().join("src/new.rs")) + .ok() + .as_deref(), + Some("pub fn n() {}\n") + ); + + // The next record builds on it: the refreshed skeleton knows the new file. + let skeleton = host.after_sandbox_submit("exp", outcome.hash()).unwrap(); + cache.import_sandbox_skeleton(&skeleton).unwrap(); + cache.reindex_working_copy().unwrap(); + write(cache_dir.path(), "src/new.rs", "pub fn n() { 2 }\n"); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let options = RecordOptions::new() + .with_all(true) + .save_to_store(false) + .apply_after_record(false); + let second = cache.record(header("again on the draft"), options).unwrap(); + let (change, _) = + atomic_core::change::Change::deserialize(&mut &second.v3_bytes().unwrap()[..]).unwrap(); + assert!( + !change + .hunks() + .iter() + .any(|h| matches!(h, atomic_core::change::GraphOp::FileAdd { .. })), + "an edit, not a second add" + ); + host.insert_submitted_change( + "exp", + &view_state(&host, "exp"), + second.hash(), + second.v3_bytes().unwrap(), + ) + .unwrap() + .expect("accepted"); + assert!(tree_of(&host, "exp").contains(&("src/new.rs".into(), b"pub fn n() { 2 }\n".to_vec()))); +} + +#[test] +fn control_a_local_sandbox_record_on_a_draft_renders_on_it() { + let host_dir = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("exp", "dev").unwrap(); + host.provision_sandbox(work.path().join("w"), "exp") + .unwrap(); + } + let sbx = Repository::open_existing(work.path().join("w")).unwrap(); + write(sbx.root(), "README.md", "hello\non the draft\n"); + record_and_apply(&sbx, "on the draft"); + drop(sbx); + let host = Repository::open_existing(host_dir.path()).unwrap(); + let tree = tree_of(&host, "exp"); + assert!( + tree.contains(&("README.md".into(), b"hello\non the draft\n".to_vec())), + "{tree:?}" + ); +} + +#[test] +fn parity_on_a_draft_view() { + let host_dir = TempDir::new().unwrap(); + let work = TempDir::new().unwrap(); + { + let mut host = Repository::init(host_dir.path()).unwrap(); + two_files(&host); + host.create_view_from("exp", "dev").unwrap(); + host.provision_sandbox(work.path().join("w"), "exp") + .unwrap(); + } + // The cache, from the draft. + let cache_dir = TempDir::new().unwrap(); + drop(Repository::init(cache_dir.path()).unwrap()); + let cache = Repository::open_sandbox(cache_dir.path(), cache_dir.path(), "exp").unwrap(); + let skeleton = { + let host = Repository::open_existing(host_dir.path()).unwrap(); + let mut live = BTreeMap::new(); + host.materialize_view_entries::<()>("exp", |entry| { + live.insert(entry.inode, entry.path.clone()); + let path = cache_dir.path().join(&entry.path); + if entry.kind != atomic_repository::ViewEntryKind::Directory { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, &entry.content).unwrap(); + } + Ok(()) + }) + .unwrap() + .unwrap(); + host.export_sandbox_skeleton("exp", &live).unwrap() + }; + cache.import_sandbox_skeleton(&skeleton).unwrap(); + cache.reindex_working_copy().unwrap(); + + let sbx = Repository::open_existing(work.path().join("w")).unwrap(); + write(sbx.root(), "README.md", "hello\non the draft\n"); + write(cache_dir.path(), "README.md", "hello\non the draft\n"); + let h = header("on the draft"); + let local = change_bytes(&sbx, h.clone()); + drop(sbx); + let host = Repository::open_existing(host_dir.path()).unwrap(); + let inodes = cache.sandbox_slice_inodes().unwrap(); + cache + .import_sandbox_slice(&host.export_sandbox_slice("exp", &inodes).unwrap()) + .unwrap(); + let remote = change_bytes(&cache, h); + assert_eq!( + local.0, remote.0, + "the cache records a different change on a draft" + ); +} diff --git a/atomic-wasm/.gitignore b/atomic-wasm/.gitignore new file mode 100644 index 00000000..01d0a084 --- /dev/null +++ b/atomic-wasm/.gitignore @@ -0,0 +1 @@ +pkg/ diff --git a/atomic-wasm/Cargo.toml b/atomic-wasm/Cargo.toml new file mode 100644 index 00000000..35a2f6a6 --- /dev/null +++ b/atomic-wasm/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "atomic-wasm" +description = "atomic-canonical for the browser: build and check atomic attestations where the signing key lives in WebCrypto" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +atomic-canonical = { workspace = true } +atomic-identity = { workspace = true } +serde_json = { workspace = true } +wasm-bindgen = "0.2" + +# The browser supplies randomness; nothing here draws any, but the +# dependency tree links getrandom, which refuses wasm32 without this. +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.2", features = ["js"] } diff --git a/atomic-wasm/README.md b/atomic-wasm/README.md new file mode 100644 index 00000000..9ec280c2 --- /dev/null +++ b/atomic-wasm/README.md @@ -0,0 +1,20 @@ +# atomic-wasm + +`atomic-canonical` for the browser. A page that holds an atomic identity's +Ed25519 key in WebCrypto signs atomic documents without the key ever leaving +the browser: this module builds the canonical document and the bytes to sign +(with the same code the CLI uses), WebCrypto signs them, and the result is an +ordinary `eddsa-jcs-2022` attestation that `atomic` verifies. + +```js +import init, { prepareAttestation, attachProof } from "./pkg/atomic_wasm.js"; +await init(); +const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); +const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); +const attested = JSON.parse(attachProof(prepared.document, publicKeyBytes, new Uint8Array(sig))); +``` + +Also: `verifyAttestation`, `didForPublicKey`, `publicKeyBase32`, `canonicalize`. + +Build with `./build.sh [out-dir]`; `node smoke.mjs` (Node ≥ 22, which has +WebCrypto Ed25519) signs, verifies and tamper-checks a document end to end. diff --git a/atomic-wasm/build.sh b/atomic-wasm/build.sh new file mode 100755 index 00000000..2cbb34b8 --- /dev/null +++ b/atomic-wasm/build.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Build the browser module: pkg/atomic_wasm{.js,_bg.wasm}. +# Needs the wasm32 target and a wasm-bindgen CLI matching the locked +# wasm-bindgen crate: +# rustup target add wasm32-unknown-unknown +# cargo install wasm-bindgen-cli --version +set -eu +here=$(cd "$(dirname "$0")" && pwd) +cargo build -p atomic-wasm --target wasm32-unknown-unknown --release +wasm-bindgen --target web --out-dir "${1:-$here/pkg}" \ + "$here/../target/wasm32-unknown-unknown/release/atomic_wasm.wasm" diff --git a/atomic-wasm/smoke.mjs b/atomic-wasm/smoke.mjs new file mode 100644 index 00000000..6fba36a7 --- /dev/null +++ b/atomic-wasm/smoke.mjs @@ -0,0 +1,13 @@ +import { readFileSync } from "node:fs"; +import init, { prepareAttestation, attachProof, verifyAttestation, didForPublicKey } from "./pkg/atomic_wasm.js"; +await init({ module_or_path: readFileSync(new URL("./pkg/atomic_wasm_bg.wasm", import.meta.url)) }); +const kp = await crypto.subtle.generateKey("Ed25519", false, ["sign", "verify"]); +const pub = new Uint8Array(await crypto.subtle.exportKey("raw", kp.publicKey)); +const doc = { "@type": "ExampleLogin", nonce: "abc", "é": [1, 2.5, "x"] }; +const p = prepareAttestation(JSON.stringify(doc), pub); +const sig = new Uint8Array(await crypto.subtle.sign("Ed25519", kp.privateKey, p.signingBytes)); +const attested = attachProof(p.document, pub, sig); +verifyAttestation(attested, pub); +console.log(didForPublicKey(pub)); +console.log(attested); +try { verifyAttestation(attested.replace('"abc"', '"abd"'), pub); console.log("TAMPER NOT DETECTED"); } catch (e) { console.log("tamper rejected:", e.message); } diff --git a/atomic-wasm/src/lib.rs b/atomic-wasm/src/lib.rs new file mode 100644 index 00000000..e4c2d55d --- /dev/null +++ b/atomic-wasm/src/lib.rs @@ -0,0 +1,107 @@ +//! atomic-canonical for the browser. +//! +//! A web page that holds an atomic identity's key in WebCrypto (as a +//! non-extractable Ed25519 key) signs atomic documents without the key ever +//! reaching Rust — or leaving the browser. This crate does everything *but* +//! the signing, with the same code the CLI uses, so the result is an ordinary +//! atomic attestation (`eddsa-jcs-2022`) that `atomic` verifies: +//! +//! ```js +//! const prepared = prepareAttestation(JSON.stringify(doc), publicKeyBytes); +//! const sig = await crypto.subtle.sign("Ed25519", key, prepared.signingBytes); +//! const attested = attachProof(prepared.document, publicKeyBytes, new Uint8Array(sig)); +//! ``` +//! +//! Every function takes and returns JSON as strings and keys/signatures as +//! bytes, so there is no JS object model to keep in step with the Rust types. + +use atomic_canonical::{did, jcs, proof}; +use atomic_identity::keypair::PublicKey; +use atomic_identity::signing::Signature; +use wasm_bindgen::prelude::*; + +fn public_key(bytes: &[u8]) -> Result { + let bytes: &[u8; 32] = bytes + .try_into() + .map_err(|_| JsError::new("an Ed25519 public key is 32 bytes"))?; + PublicKey::from_bytes(bytes).map_err(|e| JsError::new(&e.to_string())) +} + +fn parse(json: &str) -> Result { + serde_json::from_str(json).map_err(|e| JsError::new(&format!("not JSON: {e}"))) +} + +/// A document ready to sign: the document to hand back to [`attach_proof`], +/// and the bytes the signature must cover. +#[wasm_bindgen] +pub struct Prepared { + document: String, + signing_bytes: Vec, +} + +#[wasm_bindgen] +impl Prepared { + /// The document with `attributedTo` and `contentHash` filled in (JSON). + #[wasm_bindgen(getter)] + pub fn document(&self) -> String { + self.document.clone() + } + + /// What to sign with the identity's Ed25519 key. + #[wasm_bindgen(getter, js_name = signingBytes)] + pub fn signing_bytes(&self) -> Vec { + self.signing_bytes.clone() + } +} + +/// Fill in the author and content hash of `document` (JSON) for the key +/// `public_key`, and return it with the bytes to sign. +#[wasm_bindgen(js_name = prepareAttestation)] +pub fn prepare_attestation(document: &str, public_key_bytes: &[u8]) -> Result { + let pk = public_key(public_key_bytes)?; + let prepared = proof::prepare_attestation(parse(document)?, &pk); + Ok(Prepared { + document: prepared.value.to_string(), + signing_bytes: prepared.signing_bytes, + }) +} + +/// Attach the proof for `signature` (64 bytes, over the prepared signing +/// bytes) and check it verifies. Returns the attested document (JSON). +#[wasm_bindgen(js_name = attachProof)] +pub fn attach_proof( + document: &str, + public_key_bytes: &[u8], + signature: &[u8], +) -> Result { + let pk = public_key(public_key_bytes)?; + let sig = Signature::from_slice(signature).map_err(|e| JsError::new(&e.to_string()))?; + let attested = proof::attach_proof(parse(document)?, &pk, &sig); + proof::verify_value(&attested, &pk).map_err(|e| JsError::new(&e.to_string()))?; + Ok(attested.to_string()) +} + +/// Check an attested document (JSON) against a public key. +#[wasm_bindgen(js_name = verifyAttestation)] +pub fn verify_attestation(document: &str, public_key_bytes: &[u8]) -> Result<(), JsError> { + let pk = public_key(public_key_bytes)?; + proof::verify_value(&parse(document)?, &pk).map_err(|e| JsError::new(&e.to_string())) +} + +/// The `did:atomic` identifier for a public key. +#[wasm_bindgen(js_name = didForPublicKey)] +pub fn did_for_public_key(public_key_bytes: &[u8]) -> Result { + Ok(did::did_for_public_key(&public_key(public_key_bytes)?)) +} + +/// The key's base32 form — what atomic writes as the `kid` of its tokens. +#[wasm_bindgen(js_name = publicKeyBase32)] +pub fn public_key_base32(public_key_bytes: &[u8]) -> Result { + Ok(public_key(public_key_bytes)?.to_base32()) +} + +/// RFC 8785 canonical JSON of `document` — the bytes atomic hashes and signs. +#[wasm_bindgen] +pub fn canonicalize(document: &str) -> Result { + Ok(jcs::canonicalize(&parse(document)?)) +}