diff --git a/AGENTS.md b/AGENTS.md index 9d155bf..3433d3f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,6 +95,7 @@ When reviewing or writing code, check for: - ❌ Unclear variable names (x, temp, data) ## Project operations +- run tests with `cargo test -q` to keep output minimal (passing test names are suppressed, failures still shown in full); the test logger defaults to `Warn` for third-party crates — raise levels via `storage_tests/storage_testing_config.toml` or `RUST_LOG` when debugging - to fully rebuild project and restart docker compose services use `./dev.sh` - to rebuild just UI/frontend `npm --prefix site run build` - to rebuild just backend `./dev.sh -b` diff --git a/Cargo.lock b/Cargo.lock index d871408..f9b81d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,12 +46,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "anstream" version = "0.6.21" @@ -132,6 +126,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "argon2" version = "0.5.3" @@ -163,7 +166,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -198,7 +201,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -209,9 +212,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.10" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1856b1b48b65f71a4dd940b1c0931f9a7b646d4a924b9828ffefc1454714668a" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -221,6 +224,7 @@ dependencies = [ "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", @@ -234,9 +238,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.9" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86590e57ea40121d47d3f2e131bfd873dea15d78dc2f4604f4734537ad9e56c4" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -270,9 +274,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.14" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe0fd441565b0b318c76e7206c8d1d0b0166b3e986cf30e890b61feb6192045" +checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -284,9 +288,12 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", + "bytes-utils", "fastrand", "http 0.2.12", + "http 1.3.1", "http-body 0.4.6", + "http-body 1.0.1", "percent-encoding", "pin-project-lite", "tracing", @@ -295,10 +302,11 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.112.0" +version = "1.145.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee73a27721035c46da0572b390a69fbdb333d0177c24f3d8f7ff952eeb96690" +checksum = "f0e6320417a37c8a62f78b443d0b4cf628b57cd340a09b0eb56173d47cc94e93" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-sigv4", @@ -307,54 +315,60 @@ dependencies = [ "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "bytes", "fastrand", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.3.1", - "http-body 0.4.6", - "lru 0.12.5", + "http-body 1.0.1", + "lru 0.18.4", "percent-encoding", "regex-lite", - "sha2 0.10.9", + "sha2 0.11.0", "tracing", "url", ] [[package]] name = "aws-sdk-sts" -version = "1.92.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c7808adcff8333eaa76a849e6de926c6ac1a1268b9fd6afe32de9c29ef29d2" +checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "fastrand", "http 0.2.12", + "http 1.3.1", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.3.6" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -362,16 +376,15 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "crypto-bigint 0.5.5", + "crypto-bigint", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.3.1", "p256", "percent-encoding", - "ring", - "sha2 0.10.9", + "sha2 0.11.0", "subtle", "time", "tracing", @@ -380,9 +393,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.6" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "127fcfad33b7dfc531141fda7e1c402ac65f88aca5511a4d31e2e3d2cd01ce9c" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -391,29 +404,30 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.63.11" +version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95bd108f7b3563598e4dc7b62e1388c9982324a2abd622442167012690184591" +checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", "crc-fast", "hex", - "http 0.2.12", - "http-body 0.4.6", - "md-5", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "md-5 0.11.0", "pin-project-lite", - "sha1", - "sha2 0.10.9", + "sha1 0.11.0", + "sha2 0.11.0", "tracing", ] [[package]] name = "aws-smithy-eventstream" -version = "0.60.13" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29a304f8319781a39808847efb39561351b1bb76e933da7aa90232673638658" +checksum = "6de526c7b567420a31bc283657a7921b45c4cafe0827fdf2490713dcc770c28f" dependencies = [ "aws-smithy-types", "bytes", @@ -422,9 +436,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.5" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -433,9 +447,9 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 0.2.12", "http 1.3.1", - "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", "percent-encoding", "pin-project-lite", "pin-utils", @@ -444,9 +458,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "623254723e8dfd535f566ee7b2381645f8981da086b5c4aa26c0c41582bb1d2c" +checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -464,7 +478,7 @@ dependencies = [ "pin-project-lite", "rustls 0.21.12", "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -474,43 +488,49 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.7" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1881b1ea6d313f9890710d65c158bdab6fb08c91ea825f74c1c8c357baf4cc" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.8" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d28a63441360c477465f80c7abac3b9c4d075ca638f982e605b7dc2a2c7156c9" +checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.9.4" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bbe9d018d646b96c7be063dd07987849862b0e6d07c778aad7d93d1be6c1ef0" +checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", "aws-smithy-observability", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "bytes", "fastrand", @@ -518,6 +538,7 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", + "http-body-util", "pin-project-lite", "pin-utils", "tokio", @@ -526,11 +547,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7204f9fd94749a7c53b26da1b961b4ac36bf070ef1e0b94bb09f79d4f6c193" +checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", @@ -541,11 +563,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.3.1", +] + [[package]] name = "aws-smithy-types" -version = "1.3.4" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f535879a207fce0db74b679cfc3e91a3159c8144d717d55f5832aea9eef46e" +checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" dependencies = [ "base64-simd", "bytes", @@ -569,22 +613,26 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.12" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.10" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d79fb68e3d7fe5d4833ea34dc87d2e97d26d3086cb3da660bb6b1f76d98680b6" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "rustc_version", "tracing", @@ -617,7 +665,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.6", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -680,14 +728,14 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] name = "base16ct" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" @@ -695,12 +743,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -732,7 +774,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "log", "prettyplease", "proc-macro2", @@ -740,7 +782,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -820,9 +862,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -882,12 +924,6 @@ dependencies = [ "wasm-bindgen-test", ] -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.2.46" @@ -945,33 +981,6 @@ dependencies = [ "stacker", ] -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -1014,7 +1023,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1032,6 +1041,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cobs" version = "0.3.0" @@ -1160,15 +1175,12 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc-fast" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ddc2d09feefeee8bd78101665bd8645637828fa9317f9f292496dbbd8c65ff3" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "crc", "digest 0.10.7", - "rand 0.9.2", - "regex", - "rustversion", + "spin 0.10.1", ] [[package]] @@ -1180,42 +1192,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - [[package]] name = "critical-section" version = "1.2.0" @@ -1242,16 +1218,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - [[package]] name = "crossbeam-epoch" version = "0.9.18" @@ -1307,26 +1273,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-bigint" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - [[package]] name = "crypto-bigint" version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ + "generic-array", "rand_core 0.6.4", "subtle", + "zeroize", ] [[package]] @@ -1348,6 +1304,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "current_semver" version = "0.1.1" @@ -1356,7 +1321,7 @@ checksum = "61c60526ae35a7665328ec3de4f4bef07ef9630f824d73092d6f863c3ab049fe" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1404,7 +1369,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1426,7 +1391,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1448,16 +1413,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" -[[package]] -name = "der" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" -dependencies = [ - "const-oid 0.9.6", - "zeroize", -] - [[package]] name = "der" version = "0.7.10" @@ -1486,7 +1441,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1507,7 +1462,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1517,7 +1472,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1537,7 +1492,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", "unicode-xid", ] @@ -1568,6 +1523,7 @@ dependencies = [ "block-buffer 0.12.0", "const-oid 0.10.2", "crypto-common 0.2.1", + "ctutils", ] [[package]] @@ -1590,7 +1546,7 @@ source = "git+https://github.com/kshcherban/digestible.git#3a32cbac4ddbb25315da7 dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1611,7 +1567,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1622,7 +1578,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -1651,14 +1607,16 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" -version = "0.14.8" +version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.6.1", + "der", + "digest 0.10.7", "elliptic-curve", "rfc6979", - "signature 1.6.4", + "signature", + "spki", ] [[package]] @@ -1685,18 +1643,18 @@ dependencies = [ [[package]] name = "elliptic-curve" -version = "0.12.3" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", - "crypto-bigint 0.4.9", - "der 0.6.1", + "crypto-bigint", "digest 0.10.7", "ff", "generic-array", "group", - "pkcs8 0.9.0", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", "sec1", "subtle", @@ -1753,7 +1711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1796,9 +1754,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ "rand_core 0.6.4", "subtle", @@ -1967,7 +1925,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -2026,6 +1984,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2076,9 +2035,9 @@ dependencies = [ [[package]] name = "group" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", "rand_core 0.6.4", @@ -2123,17 +2082,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - [[package]] name = "handlebars" version = "6.3.2" @@ -2198,6 +2146,17 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -2228,7 +2187,7 @@ dependencies = [ "http 1.3.1", "httpdate", "mime", - "sha1", + "sha1 0.10.6", ] [[package]] @@ -2260,12 +2219,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - [[package]] name = "hex" version = "0.4.3" @@ -2278,7 +2231,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -2290,6 +2243,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.2", +] + [[package]] name = "home" version = "0.5.12" @@ -2434,7 +2396,6 @@ dependencies = [ "hyper 0.14.32", "log", "rustls 0.21.12", - "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] @@ -2449,7 +2410,7 @@ dependencies = [ "hyper 1.8.1", "hyper-util", "rustls 0.23.35", - "rustls-native-certs 0.8.2", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -2692,17 +2653,6 @@ dependencies = [ "serde", ] -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -2718,15 +2668,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -2738,9 +2679,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jobserver" @@ -2775,7 +2716,7 @@ dependencies = [ "pem", "serde", "serde_json", - "signature 2.2.0", + "signature", "simple_asn1", ] @@ -2906,20 +2847,20 @@ checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "lru" -version = "0.12.5" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +checksum = "96051b46fc183dc9cd4a223960ef37b9af631b55191852a8274bfef064cda20f" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.0", ] [[package]] name = "lru" -version = "0.16.2" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96051b46fc183dc9cd4a223960ef37b9af631b55191852a8274bfef064cda20f" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ - "hashbrown 0.16.0", + "hashbrown 0.17.1", ] [[package]] @@ -2981,6 +2922,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.2", +] + [[package]] name = "memchr" version = "2.7.6" @@ -3075,7 +3026,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3166,7 +3117,6 @@ dependencies = [ name = "nr-api" version = "1.0.0-BETA" dependencies = [ - "anyhow", "bytes", "chrono", "reqwest", @@ -3174,8 +3124,6 @@ dependencies = [ "serde_json", "thiserror 2.0.17", "tokio", - "tracing-appender", - "tracing-subscriber", "url", "uuid", ] @@ -3216,11 +3164,10 @@ dependencies = [ name = "nr-macros" version = "1.0.0-BETA" dependencies = [ - "anyhow", "prettyplease", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3250,7 +3197,7 @@ dependencies = [ "hyper-util", "ipnet", "lru 0.16.2", - "md-5", + "md-5 0.10.6", "mime", "mime_guess", "nr-core", @@ -3259,7 +3206,7 @@ dependencies = [ "postcard", "serde", "serde_json", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "sha3", "strum", @@ -3271,8 +3218,6 @@ dependencies = [ "tokio-util", "toml", "tracing", - "tracing-appender", - "tracing-subscriber", "url", "utoipa", "uuid", @@ -3294,7 +3239,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3418,12 +3363,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "openssl-probe" version = "0.1.6" @@ -3533,12 +3472,13 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "p256" -version = "0.11.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", "elliptic-curve", + "primeorder", "sha2 0.10.9", ] @@ -3637,7 +3577,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3683,7 +3623,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3703,7 +3643,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -3724,19 +3664,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.10", - "pkcs8 0.10.2", - "spki 0.7.3", -] - -[[package]] -name = "pkcs8" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" -dependencies = [ - "der 0.6.1", - "spki 0.6.0", + "der", + "pkcs8", + "spki", ] [[package]] @@ -3745,8 +3675,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.10", - "spki 0.7.3", + "der", + "spki", ] [[package]] @@ -3774,7 +3704,6 @@ dependencies = [ "casbin", "chrono", "clap", - "criterion", "cron", "current_semver", "dashmap", @@ -3796,7 +3725,7 @@ dependencies = [ "jsonwebtoken", "lettre", "maven-rs", - "md-5", + "md-5 0.10.6", "mime", "mime_guess", "mockall", @@ -3820,7 +3749,7 @@ dependencies = [ "rsa", "rust-embed", "rustls 0.23.35", - "rustls-pemfile 2.2.0", + "rustls-pemfile", "schemars", "semver", "serde", @@ -3829,7 +3758,7 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "serde_yaml", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "sha2 0.11.0", "sha3", @@ -3880,34 +3809,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - [[package]] name = "portable-atomic" version = "1.11.1" @@ -3994,14 +3895,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.110", + "syn 2.0.119", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", ] [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -4026,7 +3936,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4101,14 +4011,14 @@ dependencies = [ "once_cell", "socket2 0.6.1", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -4184,26 +4094,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "rayon" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redb" version = "3.1.0" @@ -4250,7 +4140,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4333,13 +4223,12 @@ dependencies = [ [[package]] name = "rfc6979" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "crypto-bigint 0.4.9", - "hmac", - "zeroize", + "hmac 0.12.1", + "subtle", ] [[package]] @@ -4369,7 +4258,7 @@ checksum = "d4322a2a4e8cf30771dd9f27f7f37ca9ac8fe812dddd811096a98483080dabe6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4398,10 +4287,10 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8 0.10.2", + "pkcs8", "rand_core 0.6.4", - "signature 2.2.0", - "spki 0.7.3", + "signature", + "spki", "subtle", "zeroize", ] @@ -4427,7 +4316,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.110", + "syn 2.0.119", "walkdir", ] @@ -4467,7 +4356,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4498,18 +4387,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.8.2" @@ -4519,16 +4396,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", + "security-framework", ] [[package]] @@ -4580,9 +4448,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "same-file" @@ -4622,7 +4490,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4643,31 +4511,18 @@ dependencies = [ [[package]] name = "sec1" -version = "0.3.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der 0.6.1", + "der", "generic-array", - "pkcs8 0.9.0", + "pkcs8", "subtle", "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.5.1" @@ -4738,7 +4593,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4749,7 +4604,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -4843,6 +4698,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] + [[package]] name = "sha1_smol" version = "1.0.1" @@ -4935,16 +4801,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" -dependencies = [ - "digest 0.10.7", - "rand_core 0.6.4", -] - [[package]] name = "signature" version = "2.2.0" @@ -5036,14 +4892,10 @@ dependencies = [ ] [[package]] -name = "spki" -version = "0.6.0" +name = "spin" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" -dependencies = [ - "base64ct", - "der 0.6.1", -] +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -5052,7 +4904,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.10", + "der", ] [[package]] @@ -5126,7 +4978,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5149,7 +5001,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.110", + "syn 2.0.119", "tokio", "url", ] @@ -5177,17 +5029,17 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", "rand 0.8.5", "rsa", "serde", - "sha1", + "sha1 0.10.6", "sha2 0.10.9", "smallvec", "sqlx-core", @@ -5217,11 +5069,11 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "rand 0.8.5", @@ -5329,7 +5181,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5351,9 +5203,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.110" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -5377,7 +5229,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5442,7 +5294,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5460,7 +5312,7 @@ dependencies = [ "bollard-stubs", "futures", "hex", - "hmac", + "hmac 0.12.1", "log", "rand 0.8.5", "serde", @@ -5503,7 +5355,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5514,7 +5366,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5589,16 +5441,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "tinyvec" version = "1.10.0" @@ -5616,9 +5458,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -5638,7 +5480,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -5686,13 +5528,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -5813,9 +5656,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -5837,20 +5680,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -5934,7 +5777,7 @@ dependencies = [ "httparse", "log", "rand 0.9.2", - "sha1", + "sha1 0.10.6", "thiserror 2.0.17", "utf-8", ] @@ -6096,7 +5939,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.110", + "syn 2.0.119", "url", "uuid", ] @@ -6235,7 +6078,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -6269,7 +6112,7 @@ checksum = "085b2df989e1e6f9620c1311df6c996e83fe16f57792b272ce1e024ac16a90f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -6355,7 +6198,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -6404,7 +6247,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -6415,7 +6258,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -6764,7 +6607,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", "synstructure", ] @@ -6785,7 +6628,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] @@ -6805,7 +6648,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", "synstructure", ] @@ -6845,7 +6688,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.119", ] [[package]] diff --git a/Dockerfile b/Dockerfile index 3c823b8..0118794 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN --mount=type=cache,target=/root/.npm npm run build ############################ # Rust build stage ############################ -FROM rust:1.95.0 AS rust-base +FROM rust:1.98.0 AS rust-base WORKDIR /app RUN apt-get update \ diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..f5a1f32 --- /dev/null +++ b/TODO.md @@ -0,0 +1,33 @@ +# S3 performance and robustness plan + +This checklist records the S3 performance review and its implementation status. + +## Engineering work + +- [x] Eliminate duplicate Docker blob downloads. Preserve verified digests and deliver verified + content from the same read, including large objects that use bounded temporary files. +- [x] Replace quadratic S3 append read/rewrite behavior for Docker uploads with bounded local + staging and one guarded upload during finalization. Preserve the generic storage append + contract for other repository types. +- [x] Bound total memory and temporary storage consumption. Stream large S3 writes and cache + reads, apply shared body and temporary-file budgets, and spool incoming upload chunks in + bounded buffers. +- [x] Guard cache publication against concurrent mutations. Use per-object generations and + per-key miss coordination so stale in-flight reads cannot replace newer cache entries. +- [x] Reduce metadata round trips and listing contention. Memoize safe ancestor probes, coordinate + cold manifest loads per repository, and paginate manifest listings directly from S3. +- [x] Add Docker accounting reconciliation. Reconcile PostgreSQL rows with storage inventory, + backfill missing manifest graphs, correct size drift, remove missing rows, and use revisions + to avoid overwriting concurrent writes. +- [x] Document S3 cache integrity behavior, Docker upload staging, resource limits, and the + one-writer-per-S3-repository deployment constraint. + +## Validation work + +- [x] Add regression coverage for cache publication races, concurrent cache misses, staged + appends, and accounting backfill detection. +- [x] Run the workspace Rust test suite, S3 storage tests, Clippy, formatting checks, and the + real MinIO S3 integration suite. +- [ ] Benchmark cold and warm pulls plus chunked pushes at 1, 16, and 64 concurrent clients, + using 1 MiB, 64 MiB, and 1 GiB artifacts. Record p95 time to first byte, throughput, peak + memory, S3 request counts, and transferred bytes per delivered byte. diff --git a/benches/search_db.rs b/benches/search_db.rs deleted file mode 100644 index cc6a6d3..0000000 --- a/benches/search_db.rs +++ /dev/null @@ -1,221 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use criterion::{criterion_group, criterion_main, Criterion}; -use pkgly::{ - app::api::search::query_parser::SearchQuery, - repository::repo_type::NewRepository, - search::PackageSearchRepository, -}; -use nr_core::{ - database::entities::{ - project::{ - NewProject, - versions::NewVersion, - }, - storage::NewDBStorage, - }, - repository::project::{ReleaseType, VersionData}, - storage::StorageName, -}; -use serde_json::json; -use sqlx::PgPool; -use tokio::runtime::Runtime; -use uuid::Uuid; - -const BENCH_ROW_COUNT: usize = 10_000; - -fn search_db_query(c: &mut Criterion) { - let dsn = match std::env::var("PKGLY_SEARCH_BENCH_DSN") { - Ok(value) => value, - Err(_) => { - c.bench_function("search_db_query/skipped", |b| b.iter(|| ())); - return; - } - }; - - let runtime = Runtime::new().expect("start tokio runtime"); - let (pool, repository_id) = runtime.block_on(async { - let pool = PgPool::connect(&dsn) - .await - .expect("connect benchmark database"); - let repository_id = seed_dataset(&pool) - .await - .expect("seed benchmark dataset"); - (pool, repository_id) - }); - let pool = Arc::new(pool); - let query = SearchQuery::default(); - - c.bench_function("search_db_query", |b| { - b.to_async(&runtime).iter(|| { - let pool = pool.clone(); - let query = query.clone(); - async move { - let searcher = PackageSearchRepository::new(pool.as_ref()); - let rows = searcher - .fetch_repository_rows(repository_id, &query, 25) - .await - .expect("query repository"); - criterion::black_box(rows.len()); - } - }); - }); -} - -async fn seed_dataset(pool: &PgPool) -> Result { - reset_database(pool).await?; - let storage_id = insert_storage(pool).await?; - let repository_id = insert_repository(pool, storage_id).await?; - for idx in 0..BENCH_ROW_COUNT { - let package = format!("package-{idx}"); - let version = format!("1.0.{idx}"); - insert_package(pool, repository_id, &package, &version).await?; - } - Ok(repository_id) -} - -async fn insert_storage(pool: &PgPool) -> Result { - let name = StorageName::new(format!("bench-storage-{}", Uuid::new_v4().simple())) - .expect("valid storage name"); - let storage = NewDBStorage::new("Local".into(), name, json!({ "path": "/tmp" })); - let inserted = storage.insert(pool).await?.expect("insert storage row"); - Ok(inserted.id) -} - -async fn insert_repository(pool: &PgPool, storage_id: Uuid) -> Result { - let name = format!("bench-repo-{}", Uuid::new_v4().simple()); - let repo = NewRepository { - name, - uuid: Uuid::new_v4(), - repository_type: "npm".into(), - configs: HashMap::with_hasher(Default::default()), - }; - let row = repo.insert(storage_id, pool).await?; - Ok(row.id) -} - -async fn insert_package( - pool: &PgPool, - repository_id: Uuid, - package: &str, - version: &str, -) -> Result<(), sqlx::Error> { - let project = NewProject { - scope: None, - project_key: package.to_string(), - name: package.to_string(), - description: None, - repository: repository_id, - storage_path: format!("packages/{package}"), - } - .insert(pool) - .await?; - - let mut extra = VersionData::default(); - extra.extra = Some(json!({ - "size": 512, - "filename": format!("packages/{package}/{package}-{version}.tgz"), - })); - - NewVersion { - project_id: project.id, - repository_id, - version: version.to_string(), - release_type: ReleaseType::Stable, - version_path: format!("packages/{package}/{version}/{package}-{version}.tgz"), - publisher: None, - version_page: None, - extra, - } - .insert(pool) - .await?; - Ok(()) -} - -async fn reset_database(pool: &PgPool) -> Result<(), sqlx::Error> { - sqlx::query("DROP TABLE IF EXISTS project_versions, projects, repositories, storages CASCADE") - .execute(pool) - .await?; - ensure_schema(pool).await?; - Ok(()) -} - -async fn ensure_schema(pool: &PgPool) -> Result<(), sqlx::Error> { - sqlx::query(r#"CREATE EXTENSION IF NOT EXISTS "pgcrypto""#) - .execute(pool) - .await?; - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS storages ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - storage_type TEXT NOT NULL, - name TEXT NOT NULL, - config JSONB NOT NULL, - active BOOLEAN NOT NULL DEFAULT TRUE, - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT storages_name_key UNIQUE (name) - ) - "#, - ) - .execute(pool) - .await?; - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS repositories ( - id UUID PRIMARY KEY, - storage_id UUID NOT NULL, - name TEXT NOT NULL, - repository_type TEXT NOT NULL, - visibility VARCHAR(64) NOT NULL DEFAULT 'Public', - active BOOLEAN NOT NULL DEFAULT TRUE, - storage_usage_bytes BIGINT, - storage_usage_updated_at TIMESTAMPTZ, - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() - ) - "#, - ) - .execute(pool) - .await?; - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS projects ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - scope TEXT, - key TEXT NOT NULL, - name TEXT NOT NULL, - description TEXT, - repository_id UUID NOT NULL, - path TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() - ) - "#, - ) - .execute(pool) - .await?; - sqlx::query( - r#" - CREATE TABLE IF NOT EXISTS project_versions ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - project_id UUID NOT NULL, - repository_id UUID NOT NULL, - version TEXT NOT NULL, - release_type TEXT NOT NULL, - path TEXT NOT NULL, - publisher INTEGER, - version_page TEXT, - extra JSONB NOT NULL DEFAULT '{}'::jsonb, - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() - ) - "#, - ) - .execute(pool) - .await?; - Ok(()) -} - -criterion_group!(benches, search_db_query); -criterion_main!(benches); diff --git a/crates/core/migrations/20260904120000_s3_region_strings.down.sql b/crates/core/migrations/20260904120000_s3_region_strings.down.sql new file mode 100644 index 0000000..c24e4d2 --- /dev/null +++ b/crates/core/migrations/20260904120000_s3_region_strings.down.sql @@ -0,0 +1,44 @@ +-- ABOUTME: Reverses known S3 raw region identifiers for legacy deployments. +-- ABOUTME: Leaves newer identifiers unchanged because enum-only binaries cannot represent them. +UPDATE storages +SET config = jsonb_set( + config, + '{settings,region}', + to_jsonb(CASE config #>> '{settings,region}' + WHEN 'us-east-1' THEN 'UsEast1' + WHEN 'us-east-2' THEN 'UsEast2' + WHEN 'us-west-1' THEN 'UsWest1' + WHEN 'us-west-2' THEN 'UsWest2' + WHEN 'ca-central-1' THEN 'CaCentral1' + WHEN 'af-south-1' THEN 'AfSouth1' + WHEN 'ap-east-1' THEN 'ApEast1' + WHEN 'ap-south-1' THEN 'ApSouth1' + WHEN 'ap-northeast-1' THEN 'ApNortheast1' + WHEN 'ap-northeast-2' THEN 'ApNortheast2' + WHEN 'ap-northeast-3' THEN 'ApNortheast3' + WHEN 'ap-southeast-1' THEN 'ApSoutheast1' + WHEN 'ap-southeast-2' THEN 'ApSoutheast2' + WHEN 'cn-north-1' THEN 'CnNorth1' + WHEN 'cn-northwest-1' THEN 'CnNorthwest1' + WHEN 'eu-north-1' THEN 'EuNorth1' + WHEN 'eu-central-1' THEN 'EuCentral1' + WHEN 'eu-central-2' THEN 'EuCentral2' + WHEN 'eu-west-1' THEN 'EuWest1' + WHEN 'eu-west-2' THEN 'EuWest2' + WHEN 'eu-west-3' THEN 'EuWest3' + WHEN 'il-central-1' THEN 'IlCentral1' + WHEN 'me-south-1' THEN 'MeSouth1' + WHEN 'sa-east-1' THEN 'SaEast1' + ELSE config #>> '{settings,region}' + END), + true +) +WHERE lower(storage_type) = 's3' + AND config #>> '{type}' = 'S3' + AND config #>> '{settings,region}' IN ( + 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', 'ca-central-1', 'af-south-1', + 'ap-east-1', 'ap-south-1', 'ap-northeast-1', 'ap-northeast-2', 'ap-northeast-3', + 'ap-southeast-1', 'ap-southeast-2', 'cn-north-1', 'cn-northwest-1', 'eu-north-1', + 'eu-central-1', 'eu-central-2', 'eu-west-1', 'eu-west-2', 'eu-west-3', 'il-central-1', + 'me-south-1', 'sa-east-1' + ); diff --git a/crates/core/migrations/20260904120000_s3_region_strings.up.sql b/crates/core/migrations/20260904120000_s3_region_strings.up.sql new file mode 100644 index 0000000..0c10f3b --- /dev/null +++ b/crates/core/migrations/20260904120000_s3_region_strings.up.sql @@ -0,0 +1,44 @@ +-- ABOUTME: Migrates legacy S3 enum region tokens to canonical raw identifiers. +-- ABOUTME: Keeps persisted S3 configuration compatible with string-based regions. +UPDATE storages +SET config = jsonb_set( + config, + '{settings,region}', + to_jsonb(CASE config #>> '{settings,region}' + WHEN 'UsEast1' THEN 'us-east-1' + WHEN 'UsEast2' THEN 'us-east-2' + WHEN 'UsWest1' THEN 'us-west-1' + WHEN 'UsWest2' THEN 'us-west-2' + WHEN 'CaCentral1' THEN 'ca-central-1' + WHEN 'AfSouth1' THEN 'af-south-1' + WHEN 'ApEast1' THEN 'ap-east-1' + WHEN 'ApSouth1' THEN 'ap-south-1' + WHEN 'ApNortheast1' THEN 'ap-northeast-1' + WHEN 'ApNortheast2' THEN 'ap-northeast-2' + WHEN 'ApNortheast3' THEN 'ap-northeast-3' + WHEN 'ApSoutheast1' THEN 'ap-southeast-1' + WHEN 'ApSoutheast2' THEN 'ap-southeast-2' + WHEN 'CnNorth1' THEN 'cn-north-1' + WHEN 'CnNorthwest1' THEN 'cn-northwest-1' + WHEN 'EuNorth1' THEN 'eu-north-1' + WHEN 'EuCentral1' THEN 'eu-central-1' + WHEN 'EuCentral2' THEN 'eu-central-2' + WHEN 'EuWest1' THEN 'eu-west-1' + WHEN 'EuWest2' THEN 'eu-west-2' + WHEN 'EuWest3' THEN 'eu-west-3' + WHEN 'IlCentral1' THEN 'il-central-1' + WHEN 'MeSouth1' THEN 'me-south-1' + WHEN 'SaEast1' THEN 'sa-east-1' + ELSE config #>> '{settings,region}' + END), + true +) +WHERE lower(storage_type) = 's3' + AND config #>> '{type}' = 'S3' + AND config #>> '{settings,region}' IN ( + 'UsEast1', 'UsEast2', 'UsWest1', 'UsWest2', 'CaCentral1', 'AfSouth1', + 'ApEast1', 'ApSouth1', 'ApNortheast1', 'ApNortheast2', 'ApNortheast3', + 'ApSoutheast1', 'ApSoutheast2', 'CnNorth1', 'CnNorthwest1', 'EuNorth1', + 'EuCentral1', 'EuCentral2', 'EuWest1', 'EuWest2', 'EuWest3', 'IlCentral1', + 'MeSouth1', 'SaEast1' + ); diff --git a/crates/core/migrations/20260905140000_package_files_referenced_size.down.sql b/crates/core/migrations/20260905140000_package_files_referenced_size.down.sql new file mode 100644 index 0000000..f30deaa --- /dev/null +++ b/crates/core/migrations/20260905140000_package_files_referenced_size.down.sql @@ -0,0 +1,3 @@ +-- ABOUTME: Drops the persisted referenced-size column when rolling back the listing change. +ALTER TABLE package_files +DROP COLUMN referenced_size_bytes; diff --git a/crates/core/migrations/20260905140000_package_files_referenced_size.up.sql b/crates/core/migrations/20260905140000_package_files_referenced_size.up.sql new file mode 100644 index 0000000..8ba74ea --- /dev/null +++ b/crates/core/migrations/20260905140000_package_files_referenced_size.up.sql @@ -0,0 +1,4 @@ +-- ABOUTME: Adds a persisted referenced-size column so Docker package listings avoid storage reads. +-- ABOUTME: Holds the total stored bytes of a manifest and its referenced blobs for the UI. +ALTER TABLE package_files +ADD COLUMN referenced_size_bytes BIGINT; diff --git a/crates/core/migrations/20260906120000_docker_objects.down.sql b/crates/core/migrations/20260906120000_docker_objects.down.sql new file mode 100644 index 0000000..c34dc2d --- /dev/null +++ b/crates/core/migrations/20260906120000_docker_objects.down.sql @@ -0,0 +1,4 @@ +-- ABOUTME: Removes database-backed Docker object accounting. +-- ABOUTME: Leaves catalog rows available for storage-based size reconstruction. +DROP FUNCTION docker_referenced_size(UUID, TEXT); +DROP TABLE docker_objects; diff --git a/crates/core/migrations/20260906120000_docker_objects.up.sql b/crates/core/migrations/20260906120000_docker_objects.up.sql new file mode 100644 index 0000000..02e3da6 --- /dev/null +++ b/crates/core/migrations/20260906120000_docker_objects.up.sql @@ -0,0 +1,26 @@ +-- ABOUTME: Records stored Docker object sizes and direct manifest references. +-- ABOUTME: Calculates distinct reachable cached bytes without object-store lookups. +CREATE TABLE docker_objects ( + repository_id UUID NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + path TEXT NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0), + references_paths TEXT[] NOT NULL DEFAULT '{}', + PRIMARY KEY (repository_id, path) +); + +CREATE FUNCTION docker_referenced_size(repo UUID, root_path TEXT) +RETURNS BIGINT LANGUAGE SQL STABLE AS $$ + WITH RECURSIVE reachable(path) AS ( + SELECT path FROM docker_objects WHERE repository_id = repo AND path = root_path + UNION + SELECT unnest(objects.references_paths) + FROM docker_objects objects JOIN reachable ON objects.path = reachable.path + WHERE objects.repository_id = repo + ) + SELECT SUM(objects.size_bytes)::BIGINT + FROM reachable JOIN docker_objects objects USING (path) + WHERE objects.repository_id = repo +$$; + +-- Scalar snapshots cannot establish which referenced objects are still stored. +UPDATE package_files SET referenced_size_bytes = NULL; diff --git a/crates/core/migrations/20260906130000_docker_object_revisions.down.sql b/crates/core/migrations/20260906130000_docker_object_revisions.down.sql new file mode 100644 index 0000000..464a989 --- /dev/null +++ b/crates/core/migrations/20260906130000_docker_object_revisions.down.sql @@ -0,0 +1,4 @@ +-- ABOUTME: Removes optimistic-concurrency revisions from Docker accounting rows. +-- ABOUTME: Restores the Docker accounting table to its pre-revision schema. +ALTER TABLE docker_objects + DROP COLUMN revision; diff --git a/crates/core/migrations/20260906130000_docker_object_revisions.up.sql b/crates/core/migrations/20260906130000_docker_object_revisions.up.sql new file mode 100644 index 0000000..aaf66cb --- /dev/null +++ b/crates/core/migrations/20260906130000_docker_object_revisions.up.sql @@ -0,0 +1,4 @@ +-- ABOUTME: Adds optimistic-concurrency revisions to Docker object accounting rows. +-- ABOUTME: Lets reconciliation avoid overwriting writes committed after its scan. +ALTER TABLE docker_objects + ADD COLUMN revision BIGINT NOT NULL DEFAULT 0; diff --git a/crates/core/src/database/entities/docker_object.rs b/crates/core/src/database/entities/docker_object.rs new file mode 100644 index 0000000..52b7775 --- /dev/null +++ b/crates/core/src/database/entities/docker_object.rs @@ -0,0 +1,221 @@ +// ABOUTME: Persists Docker object sizes and direct manifest reference paths. +// ABOUTME: Provides deduplicated cached-byte totals and deletion bookkeeping. +use sqlx::PgPool; +use uuid::Uuid; + +pub struct DBDockerObject; + +impl DBDockerObject { + /// Backfills an observed object without overwriting a concurrent recorded write. + /// Propagates size conversion and database errors. + pub async fn insert_missing( + database: &PgPool, + repository: Uuid, + path: &str, + size: u64, + references: &[String], + ) -> Result<(), sqlx::Error> { + let size = i64::try_from(size).map_err(|error| sqlx::Error::Encode(Box::new(error)))?; + sqlx::query( + "INSERT INTO docker_objects (repository_id, path, size_bytes, references_paths) + VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", + ) + .bind(repository) + .bind(path) + .bind(size) + .bind(references) + .execute(database) + .await?; + Ok(()) + } + /// Records a successful object write and replaces its manifest references. + /// Returns database errors, including sizes outside PostgreSQL's BIGINT range. + pub async fn upsert( + database: &PgPool, + repository: Uuid, + path: &str, + size: u64, + references: &[String], + ) -> Result<(), sqlx::Error> { + let size = i64::try_from(size).map_err(|error| sqlx::Error::Encode(Box::new(error)))?; + sqlx::query( + "INSERT INTO docker_objects (repository_id, path, size_bytes, references_paths) + VALUES ($1, $2, $3, $4) ON CONFLICT (repository_id, path) DO UPDATE + SET size_bytes = EXCLUDED.size_bytes, + references_paths = EXCLUDED.references_paths, + revision = docker_objects.revision + 1 + WHERE (docker_objects.size_bytes, docker_objects.references_paths) + IS DISTINCT FROM (EXCLUDED.size_bytes, EXCLUDED.references_paths)", + ) + .bind(repository) + .bind(path) + .bind(size) + .bind(references) + .execute(database) + .await?; + Ok(()) + } + + /// Returns distinct reachable stored bytes, or None for an unindexed root. + /// Propagates database errors; recursion terminates even for cyclic references. + pub async fn referenced_size( + database: &PgPool, + repository: Uuid, + path: &str, + ) -> Result, sqlx::Error> { + sqlx::query_scalar("SELECT docker_referenced_size($1, $2)") + .bind(repository) + .bind(path) + .fetch_one(database) + .await + } + + /// Reports whether an indexed manifest graph is missing its root or a reachable object row. + pub async fn needs_backfill( + database: &PgPool, + repository: Uuid, + root: &str, + ) -> Result { + sqlx::query_scalar( + "WITH RECURSIVE reachable(path) AS ( + SELECT $2::text + UNION + SELECT unnest(object.references_paths) + FROM docker_objects AS object + JOIN reachable ON reachable.path = object.path + WHERE object.repository_id = $1 + ) + SELECT NOT EXISTS ( + SELECT 1 FROM docker_objects + WHERE repository_id = $1 AND path = $2 + ) OR EXISTS ( + SELECT 1 + FROM reachable + LEFT JOIN docker_objects AS object + ON object.repository_id = $1 AND object.path = reachable.path + WHERE object.path IS NULL + )", + ) + .bind(repository) + .bind(root) + .fetch_one(database) + .await + } + + /// Returns the accounting revision for one object, if it is indexed. + pub async fn revision( + database: &PgPool, + repository: Uuid, + path: &str, + ) -> Result, sqlx::Error> { + sqlx::query_scalar( + "SELECT revision FROM docker_objects WHERE repository_id = $1 AND path = $2", + ) + .bind(repository) + .bind(path) + .fetch_optional(database) + .await + } + + /// Returns every recorded object path with its stored size for reconciliation. + pub async fn list_rows( + database: &PgPool, + repository: Uuid, + ) -> Result, sqlx::Error> { + sqlx::query_as( + "SELECT path, size_bytes, revision FROM docker_objects WHERE repository_id = $1", + ) + .bind(repository) + .fetch_all(database) + .await + } + + /// Corrects a size only when the row has not changed since reconciliation observed it. + pub async fn update_size_if_revision( + database: &PgPool, + repository: Uuid, + path: &str, + size: u64, + expected_revision: i64, + ) -> Result { + let size = i64::try_from(size).map_err(|error| sqlx::Error::Encode(Box::new(error)))?; + let result = sqlx::query( + "UPDATE docker_objects + SET size_bytes = $1, revision = revision + 1 + WHERE repository_id = $2 AND path = $3 AND revision = $4", + ) + .bind(size) + .bind(repository) + .bind(path) + .bind(expected_revision) + .execute(database) + .await?; + Ok(result.rows_affected() == 1) + } + + /// Rewrites an observed object graph only when its accounting revision is unchanged. + pub async fn upsert_if_revision( + database: &PgPool, + repository: Uuid, + path: &str, + size: u64, + references: &[String], + expected_revision: i64, + ) -> Result { + let size = i64::try_from(size).map_err(|error| sqlx::Error::Encode(Box::new(error)))?; + let result = sqlx::query( + "UPDATE docker_objects + SET size_bytes = $1, references_paths = $2, revision = revision + 1 + WHERE repository_id = $3 AND path = $4 AND revision = $5", + ) + .bind(size) + .bind(references) + .bind(repository) + .bind(path) + .bind(expected_revision) + .execute(database) + .await?; + Ok(result.rows_affected() == 1) + } + + /// Removes rows only when their revisions still match the reconciliation snapshot. + pub async fn delete_paths_if_revisions( + database: &PgPool, + repository: Uuid, + rows: &[(String, i64)], + ) -> Result { + if rows.is_empty() { + return Ok(0); + } + let paths: Vec<&str> = rows.iter().map(|(path, _)| path.as_str()).collect(); + let revisions: Vec = rows.iter().map(|(_, revision)| *revision).collect(); + let result = sqlx::query( + "DELETE FROM docker_objects AS object + USING unnest($2::text[], $3::bigint[]) AS candidate(path, revision) + WHERE object.repository_id = $1 + AND object.path = candidate.path + AND object.revision = candidate.revision", + ) + .bind(repository) + .bind(paths) + .bind(revisions) + .execute(database) + .await?; + Ok(result.rows_affected() as usize) + } + + /// Removes successfully deleted objects; references from other manifests remain. + /// Returns database errors without suppressing bookkeeping failures. + pub async fn delete_paths( + database: &PgPool, + repository: Uuid, + paths: &[String], + ) -> Result<(), sqlx::Error> { + sqlx::query("DELETE FROM docker_objects WHERE repository_id = $1 AND path = ANY($2)") + .bind(repository) + .bind(paths) + .execute(database) + .await?; + Ok(()) + } +} diff --git a/crates/core/src/database/entities/mod.rs b/crates/core/src/database/entities/mod.rs index 28d4cf3..68b897a 100644 --- a/crates/core/src/database/entities/mod.rs +++ b/crates/core/src/database/entities/mod.rs @@ -1,3 +1,6 @@ +// ABOUTME: Exports database entity modules. +// ABOUTME: Groups persistence APIs used throughout the application. +pub mod docker_object; pub mod package_file; pub mod project; pub mod repository; diff --git a/crates/core/src/database/entities/package_file.rs b/crates/core/src/database/entities/package_file.rs index c03d014..4931525 100644 --- a/crates/core/src/database/entities/package_file.rs +++ b/crates/core/src/database/entities/package_file.rs @@ -1,3 +1,5 @@ +// ABOUTME: Maintains paginated package catalog records. +// ABOUTME: Combines artifact metadata with database-backed Docker byte totals. use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -24,6 +26,7 @@ pub struct DBPackageFile { pub name: String, pub path: String, pub size_bytes: i64, + pub referenced_size_bytes: Option, pub content_digest: Option, pub upstream_digest: Option, pub modified_at: DateTime, @@ -69,6 +72,7 @@ pub struct PackageFileUpsertInput { pub name: String, pub path: String, pub size_bytes: i64, + pub referenced_size_bytes: Option, pub content_digest: Option, pub upstream_digest: Option, pub modified_at: DateTime, @@ -89,12 +93,13 @@ impl DBPackageFile { name, path, size_bytes, + referenced_size_bytes, content_digest, upstream_digest, modified_at, deleted_at ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NULL) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NULL) ON CONFLICT (repository_id, path_ci) DO UPDATE SET project_id = EXCLUDED.project_id, @@ -103,6 +108,7 @@ impl DBPackageFile { name = EXCLUDED.name, path = EXCLUDED.path, size_bytes = EXCLUDED.size_bytes, + referenced_size_bytes = EXCLUDED.referenced_size_bytes, content_digest = EXCLUDED.content_digest, upstream_digest = EXCLUDED.upstream_digest, modified_at = EXCLUDED.modified_at, @@ -117,6 +123,7 @@ impl DBPackageFile { name, path, size_bytes, + referenced_size_bytes, content_digest, upstream_digest, modified_at, @@ -132,6 +139,7 @@ impl DBPackageFile { .bind(input.name) .bind(input.path) .bind(input.size_bytes.max(0)) + .bind(input.referenced_size_bytes.filter(|size| *size > 0)) .bind(input.content_digest) .bind(input.upstream_digest) .bind(input.modified_at) @@ -262,6 +270,7 @@ impl DBPackageFile { name, path, size_bytes, + referenced_size_bytes, content_digest, upstream_digest, modified_at, @@ -335,6 +344,7 @@ async fn query_page( name, path, size_bytes, + docker_referenced_size(repository_id, path) AS referenced_size_bytes, content_digest, upstream_digest, modified_at, @@ -409,7 +419,9 @@ fn sort_expression(sort_by: PackageFileSortBy) -> &'static str { PackageFileSortBy::Modified => "modified_at", PackageFileSortBy::Package => "LOWER(package) COLLATE \"C\"", PackageFileSortBy::Name => "LOWER(name) COLLATE \"C\"", - PackageFileSortBy::Size => "size_bytes", + PackageFileSortBy::Size => { + "COALESCE(docker_referenced_size(repository_id, path), size_bytes)" + } PackageFileSortBy::Path => "LOWER(path) COLLATE \"C\"", PackageFileSortBy::Digest => { "LOWER(COALESCE(content_digest, upstream_digest, '')) COLLATE \"C\"" @@ -433,6 +445,7 @@ fn build_upsert_input(project: &DBProject, version: &DBProjectVersion) -> Packag name, path, size_bytes: proxy_meta.size.unwrap_or_default() as i64, + referenced_size_bytes: None, content_digest: None, upstream_digest: normalize_digest(proxy_meta.upstream_digest.as_deref()), modified_at, @@ -482,6 +495,7 @@ fn build_upsert_input(project: &DBProject, version: &DBProjectVersion) -> Packag name, path, size_bytes, + referenced_size_bytes: None, content_digest, upstream_digest: None, modified_at: version.updated_at, diff --git a/crates/core/src/database/entities/package_file/tests.rs b/crates/core/src/database/entities/package_file/tests.rs index 0f02781..4308bfc 100644 --- a/crates/core/src/database/entities/package_file/tests.rs +++ b/crates/core/src/database/entities/package_file/tests.rs @@ -1,3 +1,5 @@ +// ABOUTME: Tests package catalog normalization and ordering expressions. +// ABOUTME: Verifies stable sorting and digest handling. #![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] use super::{ @@ -36,7 +38,10 @@ fn sort_expression_covers_all_variants() { sort_expression(PackageFileSortBy::Name), "LOWER(name) COLLATE \"C\"" ); - assert_eq!(sort_expression(PackageFileSortBy::Size), "size_bytes"); + assert_eq!( + sort_expression(PackageFileSortBy::Size), + "COALESCE(docker_referenced_size(repository_id, path), size_bytes)" + ); assert_eq!( sort_expression(PackageFileSortBy::Path), "LOWER(path) COLLATE \"C\"" diff --git a/crates/core/src/database/entities/project/members.rs b/crates/core/src/database/entities/project/members.rs index 81211eb..85df63a 100644 --- a/crates/core/src/database/entities/project/members.rs +++ b/crates/core/src/database/entities/project/members.rs @@ -1,10 +1,7 @@ +use crate::database::prelude::*; use serde::Serialize; use utoipa::ToSchema; use uuid::Uuid; -mod new; -pub use new::*; - -use crate::database::prelude::*; /// On the first push. The pusher will be added as a project member with write and manage permissions #[derive(Debug, Clone, PartialEq, Eq, Serialize, FromRow, ToSchema, TableType)] diff --git a/crates/core/src/database/entities/project/members/new.rs b/crates/core/src/database/entities/project/members/new.rs deleted file mode 100644 index e812193..0000000 --- a/crates/core/src/database/entities/project/members/new.rs +++ /dev/null @@ -1,43 +0,0 @@ -use serde::{Deserialize, Serialize}; -use sqlx::PgPool; -use utoipa::ToSchema; -use uuid::Uuid; - -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)] -pub struct NewProjectMember { - pub user_id: i32, - pub project_id: Uuid, - pub can_write: bool, - pub can_manage: bool, -} -impl NewProjectMember { - pub fn new_owner(user_id: i32, project: Uuid) -> Self { - Self { - user_id, - project_id: project, - can_write: true, - can_manage: true, - } - } - pub async fn insert_no_return(self, db: &PgPool) -> Result<(), sqlx::Error> { - let Self { - user_id, - project_id, - can_write, - can_manage, - } = self; - sqlx::query( - r#" - INSERT INTO project_members (user_id, project_id, can_write, can_manage) - VALUES ($1, $2, $3, $4) - "#, - ) - .bind(user_id) - .bind(project_id) - .bind(can_write) - .bind(can_manage) - .execute(db) - .await?; - Ok(()) - } -} diff --git a/crates/core/src/database/entities/project/mod.rs b/crates/core/src/database/entities/project/mod.rs index ac15e02..7b4e63c 100644 --- a/crates/core/src/database/entities/project/mod.rs +++ b/crates/core/src/database/entities/project/mod.rs @@ -11,7 +11,6 @@ pub use new::*; use crate::{database::prelude::*, repository::project::ReleaseType}; pub mod info; pub mod members; -pub mod update; pub mod versions; /// Implemented on different types of Project query result. Such as ProjectLookupResult pub trait ProjectDBType: for<'r> FromRow<'r, PgRow> + Unpin + Send + Sync + TableQuery { diff --git a/crates/core/src/database/entities/project/update.rs b/crates/core/src/database/entities/project/update.rs deleted file mode 100644 index 8b13789..0000000 --- a/crates/core/src/database/entities/project/update.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/crates/core/src/database/entities/repository.rs b/crates/core/src/database/entities/repository.rs index 89f77f4..7c9300c 100644 --- a/crates/core/src/database/entities/repository.rs +++ b/crates/core/src/database/entities/repository.rs @@ -6,8 +6,6 @@ use sqlx::{PgPool, Row, postgres::PgRow, prelude::FromRow, types::Json}; use tracing::info; use utoipa::ToSchema; use uuid::Uuid; -mod hostname; -pub use hostname::*; mod virtual_member; pub use virtual_member::*; diff --git a/crates/core/src/database/entities/repository/hostname.rs b/crates/core/src/database/entities/repository/hostname.rs deleted file mode 100644 index e4d8b40..0000000 --- a/crates/core/src/database/entities/repository/hostname.rs +++ /dev/null @@ -1,59 +0,0 @@ -use serde::Serialize; -use sqlx::prelude::FromRow; -use utoipa::ToSchema; -use uuid::Uuid; - -use crate::database::DateTime; -/// A hostname that is associated with a repository. -/// -/// Table: `repository_hostnames` - -#[derive(Debug, Clone, Serialize, FromRow, ToSchema)] -pub struct DBRepositoryHostname { - pub id: i32, - pub repository_id: Uuid, - pub hostname: String, - pub updated_at: DateTime, - pub created_at: DateTime, -} -impl DBRepositoryHostname { - pub async fn is_hostname_available( - database: &sqlx::PgPool, - hostname: &str, - ) -> Result { - let result = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM repository_hostnames WHERE hostname = $1)", - ) - .bind(hostname) - .fetch_one(database) - .await?; - Ok(result) - } - pub async fn get_by_hostname( - database: &sqlx::PgPool, - hostname: &str, - ) -> Result, sqlx::Error> { - let result = sqlx::query_as("SELECT * FROM repository_hostnames WHERE hostname = $1") - .bind(hostname) - .fetch_optional(database) - .await?; - Ok(result) - } - - pub async fn get_by_repository_id( - database: &sqlx::PgPool, - repository_id: Uuid, - ) -> Result, sqlx::Error> { - let result = sqlx::query_as("SELECT * FROM repository_hostnames WHERE repository_id = $1") - .bind(repository_id) - .fetch_all(database) - .await?; - Ok(result) - } - pub async fn get_all(database: &sqlx::PgPool) -> Result, sqlx::Error> { - let result = sqlx::query_as("SELECT * FROM repository_hostnames") - .fetch_all(database) - .await?; - Ok(result) - } -} diff --git a/crates/core/src/database/entities/user/events.rs b/crates/core/src/database/entities/user/events.rs deleted file mode 100644 index e69de29..0000000 diff --git a/crates/core/src/repository/project/proxy.rs b/crates/core/src/repository/project/proxy.rs index 3d57394..a822cbd 100644 --- a/crates/core/src/repository/project/proxy.rs +++ b/crates/core/src/repository/project/proxy.rs @@ -1,3 +1,5 @@ +// ABOUTME: Defines metadata for cached upstream artifacts. +// ABOUTME: Carries Docker reference paths alongside proxy artifact identities. use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -26,6 +28,8 @@ pub struct ProxyArtifactMeta { pub upstream_digest: Option, pub upstream_url: Option, pub size: Option, + #[serde(default)] + pub docker_references: Option>, pub fetched_at: DateTime, } @@ -44,6 +48,7 @@ impl ProxyArtifactMeta { upstream_url: None, size: None, fetched_at: None, + docker_references: None, } } @@ -61,6 +66,7 @@ pub struct ProxyArtifactMetaBuilder { upstream_digest: Option, upstream_url: Option, size: Option, + docker_references: Option>, fetched_at: Option>, } @@ -85,6 +91,12 @@ impl ProxyArtifactMetaBuilder { self } + /// Records direct Docker object paths without fetching referenced content. + pub fn docker_references(mut self, references: Vec) -> Self { + self.docker_references = Some(references); + self + } + pub fn fetched_at(mut self, fetched_at: DateTime) -> Self { self.fetched_at = Some(fetched_at); self @@ -100,6 +112,7 @@ impl ProxyArtifactMetaBuilder { upstream_digest: self.upstream_digest, upstream_url: self.upstream_url, size: self.size, + docker_references: self.docker_references, fetched_at: self.fetched_at.unwrap_or_else(Utc::now), } } @@ -152,6 +165,7 @@ mod tests { ProxyArtifactMeta::builder("numpy", "numpy", "packages/numpy/numpy-2.1.0.whl") .version("2.1.0") .size(4_096) + .docker_references(vec!["v2/numpy/blobs/sha256:abc".into()]) .upstream_digest("sha256:deadbeef") .fetched_at(timestamp) .build() diff --git a/crates/core/src/testing/logging.rs b/crates/core/src/testing/logging/mod.rs similarity index 67% rename from crates/core/src/testing/logging.rs rename to crates/core/src/testing/logging/mod.rs index 95c5838..7aa79a9 100644 --- a/crates/core/src/testing/logging.rs +++ b/crates/core/src/testing/logging/mod.rs @@ -2,10 +2,11 @@ use std::sync::Once; use ahash::{HashMap, HashMapExt}; use serde::{Deserialize, Serialize}; -use tracing::{debug, error, info, trace, warn}; use tracing_subscriber::{Layer, filter::Targets, layer::SubscriberExt, util::SubscriberInitExt}; use crate::logging::{LevelSerde, LoggingLevels}; +#[cfg(test)] +mod tests; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TestingLoggerConfig { pub levels: LoggingLevels, @@ -16,36 +17,26 @@ impl TestingLoggerConfig { ONCE.call_once(|| { let targets: Targets = self.levels.into(); let stdout_log = tracing_subscriber::fmt::layer() - .pretty() .without_time() .with_thread_ids(false) .with_thread_names(false); tracing_subscriber::registry() .with(stdout_log.with_filter(targets)) .init(); - info!("Logger initialized"); }); - trace!("This is a trace message"); - debug!("This is a debug message"); - info!("This is an info message"); - warn!("This is a warning message"); - error!("This is an error message"); } } impl Default for TestingLoggerConfig { fn default() -> Self { + // ponytail: default=Warn keeps third-party (aws_sdk, sqlx, hyper) spam out of test + // output; raise per-crate levels in storage_testing_config.toml or RUST_LOG if needed. let mut others = HashMap::new(); others.insert("pkgly".to_string(), LevelSerde::Debug); others.insert("nr_core".to_string(), LevelSerde::Debug); others.insert("nr_storage".to_string(), LevelSerde::Debug); - - others.insert("h2".to_string(), LevelSerde::Warn); - others.insert("tower".to_string(), LevelSerde::Warn); - others.insert("tonic".to_string(), LevelSerde::Warn); - others.insert("hyper_util".to_string(), LevelSerde::Warn); Self { levels: LoggingLevels { - default: LevelSerde::Debug, + default: LevelSerde::Warn, others, }, } diff --git a/crates/core/src/testing/logging/tests.rs b/crates/core/src/testing/logging/tests.rs new file mode 100644 index 0000000..60ca777 --- /dev/null +++ b/crates/core/src/testing/logging/tests.rs @@ -0,0 +1,18 @@ +// ABOUTME: Tests that the default test logger config keeps third-party debug spam out. +#![allow(clippy::expect_used, clippy::panic, clippy::todo, clippy::unwrap_used)] +use super::*; + +#[test] +fn default_levels_silence_third_party_crates() { + let config = TestingLoggerConfig::default(); + assert_eq!(config.levels.default, LevelSerde::Warn); + for crate_name in ["pkgly", "nr_core", "nr_storage"] { + assert_eq!( + config.levels.others.get(crate_name), + Some(&LevelSerde::Debug), + "{crate_name} should stay at Debug" + ); + } + assert!(!config.levels.others.contains_key("aws_smithy_runtime")); + assert!(!config.levels.others.contains_key("h2")); +} diff --git a/crates/core/src/testing/mod.rs b/crates/core/src/testing/mod.rs index 4ed7302..206c778 100644 --- a/crates/core/src/testing/mod.rs +++ b/crates/core/src/testing/mod.rs @@ -45,7 +45,7 @@ impl TestCore { let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| log.into()); LOGGING_INIT.call_once(|| { - let stdout_log = tracing_subscriber::fmt::layer().pretty(); + let stdout_log = tracing_subscriber::fmt::layer(); match tracing_subscriber::registry() .with(stdout_log.with_filter(env_filter)) .try_init() diff --git a/crates/macros/Cargo.toml b/crates/macros/Cargo.toml index 48bcaed..36c262d 100644 --- a/crates/macros/Cargo.toml +++ b/crates/macros/Cargo.toml @@ -16,4 +16,3 @@ syn = { version = "2", features = ["full", "extra-traits", "parsing"] } [dev-dependencies] prettyplease = "0.2" -anyhow = "1" diff --git a/crates/macros/src/dyn_repository_handler.rs b/crates/macros/src/dyn_repository_handler.rs index 6b7189c..e2dc059 100644 --- a/crates/macros/src/dyn_repository_handler.rs +++ b/crates/macros/src/dyn_repository_handler.rs @@ -1,3 +1,5 @@ +// ABOUTME: Generates repository dispatch implementations for repository enums. +// ABOUTME: Bounds read-request future sizes by allocating backend handlers on the heap. use proc_macro2::TokenStream; use quote::quote; use syn::{ @@ -169,7 +171,7 @@ pub(crate) fn expand(derive_input: DeriveInput) -> Result { ) -> Result { match self { #( - #ident::#variants(variant) => variant.handle_get(request).await.map_err(Self::Error::from), + #ident::#variants(variant) => Box::pin(variant.handle_get(request)).await.map_err(Self::Error::from), )* } } @@ -220,7 +222,7 @@ pub(crate) fn expand(derive_input: DeriveInput) -> Result { ) -> Result { match self { #( - #ident::#variants(variant) => variant.handle_head(request).await.map_err(Self::Error::from), + #ident::#variants(variant) => Box::pin(variant.handle_head(request)).await.map_err(Self::Error::from), )* } } diff --git a/crates/macros/src/lib.rs b/crates/macros/src/lib.rs index c768596..2dac316 100644 --- a/crates/macros/src/lib.rs +++ b/crates/macros/src/lib.rs @@ -1,22 +1,9 @@ pub(crate) mod dyn_repository_handler; pub(crate) mod nu_type; -pub(crate) mod repository_config; pub(crate) mod serde; use proc_macro::TokenStream; -pub(crate) mod utils; use syn::{DeriveInput, parse_macro_input}; pub(crate) mod scopes; -#[proc_macro_derive(RepositoryConfig, attributes(repository_config))] -pub fn repository_config(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input as DeriveInput); - // Check if its an enum - let result = repository_config::expand(input); - match result { - Ok(ok) => ok.into(), - Err(err) => err.to_compile_error().into(), - } -} - #[proc_macro_derive(DynRepositoryHandler, attributes(repository_handler))] pub fn dyn_repository_handler(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); diff --git a/crates/macros/src/repository_config.rs b/crates/macros/src/repository_config.rs deleted file mode 100644 index c9c9e2c..0000000 --- a/crates/macros/src/repository_config.rs +++ /dev/null @@ -1,127 +0,0 @@ -use proc_macro2::TokenStream; -use quote::quote; -use syn::{ - DeriveInput, Ident, LitStr, Result, - parse::{Parse, ParseStream}, -}; -mod keywords { - use syn::custom_keyword; - custom_keyword!(name); -} -#[derive(Debug)] -pub struct ContainerAttrs { - pub config_name: LitStr, -} - -impl Parse for ContainerAttrs { - fn parse(input: ParseStream) -> syn::Result { - let mut name: Option = None; - while !input.is_empty() { - if input.peek(syn::Token![,]) { - let _: syn::Token![,] = input.parse()?; - } - let lookahead = input.lookahead1(); - if lookahead.peek(keywords::name) { - let _ = input.parse::()?; - let _: syn::Token![=] = input.parse()?; - name = Some(input.parse()?); - } else { - return Err(lookahead.error()); - } - } - let attr = Self { - config_name: name.ok_or_else(|| syn::Error::new(input.span(), "Missing name opt"))?, - }; - Ok(attr) - } -} - -pub(crate) fn expand(derive_input: DeriveInput) -> Result { - let container_attr = derive_input - .attrs - .iter() - .find(|v| v.path().is_ident("repository_config")) - .map(|v| v.parse_args::()) - .transpose()? - .ok_or_else(|| { - syn::Error::new(derive_input.ident.span(), "Missing #[repository_config]") - })?; - let ContainerAttrs { config_name } = container_attr; - let config = derive_input.ident.clone(); - let result = quote! { - pub async fn get_config( - storage_handler: actix_web::web::Data>, - database: actix_web::web::Data, - auth: crate::authentication::Authentication, - path_params: actix_web::web::Path<(String, String)>, - ) -> actix_web::Result { - use crate::storage::models::Storage; - use crate::system::permissions_checker::CanIDo; - let user = auth.get_user(&database).await??; - user.can_i_edit_repos()?; - let (storage_name, repository_name) = path_params.into_inner(); - let storage = crate::helpers::get_storage!(storage_handler, storage_name); - let repository = crate::helpers::get_repository!(storage, repository_name); - if let crate::repository::handler::DynamicRepositoryHandler::Maven( repository) = repository.as_ref() { - if let crate::repository::maven::MavenHandler::$maven_type(ref repository) = repository { - let value = crate::repository::settings::RepositoryConfigHandler::<#config>::get(repository); - return Ok(actix_web::HttpResponse::Ok().json(value)); - } - } - return Ok(actix_web::HttpResponse::BadRequest().body("Repository type not supported".to_string())); - } - pub async fn set_config( - storage_handler: actix_web::web::Data>, - database: actix_web::web::Data, - auth: crate::authentication::Authentication, - path_params: actix_web::web::Path<(String, String)>, - body: actix_web::web::Json<$config>, - ) -> actix_web::Result { - use crate::storage::models::Storage; - use crate::repository::handler::Repository; - use crate::system::permissions_checker::CanIDo; - let user = auth.get_user(&database).await??; - user.can_i_edit_repos()?; - let (storage_name, repository_name) = path_params.into_inner(); - let storage = crate::helpers::get_storage!(storage_handler, storage_name); - let (name,mut repository) = crate::helpers::take_repository!(storage, repository_name); - let body = body.into_inner(); - - let result = if let crate::repository::handler::DynamicRepositoryHandler::Maven(ref mut repository) = repository { - if let crate::repository::maven::MavenHandler::$maven_type(ref mut repository) = repository { - let _value = crate::repository::settings::RepositoryConfigHandler::<$config>::get(repository); - let value = crate::repository::settings::RepositoryConfigHandler::<$config>::update( repository, body).map(|_| true); - if let Err(e) = storage.save_repository_config(repository.get_repository(), crate::repository::settings::RepositoryConfigHandler::<#config>::get(repository)).await{ - tracing::error!("{}", e); - } - value - }else{ - Ok(false) - } - }else { - Ok(false) - }; - storage.add_repository_for_updating(name, repository,false).await.expect("Failed to add repository for updating"); - if result?{ - Ok(actix_web::HttpResponse::NoContent().finish()) - }else{ - Ok(actix_web::HttpResponse::BadRequest().body("Repository type not supported".to_string())) - } - } - pub fn init(cfg: &mut actix_web::web::ServiceConfig) { - cfg.service(actix_web::web::resource([concat!("/repositories/{storage}/{repository}/config/", #config_name)]) - .route(actix_web::web::get().to(get_config)) - .route(actix_web::web::put().to(set_config))); - } - }; - let module_name: Ident = Ident::new( - &format!("web_{}", config_name.value()), - derive_input.ident.span(), - ); - let wrapped = quote! { - pub mod #module_name { - #result - } - }; - Ok(wrapped) -} diff --git a/crates/macros/src/utils/mod.rs b/crates/macros/src/utils/mod.rs deleted file mode 100644 index 6240a82..0000000 --- a/crates/macros/src/utils/mod.rs +++ /dev/null @@ -1,148 +0,0 @@ -#![allow(dead_code)] -use std::borrow::Cow; - -use proc_macro2::TokenStream; -use quote::{ToTokens, format_ident, quote}; -use syn::{Ident, LitStr}; -pub struct DisplayStringEnum<'ident, 'entries, T: StringEnum> { - pub ident: &'ident Ident, - pub entries: &'entries [T], -} -impl ToTokens for DisplayStringEnum<'_, '_, T> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let formatter_ident = format_ident!("formatter"); - let ident = self.ident; - let entries: Vec<_> = self - .entries - .iter() - .map(|entry| StringEnum::write_str(entry, &formatter_ident)) - .collect(); - let result = quote! { - #[automatically_derived] - impl std::fmt::Display for #ident { - fn fmt(&self, #formatter_ident: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - #(#entries)* - } - } - } - }; - tokens.extend(result); - } -} -/// Adds TryFrom<&str> and TryFrom for the type -/// -/// By passing them to from_str -pub struct OtherTryStringConverts<'ident, 'error> { - pub ident: &'ident Ident, - pub error: &'error Ident, -} - -impl ToTokens for OtherTryStringConverts<'_, '_> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let ident = self.ident; - let error = self.error; - let result = quote! { - #[automatically_derived] - impl std::convert::TryFrom<&str> for #ident { - type Error = #error; - #[inline(always)] - fn try_from(value: &str) -> Result { - use std::str::FromStr; - Self::from_str(value) - } - } - #[automatically_derived] - impl std::convert::TryFrom for #ident { - type Error = #error; - #[inline(always)] - fn try_from(value: String) -> Result { - use std::str::FromStr; - Self::from_str(&value) - } - } - }; - tokens.extend(result); - } -} - -pub trait StringEnum { - fn variant_name(&self) -> &Ident; - - fn str_value(&self) -> Cow<'_, LitStr>; - - fn as_ref_str(&self) -> TokenStream { - let variant_name = self.variant_name(); - let str_value = self.str_value(); - quote! { - Self::#variant_name => #str_value, - } - } - fn write_str(&self, formatter: &Ident) -> TokenStream { - let variant_name = self.variant_name(); - let str_value = self.str_value(); - quote! { - Self::#variant_name => #formatter.write_str(#str_value), - } - } - #[allow(clippy::wrong_self_convention)] - fn from_str_impl(&self) -> TokenStream { - let variant_name = self.variant_name(); - let str_value = self.str_value(); - quote! { - #str_value => Ok(Self::#variant_name), - } - } - #[allow(dead_code)] - #[allow(clippy::wrong_self_convention)] - fn from_str_no_error(&self) -> TokenStream { - let variant_name = self.variant_name(); - let str_value = self.str_value(); - quote! { - #str_value => Self::#variant_name, - } - } -} - -pub struct AsRefImpl<'ident, 'error, 'entries, T: StringEnum> { - pub ident: &'ident Ident, - pub entries: &'entries [T], - pub error: &'error Ident, -} -impl ToTokens for AsRefImpl<'_, '_, '_, T> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let ident = self.ident; - let entries = self.entries; - let error = self.error; - let as_str = entries - .iter() - .map(StringEnum::as_ref_str) - .collect::>(); - let from_string = entries - .iter() - .map(StringEnum::from_str_impl) - .collect::>(); - let other_converts = OtherTryStringConverts { ident, error }; - let result = quote! { - #[automatically_derived] - impl std::convert::AsRef for #ident { - fn as_ref(&self) -> &str { - match self { - #(#as_str)* - } - } - } - impl std::str::FromStr for #ident { - type Err = #error; - fn from_str(value: &str) -> Result { - match value { - #(#from_string)* - _ => Err(#error::from(value.to_owned())), - } - } - } - #other_converts - }; - tokens.extend(result); - } -} diff --git a/crates/nr-api/Cargo.toml b/crates/nr-api/Cargo.toml index 5a1d006..93eb427 100644 --- a/crates/nr-api/Cargo.toml +++ b/crates/nr-api/Cargo.toml @@ -17,8 +17,5 @@ bytes.workspace = true url.workspace = true [dev-dependencies] tokio.workspace = true -anyhow = "1" -tracing-subscriber.workspace = true -tracing-appender.workspace = true [lints] workspace = true diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index c64d8d7..4e3652a 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -46,7 +46,7 @@ aws-config = { version = "1", default-features = false, features = ["behavior-ve aws-credential-types = "1" aws-sdk-s3 = "1" aws-types = "1" -aws-smithy-types = "1" +aws-smithy-types = { version = "1", features = ["rt-tokio"] } aws-smithy-runtime-api = "1" lru = "0.16" hex = "0.4" @@ -59,8 +59,6 @@ workspace = true [dev-dependencies] toml.workspace = true -tracing-subscriber.workspace = true -tracing-appender.workspace = true nr-core = { workspace = true, features = ["testing"] } anyhow = { workspace = true } parking_lot.workspace = true diff --git a/crates/storage/src/dyn_storage.rs b/crates/storage/src/dyn_storage.rs index 19ec025..930864d 100644 --- a/crates/storage/src/dyn_storage.rs +++ b/crates/storage/src/dyn_storage.rs @@ -1,3 +1,5 @@ +// ABOUTME: Dispatches repository storage operations to local and S3 backends. +// ABOUTME: Keeps backend-specific operations behind the shared storage interface. use nr_core::storage::StoragePath; use uuid::Uuid; @@ -47,8 +49,7 @@ impl Storage for DynStorage { .save_file(repository, file, location) .await .map_err(Into::into), - DynStorage::S3(storage) => storage - .save_file(repository, file, location) + DynStorage::S3(storage) => Box::pin(storage.save_file(repository, file, location)) .await .map_err(Into::into), } @@ -134,8 +135,8 @@ impl Storage for DynStorage { .open_file(repository, location) .await .map_err(Into::into), - DynStorage::S3(storage) => storage - .open_file(repository, location) + // Keep the S3 read state machine out of every caller's future, including local reads. + DynStorage::S3(storage) => Box::pin(storage.open_file(repository, location)) .await .map_err(Into::into), } diff --git a/crates/storage/src/fs/file_meta.rs b/crates/storage/src/fs/file_meta.rs index 84db800..581f902 100644 --- a/crates/storage/src/fs/file_meta.rs +++ b/crates/storage/src/fs/file_meta.rs @@ -334,11 +334,9 @@ impl LocationMeta { } #[instrument( level = "debug", - skip(path), + skip(self, path), fields( path = ?path.as_ref(), - path.meta = Empty, - created = Empty, ) )] pub(crate) fn save_meta(&self, path: impl AsRef) -> Result<(), LocalStorageError> { diff --git a/crates/storage/src/local/mod.rs b/crates/storage/src/local/mod.rs index a5c0cbd..fb578e4 100644 --- a/crates/storage/src/local/mod.rs +++ b/crates/storage/src/local/mod.rs @@ -328,7 +328,7 @@ impl LocalStorageInner { new_directory_start, }) } - #[instrument(skip(location))] + #[instrument(skip(self, location))] pub fn get_path(&self, repository: &Uuid, location: &StoragePath) -> PathBuf { let location: PathBuf = location.into(); let path = self.config.path.join(repository.to_string()); @@ -373,7 +373,7 @@ impl LocalStorageInner { content: StorageFileReader::from(file), }) } - #[instrument(skip(path), fields(entries.read, entries.skipped))] + #[instrument(skip(self, path), fields(entries.read, entries.skipped))] pub async fn open_folder(&self, path: PathBuf) -> Result { let mut set = JoinSet::, LocalStorageError>>::new(); let current_span = Span::current(); diff --git a/crates/storage/src/s3/mod.rs b/crates/storage/src/s3/mod.rs index 55e5684..52d27b6 100644 --- a/crates/storage/src/s3/mod.rs +++ b/crates/storage/src/s3/mod.rs @@ -1,8 +1,9 @@ -#![allow(dead_code)] +// ABOUTME: Implements S3-backed repository storage and object metadata handling. +// ABOUTME: Provides guarded object mutation, listing, streaming, and local caching. use std::{ - borrow::Cow, collections::VecDeque, env, + future::Future, io::ErrorKind, net::IpAddr, num::NonZeroUsize, @@ -11,53 +12,51 @@ use std::{ pin::Pin, str::FromStr, sync::{Arc, OnceLock}, + time::{SystemTime, UNIX_EPOCH}, }; use aws_config::BehaviorVersion; use aws_config::sts::AssumeRoleProvider; use aws_credential_types::{Credentials as AwsCredentials, provider::SharedCredentialsProvider}; -use aws_sdk_s3::{ - Client as AwsS3Client, - types::{CommonPrefix, Tag}, -}; +use aws_sdk_s3::{Client as AwsS3Client, types::CommonPrefix}; use aws_smithy_runtime_api::client::dns::{DnsFuture, ResolveDns, ResolveDnsError}; -use aws_smithy_runtime_api::client::result::SdkError; +use aws_smithy_runtime_api::client::{orchestrator::HttpResponse, result::SdkError}; use aws_smithy_types::byte_stream::ByteStream; +use aws_smithy_types::error::metadata::ProvideErrorMetadata; use aws_types::{SdkConfig, region::Region}; -use bytes::Bytes; -use chrono::{FixedOffset, Local}; +use bytes::{Bytes, BytesMut}; +use chrono::{DateTime as ChronoDateTime, FixedOffset, Local, Utc}; use futures::future::BoxFuture; use hex::encode; use lru::LruCache; use mime::Mime; use nr_core::storage::{FileHashes, FileTypeCheck, SerdeMime, StoragePath}; -use regions::{CustomRegion, S3StorageRegion}; +use regions::CustomRegion; use sha2::{Digest, Sha256}; use sysinfo::System; use tokio::{ fs, - io::BufReader, - sync::Mutex, + io::{AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader, ReadBuf}, + sync::{Mutex, OwnedSemaphorePermit, Semaphore}, task, time::{Duration, Instant}, }; use url::{Host, Url}; pub mod regions; -use serde::{Deserialize, Serialize}; -use tracing::{debug, error, info, instrument, warn}; -use utoipa::ToSchema; -pub mod tags; -use ahash::HashSet; +use ahash::{HashMap, HashSet}; use ipnet::IpNet; -use parking_lot::RwLock; +use parking_lot::{Mutex as ParkingMutex, RwLock}; +use serde::{Deserialize, Deserializer, Serialize}; +use tracing::{debug, info, instrument, warn}; +use utoipa::ToSchema; use uuid::Uuid; #[derive(Debug, thiserror::Error)] pub enum S3StorageError { #[error("No Region Provided")] NoRegionSpecified, - #[error("AWS SDK error: {0}")] - AwsSdkError(String), + #[error("AWS SDK error ({kind:?}): {message}")] + AwsSdkError { kind: S3ErrorKind, message: String }, #[error("Bucket Does Not Exist {0}")] BucketDoesNotExist(String), #[error("IO Error: {0}")] @@ -67,21 +66,168 @@ pub enum S3StorageError { #[error(transparent)] InvalidConfigType(#[from] InvalidConfigType), - #[error("Missing Tag: {0}")] - MissingTag(Cow<'static, str>), - #[error(transparent)] PathCollision(#[from] PathCollisionError), #[error("S3 endpoint is blocked by egress policy")] BlockedEndpoint, } + +/// Broadly classifies S3 failures so callers can choose safe retry or conflict behavior. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum S3ErrorKind { + NotFound, + AccessDenied, + Throttled, + Conflict, + Network, + Other, +} + impl S3StorageError { - pub fn static_missing_tag(tag: &'static str) -> Self { - S3StorageError::MissingTag(tag.into()) + /// Returns the S3-specific classification when this is an AWS SDK failure. + pub fn kind(&self) -> Option { + match self { + Self::AwsSdkError { kind, .. } => Some(*kind), + _ => None, + } + } + + /// Returns whether the failure means the requested object or bucket was not found. + pub fn is_not_found(&self) -> bool { + self.kind() == Some(S3ErrorKind::NotFound) + } + + /// Returns whether a conditional mutation failed because the object changed. + pub fn is_conflict(&self) -> bool { + self.kind() == Some(S3ErrorKind::Conflict) + } + + /// Returns whether retrying may succeed without changing the request semantics. + pub fn is_retryable(&self) -> bool { + matches!( + self.kind(), + Some(S3ErrorKind::Network | S3ErrorKind::Throttled) + ) + } + + /// Converts an SDK error into a classified failure while retaining its full display context. + pub fn from_sdk_error(err: SdkError) -> Self + where + E: std::error::Error + std::fmt::Display + ProvideErrorMetadata + 'static, + { + let kind = match &err { + SdkError::TimeoutError(_) | SdkError::DispatchFailure(_) => S3ErrorKind::Network, + SdkError::ServiceError(context) => { + classify_service_error(context.err().code(), context.raw().status().as_u16()) + } + SdkError::ConstructionFailure(_) | SdkError::ResponseError(_) => S3ErrorKind::Other, + _ => S3ErrorKind::Other, + }; + let message = format!( + "{}", + aws_smithy_types::error::display::DisplayErrorContext(&err) + ); + Self::AwsSdkError { kind, message } + } + + fn aws_message(message: impl Into) -> Self { + let message = message.into(); + Self::AwsSdkError { + kind: classify_error_message(&message), + message, + } + } +} + +fn classify_service_error(code: Option<&str>, status: u16) -> S3ErrorKind { + let code = code.unwrap_or_default().to_ascii_lowercase(); + if status == 409 + || status == 412 + || matches!( + code.as_str(), + "preconditionfailed" | "conditionalrequestconflict" + ) + { + return S3ErrorKind::Conflict; + } + if status == 404 || matches!(code.as_str(), "nosuchkey" | "nosuchbucket" | "notfound") { + return S3ErrorKind::NotFound; + } + if status == 401 || status == 403 || code == "accessdenied" { + return S3ErrorKind::AccessDenied; + } + if status == 429 + || status == 503 + || code.contains("throttl") + || matches!( + code.as_str(), + "slowdown" | "requestlimitexceeded" | "toomanyrequests" | "serviceunavailable" + ) + { + return S3ErrorKind::Throttled; + } + S3ErrorKind::Other +} + +fn classify_error_message(message: &str) -> S3ErrorKind { + let lower = message.to_ascii_lowercase(); + if lower.contains("preconditionfailed") + || lower.contains("precondition failed") + || lower.contains("conditionalrequestconflict") + || contains_status_code(&lower, 409) + || contains_status_code(&lower, 412) + { + return S3ErrorKind::Conflict; + } + if lower.contains("nosuchkey") + || lower.contains("no such key") + || lower.contains("nosuchbucket") + || lower.contains("no such bucket") + || lower.contains("notfound") + || lower.contains("not found") + || contains_status_code(&lower, 404) + { + return S3ErrorKind::NotFound; + } + if lower.contains("accessdenied") + || lower.contains("access denied") + || contains_status_code(&lower, 401) + || contains_status_code(&lower, 403) + { + return S3ErrorKind::AccessDenied; } - pub fn from_sdk_error(err: impl std::fmt::Display) -> Self { - S3StorageError::AwsSdkError(err.to_string()) + if lower.contains("slowdown") + || lower.contains("throttl") + || lower.contains("requestlimitexceeded") + || lower.contains("toomanyrequests") + || lower.contains("serviceunavailable") + || contains_status_code(&lower, 429) + || contains_status_code(&lower, 503) + { + return S3ErrorKind::Throttled; } + if lower.contains("timeout") + || lower.contains("timed out") + || lower.contains("deadline") + || lower.contains("dispatch") + || lower.contains("connection") + || lower.contains("network") + { + return S3ErrorKind::Network; + } + S3ErrorKind::Other +} + +fn contains_status_code(message: &str, code: u16) -> bool { + [ + format!("status code: {code}"), + format!("status: {code}"), + format!("status={code}"), + format!("http {code}"), + format!("http status {code}"), + ] + .iter() + .any(|pattern| message.contains(pattern)) } #[derive(Debug, Clone, Default)] @@ -313,23 +459,72 @@ impl AdaptiveBufferConfig { } } +#[derive(Debug, Clone, Copy)] struct MemorySnapshot { total_bytes: u64, available_bytes: u64, } +#[derive(Debug, Default)] +struct MemorySnapshotCache { + captured_at: Option, + snapshot: Option, +} + +impl MemorySnapshotCache { + fn get_or_capture(&mut self, now: Instant, capture: F) -> Option + where + F: FnOnce() -> Option, + { + if let Some(captured_at) = self.captured_at + && now + .checked_duration_since(captured_at) + .is_some_and(|elapsed| elapsed < MEMORY_SNAPSHOT_TTL) + { + return self.snapshot; + } + + let snapshot = capture(); + self.captured_at = Some(now); + self.snapshot = snapshot; + snapshot + } +} + +static MEMORY_SNAPSHOT_CACHE: OnceLock> = OnceLock::new(); + impl MemorySnapshot { + #[cfg(test)] + fn from_values(total_bytes: u64, available_bytes: u64) -> Self { + Self { + total_bytes, + available_bytes: available_bytes.min(total_bytes), + } + } + fn capture() -> Option { + let cache = + MEMORY_SNAPSHOT_CACHE.get_or_init(|| ParkingMutex::new(MemorySnapshotCache::default())); + cache + .lock() + .get_or_capture(Instant::now(), Self::capture_uncached) + } + + fn capture_uncached() -> Option { let mut system = System::new(); system.refresh_memory(); - let total = system.total_memory(); + let host_total = system.total_memory(); + let host_available = system.available_memory(); + let cgroup = system + .cgroup_limits() + .map(|limits| (limits.total_memory, limits.free_memory)); + let (total, available) = effective_memory_limits(host_total, host_available, cgroup); if total == 0 { return None; } - let available = system.available_memory(); Some(Self { - total_bytes: total.saturating_mul(1024), - available_bytes: available.saturating_mul(1024), + total_bytes: total, + available_bytes: available, }) } @@ -342,6 +537,19 @@ impl MemorySnapshot { } } +fn effective_memory_limits( + host_total: u64, + host_available: u64, + cgroup: Option<(u64, u64)>, +) -> (u64, u64) { + match cgroup { + Some((total, available)) if total > 0 && total < host_total => { + (total, available.min(total)) + } + _ => (host_total, host_available.min(host_total)), + } +} + impl Default for S3CacheConfig { fn default() -> Self { Self { @@ -364,7 +572,8 @@ fn default_cache_entry_limit() -> usize { #[derive(Clone, Serialize, Deserialize, PartialEq, ToSchema)] pub struct S3Config { pub bucket_name: String, - pub region: Option, + #[serde(default, deserialize_with = "deserialize_region")] + pub region: Option, /// Custom region takes precedence over the region field #[serde(flatten)] pub custom_region: Option, @@ -378,6 +587,45 @@ pub struct S3Config { pub adaptive_buffer: AdaptiveBufferConfig, } +fn deserialize_region<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + Ok(value.map(|region| legacy_region_id(®ion).unwrap_or(region))) +} + +fn legacy_region_id(value: &str) -> Option { + let region = match value { + "UsEast1" => "us-east-1", + "UsEast2" => "us-east-2", + "UsWest1" => "us-west-1", + "UsWest2" => "us-west-2", + "CaCentral1" => "ca-central-1", + "AfSouth1" => "af-south-1", + "ApEast1" => "ap-east-1", + "ApSouth1" => "ap-south-1", + "ApNortheast1" => "ap-northeast-1", + "ApNortheast2" => "ap-northeast-2", + "ApNortheast3" => "ap-northeast-3", + "ApSoutheast1" => "ap-southeast-1", + "ApSoutheast2" => "ap-southeast-2", + "CnNorth1" => "cn-north-1", + "CnNorthwest1" => "cn-northwest-1", + "EuNorth1" => "eu-north-1", + "EuCentral1" => "eu-central-1", + "EuCentral2" => "eu-central-2", + "EuWest1" => "eu-west-1", + "EuWest2" => "eu-west-2", + "EuWest3" => "eu-west-3", + "IlCentral1" => "il-central-1", + "MeSouth1" => "me-south-1", + "SaEast1" => "sa-east-1", + _ => return None, + }; + Some(region.to_owned()) +} + impl std::fmt::Debug for S3Config { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("S3Config") @@ -409,8 +657,13 @@ impl S3Config { .unwrap_or_else(|| "custom-endpoint".into()); return Ok(Region::new(name)); } - if let Some(region) = &self.region { - return Ok((*region).into()); + if let Some(region) = self + .region + .as_deref() + .map(str::trim) + .filter(|region| !region.is_empty()) + { + return Ok(Region::new(region.to_owned())); } Err(S3StorageError::NoRegionSpecified) } @@ -423,18 +676,12 @@ impl S3Config { self.cache.enabled && self.cache.max_bytes > 0 } } -#[derive(Debug, Clone)] -pub struct S3MetaTags { - pub name: String, - pub mime_type: Option, - pub is_directory: bool, -} - #[derive(Debug)] pub(super) struct S3DiskCache { dir: PathBuf, max_bytes: u64, state: Mutex, + publish_lock: Mutex<()>, } #[derive(Debug)] @@ -442,19 +689,40 @@ struct CacheState { entries: LruCache, current_bytes: u64, failed_deletions: VecDeque, + /// Monotonic mutation counter per object key. In-flight S3 reads capture a generation and + /// only publish into the cache when it is unchanged, so a GET that finishes after an + /// overwrite or deletion cannot resurrect stale content. + generations: HashMap, } #[derive(Debug, Clone)] struct CacheEntry { relative_path: PathBuf, size: u64, + digest: String, content_type: Option, + last_modified: Option>, } -#[derive(Debug, Clone)] +#[derive(Debug)] struct CachedObject { - bytes: Bytes, + file: tokio::fs::File, + size: u64, + digest: String, + content_type: Option, + last_modified: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CacheMetadata { + version: u8, + key: String, + relative_path: PathBuf, + size: u64, + digest: String, content_type: Option, + last_modified: Option>, + cached_at_ms: u128, } #[derive(Debug, Clone)] @@ -464,6 +732,13 @@ struct FailedDeletion { next_retry: Instant, } +/// Controls how cache publication interacts with per-object generation counters. +#[derive(Debug, Clone, Copy)] +enum PublishGuard { + Unconditional, + IfGeneration(u64), +} + impl FailedDeletion { fn new(relative_path: PathBuf) -> Self { Self { @@ -518,25 +793,26 @@ impl CacheState { impl S3DiskCache { async fn new(config: &S3CacheConfig, storage_name: &str) -> Result { if config.max_bytes == 0 { - return Err(S3StorageError::AwsSdkError( - "cache max_bytes must be greater than zero".into(), + return Err(S3StorageError::aws_message( + "cache max_bytes must be greater than zero", )); } - let dir = config - .path - .clone() - .unwrap_or_else(|| default_cache_dir(storage_name)); + let dir = resolve_cache_dir(config, storage_name); fs::create_dir_all(&dir).await?; let capacity = NonZeroUsize::new(config.max_entries.max(1)).unwrap_or(NonZeroUsize::MIN); + let entries = Self::recover_entries(&dir, capacity, config.max_bytes).await?; + let current_bytes = entries.iter().map(|(_, entry)| entry.size).sum(); let state = CacheState { - entries: LruCache::new(capacity), - current_bytes: 0, + entries, + current_bytes, failed_deletions: VecDeque::new(), + generations: HashMap::default(), }; Ok(Self { dir, max_bytes: config.max_bytes, state: Mutex::new(state), + publish_lock: Mutex::new(()), }) } @@ -547,74 +823,496 @@ impl S3DiskCache { PathBuf::from(prefix).join(rest) } + fn metadata_filename(relative: &std::path::Path) -> PathBuf { + let Some(file_name) = relative.file_name().and_then(|name| name.to_str()) else { + return relative.with_extension("meta.json"); + }; + let base_name = file_name + .split_once(".gen-") + .map_or(file_name, |(base, _)| base); + relative + .parent() + .map(|parent| parent.join(format!("{base_name}.meta.json"))) + .unwrap_or_else(|| PathBuf::from(format!("{base_name}.meta.json"))) + } + + fn generated_filename(key: &str) -> PathBuf { + let base = Self::hashed_filename(key); + let Some(file_name) = base.file_name().and_then(|name| name.to_str()) else { + return base; + }; + base.parent() + .map(|parent| parent.join(format!("{file_name}.gen-{}", Uuid::new_v4().simple()))) + .unwrap_or_else(|| { + PathBuf::from(format!("{file_name}.gen-{}", Uuid::new_v4().simple())) + }) + } + + fn is_owned_generation(name: &str) -> bool { + let Some((base, generation)) = name.split_once(".gen-") else { + return false; + }; + base.len() == 62 + && base.chars().all(|ch| ch.is_ascii_hexdigit()) + && !generation.is_empty() + && generation.chars().all(|ch| ch.is_ascii_hexdigit()) + } + + fn is_owned_legacy_content(name: &str) -> bool { + name.len() == 62 && name.chars().all(|ch| ch.is_ascii_hexdigit()) + } + + fn is_owned_metadata(name: &str) -> bool { + name.strip_suffix(".meta.json") + .is_some_and(Self::is_owned_legacy_content) + } + + fn is_owned_temp(name: &str) -> bool { + let Some((base, suffix)) = name.split_once(".tmp-") else { + return false; + }; + if suffix.is_empty() || !suffix.chars().all(|ch| ch.is_ascii_hexdigit()) { + return false; + } + Self::is_owned_legacy_content(base) + || base + .strip_suffix(".meta") + .is_some_and(Self::is_owned_legacy_content) + } + + fn is_safe_relative_path(path: &std::path::Path) -> bool { + !path.is_absolute() + && path + .components() + .all(|component| matches!(component, std::path::Component::Normal(_))) + } + + async fn verify_content_file( + path: &std::path::Path, + expected_size: u64, + expected_digest: &str, + ) -> bool { + let Ok(mut file) = fs::File::open(path).await else { + return false; + }; + let mut hasher = Sha256::new(); + let mut buffer = vec![0u8; 64 * 1024]; + let mut size = 0u64; + loop { + let read = match file.read(&mut buffer).await { + Ok(read) => read, + Err(_) => return false, + }; + if read == 0 { + break; + } + size = size.saturating_add(read as u64); + hasher.update(&buffer[..read]); + } + size == expected_size && encode(hasher.finalize()) == expected_digest + } + + async fn recover_entries( + dir: &std::path::Path, + capacity: NonZeroUsize, + max_bytes: u64, + ) -> Result, S3StorageError> { + let mut recovered = Vec::new(); + let mut owned_prefixes = Vec::new(); + let mut valid_content = HashSet::default(); + let mut valid_metadata = HashSet::default(); + let mut prefixes = fs::read_dir(dir).await?; + while let Some(prefix_entry) = prefixes.next_entry().await? { + let prefix_path = prefix_entry.path(); + let prefix_name = prefix_entry.file_name(); + let prefix_name = prefix_name.to_string_lossy(); + if !prefix_entry.file_type().await?.is_dir() + || prefix_name.len() != 2 + || !prefix_name.chars().all(|ch| ch.is_ascii_hexdigit()) + { + continue; + } + owned_prefixes.push(prefix_path.clone()); + + let mut files = fs::read_dir(&prefix_path).await?; + while let Some(file_entry) = files.next_entry().await? { + let file_name = file_entry.file_name().to_string_lossy().into_owned(); + if !Self::is_owned_metadata(&file_name) { + continue; + } + let metadata_path = file_entry.path(); + if !file_entry.file_type().await?.is_file() { + let _ = fs::remove_file(&metadata_path).await; + continue; + } + let relative_metadata = metadata_path + .strip_prefix(dir) + .map(PathBuf::from) + .map_err(|_| std::io::Error::other("cache metadata escaped root"))?; + let metadata = fs::read(&metadata_path) + .await + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let Some(metadata) = metadata else { + let _ = fs::remove_file(&metadata_path).await; + continue; + }; + let relative_content = metadata.relative_path.clone(); + let expected_content = Self::hashed_filename(&metadata.key); + let expected_metadata = Self::metadata_filename(&expected_content); + let valid_layout = Self::is_safe_relative_path(&relative_content) + && relative_content.parent() == expected_content.parent() + && relative_content + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.starts_with( + expected_content + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(), + ) && Self::is_owned_generation(name) + }); + if metadata.version != CACHE_FORMAT_VERSION + || relative_metadata != expected_metadata + || !valid_layout + || metadata.size > max_bytes + || metadata.digest.len() != 64 + { + let _ = fs::remove_file(&metadata_path).await; + if valid_layout { + let _ = fs::remove_file(dir.join(&relative_content)).await; + } + continue; + } + + let content_path = dir.join(&relative_content); + let valid = if fs::symlink_metadata(&content_path) + .await + .map(|metadata| metadata.file_type().is_file()) + .unwrap_or(false) + { + Self::verify_content_file(&content_path, metadata.size, &metadata.digest).await + } else { + false + }; + if !valid { + let _ = fs::remove_file(&metadata_path).await; + let _ = fs::remove_file(content_path).await; + continue; + } + + valid_content.insert(relative_content.clone()); + valid_metadata.insert(relative_metadata); + recovered.push(( + metadata.cached_at_ms, + metadata.key, + CacheEntry { + relative_path: relative_content, + size: metadata.size, + digest: metadata.digest, + content_type: metadata.content_type, + last_modified: metadata.last_modified, + }, + )); + } + } + + // Remove only artifacts from the recognized cache layout. User files in the configured + // directory, including files inside owned-looking prefixes, remain untouched. + for prefix_path in owned_prefixes { + let mut files = fs::read_dir(&prefix_path).await?; + while let Some(file_entry) = files.next_entry().await? { + let name = file_entry.file_name().to_string_lossy().into_owned(); + let relative = file_entry + .path() + .strip_prefix(dir) + .map(PathBuf::from) + .map_err(|_| std::io::Error::other("cache artifact escaped root"))?; + let recognized = if Self::is_owned_metadata(&name) { + !valid_metadata.contains(&relative) + } else if Self::is_owned_legacy_content(&name) + || Self::is_owned_generation(&name) + || Self::is_owned_temp(&name) + { + !valid_content.contains(&relative) + } else { + false + }; + if recognized { + let _ = fs::remove_file(file_entry.path()).await; + } + } + } + + recovered.sort_by_key(|(cached_at, _, _)| *cached_at); + let mut entries: LruCache = LruCache::new(capacity); + let mut current_bytes = 0u64; + let mut evicted: Vec = Vec::new(); + for (_, key, entry) in recovered { + if let Some(old) = entries.pop(&key) { + current_bytes = current_bytes.saturating_sub(old.size); + evicted.push(old.relative_path); + } + if entries.len() >= capacity.get() + && let Some((_, old)) = entries.pop_lru() + { + current_bytes = current_bytes.saturating_sub(old.size); + evicted.push(old.relative_path); + } + entries.put(key, entry.clone()); + current_bytes = current_bytes.saturating_add(entry.size); + while current_bytes > max_bytes { + if let Some((_, old)) = entries.pop_lru() { + current_bytes = current_bytes.saturating_sub(old.size); + evicted.push(old.relative_path); + } else { + break; + } + } + } + for relative in evicted { + Self::remove_owned_files(dir, &relative).await; + } + Ok(entries) + } + + async fn remove_owned_files(dir: &std::path::Path, relative: &std::path::Path) { + let _ = fs::remove_file(dir.join(relative)).await; + let _ = fs::remove_file(dir.join(Self::metadata_filename(relative))).await; + } + async fn get(&self, key: &str) -> Result, S3StorageError> { self.retry_failed_deletions().await; - let (relative_path, content_type) = { + let entry = { let mut state = self.state.lock().await; match state.entries.get(key) { - Some(entry) => (entry.relative_path.clone(), entry.content_type.clone()), + Some(entry) => entry.clone(), None => return Ok(None), } }; - let path = self.dir.join(relative_path); - match fs::read(&path).await { - Ok(data) => Ok(Some(CachedObject { - bytes: Bytes::from(data), - content_type, + let path = self.dir.join(&entry.relative_path); + let metadata = match fs::symlink_metadata(&path).await { + Ok(metadata) => metadata, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + self.remove_if_matches(key, &entry).await?; + return Ok(None); + } + Err(err) => return Err(err.into()), + }; + // Cache reads open the file instead of hashing full contents into memory. Digest + // integrity is checked when entries are written and during startup recovery; the cheap + // size check here catches truncation and torn writes. + if !metadata.file_type().is_file() || metadata.len() != entry.size { + self.remove_if_matches(key, &entry).await?; + return Ok(None); + } + match fs::File::open(&path).await { + Ok(file) => Ok(Some(CachedObject { + file, + size: entry.size, + digest: entry.digest, + content_type: entry.content_type, + last_modified: entry.last_modified, })), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + self.remove_if_matches(key, &entry).await?; + Ok(None) + } Err(err) => Err(err.into()), } } + /// Returns the current mutation generation for an object key (0 when never mutated). + async fn generation_for(&self, key: &str) -> u64 { + self.state + .lock() + .await + .generations + .get(key) + .copied() + .unwrap_or(0) + } + + #[cfg(test)] async fn put( &self, key: &str, data: Bytes, content_type: Option<&str>, ) -> Result<(), S3StorageError> { + self.put_with_metadata(key, data, content_type, None).await + } + + async fn put_with_metadata( + &self, + key: &str, + data: Bytes, + content_type: Option<&str>, + last_modified: Option>, + ) -> Result<(), S3StorageError> { + self.retry_failed_deletions().await; + if data.len() as u64 > self.max_bytes { + self.remove(key).await?; + return Ok(()); + } + self.publish( + key, + data, + content_type, + last_modified, + PublishGuard::Unconditional, + ) + .await + .map(|_| ()) + } + + /// Publishes cache content captured by an in-flight S3 read only when the object was not + /// mutated since the read started. Returns false (without touching the cache) when stale. + async fn put_if_generation( + &self, + key: &str, + data: Bytes, + content_type: Option<&str>, + last_modified: Option>, + generation: u64, + ) -> Result { self.retry_failed_deletions().await; - let relative = Self::hashed_filename(key); + if data.len() as u64 > self.max_bytes { + self.remove_if_generation(key, generation).await?; + return Ok(false); + } + self.publish( + key, + data, + content_type, + last_modified, + PublishGuard::IfGeneration(generation), + ) + .await + } + + /// Atomically publishes a cache entry. `Unconditional` writes bump the object generation; + /// `IfGeneration` writes verify the generation is unchanged before inserting the entry. + async fn publish( + &self, + key: &str, + data: Bytes, + content_type: Option<&str>, + last_modified: Option>, + guard: PublishGuard, + ) -> Result { + let relative = Self::generated_filename(key); let path = self.dir.join(&relative); if let Some(parent) = path.parent() { fs::create_dir_all(parent).await?; } - fs::write(&path, data.as_ref()).await?; - let mut removed = Vec::new(); + let temp_path = path.with_extension(format!("tmp-{}", Uuid::new_v4().simple())); + fs::write(&temp_path, data.as_ref()).await?; + let digest = hex::encode(Sha256::digest(data.as_ref())); + let cached_at_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let metadata = CacheMetadata { + version: CACHE_FORMAT_VERSION, + key: key.to_owned(), + relative_path: relative.clone(), + size: data.len() as u64, + digest: digest.clone(), + content_type: content_type.map(str::to_owned), + last_modified, + cached_at_ms, + }; + let metadata_relative = Self::metadata_filename(&relative); + let metadata_path = self.dir.join(&metadata_relative); + let metadata_temp = + metadata_path.with_extension(format!("tmp-{}", Uuid::new_v4().simple())); + let metadata_bytes = serde_json::to_vec(&metadata) + .map_err(|error| std::io::Error::other(error.to_string()))?; + fs::write(&metadata_temp, metadata_bytes).await?; + let mut removed_entries = Vec::new(); + let mut superseded_generations = Vec::new(); + let _publication = self.publish_lock.lock().await; + if let PublishGuard::IfGeneration(expected) = guard + && self + .state + .lock() + .await + .generations + .get(key) + .copied() + .unwrap_or(0) + != expected + { + // The object changed while the S3 read was in flight. Discard the temporary files + // without touching the current cache entry. + let _ = fs::remove_file(&temp_path).await; + let _ = fs::remove_file(&metadata_temp).await; + return Ok(false); + } + fs::rename(&temp_path, &path).await?; + fs::rename(&metadata_temp, &metadata_path).await?; { let mut state = self.state.lock().await; + if matches!(guard, PublishGuard::Unconditional) { + state + .generations + .entry(key.to_owned()) + .and_modify(|count| *count = count.wrapping_add(1)) + .or_insert(1); + } if let Some(old) = state.entries.pop(key) { state.current_bytes = state.current_bytes.saturating_sub(old.size); - removed.push(old.relative_path); + if old.relative_path != relative { + superseded_generations.push(old.relative_path); + } + } + if state.entries.len() >= state.entries.cap().get() + && let Some((_, evicted)) = state.entries.pop_lru() + { + state.current_bytes = state.current_bytes.saturating_sub(evicted.size); + removed_entries.push(evicted.relative_path); } state.entries.put( key.to_string(), CacheEntry { - relative_path: relative, + relative_path: relative.clone(), size: data.len() as u64, - content_type: content_type.map(|c| c.to_string()), + digest, + content_type: content_type.map(str::to_owned), + last_modified, }, ); state.current_bytes = state.current_bytes.saturating_add(data.len() as u64); while state.current_bytes > self.max_bytes { if let Some((_, evicted)) = state.entries.pop_lru() { state.current_bytes = state.current_bytes.saturating_sub(evicted.size); - removed.push(evicted.relative_path); + removed_entries.push(evicted.relative_path); } else { break; } } } - for rel in removed { + for rel in superseded_generations { + self.delete_content_path(rel, false).await; + } + for rel in removed_entries { self.delete_relative_path(rel).await; } - Ok(()) + Ok(true) } async fn remove(&self, key: &str) -> Result<(), S3StorageError> { self.retry_failed_deletions().await; + let _publication = self.publish_lock.lock().await; let removed = { let mut state = self.state.lock().await; + state + .generations + .entry(key.to_owned()) + .and_modify(|count| *count = count.wrapping_add(1)) + .or_insert(1); state.entries.pop(key).map(|entry| { state.current_bytes = state.current_bytes.saturating_sub(entry.size); entry.relative_path @@ -626,8 +1324,74 @@ impl S3DiskCache { Ok(()) } + /// Removes an entry only when its generation still matches an in-flight read. + async fn remove_if_generation( + &self, + key: &str, + expected_generation: u64, + ) -> Result<(), S3StorageError> { + self.retry_failed_deletions().await; + let _publication = self.publish_lock.lock().await; + let removed = { + let mut state = self.state.lock().await; + if state.generations.get(key).copied().unwrap_or(0) != expected_generation { + return Ok(()); + } + state + .generations + .entry(key.to_owned()) + .and_modify(|count| *count = count.wrapping_add(1)) + .or_insert(1); + state.entries.pop(key).map(|entry| { + state.current_bytes = state.current_bytes.saturating_sub(entry.size); + entry.relative_path + }) + }; + if let Some(relative) = removed { + self.delete_relative_path(relative).await; + } + Ok(()) + } + + async fn remove_if_matches( + &self, + key: &str, + expected: &CacheEntry, + ) -> Result<(), S3StorageError> { + self.retry_failed_deletions().await; + let _publication = self.publish_lock.lock().await; + let removed = { + let mut state = self.state.lock().await; + let matches = state.entries.peek(key).is_some_and(|entry| { + entry.relative_path == expected.relative_path && entry.digest == expected.digest + }); + if matches { + state + .generations + .entry(key.to_owned()) + .and_modify(|count| *count = count.wrapping_add(1)) + .or_insert(1); + state.entries.pop(key).map(|entry| { + state.current_bytes = state.current_bytes.saturating_sub(entry.size); + entry.relative_path + }) + } else { + None + } + }; + if let Some(relative) = removed { + self.delete_relative_path(relative).await; + } + Ok(()) + } + async fn delete_relative_path(&self, relative: PathBuf) { + self.delete_content_path(relative, true).await; + } + + async fn delete_content_path(&self, relative: PathBuf, remove_metadata: bool) { let path = self.dir.join(&relative); + let metadata_path = self.dir.join(Self::metadata_filename(&relative)); match fs::remove_file(&path).await { Ok(_) => { debug!(path = %relative.display(), "Removed cache entry"); @@ -644,6 +1408,9 @@ impl S3DiskCache { self.enqueue_failed_deletion(relative).await; } } + if remove_metadata { + let _ = fs::remove_file(metadata_path).await; + } } async fn enqueue_failed_deletion(&self, relative_path: PathBuf) { @@ -708,28 +1475,232 @@ fn default_cache_dir(storage_name: &str) -> PathBuf { .join("s3-cache") .join(sanitized) } + +fn resolve_cache_dir(config: &S3CacheConfig, storage_name: &str) -> PathBuf { + config + .path + .as_ref() + .filter(|path| !path.as_os_str().is_empty()) + .cloned() + .unwrap_or_else(|| default_cache_dir(storage_name)) +} + #[derive(Debug)] pub struct S3StorageInner { pub config: S3Config, pub storage_config: StorageConfigInner, pub client: AwsS3Client, cache: Option>, + cache_load_locks: ParkingMutex>>>, + manifest_cache: ParkingMutex, + manifest_load_locks: ParkingMutex>>>, + /// Local-disk staging for append-style uploads, keyed by full S3 object path. + /// Appends accumulate locally; the single upload happens when the object is moved. + append_staging: ParkingMutex>, + /// Per-path locks serializing staging mutations so chunks for one upload cannot + /// interleave with its finalization while unrelated uploads proceed. + append_operation_locks: ParkingMutex>>>, + /// Shared capacity for spooled incoming upload chunk files, accounted together with + /// staged bytes against MAX_STAGED_BYTES so uploads cannot exceed the shared budget. + upload_spool_budget: Arc, + staging_dir: PathBuf, + /// Memoized ancestor paths that were verified clear of concrete objects. + creation_probes: ParkingMutex>, + /// Changes whenever a concrete object is created, preventing an in-flight negative probe + /// from being cached after a concurrent write completes. + creation_probe_generation: ParkingMutex, } -impl S3StorageInner { - fn bucket(&self) -> &str { - &self.config.bucket_name - } - fn aws_client(&self) -> &AwsS3Client { - &self.client - } - pub async fn load_client(config: &S3Config) -> Result { - let region = config.resolved_region()?; - debug!(%region, bucket = %config.bucket_name, "Connecting to S3 bucket"); - let (base_config, static_provider) = build_base_config(config, ®ion).await?; +/// Local staging state for one append-style upload. +#[derive(Debug, Clone)] +struct StagedAppend { + path: PathBuf, + size: u64, + /// S3 ETag of the pre-existing object when staging started, if any. + base_etag: Option, + /// Shared temporary-storage capacity held for this staged file. + budget_permits: Vec>, + reserved_units: usize, +} + +#[derive(Debug)] +struct ManifestCache { + entries: LruCache, + generations: HashMap, +} + +#[derive(Debug, Clone)] +struct CachedManifestList { + items: Vec, + expires_at: Instant, +} + +impl ManifestCache { + fn new() -> Self { + Self { + entries: LruCache::new(NonZeroUsize::new(256).unwrap_or(NonZeroUsize::MIN)), + generations: HashMap::default(), + } + } + + fn get(&mut self, repository: Uuid, now: Instant) -> Option> { + let cached = self.entries.get(&repository)?; + if cached.expires_at <= now { + self.entries.pop(&repository); + return None; + } + Some(cached.items.clone()) + } + + fn insert(&mut self, repository: Uuid, items: Vec, now: Instant) { + self.entries.put( + repository, + CachedManifestList { + items, + expires_at: now + MANIFEST_CACHE_TTL, + }, + ); + } + + fn insert_if_generation( + &mut self, + repository: Uuid, + items: Vec, + now: Instant, + generation: u64, + ) { + if self.generations.get(&repository).copied().unwrap_or(0) == generation { + self.insert(repository, items, now); + } + } + + fn generation(&self, repository: Uuid) -> u64 { + self.generations.get(&repository).copied().unwrap_or(0) + } + + fn invalidate(&mut self, repository: Uuid) { + self.generations + .entry(repository) + .and_modify(|generation| *generation = generation.wrapping_add(1)) + .or_insert(1); + self.entries.pop(&repository); + } +} + +const S3_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const S3_CONTROL_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(30); +const S3_CONTROL_TIMEOUT: Duration = Duration::from_secs(90); +const S3_COPY_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(5 * 60); +const S3_COPY_TIMEOUT: Duration = Duration::from_secs(20 * 60); +const MULTIPART_COPY_THRESHOLD: u64 = 5 * 1024 * 1024 * 1024; +const MULTIPART_COPY_PART_SIZE: u64 = 64 * 1024 * 1024; + +fn control_timeout_config() -> aws_smithy_types::timeout::TimeoutConfig { + aws_smithy_types::timeout::TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .read_timeout(S3_CONTROL_ATTEMPT_TIMEOUT) + .operation_attempt_timeout(S3_CONTROL_ATTEMPT_TIMEOUT) + .operation_timeout(S3_CONTROL_TIMEOUT) + .build() +} + +fn copy_timeout_config() -> aws_smithy_types::timeout::TimeoutConfig { + aws_smithy_types::timeout::TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .read_timeout(S3_CONTROL_ATTEMPT_TIMEOUT) + .operation_attempt_timeout(S3_COPY_ATTEMPT_TIMEOUT) + .operation_timeout(S3_COPY_TIMEOUT) + .build() +} + +fn streaming_timeout_config() -> aws_smithy_types::timeout::TimeoutConfig { + aws_smithy_types::timeout::TimeoutConfig::builder() + .connect_timeout(S3_CONNECT_TIMEOUT) + .read_timeout(S3_CONTROL_ATTEMPT_TIMEOUT) + .disable_operation_attempt_timeout() + .disable_operation_timeout() + .build() +} + +fn timeout_override( + timeout_config: aws_smithy_types::timeout::TimeoutConfig, +) -> aws_sdk_s3::config::Builder { + aws_sdk_s3::config::Builder::new().timeout_config(timeout_config) +} + +async fn with_timeout(timeout_duration: Duration, future: F) -> Result +where + F: Future>>, + E: std::error::Error + std::fmt::Display + ProvideErrorMetadata + 'static, +{ + match tokio::time::timeout(timeout_duration, future).await { + Ok(result) => result.map_err(S3StorageError::from_sdk_error), + Err(_) => Err(S3StorageError::aws_message(format!( + "S3 request timed out after {} seconds", + timeout_duration.as_secs() + ))), + } +} + +async fn next_control_page( + deadline: Instant, + future: F, +) -> Result, S3StorageError> +where + F: Future>>>, + E: std::error::Error + std::fmt::Display + ProvideErrorMetadata + 'static, +{ + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(S3StorageError::aws_message( + "S3 control operation exceeded its deadline", + )); + } + let result = tokio::time::timeout(remaining, future) + .await + .map_err(|_| S3StorageError::aws_message("S3 LIST request timed out"))?; + result + .map(|page| page.map_err(S3StorageError::from_sdk_error)) + .transpose() +} + +fn encode_copy_source_component(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + encoded.push(byte as char); + } else { + encoded.push('%'); + encoded.push_str(&format!("{byte:02X}")); + } + } + encoded +} + +fn copy_source(bucket: &str, key: &str) -> String { + format!( + "/{}/{}", + encode_copy_source_component(bucket), + encode_copy_source_component(key) + ) +} + +impl S3StorageInner { + fn bucket(&self) -> &str { + &self.config.bucket_name + } + fn aws_client(&self) -> &AwsS3Client { + &self.client + } + pub async fn load_client(config: &S3Config) -> Result { + let region = config.resolved_region()?; + debug!(%region, bucket = %config.bucket_name, "Connecting to S3 bucket"); + + let (base_config, static_provider) = build_base_config(config, ®ion).await?; let mut builder = aws_sdk_s3::config::Builder::from(&base_config).force_path_style(config.path_style); + builder = builder.timeout_config(control_timeout_config()); let http_client = aws_smithy_http_client::Builder::new() .tls_provider(aws_smithy_http_client::tls::Provider::rustls( @@ -772,17 +1743,17 @@ impl S3StorageInner { } let client = AwsS3Client::from_conf(builder.build()); - match client - .head_bucket() - .bucket(&config.bucket_name) - .send() - .await + match with_timeout( + S3_CONTROL_TIMEOUT, + client.head_bucket().bucket(&config.bucket_name).send(), + ) + .await { Ok(_) => Ok(client), - Err(SdkError::ServiceError(err)) if err.err().is_not_found() => Err( - S3StorageError::BucketDoesNotExist(config.bucket_name.clone()), - ), - Err(err) => Err(S3StorageError::from_sdk_error(err)), + Err(error) if error.is_not_found() => Err(S3StorageError::BucketDoesNotExist( + config.bucket_name.clone(), + )), + Err(error) => Err(error), } } @@ -796,6 +1767,59 @@ impl S3StorageInner { let cache = S3DiskCache::new(&config.cache, &storage.storage_name).await?; Ok(Some(Arc::new(cache))) } + + /// Creates a fresh staging directory for append uploads and removes only stale leftovers from + /// previous runs, leaving active staging directories untouched. + async fn prepare_staging_dir(storage_name: &str) -> Result { + let prefix = format!("pkgly-s3-staging-{storage_name}-"); + let dir = std::env::temp_dir().join(format!("{prefix}{}", Uuid::new_v4().simple())); + let mut entries = fs::read_dir(std::env::temp_dir()).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name().to_string_lossy().into_owned(); + if !name.starts_with(&prefix) { + continue; + } + let owner = fs::read_to_string(entry.path().join(".owner")).await.ok(); + let owned_by_current_process = owner + .as_deref() + .and_then(|value| value.trim().parse::().ok()) + .is_some_and(|pid| pid == std::process::id()); + if !owned_by_current_process { + let _ = fs::remove_dir_all(entry.path()).await; + } + } + fs::create_dir_all(&dir).await?; + fs::write(dir.join(".owner"), std::process::id().to_string()).await?; + Ok(dir) + } + + /// Acquires budget for buffering a body of the given size, or None when it exceeds the + /// shared budget (callers must stream instead of buffering in that case). + async fn acquire_body_budget(&self, size: u64) -> Option { + let permits = size.div_ceil(BODY_BUDGET_PERMIT_BYTES); + let total_permits = BODY_BUDGET_BYTES / BODY_BUDGET_PERMIT_BYTES; + if permits > total_permits { + return None; + } + global_body_budget() + .acquire_many_owned(permits as u32) + .await + .ok() + } + + /// Captures the cache generation for a location, or None when caching is disabled. + async fn cache_generation(&self, repository: &Uuid, location: &StoragePath) -> Option { + if !self.should_cache(location) { + return None; + } + let cache = self.cache.as_ref()?; + Some( + cache + .generation_for(&self.cache_key(repository, location)) + .await, + ) + } + pub fn s3_path(&self, repository: &Uuid, path: &StoragePath) -> String { format!("{}/{}", repository, path) } @@ -834,6 +1858,42 @@ impl S3StorageInner { key.ends_with(".nr-meta") || key.split('/').any(|part| part == ".nr-meta") } + async fn list_directory_entries( + &self, + prefix: &str, + ) -> Result<(Vec, Vec), S3StorageError> { + let mut paginator = self + .aws_client() + .list_objects_v2() + .bucket(self.bucket()) + .prefix(prefix) + .delimiter("/") + .max_keys(1000) + .into_paginator() + .send(); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; + let mut objects = Vec::new(); + let mut prefixes = Vec::new(); + while let Some(page) = next_control_page(deadline, paginator.next()).await? { + objects.extend(page.contents().iter().filter_map(|object| { + let key = object.key()?.to_owned(); + let size = object.size().unwrap_or_default().max(0) as u64; + Some(DirectoryObject { + key, + size, + last_modified: s3_last_modified(object.last_modified()), + }) + })); + prefixes.extend( + page.common_prefixes() + .iter() + .filter_map(CommonPrefix::prefix) + .map(str::to_owned), + ); + } + Ok((objects, prefixes)) + } + async fn cache_get( &self, repository: &Uuid, @@ -855,6 +1915,7 @@ impl S3StorageInner { location: &StoragePath, data: Bytes, content_type: Option, + last_modified: Option>, ) -> Result<(), S3StorageError> { if !self.should_cache(location) { return Ok(()); @@ -862,7 +1923,9 @@ impl S3StorageInner { if let Some(cache) = &self.cache { let key = self.cache_key(repository, location); let data_len = data.len(); - cache.put(&key, data, content_type.as_deref()).await?; + cache + .put_with_metadata(&key, data, content_type.as_deref(), last_modified) + .await?; debug!( repository = %repository, path = %location, @@ -887,6 +1950,466 @@ impl S3StorageInner { } Ok(()) } + + /// Publishes GET-captured content into the cache only when the object generation is + /// unchanged since the read started, so stale content never overwrites newer cache entries. + async fn cache_put_if_generation( + &self, + repository: &Uuid, + location: &StoragePath, + data: Bytes, + content_type: Option, + last_modified: Option>, + generation: u64, + ) -> Result<(), S3StorageError> { + if !self.should_cache(location) { + return Ok(()); + } + if let Some(cache) = &self.cache { + let key = self.cache_key(repository, location); + cache + .put_if_generation( + &key, + data, + content_type.as_deref(), + last_modified, + generation, + ) + .await?; + } + Ok(()) + } + + /// Ensures a local staging file exists for an append-style upload, seeding it with any + /// existing S3 object content. Returns the staging path, current size, and base ETag. + async fn ensure_staging( + &self, + path: &str, + ) -> Result<(PathBuf, u64, Option), S3StorageError> { + let existing = { + let staging = self.append_staging.lock(); + staging + .get(path) + .map(|entry| (entry.path.clone(), entry.size, entry.base_etag.clone())) + }; + if let Some(existing) = existing { + return Ok(existing); + } + let staging_path = self.staging_dir.join(Uuid::new_v4().simple().to_string()); + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&staging_path) + .await?; + let (size, base_etag, budget_permits, reserved_units) = match self + .aws_client() + .get_object() + .bucket(self.bucket()) + .key(path) + .customize() + .config_override(timeout_override(streaming_timeout_config())) + .send() + .await + { + Ok(response) => { + let Some(etag) = response.e_tag().map(str::to_owned) else { + drop(file); + let _ = fs::remove_file(&staging_path).await; + return Err(S3StorageError::aws_message( + "S3 existing object did not return an ETag; refusing an unguarded append", + )); + }; + let Some(size) = response + .content_length() + .and_then(|size| u64::try_from(size).ok()) + else { + drop(file); + let _ = fs::remove_file(&staging_path).await; + return Err(S3StorageError::aws_message( + "S3 existing object did not return a valid content length", + )); + }; + let reserved_units = size.div_ceil(S3_UPLOAD_SPOOL_PERMIT_BYTES) as usize; + let budget_permits = if reserved_units == 0 { + Vec::new() + } else { + match self + .upload_spool_budget + .clone() + .try_acquire_many_owned(reserved_units as u32) + { + Ok(permit) => vec![Arc::new(permit)], + Err(_) => { + drop(file); + let _ = fs::remove_file(&staging_path).await; + return Err(S3StorageError::aws_message( + "S3 append staging byte capacity exceeded", + )); + } + } + }; + let mut stream = response.body.into_async_read(); + let copied = match tokio::io::copy(&mut stream, &mut file).await { + Ok(copied) => copied, + Err(error) => { + drop(file); + let _ = fs::remove_file(&staging_path).await; + return Err(error.into()); + } + }; + if copied != size { + drop(file); + let _ = fs::remove_file(&staging_path).await; + return Err(S3StorageError::aws_message( + "S3 existing object length changed while staging", + )); + } + (copied, Some(etag), budget_permits, reserved_units) + } + Err(error) => { + let error = S3StorageError::from_sdk_error(error); + if error.is_not_found() { + (0, None, Vec::new(), 0) + } else { + drop(file); + let _ = fs::remove_file(&staging_path).await; + return Err(error); + } + } + }; + if let Err(error) = file.flush().await { + drop(file); + let _ = fs::remove_file(&staging_path).await; + return Err(error.into()); + } + // Publish the staging entry; a concurrent first-append may have won, so discard ours. + enum StagingDecision { + Ready(PathBuf, u64, Option), + Existing(PathBuf, u64, Option), + CapacityExceeded, + } + let decision = { + let mut staging = self.append_staging.lock(); + if let Some(existing) = staging.get(path) { + StagingDecision::Existing( + existing.path.clone(), + existing.size, + existing.base_etag.clone(), + ) + } else if staging.len() >= MAX_STAGED_UPLOADS { + StagingDecision::CapacityExceeded + } else { + staging.insert( + path.to_string(), + StagedAppend { + path: staging_path.clone(), + size, + base_etag: base_etag.clone(), + budget_permits, + reserved_units, + }, + ); + StagingDecision::Ready(staging_path.clone(), size, base_etag) + } + }; + match decision { + StagingDecision::Ready(result, size, base_etag) => Ok((result, size, base_etag)), + StagingDecision::Existing(result, size, base_etag) => { + drop(file); + let _ = fs::remove_file(&staging_path).await; + Ok((result, size, base_etag)) + } + StagingDecision::CapacityExceeded => { + drop(file); + let _ = fs::remove_file(&staging_path).await; + Err(S3StorageError::aws_message( + "S3 append staging capacity exceeded; too many concurrent uploads", + )) + } + } + } + + /// Removes staging state for an object, returning whether staging existed. + async fn remove_staging(&self, path: &str) -> bool { + let operation_lock = self.staging_operation_lock(path); + let _operation = operation_lock.lock().await; + let Some(entry) = self.append_staging.lock().remove(path) else { + return false; + }; + let _ = fs::remove_file(entry.path).await; + true + } + + /// Drops staging entries for a repository prefix (used during repository deletion). + async fn remove_staging_prefix(&self, repository: &Uuid) { + let prefix = format!("{repository}/"); + let entries: Vec<(String, PathBuf)> = { + let staging = self.append_staging.lock(); + staging + .iter() + .filter(|(key, _)| key.starts_with(&prefix)) + .map(|(key, entry)| (key.clone(), entry.path.clone())) + .collect() + }; + for (key, path) in entries { + let operation_lock = self.staging_operation_lock(&key); + let _operation = operation_lock.lock().await; + self.append_staging.lock().remove(&key); + let _ = fs::remove_file(path).await; + } + } + + async fn cleanup_staging(&self) -> Result<(), S3StorageError> { + let keys: Vec = self.append_staging.lock().keys().cloned().collect(); + for key in keys { + let operation_lock = self.staging_operation_lock(&key); + let _operation = operation_lock.lock().await; + let staged_path = self + .append_staging + .lock() + .remove(&key) + .map(|entry| entry.path); + if let Some(path) = staged_path { + let _ = fs::remove_file(path).await; + } + } + match fs::remove_dir_all(&self.staging_dir).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + } + + /// Records that a concrete object was created at `path`, invalidating any memoized + /// "clear ancestor" probe for that path. + fn note_object_created(&self, path: &str) { + let mut generation = self.creation_probe_generation.lock(); + *generation = generation.wrapping_add(1); + self.creation_probes.lock().pop(path); + } + + fn creation_probe_generation(&self) -> u64 { + *self.creation_probe_generation.lock() + } + + /// Returns a per-repository lock coordinating cold manifest listing loads. + async fn manifest_load_lock(&self, repository: Uuid) -> Arc> { + let mut locks = self.manifest_load_locks.lock(); + if locks.len() > MANIFEST_LOAD_LOCK_CAPACITY { + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + } + locks + .entry(repository) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } + + fn cache_load_lock(&self, key: &str) -> Arc> { + let mut locks = self.cache_load_locks.lock(); + if locks.len() > CACHE_LOAD_LOCK_CAPACITY { + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + } + locks + .entry(key.to_owned()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } + + /// Returns the per-path lock coordinating staging appends with finalization and + /// cleanup for a single upload, leaving unrelated uploads uncontended. + fn staging_operation_lock(&self, path: &str) -> Arc> { + let mut locks = self.append_operation_locks.lock(); + if locks.len() > CACHE_LOAD_LOCK_CAPACITY { + locks.retain(|_, lock| Arc::strong_count(lock) > 1); + } + locks + .entry(path.to_owned()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } + + /// Atomically reserves `additional` staged bytes for one upload under the shared + /// budget. Returns the newly acquired permit units, or None when the upload has no + /// staging entry or the budget would be exceeded. + fn reserve_staged_size(&self, path: &str, additional: u64) -> Option { + let mut staging = self.append_staging.lock(); + let entry = staging.get_mut(path)?; + let new_size = entry.size.saturating_add(additional); + let required_units = new_size.div_ceil(S3_UPLOAD_SPOOL_PERMIT_BYTES) as usize; + let additional_units = required_units.saturating_sub(entry.reserved_units); + if additional_units > 0 { + let permit = self + .upload_spool_budget + .clone() + .try_acquire_many_owned(additional_units as u32) + .ok()?; + entry.budget_permits.push(Arc::new(permit)); + entry.reserved_units = required_units; + } + entry.size = new_size; + Some(additional_units) + } + + /// Rolls back a reservation after a failed append. + fn release_staged_size(&self, path: &str, amount: u64, reserved_units: usize) { + if let Some(entry) = self.append_staging.lock().get_mut(path) { + entry.size = entry.size.saturating_sub(amount); + if reserved_units > 0 { + entry.reserved_units = entry.reserved_units.saturating_sub(reserved_units); + entry.budget_permits.pop(); + } + } + } + + async fn multipart_copy( + &self, + destination: &str, + source: &str, + source_etag: &str, + object_size: u64, + head: &aws_sdk_s3::operation::head_object::HeadObjectOutput, + ) -> Result<(), S3StorageError> { + let mut create = self + .aws_client() + .create_multipart_upload() + .bucket(self.bucket()) + .key(destination); + if let Some(cache_control) = head.cache_control() { + create = create.cache_control(cache_control); + } + if let Some(content_disposition) = head.content_disposition() { + create = create.content_disposition(content_disposition); + } + if let Some(content_encoding) = head.content_encoding() { + create = create.content_encoding(content_encoding); + } + if let Some(content_language) = head.content_language() { + create = create.content_language(content_language); + } + if let Some(content_type) = head.content_type() { + create = create.content_type(content_type); + } + if let Some(expires) = head.expires_string().and_then(|value| { + aws_smithy_types::DateTime::from_str( + value, + aws_smithy_types::date_time::Format::HttpDate, + ) + .ok() + }) { + create = create.expires(expires); + } + if let Some(redirect) = head.website_redirect_location() { + create = create.website_redirect_location(redirect); + } + if let Some(metadata) = head.metadata() { + for (key, value) in metadata { + create = create.metadata(key, value); + } + } + let copy_deadline = Instant::now() + S3_COPY_TIMEOUT; + let create_timeout = copy_deadline + .saturating_duration_since(Instant::now()) + .min(S3_CONTROL_TIMEOUT); + let created = with_timeout(create_timeout, create.send()).await?; + let Some(upload_id) = created.upload_id() else { + return Err(S3StorageError::aws_message( + "S3 multipart copy did not return an upload ID", + )); + }; + let upload_id = upload_id.to_owned(); + let part_size = MULTIPART_COPY_PART_SIZE.max(object_size.div_ceil(10_000)); + let part_count = object_size.div_ceil(part_size); + let mut completed_parts = Vec::with_capacity(part_count as usize); + + for part in 0..part_count { + let start = part * part_size; + let end = (start + part_size).min(object_size) - 1; + let remaining = copy_deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(S3StorageError::aws_message( + "S3 multipart copy exceeded its operation deadline", + )); + } + let result = with_timeout( + remaining, + self.aws_client() + .upload_part_copy() + .bucket(self.bucket()) + .key(destination) + .upload_id(&upload_id) + .part_number((part + 1) as i32) + .copy_source(source) + .copy_source_range(format!("bytes={start}-{end}")) + .copy_source_if_match(source_etag) + .customize() + .config_override(timeout_override(copy_timeout_config())) + .send(), + ) + .await; + let result = match result { + Ok(result) => result, + Err(error) => { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(error); + } + }; + let Some(etag) = result.copy_part_result().and_then(|part| part.e_tag()) else { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(S3StorageError::aws_message( + "S3 multipart copy part did not return an ETag", + )); + }; + completed_parts.push( + aws_sdk_s3::types::CompletedPart::builder() + .part_number((part + 1) as i32) + .e_tag(etag) + .build(), + ); + } + + let complete = self + .aws_client() + .complete_multipart_upload() + .bucket(self.bucket()) + .key(destination) + .upload_id(upload_id.clone()) + .multipart_upload( + aws_sdk_s3::types::CompletedMultipartUpload::builder() + .set_parts(Some(completed_parts)) + .build(), + ); + let remaining = copy_deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(S3StorageError::aws_message( + "S3 multipart copy exceeded its operation deadline", + )); + } + if let Err(error) = with_timeout(remaining.min(S3_CONTROL_TIMEOUT), complete.send()).await { + self.abort_multipart_copy(destination, &upload_id).await; + return Err(error); + } + Ok(()) + } + + async fn abort_multipart_copy(&self, destination: &str, upload_id: &str) { + let _ = with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .abort_multipart_upload() + .bucket(self.bucket()) + .key(destination) + .upload_id(upload_id) + .send(), + ) + .await; + } + + fn invalidate_manifest_cache(&self, repository: Uuid) { + self.manifest_cache.lock().invalidate(repository); + } + pub async fn get_path_for_creation( &self, repository: Uuid, @@ -902,30 +2425,41 @@ impl S3StorageInner { conflicting_path.push_mut(part.as_ref()); let is_last = iter.peek().is_none(); - let exists_as_object = self.does_path_exist(&path).await?; - - if exists_as_object && !is_last { - // A parent segment is a concrete object, so we cannot place a child under it. - return Err(PathCollisionError { - path: location.clone(), - conflicts_with: conflicting_path, + if !is_last { + // Memoize ancestor probes: the S3 object set changes only through this storage + // instance, and every concrete object creation invalidates its own path. + let already_checked = self.creation_probes.lock().peek(&path).is_some(); + if !already_checked { + let probe_generation = self.creation_probe_generation(); + if self.does_path_exist(&path).await? { + // A parent segment is a concrete object, so we cannot place a child under it. + return Err(PathCollisionError { + path: location.clone(), + conflicts_with: conflicting_path, + } + .into()); + } + if self.creation_probe_generation() == probe_generation { + self.creation_probes.lock().put(path.clone(), ()); + } } - .into()); } - // If this is the last segment, overwriting an existing object is allowed. } Ok(path) } #[instrument] async fn does_path_exist(&self, path: &str) -> Result { - let result = self - .aws_client() - .head_object() - .bucket(self.bucket()) - .key(path) - .send() - .await; + let result = tokio::time::timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .head_object() + .bucket(self.bucket()) + .key(path) + .send(), + ) + .await + .map_err(|_| S3StorageError::aws_message("S3 HEAD request timed out"))?; match result { Ok(_) => Ok(true), @@ -933,122 +2467,24 @@ impl S3StorageInner { Err(err) => Err(S3StorageError::from_sdk_error(err)), } } - #[instrument] - fn is_directory_from_result( - &self, - result: &aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output, - path: &str, - ) -> (bool, Option) { - let contents = result.contents(); - let prefixes = result.common_prefixes(); - let is_contents_empty = contents.is_empty(); - let has_prefixes = !prefixes.is_empty(); - - if is_contents_empty && !has_prefixes { - return (true, None); - } - if path.ends_with('/') && !is_contents_empty { - return (true, None); - } - - let path_with_slash = format!("{}/", path); - if let Some(match_prefix) = prefixes - .iter() - .filter_map(CommonPrefix::prefix) - .find(|prefix| *prefix == path_with_slash) - { - return (true, Some(match_prefix.to_string())); - } - - (false, None) - } - - #[instrument] - async fn is_directory(&self, path: &str) -> Result { - let list = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(path.to_owned()) - .delimiter("/") - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - - Ok(self.is_directory_from_result(&list, path).0) - } - async fn get_directory_meta( &self, path: &str, + modified: Option>, ) -> Result>, S3StorageError> { let file_file = FileType::Directory(DirectoryFileType { file_count: 0 }); let name = path.split_once('/').map(|(_, rest)| rest).unwrap_or(path); + let modified = modified.unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta { name: name.to_owned(), file_type: file_file, - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified, + created: modified, }; Ok(Some(meta)) } - #[instrument] - async fn get_object_tagging(&self, path: &str) -> Result>, S3StorageError> { - let response = self - .aws_client() - .get_object_tagging() - .bucket(self.bucket()) - .key(path) - .send() - .await; - - match response { - Ok(output) => Ok(Some(output.tag_set().to_vec())), - Err(SdkError::ServiceError(err)) - if err - .err() - .meta() - .code() - .is_some_and(|code| code == "NoSuchKey") => - { - Ok(None) - } - Err(err) => Err(S3StorageError::from_sdk_error(err)), - } - } - - async fn get_meta_tags(&self, path: &str) -> Result, S3StorageError> { - let Some(tags) = self.get_object_tagging(path).await? else { - return Ok(None); - }; - - let name = tags - .iter() - .find(|tag| tag.key() == tags::NAME) - .map(|tag| tag.value().to_string()) - .ok_or_else(|| S3StorageError::static_missing_tag(tags::NAME))?; - - let mime_type = tags - .iter() - .find(|tag| tag.key() == tags::MIME_TYPE) - .map(|tag| Mime::from_str(tag.value())) - .transpose(); - let mime_type = match mime_type { - Ok(ok) => ok, - Err(e) => { - error!(?e, ?path, "Failed to parse mime type"); - None - } - }; - - Ok(Some(S3MetaTags { - name, - mime_type, - is_directory: false, - })) - } } async fn build_base_config( @@ -1091,47 +2527,95 @@ fn default_session_name() -> String { format!("pkgly-{}", Uuid::new_v4().simple()) } -fn bytes_to_stream(bytes: FileContentBytes) -> (ByteStream, usize) { - match bytes { - FileContentBytes::Content(content) => { - let len = content.len(); - (ByteStream::from(content), len) - } - FileContentBytes::Bytes(bytes) => { - let len = bytes.len(); - (ByteStream::from(bytes.to_vec()), len) - } - } +fn bytes_to_stream(bytes: Bytes) -> (ByteStream, usize) { + let len = bytes.len(); + (ByteStream::from(bytes), len) } -async fn file_into_bytes(file: FileContent) -> Result { +async fn file_into_bytes(file: FileContent) -> Result { let bytes = task::spawn_blocking(move || FileContentBytes::try_from(file)).await??; - Ok(bytes) + Ok(match bytes { + FileContentBytes::Content(content) => Bytes::from(content), + FileContentBytes::Bytes(bytes) => bytes, + }) } async fn collect_body(stream: ByteStream) -> Result { let aggregated = stream .collect() .await - .map_err(|err| S3StorageError::AwsSdkError(err.to_string()))?; + .map_err(|err| S3StorageError::aws_message(err.to_string()))?; Ok(aggregated.into_bytes()) } const DEFAULT_MIN_BUFFERED_OBJECT_BYTES: u64 = 1024 * 1024; // 1 MiB const DEFAULT_MAX_BUFFERED_OBJECT_BYTES: u64 = 8 * 1024 * 1024; // 8 MiB const DEFAULT_MEMORY_PRESSURE_THRESHOLD: f64 = 0.75; +const MEMORY_SNAPSHOT_TTL: Duration = Duration::from_secs(5); +const CACHE_FORMAT_VERSION: u8 = 1; +const MANIFEST_CACHE_TTL: Duration = Duration::from_secs(30); const FAILED_DELETION_QUEUE_LIMIT: usize = 1024; const FAILED_DELETION_MAX_RETRIES_PER_TICK: usize = 64; const FAILED_DELETION_BASE_DELAY_MS: u64 = 100; const FAILED_DELETION_MAX_DELAY_MS: u64 = 30_000; const FAILED_DELETION_BACKOFF_CUTOFF: u32 = 8; +// Shared budget for concurrently buffered S3 bodies (GET bodies, cache pre-warming reads). +// Permits are 1 MiB each; operations needing more than the budget stream instead of buffering. +const BODY_BUDGET_BYTES: u64 = 256 * 1024 * 1024; +const BODY_BUDGET_PERMIT_BYTES: u64 = 1024 * 1024; +static GLOBAL_BODY_BUDGET: OnceLock> = OnceLock::new(); + +// Staged (local-disk) append uploads bound concurrent in-flight Docker pushes. The flat cap makes +// capacity exhaustion explicit instead of allowing uploads to spill without a bound. Incoming +// upload chunk files spool against the same budget through `upload_spool_budget`, so staged +// bytes plus in-flight spool files never exceed MAX_STAGED_BYTES. +const MAX_STAGED_UPLOADS: usize = 64; +const MAX_STAGED_BYTES: u64 = 64 * 1024 * 1024 * 1024; +/// Permit granularity of the shared upload spool budget (see `S3Storage::upload_spool_budget`). +pub const S3_UPLOAD_SPOOL_PERMIT_BYTES: u64 = 1024 * 1024; +const CREATION_PROBE_CACHE_CAPACITY: usize = 4096; +const CACHE_LOAD_LOCK_CAPACITY: usize = 4096; +const MANIFEST_LOAD_LOCK_CAPACITY: usize = 1024; + #[derive(Debug, Copy, Clone, PartialEq, Eq)] enum BodyRetrievalStrategy { BufferAndCache, StreamWithoutCache, } +/// Keeps a buffered response's memory reservation until its bytes have been consumed. +struct BudgetedBytesReader { + bytes: Bytes, + offset: usize, + _permit: OwnedSemaphorePermit, +} + +impl AsyncRead for BudgetedBytesReader { + fn poll_read( + mut self: Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> std::task::Poll> { + if self.offset < self.bytes.len() && buffer.remaining() > 0 { + let count = (self.bytes.len() - self.offset).min(buffer.remaining()); + buffer.put_slice(&self.bytes[self.offset..self.offset + count]); + self.offset += count; + } + std::task::Poll::Ready(Ok(())) + } +} + +fn global_body_budget() -> Arc { + GLOBAL_BODY_BUDGET + .get_or_init(|| { + Arc::new(Semaphore::new( + (BODY_BUDGET_BYTES / BODY_BUDGET_PERMIT_BYTES) as usize, + )) + }) + .clone() +} + impl BodyRetrievalStrategy { fn from_content_length( length: Option, @@ -1160,8 +2644,23 @@ pub struct S3ListedObject { /// Key relative to the repository root (e.g. `packages/pkg/file.tgz`). pub key: String, pub size: u64, + /// Provider object timestamp, when the S3 response supplied a usable value. pub last_modified: Option>, } + +fn s3_last_modified( + value: Option<&aws_smithy_types::DateTime>, +) -> Option> { + let system_time: SystemTime = (*value?).try_into().ok()?; + Some(ChronoDateTime::::from(system_time).fixed_offset()) +} + +#[derive(Debug, Clone)] +struct DirectoryObject { + key: String, + size: u64, + last_modified: Option>, +} #[derive(Debug, Clone)] pub struct S3Storage(Arc); new_type_arc_type!(S3Storage(S3StorageInner)); @@ -1174,6 +2673,7 @@ impl Storage for S3Storage { #[instrument(name = "Storage::unload", fields(storage_type = "s3"))] async fn unload(&self) -> Result<(), S3StorageError> { info!("Unloading S3 Storage"); + self.cleanup_staging().await?; Ok(()) } #[instrument(fields(storage_type = "s3"))] @@ -1186,7 +2686,7 @@ impl Storage for S3Storage { #[instrument( name = "Storage::save_file", fields(storage_type = "s3", repository = %repository, path = %location), - skip(file) + skip(self, file) )] async fn save_file( &self, @@ -1195,6 +2695,7 @@ impl Storage for S3Storage { location: &StoragePath, ) -> Result<(usize, bool), S3StorageError> { let path = self.get_path_for_creation(repository, location).await?; + self.note_object_created(&path); let already_exists = self.does_path_exist(&path).await?; if already_exists { debug!("File already exists, overwriting"); @@ -1204,27 +2705,102 @@ impl Storage for S3Storage { } else { "application/octet-stream" }; - let file_as_bytes = file_into_bytes(file).await?; - let cache_buffer = file_as_bytes.clone_into_bytes(); - let (body, size) = bytes_to_stream(file_as_bytes); - self.aws_client() - .put_object() - .bucket(self.bucket()) - .key(&path) - .body(body) - .content_type(content_type) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - debug!(path = %path, "File saved to S3"); - self.cache_put( - &repository, - location, - cache_buffer, - Some(content_type.to_string()), - ) - .await?; - Ok((size, !already_exists)) + let size = match file { + // Stream paths directly into S3 instead of buffering whole files in memory. + FileContent::Path(ref file_path) => { + let size = fs::metadata(file_path).await?.len(); + let cacheable = self.should_cache(location) + && size <= self.cache.as_ref().map_or(0, |cache| cache.max_bytes) + && size <= BODY_BUDGET_BYTES; + let body = ByteStream::read_from() + .path(file_path.clone()) + .build() + .await + .map_err(|error| S3StorageError::IOError(std::io::Error::other(error)))?; + self.aws_client() + .put_object() + .bucket(self.bucket()) + .key(&path) + .body(body) + .content_type(content_type) + .customize() + .config_override(timeout_override(streaming_timeout_config())) + .send() + .await + .map_err(S3StorageError::from_sdk_error)?; + self.note_object_created(&path); + debug!(path = %path, "File streamed to S3"); + self.invalidate_manifest_cache(repository); + let modified = Local::now().fixed_offset(); + if cacheable { + if let Some(_budget) = self.acquire_body_budget(size).await { + let data = fs::read(file_path).await?; + self.cache_put( + &repository, + location, + Bytes::from(data), + Some(content_type.to_string()), + Some(modified), + ) + .await?; + } else { + self.cache_remove(&repository, location).await?; + } + } else { + self.cache_remove(&repository, location).await?; + } + size + } + other => { + let body_size = match &other { + FileContent::Content(content) => content.len() as u64, + FileContent::Bytes(bytes) => bytes.len() as u64, + FileContent::Path(_) => 0, + }; + let _budget = if body_size <= BODY_BUDGET_BYTES { + self.acquire_body_budget(body_size).await + } else { + None + }; + let file_as_bytes = file_into_bytes(other).await?; + let size = file_as_bytes.len() as u64; + let cache_buffer = (self.should_cache(location) + && size <= self.cache.as_ref().map_or(0, |cache| cache.max_bytes) + && size <= BODY_BUDGET_BYTES) + .then(|| file_as_bytes.clone()); + let (body, _) = bytes_to_stream(file_as_bytes); + self.aws_client() + .put_object() + .bucket(self.bucket()) + .key(&path) + .body(body) + .content_type(content_type) + .customize() + .config_override(timeout_override(streaming_timeout_config())) + .send() + .await + .map_err(S3StorageError::from_sdk_error)?; + self.note_object_created(&path); + debug!(path = %path, "File saved to S3"); + self.invalidate_manifest_cache(repository); + let modified = Local::now().fixed_offset(); + if let Some(cache_buffer) = cache_buffer { + self.cache_put( + &repository, + location, + cache_buffer, + Some(content_type.to_string()), + Some(modified), + ) + .await?; + } else { + self.cache_remove(&repository, location).await?; + } + size + } + }; + let size_usize = usize::try_from(size).unwrap_or(usize::MAX); + Ok((size_usize, !already_exists)) } #[instrument(name = "Storage::append_file", fields(storage_type = "s3"))] async fn append_file( @@ -1233,53 +2809,66 @@ impl Storage for S3Storage { file: FileContent, location: &StoragePath, ) -> Result { - // S3 doesn't support native append operations - // We need to read, append, and write back - // This is still O(n) for S3 since network I/O dominates let path = self.get_path_for_creation(repository, location).await?; - - let mut combined_buffer = if self.does_path_exist(&path).await? { - let response = self - .aws_client() - .get_object() - .bucket(self.bucket()) - .key(&path) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - collect_body(response.body).await?.to_vec() - } else { - Vec::new() + let (current_bytes, current_etag) = match self + .aws_client() + .get_object() + .bucket(self.bucket()) + .key(&path) + .customize() + .config_override(timeout_override(streaming_timeout_config())) + .send() + .await + { + Ok(response) => { + let Some(etag) = response.e_tag().map(str::to_owned) else { + return Err(S3StorageError::aws_message( + "S3 existing object did not return an ETag; refusing an unguarded append", + )); + }; + (collect_body(response.body).await?, Some(etag)) + } + Err(error) => { + let error = S3StorageError::from_sdk_error(error); + if error.is_not_found() { + (Bytes::new(), None) + } else { + return Err(error); + } + } }; - let appended = file_into_bytes(file).await?; - combined_buffer.extend_from_slice(appended.as_ref()); - - let combined_bytes = Bytes::from(combined_buffer); - + let mut combined_buffer = BytesMut::with_capacity(current_bytes.len() + appended.len()); + combined_buffer.extend_from_slice(¤t_bytes); + combined_buffer.extend_from_slice(&appended); + let combined_bytes = combined_buffer.freeze(); let content_type = if location.is_directory() { "application/x-directory" } else { "application/octet-stream" }; - let size = combined_bytes.len(); - self.aws_client() + let mut request = self + .aws_client() .put_object() .bucket(self.bucket()) .key(&path) .content_type(content_type) - .body(ByteStream::from(combined_bytes.clone())) + .body(ByteStream::from(combined_bytes.clone())); + request = match current_etag { + Some(etag) => request.if_match(etag), + None => request.if_none_match("*"), + }; + request + .customize() + .config_override(timeout_override(streaming_timeout_config())) .send() .await .map_err(S3StorageError::from_sdk_error)?; - self.cache_put( - &repository, - location, - combined_bytes, - Some(content_type.to_string()), - ) - .await?; - Ok(size) + let appended_size = appended.len(); + self.note_object_created(&path); + self.cache_remove(&repository, location).await?; + self.invalidate_manifest_cache(repository); + Ok(appended_size) } #[instrument(name = "Storage::put_repository_meta", fields(storage_type = "s3"))] async fn put_repository_meta( @@ -1307,10 +2896,8 @@ impl Storage for S3Storage { .list_objects_v2() .bucket(self.bucket()) .prefix(prefix) - .max_keys(1) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + .max_keys(1); + let probe = with_timeout(S3_CONTROL_TIMEOUT, probe.send()).await?; if probe.key_count().unwrap_or(0) == 0 { return Err(S3StorageError::IOError(std::io::Error::new( std::io::ErrorKind::NotFound, @@ -1325,17 +2912,20 @@ impl Storage for S3Storage { .map(ByteStream::from) .map_err(|err| S3StorageError::IOError(std::io::Error::other(err)))?; - self.aws_client() - .put_object() - .bucket(self.bucket()) - .key(meta_path) - .content_type("application/json") - .body(body) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .put_object() + .bucket(self.bucket()) + .key(meta_path) + .content_type("application/json") + .body(body) + .send(), + ) + .await?; // Repository meta is small; we intentionally do not cache it to avoid polluting the blob cache. + self.invalidate_manifest_cache(repository); Ok(()) } #[instrument(name = "Storage::get_repository_meta", fields(storage_type = "s3"))] @@ -1347,17 +2937,19 @@ impl Storage for S3Storage { let meta_location = S3StorageInner::meta_storage_path(location); let meta_path = self.s3_path(&repository, &meta_location); - let response = match self - .aws_client() - .get_object() - .bucket(self.bucket()) - .key(&meta_path) - .send() - .await + let response = match with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .get_object() + .bucket(self.bucket()) + .key(&meta_path) + .send(), + ) + .await { Ok(resp) => resp, - Err(SdkError::ServiceError(err)) if err.err().is_no_such_key() => return Ok(None), - Err(err) => return Err(S3StorageError::from_sdk_error(err)), + Err(error) if error.is_not_found() => return Ok(None), + Err(error) => return Err(error), }; let body = collect_body(response.body).await?; @@ -1376,18 +2968,22 @@ impl Storage for S3Storage { location: &StoragePath, ) -> Result { let path = self.s3_path(&repository, location); + let had_staging = self.remove_staging(&path).await; let exists = self.does_path_exist(&path).await?; if !exists { - return Ok(false); + return Ok(had_staging); } - self.aws_client() - .delete_object() - .bucket(self.bucket()) - .key(&path) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .delete_object() + .bucket(self.bucket()) + .key(&path) + .send(), + ) + .await?; self.cache_remove(&repository, location).await?; + self.invalidate_manifest_cache(repository); Ok(true) } #[instrument( @@ -1404,54 +3000,161 @@ impl Storage for S3Storage { let from_path = self.s3_path(&repository, from); let to_path = self.s3_path(&repository, to); - // Check if source exists - if !self.does_path_exist(&from_path).await? { - return Ok(false); - } - - // For S3, we need to copy and then delete since there's no native rename - // Read the object - let response = self - .aws_client() - .get_object() - .bucket(self.bucket()) - .key(&from_path) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - let bytes = collect_body(response.body).await?; - - // Get content type from original object metadata (if available) - let content_type = if to.is_directory() { - "application/x-directory" - } else { - "application/octet-stream" - }; - - // Write to new location - self.aws_client() - .put_object() - .bucket(self.bucket()) - .key(&to_path) - .content_type(content_type) - .body(ByteStream::from(bytes.clone())) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - - // Delete original - self.aws_client() - .delete_object() - .bucket(self.bucket()) - .key(&from_path) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + // Finalize a staged append-style upload: upload the local staging file once and delete + // the pre-existing S3 source object (if any) with the same conditional guard as before. + // Serialize with appends and cleanup for this upload only, so the S3 transfer does not + // block unrelated uploads; drop the guard before an ordinary server-side move. + let staging_lock = self.staging_operation_lock(&from_path); + let staging_operation = staging_lock.lock().await; + let staged = self.append_staging.lock().get(&from_path).cloned(); + if let Some(staged) = staged { + self.note_object_created(&to_path); + let content_type = if to.is_directory() { + "application/x-directory" + } else { + "application/octet-stream" + }; + let body = ByteStream::read_from() + .path(staged.path.clone()) + .build() + .await + .map_err(|error| S3StorageError::IOError(std::io::Error::other(error)))?; + let mut request = self + .aws_client() + .put_object() + .bucket(self.bucket()) + .key(&to_path) + .content_type(content_type) + .body(body); + request = request.if_none_match("*"); + let upload_result = request + .customize() + .config_override(timeout_override(streaming_timeout_config())) + .send() + .await; + if let Err(error) = upload_result { + let error = S3StorageError::from_sdk_error(error); + if !error.is_conflict() { + return Err(error); + } + // A previous attempt may have uploaded the destination before failing while + // deleting the source. Treat a same-sized destination as an idempotent retry. + let destination = match with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .head_object() + .bucket(self.bucket()) + .key(&to_path) + .send(), + ) + .await + { + Ok(destination) => destination, + Err(head_error) if head_error.is_not_found() => return Err(error), + Err(_) => return Err(error), + }; + let destination_size = + destination.content_length().unwrap_or_default().max(0) as u64; + if destination_size != staged.size { + return Err(error); + } + } + self.note_object_created(&to_path); + if let Some(etag) = staged.base_etag.clone() + && let Err(error) = with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .delete_object() + .bucket(self.bucket()) + .key(&from_path) + .if_match(etag) + .send(), + ) + .await + { + warn!(path = %from_path, %error, "Conditional source deletion failed after staged finalize"); + self.cache_remove(&repository, to).await?; + self.cache_remove(&repository, from).await?; + self.invalidate_manifest_cache(repository); + return Err(error); + } + self.append_staging.lock().remove(&from_path); + let _ = fs::remove_file(&staged.path).await; + self.cache_remove(&repository, to).await?; + self.cache_remove(&repository, from).await?; + self.invalidate_manifest_cache(repository); + return Ok(true); + } + drop(staging_operation); - self.cache_remove(&repository, from).await?; - self.cache_put(&repository, to, bytes, Some(content_type.to_string())) + let head = match with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .head_object() + .bucket(self.bucket()) + .key(&from_path) + .send(), + ) + .await + { + Ok(head) => head, + Err(error) if error.is_not_found() => { + return Ok(false); + } + Err(error) => return Err(error), + }; + self.note_object_created(&to_path); + let Some(source_etag) = head.e_tag().map(str::to_owned) else { + return Err(S3StorageError::aws_message( + "S3 source object did not return an ETag; refusing an unguarded move", + )); + }; + let object_size = head.content_length().unwrap_or_default().max(0) as u64; + let source = copy_source(self.bucket(), &from_path); + + if object_size <= MULTIPART_COPY_THRESHOLD { + with_timeout( + S3_COPY_TIMEOUT, + self.aws_client() + .copy_object() + .bucket(self.bucket()) + .key(&to_path) + .copy_source(source.clone()) + .copy_source_if_match(source_etag.clone()) + .customize() + .config_override(timeout_override(copy_timeout_config())) + .send(), + ) .await?; + } else { + self.multipart_copy(&to_path, &source, &source_etag, object_size, &head) + .await?; + } + self.note_object_created(&to_path); + + // The destination changed as soon as the copy completed. Drop any stale destination + // cache before attempting the guarded source deletion. + self.cache_remove(&repository, to).await?; + let delete_result = with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .delete_object() + .bucket(self.bucket()) + .key(&from_path) + .if_match(source_etag) + .send(), + ) + .await; + if let Err(error) = delete_result { + // A failed conditional delete may mean the source changed or that the request was + // accepted before the connection failed; both cache entries are unsafe to retain. + self.cache_remove(&repository, from).await?; + self.invalidate_manifest_cache(repository); + return Err(error); + } + self.cache_remove(&repository, from).await?; + self.invalidate_manifest_cache(repository); Ok(true) } #[instrument( @@ -1464,92 +3167,121 @@ impl Storage for S3Storage { repository: uuid::Uuid, location: &StoragePath, ) -> Result>, S3StorageError> { - if let Some(cached) = self.cache_get(&repository, location).await? { + // Staged append uploads exist only locally until finalized; report their size directly. + let path = self.s3_path(&repository, location); + if let Some(entry) = self.append_staging.lock().get(&path) { + let modified = Local::now().fixed_offset(); + return Ok(Some(StorageFileMeta:: { + name: location.to_string(), + file_type: FileType::File(FileFileType { + file_size: entry.size, + mime_type: None, + file_hash: FileHashes::default(), + }), + modified, + created: modified, + })); + } + + // Metadata describes the S3 object even if its local content copy is evicted + // or damaged. Only content reads need to read and verify the cache file. + let cached = if self.should_cache(location) { + if let Some(cache) = &self.cache { + let key = self.cache_key(&repository, location); + cache.state.lock().await.entries.get(&key).cloned() + } else { + None + } + } else { + None + }; + if let Some(cached) = cached { let mime_type = cached .content_type .as_deref() .and_then(|ct| Mime::from_str(ct).ok()) .map(SerdeMime); - let size = cached.bytes.len() as u64; + let size = cached.size; + let modified = cached + .last_modified + .unwrap_or_else(|| Local::now().fixed_offset()); return Ok(Some(StorageFileMeta:: { name: location.to_string(), file_type: FileType::File(FileFileType { file_size: size, mime_type, - file_hash: FileHashes::default(), + file_hash: FileHashes { + md5: None, + sha1: None, + sha2_256: Some(cached.digest), + sha3_256: None, + }, }), - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified, + created: modified, })); } let path = self.s3_path(&repository, location); - let head = match self - .aws_client() - .head_object() - .bucket(self.bucket()) - .key(&path) - .send() - .await + let head = match with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .head_object() + .bucket(self.bucket()) + .key(&path) + .send(), + ) + .await { Ok(head) => head, - Err(SdkError::ServiceError(err)) if err.err().is_not_found() => { + Err(error) if error.is_not_found() => { // Maybe this is a directory prefix without a placeholder object. let prefix = if path.ends_with('/') { path.clone() } else { format!("{}/", path) }; - let list = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(prefix.clone()) - .delimiter("/") - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; - - if list.key_count().unwrap_or(0) == 0 { + let (objects, prefixes) = self.list_directory_entries(&prefix).await?; + if objects.is_empty() && prefixes.is_empty() { return Ok(None); } - let mut count: u64 = 0; - for obj in list.contents() { - if let Some(key) = obj.key() { - if key == prefix || S3StorageInner::is_hidden_file(key) { - continue; - } - count += 1; - } - } - for pref in list - .common_prefixes() + let count = objects .iter() - .filter_map(CommonPrefix::prefix) - { - if S3StorageInner::is_hidden_file(pref) { - continue; - } - count += 1; - } + .filter(|object| { + object.key != prefix && !S3StorageInner::is_hidden_file(&object.key) + }) + .count() + + prefixes + .iter() + .filter(|prefix| !S3StorageInner::is_hidden_file(prefix)) + .count(); + let modified = objects + .iter() + .find(|object| object.key == prefix) + .and_then(|object| object.last_modified) + .unwrap_or_else(|| Local::now().fixed_offset()); let dir_meta = StorageFileMeta:: { name: location.to_string(), - file_type: FileType::Directory(DirectoryFileType { file_count: count }), - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + file_type: FileType::Directory(DirectoryFileType { + file_count: count as u64, + }), + modified, + created: modified, }; return Ok(Some(dir_meta)); } - Err(err) => return Err(S3StorageError::from_sdk_error(err)), + Err(error) => return Err(error), }; let content_type = head.content_type().map(|ct| ct.to_string()); if content_type .as_deref() .is_some_and(|ct| ct == "application/x-directory") - && let Some(meta) = self.get_directory_meta(&path).await? + && let Some(meta) = self + .get_directory_meta(&path, s3_last_modified(head.last_modified())) + .await? { return Ok(Some(meta)); } @@ -1566,7 +3298,8 @@ impl Storage for S3Storage { .unwrap_or_default() .map(SerdeMime); - let modified = Local::now().fixed_offset(); + let modified = + s3_last_modified(head.last_modified()).unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta:: { name: location.to_string(), @@ -1589,59 +3322,143 @@ impl Storage for S3Storage { repository: uuid::Uuid, location: &StoragePath, ) -> Result, S3StorageError> { + // Staged append uploads live only on local disk until finalized; serve them directly. + let path = self.s3_path(&repository, location); + let staged = self.append_staging.lock().get(&path).cloned(); + if let Some(staged) = staged { + let modified = Local::now().fixed_offset(); + let meta = StorageFileMeta:: { + name: location.to_string(), + file_type: FileFileType { + file_size: staged.size, + mime_type: None, + file_hash: FileHashes::default(), + }, + modified, + created: modified, + }; + let file = fs::File::open(&staged.path).await?; + return Ok(Some(StorageFile::File { + meta, + content: crate::StorageFileReader::File(file), + })); + } if let Some(cached) = self.cache_get(&repository, location).await? { let mime_type = cached .content_type .as_deref() .and_then(|ct| Mime::from_str(ct).ok()) .map(SerdeMime); - let size = cached.bytes.len() as u64; + let modified = cached + .last_modified + .unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta:: { name: location.to_string(), file_type: FileFileType { - file_size: size, + file_size: cached.size, mime_type, - file_hash: FileHashes::default(), + // Retain the digest verified when the cache entry was written so callers + // (e.g. Docker blob delivery) can skip re-hashing the content. + file_hash: FileHashes { + md5: None, + sha1: None, + sha2_256: Some(cached.digest), + sha3_256: None, + }, }, - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified, + created: modified, }; let result = StorageFile::File { meta, - content: crate::StorageFileReader::Bytes(FileContentBytes::Bytes(cached.bytes)), + content: crate::StorageFileReader::File(cached.file), }; return Ok(Some(result)); } + let cache_allowed = self.should_cache(location); + let cache_load_lock = + cache_allowed.then(|| self.cache_load_lock(&self.cache_key(&repository, location))); + let cache_load_guard = if let Some(lock) = cache_load_lock.as_ref() { + Some(lock.lock().await) + } else { + None + }; + // Another request may have populated the cache while this request was waiting for the + // per-key lock. + if cache_allowed && let Some(cached) = self.cache_get(&repository, location).await? { + let mime_type = cached + .content_type + .as_deref() + .and_then(|ct| Mime::from_str(ct).ok()) + .map(SerdeMime); + let modified = cached + .last_modified + .unwrap_or_else(|| Local::now().fixed_offset()); + let meta = StorageFileMeta:: { + name: location.to_string(), + file_type: FileFileType { + file_size: cached.size, + mime_type, + file_hash: FileHashes { + md5: None, + sha1: None, + sha2_256: Some(cached.digest), + sha3_256: None, + }, + }, + modified, + created: modified, + }; + return Ok(Some(StorageFile::File { + meta, + content: crate::StorageFileReader::File(cached.file), + })); + } let path = self.s3_path(&repository, location); + // Capture the generation before starting the S3 request. A mutation that completes while + // the response is in flight must prevent this response from being cached. + let cache_generation = if cache_allowed { + self.cache_generation(&repository, location).await + } else { + None + }; let response = match self .aws_client() .get_object() .bucket(self.bucket()) .key(&path) + .customize() + .config_override(timeout_override(streaming_timeout_config())) .send() .await { Ok(resp) => resp, Err(SdkError::ServiceError(err)) if err.err().is_no_such_key() => { - return self.collect_directory(repository, location).await; + return self.collect_directory(repository, location, None).await; } Err(err) => return Err(S3StorageError::from_sdk_error(err)), }; let response_content_type = response.content_type().map(|ct| ct.to_string()); + let response_last_modified = s3_last_modified(response.last_modified()); if response_content_type .as_deref() .map(|ct| ct == "application/x-directory") .unwrap_or(false) { - return self.collect_directory(repository, location).await; + return self + .collect_directory(repository, location, response_last_modified) + .await; } let response_length_opt = response .content_length() .and_then(|len| len.try_into().ok()); let response_length = response_length_opt.unwrap_or_default(); - let cache_allowed = self.should_cache(location); - let buffer_limit = self.config.adaptive_buffer.buffer_limit_bytes(); + let buffer_limit = self + .config + .adaptive_buffer + .buffer_limit_bytes() + .min(BODY_BUDGET_BYTES); let strategy = BodyRetrievalStrategy::from_content_length( response_length_opt, cache_allowed, @@ -1665,6 +3482,7 @@ impl Storage for S3Storage { ); } + let modified = response_last_modified.unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta:: { name: location.to_string(), file_type: FileFileType { @@ -1677,25 +3495,36 @@ impl Storage for S3Storage { .map(SerdeMime), file_hash: FileHashes::default(), }, - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified, + created: modified, }; let content = match strategy { BodyRetrievalStrategy::BufferAndCache => { + let budget = self + .acquire_body_budget(response_length) + .await + .ok_or_else(|| S3StorageError::aws_message("S3 body budget is unavailable"))?; let body = collect_body(response.body).await?; - if strategy.should_cache() { - self.cache_put( + if let Some(generation) = cache_generation { + self.cache_put_if_generation( &repository, location, body.clone(), response_content_type.clone(), + Some(modified), + generation, ) .await?; } - crate::StorageFileReader::Bytes(FileContentBytes::Bytes(body)) + crate::StorageFileReader::AsyncReader(Box::pin(BudgetedBytesReader { + bytes: body, + offset: 0, + _permit: budget, + })) } BodyRetrievalStrategy::StreamWithoutCache => byte_stream_to_reader(response.body), }; + drop(cache_load_guard); let result = StorageFile::File { meta, content }; Ok(Some(result)) @@ -1731,7 +3560,9 @@ impl Storage for S3Storage { skip(self) )] async fn delete_repository(&self, repository: uuid::Uuid) -> Result<(), S3StorageError> { + self.remove_staging_prefix(&repository).await; let prefix = format!("{repository}/"); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; let mut continuation: Option = None; loop { @@ -1746,10 +3577,13 @@ impl Storage for S3Storage { request = request.continuation_token(token); } - let response = request - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(S3StorageError::aws_message( + "S3 repository deletion exceeded its control deadline", + )); + } + let response = with_timeout(remaining, request.send()).await?; let paths: Vec = response .contents() @@ -1776,6 +3610,7 @@ impl Storage for S3Storage { break; } + self.invalidate_manifest_cache(repository); Ok(()) } @@ -1818,30 +3653,23 @@ impl Storage for S3Storage { format!("{base_prefix}/") }; - let list = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(prefix.clone()) - .delimiter("/") - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + let (objects, prefixes) = self.list_directory_entries(&prefix).await?; + let observed_at = Local::now().fixed_offset(); // Convert objects to StorageFileMeta entries using names relative to the requested // directory (not repository root) so callers can safely append child segments. let mut entries: Vec> = Vec::new(); - for obj in list.contents() { - let Some(key) = obj.key() else { continue }; + for obj in objects { + let key = obj.key; if key == prefix { // Directory placeholder object continue; } - if S3StorageInner::is_hidden_file(key) { + if S3StorageInner::is_hidden_file(&key) { continue; } - let full_name = S3StorageInner::strip_repository_prefix(&repository, key); + let full_name = S3StorageInner::strip_repository_prefix(&repository, &key); let Some(relative_name) = full_name.strip_prefix(&base_prefix) else { continue; }; @@ -1849,26 +3677,21 @@ impl Storage for S3Storage { continue; } - let size: u64 = obj.size().unwrap_or(0i64).max(0) as u64; let meta = StorageFileMeta:: { name: relative_name.to_string(), file_type: FileType::File(FileFileType { - file_size: size, + file_size: obj.size, mime_type: None, file_hash: FileHashes::default(), }), - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified: obj.last_modified.unwrap_or(observed_at), + created: obj.last_modified.unwrap_or(observed_at), }; entries.push(meta); } - for prefix_entry in list - .common_prefixes() - .iter() - .filter_map(CommonPrefix::prefix) - { - let full_name = S3StorageInner::strip_repository_prefix(&repository, prefix_entry); + for prefix_entry in prefixes { + let full_name = S3StorageInner::strip_repository_prefix(&repository, &prefix_entry); let Some(relative_name) = full_name.strip_prefix(&base_prefix) else { continue; }; @@ -1884,8 +3707,8 @@ impl Storage for S3Storage { let meta = StorageFileMeta:: { name: cleaned.to_string(), file_type: FileType::Directory(DirectoryFileType { file_count: 0 }), - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified: observed_at, + created: observed_at, }; entries.push(meta); } @@ -1907,6 +3730,88 @@ impl Storage for S3Storage { } impl S3Storage { + /// Drops a cached object so maintenance reads observe the backing S3 object. + pub async fn invalidate_cached_file( + &self, + repository: Uuid, + location: &StoragePath, + ) -> Result<(), S3StorageError> { + self.cache_remove(&repository, location).await + } + + /// Returns the shared capacity budget for spooled incoming upload chunk files. Upload + /// handlers reserve permits (1 MiB each, see `S3_UPLOAD_SPOOL_PERMIT_BYTES`) before + /// writing spool bytes so in-flight uploads stay within the staged temporary-storage + /// budget; the reservation is held until the spooled bytes have been consumed. + pub fn upload_spool_budget(&self) -> Arc { + Arc::clone(&self.upload_spool_budget) + } + + /// Appends a Docker upload chunk to local staging. The staged bytes are uploaded only when + /// the upload is finalized through `move_file`. + pub async fn append_file_staged( + &self, + repository: Uuid, + file: FileContent, + location: &StoragePath, + ) -> Result { + let path = self.get_path_for_creation(repository, location).await?; + // Serialize with finalization and cleanup for this upload only; unrelated uploads + // keep making progress while this append runs. + let operation_lock = self.staging_operation_lock(&path); + let _operation = operation_lock.lock().await; + self.note_object_created(&path); + let (staging_path, current_size, _) = self.ensure_staging(&path).await?; + let mut staging_file = fs::OpenOptions::new() + .append(true) + .open(&staging_path) + .await?; + let incoming_size = match &file { + FileContent::Path(file_path) => fs::metadata(file_path).await?.len(), + FileContent::Content(content) => content.len() as u64, + FileContent::Bytes(bytes) => bytes.len() as u64, + }; + // Reserve shared capacity before writing: staged bytes across every upload plus + // outstanding spool reservations must stay within MAX_STAGED_BYTES. The reservation + // and the size bump happen atomically so concurrent appends cannot collectively + // exceed the budget; a failed write rolls the reservation back. + let Some(reserved_units) = self.reserve_staged_size(&path, incoming_size) else { + return Err(S3StorageError::aws_message( + "S3 append staging byte capacity exceeded", + )); + }; + let appended_size = match file { + FileContent::Path(file_path) => match fs::File::open(file_path).await { + Ok(mut source) => tokio::io::copy(&mut source, &mut staging_file) + .await + .map(|size| size as usize), + Err(error) => Err(error), + }, + FileContent::Content(content) => staging_file + .write_all(&content) + .await + .map(|()| content.len()), + FileContent::Bytes(bytes) => staging_file.write_all(&bytes).await.map(|()| bytes.len()), + }; + let appended_size = match appended_size { + Ok(size) => size, + Err(error) => { + let _ = staging_file.set_len(current_size).await; + self.release_staged_size(&path, incoming_size, reserved_units); + return Err(error.into()); + } + }; + if let Err(error) = staging_file.flush().await { + let _ = staging_file.set_len(current_size).await; + self.release_staged_size(&path, incoming_size, reserved_units); + return Err(error.into()); + } + self.note_object_created(&path); + self.cache_remove(&repository, location).await?; + self.invalidate_manifest_cache(repository); + Ok(appended_size) + } + /// List all objects for a repository under an optional prefix, returning repository-relative /// keys. Uses S3's paginator to minimize the number of API calls while avoiding per-directory /// traversal. @@ -1934,12 +3839,11 @@ impl S3Storage { .max_keys(1000) .into_paginator() .send(); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; let mut objects = Vec::new(); - while let Some(page) = paginator.next().await { - let page = page.map_err(S3StorageError::from_sdk_error)?; - + while let Some(page) = next_control_page(deadline, paginator.next()).await? { for obj in page.contents() { let Some(key) = obj.key() else { continue }; if S3StorageInner::is_hidden_file(key) { @@ -1954,7 +3858,7 @@ impl S3Storage { } let size = obj.size().unwrap_or(0i64).max(0) as u64; - let last_modified = None; + let last_modified = s3_last_modified(obj.last_modified()); objects.push(S3ListedObject { key: repo_relative.to_string(), @@ -1975,11 +3879,12 @@ impl S3Storage { fields(storage_type = "s3", ?repository), skip(self) )] - pub async fn list_docker_manifests( + async fn load_docker_manifests( &self, repository: Uuid, ) -> Result, S3StorageError> { let mut manifests = Vec::new(); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; let mut queue = VecDeque::new(); queue.push_back(format!("{}/v2/", repository)); @@ -1995,9 +3900,7 @@ impl S3Storage { .into_paginator() .send(); - while let Some(page) = paginator.next().await { - let page = page.map_err(S3StorageError::from_sdk_error)?; - + while let Some(page) = next_control_page(deadline, paginator.next()).await? { for p in page .common_prefixes() .iter() @@ -2019,8 +3922,9 @@ impl S3Storage { .into_paginator() .send(); - while let Some(mpage) = manifest_pages.next().await { - let mpage = mpage.map_err(S3StorageError::from_sdk_error)?; + while let Some(mpage) = + next_control_page(deadline, manifest_pages.next()).await? + { for obj in mpage.contents() { let Some(key) = obj.key() else { continue }; if S3StorageInner::is_hidden_file(key) { @@ -2036,7 +3940,7 @@ impl S3Storage { manifests.push(S3ListedObject { key: repo_relative.to_string(), size, - last_modified: None, + last_modified: s3_last_modified(obj.last_modified()), }); } } @@ -2051,6 +3955,36 @@ impl S3Storage { Ok(manifests) } + /// List Docker manifests using a short-lived per-repository index. + pub async fn list_docker_manifests( + &self, + repository: Uuid, + ) -> Result, S3StorageError> { + let cached = self.manifest_cache.lock().get(repository, Instant::now()); + if let Some(cached) = cached { + return Ok(cached); + } + // Coordinate cold loads per repository instead of globally so a slow repository listing + // does not serialize listings of unrelated repositories. + let load_lock = self.manifest_load_lock(repository).await; + let _load_guard = load_lock.lock().await; + let generation = { + let mut cache = self.manifest_cache.lock(); + if let Some(cached) = cache.get(repository, Instant::now()) { + return Ok(cached); + } + cache.generation(repository) + }; + let manifests = self.load_docker_manifests(repository).await?; + self.manifest_cache.lock().insert_if_generation( + repository, + manifests.clone(), + Instant::now(), + generation, + ); + Ok(manifests) + } + /// Paginate Docker manifest objects without loading the entire repository into memory. /// /// Returns the requested page of manifest objects (ordered lexicographically by key) @@ -2066,87 +4000,40 @@ impl S3Storage { start: usize, limit: usize, ) -> Result<(Vec, usize), S3StorageError> { - let mut items = Vec::with_capacity(limit); + let load_lock = self.manifest_load_lock(repository).await; + let _load_guard = load_lock.lock().await; + let prefix = format!("{repository}/v2/"); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; + let mut paginator = self + .aws_client() + .list_objects_v2() + .bucket(self.bucket()) + .prefix(prefix) + .max_keys(1000) + .into_paginator() + .send(); let mut total = 0usize; - - // Breadth-first traversal that skips heavy prefixes (`blobs`, uploads) while preserving - // lexicographic order of manifest keys. - let mut queue = VecDeque::new(); - queue.push_back(format!("{}/v2/", repository)); - - while let Some(prefix) = queue.pop_front() { - let mut paginator = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(prefix.clone()) - .delimiter("/") - .max_keys(1000) - .into_paginator() - .send(); - - while let Some(page) = paginator.next().await { - let page = page.map_err(S3StorageError::from_sdk_error)?; - - // Descend into sub-prefixes (directories) - for p in page - .common_prefixes() - .iter() - .filter_map(CommonPrefix::prefix) + let mut items = Vec::with_capacity(limit); + while let Some(page) = next_control_page(deadline, paginator.next()).await? { + for object in page.contents() { + let Some(key) = object.key() else { continue }; + let relative = S3StorageInner::strip_repository_prefix(&repository, key); + if S3StorageInner::is_hidden_file(key) + || !relative.contains("/manifests/") + || relative.ends_with(".nr-docker-tagmeta") { - if p.ends_with("blobs/") || p.ends_with("uploads/") || p.ends_with("_uploads/") - { - continue; - } - - if p.ends_with("manifests/") { - // List manifest objects directly under this prefix (no delimiter) - let mut manifest_pages = self - .aws_client() - .list_objects_v2() - .bucket(self.bucket()) - .prefix(p) - .max_keys(1000) - .into_paginator() - .send(); - - while let Some(mpage) = manifest_pages.next().await { - let mpage = mpage.map_err(S3StorageError::from_sdk_error)?; - for obj in mpage.contents() { - let Some(key) = obj.key() else { continue }; - if S3StorageInner::is_hidden_file(key) { - continue; - } - - total += 1; - if total <= start { - continue; - } - if items.len() >= limit { - continue; - } - - let repo_relative = - S3StorageInner::strip_repository_prefix(&repository, key); - if repo_relative.is_empty() { - continue; - } - - let size = obj.size().unwrap_or(0i64).max(0) as u64; - items.push(S3ListedObject { - key: repo_relative.to_string(), - size, - last_modified: None, - }); - } - } - } else { - queue.push_back(p.to_string()); - } + continue; + } + if total >= start && items.len() < limit { + items.push(S3ListedObject { + key: relative.to_string(), + size: object.size().unwrap_or_default().max(0) as u64, + last_modified: s3_last_modified(object.last_modified()), + }); } + total = total.saturating_add(1); } } - Ok((items, total)) } @@ -2154,6 +4041,7 @@ impl S3Storage { &self, repository: Uuid, location: &StoragePath, + modified: Option>, ) -> Result, S3StorageError> { let Some(stream) = self.stream_directory(repository, location).await? else { return Ok(None); @@ -2162,15 +4050,16 @@ impl S3Storage { let file_count = stream.number_of_files(); let files = collect_directory_stream(stream) .await - .map_err(|err| S3StorageError::AwsSdkError(err.to_string()))?; + .map_err(|err| S3StorageError::aws_message(err.to_string()))?; + let observed_at = modified.unwrap_or_else(|| Local::now().fixed_offset()); let meta = StorageFileMeta:: { name: location.to_string(), file_type: DirectoryFileType { file_count: file_count.max(files.len() as u64), }, - modified: Local::now().fixed_offset(), - created: Local::now().fixed_offset(), + modified: observed_at, + created: observed_at, }; Ok(Some(StorageFile::Directory { meta, files })) @@ -2194,6 +4083,9 @@ impl S3Storage { if paths.is_empty() { return Ok(0); } + for path in paths { + self.remove_staging(&self.s3_path(&repository, path)).await; + } use aws_sdk_s3::types::ObjectIdentifier; @@ -2209,7 +4101,7 @@ impl S3Storage { let obj_id = ObjectIdentifier::builder() .key(key) .build() - .map_err(|err| S3StorageError::AwsSdkError(err.to_string()))?; + .map_err(|err| S3StorageError::aws_message(err.to_string()))?; object_ids.push(obj_id); } @@ -2217,23 +4109,28 @@ impl S3Storage { continue; } - let response = self - .aws_client() - .delete_objects() - .bucket(self.bucket()) - .delete( - aws_sdk_s3::types::Delete::builder() - .set_objects(Some(object_ids)) - .quiet(true) // Don't return deleted objects in response - .build() - .map_err(|err| S3StorageError::AwsSdkError(err.to_string()))?, - ) - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + let response = with_timeout( + S3_CONTROL_TIMEOUT, + self.aws_client() + .delete_objects() + .bucket(self.bucket()) + .delete( + aws_sdk_s3::types::Delete::builder() + .set_objects(Some(object_ids)) + .quiet(true) // Don't return deleted objects in response + .build() + .map_err(|err| S3StorageError::aws_message(err.to_string()))?, + ) + .send(), + ) + .await?; // Count successful deletions (errors() returns objects that failed) let failed = response.errors(); + for path in chunk { + self.cache_remove(&repository, path).await?; + } + self.invalidate_manifest_cache(repository); if !failed.is_empty() { let (code, message, key) = failed .first() @@ -2255,17 +4152,12 @@ impl S3Storage { keys_sample = ?keys_for_log.get(0..5).map(|v| v.to_vec()), "S3 delete_objects reported errors" ); - return Err(S3StorageError::AwsSdkError(format!( + return Err(S3StorageError::aws_message(format!( "delete_objects failed for key {key}: {code} - {message}" ))); } deleted_count += chunk.len(); - - // Remove from cache - for path in chunk { - self.cache_remove(&repository, path).await?; - } } debug!( @@ -2282,6 +4174,7 @@ impl S3Storage { /// Skips internal Pkgly metadata objects. pub async fn repository_size_bytes(&self, repository: Uuid) -> Result { let prefix = format!("{repository}/"); + let deadline = Instant::now() + S3_CONTROL_TIMEOUT; let mut continuation: Option = None; let mut total: u64 = 0; @@ -2296,10 +4189,13 @@ impl S3Storage { request = request.continuation_token(token); } - let response = request - .send() - .await - .map_err(S3StorageError::from_sdk_error)?; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(S3StorageError::aws_message( + "S3 repository size calculation exceeded its control deadline", + )); + } + let response = with_timeout(remaining, request.send()).await?; for obj in response.contents() { if let Some(key) = obj.key() { @@ -2350,11 +4246,25 @@ impl StaticStorageFactory for S3StorageFactory { ) -> Result { let client = S3StorageInner::load_client(&type_config).await?; let cache = S3StorageInner::build_cache(&type_config, &inner).await?; + let staging_dir = S3StorageInner::prepare_staging_dir(&inner.storage_name).await?; let inner = S3StorageInner { config: type_config, storage_config: inner, client, cache, + cache_load_locks: ParkingMutex::new(HashMap::default()), + manifest_cache: ParkingMutex::new(ManifestCache::new()), + manifest_load_locks: ParkingMutex::new(HashMap::default()), + append_staging: ParkingMutex::new(HashMap::default()), + append_operation_locks: ParkingMutex::new(HashMap::default()), + upload_spool_budget: Arc::new(Semaphore::new( + (MAX_STAGED_BYTES / S3_UPLOAD_SPOOL_PERMIT_BYTES) as usize, + )), + staging_dir, + creation_probes: ParkingMutex::new(LruCache::new( + NonZeroUsize::new(CREATION_PROBE_CACHE_CAPACITY).unwrap_or(NonZeroUsize::MIN), + )), + creation_probe_generation: ParkingMutex::new(0), }; let storage = S3Storage::from(inner); Ok(storage) @@ -2388,11 +4298,26 @@ impl StorageFactory for S3StorageFactory { let storage_config = config.storage_config; let client = S3StorageInner::load_client(&s3_config).await?; let cache = S3StorageInner::build_cache(&s3_config, &storage_config).await?; + let staging_dir = + S3StorageInner::prepare_staging_dir(&storage_config.storage_name).await?; let inner = S3StorageInner { config: s3_config, storage_config, client, cache, + cache_load_locks: ParkingMutex::new(HashMap::default()), + manifest_cache: ParkingMutex::new(ManifestCache::new()), + manifest_load_locks: ParkingMutex::new(HashMap::default()), + append_staging: ParkingMutex::new(HashMap::default()), + append_operation_locks: ParkingMutex::new(HashMap::default()), + upload_spool_budget: Arc::new(Semaphore::new( + (MAX_STAGED_BYTES / S3_UPLOAD_SPOOL_PERMIT_BYTES) as usize, + )), + staging_dir, + creation_probes: ParkingMutex::new(LruCache::new( + NonZeroUsize::new(CREATION_PROBE_CACHE_CAPACITY).unwrap_or(NonZeroUsize::MIN), + )), + creation_probe_generation: ParkingMutex::new(0), }; let storage = S3Storage::from(inner); Ok(DynStorage::S3(storage)) diff --git a/crates/storage/src/s3/regions.rs b/crates/storage/src/s3/regions.rs index c96352a..0d261b2 100644 --- a/crates/storage/src/s3/regions.rs +++ b/crates/storage/src/s3/regions.rs @@ -1,98 +1,65 @@ -use aws_types::region::Region; +// ABOUTME: Defines S3 region suggestions and custom endpoint configuration. +// ABOUTME: Region values remain raw strings so new providers need no code release. use serde::{Deserialize, Serialize}; -use std::fmt::{Display, Formatter}; -use strum::EnumIter; use url::Url; use utoipa::ToSchema; -#[derive(Clone, Debug, Eq, Copy, PartialEq, Serialize, Deserialize, ToSchema, EnumIter)] -pub enum S3StorageRegion { - /// us-east-1 - UsEast1, - /// us-east-2 - UsEast2, - /// us-west-1 - UsWest1, - /// us-west-2 - UsWest2, - /// ca-central-1 - CaCentral1, - /// af-south-1 - AfSouth1, - /// ap-east-1 - ApEast1, - /// ap-south-1 - ApSouth1, - /// ap-northeast-1 - ApNortheast1, - /// ap-northeast-2 - ApNortheast2, - /// ap-northeast-3 - ApNortheast3, - /// ap-southeast-1 - ApSoutheast1, - /// ap-southeast-2 - ApSoutheast2, - /// cn-north-1 - CnNorth1, - /// cn-northwest-1 - CnNorthwest1, - /// eu-north-1 - EuNorth1, - /// eu-central-1 - EuCentral1, - /// eu-central-2 - EuCentral2, - /// eu-west-1 - EuWest1, - /// eu-west-2 - EuWest2, - /// eu-west-3 - EuWest3, - /// il-central-1 - IlCentral1, - /// me-south-1 - MeSouth1, - /// sa-east-1 - SaEast1, -} -impl Display for S3StorageRegion { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let value = match self { - S3StorageRegion::UsEast1 => "us-east-1", - S3StorageRegion::UsEast2 => "us-east-2", - S3StorageRegion::UsWest1 => "us-west-1", - S3StorageRegion::UsWest2 => "us-west-2", - S3StorageRegion::CaCentral1 => "ca-central-1", - S3StorageRegion::AfSouth1 => "af-south-1", - S3StorageRegion::ApEast1 => "ap-east-1", - S3StorageRegion::ApSouth1 => "ap-south-1", - S3StorageRegion::ApNortheast1 => "ap-northeast-1", - S3StorageRegion::ApNortheast2 => "ap-northeast-2", - S3StorageRegion::ApNortheast3 => "ap-northeast-3", - S3StorageRegion::ApSoutheast1 => "ap-southeast-1", - S3StorageRegion::ApSoutheast2 => "ap-southeast-2", - S3StorageRegion::CnNorth1 => "cn-north-1", - S3StorageRegion::CnNorthwest1 => "cn-northwest-1", - S3StorageRegion::EuNorth1 => "eu-north-1", - S3StorageRegion::EuCentral1 => "eu-central-1", - S3StorageRegion::EuCentral2 => "eu-central-2", - S3StorageRegion::EuWest1 => "eu-west-1", - S3StorageRegion::EuWest2 => "eu-west-2", - S3StorageRegion::EuWest3 => "eu-west-3", - S3StorageRegion::IlCentral1 => "il-central-1", - S3StorageRegion::MeSouth1 => "me-south-1", - S3StorageRegion::SaEast1 => "sa-east-1", - }; - f.write_str(value) - } -} +/// Canonical region identifiers offered as UI suggestions. +/// +/// The S3 configuration accepts any non-empty region string; this list is only a convenience for +/// commonly used AWS partitions and does not constrain configuration or deserialization. +pub const KNOWN_S3_REGIONS: &[&str] = &[ + "af-south-1", + "ap-east-1", + "ap-east-2", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-south-2", + "ap-southeast-1", + "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", + "ap-southeast-5", + "ap-southeast-6", + "ap-southeast-7", + "ca-central-1", + "ca-west-1", + "cn-north-1", + "cn-northwest-1", + "eu-central-1", + "eu-central-2", + "eu-central-3", + "eu-north-1", + "eu-south-1", + "eu-south-2", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "eusc-de-east-1", + "eusc-de-east-2", + "il-central-1", + "me-central-1", + "me-south-1", + "mx-central-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-gov-east-1", + "us-gov-west-1", + "us-iso-east-1", + "us-iso-west-1", + "us-isob-east-1", + "us-isob-west-1", + "us-isof-east-1", + "us-isof-south-1", + "us-isof-south-2", + "us-west-1", + "us-west-2", +]; -impl From for Region { - fn from(value: S3StorageRegion) -> Self { - Region::new(value.to_string()) - } -} +/// An optional custom endpoint and the region label used to sign requests to it. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, ToSchema)] pub struct CustomRegion { pub custom_region: Option, diff --git a/crates/storage/src/s3/tags.rs b/crates/storage/src/s3/tags.rs deleted file mode 100644 index d150c95..0000000 --- a/crates/storage/src/s3/tags.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub static NAME: &str = "NR-NAME"; -pub static MIME_TYPE: &str = "NR-MIME-TYPE"; diff --git a/crates/storage/src/s3/tests.rs b/crates/storage/src/s3/tests.rs index afab828..288a8df 100644 --- a/crates/storage/src/s3/tests.rs +++ b/crates/storage/src/s3/tests.rs @@ -1,12 +1,28 @@ +// ABOUTME: Exercises S3 configuration, caching, and storage behavior. +// ABOUTME: Provides a deterministic HTTP S3 test service for protocol-level tests. #![allow(clippy::expect_used, clippy::panic, clippy::todo, clippy::unwrap_used)] use super::{ AdaptiveBufferConfig, BodyRetrievalStrategy, CustomRegion, DEFAULT_MAX_BUFFERED_OBJECT_BYTES, - S3CacheConfig, S3Config, S3Credentials, S3DiskCache, S3StorageRegion, + MemorySnapshot, MemorySnapshotCache, S3CacheConfig, S3Config, S3Credentials, S3DiskCache, + default_cache_dir, resolve_cache_dir, +}; +use aws_smithy_runtime_api::{ + client::{ + orchestrator::HttpResponse as SmithyHttpResponse, result::SdkError as SmithySdkError, + }, + http::StatusCode as SmithyStatusCode, +}; +use aws_smithy_types::{ + body::SdkBody, + error::{ErrorMetadata, metadata::ProvideErrorMetadata}, }; use bytes::Bytes; +use chrono::{FixedOffset, TimeZone}; use tempfile::tempdir; use tokio::{ fs, + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, time::{Duration, sleep}, }; @@ -49,7 +65,7 @@ fn role_detection_prefers_non_empty_strings() { fn custom_region_returns_endpoint_and_name() { let config = S3Config { bucket_name: "pkgly".into(), - region: Some(S3StorageRegion::UsEast1), + region: Some("us-east-1".into()), custom_region: Some(CustomRegion { custom_region: Some("minio".into()), endpoint: "https://minio.local".parse().unwrap(), @@ -67,6 +83,205 @@ fn custom_region_returns_endpoint_and_name() { assert!(config.custom_endpoint().is_some()); } +#[test] +fn raw_region_values_are_passed_through() { + let config = S3Config { + bucket_name: "pkgly".into(), + region: Some("eu-central-99".into()), + custom_region: None, + credentials: S3Credentials::default(), + path_style: true, + cache: S3CacheConfig::default(), + adaptive_buffer: AdaptiveBufferConfig::default(), + }; + + assert_eq!( + config.resolved_region().expect("region").as_ref(), + "eu-central-99" + ); +} + +#[test] +fn copy_source_encodes_bucket_and_key() { + assert_eq!( + super::copy_source("bucket", "folder/file name+one"), + "/bucket/folder%2Ffile%20name%2Bone" + ); +} + +#[test] +fn sdk_error_messages_keep_actionable_kinds() { + assert_eq!( + super::S3StorageError::aws_message("PreconditionFailed: status code: 412").kind(), + Some(super::S3ErrorKind::Conflict) + ); + assert_eq!( + super::S3StorageError::aws_message("AccessDenied: status code: 403").kind(), + Some(super::S3ErrorKind::AccessDenied) + ); + assert_eq!( + super::S3StorageError::aws_message("NoSuchBucket").kind(), + Some(super::S3ErrorKind::NotFound) + ); + assert_eq!( + super::S3StorageError::aws_message("HTTP status 404").kind(), + Some(super::S3ErrorKind::NotFound) + ); + assert!(super::S3StorageError::aws_message("SlowDown: status code: 429").is_retryable()); + assert!(super::S3StorageError::aws_message("status code: 503").is_retryable()); + assert!(super::S3StorageError::aws_message("dispatch failure").is_retryable()); + assert!(super::S3StorageError::aws_message("operation deadline exceeded").is_retryable()); +} + +#[derive(Debug)] +struct MetadataOnlyError(ErrorMetadata); + +impl std::fmt::Display for MetadataOnlyError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("opaque service failure") + } +} + +impl std::error::Error for MetadataOnlyError {} + +impl ProvideErrorMetadata for MetadataOnlyError { + fn meta(&self) -> &ErrorMetadata { + &self.0 + } +} + +fn smithy_response(status: u16) -> SmithyHttpResponse { + SmithyHttpResponse::new( + SmithyStatusCode::try_from(status).expect("valid status"), + SdkBody::empty(), + ) +} + +#[test] +fn sdk_error_classification_uses_smithy_metadata_and_status() { + let denied = SmithySdkError::service_error( + MetadataOnlyError(ErrorMetadata::builder().code("AccessDenied").build()), + smithy_response(500), + ); + assert_eq!( + super::S3StorageError::from_sdk_error(denied).kind(), + Some(super::S3ErrorKind::AccessDenied) + ); + + let conflict = SmithySdkError::service_error( + MetadataOnlyError(ErrorMetadata::builder().code("OpaqueCode").build()), + smithy_response(412), + ); + assert_eq!( + super::S3StorageError::from_sdk_error(conflict).kind(), + Some(super::S3ErrorKind::Conflict) + ); +} + +#[test] +fn timeout_profiles_define_attempt_and_operation_deadlines() { + let control = super::control_timeout_config(); + assert_eq!( + control.operation_attempt_timeout(), + Some(Duration::from_secs(30)) + ); + assert_eq!(control.operation_timeout(), Some(Duration::from_secs(90))); + + let copy = super::copy_timeout_config(); + assert_eq!( + copy.operation_attempt_timeout(), + Some(Duration::from_secs(5 * 60)) + ); + assert_eq!(copy.operation_timeout(), Some(Duration::from_secs(20 * 60))); + + let streaming = super::streaming_timeout_config(); + assert_eq!(streaming.operation_attempt_timeout(), None); + assert_eq!(streaming.operation_timeout(), None); + assert_eq!(streaming.connect_timeout(), Some(Duration::from_secs(5))); + assert_eq!(streaming.read_timeout(), Some(Duration::from_secs(30))); +} + +#[test] +fn legacy_region_enum_values_deserialize_to_raw_ids() { + let config: S3Config = serde_json::from_value(serde_json::json!({ + "bucket_name": "pkgly", + "region": "UsEast1", + "credentials": {}, + })) + .expect("legacy region should remain readable"); + + assert_eq!(config.region.as_deref(), Some("us-east-1")); +} + +#[test] +fn every_legacy_region_token_is_migrated_by_deserializer() { + let mappings = [ + ("UsEast1", "us-east-1"), + ("UsEast2", "us-east-2"), + ("UsWest1", "us-west-1"), + ("UsWest2", "us-west-2"), + ("CaCentral1", "ca-central-1"), + ("AfSouth1", "af-south-1"), + ("ApEast1", "ap-east-1"), + ("ApSouth1", "ap-south-1"), + ("ApNortheast1", "ap-northeast-1"), + ("ApNortheast2", "ap-northeast-2"), + ("ApNortheast3", "ap-northeast-3"), + ("ApSoutheast1", "ap-southeast-1"), + ("ApSoutheast2", "ap-southeast-2"), + ("CnNorth1", "cn-north-1"), + ("CnNorthwest1", "cn-northwest-1"), + ("EuNorth1", "eu-north-1"), + ("EuCentral1", "eu-central-1"), + ("EuCentral2", "eu-central-2"), + ("EuWest1", "eu-west-1"), + ("EuWest2", "eu-west-2"), + ("EuWest3", "eu-west-3"), + ("IlCentral1", "il-central-1"), + ("MeSouth1", "me-south-1"), + ("SaEast1", "sa-east-1"), + ]; + for (legacy, expected) in mappings { + let config: S3Config = serde_json::from_value(serde_json::json!({ + "bucket_name": "pkgly", + "region": legacy, + "credentials": {}, + })) + .expect("legacy region should deserialize"); + assert_eq!(config.region.as_deref(), Some(expected)); + } +} + +#[test] +fn unknown_region_round_trips_as_raw_string() { + let config: S3Config = serde_json::from_value(serde_json::json!({ + "bucket_name": "pkgly", + "region": "provider-special-1", + "credentials": {}, + })) + .expect("raw region should deserialize"); + let serialized = serde_json::to_value(config).expect("config should serialize"); + assert_eq!(serialized["region"], "provider-special-1"); +} + +#[test] +fn blank_region_is_rejected_without_custom_endpoint() { + let config = S3Config { + bucket_name: "pkgly".into(), + region: Some(" ".into()), + custom_region: None, + credentials: S3Credentials::default(), + path_style: true, + cache: S3CacheConfig::default(), + adaptive_buffer: AdaptiveBufferConfig::default(), + }; + + assert!(matches!( + config.resolved_region(), + Err(super::S3StorageError::NoRegionSpecified) + )); +} + #[test] fn body_strategy_caches_small_objects() { let limit = DEFAULT_MAX_BUFFERED_OBJECT_BYTES; @@ -103,6 +318,21 @@ fn cache_config_with_dir(dir: &std::path::Path) -> S3CacheConfig { } } +#[test] +fn empty_cache_path_uses_storage_default() { + let config = S3CacheConfig { + enabled: true, + path: Some(std::path::PathBuf::new()), + max_bytes: 8, + max_entries: 4, + }; + + assert_eq!( + resolve_cache_dir(&config, "blank-cache-path"), + default_cache_dir("blank-cache-path") + ); +} + #[tokio::test] async fn disk_cache_retries_failed_deletions_on_next_put() { let temp_dir = tempdir().expect("tempdir"); @@ -115,7 +345,15 @@ async fn disk_cache_retries_failed_deletions_on_next_put() { .await .expect("initial write"); - let relative = S3DiskCache::hashed_filename("first"); + let relative = { + let state = cache.state.lock().await; + state + .entries + .peek("first") + .expect("first cache entry") + .relative_path + .clone() + }; let disk_path = cache.dir.join(&relative); fs::remove_file(&disk_path) .await @@ -150,407 +388,2716 @@ async fn disk_cache_retries_failed_deletions_on_next_put() { assert!(!exists, "failed deletions get retried before new puts"); } -#[test] -fn adaptive_buffer_respects_pressure_threshold() { - let config = AdaptiveBufferConfig { - min_buffer_bytes: 1024 * 1024, - max_buffer_bytes: 16 * 1024 * 1024, - memory_pressure_threshold: 0.5, +#[tokio::test] +async fn disk_cache_recovers_entries_and_preserves_unrelated_files() { + let temp_dir = tempdir().expect("tempdir"); + let sentinel = temp_dir.path().join("sentinel.txt"); + fs::write(&sentinel, b"leave me alone") + .await + .expect("sentinel"); + + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put( + "recover", + Bytes::from_static(b"persisted"), + Some("text/plain"), + ) + .await + .expect("write cache entry"); + drop(cache); - assert_eq!(config.limit_for_pressure(0.0), 16 * 1024 * 1024); - let mid = config.limit_for_pressure(0.25); - assert!(mid < 16 * 1024 * 1024 && mid > 1024 * 1024); - assert_eq!(config.limit_for_pressure(0.5), 1024 * 1024); - assert_eq!(config.limit_for_pressure(0.9), 1024 * 1024); + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("recovered cache"); + let object = recovered + .get("recover") + .await + .expect("cache read") + .expect("recovered object"); + let mut content = Vec::new(); + let mut file = object.file; + tokio::io::AsyncReadExt::read_to_end(&mut file, &mut content) + .await + .expect("cache content"); + assert_eq!(content, b"persisted"); + assert_eq!(object.content_type.as_deref(), Some("text/plain")); + assert_eq!( + fs::read(&sentinel).await.expect("sentinel read"), + b"leave me alone" + ); } -// --------------------------------------------------------------------------- -// delete_repository integration tests against a lightweight mock S3 endpoint. -// --------------------------------------------------------------------------- -use super::{S3Storage, S3StorageInner}; -use crate::{Storage, StorageConfigInner}; -use aws_config::BehaviorVersion; -use aws_credential_types::{Credentials as AwsCredentials, provider::SharedCredentialsProvider}; -use aws_sdk_s3::config::Builder as S3ConfigBuilder; -use aws_types::region::Region; -use bytes::Buf; -use http_body_util::{BodyExt, Full}; -use hyper::{ - Request, Response, StatusCode, body::Incoming, server::conn::http1, service::service_fn, -}; -use hyper_util::rt::TokioIo; -use parking_lot::Mutex; -use std::{collections::VecDeque, convert::Infallible, net::SocketAddr, sync::Arc}; -use tokio::net::TcpListener; -use uuid::Uuid; - -type RecordedBody = bytes::Bytes; -type Responder = - Box) -> Response> + Send + 'static>; +#[tokio::test] +async fn disk_cache_recovers_latest_same_key_publication() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("replaced", Bytes::from_static(b"old"), Some("text/plain")) + .await + .expect("initial cache entry"); + cache + .put( + "replaced", + Bytes::from_static(b"latest"), + Some("text/plain"), + ) + .await + .expect("replacement cache entry"); + drop(cache); -#[derive(Clone, Debug)] -struct RecordedRequest { - method: hyper::Method, - uri: hyper::Uri, - body: RecordedBody, + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("recovered cache"); + let object = recovered + .get("replaced") + .await + .expect("cache read") + .expect("latest entry should recover"); + let mut content = Vec::new(); + let mut file = object.file; + tokio::io::AsyncReadExt::read_to_end(&mut file, &mut content) + .await + .expect("cache content"); + assert_eq!(content, b"latest"); } -struct MockS3Server { - address: SocketAddr, - requests: Arc>>, - shutdown: tokio::sync::oneshot::Sender<()>, - task: tokio::task::JoinHandle<()>, -} +#[tokio::test] +async fn disk_cache_recovery_preserves_object_timestamp() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + let timestamp = FixedOffset::east_opt(0) + .unwrap() + .with_ymd_and_hms(2025, 1, 1, 0, 0, 0) + .unwrap(); + cache + .put_with_metadata( + "timestamp", + Bytes::from_static(b"payload"), + Some("text/plain"), + Some(timestamp), + ) + .await + .expect("write cache entry"); + drop(cache); -impl MockS3Server { - async fn start(responders: Vec) -> Self { - let listener = TcpListener::bind(("127.0.0.1", 0)) + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("recovered cache"); + assert_eq!( + recovered + .get("timestamp") .await - .expect("bind mock s3"); - let address = listener.local_addr().expect("address"); - let requests = Arc::new(Mutex::new(Vec::new())); - let responder_queue = Arc::new(Mutex::new(VecDeque::from(responders))); - - let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>(); - let requests_clone = Arc::clone(&requests); - let responders_clone = Arc::clone(&responder_queue); + .expect("cache read") + .expect("entry") + .last_modified, + Some(timestamp) + ); +} - let task = tokio::spawn(async move { - loop { - tokio::select! { - _ = &mut shutdown_rx => break, - accept = listener.accept() => { - let (stream, _) = match accept { - Ok(ok) => ok, - Err(err) => { - eprintln!("mock s3 accept error: {err}"); - continue; - } - }; - let requests = Arc::clone(&requests_clone); - let responders = Arc::clone(&responders_clone); - tokio::spawn(async move { - let service = service_fn(move |req: Request| { - handle_request(req, Arc::clone(&requests), Arc::clone(&responders)) - }); - if let Err(err) = http1::Builder::new() - .serve_connection(TokioIo::new(stream), service) - .await - { - eprintln!("mock s3 connection error: {err}"); - } - }); - } - } - } - }); +#[tokio::test] +async fn disk_cache_discards_missing_or_malformed_sidecars() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("malformed", Bytes::from_static(b"payload"), None) + .await + .expect("malformed entry"); + cache + .put("missing-sidecar", Bytes::from_static(b"payload"), None) + .await + .expect("missing sidecar entry"); + let (malformed_meta, missing_meta) = { + let state = cache.state.lock().await; + let malformed = state.entries.peek("malformed").expect("malformed entry"); + let missing = state + .entries + .peek("missing-sidecar") + .expect("missing entry"); + ( + cache + .dir + .join(S3DiskCache::metadata_filename(&malformed.relative_path)), + cache + .dir + .join(S3DiskCache::metadata_filename(&missing.relative_path)), + ) + }; + fs::write(&malformed_meta, b"not json") + .await + .expect("malformed sidecar"); + fs::remove_file(&missing_meta) + .await + .expect("remove sidecar"); + drop(cache); - Self { - address, - requests, - shutdown: shutdown_tx, - task, - } - } + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("recovered cache"); + assert!( + recovered + .get("malformed") + .await + .expect("malformed read") + .is_none() + ); + assert!( + recovered + .get("missing-sidecar") + .await + .expect("missing read") + .is_none() + ); +} - fn endpoint(&self) -> String { - format!("http://{}", self.address) - } +#[tokio::test] +async fn disk_cache_runtime_same_size_mutation_is_not_rehashed() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = cache_config_with_dir(temp_dir.path()); + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("corrupt", Bytes::from_static(b"original"), None) + .await + .expect("write cache entry"); + + let relative = { + let state = cache.state.lock().await; + state + .entries + .peek("corrupt") + .expect("cache entry") + .relative_path + .clone() + }; + fs::write(cache.dir.join(relative), b"tampered") + .await + .expect("corrupt cache entry"); - fn take_requests(&self) -> Vec { - self.requests.lock().clone() - } + let object = cache + .get("corrupt") + .await + .expect("cache read") + .expect("same-sized content remains indexed"); + let mut content = Vec::new(); + let mut file = object.file; + tokio::io::AsyncReadExt::read_to_end(&mut file, &mut content) + .await + .expect("cache content"); + assert_eq!(content, b"tampered"); +} - async fn shutdown(self) { - let _ = self.shutdown.send(()); - let _ = self.task.await; - } +#[tokio::test] +async fn disk_cache_recovers_generation_and_removes_owned_orphans() { + let temp_dir = tempdir().expect("tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("kept", Bytes::from_static(b"kept"), None) + .await + .expect("write cache entry"); + let content_path = { + let state = cache.state.lock().await; + state + .entries + .peek("kept") + .expect("entry") + .relative_path + .clone() + }; + let parent = temp_dir.path().join(content_path.parent().expect("parent")); + let base = S3DiskCache::hashed_filename("orphan"); + let orphan_generation = temp_dir + .path() + .join(base.parent().expect("base parent")) + .join(format!( + "{}.gen-abcdef", + base.file_name().expect("base file").to_string_lossy() + )); + fs::create_dir_all(orphan_generation.parent().unwrap()) + .await + .expect("orphan parent"); + fs::write(&orphan_generation, b"orphan") + .await + .expect("orphan generation"); + let legacy = temp_dir.path().join("aa").join("0".repeat(62)); + fs::create_dir_all(legacy.parent().unwrap()) + .await + .expect("legacy parent"); + fs::write(&legacy, b"legacy") + .await + .expect("legacy artifact"); + let unrelated = parent.join("notes.txt"); + fs::write(&unrelated, b"keep this") + .await + .expect("unrelated"); + let unrelated_temp = parent.join("notes.tmp-abcdef"); + fs::write(&unrelated_temp, b"keep this too") + .await + .expect("unrelated temp"); + let unrelated_metadata = parent.join("notes.meta.json"); + fs::write(&unrelated_metadata, b"keep this metadata too") + .await + .expect("unrelated metadata"); + drop(cache); + + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("reopen cache"); + assert!(recovered.get("kept").await.expect("cache read").is_some()); + assert!( + !fs::try_exists(&orphan_generation) + .await + .expect("orphan exists") + ); + assert!(!fs::try_exists(&legacy).await.expect("legacy exists")); + assert!(fs::try_exists(&unrelated).await.expect("unrelated exists")); + assert!( + fs::try_exists(&unrelated_temp) + .await + .expect("unrelated temp exists") + ); + assert!( + fs::try_exists(&unrelated_metadata) + .await + .expect("unrelated metadata exists") + ); } -async fn handle_request( - mut req: Request, - requests: Arc>>, - responders: Arc>>, -) -> Result>, Infallible> { - let body_bytes = req.body_mut().collect().await.unwrap().to_bytes(); - let (parts, _) = req.into_parts(); - let req_with_body = Request::from_parts(parts, body_bytes.clone()); +#[cfg(unix)] +#[tokio::test] +async fn disk_cache_rejects_symlinked_content() { + use std::os::unix::fs::symlink; - requests.lock().push(RecordedRequest { - method: req_with_body.method().clone(), - uri: req_with_body.uri().clone(), - body: body_bytes.clone(), - }); + let temp_dir = tempdir().expect("tempdir"); + let outside = tempdir().expect("outside tempdir"); + let cache_config = S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("cache"); + cache + .put("symlink", Bytes::from_static(b"payload"), None) + .await + .expect("write cache entry"); + let relative = { + let state = cache.state.lock().await; + state + .entries + .peek("symlink") + .expect("entry") + .relative_path + .clone() + }; + let content_path = cache.dir.join(relative); + let target = outside.path().join("target"); + fs::write(&target, b"secret").await.expect("target"); + fs::remove_file(&content_path) + .await + .expect("remove content"); + symlink(&target, &content_path).expect("symlink"); + drop(cache); - let mut queue = responders.lock(); - let resp = if let Some(responder) = queue.front_mut() { - responder(req_with_body) - } else { - Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Full::new(bytes::Bytes::from_static( - b"no responder for request", - ))) - .unwrap() + let recovered = S3DiskCache::new(&cache_config, "test-cache") + .await + .expect("reopen cache"); + assert!( + recovered + .get("symlink") + .await + .expect("cache read") + .is_none() + ); +} + +#[tokio::test] +async fn disk_cache_recovery_enforces_entry_capacity() { + let temp_dir = tempdir().expect("tempdir"); + let write_config = S3CacheConfig { + max_entries: 4, + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&write_config, "test-cache") + .await + .expect("cache"); + cache + .put("old", Bytes::from_static(b"old"), None) + .await + .expect("old entry"); + sleep(Duration::from_millis(2)).await; + cache + .put("new", Bytes::from_static(b"new"), None) + .await + .expect("new entry"); + drop(cache); + + let read_config = S3CacheConfig { + max_entries: 1, + ..write_config }; - // pop only after a successful response to preserve strict ordering on errors - queue.pop_front(); - Ok(resp) + let recovered = S3DiskCache::new(&read_config, "test-cache") + .await + .expect("reopen cache"); + assert!(recovered.get("old").await.expect("old read").is_none()); + assert!(recovered.get("new").await.expect("new read").is_some()); } -fn build_s3_storage(endpoint: &str, bucket: &str) -> S3Storage { - let mut storage_config = StorageConfigInner::test_config(); - storage_config.storage_type = "s3".into(); +#[tokio::test] +async fn disk_cache_does_not_insert_oversized_objects() { + let temp_dir = tempdir().expect("tempdir"); + let config = S3CacheConfig { + max_bytes: 3, + ..cache_config_with_dir(temp_dir.path()) + }; + let cache = S3DiskCache::new(&config, "test-cache") + .await + .expect("cache"); + cache + .put("too-large", Bytes::from_static(b"old"), None) + .await + .expect("cache original object"); + cache + .put("too-large", Bytes::from_static(b"four"), None) + .await + .expect("oversized writes are ignored"); + assert!(cache.get("too-large").await.expect("cache read").is_none()); +} - let client_config = S3ConfigBuilder::new() - .region(Region::new("us-east-1")) - .behavior_version(BehaviorVersion::latest()) - .force_path_style(true) - .endpoint_url(endpoint) - .credentials_provider(SharedCredentialsProvider::new(AwsCredentials::new( - "AKIA", "SECRET", None, None, "mock", - ))) - .build(); +#[tokio::test] +#[ignore = "requires a real MinIO endpoint in PKGLY_TEST_MINIO_ENDPOINT"] +async fn oversized_s3_mutations_invalidate_cached_content() { + let endpoint = std::env::var("PKGLY_TEST_MINIO_ENDPOINT").expect("MinIO endpoint"); + let directory = tempdir().expect("cache directory"); + let cache = Arc::new( + S3DiskCache::new( + &S3CacheConfig { + max_bytes: 3, + ..cache_config_with_dir(directory.path()) + }, + "mutation-test", + ) + .await + .expect("cache"), + ); + let bucket = format!("mutation-{}", Uuid::new_v4()); + let mut storage = build_s3_storage_with_cache(&endpoint, &bucket, cache); + let client = aws_sdk_s3::Client::from_conf( + S3ConfigBuilder::new() + .region(Region::new("us-east-1")) + .behavior_version(BehaviorVersion::latest()) + .force_path_style(true) + .endpoint_url(&endpoint) + .credentials_provider(SharedCredentialsProvider::new(AwsCredentials::new( + "minioadmin", + "minioadmin", + None, + None, + "minio-test", + ))) + .build(), + ); + Arc::get_mut(&mut storage.0) + .expect("exclusive storage") + .client = client.clone(); + client + .create_bucket() + .bucket(&bucket) + .send() + .await + .expect("create bucket"); + let repository = Uuid::new_v4(); + for append in [false, true] { + let path = StoragePath::from(if append { "append" } else { "overwrite" }); + storage + .save_file(repository, FileContent::from(&b"old"[..]), &path) + .await + .expect("store cached original"); + if append { + storage + .append_file(repository, FileContent::from(&b"!"[..]), &path) + .await + .expect("append beyond capacity"); + } else { + storage + .save_file(repository, FileContent::from(&b"replacement"[..]), &path) + .await + .expect("overwrite beyond capacity"); + } + let expected: &[u8] = if append { b"old!" } else { b"replacement" }; + let Some(crate::StorageFile::File { mut content, .. }) = storage + .open_file(repository, &path) + .await + .expect("read object") + else { + panic!("missing object") + }; + let mut actual = Vec::new(); + content + .read_to_end(&mut actual) + .await + .expect("read content"); + assert_eq!(actual, expected, "append={append}"); + } + storage + .delete_repository(repository) + .await + .expect("delete objects"); + client + .delete_bucket() + .bucket(bucket) + .send() + .await + .expect("delete bucket"); +} - let client = aws_sdk_s3::Client::from_conf(client_config); - let config = S3Config { - bucket_name: bucket.into(), - region: Some(S3StorageRegion::UsEast1), - custom_region: Some(CustomRegion { - custom_region: Some("us-east-1".into()), - endpoint: endpoint.parse().unwrap(), - }), - credentials: S3Credentials::new_access_key("AKIA", "SECRET"), - path_style: true, - cache: S3CacheConfig::default(), - adaptive_buffer: AdaptiveBufferConfig::default(), +#[tokio::test] +async fn cached_file_information_uses_memory_without_reading_cached_content() { + let directory = tempdir().expect("cache directory"); + let cache = Arc::new( + S3DiskCache::new(&cache_config_with_dir(directory.path()), "metadata-test") + .await + .expect("cache"), + ); + // A local listener with no accept loop makes any accidental S3 request time out. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("listener"); + let endpoint = format!("http://{}", listener.local_addr().expect("address")); + let storage = build_s3_storage_with_cache(&endpoint, "metadata-test", cache.clone()); + let repository = Uuid::new_v4(); + let path = StoragePath::from("blob"); + let key = storage.cache_key(&repository, &path); + cache + .put(&key, Bytes::from_static(b"content"), Some("text/plain")) + .await + .expect("cache content"); + let entry = cache + .state + .lock() + .await + .entries + .peek(&key) + .expect("entry") + .clone(); + fs::remove_file(cache.dir.join(entry.relative_path)) + .await + .expect("remove cache file"); + + let information = tokio::time::timeout( + Duration::from_secs(1), + storage.get_file_information(repository, &path), + ) + .await + .expect("metadata must not contact storage") + .expect("metadata") + .expect("known object"); + let crate::FileType::File(file) = information.file_type else { + panic!("expected file") }; + assert_eq!(file.file_size, 7); + assert_eq!( + file.mime_type.expect("content type").to_string(), + "text/plain" + ); +} - let inner = S3StorageInner { - config, - storage_config, - client, - cache: None, +#[tokio::test] +async fn disk_cache_concurrent_publications_leave_a_valid_entry() { + let temp_dir = tempdir().expect("tempdir"); + let config = S3CacheConfig { + max_bytes: 128, + ..cache_config_with_dir(temp_dir.path()) }; + let cache = Arc::new( + S3DiskCache::new(&config, "test-cache") + .await + .expect("cache"), + ); + let mut tasks = Vec::new(); + for index in 0..8u8 { + let cache = Arc::clone(&cache); + tasks.push(tokio::spawn(async move { + cache + .put("same-key", Bytes::from(vec![index; 8]), None) + .await + .expect("concurrent publication"); + })); + } + for task in tasks { + task.await.expect("publication task"); + } + let object = cache + .get("same-key") + .await + .expect("cache read") + .expect("entry remains"); + assert_eq!(object.size, 8); + let mut content = Vec::new(); + let mut file = object.file; + tokio::io::AsyncReadExt::read_to_end(&mut file, &mut content) + .await + .expect("read cache file"); + assert_eq!(content.len(), 8); +} - S3Storage::from(inner) +#[tokio::test] +async fn stale_cache_publication_does_not_remove_newer_entry() { + let temp_dir = tempdir().expect("tempdir"); + let cache = S3DiskCache::new(&cache_config_with_dir(temp_dir.path()), "test-cache") + .await + .expect("cache"); + + cache + .put("same-key", Bytes::from_static(b"old"), None) + .await + .expect("initial publication"); + let generation = cache.generation_for("same-key").await; + cache + .put("same-key", Bytes::from_static(b"new"), None) + .await + .expect("newer publication"); + + assert!( + !cache + .put_if_generation( + "same-key", + Bytes::from_static(b"stale"), + None, + None, + generation, + ) + .await + .expect("stale publication") + ); + + let object = cache + .get("same-key") + .await + .expect("cache read") + .expect("newer entry remains"); + let mut content = Vec::new(); + let mut file = object.file; + tokio::io::AsyncReadExt::read_to_end(&mut file, &mut content) + .await + .expect("cache content"); + assert_eq!(content, b"new"); } -fn list_response_body(prefix: &str, keys: &[&str], truncated: bool, token: Option<&str>) -> String { - let contents = keys - .iter() - .map(|k| format!("{}", k)) - .collect::>() - .join(""); - let token_xml = token - .map(|t| format!("{t}")) - .unwrap_or_default(); - format!( - r#" - mock-bucket - {prefix} - {} - {} - {contents} - {token_xml} -"#, - keys.len(), - if truncated { "true" } else { "false" } - ) +#[tokio::test] +async fn oversized_stale_publication_does_not_evict_newer_entry() { + let temp_dir = tempdir().expect("tempdir"); + let cache = S3DiskCache::new(&cache_config_with_dir(temp_dir.path()), "test-cache") + .await + .expect("cache"); + cache + .put("same-key", Bytes::from_static(b"new"), None) + .await + .expect("newer publication"); + + assert!( + !cache + .put_if_generation( + "same-key", + Bytes::from_static(b"stale-too-large"), + None, + None, + 0, + ) + .await + .expect("stale publication") + ); + assert!(cache.get("same-key").await.expect("cache read").is_some()); } -fn delete_ok_response() -> Response> { - Response::builder() - .status(StatusCode::OK) - .body(Full::new(bytes::Bytes::from_static( - br#""#, - ))) - .unwrap() +#[test] +fn adaptive_buffer_respects_pressure_threshold() { + let config = AdaptiveBufferConfig { + min_buffer_bytes: 1024 * 1024, + max_buffer_bytes: 16 * 1024 * 1024, + memory_pressure_threshold: 0.5, + }; + + assert_eq!(config.limit_for_pressure(0.0), 16 * 1024 * 1024); + let mid = config.limit_for_pressure(0.25); + assert!(mid < 16 * 1024 * 1024 && mid > 1024 * 1024); + assert_eq!(config.limit_for_pressure(0.5), 1024 * 1024); + assert_eq!(config.limit_for_pressure(0.9), 1024 * 1024); +} + +#[test] +fn memory_snapshot_values_are_already_bytes() { + let snapshot = MemorySnapshot::from_values(100, 25); + assert_eq!(snapshot.total_bytes, 100); + assert_eq!(snapshot.available_bytes, 25); + assert!((snapshot.pressure() - 0.75).abs() < f64::EPSILON); +} + +#[test] +fn memory_snapshot_cache_reuses_values_until_expiry() { + let mut cache = MemorySnapshotCache::default(); + let start = tokio::time::Instant::now(); + let mut captures = 0; + + let first = cache.get_or_capture(start, || { + captures += 1; + Some(MemorySnapshot::from_values(100, 50)) + }); + let second = cache.get_or_capture(start + Duration::from_secs(1), || { + captures += 1; + Some(MemorySnapshot::from_values(200, 100)) + }); + let third = cache.get_or_capture(start + Duration::from_secs(6), || { + captures += 1; + Some(MemorySnapshot::from_values(200, 100)) + }); + + assert_eq!(captures, 2); + assert_eq!(first.expect("first").total_bytes, 100); + assert_eq!(second.expect("second").total_bytes, 100); + assert_eq!(third.expect("third").total_bytes, 200); +} + +#[test] +fn memory_snapshot_cache_reuses_zero_memory_result_until_expiry() { + let mut cache = MemorySnapshotCache::default(); + let start = tokio::time::Instant::now(); + let mut captures = 0; + + let first = cache.get_or_capture(start, || { + captures += 1; + None + }); + let second = cache.get_or_capture(start + Duration::from_secs(1), || { + captures += 1; + Some(MemorySnapshot::from_values(100, 50)) + }); + + assert!(first.is_none()); + assert!(second.is_none()); + assert_eq!(captures, 1); +} + +#[test] +fn memory_limits_prefer_valid_lower_cgroup_limit() { + assert_eq!( + super::effective_memory_limits(1_000, 900, Some((400, 500))), + (400, 400) + ); + assert_eq!( + super::effective_memory_limits(1_000, 900, Some((2_000, 100))), + (1_000, 900) + ); + assert_eq!(super::effective_memory_limits(0, 0, Some((0, 0))), (0, 0)); +} + +#[test] +fn zero_total_memory_pressure_is_safely_saturated() { + let snapshot = MemorySnapshot::from_values(0, 0); + assert_eq!(snapshot.pressure(), 1.0); } -fn respond_list(body: String) -> Responder { - Box::new(move |_| { - Response::builder() - .status(StatusCode::OK) - .body(Full::new(bytes::Bytes::from(body.clone()))) +// --------------------------------------------------------------------------- +// delete_repository integration tests against a lightweight mock S3 endpoint. +// --------------------------------------------------------------------------- +use super::{S3Storage, S3StorageInner}; +use crate::{FileContent, Storage, StorageConfigInner}; +use aws_config::BehaviorVersion; +use aws_credential_types::{Credentials as AwsCredentials, provider::SharedCredentialsProvider}; +use aws_sdk_s3::config::Builder as S3ConfigBuilder; +use aws_smithy_types::retry::RetryConfig; +use aws_types::region::Region; +use bytes::Buf; +use http_body_util::{BodyExt, Full}; +use hyper::{ + Request, Response, StatusCode, body::Incoming, header, server::conn::http1, service::service_fn, +}; +use hyper_util::rt::TokioIo; +use nr_core::storage::StoragePath; +use parking_lot::Mutex; +use std::{collections::VecDeque, convert::Infallible, net::SocketAddr, sync::Arc}; +use tokio::net::TcpListener; +use uuid::Uuid; + +type RecordedBody = bytes::Bytes; +type Responder = Box) -> ResponsePlan + Send + 'static>; + +struct ResponsePlan { + delay: Duration, + response: Response>, +} + +impl ResponsePlan { + fn immediate(response: Response>) -> Self { + Self { + delay: Duration::ZERO, + response, + } + } + + fn delayed(delay: Duration, response: Response>) -> Self { + Self { delay, response } + } +} + +#[derive(Clone, Debug)] +struct RecordedRequest { + method: hyper::Method, + uri: hyper::Uri, + body: RecordedBody, + headers: hyper::HeaderMap, +} + +struct MockS3Server { + address: SocketAddr, + requests: Arc>>, + shutdown: tokio::sync::oneshot::Sender<()>, + task: tokio::task::JoinHandle<()>, +} + +impl MockS3Server { + async fn start(responders: Vec) -> Self { + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind mock s3"); + let address = listener.local_addr().expect("address"); + let requests = Arc::new(Mutex::new(Vec::new())); + let responder_queue = Arc::new(Mutex::new(VecDeque::from(responders))); + + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + let requests_clone = Arc::clone(&requests); + let responders_clone = Arc::clone(&responder_queue); + + let task = tokio::spawn(async move { + loop { + tokio::select! { + _ = &mut shutdown_rx => break, + accept = listener.accept() => { + let (stream, _) = match accept { + Ok(ok) => ok, + Err(err) => { + eprintln!("mock s3 accept error: {err}"); + continue; + } + }; + let requests = Arc::clone(&requests_clone); + let responders = Arc::clone(&responders_clone); + tokio::spawn(async move { + let service = service_fn(move |req: Request| { + handle_request(req, Arc::clone(&requests), Arc::clone(&responders)) + }); + if let Err(err) = http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await + { + eprintln!("mock s3 connection error: {err}"); + } + }); + } + } + } + }); + + Self { + address, + requests, + shutdown: shutdown_tx, + task, + } + } + + fn endpoint(&self) -> String { + format!("http://{}", self.address) + } + + fn take_requests(&self) -> Vec { + self.requests.lock().clone() + } + + async fn shutdown(self) { + let _ = self.shutdown.send(()); + let _ = self.task.await; + } +} + +async fn handle_request( + mut req: Request, + requests: Arc>>, + responders: Arc>>, +) -> Result>, Infallible> { + let body_bytes = req.body_mut().collect().await.unwrap().to_bytes(); + let (parts, _) = req.into_parts(); + let req_with_body = Request::from_parts(parts, body_bytes.clone()); + + requests.lock().push(RecordedRequest { + method: req_with_body.method().clone(), + uri: req_with_body.uri().clone(), + body: body_bytes.clone(), + headers: req_with_body.headers().clone(), + }); + + let plan = { + let mut queue = responders.lock(); + let plan = if let Some(responder) = queue.front_mut() { + responder(req_with_body) + } else { + ResponsePlan::immediate( + Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Full::new(bytes::Bytes::from_static( + b"no responder for request", + ))) + .unwrap(), + ) + }; + // pop only after a successful response to preserve strict ordering on errors + queue.pop_front(); + plan + }; + if !plan.delay.is_zero() { + sleep(plan.delay).await; + } + Ok(plan.response) +} + +fn build_s3_storage(endpoint: &str, bucket: &str) -> S3Storage { + let mut storage_config = StorageConfigInner::test_config(); + storage_config.storage_type = "s3".into(); + + let client_config = S3ConfigBuilder::new() + .region(Region::new("us-east-1")) + .behavior_version(BehaviorVersion::latest()) + .force_path_style(true) + .endpoint_url(endpoint) + .retry_config(RetryConfig::standard().with_max_attempts(1)) + .credentials_provider(SharedCredentialsProvider::new(AwsCredentials::new( + "AKIA", "SECRET", None, None, "mock", + ))) + .build(); + + let client = aws_sdk_s3::Client::from_conf(client_config); + let config = S3Config { + bucket_name: bucket.into(), + region: Some("us-east-1".into()), + custom_region: Some(CustomRegion { + custom_region: Some("us-east-1".into()), + endpoint: endpoint.parse().unwrap(), + }), + credentials: S3Credentials::new_access_key("AKIA", "SECRET"), + path_style: true, + cache: S3CacheConfig::default(), + adaptive_buffer: AdaptiveBufferConfig::default(), + }; + + let inner = S3StorageInner { + config, + storage_config, + client, + cache: None, + cache_load_locks: parking_lot::Mutex::new(ahash::HashMap::default()), + manifest_cache: parking_lot::Mutex::new(super::ManifestCache::new()), + manifest_load_locks: parking_lot::Mutex::new(ahash::HashMap::default()), + append_staging: parking_lot::Mutex::new(ahash::HashMap::default()), + append_operation_locks: parking_lot::Mutex::new(ahash::HashMap::default()), + upload_spool_budget: Arc::new(tokio::sync::Semaphore::new( + (super::MAX_STAGED_BYTES / super::S3_UPLOAD_SPOOL_PERMIT_BYTES) as usize, + )), + staging_dir: test_staging_dir(), + creation_probes: parking_lot::Mutex::new(lru::LruCache::new( + std::num::NonZeroUsize::new(4096).unwrap(), + )), + creation_probe_generation: parking_lot::Mutex::new(0), + }; + + S3Storage::from(inner) +} + +fn test_staging_dir() -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("pkgly-test-staging-{}", Uuid::new_v4())); + std::fs::create_dir_all(&dir).expect("staging dir"); + dir +} + +fn build_s3_storage_with_cache(endpoint: &str, bucket: &str, cache: Arc) -> S3Storage { + let mut storage_config = StorageConfigInner::test_config(); + storage_config.storage_type = "s3".into(); + + let client_config = S3ConfigBuilder::new() + .region(Region::new("us-east-1")) + .behavior_version(BehaviorVersion::latest()) + .force_path_style(true) + .endpoint_url(endpoint) + .retry_config(RetryConfig::standard().with_max_attempts(1)) + .credentials_provider(SharedCredentialsProvider::new(AwsCredentials::new( + "AKIA", "SECRET", None, None, "mock", + ))) + .build(); + + let client = aws_sdk_s3::Client::from_conf(client_config); + let config = S3Config { + bucket_name: bucket.into(), + region: Some("us-east-1".into()), + custom_region: Some(CustomRegion { + custom_region: Some("us-east-1".into()), + endpoint: endpoint.parse().unwrap(), + }), + credentials: S3Credentials::new_access_key("AKIA", "SECRET"), + path_style: true, + cache: S3CacheConfig { + enabled: true, + path: Some(cache.dir.clone()), + max_bytes: cache.max_bytes, + max_entries: 4, + }, + adaptive_buffer: AdaptiveBufferConfig::default(), + }; + + S3Storage::from(S3StorageInner { + config, + storage_config, + client, + cache: Some(cache), + cache_load_locks: parking_lot::Mutex::new(ahash::HashMap::default()), + manifest_cache: parking_lot::Mutex::new(super::ManifestCache::new()), + manifest_load_locks: parking_lot::Mutex::new(ahash::HashMap::default()), + append_staging: parking_lot::Mutex::new(ahash::HashMap::default()), + append_operation_locks: parking_lot::Mutex::new(ahash::HashMap::default()), + upload_spool_budget: Arc::new(tokio::sync::Semaphore::new( + (super::MAX_STAGED_BYTES / super::S3_UPLOAD_SPOOL_PERMIT_BYTES) as usize, + )), + staging_dir: test_staging_dir(), + creation_probes: parking_lot::Mutex::new(lru::LruCache::new( + std::num::NonZeroUsize::new(4096).unwrap(), + )), + creation_probe_generation: parking_lot::Mutex::new(0), + }) +} + +fn list_response_body(prefix: &str, keys: &[&str], truncated: bool, token: Option<&str>) -> String { + let contents = keys + .iter() + .map(|k| format!("{}", k)) + .collect::>() + .join(""); + let token_xml = token + .map(|t| format!("{t}")) + .unwrap_or_default(); + format!( + r#" + mock-bucket + {prefix} + {} + {} + {contents} + {token_xml} +"#, + keys.len(), + if truncated { "true" } else { "false" } + ) +} + +fn list_response_body_with_metadata( + prefix: &str, + objects: &[(&str, u64, Option<&str>)], + common_prefixes: &[&str], + truncated: bool, + token: Option<&str>, +) -> String { + let contents = objects + .iter() + .map(|(key, size, last_modified)| { + let last_modified = last_modified + .map(|value| format!("{value}")) + .unwrap_or_default(); + format!("{key}{last_modified}{size}") + }) + .collect::>() + .join(""); + let prefixes = common_prefixes + .iter() + .map(|prefix| format!("{prefix}")) + .collect::>() + .join(""); + let token_xml = token + .map(|value| format!("{value}")) + .unwrap_or_default(); + format!( + r#" + mock-bucket + {prefix} + {} + {} + {contents} + {prefixes} + {token_xml} +"#, + objects.len() + common_prefixes.len(), + if truncated { "true" } else { "false" } + ) +} + +fn delete_ok_response() -> Response> { + Response::builder() + .status(StatusCode::OK) + .body(Full::new(bytes::Bytes::from_static( + br#""#, + ))) + .unwrap() +} + +fn respond_list(body: String) -> Responder { + Box::new(move |_| { + ResponsePlan::immediate( + Response::builder() + .status(StatusCode::OK) + .body(Full::new(bytes::Bytes::from(body.clone()))) + .unwrap(), + ) + }) +} + +fn respond_delete(assert_keys: Vec) -> Responder { + Box::new(move |req| { + let body = req.into_body(); + let body_text = std::str::from_utf8(body.chunk()).expect("utf8 delete body"); + for key in &assert_keys { + assert!( + body_text.contains(key), + "delete payload should contain key {key}, payload was {body_text}" + ); + } + ResponsePlan::immediate(delete_ok_response()) + }) +} + +fn respond_response(response: Response>) -> Responder { + Box::new(move |_| ResponsePlan::immediate(response.clone())) +} + +fn respond_delayed(delay: Duration, response: Response>) -> Responder { + Box::new(move |_| ResponsePlan::delayed(delay, response.clone())) +} + +fn response_with_body( + status: StatusCode, + body: impl Into, +) -> Response> { + Response::builder() + .status(status) + .body(Full::new(body.into())) + .unwrap() +} + +fn not_found_response(code: &str) -> Response> { + response_with_body( + StatusCode::NOT_FOUND, + bytes::Bytes::from(format!( + r#"{code}missing"# + )), + ) +} + +fn conditional_error_response(status: StatusCode, code: &str) -> Response> { + response_with_body( + status, + bytes::Bytes::from(format!( + r#"{code}conditional write failed"# + )), + ) +} + +fn head_response( + status: StatusCode, + etag: Option<&str>, + size: Option, + content_type: Option<&str>, + last_modified: Option<&str>, +) -> Response> { + let mut builder = Response::builder().status(status); + if let Some(etag) = etag { + builder = builder.header(header::ETAG, etag); + } + if let Some(size) = size { + builder = builder.header(header::CONTENT_LENGTH, size.to_string()); + } + if let Some(content_type) = content_type { + builder = builder.header(header::CONTENT_TYPE, content_type); + } + if let Some(last_modified) = last_modified { + builder = builder.header(header::LAST_MODIFIED, last_modified); + } + builder.body(Full::new(bytes::Bytes::new())).unwrap() +} + +fn get_response( + body: &'static [u8], + etag: Option<&str>, + content_type: Option<&str>, + last_modified: Option<&str>, +) -> Response> { + let mut builder = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_LENGTH, body.len().to_string()); + if let Some(etag) = etag { + builder = builder.header(header::ETAG, etag); + } + if let Some(content_type) = content_type { + builder = builder.header(header::CONTENT_TYPE, content_type); + } + if let Some(last_modified) = last_modified { + builder = builder.header(header::LAST_MODIFIED, last_modified); + } + builder + .body(Full::new(bytes::Bytes::from_static(body))) + .unwrap() +} + +fn copy_response(etag: &str) -> Response> { + response_with_body( + StatusCode::OK, + bytes::Bytes::from(format!( + r#"{etag}2025-01-01T00:00:00.000Z"# + )), + ) +} + +fn multipart_create_response(upload_id: &str) -> Response> { + response_with_body( + StatusCode::OK, + bytes::Bytes::from(format!( + r#"{upload_id}"# + )), + ) +} + +fn multipart_part_response(etag: &str) -> Response> { + response_with_body( + StatusCode::OK, + bytes::Bytes::from(format!( + r#"{etag}2025-01-01T00:00:00.000Z"# + )), + ) +} + +fn multipart_complete_response() -> Response> { + response_with_body( + StatusCode::OK, + bytes::Bytes::from_static( + br#"\"complete\""#, + ), + ) +} + +fn parse_deleted_keys(body: &RecordedBody) -> Vec { + let text = String::from_utf8_lossy(body); + text.split("") + .skip(1) + .filter_map(|part| part.split("").next()) + .map(|s| s.to_string()) + .collect() +} + +#[tokio::test] +async fn delayed_responder_can_be_queued_for_timeout_tests() { + let server = MockS3Server::start(vec![respond_delayed( + Duration::from_millis(1), + response_with_body(StatusCode::OK, bytes::Bytes::new()), + )]) + .await; + let mut stream = TcpStream::connect(server.address) + .await + .expect("connect delayed responder"); + stream + .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .expect("write request"); + let mut response = Vec::new(); + stream + .read_to_end(&mut response) + .await + .expect("read response"); + assert!(response.starts_with(b"HTTP/1.1 200 OK")); + server.shutdown().await; +} + +#[tokio::test] +async fn delete_repository_removes_all_files() { + let repository = Uuid::new_v4(); + let key_one = format!("{repository}/packages/a.bin"); + let key_two = format!("{repository}/packages/nested/b.bin"); + + let server = MockS3Server::start(vec![ + respond_list(list_response_body( + &format!("{repository}/"), + &[&key_one, &key_two], + false, + None, + )), + respond_delete(vec![key_one.clone(), key_two.clone()]), + ]) + .await; + + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .delete_repository(repository) + .await + .expect("delete_repository should succeed"); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 2, "one list and one delete call expected"); + let delete_keys = parse_deleted_keys(&requests[1].body); + assert_eq!(delete_keys.len(), 2); + assert!(delete_keys.contains(&key_one)); + assert!(delete_keys.contains(&key_two)); + + server.shutdown().await; +} + +#[tokio::test] +async fn delete_repository_handles_pagination() { + let repository = Uuid::new_v4(); + let first_page_keys = vec![ + format!("{repository}/page1/one"), + format!("{repository}/page1/two"), + ]; + let second_page_keys = vec![format!("{repository}/page2/three")]; + + let server = MockS3Server::start(vec![ + respond_list(list_response_body( + &format!("{repository}/"), + &first_page_keys + .iter() + .map(|s| s.as_str()) + .collect::>(), + true, + Some("token-1"), + )), + respond_delete(first_page_keys.clone()), + respond_list(list_response_body( + &format!("{repository}/"), + &second_page_keys + .iter() + .map(|s| s.as_str()) + .collect::>(), + false, + None, + )), + respond_delete(second_page_keys.clone()), + ]) + .await; + + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .delete_repository(repository) + .await + .expect("delete_repository should succeed with pagination"); + + let requests = server.take_requests(); + assert_eq!( + requests.len(), + 4, + "list/delete/list/delete sequence expected" + ); + assert!( + requests[2] + .uri + .query() + .unwrap_or_default() + .contains("continuation-token=token-1"), + "second list should carry continuation token" + ); + + server.shutdown().await; +} + +#[tokio::test] +async fn delete_repository_is_idempotent_on_empty_prefix() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_list(list_response_body( + &format!("{repository}/"), + &[], + false, + None, + ))]) + .await; + + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .delete_repository(repository) + .await + .expect("empty prefixes should be handled gracefully"); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 1, "only a list request is expected"); + + server.shutdown().await; +} + +#[tokio::test] +async fn delete_repository_preserves_other_repositories() { + let repository = Uuid::new_v4(); + let other_repo = Uuid::new_v4(); + + let keys = [ + format!("{repository}/packages/a.bin"), + format!("{other_repo}/packages/should-not-delete"), + ]; + + let server = MockS3Server::start(vec![ + respond_list(list_response_body( + &format!("{repository}/"), + &keys.iter().map(|s| s.as_str()).collect::>(), + false, + None, + )), + respond_delete(vec![format!("{repository}/packages/a.bin")]), + ]) + .await; + + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage + .delete_repository(repository) + .await + .expect("delete_repository should ignore other repo keys"); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 2); + let deleted = parse_deleted_keys(&requests[1].body); + assert_eq!(deleted, vec![format!("{repository}/packages/a.bin")]); + assert!( + !deleted + .iter() + .any(|key| key.contains(&other_repo.to_string())), + "keys from other repositories must not be deleted" + ); + + server.shutdown().await; +} + +#[tokio::test] +async fn append_uses_if_match_and_returns_appended_bytes() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(get_response( + b"old", + Some("\"etag-old\""), + Some("application/octet-stream"), + None, + )), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let appended = storage + .append_file_staged( + repository, + FileContent::Bytes(Bytes::from_static(b"new")), + &StoragePath::from("file.bin"), + ) + .await + .expect("append should succeed"); + assert_eq!(appended, 3); + + storage + .move_file( + repository, + &StoragePath::from("file.bin"), + &StoragePath::from("final.bin"), + ) + .await + .expect("finalize should upload staged content"); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[0].method, hyper::Method::GET); + assert_eq!(requests[1].method, hyper::Method::PUT); + assert!( + requests[1] + .body + .windows(b"oldnew".len()) + .any(|body| body == b"oldnew") + ); + assert_eq!(requests[1].headers.get(header::IF_NONE_MATCH).unwrap(), "*"); + assert!(requests[1].headers.get(header::IF_MATCH).is_none()); + assert_eq!(requests[2].method, hyper::Method::DELETE); + assert_eq!( + requests[2].headers.get(header::IF_MATCH).unwrap(), + "\"etag-old\"" + ); + server.shutdown().await; +} + +#[tokio::test] +async fn append_uses_if_none_match_for_missing_object() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NoSuchKey")), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let appended = storage + .append_file_staged( + repository, + FileContent::Content(b"new".to_vec()), + &StoragePath::from("file.bin"), + ) + .await + .expect("append should create missing object"); + assert_eq!(appended, 3); + + storage + .move_file( + repository, + &StoragePath::from("file.bin"), + &StoragePath::from("final.bin"), + ) + .await + .expect("finalize should upload staged content"); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].method, hyper::Method::GET); + assert_eq!(requests[1].headers.get(header::IF_NONE_MATCH).unwrap(), "*"); + assert!(requests[1].headers.get(header::IF_MATCH).is_none()); + assert!( + requests[1] + .body + .windows(b"new".len()) + .any(|body| body == b"new") + ); + server.shutdown().await; +} + +#[tokio::test] +async fn append_conflict_preserves_cache_and_returns_conflict() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(get_response(b"old", Some("\"etag-old\""), None, None)), + respond_response(conditional_error_response( + StatusCode::PRECONDITION_FAILED, + "PreconditionFailed", + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + storage + .append_file_staged( + repository, + FileContent::Bytes(Bytes::from_static(b"new")), + &StoragePath::from("file.bin"), + ) + .await + .expect("append should stage locally"); + + let error = storage + .move_file( + repository, + &StoragePath::from("file.bin"), + &StoragePath::from("final.bin"), + ) + .await + .expect_err("stale ETag should produce a conflict"); + + assert!(error.is_conflict()); + assert_eq!(server.take_requests().len(), 3); + server.shutdown().await; +} + +#[tokio::test] +async fn append_rejects_existing_object_without_etag() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_response(get_response( + b"old", None, None, None, + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let error = storage + .append_file( + repository, + FileContent::Content(b"new".to_vec()), + &StoragePath::from("file.bin"), + ) + .await + .expect_err("append must not overwrite without an ETag"); + + assert!(matches!(error.kind(), Some(super::S3ErrorKind::Other))); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} + +#[tokio::test] +async fn append_checks_concrete_ancestors_without_rechecking_final_target() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NotFound")), + respond_response(not_found_response("NoSuchKey")), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + storage + .append_file_staged( + repository, + FileContent::Content(b"new".to_vec()), + &StoragePath::from("parent/file.bin"), + ) + .await + .expect("append should succeed"); + let requests = server.take_requests(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].method, hyper::Method::HEAD); + assert_eq!(requests[1].method, hyper::Method::GET); + server.shutdown().await; +} + +#[tokio::test] +async fn repeated_appends_stage_locally_without_per_chunk_s3_transfers() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NoSuchKey")), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let path = StoragePath::from("file.bin"); + + storage + .append_file_staged(repository, FileContent::Content(b"one".to_vec()), &path) + .await + .expect("first append"); + storage + .append_file_staged(repository, FileContent::Content(b"two".to_vec()), &path) + .await + .expect("second append"); + storage + .append_file_staged(repository, FileContent::Content(b"three".to_vec()), &path) + .await + .expect("third append"); + + // Only the first append contacts S3 (to check for pre-existing content); subsequent + // appends are pure local disk writes, and the single upload happens on finalize. + assert_eq!(server.take_requests().len(), 1); + + storage + .move_file(repository, &path, &StoragePath::from("final.bin")) + .await + .expect("finalize"); + let requests = server.take_requests(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[1].method, hyper::Method::PUT); + assert!( + requests[1] + .body + .windows(b"onetwothree".len()) + .any(|body| body == b"onetwothree") + ); + server.shutdown().await; +} + +#[tokio::test] +async fn save_rejects_concrete_object_ancestor() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_response(head_response( + StatusCode::OK, + Some("\"parent\""), + Some(1), + None, + None, + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let error = storage + .save_file( + repository, + FileContent::Content(b"new".to_vec()), + &StoragePath::from("parent/file.bin"), + ) + .await + .expect_err("concrete parent objects must collide"); + assert!(matches!(error, super::S3StorageError::PathCollision(_))); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} + +#[tokio::test] +async fn get_file_information_paginates_virtual_directory_and_counts_all_entries() { + let repository = Uuid::new_v4(); + let prefix = format!("{repository}/dir/"); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NotFound")), + respond_list(list_response_body_with_metadata( + &prefix, + &[( + &format!("{prefix}a.bin"), + 3, + Some("2025-01-01T00:00:00.000Z"), + )], + &[&format!("{prefix}nested/")], + true, + Some("page-2"), + )), + respond_list(list_response_body_with_metadata( + &prefix, + &[( + &format!("{prefix}b.bin"), + 4, + Some("2025-01-02T00:00:00.000Z"), + )], + &[], + false, + None, + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let meta = storage + .get_file_information(repository, &StoragePath::from("dir")) + .await + .expect("directory lookup should succeed") + .expect("directory should exist"); + match meta.file_type { + super::FileType::Directory(directory) => assert_eq!(directory.file_count, 3), + other => panic!("expected directory metadata, got {other:?}"), + } + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert!( + requests[2] + .uri + .query() + .unwrap_or_default() + .contains("continuation-token=page-2") + ); + server.shutdown().await; +} + +#[tokio::test] +async fn virtual_directory_uses_placeholder_timestamp_when_available() { + let repository = Uuid::new_v4(); + let prefix = format!("{repository}/dir/"); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NoSuchKey")), + respond_list(list_response_body_with_metadata( + &prefix, + &[(&prefix, 0, Some("2025-01-04T00:00:00.000Z"))], + &[], + false, + None, + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let meta = storage + .get_file_information(repository, &StoragePath::from("dir")) + .await + .expect("directory lookup") + .expect("directory") + .modified; + assert_eq!(meta.to_rfc3339(), "2025-01-04T00:00:00+00:00"); + server.shutdown().await; +} + +#[tokio::test] +async fn object_timestamps_are_propagated_from_head_and_list() { + let repository = Uuid::new_v4(); + let timestamp = "Wed, 01 Jan 2025 00:00:00 GMT"; + let server = MockS3Server::start(vec![respond_response(head_response( + StatusCode::OK, + Some("\"etag\""), + Some(7), + Some("text/plain"), + Some(timestamp), + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let meta = storage + .get_file_information(repository, &StoragePath::from("file.txt")) + .await + .expect("head should succeed") + .expect("file should exist"); + assert_eq!(meta.modified.to_rfc3339(), "2025-01-01T00:00:00+00:00"); + assert_eq!(meta.created, meta.modified); + server.shutdown().await; +} + +#[tokio::test] +async fn open_file_propagates_get_last_modified() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_response(get_response( + b"payload", + Some("\"etag\""), + Some("text/plain"), + Some("Wed, 01 Jan 2025 00:00:00 GMT"), + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let file = storage + .open_file(repository, &StoragePath::from("file.txt")) + .await + .expect("get should succeed") + .expect("file should exist"); + let (_, meta) = file.file().expect("file result"); + assert_eq!(meta.modified.to_rfc3339(), "2025-01-01T00:00:00+00:00"); + assert_eq!(meta.created, meta.modified); + server.shutdown().await; +} + +#[tokio::test] +async fn open_file_persists_request_timestamp_when_get_omits_last_modified() { + let repository = Uuid::new_v4(); + let temp_dir = tempdir().expect("cache directory"); + let cache = Arc::new( + S3DiskCache::new( + &S3CacheConfig { + max_bytes: 64, + ..cache_config_with_dir(temp_dir.path()) + }, + "test-cache", + ) + .await + .expect("cache"), + ); + let server = MockS3Server::start(vec![respond_response(get_response( + b"payload", + Some("\"etag\""), + Some("text/plain"), + None, + ))]) + .await; + let storage = build_s3_storage_with_cache(&server.endpoint(), "mock-bucket", cache); + + let first = storage + .open_file(repository, &StoragePath::from("file.txt")) + .await + .expect("first get") + .expect("file") + .file() + .expect("file result") + .1 + .modified; + let second = storage + .open_file(repository, &StoragePath::from("file.txt")) + .await + .expect("cache get") + .expect("cached file") + .file() + .expect("cached file result") + .1 + .modified; + + assert_eq!(first, second); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} + +#[tokio::test] +async fn list_repository_objects_preserves_last_modified() { + let repository = Uuid::new_v4(); + let prefix = format!("{repository}/"); + let server = MockS3Server::start(vec![respond_list(list_response_body_with_metadata( + &prefix, + &[( + &format!("{prefix}manifest.json"), + 9, + Some("2025-01-03T00:00:00.000Z"), + )], + &[], + false, + None, + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let objects = storage + .list_repository_objects(repository, None) + .await + .expect("list should succeed"); + assert_eq!(objects.len(), 1); + assert_eq!(objects[0].key, "manifest.json"); + assert_eq!(objects[0].size, 9); + assert_eq!( + objects[0].last_modified.unwrap().to_rfc3339(), + "2025-01-03T00:00:00+00:00" + ); + server.shutdown().await; +} + +#[tokio::test] +async fn control_timeout_is_classified_as_network() { + let error = super::with_timeout(Duration::from_millis(5), async { + tokio::time::sleep(Duration::from_millis(50)).await; + Ok::<(), SmithySdkError>(()) + }) + .await + .expect_err("delayed control call should time out"); + assert!(matches!(error.kind(), Some(super::S3ErrorKind::Network))); +} + +#[tokio::test] +async fn copy_move_uses_guarded_server_side_copy_without_get() { + let repository = Uuid::new_v4(); + let source = StoragePath::from("folder/file name.bin"); + let destination = StoragePath::from("folder/moved.bin"); + let server = MockS3Server::start(vec![ + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(12), + Some("application/octet-stream"), + None, + )), + respond_response(copy_response("\"destination-etag\"")), + respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock bucket"); + + assert!( + storage + .move_file(repository, &source, &destination) + .await + .expect("move should succeed") + ); + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[0].method, hyper::Method::HEAD); + assert_eq!(requests[1].method, hyper::Method::PUT); + assert_eq!(requests[2].method, hyper::Method::DELETE); + assert!( + requests + .iter() + .all(|request| request.method != hyper::Method::GET) + ); + assert_eq!( + requests[1] + .headers + .get("x-amz-copy-source") + .unwrap() + .to_str() + .unwrap(), + format!("/mock%20bucket/{repository}%2Ffolder%2Ffile%20name.bin") + ); + assert_eq!( + requests[1] + .headers + .get("x-amz-copy-source-if-match") + .unwrap(), + "\"source-etag\"" + ); + assert_eq!( + requests[2].headers.get(header::IF_MATCH).unwrap(), + "\"source-etag\"" + ); + server.shutdown().await; +} + +#[tokio::test] +async fn exactly_five_gib_uses_single_copy_object() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(super::MULTIPART_COPY_THRESHOLD), + None, + None, + )), + respond_response(copy_response("\"destination-etag\"")), + respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + assert!( + storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect("boundary move should succeed") + ); + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[1].method, hyper::Method::PUT); + assert!( + requests[1].uri.query().is_none() || !requests[1].uri.query().unwrap().contains("uploadId") + ); + server.shutdown().await; +} + +#[tokio::test] +async fn move_returns_false_for_missing_source() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_response(not_found_response("NoSuchKey"))]).await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + assert!( + !storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect("missing move should be idempotent") + ); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} + +#[tokio::test] +async fn move_copy_conflict_does_not_delete_source() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(1), + None, + None, + )), + respond_response(conditional_error_response( + StatusCode::PRECONDITION_FAILED, + "PreconditionFailed", + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let error = storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect_err("copy conflict should be returned"); + assert!(error.is_conflict()); + let requests = server.take_requests(); + assert_eq!(requests.len(), 2); + assert!( + requests + .iter() + .all(|request| request.method != hyper::Method::DELETE) + ); + server.shutdown().await; +} + +#[tokio::test] +async fn move_delete_conflict_leaves_both_objects() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(1), + None, + None, + )), + respond_response(copy_response("\"destination-etag\"")), + respond_response(conditional_error_response( + StatusCode::PRECONDITION_FAILED, + "PreconditionFailed", + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let error = storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect_err("delete conflict should be returned"); + assert!(error.is_conflict()); + let requests = server.take_requests(); + assert_eq!(requests.len(), 3); + assert_eq!( + requests[2].headers.get(header::IF_MATCH).unwrap(), + "\"source-etag\"" + ); + server.shutdown().await; +} + +#[tokio::test] +async fn multipart_copy_aborts_after_failed_part() { + let repository = Uuid::new_v4(); + let size = super::MULTIPART_COPY_THRESHOLD + 1; + let server = MockS3Server::start(vec![ + respond_response(head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(size), + None, + None, + )), + respond_response(multipart_create_response("upload-1")), + respond_response(response_with_body( + StatusCode::INTERNAL_SERVER_ERROR, + bytes::Bytes::from_static(b"copy part failed"), + )), + respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let error = storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect_err("part failure should abort multipart copy"); + assert!(!error.is_conflict()); + let requests = server.take_requests(); + assert_eq!(requests.len(), 4); + assert_eq!(requests[3].method, hyper::Method::DELETE); + assert!( + requests[3] + .uri + .query() + .unwrap_or_default() + .contains("uploadId=upload-1") + ); + server.shutdown().await; +} + +#[tokio::test] +async fn large_move_uses_multipart_copy_ranges_and_completes() { + let repository = Uuid::new_v4(); + let size = super::MULTIPART_COPY_THRESHOLD + 1; + let part_size = super::MULTIPART_COPY_PART_SIZE.max(size.div_ceil(10_000)); + let part_count = size.div_ceil(part_size); + let mut source_head = head_response( + StatusCode::OK, + Some("\"source-etag\""), + Some(size), + Some("application/octet-stream"), + None, + ); + source_head + .headers_mut() + .insert(header::CACHE_CONTROL, "max-age=3600".parse().unwrap()); + source_head.headers_mut().insert( + header::CONTENT_DISPOSITION, + "attachment; filename=artifact.bin".parse().unwrap(), + ); + source_head + .headers_mut() + .insert(header::CONTENT_ENCODING, "gzip".parse().unwrap()); + source_head + .headers_mut() + .insert(header::CONTENT_LANGUAGE, "en".parse().unwrap()); + source_head.headers_mut().insert( + header::EXPIRES, + "Wed, 21 Oct 2037 07:28:00 GMT".parse().unwrap(), + ); + source_head.headers_mut().insert( + header::HeaderName::from_static("x-amz-website-redirect-location"), + "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/downloads/artifact.bin".parse().unwrap(), + ); + source_head.headers_mut().insert( + header::HeaderName::from_static("x-amz-meta-owner"), + "pkgly".parse().unwrap(), + ); + let mut responders = vec![ + respond_response(source_head), + respond_response(multipart_create_response("upload-1")), + ]; + for part in 0..part_count { + responders.push(respond_response(multipart_part_response(&format!( + "\"part-{part}\"" + )))); + } + responders.push(respond_response(multipart_complete_response())); + responders.push(respond_response(response_with_body( + StatusCode::NO_CONTENT, + bytes::Bytes::new(), + ))); + let server = MockS3Server::start(responders).await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + assert!( + storage + .move_file( + repository, + &StoragePath::from("source.bin"), + &StoragePath::from("destination.bin"), + ) + .await + .expect("multipart move should succeed") + ); + + let requests = server.take_requests(); + assert_eq!(requests.len(), 2 + part_count as usize + 2); + let create_headers = &requests[1].headers; + assert_eq!( + create_headers.get(header::CACHE_CONTROL).unwrap(), + "max-age=3600" + ); + assert_eq!( + create_headers.get(header::CONTENT_DISPOSITION).unwrap(), + "attachment; filename=artifact.bin" + ); + assert_eq!( + create_headers.get(header::CONTENT_ENCODING).unwrap(), + "gzip" + ); + assert_eq!(create_headers.get(header::CONTENT_LANGUAGE).unwrap(), "en"); + assert_eq!( + create_headers.get(header::EXPIRES).unwrap(), + "Wed, 21 Oct 2037 07:28:00 GMT" + ); + assert_eq!( + create_headers + .get("x-amz-website-redirect-location") + .unwrap(), + "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/downloads/artifact.bin" + ); + assert_eq!(create_headers.get("x-amz-meta-owner").unwrap(), "pkgly"); + let part_requests = &requests[2..2 + part_count as usize]; + assert_eq!( + part_requests[0] + .headers + .get("x-amz-copy-source-range") + .unwrap(), + "bytes=0-67108863" + ); + assert!( + part_requests + .last() .unwrap() - }) + .headers + .get("x-amz-copy-source-range") + .is_some() + ); + assert_eq!( + requests[2 + part_count as usize].method, + hyper::Method::POST + ); + assert_eq!(requests.last().unwrap().method, hyper::Method::DELETE); + server.shutdown().await; } -fn respond_delete(assert_keys: Vec) -> Responder { - Box::new(move |req| { - let body = req.into_body(); - let body_text = std::str::from_utf8(body.chunk()).expect("utf8 delete body"); - for key in &assert_keys { - assert!( - body_text.contains(key), - "delete payload should contain key {key}, payload was {body_text}" - ); - } - delete_ok_response() - }) +#[tokio::test] +async fn manifest_pages_share_one_cached_traversal() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_list(list_response_body_with_metadata( + &format!("{repository}/v2/"), + &[], + &[], + false, + None, + )), + respond_list(list_response_body_with_metadata( + &format!("{repository}/v2/"), + &[], + &[], + false, + None, + )), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let first = storage + .list_docker_manifests_paginated(repository, 0, 10) + .await + .expect("first page"); + let second = storage + .list_docker_manifests_paginated(repository, 10, 10) + .await + .expect("second page"); + assert!(first.0.is_empty()); + assert_eq!(first.1, 0); + assert!(second.0.is_empty()); + assert_eq!(server.take_requests().len(), 2); + server.shutdown().await; } -fn parse_deleted_keys(body: &RecordedBody) -> Vec { - let text = String::from_utf8_lossy(body); - text.split("") - .skip(1) - .filter_map(|part| part.split("").next()) - .map(|s| s.to_string()) - .collect() +#[tokio::test] +async fn concurrent_manifest_requests_share_one_traversal() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![respond_list(list_response_body_with_metadata( + &format!("{repository}/v2/"), + &[], + &[], + false, + None, + ))]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let first_storage = storage.clone(); + let first = tokio::spawn(async move { + first_storage + .list_docker_manifests(repository) + .await + .expect("first manifest request") + }); + let second_storage = storage.clone(); + let second = tokio::spawn(async move { + second_storage + .list_docker_manifests(repository) + .await + .expect("second manifest request") + }); + assert!(first.await.expect("first task").is_empty()); + assert!(second.await.expect("second task").is_empty()); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; } #[tokio::test] -async fn delete_repository_removes_all_files() { +async fn concurrent_cache_misses_share_one_object_download() { let repository = Uuid::new_v4(); - let key_one = format!("{repository}/packages/a.bin"); - let key_two = format!("{repository}/packages/nested/b.bin"); + let directory = tempdir().expect("cache directory"); + let cache = Arc::new( + S3DiskCache::new( + &cache_config_with_dir(directory.path()), + "cache-miss-coordination", + ) + .await + .expect("cache"), + ); + let server = MockS3Server::start(vec![respond_delayed( + Duration::from_millis(50), + get_response(b"payload", Some("\"etag\""), None, None), + )]) + .await; + let storage = build_s3_storage_with_cache(&server.endpoint(), "mock-bucket", cache); + let path = StoragePath::from("blob"); + + let first_storage = storage.clone(); + let first_path = path.clone(); + let first = tokio::spawn(async move { + first_storage + .open_file(repository, &first_path) + .await + .expect("first object") + }); + sleep(Duration::from_millis(5)).await; + let second_storage = storage.clone(); + let second_path = path.clone(); + let second = tokio::spawn(async move { + second_storage + .open_file(repository, &second_path) + .await + .expect("second object") + }); + + assert!(first.await.expect("first task").is_some()); + assert!(second.await.expect("second task").is_some()); + assert_eq!(server.take_requests().len(), 1); + server.shutdown().await; +} +#[tokio::test] +async fn mutation_during_manifest_load_prevents_stale_cache_publication() { + let repository = Uuid::new_v4(); + let root = format!("{repository}/v2/"); let server = MockS3Server::start(vec![ - respond_list(list_response_body( - &format!("{repository}/"), - &[&key_one, &key_two], + respond_delayed( + Duration::from_millis(75), + response_with_body( + StatusCode::OK, + bytes::Bytes::from(list_response_body_with_metadata( + &root, + &[], + &[], + false, + None, + )), + ), + ), + respond_response(not_found_response("NotFound")), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + respond_list(list_response_body_with_metadata( + &root, + &[], + &[], false, None, )), - respond_delete(vec![key_one.clone(), key_two.clone()]), ]) .await; - let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let loading_storage = storage.clone(); + let loading = tokio::spawn(async move { + loading_storage + .list_docker_manifests(repository) + .await + .expect("manifest traversal") + }); + for _ in 0..100 { + if !server.take_requests().is_empty() { + break; + } + sleep(Duration::from_millis(1)).await; + } + assert_eq!( + server.take_requests().len(), + 1, + "manifest load should start" + ); + storage - .delete_repository(repository) + .save_file( + repository, + FileContent::Content(b"manifest".to_vec()), + &StoragePath::from("manifest.json"), + ) .await - .expect("delete_repository should succeed"); - - let requests = server.take_requests(); - assert_eq!(requests.len(), 2, "one list and one delete call expected"); - let delete_keys = parse_deleted_keys(&requests[1].body); - assert_eq!(delete_keys.len(), 2); - assert!(delete_keys.contains(&key_one)); - assert!(delete_keys.contains(&key_two)); + .expect("save during traversal"); + assert!(loading.await.expect("manifest task").is_empty()); + storage + .list_docker_manifests(repository) + .await + .expect("next read must traverse again"); + assert_eq!(server.take_requests().len(), 4); server.shutdown().await; } #[tokio::test] -async fn delete_repository_handles_pagination() { +async fn manifest_loader_sorts_nested_results_before_pagination() { let repository = Uuid::new_v4(); - let first_page_keys = vec![ - format!("{repository}/page1/one"), - format!("{repository}/page1/two"), - ]; - let second_page_keys = vec![format!("{repository}/page2/three")]; + let root = format!("{repository}/v2/"); + let page = list_response_body_with_metadata( + &root, + &[ + ( + &format!("{repository}/v2/library/image/manifests/a"), + 2, + Some("2025-01-01T00:00:00.000Z"), + ), + ( + &format!("{repository}/v2/library/image/manifests/z"), + 1, + Some("2025-01-01T00:00:00.000Z"), + ), + ], + &[], + false, + None, + ); + let server = MockS3Server::start(vec![respond_list(page.clone()), respond_list(page)]).await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + let (first, total) = storage + .list_docker_manifests_paginated(repository, 0, 1) + .await + .expect("first manifest page"); + let (second, second_total) = storage + .list_docker_manifests_paginated(repository, 1, 1) + .await + .expect("second manifest page"); + assert_eq!(total, 2); + assert_eq!(second_total, 2); + assert_eq!(first[0].key, "v2/library/image/manifests/a"); + assert_eq!(second[0].key, "v2/library/image/manifests/z"); + assert_eq!(server.take_requests().len(), 2); + server.shutdown().await; +} + +#[test] +fn manifest_cache_expires_and_invalidates_entries() { + let repository = Uuid::new_v4(); + let now = tokio::time::Instant::now(); + let mut cache = super::ManifestCache::new(); + cache.insert(repository, Vec::new(), now); + assert!( + cache + .get(repository, now + Duration::from_secs(29)) + .is_some() + ); + assert!( + cache + .get(repository, now + Duration::from_secs(30)) + .is_none() + ); + cache.insert(repository, Vec::new(), now); + cache.invalidate(repository); + assert!(cache.get(repository, now).is_none()); +} +#[tokio::test] +async fn failed_manifest_traversal_is_not_cached() { + let repository = Uuid::new_v4(); + let root = format!("{repository}/v2/"); let server = MockS3Server::start(vec![ - respond_list(list_response_body( - &format!("{repository}/"), - &first_page_keys - .iter() - .map(|s| s.as_str()) - .collect::>(), - true, - Some("token-1"), + respond_response(response_with_body( + StatusCode::INTERNAL_SERVER_ERROR, + bytes::Bytes::from_static(b"temporary failure"), )), - respond_delete(first_page_keys.clone()), - respond_list(list_response_body( - &format!("{repository}/"), - &second_page_keys - .iter() - .map(|s| s.as_str()) - .collect::>(), + respond_list(list_response_body_with_metadata( + &root, + &[], + &[], false, None, )), - respond_delete(second_page_keys.clone()), ]) .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + + assert!(storage.list_docker_manifests(repository).await.is_err()); + assert!( + storage + .list_docker_manifests(repository) + .await + .expect("retry should load") + .is_empty() + ); + assert_eq!(server.take_requests().len(), 2); + server.shutdown().await; +} +#[tokio::test] +async fn successful_save_invalidates_manifest_cache() { + let repository = Uuid::new_v4(); + let root = format!("{repository}/v2/"); + let server = MockS3Server::start(vec![ + respond_list(list_response_body_with_metadata( + &root, + &[], + &[], + false, + None, + )), + respond_response(not_found_response("NotFound")), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + respond_list(list_response_body_with_metadata( + &root, + &[], + &[], + false, + None, + )), + ]) + .await; let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + storage - .delete_repository(repository) + .list_docker_manifests(repository) .await - .expect("delete_repository should succeed with pagination"); + .expect("initial traversal"); + storage + .save_file( + repository, + FileContent::Content(b"manifest".to_vec()), + &StoragePath::from("manifest.json"), + ) + .await + .expect("save should succeed"); + storage + .list_docker_manifests(repository) + .await + .expect("traversal after invalidation"); + assert_eq!(server.take_requests().len(), 4); + server.shutdown().await; +} - let requests = server.take_requests(); - assert_eq!( - requests.len(), - 4, - "list/delete/list/delete sequence expected" +#[tokio::test] +async fn append_file_staged_accounts_shared_spool_capacity() { + let repository = Uuid::new_v4(); + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NoSuchKey")), + respond_response(not_found_response("NoSuchKey")), + respond_response(response_with_body(StatusCode::OK, bytes::Bytes::new())), + ]) + .await; + let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let path = StoragePath::from("file.bin"); + + // Hold the entire shared spool budget so no capacity remains for staged bytes; + // the append must be rejected instead of exceeding the budget. + let total_units = (super::MAX_STAGED_BYTES / super::S3_UPLOAD_SPOOL_PERMIT_BYTES) as u32; + let hoard = storage + .upload_spool_budget() + .try_acquire_many_owned(total_units) + .expect("budget should start empty"); + + let error = storage + .append_file_staged( + repository, + FileContent::Bytes(Bytes::from_static(b"data")), + &path, + ) + .await + .expect_err("append must respect the shared spool capacity"); + assert!( + error.to_string().contains("capacity exceeded"), + "unexpected error: {error}" ); assert!( - requests[2] - .uri - .query() - .unwrap_or_default() - .contains("continuation-token=token-1"), - "second list should carry continuation token" + storage + .append_staging + .lock() + .values() + .map(|entry| entry.size) + .sum::() + == 0, + "a rejected append must not consume shared budget" ); + // Releasing the spool reservation frees capacity for the staged bytes. + drop(hoard); + let appended = storage + .append_file_staged( + repository, + FileContent::Bytes(Bytes::from_static(b"data")), + &path, + ) + .await + .expect("append succeeds once spool capacity is released"); + assert_eq!(appended, 4); server.shutdown().await; } #[tokio::test] -async fn delete_repository_is_idempotent_on_empty_prefix() { +async fn staged_bytes_hold_shared_spool_capacity() { let repository = Uuid::new_v4(); - let server = MockS3Server::start(vec![respond_list(list_response_body( - &format!("{repository}/"), - &[], - false, - None, - ))]) + let server = MockS3Server::start(vec![ + respond_response(not_found_response("NoSuchKey")), + respond_response(not_found_response("NoSuchKey")), + ]) .await; - let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let budget = storage.upload_spool_budget(); + let initial = budget.available_permits(); + let path = StoragePath::from("file.bin"); + storage - .delete_repository(repository) + .append_file_staged( + repository, + FileContent::Bytes(Bytes::from_static(b"data")), + &path, + ) .await - .expect("empty prefixes should be handled gracefully"); - - let requests = server.take_requests(); - assert_eq!(requests.len(), 1, "only a list request is expected"); + .expect("append should reserve shared capacity"); + assert_eq!(budget.available_permits(), initial - 1); + assert!( + storage + .delete_file(repository, &path) + .await + .expect("delete staged upload") + ); + assert_eq!(budget.available_permits(), initial); server.shutdown().await; } #[tokio::test] -async fn delete_repository_preserves_other_repositories() { +async fn staged_finalization_does_not_block_unrelated_uploads() { let repository = Uuid::new_v4(); - let other_repo = Uuid::new_v4(); - - let keys = [ - format!("{repository}/packages/a.bin"), - format!("{other_repo}/packages/should-not-delete"), - ]; - let server = MockS3Server::start(vec![ - respond_list(list_response_body( - &format!("{repository}/"), - &keys.iter().map(|s| s.as_str()).collect::>(), - false, - None, - )), - respond_delete(vec![format!("{repository}/packages/a.bin")]), + respond_response(not_found_response("NoSuchKey")), + respond_delayed( + Duration::from_millis(400), + response_with_body(StatusCode::OK, bytes::Bytes::new()), + ), + respond_response(not_found_response("NoSuchKey")), ]) .await; - let storage = build_s3_storage(&server.endpoint(), "mock-bucket"); + let path_a = StoragePath::from("a.bin"); + let path_b = StoragePath::from("b.bin"); + storage - .delete_repository(repository) + .append_file_staged( + repository, + FileContent::Bytes(Bytes::from_static(b"aaa")), + &path_a, + ) .await - .expect("delete_repository should ignore other repo keys"); - - let requests = server.take_requests(); - assert_eq!(requests.len(), 2); - let deleted = parse_deleted_keys(&requests[1].body); - assert_eq!(deleted, vec![format!("{repository}/packages/a.bin")]); + .expect("stage upload a"); + + let finalize = tokio::spawn({ + let storage = storage.clone(); + async move { + storage + .move_file(repository, &path_a, &StoragePath::from("final-a.bin")) + .await + } + }); + // Give the finalization time to enter its delayed S3 transfer while holding the + // per-upload lock; an unrelated upload must proceed without waiting for it. + sleep(Duration::from_millis(50)).await; + let started = std::time::Instant::now(); + storage + .append_file_staged( + repository, + FileContent::Bytes(Bytes::from_static(b"bbb")), + &path_b, + ) + .await + .expect("unrelated append must not block behind finalization"); assert!( - !deleted - .iter() - .any(|key| key.contains(&other_repo.to_string())), - "keys from other repositories must not be deleted" + started.elapsed() < Duration::from_millis(300), + "unrelated append was blocked behind the finalization transfer" ); - + finalize + .await + .expect("finalize task joins") + .expect("finalization succeeds"); server.shutdown().await; } diff --git a/docs/docs/knowledge/search.md b/docs/docs/knowledge/search.md index 40a5ab4..654de7f 100644 --- a/docs/docs/knowledge/search.md +++ b/docs/docs/knowledge/search.md @@ -116,15 +116,6 @@ Recommended Prometheus alerts: - High p95 query latency (>50 ms) sustained for 5 minutes. - Zero rows returned while metadata exists (implies filters overly strict or missing indexes). -### Benchmark harness -`benches/search_db.rs` seeds 10k manifests and exercises the search query. Run it locally or in CI with: - -```bash -PKGLY_SEARCH_BENCH_DSN=postgres://user:pass@localhost:5432/pkgly_bench cargo bench search_db_query -``` - -Use this to validate schema/index changes before rollout. - ## 6. Troubleshooting Checklist | Symptom | Probable cause | Action | diff --git a/docs/docs/repositoryTypes/docker/index.md b/docs/docs/repositoryTypes/docker/index.md index 206c579..ec2ab11 100644 --- a/docs/docs/repositoryTypes/docker/index.md +++ b/docs/docs/repositoryTypes/docker/index.md @@ -26,6 +26,24 @@ The Docker repository supports multiple manifest formats: - **OCI Image Manifest** - Open Container Initiative image format - **OCI Image Index** - Multi-platform image manifests (manifest lists) +## Cached package sizes + +The package list reports the stored manifest bytes plus its distinct reachable cached +manifests and blobs. Missing layers and platforms contribute no bytes. A shared layer +counts once within each listed image; adding together multiple image rows is therefore +not a measure of total repository disk usage. + +Pkgly records object sizes after successful writes and stores direct manifest references +in PostgreSQL. Package listings and size sorting calculate totals from that inventory; +they do not read blobs or probe S3 or the filesystem for indexed images. Blob arrivals, +tag replacements, and successful deletions change the inventory, so totals survive +restarts and do not depend on the order of concurrent pulls. + +Images cached before object accounting are indexed on their first package listing. +That backfill reads their manifests and looks up uncatalogued blob metadata once. +Until a legacy image is indexed, size sorting uses its manifest-file size. Changes made +directly in the storage backend bypass accounting; manage cached content through Pkgly. + ## URL Structure Unlike other repository types, Docker repositories use a special URL structure to maintain compatibility with Docker clients: diff --git a/docs/docs/sysAdmin/index.md b/docs/docs/sysAdmin/index.md index 30fc573..0d3ff7e 100644 --- a/docs/docs/sysAdmin/index.md +++ b/docs/docs/sysAdmin/index.md @@ -18,7 +18,8 @@ Please use one of the following options for your build ### SSL After installation you can add SSL -Edit cfg/pkgly.toml +Edit cfg/pkgly.toml. A complete configuration example is available in the +[example config.toml](https://github.com/kshcherban/pkgly/blob/main/examples/config.toml). Under the application section diff --git a/docs/docs/sysAdmin/s3.md b/docs/docs/sysAdmin/s3.md index 91eae15..9833364 100644 --- a/docs/docs/sysAdmin/s3.md +++ b/docs/docs/sysAdmin/s3.md @@ -8,8 +8,12 @@ Use this guide to back Pkgly repositories with an Amazon S3 bucket or any S3-com - You run Pkgly in Kubernetes or another stateless platform and do not want to manage block volumes. - You already store artifacts in an S3-compatible service and want Pkgly to sit in front of that bucket. -**Limitations (as of this commit):** -- `move_file` is implemented as `GetObject` + `PutObject` + `DeleteObject`. Renaming large blobs consumes bandwidth proportional to object size. +**Operational behavior:** +- Moves use an ETag-guarded server-side `CopyObject`; objects larger than 5 GiB use multipart copy. The source is deleted only after a successful copy. If the guarded delete conflicts, both source and destination remain so no changed source data is lost. +- Generic storage appends use a read/modify/write sequence guarded by `If-Match` (or `If-None-Match: *` for creation). Docker upload sessions use local disk staging instead: chunks are streamed to a bounded temporary file and uploaded once when the digest is finalized. A conditional conflict is returned to the caller, which should retry with a fresh object version. +- Successful overwrites and appends invalidate the previous disk-cache entry; objects larger than + the cache capacity are not retained. +- Metadata lookups for disk-cached objects use the in-memory size, content type, and modification time without reading the cached body or contacting S3. Cache entries are verified by size and SHA-256 when written and during startup recovery; normal hits only perform a bounded file-type and size check. Cache misses use S3 metadata requests. ## Prerequisites @@ -21,16 +25,9 @@ Use this guide to back Pkgly repositories with an Amazon S3 bucket or any S3-com - `s3:GetObject` - `s3:PutObject` - `s3:DeleteObject` - - `s3:GetObjectTagging` - `s3:HeadObject` - **IAM role to assume** (`role_arn`, optional `role_session_name` / `external_id`). Pkgly calls `AssumeRole` through the AWS SDK and uses those temporary credentials for all S3 traffic. - **Instance/container profile** – leave every credential field blank and Pkgly will fall back to the AWS SDK default chain (environment variables, shared config/credentials files, ECS/EKS credentials, IMDS, etc.). - - `s3:ListBucket` - - `s3:GetObject` - - `s3:PutObject` - - `s3:DeleteObject` - - `s3:GetObjectTagging` - - `s3:HeadObject` 4. **Administrative login** that carries the `StorageManager` capability (system manager or admin) so you can call `/api/storage/**`. ## Bucket layout expectations @@ -48,7 +45,7 @@ curl -s -H "Authorization: Bearer $PKGLY_TOKEN" \ https://pkgly.example.com/api/storage/s3/regions | jq ``` -Pick one of the values (e.g., `UsEast1`). For S3-compatible endpoints that report a custom name (MinIO, Ceph), skip this list and plan to provide a custom endpoint instead. +Pick one of the canonical raw identifiers (for example, `us-east-1`). The list is only a convenience: Pkgly accepts any non-empty provider region string, including identifiers not yet listed. For S3-compatible endpoints that report a custom name (MinIO, Ceph), use a custom endpoint and optional signing-region label. ## Step 2 – Decide between path-style and virtual-hosted-style URLs @@ -68,7 +65,7 @@ curl -X POST https://pkgly.example.com/api/storage/new/s3 \ "type": "S3", "settings": { "bucket_name": "pkgly-prod-artifacts", - "region": "UsEast1", + "region": "us-east-1", "credentials": { "access_key": "AKIA...", "secret_key": "••••••••", @@ -96,6 +93,15 @@ Notes: - Credentials are stored encrypted at rest inside the `storages` table. Rotate them the same way you created them: create a new storage (or future update endpoint once available) and reassign repositories to it. - During creation Pkgly invokes `S3StorageFactory::test_storage_config()`. If anything fails (missing region, auth error, bucket not found) the API responds with `400 Invalid Storage Config` and the precise driver error appears in the response body and server logs. +### Region migration and rollback + +The region field is stored as a raw string. The migration translates all legacy enum tokens (such +as `UsEast1`) to canonical identifiers such as `us-east-1`; newly serialized configurations always +use the raw form. Unknown provider identifiers are accepted and preserved unchanged. Rolling the +migration back converts only the original 24 identifiers and deliberately leaves newer or custom +values untouched. An enum-only binary cannot read those untouched values, so change them manually +before downgrading to such a binary. + ### Custom endpoint example (MinIO, DigitalOcean Spaces, etc.) The S3 driver flattens the optional `CustomRegion` struct, so provide `custom_region` (name label, optional) and `endpoint` (full URL) directly inside `settings`. When `endpoint` is present it takes precedence over `region`. @@ -130,7 +136,7 @@ To keep hot artifacts on the node and avoid repeated S3 downloads, enable the ca "type": "S3", "settings": { "bucket_name": "pkgly-artifacts", - "region": "UsEast1", + "region": "us-east-1", "credentials": { "role_arn": "arn:aws:iam::123:role/pkgly" }, "path_style": true, "cache": { @@ -144,7 +150,34 @@ To keep hot artifacts on the node and avoid repeated S3 downloads, enable the ca } ``` -Leave `path` blank to fall back to `$(TMPDIR)/pkgly/s3-cache/`. The cache runs fully asynchronously—blocking file reads/writes are offloaded to Tokio’s blocking pool—so enabling it will not starve the HTTP runtime. +Leave `path` blank to fall back to `$(TMPDIR)/pkgly/s3-cache/`. Prefer a directory dedicated to one Pkgly S3 storage. The cache writes versioned content sidecars atomically, verifies size and SHA-256 on writes and restart recovery, and rebuilds its LRU index after restart while preserving unrelated files in the configured directory. + +Cache hits retain the S3 object `Last-Modified` timestamp. Files without a usable provider timestamp +fall back to the request time; virtual directories and common prefixes are always assigned one +request-time timestamp because S3 does not expose a directory creation time. Docker manifest indexes +are kept in a per-storage LRU cache for 30 seconds and are refreshed after that window or after a +successful mutation. + +The driver uses fixed internal deadlines for control traffic: 30 seconds per attempt and 90 seconds +for the complete control operation. Copy and multipart-copy requests allow five minutes per attempt +and 20 minutes for the operation. Streamed GET and PUT operations intentionally have no wall-clock +deadline so legitimate large transfers can finish; the SDK still stops stalled connections. + +The configured cache directory should be dedicated to this storage. Pkgly only removes recognized +cache sidecars, generation files, and legacy hash-layout artifacts, so unrelated files are retained, +but sharing a directory between multiple Pkgly caches can still cause avoidable churn. + +### Upload resource limits + +The S3 driver shares a 256 MiB process-wide budget for buffered bodies and streams larger or +unknown-length responses. Docker upload chunks are spooled incrementally, with at most 64 staged +uploads and 64 GiB of staged bytes per process. Incoming chunk files draw from the same 64 GiB +budget as staged bytes, and appends reserve their capacity before writing, so concurrent push +sessions cannot collectively exceed the limit; exceeding it rejects the request until capacity +frees up. Proxy downloads to temporary files reserve their advertised size upfront and fail fast +when the shared 64 GiB temporary-file budget is exhausted instead of blocking. Unfinished sessions +are in-memory and expire when the process restarts; graceful shutdown removes their temporary +files. Run one Pkgly writer for a repository when using Docker upload staging. ### IAM role / default chain example @@ -157,7 +190,7 @@ To avoid long-lived keys entirely, leave the key fields blank and provide only t "type": "S3", "settings": { "bucket_name": "pkgly-artifacts", - "region": "UsEast1", + "region": "us-east-1", "credentials": { "role_arn": "arn:aws:iam::123456789012:role/pkgly-deploy", "role_session_name": "pkgly-ci", @@ -187,14 +220,18 @@ To avoid long-lived keys entirely, leave the key fields blank and provide only t | Symptom | Likely cause | Fix | | --- | --- | --- | -| `Invalid Storage Config: NoRegionSpecified` | Neither `region` nor `endpoint` provided | Supply one of the supported regions or a custom endpoint | -| `AWS SDK error: InvalidAccessKeyId` | Wrong credentials or user lacks permission | Regenerate the key pair or fix the role policy so it has the required bucket-scoped IAM actions | +| `Invalid Storage Config: NoRegionSpecified` | Neither `region` nor `endpoint` provided | Supply a non-empty provider region identifier or a custom endpoint | +| `AWS SDK error (AccessDenied): InvalidAccessKeyId` | Wrong credentials or user lacks permission | Regenerate the key pair or fix the role policy so it has the required bucket-scoped IAM actions | | Cached file never expires | Cache size too small or directory not writable | Increase `cache.max_bytes` / `max_entries` or ensure Pkgly can write to the cache path | | `Bucket Does Not Exist` | Typo in `bucket_name` or Pkgly lacks access to the bucket | Verify the bucket name and the IAM policy’s `Resource` list | -| Uploads stall or time out | Pkgly copying large files via `move_file` or `append_file` | Avoid mass renames; delete + re-upload is faster until the driver adopts server-side copy | +| Uploads stall or time out | The endpoint is unreachable or a control request exceeded its deadline | Check endpoint reachability and logs; control operations use 30-second attempt and 90-second total limits, while streamed transfers have stalled-read protection without a wall-clock cap | ## Operational tips +- Repository GET/HEAD dispatch, Docker manifest refresh/download, and S3 read/write operations use heap-allocated futures to limit nested request stack frames. This also protects local-storage Docker proxies, whose shared storage interface includes S3 operations, in debug builds with the runtime's default worker stack size. - Keep the bucket lifecycle rules aligned with Pkgly retention policies. Pkgly never deletes repositories automatically, so lifecycle rules that expire objects will surface as 404s to clients. -- Monitor object count and size to detect runaway storages; Pkgly’s S3 driver currently lacks the optimized directory streaming used by the local backend, so list-heavy operations will cost extra API calls. +- Monitor object count and size to detect runaway storages. Directory and manifest listings consume paginated S3 LIST calls; Docker manifest indexes are cached per repository for up to 30 seconds and are invalidated after successful mutations. - Back up the `storages` table whenever you rotate credentials—losing it means Pkgly forgets how to talk to the bucket even though the data still exists. + +Direct uploads larger than 5 GiB, distributed cache invalidation across Pkgly nodes, automatic +egress-policy discovery, and cache metrics endpoints are outside this driver’s scope. diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100644 index eb4af31..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -exec /app/pkgly start --config /etc/pkgly/pkgly.toml diff --git a/pkgly/Cargo.toml b/pkgly/Cargo.toml index 1e8ace3..dfe2382 100644 --- a/pkgly/Cargo.toml +++ b/pkgly/Cargo.toml @@ -11,7 +11,6 @@ build = "build.rs" mockall = "0.14" pretty_assertions = "1.1" rsa = { version = "0.9" } -criterion = "0.5" testcontainers = "0.14" opentelemetry_sdk = { workspace = true, features = ["testing", "experimental_metrics_custom_reader"] } [dependencies] diff --git a/pkgly/src/app/api/repository/packages.rs b/pkgly/src/app/api/repository/packages.rs index ad13de4..c3f795d 100644 --- a/pkgly/src/app/api/repository/packages.rs +++ b/pkgly/src/app/api/repository/packages.rs @@ -21,8 +21,6 @@ use chrono::{DateTime, FixedOffset}; use http::header::HeaderValue; #[cfg(test)] use nr_storage::StorageFileMeta; -#[cfg(test)] -use nr_storage::s3::S3Storage; use nr_storage::{DynStorage, FileType, Storage, StorageError, StorageFile}; use serde::{Deserialize, Serialize}; #[cfg(test)] @@ -36,10 +34,6 @@ use tracing::{debug, instrument, warn}; use utoipa::{IntoParams, ToSchema}; use uuid::Uuid; -#[cfg(test)] -use crate::repository::docker::metadata::collect_manifest_entries; -#[cfg(test)] -use crate::repository::helm::HelmChartVersionExtra; use crate::{ app::{ Pkgly, @@ -52,9 +46,7 @@ use crate::{ DynRepository, Repository, docker::{ DockerRegistry, - metadata::{ - calculate_referenced_manifest_size, docker_package_key, split_manifest_cache_path, - }, + metadata::{backfill_manifest_objects, docker_package_key, split_manifest_cache_path}, types::{Manifest as DockerManifest, MediaType}, }, go::GoRepository, @@ -69,7 +61,7 @@ use crate::{ }; use ahash::{HashSet, HashSetExt}; #[cfg(test)] -use nr_core::repository::project::{CargoPackageMetadata, DebPackageMetadata, VersionData}; +use nr_core::repository::project::{CargoPackageMetadata, VersionData}; use nr_core::user::permissions::{HasPermissions, RepositoryActions}; #[cfg(test)] use nr_core::utils::base64_utils; @@ -471,10 +463,7 @@ async fn collect_go_package_page( #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum PackageStrategy { - #[allow(dead_code)] - PackagesDirectory { - base: Option<&'static str>, - }, + PackagesDirectory { base: Option<&'static str> }, MavenHosted, MavenProxy, PythonHosted, @@ -751,20 +740,22 @@ pub async fn list_cached_packages( }; if is_docker_repository { - let manifest_path = StoragePath::from(entry.cache_path.as_str()); - match calculate_referenced_manifest_size(&storage, repository.id(), &manifest_path) + if let Some(size) = row.referenced_size_bytes { + entry.size = size.max(0) as u64; + } else { + match backfill_manifest_objects( + &site.database, + &storage, + repository.id(), + &entry.cache_path, + ) .await - { - Ok(Some(size)) => { - entry.size = size; - } - Ok(None) => {} - Err(err) => { - warn!( - ?err, - cache_path = %entry.cache_path, - "Failed to calculate Docker referenced size" - ); + { + Ok(Some(size)) => entry.size = size, + Ok(None) => {} + Err(err) => { + warn!(?err, cache_path = %entry.cache_path, "Failed to backfill Docker object accounting") + } } } } else if is_maven_repository { @@ -838,10 +829,6 @@ async fn calculate_stored_path_size( } } -#[cfg(test)] -#[allow(dead_code)] -const MAX_STORAGE_CONCURRENCY: usize = 8; - #[cfg(test)] async fn map_ordered_concurrent( items: Vec, @@ -870,98 +857,6 @@ where Ok(ordered.into_iter().map(|(_, value)| value).collect()) } -#[cfg(test)] -#[allow(dead_code)] -async fn list_directory_packages( - repository: DynRepository, - page: usize, - per_page_raw: usize, - base: Option<&str>, - search: Option<&str>, -) -> Result { - let storage = repository.get_storage(); - let response = - collect_directory_package_page(&storage, repository.id(), base, page, per_page_raw, search) - .await?; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -fn file_name_from_path(path: &str) -> String { - path.rsplit('/') - .next() - .filter(|value| !value.is_empty()) - .unwrap_or(path) - .to_string() -} - -#[cfg(test)] -#[allow(dead_code)] -async fn load_single_file_entry( - storage: DynStorage, - repository_id: Uuid, - package: String, - cache_path: String, - updated_at: DateTime, -) -> Result { - let storage_path = nr_core::storage::StoragePath::from(cache_path.as_str()); - if let Some(StorageFile::File { meta, .. }) = - storage.open_file(repository_id, &storage_path).await? - { - return Ok(PackageFileEntry { - package, - name: file_name_from_path(&cache_path), - cache_path, - blob_digest: blob_digest_from_file_type(&meta.file_type), - size: meta.file_type.file_size, - modified: meta.modified, - }); - } - - Ok(PackageFileEntry { - package, - name: file_name_from_path(&cache_path), - cache_path, - blob_digest: None, - size: 0, - modified: updated_at, - }) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_directory_packages_s3( - repository_id: Uuid, - storage: S3Storage, - page: usize, - per_page_raw: usize, - base: Option<&str>, - search: Option<&str>, -) -> Result { - let objects = storage - .list_repository_objects(repository_id, base) - .await - .map_err(StorageError::from)?; - - let mut objects: Vec = objects - .into_iter() - .map(|obj| PackageObject { - key: obj.key, - size: obj.size, - modified: obj - .last_modified - .unwrap_or_else(|| chrono::Local::now().fixed_offset()), - }) - .collect(); - - // Ensure deterministic ordering independent of S3 pagination - objects.sort_by(|a, b| a.key.cmp(&b.key)); - - let response = build_package_page_from_objects(objects, base, page, per_page_raw, search); - - Ok(ResponseBuilder::ok().json(&response)) -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[cfg(test)] enum GoFileKind { @@ -1028,16 +923,6 @@ struct GoDeletionResult { missing: Vec, } -#[derive(sqlx::FromRow)] -#[cfg(test)] -#[allow(dead_code)] -struct DebPackageRow { - project_name: String, - version: String, - extra: SqlxJson, - created_at: DateTime, -} - #[derive(sqlx::FromRow)] #[cfg(test)] struct HostedCatalogRow { @@ -1119,68 +1004,6 @@ async fn fetch_maven_catalog_page( .await } -#[cfg(test)] -#[allow(dead_code)] -async fn fetch_maven_proxy_catalog_page( - database: &PgPool, - repository_id: Uuid, - per_page: usize, - offset: i64, - search: Option<&str>, -) -> Result, sqlx::Error> { - if let Some(term) = search { - let pattern = format!("%{}%", term.to_lowercase()); - return sqlx::query_as::<_, ProxyCatalogRow>( - r#" - SELECT - p.key AS project_key, - pv.version, - pv.path AS cache_path, - pv.extra AS version_data, - pv.updated_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE pv.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - ORDER BY LOWER(p.key) COLLATE "C", LOWER(pv.version) COLLATE "C" - LIMIT $3 OFFSET $4 - "#, - ) - .bind(repository_id) - .bind(pattern) - .bind(per_page as i64) - .bind(offset) - .fetch_all(database) - .await; - } - - sqlx::query_as::<_, ProxyCatalogRow>( - r#" - SELECT - p.key AS project_key, - pv.version, - pv.path AS cache_path, - pv.extra AS version_data, - pv.updated_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE pv.repository_id = $1 - ORDER BY LOWER(p.key) COLLATE "C", LOWER(pv.version) COLLATE "C" - LIMIT $2 OFFSET $3 - "#, - ) - .bind(repository_id) - .bind(per_page as i64) - .bind(offset) - .fetch_all(database) - .await -} - #[cfg(test)] async fn fetch_php_catalog_page( database: &PgPool, @@ -1193,70 +1016,6 @@ async fn fetch_php_catalog_page( fetch_maven_catalog_page(database, repository_id, per_page, offset, search).await } -#[cfg(test)] -#[allow(dead_code)] -async fn fetch_php_proxy_catalog_page( - database: &PgPool, - repository_id: Uuid, - per_page: usize, - offset: i64, - search: Option<&str>, -) -> Result, sqlx::Error> { - if let Some(term) = search { - let pattern = format!("%{}%", term.to_lowercase()); - return sqlx::query_as::<_, HostedCatalogRow>( - r#" - SELECT - p.key AS project_key, - pv.version AS version, - pv.path AS version_path, - pv.extra AS version_data, - pv.updated_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND (pv.extra->'extra'->>'size') IS NOT NULL - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - ORDER BY LOWER(p.key) COLLATE "C", LOWER(pv.version) COLLATE "C" - LIMIT $3 OFFSET $4 - "#, - ) - .bind(repository_id) - .bind(pattern) - .bind(per_page as i64) - .bind(offset) - .fetch_all(database) - .await; - } - - sqlx::query_as::<_, HostedCatalogRow>( - r#" - SELECT - p.key AS project_key, - pv.version AS version, - pv.path AS version_path, - pv.extra AS version_data, - pv.updated_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND (pv.extra->'extra'->>'size') IS NOT NULL - ORDER BY LOWER(p.key) COLLATE "C", LOWER(pv.version) COLLATE "C" - LIMIT $2 OFFSET $3 - "#, - ) - .bind(repository_id) - .bind(per_page as i64) - .bind(offset) - .fetch_all(database) - .await -} - #[cfg(test)] async fn fetch_npm_proxy_catalog_page( database: &PgPool, @@ -1329,14 +1088,6 @@ async fn fetch_proxy_catalog_page( .await } -#[cfg(test)] -#[allow(dead_code)] -fn deb_metadata(data: &VersionData) -> Option { - data.extra - .as_ref() - .and_then(|value| serde_json::from_value(value.clone()).ok()) -} - fn go_related_paths(path: &str) -> Option> { for suffix in GO_FILE_SUFFIXES.iter() { if let Some(base) = path.strip_suffix(suffix) { @@ -1656,1804 +1407,124 @@ fn build_go_entries_from_directory( versions.into_values().map(|(entry, _)| entry).collect() } -#[allow(dead_code)] #[cfg(test)] -async fn list_go_packages( - repository: DynRepository, - base: &str, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let storage = repository.get_storage(); - let response = - collect_go_package_page(&storage, repository.id(), base, page, per_page_raw, search) - .await?; - Ok(ResponseBuilder::ok().json(&response)) +fn build_cargo_package_entry( + crate_name: &str, + project_key: &str, + version: &str, + updated_at: DateTime, + metadata: &CargoPackageMetadata, +) -> PackageFileEntry { + PackageFileEntry { + package: crate_name.to_string(), + name: version.to_string(), + cache_path: cargo_cache_path(project_key, version), + blob_digest: normalize_sha256_digest(&metadata.checksum), + size: metadata.crate_size, + modified: updated_at, + } } #[cfg(test)] -#[allow(dead_code)] -async fn list_helm_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); +fn cargo_cache_path(project_key: &str, version: &str) -> String { + format!( + "crates/{key}/{ver}/{key}-{ver}.crate", + key = project_key, + ver = version + ) +} - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository.id()) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository.id()) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - if offset >= total_versions { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - let rows = if let Some(pattern) = &search_pattern { - sqlx::query( - r#" - SELECT - p.name AS chart_name, - pv.version, - pv.path, - pv.extra, - pv.updated_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - ORDER BY p.name ASC, pv.version ASC - LIMIT $3 OFFSET $4 - "#, - ) - .bind(repository.id()) - .bind(pattern) - .bind(per_page as i64) - .bind(offset) - .fetch_all(&site.database) - .await? - } else { - sqlx::query( - r#" - SELECT - p.name AS chart_name, - pv.version, - pv.path, - pv.extra, - pv.updated_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - ORDER BY p.name ASC, pv.version ASC - LIMIT $2 OFFSET $3 - "#, - ) - .bind(repository.id()) - .bind(per_page as i64) - .bind(offset) - .fetch_all(&site.database) - .await? - }; - - let mut items = Vec::with_capacity(rows.len()); - - for row in rows { - let chart_name: String = row.try_get("chart_name")?; - let version: String = row.try_get("version")?; - let path: String = row.try_get("path")?; - let version_data: sqlx::types::Json = row.try_get("extra")?; - let updated_at: DateTime = row.try_get("updated_at")?; - - let Some(extra_value) = version_data.0.extra else { - debug!( - chart = %chart_name, - version = %version, - "Skipping Helm version without extra metadata" - ); - continue; - }; - let chart_extra: HelmChartVersionExtra = serde_json::from_value(extra_value)?; - let cache_path = if chart_extra.canonical_path.is_empty() { - path.clone() - } else { - chart_extra.canonical_path.clone() - }; - items.push(PackageFileEntry { - package: chart_name, - name: version, - cache_path, - blob_digest: Some(chart_extra.digest), - size: chart_extra.size_bytes, - modified: updated_at, - }); - } - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_cargo_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository.id()) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository.id()) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - if offset >= total_versions { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - let rows = if let Some(pattern) = &search_pattern { - sqlx::query( - r#" - SELECT - p.name AS crate_name, - p.key AS project_key, - pv.version AS version, - pv.extra AS extra, - pv.updated_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - ORDER BY p.name ASC, pv.version ASC - LIMIT $3 OFFSET $4 - "#, - ) - .bind(repository.id()) - .bind(pattern) - .bind(per_page as i64) - .bind(offset) - .fetch_all(&site.database) - .await? - } else { - sqlx::query( - r#" - SELECT - p.name AS crate_name, - p.key AS project_key, - pv.version AS version, - pv.extra AS extra, - pv.updated_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - ORDER BY p.name ASC, pv.version ASC - LIMIT $2 OFFSET $3 - "#, - ) - .bind(repository.id()) - .bind(per_page as i64) - .bind(offset) - .fetch_all(&site.database) - .await? - }; - - let mut items = Vec::with_capacity(rows.len()); - - for row in rows { - let crate_name: String = row.try_get("crate_name")?; - let project_key: String = row.try_get("project_key")?; - let version: String = row.try_get("version")?; - let version_data: sqlx::types::Json = row.try_get("extra")?; - let updated_at: DateTime = row.try_get("updated_at")?; - - let VersionData { extra, .. } = version_data.0; - let Some(extra_value) = extra else { - debug!( - crate = %crate_name, - version = %version, - "Skipping Cargo version without metadata" - ); - continue; - }; - let metadata: CargoPackageMetadata = serde_json::from_value(extra_value)?; - items.push(build_cargo_package_entry( - &crate_name, - &project_key, - &version, - updated_at, - &metadata, - )); - } - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -fn build_cargo_package_entry( - crate_name: &str, - project_key: &str, - version: &str, - updated_at: DateTime, - metadata: &CargoPackageMetadata, -) -> PackageFileEntry { - PackageFileEntry { - package: crate_name.to_string(), - name: version.to_string(), - cache_path: cargo_cache_path(project_key, version), - blob_digest: normalize_sha256_digest(&metadata.checksum), - size: metadata.crate_size, - modified: updated_at, - } -} - -#[cfg(test)] -fn cargo_cache_path(project_key: &str, version: &str) -> String { - format!( - "crates/{key}/{ver}/{key}-{ver}.crate", - key = project_key, - ver = version - ) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_npm_proxy_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE pv.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - WHERE pv.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if offset >= total_versions { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - let rows = - fetch_npm_proxy_catalog_page(&site.database, repository_id, per_page, offset, search) - .await?; - - let items: Vec = rows.iter().filter_map(proxy_entry_from_row).collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_npm_hosted_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if offset >= total_versions { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - let rows = - fetch_maven_catalog_page(&site.database, repository_id, per_page, offset, search).await?; - - let storage = repository.get_storage(); - let items = map_ordered_concurrent(rows, MAX_STORAGE_CONCURRENCY, move |row| { - let storage = storage.clone(); - async move { - load_single_file_entry( - storage, - repository_id, - row.project_key.clone(), - row.version_path.clone(), - row.updated_at, - ) - .await - } - }) - .await?; - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_npm_virtual_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - list_npm_hosted_packages(site, repository, page, per_page_raw, search).await -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_python_hosted_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if offset >= total_versions { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - let rows = - fetch_maven_catalog_page(&site.database, repository_id, per_page, offset, search).await?; - - let storage = repository.get_storage(); - let items = map_ordered_concurrent(rows, MAX_STORAGE_CONCURRENCY, move |row| { - let storage = storage.clone(); - async move { - load_single_file_entry( - storage, - repository_id, - row.project_key.clone(), - row.version_path.clone(), - row.updated_at, - ) - .await - } - }) - .await?; - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_python_proxy_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if offset >= total_versions { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - let rows = - fetch_proxy_catalog_page(&site.database, repository_id, per_page, offset, search).await?; - - let items: Vec = rows.iter().filter_map(proxy_entry_from_row).collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_go_catalog_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if offset >= total_versions { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - let rows = - fetch_maven_catalog_page(&site.database, repository_id, per_page, offset, search).await?; - - let storage = repository.get_storage(); - let items = map_ordered_concurrent(rows, MAX_STORAGE_CONCURRENCY, move |row| { - let storage = storage.clone(); - async move { - let mut entry = load_single_file_entry( - storage, - repository_id, - row.project_key.clone(), - row.version_path.clone(), - row.updated_at, - ) - .await?; - entry.name = row.version.clone(); - Ok::<_, InternalError>(entry) - } - }) - .await?; - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -fn go_entry_from_proxy_row(row: &ProxyCatalogRow) -> Option { - let (size, modified, blob_digest) = match row.version_data.0.proxy_artifact() { - Some(proxy_meta) => ( - proxy_meta.size.unwrap_or_default(), - DateTime::::from(proxy_meta.fetched_at), - proxy_meta.upstream_digest.clone(), - ), - None => (0, row.updated_at, None), - }; - Some(PackageFileEntry { - package: row.project_key.clone(), - name: row.version.clone(), - cache_path: row.cache_path.clone(), - blob_digest, - size, - modified, - }) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_go_proxy_catalog_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if offset >= total_versions { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - let rows = - fetch_proxy_catalog_page(&site.database, repository_id, per_page, offset, search).await?; - let items = rows - .iter() - .filter_map(go_entry_from_proxy_row) - .collect::>(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -fn docker_entry_from_row(row: &ProxyCatalogRow) -> Option { - let (package, size, modified, blob_digest) = match row.version_data.0.proxy_artifact() { - Some(proxy_meta) => ( - proxy_meta.package_name, - proxy_meta.size.unwrap_or_default(), - DateTime::::from(proxy_meta.fetched_at), - proxy_meta.upstream_digest.clone(), - ), - None => ( - row.project_key.clone(), - 0, - row.updated_at, - if row.version.starts_with("sha256:") { - Some(row.version.clone()) - } else { - None - }, - ), - }; - Some(PackageFileEntry { - package, - name: row.version.clone(), - cache_path: row.cache_path.clone(), - blob_digest, - size, - modified, - }) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_docker_catalog_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if offset >= total_versions { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - let rows = - fetch_proxy_catalog_page(&site.database, repository_id, per_page, offset, search).await?; - - let items: Vec = rows.iter().filter_map(docker_entry_from_row).collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_maven_hosted_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - if offset >= total_versions { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - let rows = - fetch_maven_catalog_page(&site.database, repository_id, per_page, offset, search).await?; - - let storage = repository.get_storage(); - let version_chunks = map_ordered_concurrent(rows, MAX_STORAGE_CONCURRENCY, move |row| { - let storage = storage.clone(); - async move { load_maven_version_entries(storage, repository_id, row).await } - }) - .await?; - - let items: Vec = version_chunks.into_iter().flatten().collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_php_hosted_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - if offset >= total_versions { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - let rows = - fetch_php_catalog_page(&site.database, repository_id, per_page, offset, search).await?; - - let storage = repository.get_storage(); - let version_chunks = map_ordered_concurrent(rows, MAX_STORAGE_CONCURRENCY, move |row| { - let storage = storage.clone(); - async move { load_php_version_entries(storage, repository_id, row).await } - }) - .await?; - - let items: Vec = version_chunks.into_iter().flatten().collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn load_maven_version_entries( - storage: DynStorage, - repository_id: Uuid, - row: HostedCatalogRow, -) -> Result, InternalError> { - let HostedCatalogRow { - project_key, - version, - version_path, - version_data, - updated_at: _updated_at, - } = row; - let version_data = version_data.0; - let package_label = format!("{}:{}", project_key, version); - let cache_prefix = version_path.trim_end_matches('/'); - let normalized_path = ensure_trailing_slash(&version_path); - let storage_path = nr_core::storage::StoragePath::from(normalized_path); - - if let Some(StorageFile::Directory { files, .. }) = - storage.open_file(repository_id, &storage_path).await? - { - let mut file_entries: Vec<_> = files.iter().collect(); - file_entries.sort_by(|a, b| a.name().cmp(b.name())); - - let mut items = Vec::new(); - for meta in file_entries { - if should_ignore(meta.name()) { - continue; - } - if let FileType::File(file_meta) = meta.file_type() { - let cache_path = if cache_prefix.is_empty() { - meta.name().to_string() - } else { - format!("{cache_prefix}/{}", meta.name()) - }; - items.push(PackageFileEntry { - package: package_label.clone(), - name: meta.name().to_string(), - cache_path, - blob_digest: blob_digest_from_file_type(file_meta), - size: file_meta.file_size, - modified: meta.modified().clone(), - }); - } - } - return Ok(items); - } - - if let Some(proxy_meta) = version_data.proxy_artifact() { - let modified: DateTime = proxy_meta.fetched_at.into(); - let file_name = proxy_meta - .cache_path - .rsplit('/') - .next() - .unwrap_or(&proxy_meta.cache_path) - .to_string(); - return Ok(vec![PackageFileEntry { - package: package_label, - name: file_name, - cache_path: proxy_meta.cache_path.clone(), - blob_digest: proxy_meta.upstream_digest.clone(), - size: proxy_meta.size.unwrap_or_default(), - modified, - }]); - } - - let direct_path = nr_core::storage::StoragePath::from(version_path.as_str()); - if let Some(StorageFile::File { meta, .. }) = - storage.open_file(repository_id, &direct_path).await? - { - let name = version_path - .rsplit('/') - .next() - .unwrap_or(&version_path) - .to_string(); - return Ok(vec![PackageFileEntry { - package: package_label, - name, - cache_path: version_path, - blob_digest: blob_digest_from_file_type(&meta.file_type), - size: meta.file_type.file_size, - modified: meta.modified, - }]); - } - - Ok(Vec::new()) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn load_maven_proxy_version_entries( - storage: DynStorage, - repository_id: Uuid, - row: ProxyCatalogRow, -) -> Result, InternalError> { - let ProxyCatalogRow { - project_key, - version, - cache_path, - version_data, - updated_at, - } = row; - let version_data = version_data.0; - let package_label = format!("{}:{}", project_key, version); - let cache_prefix = cache_path.trim_end_matches('/'); - let normalized_path = ensure_trailing_slash(&cache_path); - let storage_path = nr_core::storage::StoragePath::from(normalized_path); - - if let Some(StorageFile::Directory { files, .. }) = - storage.open_file(repository_id, &storage_path).await? - { - let mut file_entries: Vec<_> = files.iter().collect(); - file_entries.sort_by(|a, b| a.name().cmp(b.name())); - - let mut items = Vec::new(); - for meta in file_entries { - if should_ignore(meta.name()) { - continue; - } - if let FileType::File(file_meta) = meta.file_type() { - let child_path = if cache_prefix.is_empty() { - meta.name().to_string() - } else { - format!("{cache_prefix}/{}", meta.name()) - }; - items.push(PackageFileEntry { - package: package_label.clone(), - name: meta.name().to_string(), - cache_path: child_path, - blob_digest: blob_digest_from_file_type(file_meta), - size: file_meta.file_size, - modified: meta.modified().clone(), - }); - } - } - return Ok(items); - } - - if let Some(proxy_meta) = version_data.proxy_artifact() { - let modified: DateTime = proxy_meta.fetched_at.into(); - return Ok(vec![PackageFileEntry { - package: package_label, - name: file_name_from_path(&proxy_meta.cache_path), - cache_path: proxy_meta.cache_path, - blob_digest: proxy_meta.upstream_digest, - size: proxy_meta.size.unwrap_or_default(), - modified, - }]); - } - - let direct_path = nr_core::storage::StoragePath::from(cache_path.as_str()); - if let Some(StorageFile::File { meta, .. }) = - storage.open_file(repository_id, &direct_path).await? - { - return Ok(vec![PackageFileEntry { - package: package_label, - name: file_name_from_path(&cache_path), - cache_path, - blob_digest: blob_digest_from_file_type(&meta.file_type), - size: meta.file_type.file_size, - modified: meta.modified, - }]); - } - - let _ = updated_at; - Ok(Vec::new()) -} - -#[cfg(test)] -async fn load_php_version_entries( - storage: DynStorage, - repository_id: Uuid, - row: HostedCatalogRow, -) -> Result, InternalError> { - let HostedCatalogRow { - project_key, - version, - version_path, - version_data, - updated_at, - } = row; - - if let Some(proxy_meta) = version_data.0.proxy_artifact() { - // PHP proxy: rely on catalog metadata and only surface entries - // for versions that have a recorded cache size, which is set - // when the dist is actually cached. - if let Some(size) = proxy_meta.size { - let modified: DateTime = proxy_meta.fetched_at.into(); - let label = proxy_meta.version.clone().unwrap_or_else(|| { - proxy_meta - .cache_path - .rsplit('/') - .next() - .unwrap_or(&proxy_meta.cache_path) - .to_string() - }); - return Ok(vec![PackageFileEntry { - package: proxy_meta.package_key.clone(), - name: label, - cache_path: proxy_meta.cache_path.clone(), - blob_digest: proxy_meta.upstream_digest.clone(), - size, - modified, - }]); - } - // Metadata-only proxy rows (no cached dist) are hidden from the list. - return Ok(Vec::new()); - } - - // PHP hosted: list only versions that have a dist file in storage. - let storage_path = nr_core::storage::StoragePath::from(version_path.as_str()); - if let Some(StorageFile::File { meta, .. }) = - storage.open_file(repository_id, &storage_path).await? - { - return Ok(vec![PackageFileEntry { - package: project_key, - name: version, - cache_path: version_path, - blob_digest: blob_digest_from_file_type(&meta.file_type), - size: meta.file_type.file_size, - modified: meta.modified, - }]); - } - - let modified: DateTime = updated_at; - Ok(vec![PackageFileEntry { - package: project_key, - name: version, - cache_path: version_path, - blob_digest: None, - size: 0, - modified, - }]) -} - -#[cfg(test)] -fn proxy_entry_from_row(row: &ProxyCatalogRow) -> Option { - let (cache_path, size, modified, blob_digest) = match row.version_data.0.proxy_artifact() { - Some(proxy_meta) => ( - proxy_meta.cache_path, - proxy_meta.size.unwrap_or_default(), - DateTime::::from(proxy_meta.fetched_at), - proxy_meta.upstream_digest, - ), - None => (row.cache_path.clone(), 0, row.updated_at, None), - }; - - Some(PackageFileEntry { - package: row.project_key.clone(), - name: file_name_from_path(&cache_path), - cache_path, - blob_digest, - size, - modified, - }) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_deb_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 OR - LOWER(COALESCE(pv.extra::text, '')) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository.id()) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - "#, - ) - .bind(repository.id()) - .fetch_one(&site.database) - .await? - }; - - if total_versions == 0 { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if offset >= total_versions { - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - let rows = if let Some(pattern) = &search_pattern { - sqlx::query_as::<_, DebPackageRow>( - r#" - SELECT - p.name AS project_name, - pv.version, - pv.extra, - pv.created_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 OR - LOWER(COALESCE(pv.extra::text, '')) COLLATE "C" LIKE $2 - ) - ORDER BY pv.created_at DESC - LIMIT $3 OFFSET $4 - "#, - ) - .bind(repository.id()) - .bind(pattern) - .bind(per_page as i64) - .bind(offset) - .fetch_all(&site.database) - .await? - } else { - sqlx::query_as::<_, DebPackageRow>( - r#" - SELECT - p.name AS project_name, - pv.version, - pv.extra, - pv.created_at - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - ORDER BY pv.created_at DESC - LIMIT $2 OFFSET $3 - "#, - ) - .bind(repository.id()) - .bind(per_page as i64) - .bind(offset) - .fetch_all(&site.database) - .await? - }; - - let mut items = Vec::new(); - for row in rows { - if let Some(metadata) = deb_metadata(&row.extra.0) { - items.push(PackageFileEntry { - package: row.project_name.clone(), - name: row.version.clone(), - cache_path: metadata.filename.clone(), - blob_digest: normalize_sha256_digest(&metadata.sha256), - size: metadata.size, - modified: row.created_at, - }); - } - } - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - -#[cfg(test)] -#[allow(dead_code)] -async fn list_php_proxy_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND (pv.extra->'extra'->>'size') IS NOT NULL - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE p.repository_id = $1 - AND (pv.extra->'extra'->>'size') IS NOT NULL - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? +#[cfg(test)] +fn proxy_entry_from_row(row: &ProxyCatalogRow) -> Option { + let (cache_path, size, modified, blob_digest) = match row.version_data.0.proxy_artifact() { + Some(proxy_meta) => ( + proxy_meta.cache_path, + proxy_meta.size.unwrap_or_default(), + DateTime::::from(proxy_meta.fetched_at), + proxy_meta.upstream_digest, + ), + None => (row.cache_path.clone(), 0, row.updated_at, None), }; - if total_versions == 0 { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - if offset >= total_versions { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - let rows = - fetch_php_proxy_catalog_page(&site.database, repository_id, per_page, offset, search) - .await?; - - let storage = repository.get_storage(); - let version_chunks = map_ordered_concurrent(rows, MAX_STORAGE_CONCURRENCY, move |row| { - let storage = storage.clone(); - async move { load_php_version_entries(storage, repository_id, row).await } + let name = cache_path + .rsplit('/') + .next() + .filter(|value| !value.is_empty()) + .unwrap_or(&cache_path) + .to_string(); + Some(PackageFileEntry { + package: row.project_key.clone(), + name, + cache_path, + blob_digest, + size, + modified, }) - .await?; - - let items: Vec = version_chunks.into_iter().flatten().collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) } #[cfg(test)] -#[allow(dead_code)] -async fn list_maven_proxy_packages( - site: Pkgly, - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let offset = ((current_page - 1) * per_page) as i64; - let repository_id = repository.id(); - let search_pattern = search.map(|term| format!("%{}%", term.to_lowercase())); - - let total_versions: i64 = if let Some(pattern) = &search_pattern { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - INNER JOIN projects p ON pv.project_id = p.id - WHERE pv.repository_id = $1 - AND ( - LOWER(p.name) COLLATE "C" LIKE $2 OR - LOWER(p.key) COLLATE "C" LIKE $2 OR - LOWER(pv.version) COLLATE "C" LIKE $2 OR - LOWER(pv.path) COLLATE "C" LIKE $2 - ) - "#, - ) - .bind(repository_id) - .bind(pattern) - .fetch_one(&site.database) - .await? - } else { - sqlx::query_scalar( - r#" - SELECT COUNT(*) - FROM project_versions pv - WHERE pv.repository_id = $1 - "#, - ) - .bind(repository_id) - .fetch_one(&site.database) - .await? - }; +async fn load_php_version_entries( + storage: DynStorage, + repository_id: Uuid, + row: HostedCatalogRow, +) -> Result, InternalError> { + let HostedCatalogRow { + project_key, + version, + version_path, + version_data, + updated_at, + } = row; - if total_versions == 0 { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: 0, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); + if let Some(proxy_meta) = version_data.0.proxy_artifact() { + // PHP proxy: rely on catalog metadata and only surface entries + // for versions that have a recorded cache size, which is set + // when the dist is actually cached. + if let Some(size) = proxy_meta.size { + let modified: DateTime = proxy_meta.fetched_at.into(); + let label = proxy_meta.version.clone().unwrap_or_else(|| { + proxy_meta + .cache_path + .rsplit('/') + .next() + .unwrap_or(&proxy_meta.cache_path) + .to_string() + }); + return Ok(vec![PackageFileEntry { + package: proxy_meta.package_key.clone(), + name: label, + cache_path: proxy_meta.cache_path.clone(), + blob_digest: proxy_meta.upstream_digest.clone(), + size, + modified, + }]); + } + // Metadata-only proxy rows (no cached dist) are hidden from the list. + return Ok(Vec::new()); } - if offset >= total_versions { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); + let storage_path = nr_core::storage::StoragePath::from(version_path.as_str()); + if let Some(StorageFile::File { meta, .. }) = + storage.open_file(repository_id, &storage_path).await? + { + return Ok(vec![PackageFileEntry { + package: project_key, + name: version, + cache_path: version_path, + blob_digest: blob_digest_from_file_type(&meta.file_type), + size: meta.file_type.file_size, + modified: meta.modified, + }]); } - let rows = - fetch_maven_proxy_catalog_page(&site.database, repository_id, per_page, offset, search) - .await?; - let storage = repository.get_storage(); - let version_chunks = map_ordered_concurrent(rows, MAX_STORAGE_CONCURRENCY, move |row| { - let storage = storage.clone(); - async move { load_maven_proxy_version_entries(storage, repository_id, row).await } - }) - .await?; - - let items: Vec = version_chunks.into_iter().flatten().collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages: total_versions as usize, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) + Ok(vec![PackageFileEntry { + package: project_key, + name: version, + cache_path: version_path, + blob_digest: None, + size: 0, + modified: updated_at, + }]) } #[allow(dead_code)] @@ -3600,172 +1671,6 @@ fn derive_maven_package_label(path: &str) -> Option { Some(format!("{project_key}:{version}")) } -#[allow(dead_code)] -#[cfg(test)] -async fn list_docker_packages( - repository: DynRepository, - page: usize, - per_page_raw: usize, - search: Option<&str>, -) -> Result { - let storage = repository.get_storage(); - let per_page = per_page_raw.clamp(1, MAX_PER_PAGE); - let current_page = page.max(1); - let start = (current_page - 1) * per_page; - let search_term = search.map(|value| value.to_lowercase()); - - if search_term.is_some() { - let mut manifests = collect_manifest_entries(&storage, repository.id()) - .await - .map_err(InternalError::from)?; - - manifests.sort_by(|a, b| { - a.repository - .cmp(&b.repository) - .then(a.reference.cmp(&b.reference)) - }); - - let mut entries: Vec = manifests - .into_iter() - .filter_map(|entry| { - let pkg = PackageFileEntry { - package: entry.repository.clone(), - name: entry.reference.clone(), - cache_path: entry.cache_path.clone(), - blob_digest: if entry.reference.starts_with("sha256:") { - Some(entry.reference.clone()) - } else { - None - }, - size: entry.size, - modified: entry.modified, - }; - if let Some(term) = &search_term { - if !matches_search(&pkg, term) { - return None; - } - } - Some(pkg) - }) - .collect(); - - let total_packages = entries.len(); - if total_packages == 0 || start >= total_packages { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - let end = min(start + per_page, total_packages); - let items = entries.drain(start..end).collect(); - let response = PackageListResponse { - page: current_page, - per_page, - total_packages, - items, - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - if let DynStorage::S3(s3_storage) = storage.clone() { - let (manifests, total_packages) = s3_storage - .list_docker_manifests_paginated(repository.id(), start, per_page) - .await - .map_err(StorageError::from)?; - - if total_packages == 0 || start >= total_packages { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - let now = chrono::Local::now().fixed_offset(); - let items = manifests - .into_iter() - .filter_map(|obj| { - let repo_relative = obj.key.strip_prefix("v2/")?.to_string(); - let (repository, reference) = repo_relative.split_once("/manifests/")?; - Some(PackageFileEntry { - package: repository.to_string(), - name: reference.to_string(), - cache_path: obj.key, - blob_digest: if reference.starts_with("sha256:") { - Some(reference.to_string()) - } else { - None - }, - size: obj.size, - modified: obj.last_modified.unwrap_or(now), - }) - }) - .collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages, - items, - }; - return Ok(ResponseBuilder::ok().json(&response)); - } - - // Fallback for non-S3 storage: load manifests into memory (local FS) - let mut manifests = collect_manifest_entries(&storage, repository.id()) - .await - .map_err(InternalError::from)?; - - manifests.sort_by(|a, b| { - a.repository - .cmp(&b.repository) - .then(a.reference.cmp(&b.reference)) - }); - - let total_packages = manifests.len(); - - if total_packages == 0 || start >= total_packages { - let empty = PackageListResponse { - page: current_page, - per_page, - total_packages, - items: Vec::new(), - }; - return Ok(ResponseBuilder::ok().json(&empty)); - } - - let end = min(start + per_page, total_packages); - let items = manifests[start..end] - .iter() - .map(|entry| PackageFileEntry { - package: entry.repository.clone(), - name: entry.reference.clone(), - cache_path: entry.cache_path.clone(), - blob_digest: if entry.reference.starts_with("sha256:") { - Some(entry.reference.clone()) - } else { - None - }, - size: entry.size, - modified: entry.modified, - }) - .collect(); - - let response = PackageListResponse { - page: current_page, - per_page, - total_packages, - items, - }; - Ok(ResponseBuilder::ok().json(&response)) -} - fn is_valid_cache_path(path: &str, strategy: PackageStrategy) -> bool { match strategy { PackageStrategy::PackagesDirectory { base } => { @@ -3841,6 +1746,8 @@ pub enum DockerDeletionError { InvalidManifest(String), #[error("indexing error: {0}")] Indexing(#[from] ProxyIndexingError), + #[error("accounting error: {0}")] + Accounting(#[from] sqlx::Error), } fn docker_proxy_key_from_path(path: &str) -> Option { @@ -3852,11 +1759,14 @@ fn docker_proxy_key_from_path(path: &str) -> Option { }) } +/// Deletes a Docker manifest graph and removes successfully deleted inventory entries. +/// Returns storage, catalog, or accounting errors without suppressing failures. pub async fn delete_docker_package( storage: &nr_storage::DynStorage, repository_id: Uuid, cache_path: &str, indexer: Option<&dyn ProxyIndexing>, + database: Option<&PgPool>, ) -> Result { let (repository_name, _) = split_manifest_cache_path(cache_path).ok_or(DockerDeletionError::InvalidManifestPath)?; @@ -3906,6 +1816,15 @@ pub async fn delete_docker_package( .delete_files_batch(repository_id, &paths_vec) .await?; + if let Some(database) = database { + nr_core::database::entities::docker_object::DBDockerObject::delete_paths( + database, + repository_id, + &paths_to_delete.iter().cloned().collect::>(), + ) + .await?; + } + // Count manifests vs blobs for the result let manifest_count = paths_to_delete .iter() @@ -3943,6 +1862,7 @@ struct StreamingDockerBatchDeletion { deleted_packages: usize, deleted_objects: usize, indexer: Option>, + database: Option, } impl StreamingDockerBatchDeletion { @@ -3962,6 +1882,7 @@ impl StreamingDockerBatchDeletion { deleted_packages: 0, deleted_objects: 0, indexer, + database: None, } } @@ -3990,6 +1911,14 @@ impl StreamingDockerBatchDeletion { .delete_files_batch(self.repository_id, &paths) .await?; + if let Some(database) = &self.database { + nr_core::database::entities::docker_object::DBDockerObject::delete_paths( + database, + self.repository_id, + &drained, + ) + .await?; + } self.deleted_objects += deleted; Ok(()) } @@ -4025,8 +1954,10 @@ async fn collect_docker_deletions_batch( repository_id: Uuid, paths: &[String], indexer: Option>, + database: Option<&PgPool>, ) -> Result { let mut batch = StreamingDockerBatchDeletion::new(storage, repository_id, indexer); + batch.database = database.cloned(); for path in paths { if !is_valid_docker_manifest_path(path) { @@ -4337,10 +2268,15 @@ pub async fn delete_cached_package_paths( PackageStrategy::DockerHosted | PackageStrategy::DockerProxy ) { let docker_indexer = docker_proxy.as_ref().map(|proxy| proxy.indexer().clone()); - let batch = - collect_docker_deletions_batch(&storage, repository.id(), paths, docker_indexer) - .await - .map_err(|err| InternalError::from(OtherInternalError::new(err)))?; + let batch = collect_docker_deletions_batch( + &storage, + repository.id(), + paths, + docker_indexer, + Some(&site.database), + ) + .await + .map_err(|err| InternalError::from(OtherInternalError::new(err)))?; debug!( paths = paths.len(), diff --git a/pkgly/src/app/api/repository/packages/tests.rs b/pkgly/src/app/api/repository/packages/tests.rs index c551435..f29afc1 100644 --- a/pkgly/src/app/api/repository/packages/tests.rs +++ b/pkgly/src/app/api/repository/packages/tests.rs @@ -607,7 +607,7 @@ async fn delete_docker_manifest_removes_all_payloads() -> Result<()> { let tag_cache_path = tag_path.to_string(); let result = - delete_docker_package(&storage, repository_id, tag_cache_path.as_str(), None).await?; + delete_docker_package(&storage, repository_id, tag_cache_path.as_str(), None, None).await?; assert_eq!(result.removed_manifests, 2); assert_eq!(result.removed_blobs, 3); @@ -681,8 +681,14 @@ async fn delete_docker_manifest_handles_digest_path() -> Result<()> { } let digest_cache_path = digest_path.to_string(); - let result = - delete_docker_package(&storage, repository_id, digest_cache_path.as_str(), None).await?; + let result = delete_docker_package( + &storage, + repository_id, + digest_cache_path.as_str(), + None, + None, + ) + .await?; assert_eq!(result.removed_manifests, 1); assert_eq!(result.removed_blobs, 2); @@ -765,6 +771,7 @@ async fn delete_docker_package_notifies_indexer() -> Result<()> { repository_id, tag_path.to_string().as_str(), Some(indexer.as_ref()), + None, ) .await?; @@ -860,6 +867,7 @@ async fn collect_docker_deletions_batch_deduplicates_shared_layers() -> Result<( repository_id, &tag_paths.iter().cloned().collect::>(), None, + None, ) .await?; @@ -967,7 +975,7 @@ async fn collect_docker_deletions_batch_streams_large_batches() -> Result<()> { } let batch = - super::collect_docker_deletions_batch(&storage, repository_id, &manifest_paths, None) + super::collect_docker_deletions_batch(&storage, repository_id, &manifest_paths, None, None) .await?; assert!(batch.deleted_objects > 0); @@ -1444,6 +1452,126 @@ mod catalog_db_tests { db } + #[tokio::test] + async fn docker_object_accounting_tracks_arrivals_deletion_and_tag_replacement() { + use nr_core::database::entities::docker_object::DBDockerObject; + let _guard = DB_TEST_LOCK.lock().await; + let db = fresh_pool().await; + let storage = insert_storage(db.pool()).await; + let repository = insert_docker_repository(db.pool(), storage).await; + let root = "v2/image/manifests/latest"; + let child = "v2/image/manifests/sha256:child"; + let blob = "v2/image/blobs/sha256:blob"; + DBDockerObject::upsert( + db.pool(), + repository, + root, + 10, + &[child.into(), blob.into()], + ) + .await + .unwrap(); + assert_eq!( + DBDockerObject::referenced_size(db.pool(), repository, root) + .await + .unwrap(), + Some(10) + ); + assert!( + DBDockerObject::needs_backfill(db.pool(), repository, root) + .await + .unwrap() + ); + DBDockerObject::upsert( + db.pool(), + repository, + child, + 20, + &[blob.into(), root.into()], + ) + .await + .unwrap(); + DBDockerObject::upsert(db.pool(), repository, blob, 30, &[]) + .await + .unwrap(); + assert_eq!( + DBDockerObject::referenced_size(db.pool(), repository, root) + .await + .unwrap(), + Some(60) + ); + assert!( + !DBDockerObject::needs_backfill(db.pool(), repository, root) + .await + .unwrap() + ); + DBDockerObject::insert_missing(db.pool(), repository, root, 999, &[]) + .await + .unwrap(); + assert_eq!( + DBDockerObject::referenced_size(db.pool(), repository, root) + .await + .unwrap(), + Some(60) + ); + let roots: Vec = (0..5) + .map(|n| format!("v2/concurrent/manifests/{n}")) + .collect(); + for path in &roots { + DBDockerObject::upsert(db.pool(), repository, path, 1, &[blob.into()]) + .await + .unwrap(); + } + let writes = roots + .iter() + .map(|path| DBDockerObject::upsert(db.pool(), repository, path, 2, &[])); + for result in futures::future::join_all(writes).await { + result.unwrap(); + } + for path in roots { + assert_eq!( + DBDockerObject::referenced_size(db.pool(), repository, &path) + .await + .unwrap(), + Some(2) + ); + } + assert!( + DBDockerObject::upsert(db.pool(), repository, "overflow", u64::MAX, &[]) + .await + .is_err() + ); + DBDockerObject::delete_paths(db.pool(), repository, &[blob.into()]) + .await + .unwrap(); + assert!( + DBDockerObject::needs_backfill(db.pool(), repository, root) + .await + .unwrap() + ); + assert_eq!( + DBDockerObject::referenced_size(db.pool(), repository, root) + .await + .unwrap(), + Some(30) + ); + DBDockerObject::upsert(db.pool(), repository, root, 15, &[]) + .await + .unwrap(); + assert_eq!( + DBDockerObject::referenced_size(db.pool(), repository, root) + .await + .unwrap(), + Some(15) + ); + assert_eq!( + DBDockerObject::referenced_size(db.pool(), repository, "missing") + .await + .unwrap(), + None + ); + } + async fn reset_database(db: &TestDb) { sqlx::query( "TRUNCATE TABLE project_versions, projects, repositories, storages RESTART IDENTITY CASCADE", @@ -1980,6 +2108,90 @@ mod catalog_db_tests { site.close().await; } + #[tokio::test] + async fn docker_package_listing_uses_persisted_referenced_size_without_storage_access() { + let _guard = DB_TEST_LOCK.lock().await; + let db = fresh_pool().await; + reset_database(&db).await; + let root = tempfile::tempdir().expect("tempdir"); + let storage_id = insert_storage_at(db.pool(), root.path()).await; + let repository_id = insert_docker_repository(db.pool(), storage_id).await; + let fetched = chrono::Utc + .with_ymd_and_hms(2025, 1, 2, 12, 0, 0) + .single() + .unwrap(); + insert_proxy_version( + db.pool(), + repository_id, + "library/alpine", + "library/alpine", + "latest", + "v2/library/alpine/manifests/latest", + 10, + fetched, + ) + .await; + + nr_core::database::entities::docker_object::DBDockerObject::upsert( + db.pool(), + repository_id, + "v2/library/alpine/manifests/latest", + 4242, + &[], + ) + .await + .expect("persist object size"); + + insert_proxy_version( + db.pool(), + repository_id, + "library/small", + "library/small", + "latest", + "v2/library/small/manifests/latest", + 99999, + fetched, + ) + .await; + nr_core::database::entities::docker_object::DBDockerObject::upsert( + db.pool(), + repository_id, + "v2/library/small/manifests/latest", + 1, + &[], + ) + .await + .expect("persist smaller image"); + + let site = build_site(&db, root.path()).await; + + let response = super::list_cached_packages( + State(site.clone()), + Some(sample_auth()), + Path(repository_id), + Query(PackageListQuery { + page: 1, + per_page: 1, + q: None, + sort_by: PackageSortBy::Size, + sort_dir: PackageSortDirection::Desc, + }), + ) + .await + .expect("list packages succeeds"); + + assert_eq!(response.status(), http::StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("read response body"); + let payload: serde_json::Value = serde_json::from_slice(&body).expect("json response"); + let size = payload["items"][0]["size"].as_u64().expect("size value"); + + // No manifest or blobs were stored; the size must come from the object inventory. + assert_eq!(size, 4242); + site.close().await; + } + #[tokio::test] async fn maven_package_listing_reports_stored_version_directory_size() { let _guard = DB_TEST_LOCK.lock().await; diff --git a/pkgly/src/app/api/storage/s3.rs b/pkgly/src/app/api/storage/s3.rs index 57ab7e9..0ae9fc9 100644 --- a/pkgly/src/app/api/storage/s3.rs +++ b/pkgly/src/app/api/storage/s3.rs @@ -1,10 +1,11 @@ +// ABOUTME: Exposes S3 storage administration endpoints. +// ABOUTME: Returns raw region identifiers for editable provider configuration. use axum::{ response::{IntoResponse, Response}, routing::get, }; use nr_core::user::permissions::HasPermissions; -use nr_storage::s3::regions::S3StorageRegion; -use strum::IntoEnumIterator; +use nr_storage::s3::regions::KNOWN_S3_REGIONS; use tracing::instrument; use utoipa::OpenApi; @@ -15,7 +16,7 @@ use crate::{ }; #[derive(OpenApi)] -#[openapi(paths(region_list), components(schemas(S3StorageRegion)))] +#[openapi(paths(region_list))] pub struct S3StorageAPI; pub fn s3_storage_api() -> axum::Router { axum::Router::new().route("/regions", get(region_list)) @@ -25,7 +26,7 @@ pub fn s3_storage_api() -> axum::Router { get, path = "/regions", responses( - (status = 200, description = "A list of available regions for the S3 storage", body = Vec) + (status = 200, description = "A list of suggested raw region identifiers for S3 storage", body = Vec) ) )] #[instrument(skip(auth), fields(user = %auth.id))] @@ -33,6 +34,60 @@ pub async fn region_list(auth: Authentication) -> Result = S3StorageRegion::iter().collect(); + let regions: Vec = KNOWN_S3_REGIONS + .iter() + .map(|region| (*region).to_owned()) + .collect(); Ok(ResponseBuilder::ok().json(®ions)) } + +#[cfg(test)] +mod tests { + use super::*; + use axum::response::Response; + use chrono::DateTime; + use http_body_util::BodyExt; + use nr_core::{ + database::entities::user::{UserSafeData, auth_token::AuthToken}, + user::{Email, Username}, + }; + + fn admin_auth() -> Authentication { + let timestamp = DateTime::parse_from_rfc3339("2024-01-01T00:00:00+00:00").unwrap(); + let user = UserSafeData { + id: 1, + name: "Admin".into(), + username: Username::new("admin".into()).unwrap(), + email: Some(Email::new("admin@example.com".into()).unwrap()), + require_password_change: false, + active: true, + admin: true, + user_manager: true, + system_manager: true, + default_repository_actions: vec![], + updated_at: timestamp, + created_at: timestamp, + }; + let token = AuthToken { + id: 1, + user_id: 1, + name: Some("test".into()), + description: None, + token: "token".into(), + active: true, + source: "test".into(), + expires_at: None, + created_at: timestamp, + }; + Authentication::AuthToken(token, user) + } + + #[tokio::test] + async fn region_list_returns_raw_identifiers() { + let response: Response = region_list(admin_auth()).await.unwrap(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let regions: Vec = serde_json::from_slice(&body).unwrap(); + assert!(regions.iter().any(|region| region == "us-east-1")); + assert!(!regions.iter().any(|region| region == "UsEast1")); + } +} diff --git a/pkgly/src/app/authentication/mod.rs b/pkgly/src/app/authentication/mod.rs index dd74b91..50b0a5f 100644 --- a/pkgly/src/app/authentication/mod.rs +++ b/pkgly/src/app/authentication/mod.rs @@ -54,11 +54,6 @@ pub enum AuthenticationError { #[error("Forbidden")] Forbidden, } -impl AuthenticationError { - pub fn is_internal_error(&self) -> bool { - matches!(self, AuthenticationError::InternalError(_)) - } -} macro_rules! internal_errors { ( $($error:ty),* diff --git a/pkgly/src/app/authentication/session.rs b/pkgly/src/app/authentication/session.rs index 85d1ceb..e68cd94 100644 --- a/pkgly/src/app/authentication/session.rs +++ b/pkgly/src/app/authentication/session.rs @@ -370,15 +370,6 @@ impl SessionManager { Ok(session) } #[instrument] - pub fn create_session_default_lifespan( - &self, - user_id: i32, - user_agent: String, - ip_address: String, - ) -> Result { - self.create_session(user_id, user_agent, ip_address, self.config.lifespan) - } - #[instrument] pub fn get_session(&self, session_id: &str) -> Result, SessionError> { let sessions = self.sessions.begin_read()?; diff --git a/pkgly/src/app/email_service.rs b/pkgly/src/app/email_service.rs index 2837c4c..2098fcb 100644 --- a/pkgly/src/app/email_service.rs +++ b/pkgly/src/app/email_service.rs @@ -90,9 +90,6 @@ impl EmailAccess { warn!("Email Queue Error: {}", error); }; } - pub fn get_handlebars(&self) -> &Handlebars<'static> { - &self.email_handlebars - } #[inline] #[instrument()] pub fn build_body(&self, data: &E) -> MultiPart { diff --git a/pkgly/src/app/mod.rs b/pkgly/src/app/mod.rs index 41b62bd..684b3e9 100644 --- a/pkgly/src/app/mod.rs +++ b/pkgly/src/app/mod.rs @@ -17,7 +17,6 @@ pub mod email_service; pub mod frontend; pub mod open_api; pub mod request_logging; -pub mod resources; pub mod scheduler; pub mod storage_usage; pub mod webhooks; diff --git a/pkgly/src/app/resources.rs b/pkgly/src/app/resources.rs deleted file mode 100644 index 1e7d3fd..0000000 --- a/pkgly/src/app/resources.rs +++ /dev/null @@ -1,49 +0,0 @@ -use std::{borrow::Cow, fs::OpenOptions, io::Read, path::Path}; - -use rust_embed::RustEmbed; -use tracing::error; - -use crate::error::{IllegalStateError, InternalError}; - -#[derive(RustEmbed)] -#[folder = "$CARGO_MANIFEST_DIR/resources"] -pub struct Resources; - -impl Resources { - /// Gets the file from the resources file if it exists or defaults to the embedded file. - /// - /// # Arguments - /// - /// * `file`: Relative path to the file. - /// - /// returns: Result, InternalError> - /// Errors are returned if the IO operation fails. - /// # Panics - /// If the embedded resource is not found. - /// This should never happen. - /// If it does, it is a bug. - /// Please report it. - pub fn file_get(file: &str) -> Result, InternalError> { - let buf = Path::new("resources").join(file); - if buf.exists() { - let mut file = match OpenOptions::new().read(true).open(buf) { - Ok(ok) => ok, - Err(err) => { - error!("Unable to open the {file:?}: {}", err); - return Err(InternalError::from(err)); - } - }; - let mut buffer = Vec::with_capacity(file.metadata()?.len() as usize); - file.read_to_end(&mut buffer)?; - Ok(Cow::Owned(buffer)) - } else { - Resources::get(file) - .map(|resource| resource.data) - .ok_or_else(|| { - InternalError::from(IllegalStateError( - "Embedded resource was not found during file lookup", - )) - }) - } - } -} diff --git a/pkgly/src/app/site.rs b/pkgly/src/app/site.rs index e68b546..1ee1a3e 100644 --- a/pkgly/src/app/site.rs +++ b/pkgly/src/app/site.rs @@ -68,7 +68,7 @@ use super::{ webhooks::WebhookService, }; use current_semver::current_semver; -use http::{HeaderName, Uri}; +use http::HeaderName; #[derive(Debug, Default)] pub struct InternalServices { @@ -551,10 +551,6 @@ impl Pkgly { .filter(|cfg| cfg.enabled) } - pub fn sso_settings_raw(&self) -> Option { - self.inner.general_security_settings.read().sso.clone() - } - pub fn oauth2_settings(&self) -> Option { self.inner .general_security_settings @@ -583,20 +579,6 @@ impl Pkgly { Ok(()) } - pub async fn check_oauth_permission( - &self, - subject: &str, - object: &str, - action: &str, - ) -> anyhow::Result> { - if let Some(rbac) = self.oauth2_rbac() { - let decision = rbac.enforce(subject, object, action).await?; - Ok(Some(decision)) - } else { - Ok(None) - } - } - pub async fn update_oauth2_settings( &self, settings: Option, @@ -729,14 +711,6 @@ impl Pkgly { self.ensure_upload_state_handle(repository, upload_id, true) } - fn ensure_blob_upload_state_handle( - &self, - repository: Uuid, - upload_id: &str, - ) -> BlobUploadStateHandle { - self.ensure_upload_state_handle(repository, upload_id, false) - } - pub fn update_upload_state_handle(&self, handle: &BlobUploadStateHandle, chunk: &[u8]) -> u64 { let mut guard = handle.lock(); guard.update(chunk); @@ -747,36 +721,11 @@ impl Pkgly { handle.lock().length } - pub fn begin_blob_upload_state(&self, repository: Uuid, upload_id: &str) { - self.ensure_blob_upload_state_handle(repository, upload_id); - } - /// Begin blob upload state for Docker (SHA256 only) pub fn begin_docker_blob_upload_state(&self, repository: Uuid, upload_id: &str) { self.ensure_docker_blob_upload_state_handle(repository, upload_id); } - pub fn update_blob_upload_state(&self, repository: Uuid, upload_id: &str, chunk: &[u8]) -> u64 { - let state = self.ensure_blob_upload_state_handle(repository, upload_id); - self.update_upload_state_handle(&state, chunk) - } - - /// Update blob upload state for Docker (ensures SHA256-only hashing) - pub fn update_docker_blob_upload_state( - &self, - repository: Uuid, - upload_id: &str, - chunk: &[u8], - ) -> u64 { - let state = self.ensure_docker_blob_upload_state_handle(repository, upload_id); - self.update_upload_state_handle(&state, chunk) - } - - pub fn current_blob_upload_length(&self, repository: Uuid, upload_id: &str) -> Option { - self.get_upload_state_handle(repository, upload_id) - .map(|handle| handle.lock().length) - } - pub fn finalize_blob_upload_state( &self, repository: Uuid, @@ -801,11 +750,6 @@ impl Pkgly { .remove(&(repository, upload_id.to_owned())); } - pub fn update_app_url(&self, app_url: &Uri) { - info!(?app_url, "Updating app url"); - // TODO: Update persisted application URL if needed. - } - pub async fn update_sso_settings(&self, settings: Option) -> anyhow::Result<()> { { let mut security = self.inner.general_security_settings.write(); diff --git a/pkgly/src/app/storage_usage.rs b/pkgly/src/app/storage_usage.rs index 1c7f649..69826e8 100644 --- a/pkgly/src/app/storage_usage.rs +++ b/pkgly/src/app/storage_usage.rs @@ -52,6 +52,38 @@ async fn compute_repository_storage_usage( }; let usage = calculate_repository_storage_usage(&repository).await?; + + // Docker object accounting lives in PostgreSQL but storage is the source of truth. Repair + // drift opportunistically during the periodic usage refresh instead of scanning on every read. + if let DynRepository::Docker(docker) = &repository { + match crate::repository::docker::metadata::reconcile_docker_objects( + &site.database, + &docker.get_storage(), + repository_id, + ) + .await + { + Ok(summary) => { + if summary.backfilled > 0 || summary.corrected > 0 || summary.removed > 0 { + tracing::info!( + %repository_id, + backfilled = summary.backfilled, + corrected = summary.corrected, + removed = summary.removed, + "Reconciled Docker object accounting" + ); + } + } + Err(err) => { + tracing::warn!( + %repository_id, + %err, + "Docker object accounting reconciliation failed" + ); + } + } + } + Ok(StorageUsageComputeResult::Usage(usage)) } diff --git a/pkgly/src/app/web.rs b/pkgly/src/app/web.rs index e45e230..d120460 100644 --- a/pkgly/src/app/web.rs +++ b/pkgly/src/app/web.rs @@ -1,13 +1,6 @@ // ABOUTME: Starts the HTTP/TLS server and coordinates graceful shutdown. // ABOUTME: Logs startup build metadata and owns runtime worker configuration. -use std::{ - fs::File, - io::BufReader, - net::SocketAddr, - num::NonZeroUsize, - path::{Path, PathBuf}, - sync::Arc, -}; +use std::{fs::File, io::BufReader, net::SocketAddr, num::NonZeroUsize, path::Path, sync::Arc}; use anyhow::Context; use axum::{Router, extract::Request}; @@ -25,7 +18,7 @@ use super::Pkgly; use super::routes; use crate::app::build_info::{BuildInfo, current_build_info}; use crate::app::config::WebServer; -use crate::config::{PkglyConfig, load_config}; +use crate::config::PkglyConfig; /// Decide how many Tokio worker threads to start. pub(crate) fn resolve_worker_threads(web_server: &WebServer) -> usize { let configured = web_server @@ -44,12 +37,6 @@ pub(crate) fn startup_build_info() -> BuildInfo { current_build_info() } -#[allow(dead_code)] // Useful for callers that already hold a runtime -pub(crate) async fn start(config_path: Option) -> anyhow::Result<()> { - let config = load_config(config_path)?; - start_with_config(config).await -} - pub(crate) async fn start_with_config(config: PkglyConfig) -> anyhow::Result<()> { let PkglyConfig { web_server, diff --git a/pkgly/src/app/webhooks/mod.rs b/pkgly/src/app/webhooks/mod.rs index 05e50eb..58a7da0 100644 --- a/pkgly/src/app/webhooks/mod.rs +++ b/pkgly/src/app/webhooks/mod.rs @@ -1391,25 +1391,5 @@ async fn finalize_delivery_attempt( Ok(()) } -pub fn latest_delivery_summary( - status: Option, - last_attempt_at: Option>, - delivered_at: Option>, - last_http_status: Option, - last_error: Option, -) -> ( - Option, - Option>, - Option, - Option, -) { - ( - status, - delivered_at.or(last_attempt_at), - last_http_status, - last_error, - ) -} - #[cfg(test)] mod tests; diff --git a/pkgly/src/main.rs b/pkgly/src/main.rs index 53f8d6c..b22b86f 100644 --- a/pkgly/src/main.rs +++ b/pkgly/src/main.rs @@ -1,3 +1,4 @@ +#![recursion_limit = "256"] #![allow( elided_lifetimes_in_paths, clippy::all, @@ -217,6 +218,9 @@ fn web_start(config_path: Option) -> anyhow::Result<()> { let tokio = tokio::runtime::Builder::new_multi_thread() .worker_threads(worker_threads) .thread_name_fn(thread_name) + // Request chains through S3-backed Docker repositories nest the axum handler stack + // inside the AWS SDK state machine; std::thread defaults (2 MiB) overflow there. + .thread_stack_size(8 * 1024 * 1024) .enable_all() .build()?; tokio.block_on(app::web::start_with_config(config))?; diff --git a/pkgly/src/repository/base_tests.rs b/pkgly/src/repository/base_tests.rs deleted file mode 100644 index f9bb6f0..0000000 --- a/pkgly/src/repository/base_tests.rs +++ /dev/null @@ -1,11 +0,0 @@ -#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] - -#[derive(Debug, Clone)] -#[allow(dead_code)] -struct DummyRepository; - -#[derive(Debug)] -#[allow(dead_code)] -struct DummyError; - -// NOTE: this module intentionally stays minimal for now. diff --git a/pkgly/src/repository/commands.rs b/pkgly/src/repository/commands.rs deleted file mode 100644 index 26177d4..0000000 --- a/pkgly/src/repository/commands.rs +++ /dev/null @@ -1,11 +0,0 @@ -use schemars::{JsonSchema, Schema}; -use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, JsonSchema, Serialize, Deserialize)] -pub struct CommandDefinition { - pub name: String, - pub description: String, - /// Will be prepended with the repository type - pub key: String, - pub warn_before_run: Option, - pub request_schema: Schema, -} diff --git a/pkgly/src/repository/docker/handlers.rs b/pkgly/src/repository/docker/handlers.rs index 9a51f3f..53e5c5e 100644 --- a/pkgly/src/repository/docker/handlers.rs +++ b/pkgly/src/repository/docker/handlers.rs @@ -1,8 +1,11 @@ +// ABOUTME: Handles Docker Registry manifest and blob requests. +// ABOUTME: Records stored object sizes and references for cached-byte accounting. //! Docker Registry API V2 HTTP handlers //! //! Implements the Docker Registry HTTP API V2 specification. //! Reference: https://docs.docker.com/registry/spec/api/ +use crate::repository::proxy_indexing::ProxyIndexingError; use axum::body::Body; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; @@ -10,6 +13,7 @@ use bytes::Bytes; use chrono::Utc; use futures::StreamExt; use http::StatusCode; +use nr_core::database::entities::docker_object::DBDockerObject; use nr_core::database::entities::project::{ DBProject, NewProject, ProjectDBType, versions::{DBProjectVersion, NewVersion}, @@ -21,10 +25,13 @@ use nr_core::{ }; use nr_storage::{ DynStorage, FileContent, FileType, Storage, StorageError, StorageFile, local::LocalStorage, + s3::S3_UPLOAD_SPOOL_PERMIT_BYTES, }; use sha2::{Digest, Sha256}; -use std::{collections::BTreeSet, future::Future, io}; +use std::{collections::BTreeSet, future::Future, io, path::PathBuf, sync::Arc}; +use tempfile::{Builder, TempPath}; use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, BufWriter}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio::task::spawn_blocking; use tokio_util::io::ReaderStream; use tracing::{debug, info, instrument, warn}; @@ -468,18 +475,114 @@ where Ok(()) } -async fn collect_stream_bytes(mut stream: S) -> Result, DockerError> +/// Reserves shared spool capacity before writing so incoming upload chunk files stay +/// within the shared temporary-storage budget alongside staged uploads. Growth beyond the +/// available capacity fails the request instead of blocking while holding earlier +/// reservations; dropping the permits releases the capacity once the spool is consumed. +fn reserve_spool_capacity( + budget: Option<&Arc>, + permits: &mut Vec, + total_bytes: u64, +) -> Result<(), DockerError> { + let Some(budget) = budget else { + return Ok(()); + }; + let required = total_bytes.div_ceil(S3_UPLOAD_SPOOL_PERMIT_BYTES) as usize; + let reserved = permits + .iter() + .map(|permit| permit.num_permits()) + .sum::(); + if reserved >= required { + return Ok(()); + } + match budget + .clone() + .try_acquire_many_owned((required - reserved) as u32) + { + Ok(permit) => { + permits.push(permit); + Ok(()) + } + Err(_) => Err(DockerError::InvalidManifest( + "upload temporary storage budget exhausted".to_string(), + )), + } +} + +async fn spool_stream( + mut stream: S, + spool_budget: Option>, +) -> Result<(PathBuf, TempPath, u64, Vec), DockerError> where S: futures::Stream> + Unpin, { - let mut data = Vec::new(); + let named = Builder::new().prefix("docker-upload-chunk-").tempfile()?; + let (std_file, path) = named.into_parts(); + let path_buf = path.to_path_buf(); + let mut file = tokio::fs::File::from_std(std_file); + let mut length = 0u64; + let mut permits = Vec::new(); while let Some(chunk) = stream.next().await { let chunk = chunk.map_err(DockerError::from)?; if !chunk.is_empty() { - data.extend_from_slice(&chunk); + length = length.saturating_add(chunk.len() as u64); + reserve_spool_capacity(spool_budget.as_ref(), &mut permits, length)?; + file.write_all(&chunk).await.map_err(DockerError::from)?; } } - Ok(data) + file.flush().await.map_err(DockerError::from)?; + Ok((path_buf, path, length, permits)) +} + +/// Returns the shared spool budget to reserve against, if this storage spools uploads. +fn upload_spool_budget(storage: &DynStorage) -> Option> { + match storage { + DynStorage::S3(s3) => Some(s3.upload_spool_budget()), + _ => None, + } +} + +async fn update_upload_state_from_file( + site: &Pkgly, + handle: &BlobUploadStateHandle, + path: &std::path::Path, +) -> Result { + let mut file = tokio::fs::File::open(path) + .await + .map_err(DockerError::from)?; + let mut buffer = vec![0u8; LOCAL_UPLOAD_BUFFER_SIZE]; + loop { + let read = file.read(&mut buffer).await.map_err(DockerError::from)?; + if read == 0 { + break; + } + update_upload_state_background( + site.clone(), + handle.clone(), + Bytes::copy_from_slice(&buffer[..read]), + ) + .await?; + } + Ok(site.blob_upload_state_length(handle)) +} + +async fn append_upload_chunk( + storage: &DynStorage, + repository: Uuid, + chunk: FileContent, + location: &StoragePath, +) -> Result { + match storage { + DynStorage::S3(s3) => s3 + .append_file_staged(repository, chunk, location) + .await + .map_err(StorageError::from) + .map_err(DockerError::from), + _ => storage + .append_file(repository, chunk, location) + .await + .map_err(DockerError::from), + } } async fn update_upload_state_background( @@ -902,6 +1005,17 @@ async fn get_blob( .ok_or_else(|| DockerError::BlobNotFound(digest.to_string()))?; let size = meta.file_type.file_size; + if repo.catalog_indexing_enabled() { + DBDockerObject::upsert( + &repo.site().database, + repo.id(), + &blob_path.to_string(), + size, + &[], + ) + .await + .map_err(ProxyIndexingError::from)?; + } let stream = ReaderStream::new(reader); let mut builder = ResponseBuilder::ok(); @@ -933,6 +1047,17 @@ async fn head_blob( .ok_or_else(|| DockerError::BlobNotFound(digest.to_string()))?; let size = meta.file_type.file_size; + if repo.catalog_indexing_enabled() { + DBDockerObject::upsert( + &repo.site().database, + repo.id(), + &blob_path.to_string(), + size, + &[], + ) + .await + .map_err(ProxyIndexingError::from)?; + } let stored_digest = meta .file_type .file_hash @@ -1025,6 +1150,8 @@ async fn put_manifest( } } + let references = super::metadata::manifest_references(repository_name, &body); + // Save manifest by tag/reference let manifest_path = StoragePath::from(format!("v2/{}/manifests/{}", repository_name, reference)); @@ -1032,6 +1159,15 @@ async fn put_manifest( .save_file(repo.id(), body.clone().into(), &manifest_path) .await?; if repo.catalog_indexing_enabled() { + DBDockerObject::upsert( + &repo.site().database, + repo.id(), + &manifest_path.to_string(), + body_size, + &references, + ) + .await + .map_err(ProxyIndexingError::from)?; record_manifest_in_catalog( &repo.site().database, repo.id(), @@ -1052,6 +1188,15 @@ async fn put_manifest( .save_file(repo.id(), body.into(), &digest_path) .await?; if repo.catalog_indexing_enabled() { + DBDockerObject::upsert( + &repo.site().database, + repo.id(), + &digest_path.to_string(), + body_size, + &references, + ) + .await + .map_err(ProxyIndexingError::from)?; record_manifest_in_catalog( &repo.site().database, repo.id(), @@ -1278,16 +1423,19 @@ async fn upload_blob_chunk( .await?; } storage => { - let bytes = collect_stream_bytes(stream).await?; - if !bytes.is_empty() { - let chunk = Bytes::from(bytes); - storage - .append_file(repo.id(), FileContent::Bytes(chunk.clone()), &upload_path) - .await?; + let (chunk_path, _chunk_file, chunk_size, _spool_permits) = + spool_stream(stream, upload_spool_budget(&storage)).await?; + if chunk_size > 0 { + append_upload_chunk( + &storage, + repo.id(), + FileContent::Path(chunk_path.clone()), + &upload_path, + ) + .await?; total_size = - update_upload_state_background(site.clone(), state_handle.clone(), chunk) - .await?; + update_upload_state_from_file(&site, &state_handle, &chunk_path).await?; } } } @@ -1380,16 +1528,19 @@ async fn complete_blob_upload( .await?; } storage => { - let bytes = collect_stream_bytes(stream).await?; - if !bytes.is_empty() { - let chunk = Bytes::from(bytes); - storage - .append_file(repo.id(), FileContent::Bytes(chunk.clone()), &upload_path) - .await?; + let (chunk_path, _chunk_file, chunk_size, _spool_permits) = + spool_stream(stream, upload_spool_budget(&storage)).await?; + if chunk_size > 0 { + append_upload_chunk( + &storage, + repo.id(), + FileContent::Path(chunk_path.clone()), + &upload_path, + ) + .await?; _current_size = - update_upload_state_background(site.clone(), state_handle.clone(), chunk) - .await?; + update_upload_state_from_file(&site, &state_handle, &chunk_path).await?; } } } @@ -1434,6 +1585,17 @@ async fn complete_blob_upload( return Err(DockerError::BlobUploadNotFound(upload_id.to_string())); } + if repo.catalog_indexing_enabled() { + DBDockerObject::upsert( + &repo.site().database, + repo.id(), + &blob_path.to_string(), + finalized.length, + &[], + ) + .await + .map_err(ProxyIndexingError::from)?; + } let location = format!("/v2/{}/blobs/{}", repository_name, digest); Ok(custom_response( @@ -1524,6 +1686,7 @@ async fn delete_manifest( repo.id(), &manifest_path_str, None, + Some(&repo.site().database), ) .await { @@ -1572,6 +1735,9 @@ async fn delete_blob( repo.get_storage() .delete_file(repo.id(), &blob_path) .await?; + DBDockerObject::delete_paths(&repo.site().database, repo.id(), &[blob_path.to_string()]) + .await + .map_err(ProxyIndexingError::from)?; Ok(custom_response(StatusCode::ACCEPTED, vec![], vec![])) } diff --git a/pkgly/src/repository/docker/handlers/tests.rs b/pkgly/src/repository/docker/handlers/tests.rs index b61ee56..f64920a 100644 --- a/pkgly/src/repository/docker/handlers/tests.rs +++ b/pkgly/src/repository/docker/handlers/tests.rs @@ -29,6 +29,27 @@ fn test_stream_from_bytes( stream::iter(chunks.into_iter().map(|bytes| Ok(bytes))) } +#[test] +fn spool_capacity_tracks_owned_permit_units() { + let budget = Arc::new(tokio::sync::Semaphore::new(4)); + let mut permits = Vec::new(); + + reserve_spool_capacity( + Some(&budget), + &mut permits, + 2 * S3_UPLOAD_SPOOL_PERMIT_BYTES, + ) + .expect("initial reservation"); + reserve_spool_capacity( + Some(&budget), + &mut permits, + 3 * S3_UPLOAD_SPOOL_PERMIT_BYTES, + ) + .expect("incremental reservation"); + + assert_eq!(budget.available_permits(), 1); +} + #[tokio::test] async fn stream_writer_persists_full_payload() -> anyhow::Result<()> { let payload = (0u32..(512 * 1024)) diff --git a/pkgly/src/repository/docker/hosted.rs b/pkgly/src/repository/docker/hosted.rs index cd263fa..53c75a3 100644 --- a/pkgly/src/repository/docker/hosted.rs +++ b/pkgly/src/repository/docker/hosted.rs @@ -96,23 +96,6 @@ impl DockerHosted { Ok(Self(Arc::new(inner))) } - - pub async fn load_proxy( - repository: DBRepository, - storage: DynStorage, - site: Pkgly, - upstream_url: &str, - ) -> Result { - let mut hosted = Self::load(repository, storage, site).await?; - let upstream = Url::parse(upstream_url).map_err(|err| { - RepositoryFactoryError::InvalidConfig(super::REPOSITORY_TYPE_ID, err.to_string()) - })?; - let client = reqwest::Client::new(); - Arc::get_mut(&mut hosted.0) - .expect("no other references during construction") - .proxy = Some(ProxySettings { upstream, client }); - Ok(hosted) - } } impl Repository for DockerHosted { @@ -215,10 +198,6 @@ impl Repository for DockerHosted { } impl DockerHosted { - pub fn is_proxy(&self) -> bool { - self.0.proxy.is_some() - } - pub fn upstream(&self) -> Option<&ProxySettings> { self.0.proxy.as_ref() } diff --git a/pkgly/src/repository/docker/metadata.rs b/pkgly/src/repository/docker/metadata.rs index 76efe1e..28cc487 100644 --- a/pkgly/src/repository/docker/metadata.rs +++ b/pkgly/src/repository/docker/metadata.rs @@ -1,13 +1,16 @@ // ABOUTME: Collects Docker registry metadata for listings, browse views, and size reporting. // ABOUTME: Resolves manifest paths and computes referenced Docker content sizes. -use std::collections::{HashSet, VecDeque}; +use std::collections::VecDeque; +use ahash::{HashMap, HashSet}; use chrono::{DateTime, FixedOffset}; use nr_core::storage::StoragePath; use nr_storage::{DynStorage, FileType, Storage, StorageError, StorageFile, s3::S3Storage}; use uuid::Uuid; -use super::types::{Descriptor, Manifest, ManifestDescriptor, MediaType}; +#[cfg(test)] +use super::types::{Descriptor, ManifestDescriptor}; +use super::types::{Manifest, MediaType}; /// Represents a manifest (tag or digest) stored for a Docker image. #[derive(Debug, Clone)] @@ -102,19 +105,11 @@ pub async fn collect_manifest_entries( let mut manifest_path = manifests_path.clone(); manifest_path.push_mut(&manifest.name); - let calculated_size = calculate_referenced_manifest_size( - storage, - repository_id, - &manifest_path, - ) - .await? - .unwrap_or(file_meta.file_size); - manifests.push(DockerManifestEntry { repository: repository_name.clone(), reference: manifest.name.clone(), cache_path: manifest_path.to_string(), - size: calculated_size, + size: file_meta.file_size, modified: manifest.modified, }); } @@ -204,6 +199,9 @@ async fn read_manifest_file( repository_id: Uuid, path: &StoragePath, ) -> Result, u64)>, StorageError> { + if let DynStorage::S3(s3) = storage { + s3.invalidate_cached_file(repository_id, path).await?; + } let manifest_file = match storage.open_file(repository_id, path).await { Ok(Some(file)) => file, Ok(None) => return Ok(None), @@ -224,6 +222,7 @@ async fn read_manifest_file( } } +#[cfg(test)] pub(crate) async fn calculate_referenced_manifest_size( storage: &DynStorage, repository_id: Uuid, @@ -243,8 +242,8 @@ pub(crate) async fn calculate_referenced_manifest_size( }; let mut total = manifest_size; - let mut seen_blobs = HashSet::new(); - let mut seen_manifests = HashSet::new(); + let mut seen_blobs = HashSet::default(); + let mut seen_manifests = HashSet::default(); let mut pending_manifests = Vec::new(); add_manifest_payload_sizes( @@ -305,6 +304,220 @@ fn parse_manifest(bytes: &[u8]) -> Option { Manifest::from_bytes(bytes, media_type).ok() } +/// Extracts distinct direct object references without accessing storage. +pub(crate) fn manifest_references(repository_name: &str, bytes: &[u8]) -> Vec { + let Some(manifest) = parse_manifest(bytes) else { + return Vec::new(); + }; + let (kind, digests): (&str, Vec) = match manifest { + Manifest::DockerV2(manifest) => ( + "blobs", + std::iter::once(manifest.config) + .chain(manifest.layers) + .map(|descriptor| descriptor.digest) + .collect(), + ), + Manifest::OciImage(manifest) => ( + "blobs", + manifest + .config + .into_iter() + .chain(manifest.layers) + .map(|descriptor| descriptor.digest) + .collect(), + ), + Manifest::OciIndex(index) => ( + "manifests", + index + .manifests + .into_iter() + .map(|descriptor| descriptor.digest) + .collect(), + ), + }; + let mut paths: Vec<_> = digests + .into_iter() + .map(|digest| format!("v2/{repository_name}/{kind}/{digest}")) + .collect(); + paths.sort(); + paths.dedup(); + paths +} + +/// Summary of a Docker object accounting reconciliation pass. +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct DockerReconcileSummary { + pub backfilled: usize, + pub corrected: usize, + pub removed: usize, +} + +/// Reconciles the `docker_objects` accounting table with storage reality. +/// +/// Storage mutations and PostgreSQL updates are separate operations, so crashes or partial +/// failures can leave the table inaccurate. This backfills missing manifest graphs, corrects +/// drifted sizes, and removes rows whose objects no longer exist. It only probes objects +/// referenced by stored manifests (plus manifest rows), not every blob ever seen. +pub(crate) async fn reconcile_docker_objects( + database: &sqlx::PgPool, + storage: &DynStorage, + repository_id: Uuid, +) -> anyhow::Result { + use nr_core::database::entities::docker_object::DBDockerObject; + let mut summary = DockerReconcileSummary::default(); + + // 1. Backfill missing manifest graphs so every stored manifest has an accounting root. + let entries = collect_manifest_entries(storage, repository_id).await?; + for entry in &entries { + if DBDockerObject::needs_backfill(database, repository_id, &entry.cache_path).await? { + match backfill_manifest_objects(database, storage, repository_id, &entry.cache_path) + .await + { + Ok(_) => summary.backfilled += 1, + Err(err) => { + tracing::warn!( + %repository_id, + cache_path = %entry.cache_path, + %err, + "Failed to backfill Docker object accounting" + ); + } + } + } + } + + // 2. Compare rows with an authoritative object inventory and correct drifted sizes. + // Capture rows before scanning storage: an upload completing between the two snapshots + // is then absent from the older row snapshot, so its fresh row can never be deleted + // for missing inventory. + let rows = DBDockerObject::list_rows(database, repository_id).await?; + let inventory = if let DynStorage::S3(s3) = storage { + Some( + s3.list_repository_objects(repository_id, None) + .await? + .into_iter() + .map(|object| (object.key, object.size)) + .collect::>(), + ) + } else { + None + }; + let mut missing = Vec::new(); + for (path, stored_size, revision) in rows { + let storage_path = StoragePath::from(path.as_str()); + let observed_size = if let Some(inventory) = &inventory { + inventory.get(&path).copied() + } else { + match storage + .get_file_information(repository_id, &storage_path) + .await + { + Ok(Some(meta)) => match meta.file_type() { + FileType::File(file) => Some(file.file_size), + FileType::Directory(_) => None, + }, + Ok(None) => None, + Err(err) => { + tracing::warn!( + %repository_id, + path = %path, + %err, + "Skipping Docker object during reconciliation" + ); + continue; + } + } + }; + match observed_size { + Some(size) if size != stored_size.max(0) as u64 => { + if DBDockerObject::update_size_if_revision( + database, + repository_id, + &path, + size, + revision, + ) + .await? + { + summary.corrected += 1; + } + } + Some(_) => {} + None => missing.push((path, revision)), + } + } + if !missing.is_empty() { + summary.removed = + DBDockerObject::delete_paths_if_revisions(database, repository_id, &missing).await?; + } + + Ok(summary) +} + +/// Backfills a manifest graph by probing storage and recording each observed object. +pub(crate) async fn backfill_manifest_objects( + database: &sqlx::PgPool, + storage: &DynStorage, + repository_id: Uuid, + root: &str, +) -> anyhow::Result> { + use nr_core::database::entities::docker_object::DBDockerObject; + let Some((repository_name, _)) = split_manifest_cache_path(root) else { + return Ok(None); + }; + let mut pending = vec![root.to_string()]; + let mut visited = HashSet::default(); + // Publish children before parents so an indexed root represents a finished backfill. + let mut objects = Vec::new(); + while let Some(path) = pending.pop() { + if !visited.insert(path.clone()) { + continue; + } + // Capture the revision before reading storage: a concurrent update to this object + // bumps its revision, so publishing the bytes read here becomes a guarded no-op + // instead of overwriting the newer row with stale references. + let revision = DBDockerObject::revision(database, repository_id, &path).await?; + let storage_path = StoragePath::from(path.as_str()); + if split_manifest_cache_path(&path).is_some() { + if let Some((bytes, size)) = + read_manifest_file(storage, repository_id, &storage_path).await? + { + let references = manifest_references(&repository_name, &bytes); + pending.extend(references.clone()); + objects.push((path, size, references, revision)); + } + } else if let Some(meta) = storage + .get_file_information(repository_id, &storage_path) + .await? + && let FileType::File(file) = meta.file_type() + { + objects.push((path, file.file_size, Vec::new(), revision)); + } + } + for (path, size, references, revision) in objects.into_iter().rev() { + if let Some(revision) = revision { + let _ = DBDockerObject::upsert_if_revision( + database, + repository_id, + &path, + size, + &references, + revision, + ) + .await?; + } else { + DBDockerObject::insert_missing(database, repository_id, &path, size, &references) + .await?; + } + } + Ok( + DBDockerObject::referenced_size(database, repository_id, root) + .await? + .map(|size| size as u64), + ) +} + +#[cfg(test)] async fn add_manifest_payload_sizes( storage: &DynStorage, repository_id: Uuid, @@ -368,6 +581,7 @@ async fn add_manifest_payload_sizes( Ok(()) } +#[cfg(test)] async fn add_blob_descriptor_size( storage: &DynStorage, repository_id: Uuid, @@ -384,10 +598,14 @@ async fn add_blob_descriptor_size( "v2/{}/blobs/{}", repository_name, descriptor.digest )); - if let Some(StorageFile::File { meta, .. }) = - storage.open_file(repository_id, &blob_path).await? + // Only the stored size is needed here, so retrieve metadata instead of downloading the blob + // body. This avoids one full-body S3 GET per layer per listing row. + if let Some(meta) = storage + .get_file_information(repository_id, &blob_path) + .await? + && let FileType::File(file_meta) = meta.file_type() { - *total += meta.file_type.file_size; + *total += file_meta.file_size; } Ok(()) } diff --git a/pkgly/src/repository/docker/metadata/tests.rs b/pkgly/src/repository/docker/metadata/tests.rs index 2fcf23f..243085c 100644 --- a/pkgly/src/repository/docker/metadata/tests.rs +++ b/pkgly/src/repository/docker/metadata/tests.rs @@ -18,6 +18,30 @@ const CHILD_DIGEST: &str = const MISSING_CHILD_DIGEST: &str = "sha256:4444444444444444444444444444444444444444444444444444444444444444"; +#[test] +fn manifest_references_deduplicate_paths_without_using_declared_sizes() { + let paths = manifest_references("library/alpine", &image_manifest(999, 999)); + assert_eq!( + paths, + vec![ + format!("v2/library/alpine/blobs/{CONFIG_DIGEST}"), + format!("v2/library/alpine/blobs/{LAYER_DIGEST}"), + ] + ); + let index = serde_json::to_vec(&json!({ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [{"mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": CHILD_DIGEST, "size": 999}] + })) + .unwrap(); + assert_eq!( + manifest_references("library/alpine", &index), + vec![format!("v2/library/alpine/manifests/{CHILD_DIGEST}")] + ); + assert!(manifest_references("library/alpine", b"invalid").is_empty()); +} + async fn save_bytes( storage: &DynStorage, repository_id: Uuid, @@ -59,6 +83,31 @@ fn image_manifest(config_size: u64, layer_size: u64) -> Vec { .expect("serialize manifest") } +#[tokio::test] +async fn catalog_entries_report_manifest_bytes_without_summing_layers() -> anyhow::Result<()> { + let storage = test_storage().await; + let repository_id = Uuid::new_v4(); + let manifest = image_manifest(10, 20); + save_bytes( + &storage, + repository_id, + "v2/library/alpine/manifests/latest", + &manifest, + ) + .await?; + save_bytes( + &storage, + repository_id, + &format!("v2/library/alpine/blobs/{LAYER_DIGEST}"), + b"layer", + ) + .await?; + let entries = collect_manifest_entries(&storage, repository_id).await?; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].size, manifest.len() as u64); + Ok(()) +} + #[tokio::test] async fn referenced_manifest_size_uses_stored_blob_sizes_and_dedupes_digests() -> anyhow::Result<()> { diff --git a/pkgly/src/repository/docker/proxy.rs b/pkgly/src/repository/docker/proxy.rs index 783924d..faf27f5 100644 --- a/pkgly/src/repository/docker/proxy.rs +++ b/pkgly/src/repository/docker/proxy.rs @@ -1,3 +1,5 @@ +// ABOUTME: Implements Docker pull-through caching and upstream authentication. +// ABOUTME: Serves manifests and blobs through the shared repository storage interface. //! Docker proxy (pull-through cache) support. //! //! This module implements a read-only proxy repository that forwards @@ -32,18 +34,22 @@ use http::{ header::{ACCEPT, CONTENT_LENGTH, CONTENT_TYPE}, }; use nr_core::{ + database::entities::docker_object::DBDockerObject, repository::{Visibility, config::RepositoryConfigType, project::ProxyArtifactMeta}, storage::StoragePath, utils::base64_utils, }; -use nr_storage::{DynStorage, FileContent, FileType, Storage, StorageFile, StorageFileReader}; +use nr_storage::{ + DynStorage, FileContent, FileContentBytes, FileType, Storage, StorageFile, StorageFileReader, +}; use parking_lot::RwLock; use reqwest::{Client, Response}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sha2::Digest; use tempfile::Builder; -use tokio::io::{AsyncRead, AsyncWriteExt}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio::time::sleep; use tracing::{info, instrument, warn}; use url::Url; @@ -56,6 +62,7 @@ use crate::{ repository::{ RepoResponse, Repository, RepositoryAuthConfigType, RepositoryFactoryError, RepositoryRequest, + docker::metadata::manifest_references, proxy_indexing::{DatabaseProxyIndexer, ProxyIndexing, ProxyIndexingError}, utils::can_read_repository_with_auth, }, @@ -471,10 +478,12 @@ fn accept_priority(token: &str) -> u8 { } } +#[derive(Debug)] struct StreamedDownload { path: tempfile::TempPath, size: u64, digest: String, + permits: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -626,11 +635,12 @@ async fn record_docker_manifest_cache_hit( cache_path: &StoragePath, digest: &str, size: u64, + references: Vec, ) -> Result<(), ProxyIndexingError> { let Some(indexer) = indexer else { return Ok(()); }; - let meta = ProxyArtifactMeta::builder( + let mut builder = ProxyArtifactMeta::builder( repository_name.to_string(), docker_proxy_package_key(repository_name), cache_path.to_string(), @@ -638,9 +648,41 @@ async fn record_docker_manifest_cache_hit( .version(reference.to_string()) .upstream_digest(digest.to_string()) .size(size) - .fetched_at(Utc::now()) - .build(); - indexer.record_cached_artifact(meta).await + .fetched_at(Utc::now()); + builder = builder.docker_references(references); + indexer.record_cached_artifact(builder.build()).await +} + +/// Grows the temporary-file reservation to cover `total_bytes`, failing fast when the +/// shared budget is exhausted. Blocking while holding earlier permits would deadlock +/// competing downloads once incomplete ones exhausted the budget, and an object larger +/// than the budget could never satisfy a blocking reservation at all; releasing the +/// partial download instead lets the caller retry once capacity frees up. +fn reserve_temp_permits( + budget: &Arc, + permits: &mut Vec, + total_bytes: u64, +) -> Result<(), DockerError> { + let required = total_bytes.div_ceil(TEMP_FILE_PERMIT_BYTES) as usize; + let reserved = permits + .iter() + .map(|permit| permit.num_permits()) + .sum::(); + if reserved >= required { + return Ok(()); + } + match budget + .clone() + .try_acquire_many_owned((required - reserved) as u32) + { + Ok(permit) => { + permits.push(permit); + Ok(()) + } + Err(_) => Err(DockerError::InvalidManifest( + "temporary file budget exhausted".to_string(), + )), + } } async fn stream_response_to_tempfile(response: Response) -> Result { @@ -649,11 +691,25 @@ async fn stream_response_to_tempfile(response: Response) -> Result().ok()); + if let Some(length) = advertised { + reserve_temp_permits(&budget, &mut permits, length)?; + } let mut stream = response.bytes_stream(); while let Some(chunk) = stream.next().await { let chunk = chunk?; total += chunk.len() as u64; + reserve_temp_permits(&budget, &mut permits, total)?; hasher.update(&chunk); file.write_all(&chunk).await?; } @@ -665,17 +721,27 @@ async fn stream_response_to_tempfile(response: Response) -> Result, } impl TempFileReader { - fn new(file: tokio::fs::File, path: tempfile::TempPath) -> Self { - Self { file, _path: path } + fn new( + file: tokio::fs::File, + path: tempfile::TempPath, + permits: Vec, + ) -> Self { + Self { + file, + _path: path, + _permits: permits, + } } } @@ -689,6 +755,59 @@ impl AsyncRead for TempFileReader { } } +/// Blobs up to this size are hashed in memory and delivered from that buffer; larger blobs are +/// hashed into a temp file and delivered from there, keeping peak memory bounded. +const BLOB_HASH_BUFFER_LIMIT: usize = 8 * 1024 * 1024; +const TEMP_FILE_BUDGET_BYTES: u64 = 64 * 1024 * 1024 * 1024; +const TEMP_FILE_PERMIT_BYTES: u64 = 1024 * 1024; +static TEMP_FILE_BUDGET: std::sync::OnceLock> = std::sync::OnceLock::new(); + +fn temp_file_budget() -> Arc { + TEMP_FILE_BUDGET + .get_or_init(|| { + Arc::new(Semaphore::new( + (TEMP_FILE_BUDGET_BYTES / TEMP_FILE_PERMIT_BYTES) as usize, + )) + }) + .clone() +} + +/// Streams a reader through a hasher into a temp file so large blobs are verified and delivered +/// from a single storage read without holding the whole object in memory. +async fn hash_reader_to_tempfile( + reader: StorageFileReader, +) -> Result { + let named = Builder::new().prefix("docker-proxy-blob-").tempfile()?; + let (std_file, path) = named.into_parts(); + let mut file = tokio::fs::File::from_std(std_file); + let mut hasher = sha2::Sha256::new(); + let mut reader = reader; + let mut buffer = vec![0u8; 64 * 1024]; + let mut total = 0u64; + let mut permits = Vec::new(); + let budget = temp_file_budget(); + + loop { + let read = reader.read(&mut buffer).await?; + if read == 0 { + break; + } + total += read as u64; + reserve_temp_permits(&budget, &mut permits, total)?; + hasher.update(&buffer[..read]); + file.write_all(&buffer[..read]).await?; + } + file.flush().await?; + + let digest = format!("sha256:{:x}", hasher.finalize()); + Ok(StreamedDownload { + path, + size: total, + digest, + permits, + }) +} + fn upstream_image_name(repository_name: &str, upstream: &ProxyUpstream) -> String { let mut segments: Vec<&str> = repository_name.split('/').collect(); @@ -787,6 +906,15 @@ impl DockerProxy { .await? { if let FileType::File(file_meta) = meta.file_type() { + DBDockerObject::upsert( + &self.0.site.database, + self.id(), + &blob_path.to_string(), + file_meta.file_size, + &[], + ) + .await + .map_err(ProxyIndexingError::from)?; let digest_value = file_meta .file_hash .sha2_256 @@ -807,6 +935,17 @@ impl DockerProxy { ) .await?; + let blob_path = format!("v2/{repository_name}/blobs/{digest}"); + DBDockerObject::upsert( + &self.0.site.database, + self.id(), + &blob_path, + blob.length, + &[], + ) + .await + .map_err(ProxyIndexingError::from)?; + Ok(if head_only { blob_head_response(&blob.digest, blob.length) } else { @@ -948,20 +1087,14 @@ async fn load_cached_manifest( actual: computed_digest, }); } - // S3 sets content-type to application/octet-stream; prefer detecting from content + // S3 sets content-type to application/octet-stream; prefer detecting from content. let content_type = content_type_from_meta .clone() .unwrap_or_else(|| manifest_media_type(&bytes, None)); - let reopened = storage - .open_file(repository_id, manifest_path) - .await? - .ok_or_else(|| DockerError::ManifestNotFound(reference.to_string()))?; - let (reader, _) = reopened - .file() - .ok_or_else(|| DockerError::InvalidManifest("Expected file, got directory".into()))?; + // Deliver from the bytes we already read instead of re-opening the object. Ok(Some(CachedManifest { - reader, + reader: StorageFileReader::Bytes(FileContentBytes::Content(bytes)), digest: computed_digest, content_type, length, @@ -1007,7 +1140,7 @@ async fn download_manifest_from_upstream( } let headers = response.headers().clone(); let streamed = stream_response_to_tempfile(response).await?; - let manifest_bytes = tokio::fs::read(streamed.path.to_path_buf()).await?; + let mut manifest_bytes = tokio::fs::read(streamed.path.to_path_buf()).await?; let mut content_type = manifest_media_type(&manifest_bytes, Some(&headers)); let mut streamed_download = streamed; @@ -1032,6 +1165,7 @@ async fn download_manifest_from_upstream( } } + manifest_bytes = modern_bytes; content_type = modern_content_type; streamed_download = modern_streamed; } else { @@ -1072,6 +1206,8 @@ async fn download_manifest_from_upstream( ) .await?; + let references = manifest_references(repository_name, &manifest_bytes); + record_docker_manifest_cache_hit( indexer, repository_name, @@ -1079,6 +1215,7 @@ async fn download_manifest_from_upstream( manifest_path, &computed_digest, streamed_download.size, + references.clone(), ) .await?; @@ -1105,6 +1242,7 @@ async fn download_manifest_from_upstream( &digest_path, &computed_digest, streamed_download.size, + references.clone(), ) .await?; } @@ -1117,6 +1255,7 @@ async fn download_manifest_from_upstream( reader: StorageFileReader::AsyncReader(Box::pin(TempFileReader::new( tokio::fs::File::open(streamed_download.path.to_path_buf()).await?, streamed_download.path, + streamed_download.permits, ))), digest: computed_digest, content_type, @@ -1206,6 +1345,8 @@ pub(crate) async fn fetch_and_cache_manifest( ); let manifest_path = StoragePath::from(format!("v2/{}/manifests/{}", repository_name, reference)); + // Refresh/download futures contain nested SDK operations. Keep them heap-allocated so + // their state does not accumulate in the surrounding HTTP request's stack frames. if let Some(cached) = load_cached_manifest(storage, repository_id, &manifest_path, reference).await? { @@ -1226,7 +1367,7 @@ pub(crate) async fn fetch_and_cache_manifest( return Err(DockerError::ManifestNotFound(reference.to_string())); } } else { - match revalidate_manifest_tag( + match Box::pin(revalidate_manifest_tag( upstream, storage, repository_id, @@ -1235,7 +1376,7 @@ pub(crate) async fn fetch_and_cache_manifest( &manifest_path, cached, indexer, - ) + )) .await? { RevalidationOutcome::Unchanged(cached) => { @@ -1254,7 +1395,7 @@ pub(crate) async fn fetch_and_cache_manifest( } else { accept }; - match download_manifest_from_upstream( + match Box::pin(download_manifest_from_upstream( upstream, storage, repository_id, @@ -1263,7 +1404,7 @@ pub(crate) async fn fetch_and_cache_manifest( override_accept, &manifest_path, indexer, - ) + )) .await { Ok(manifest) => return Ok(manifest), @@ -1279,7 +1420,7 @@ pub(crate) async fn fetch_and_cache_manifest( } info!("fetch_and_cache_manifest: cache miss or refresh failed, downloading"); - download_manifest_from_upstream( + Box::pin(download_manifest_from_upstream( upstream, storage, repository_id, @@ -1288,7 +1429,7 @@ pub(crate) async fn fetch_and_cache_manifest( Some(MODERN_UPSTREAM_ACCEPT), &manifest_path, indexer, - ) + )) .await } @@ -1426,29 +1567,45 @@ async fn load_cached_blob( })); } + // No stored digest: hash the content once and deliver from that same copy so blob delivery + // performs a single storage read instead of re-opening the object after verification. let length_usize: usize = length .try_into() .map_err(|_| DockerError::InvalidManifest("blob size overflow".to_string()))?; - let bytes = reader.read_to_vec(length_usize).await?; - let computed_digest = compute_sha256_hex(&bytes); - if digest.starts_with("sha256:") && digest != computed_digest { - return Err(DockerError::DigestMismatch { - expected: digest.to_string(), - actual: computed_digest, - }); - } - let reopened = storage - .open_file(repository_id, blob_path) - .await? - .ok_or_else(|| DockerError::BlobNotFound(digest.to_string()))?; - let (reader, meta) = reopened - .file() - .ok_or_else(|| DockerError::BlobNotFound(digest.to_string()))?; + let (reader, computed_digest) = if length_usize <= BLOB_HASH_BUFFER_LIMIT { + let bytes = reader.read_to_vec(length_usize).await?; + let computed_digest = compute_sha256_hex(&bytes); + if digest.starts_with("sha256:") && digest != computed_digest { + return Err(DockerError::DigestMismatch { + expected: digest.to_string(), + actual: computed_digest, + }); + } + ( + StorageFileReader::Bytes(FileContentBytes::Content(bytes)), + computed_digest, + ) + } else { + let streamed = hash_reader_to_tempfile(reader).await?; + let computed_digest = streamed.digest.clone(); + if digest.starts_with("sha256:") && digest != computed_digest { + return Err(DockerError::DigestMismatch { + expected: digest.to_string(), + actual: computed_digest, + }); + } + let reader = StorageFileReader::AsyncReader(Box::pin(TempFileReader::new( + tokio::fs::File::open(streamed.path.to_path_buf()).await?, + streamed.path, + streamed.permits, + ))); + (reader, computed_digest) + }; Ok(Some(CachedBlob { reader, digest: computed_digest, - length: meta.file_type.file_size, + length, })) } @@ -1514,6 +1671,7 @@ async fn download_blob_from_upstream( reader: StorageFileReader::AsyncReader(Box::pin(TempFileReader::new( tokio::fs::File::open(streamed.path.to_path_buf()).await?, streamed.path, + streamed.permits, ))), digest: computed_digest, length: streamed.size, diff --git a/pkgly/src/repository/docker/proxy/tests.rs b/pkgly/src/repository/docker/proxy/tests.rs index 6ed265c..3575f38 100644 --- a/pkgly/src/repository/docker/proxy/tests.rs +++ b/pkgly/src/repository/docker/proxy/tests.rs @@ -1,3 +1,5 @@ +// ABOUTME: Exercises Docker proxy caching, upstream requests, and manifest validation. +// ABOUTME: Checks request resource bounds and cache behavior across registry operations. #![allow(clippy::expect_used, clippy::panic, clippy::todo, clippy::unwrap_used)] use super::*; use crate::repository::{ @@ -295,6 +297,50 @@ impl ProxyIndexing for RecordingIndexer { } } +#[tokio::test] +async fn manifest_future_fits_worker_stack() -> anyhow::Result<()> { + let storage = test_storage().await; + let upstream = ProxyUpstream::new(&DockerProxyConfig { + upstream_url: "https://registry-1.docker.io".into(), + upstream_auth: None, + revalidation_ttl_seconds: default_revalidation_ttl(), + skip_tag_revalidation: false, + })?; + let future = fetch_and_cache_manifest( + &upstream, + &storage, + Uuid::new_v4(), + "library/nginx", + "alpine", + None, + None, + ); + let size = std::mem::size_of_val(&future); + assert!( + size < 16 * 1024, + "manifest future occupies {size} bytes on the request stack" + ); + Ok(()) +} + +#[test] +fn repository_read_futures_fit_worker_stack() { + fn size( + _: impl Fn(&'static crate::repository::DynRepository, RepositoryRequest) -> F, + ) -> usize { + std::mem::size_of::() + } + for (method, size) in [ + ("GET", size(|repo, request| repo.handle_get(request))), + ("HEAD", size(|repo, request| repo.handle_head(request))), + ] { + assert!( + size < 16 * 1024, + "repository {method} future occupies {size} bytes" + ); + } +} + #[tokio::test] async fn fetch_manifest_caches_locally() -> anyhow::Result<()> { let manifest = br#"{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","size":7023,"digest":"sha256:0000000000000000000000000000000000000000000000000000000000000000"},"layers":[]}"#; @@ -401,6 +447,12 @@ async fn fetch_manifest_records_proxy_index_entries() -> anyhow::Result<()> { .iter() .all(|meta| meta.cache_path.starts_with("v2/library/alpine/manifests/")) ); + // Direct references are indexed without probing their stored content. + assert!(recorded.iter().all(|meta| { + meta.docker_references + .as_ref() + .is_some_and(|paths| paths.len() == 1) + })); Ok(()) } @@ -721,6 +773,7 @@ async fn deleted_manifest_is_downloaded_again() -> anyhow::Result<()> { repository_id, manifest_path.to_string().as_str(), None, + None, ) .await .expect("docker deletion should succeed"); @@ -930,6 +983,7 @@ fn streamed_from_bytes(bytes: &[u8]) -> anyhow::Result { path, size: bytes.len() as u64, digest, + permits: Vec::new(), }) } @@ -1082,3 +1136,21 @@ async fn bearer_challenge_is_followed_for_public_token() -> anyhow::Result<()> { token_server.abort(); Ok(()) } + +#[test] +fn temp_reservations_track_units_and_fail_fast_when_exhausted() { + let budget = Arc::new(tokio::sync::Semaphore::new(4)); + let mut permits = Vec::new(); + reserve_temp_permits(&budget, &mut permits, 2 * TEMP_FILE_PERMIT_BYTES) + .expect("initial reservation"); + reserve_temp_permits(&budget, &mut permits, 3 * TEMP_FILE_PERMIT_BYTES) + .expect("incremental reservation"); + assert_eq!(budget.available_permits(), 1); + + let error = reserve_temp_permits(&budget, &mut permits, 5 * TEMP_FILE_PERMIT_BYTES) + .expect_err("reservation must fail fast when the budget is exhausted"); + assert!( + error.to_string().contains("temporary file budget"), + "unexpected error: {error}" + ); +} diff --git a/pkgly/src/repository/docker/types.rs b/pkgly/src/repository/docker/types.rs index db4f5e0..4259fb8 100644 --- a/pkgly/src/repository/docker/types.rs +++ b/pkgly/src/repository/docker/types.rs @@ -159,35 +159,6 @@ impl Manifest { Manifest::OciIndex(m) => serde_json::to_vec(m), } } - - /// Get all blob digests referenced by this manifest - pub fn get_blob_digests(&self) -> Vec { - let mut digests = Vec::new(); - - match self { - Manifest::DockerV2(m) => { - digests.push(m.config.digest.clone()); - for layer in &m.layers { - digests.push(layer.digest.clone()); - } - } - Manifest::OciImage(m) => { - if let Some(config) = &m.config { - digests.push(config.digest.clone()); - } - for layer in &m.layers { - digests.push(layer.digest.clone()); - } - } - Manifest::OciIndex(index) => { - for manifest in &index.manifests { - digests.push(manifest.digest.clone()); - } - } - } - - digests - } } /// Standard media types for Docker and OCI diff --git a/pkgly/src/repository/go/utils.rs b/pkgly/src/repository/go/utils.rs index 793331d..6a68bed 100644 --- a/pkgly/src/repository/go/utils.rs +++ b/pkgly/src/repository/go/utils.rs @@ -190,22 +190,6 @@ impl GoModuleRequest { .ok_or(RepositoryHandlerError::NotFound) } - /// Check if this is a read request - pub fn is_read_request(&self) -> bool { - matches!( - self.request_type, - GoRequestType::ListVersions - | GoRequestType::VersionInfo - | GoRequestType::GoMod - | GoRequestType::ModuleZip - | GoRequestType::Latest - | GoRequestType::GoModWithoutVersion - | GoRequestType::SumdbSupported - | GoRequestType::SumdbLookup - | GoRequestType::SumdbTile - ) - } - /// Check if this request requires a version pub fn requires_version(&self) -> bool { matches!( @@ -324,31 +308,3 @@ pub fn generate_go_module_info( pub fn generate_go_mod(module_path: &str) -> String { format!("module {}\n\ngo 1.21\n", module_path) } - -/// Validate that a version string is compatible with Go module requirements -pub fn validate_version_for_go(version: &str) -> Result<(), GoModuleError> { - let _go_version = GoVersion::new(version)?; - Ok(()) -} - -/// Check if a module path is a major version suffix -pub fn has_major_version_suffix(module_path: &str) -> bool { - if let Some((_, suffix)) = module_path.rsplit_once('/') { - suffix.starts_with('v') && suffix[1..].chars().all(|c| c.is_ascii_digit()) - } else { - false - } -} - -/// Extract major version from module path if present -pub fn extract_major_version(module_path: &str) -> Option { - if let Some((_, suffix)) = module_path.rsplit_once('/') { - if suffix.starts_with('v') && suffix.len() > 1 { - suffix[1..].parse::().ok() - } else { - None - } - } else { - None - } -} diff --git a/pkgly/src/repository/mod.rs b/pkgly/src/repository/mod.rs index 6dd57fe..c3e4833 100644 --- a/pkgly/src/repository/mod.rs +++ b/pkgly/src/repository/mod.rs @@ -25,7 +25,6 @@ mod repo_http; pub use repo_http::*; mod auth_config; pub mod cargo; -pub mod commands; pub mod deb; pub mod docker; pub mod go; @@ -99,6 +98,3 @@ pub(crate) mod test_helpers { DynStorage::Local(local) } } - -#[cfg(test)] -mod base_tests; diff --git a/pkgly/src/repository/nuget/mod.rs b/pkgly/src/repository/nuget/mod.rs index 9ed51b5..e591599 100644 --- a/pkgly/src/repository/nuget/mod.rs +++ b/pkgly/src/repository/nuget/mod.rs @@ -241,19 +241,5 @@ impl RepositoryType for NugetRepositoryType { } } -impl NugetRepository { - pub async fn resolve_project_and_version( - &self, - path: &StoragePath, - ) -> Result { - match self { - NugetRepository::Hosted(repo) => repo.resolve_project(path).await, - NugetRepository::Proxy(_) | NugetRepository::Virtual(_) => { - Ok(ProjectResolution::default()) - } - } - } -} - #[cfg(test)] mod tests; diff --git a/pkgly/src/repository/php/utils.rs b/pkgly/src/repository/php/utils.rs index b7b00bd..2404b7d 100644 --- a/pkgly/src/repository/php/utils.rs +++ b/pkgly/src/repository/php/utils.rs @@ -17,10 +17,6 @@ impl PhpPackagePathInfo { format!("{}/{}", self.vendor, self.package) } - pub fn normalized_package_name(&self) -> String { - self.package_name().to_ascii_lowercase() - } - pub fn project_storage_path(&self) -> String { format!( "{}/{}", diff --git a/pkgly/src/repository/proxy_indexing.rs b/pkgly/src/repository/proxy_indexing.rs index e614f11..7c7a36a 100644 --- a/pkgly/src/repository/proxy_indexing.rs +++ b/pkgly/src/repository/proxy_indexing.rs @@ -1,3 +1,5 @@ +// ABOUTME: Persists cached proxy artifacts in the project and package catalogs. +// ABOUTME: Tracks Docker object references alongside manifest catalog updates. use async_trait::async_trait; use nr_core::{ database::entities::project::{ @@ -130,6 +132,16 @@ impl ProxyIndexing for DatabaseProxyIndexer { return Err(ProxyIndexingError::MissingCachePath); } + if let Some(references) = &meta.docker_references { + nr_core::database::entities::docker_object::DBDockerObject::upsert( + &self.site.database, + self.repository_id, + &meta.cache_path, + meta.size.unwrap_or_default(), + references, + ) + .await?; + } let project = self.ensure_project(&meta).await?; let db = &self.site.database; diff --git a/pkgly/src/repository/python/hosted.rs b/pkgly/src/repository/python/hosted.rs index 63d7858..3c5432d 100644 --- a/pkgly/src/repository/python/hosted.rs +++ b/pkgly/src/repository/python/hosted.rs @@ -27,7 +27,7 @@ use uuid::Uuid; use super::{ PythonRepositoryError, - configs::{PythonRepositoryConfig, PythonRepositoryConfigType}, + configs::PythonRepositoryConfigType, utils::{PythonPackagePathInfo, html_escape, normalize_package_name}, }; use crate::{ @@ -59,8 +59,6 @@ pub struct PythonRepositoryInner { pub name: String, pub visibility: RwLock, pub repository: DBRepository, - #[allow(dead_code)] - pub config: PythonRepositoryConfig, pub storage: DynStorage, pub site: Pkgly, } @@ -73,7 +71,6 @@ impl PythonHosted { site: Pkgly, storage: DynStorage, repository: DBRepository, - config: PythonRepositoryConfig, ) -> Result { let visibility = RwLock::new(repository.visibility); Ok(Self(Arc::new(PythonRepositoryInner { @@ -81,7 +78,6 @@ impl PythonHosted { name: repository.name.to_string(), visibility, repository, - config, storage, site, }))) diff --git a/pkgly/src/repository/python/mod.rs b/pkgly/src/repository/python/mod.rs index cf09b90..adf7747 100644 --- a/pkgly/src/repository/python/mod.rs +++ b/pkgly/src/repository/python/mod.rs @@ -166,13 +166,7 @@ impl RepositoryType for PythonRepositoryType { match config { PythonRepositoryConfig::Hosted => { - let hosted = hosted::PythonHosted::load( - website, - storage, - repo, - PythonRepositoryConfig::Hosted, - ) - .await?; + let hosted = hosted::PythonHosted::load(website, storage, repo).await?; Ok(DynRepository::Python(PythonRepository::Hosted(hosted))) } PythonRepositoryConfig::Proxy(proxy_config) => { diff --git a/pkgly/src/repository/repo_http.rs b/pkgly/src/repository/repo_http.rs index 0ea2295..4b4235d 100644 --- a/pkgly/src/repository/repo_http.rs +++ b/pkgly/src/repository/repo_http.rs @@ -598,7 +598,6 @@ impl From>> for RepoResponse { } } } -#[allow(dead_code)] #[derive(Debug, Clone)] pub struct RepoRequestPath { storage: String, diff --git a/pkgly/src/repository/repo_http/repo_tracing.rs b/pkgly/src/repository/repo_http/repo_tracing.rs index 1cca4e4..6418ceb 100644 --- a/pkgly/src/repository/repo_http/repo_tracing.rs +++ b/pkgly/src/repository/repo_http/repo_tracing.rs @@ -2,22 +2,15 @@ use std::{error::Error, sync::Arc}; use nr_core::storage::StoragePath; use nr_storage::Storage; -use opentelemetry::{ - KeyValue, global, - metrics::{Histogram, Meter, UpDownCounter}, -}; +use opentelemetry::{KeyValue, global, metrics::Histogram}; use parking_lot::Mutex; use tracing::{Level, Span, event, field::Empty, info_span}; use super::DynRepository; use crate::repository::Repository; #[derive(Debug, Clone)] -/// Dead Code is allowed as this stuff is still in development -#[allow(dead_code)] pub struct RepositoryMetricsMeter { - meter: Meter, project_access_bytes: Histogram, - project_number_of_versions: UpDownCounter, project_write_bytes: Histogram, } impl Default for RepositoryMetricsMeter { @@ -25,10 +18,7 @@ impl Default for RepositoryMetricsMeter { let meter = global::meter("pkgly::repository::metrics"); Self { project_access_bytes: meter.u64_histogram("nr.project.access.bytes").build(), - project_number_of_versions: meter.i64_up_down_counter("nr.project.versions").build(), project_write_bytes: meter.u64_histogram("nr.project.write.bytes").build(), - - meter, } } } @@ -159,11 +149,6 @@ impl RepositoryRequestTracing { self.push_metric_and_span("project.version", &project_version); } } - pub fn add_metric_attribute(&self, key: &'static str, value: impl Into) { - let value = value.into(); - self.metrics.add_attribute(key, value); - } - fn push_metric_and_span(&self, key: &'static str, value: &str) { self.span.record(key, value); self.metrics.add_attribute(key, value.to_string()); diff --git a/pkgly/src/repository/repo_http/tests.rs b/pkgly/src/repository/repo_http/tests.rs index 654af32..1219e8b 100644 --- a/pkgly/src/repository/repo_http/tests.rs +++ b/pkgly/src/repository/repo_http/tests.rs @@ -39,7 +39,7 @@ fn repository_request_accepts_router_separator_before_path() { assert!(result.is_ok()); } -static DB_LOCK: LazyLock> = LazyLock::new(|| tokio::sync::Mutex::new(())); +static DB_LOCK: &LazyLock> = &crate::test_support::DB_TEST_LOCK; struct TestDb { pool: PgPool, diff --git a/pkgly/src/repository/staging.rs b/pkgly/src/repository/staging.rs index eab5133..88b7790 100644 --- a/pkgly/src/repository/staging.rs +++ b/pkgly/src/repository/staging.rs @@ -1,17 +1,13 @@ use std::{env, fmt::Debug, path::PathBuf, sync::Arc}; use axum::response::IntoResponse; -use bytes::Bytes; use chrono::Duration; use derive_more::derive::Deref; -use nr_core::database::entities::stages::{DBStage, NewDBStageFile}; -use redb::Result; use serde::{Deserialize, Serialize}; use thiserror::Error; -use tracing::{debug, error, instrument}; +use tracing::error; use uuid::Uuid; -use crate::app::Pkgly; #[derive(Debug, Error)] pub enum StagingManagerError { #[error("Database Error")] @@ -49,7 +45,6 @@ fn default_staging_directory() -> PathBuf { } pub struct StagingManagerInner { repository: Uuid, - site: Pkgly, } #[derive(Deref, Clone)] pub struct StagingManager(Arc); @@ -61,38 +56,3 @@ impl Debug for StagingManager { .finish() } } - -impl StagingManager { - pub async fn get_stage(&self, id: Uuid) -> Result, StagingManagerError> { - let stage = DBStage::get_stage_by_id(id, self.repository, &self.site.database).await?; - Ok(stage) - } - // This function will assume the stage exists. - #[instrument] - pub async fn add_file( - &self, - stage_id: Uuid, - file_name: String, - file: Bytes, - ) -> Result<(), StagingManagerError> { - let staging_dir = self - .site - .staging_config - .staging_dir - .join(stage_id.to_string()); - if !staging_dir.exists() { - std::fs::create_dir_all(&staging_dir)?; - } - let file_path = staging_dir.join(&file_name); - std::fs::write(file_path, file)?; - - let new_stage_file = NewDBStageFile { - stage: stage_id, - file_name, - }; - debug!(?new_stage_file, "Adding file to stage"); - let new_file = new_stage_file.insert(&self.site.database).await?; - debug!(?new_file, "File added to stage"); - Ok(()) - } -} diff --git a/pkgly/src/search/query.rs b/pkgly/src/search/query.rs index 669a0b3..1f78b25 100644 --- a/pkgly/src/search/query.rs +++ b/pkgly/src/search/query.rs @@ -9,10 +9,7 @@ use std::{sync::LazyLock, time::Instant}; use chrono::{DateTime, FixedOffset}; -use opentelemetry::{ - global, - metrics::{Histogram, Meter}, -}; +use opentelemetry::{global, metrics::Histogram}; use serde_json::Value; use sqlx::{FromRow, Postgres, QueryBuilder}; use uuid::Uuid; @@ -179,8 +176,6 @@ fn push_collated_lower(builder: &mut QueryBuilder, expression: &str) { } struct SearchMetrics { - #[allow(dead_code)] - meter: Meter, query_duration_ms: Histogram, rows_returned: Histogram, } @@ -198,7 +193,6 @@ impl SearchMetrics { .with_description("Rows returned per repository search query") .build(); Self { - meter, query_duration_ms, rows_returned, } diff --git a/pkgly/src/utils/header/date_time.rs b/pkgly/src/utils/header/date_time.rs index f197383..80d5028 100644 --- a/pkgly/src/utils/header/date_time.rs +++ b/pkgly/src/utils/header/date_time.rs @@ -1,17 +1,9 @@ use chrono::FixedOffset; use http::HeaderValue; -use crate::utils::bad_request::BadRequestErrors; - pub fn date_time_for_header(date_time: &chrono::DateTime) -> HeaderValue { let date_time = date_time.with_timezone(&chrono::Utc); let date_time = date_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string(); HeaderValue::from_str(date_time.as_str()) .unwrap_or_else(|_| panic!("Failed to convert date time to header")) } -pub fn parse_date_time( - header_value: &HeaderValue, -) -> Result, BadRequestErrors> { - let date_time = header_value.to_str()?; - chrono::DateTime::parse_from_rfc2822(date_time).map_err(BadRequestErrors::from) -} diff --git a/pkgly/src/utils/mod.rs b/pkgly/src/utils/mod.rs index c7e606c..f79a038 100644 --- a/pkgly/src/utils/mod.rs +++ b/pkgly/src/utils/mod.rs @@ -1,6 +1,3 @@ -use digestible::{Digester, Digestible, IntoBase64, byteorder::NativeEndian}; -use http::HeaderValue; -use sha2_0_11::Digest; pub mod requests; pub mod response; pub use response::*; @@ -10,12 +7,3 @@ pub mod other; pub mod request_logging; pub mod upstream; pub use requests::*; - -use self::builder::error::ResponseBuildError; - -pub fn generate_etag(data: &impl Digestible) -> Result { - let hasher = sha2_0_11::Sha256::new().into_base64(); - let result = hasher.digest::(data); - - Ok(HeaderValue::try_from(result)?) -} diff --git a/pkgly/src/utils/request_logging/access_log.rs b/pkgly/src/utils/request_logging/access_log.rs index e122792..0e9b52b 100644 --- a/pkgly/src/utils/request_logging/access_log.rs +++ b/pkgly/src/utils/request_logging/access_log.rs @@ -77,22 +77,10 @@ impl AccessLogContext { self.0.lock().storage_id = Some(storage_id); } - pub fn set_target_user_id(&self, user_id: i32) { - self.0.lock().target_user_id = Some(user_id); - } - - pub fn set_token_id(&self, token_id: i32) { - self.0.lock().token_id = Some(token_id); - } - pub fn set_audit_path(&self, path: impl Into) { self.0.lock().audit_path = Some(path.into()); } - pub fn set_audit_query(&self, query: impl Into) { - self.0.lock().audit_query = Some(query.into()); - } - pub fn set_client_address(&self, client_address: impl Into) { self.0.lock().client_address = Some(client_address.into()); } diff --git a/pkgly/src/utils/response/builder.rs b/pkgly/src/utils/response/builder.rs index c683d0c..5084d31 100644 --- a/pkgly/src/utils/response/builder.rs +++ b/pkgly/src/utils/response/builder.rs @@ -105,13 +105,6 @@ impl ResponseBuilder { Err(err) => err.into_response(), } } - /// Checks if the data is present and returns a JSON response or a not found response - pub fn json_or_not_found(self, data: &Option) -> Response { - match data { - Some(data) => self.json(data), - None => self.status(StatusCode::NOT_FOUND).empty(), - } - } pub fn html_or_err(self, html: impl Into>) -> Result { self.content_type(mime::TEXT_HTML_UTF_8) .body_or_err(html.into()) diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index ebec114..0000000 --- a/scripts/README.md +++ /dev/null @@ -1,307 +0,0 @@ -# Artifactory To Pkgly Migration - -[`artifactory_to_pkgly.py`](./artifactory_to_pkgly.py) migrates supported Artifactory `local` repositories into Pkgly hosted repositories. - -The script has two migration modes: - -- raw-copy for formats where canonical package files can be moved directly into Pkgly -- protocol-aware republish for formats where Pkgly expects the native publish API instead of raw path uploads - -The script prints one JSON object per repository. Each result includes: - -- `status` -- `discovered` -- `transferred` -- `skipped_existing` -- `skipped_filtered` -- `skipped_noncanonical` -- `skipped_unsupported_artifacts` -- `dry_run` - -## Supported Artifactory package types - -Source `packageType` values accepted by the script: - -- `maven` -- `gradle` -- `ivy` -- `sbt` -- `pypi` -- `helm` -- `composer` -- `npm` -- `nuget` -- `gems` -- `cargo` -- `go` -- `deb` -- `docker` - -Pkgly repository types created by `--create-targets`: - -- `maven|gradle|ivy|sbt -> maven` -- `pypi -> python` -- `helm -> helm` -- `composer -> php` -- `npm -> npm` -- `nuget -> nuget` -- `gems -> ruby` -- `cargo -> cargo` -- `go -> go` -- `deb -> deb` -- `docker -> docker` - -## How each type is migrated - -### Raw-copy types - -These use Artifactory file listing plus direct `PUT /repositories/{storage}/{repo}/{path}` into Pkgly: - -- `maven`, `gradle`, `ivy`, `sbt` - Maven-style paths are copied as-is. Checksum sidecars are skipped. -- `pypi` - Only distributions are copied. `simple/` and generated metadata are skipped. Files are normalized to `//`. -- `helm` - Only `.tgz` and `.tgz.prov` are copied. Target paths are normalized to `charts//-.*`. -- `composer` - Only dist ZIP archives are copied. `packages.json`, `p2/*.json`, and checksums are skipped. Target paths are normalized to `dist///.zip`. - -### Protocol-aware types - -These still discover source content in Artifactory, but they publish into Pkgly using the target repository's native API: - -- `npm` - Reads `.tgz` packages, extracts `package.json`, synthesizes an npm publish body, and publishes with `npm-command: publish`. -- `nuget` - Pushes `.nupkg` files to `/api/v2/package`. -- `gems` - Posts `.gem` files to `/api/v1/gems`. -- `cargo` - Reads `.crate` archives, extracts `Cargo.toml`, builds the crates.io publish payload, and publishes to `/api/v1/crates/new`. -- `go` - Groups canonical `/@v/.zip|mod|info` artifacts and publishes each complete version through `/upload`. -- `deb` - Uploads `.deb` files with multipart form-data. Pkgly regenerates `dists/` metadata. -- `docker` - Uses Artifactory's Docker V2 API to enumerate repositories, tags, manifests, and blobs, then pushes blobs and manifests into Pkgly's Docker V2 endpoints. - -## What gets skipped - -The script skips content on purpose in these cases: - -- generated repository metadata that Pkgly rebuilds itself -- checksum sidecars -- non-canonical Go module paths -- unsupported NuGet symbol packages (`.snupkg`, `.symbols.nupkg`) -- Docker schema1 manifests -- any leftover artifacts in a supported repository that do not map safely to a Pkgly publish flow - -By default the script skips and reports those artifacts instead of failing the whole repository. - -## Prerequisites - -- The Artifactory source repository must be `local`. -- The target Pkgly storage named by `--pkgly-storage` must already exist. -- The Pkgly credentials must be allowed to create repositories when `--create-targets` is used. -- The Pkgly credentials must have write access to the target repositories. -- For Docker migrations, the source Artifactory repository must expose the standard Docker API under `/artifactory/api/docker/{repo}/v2/...`. - -## Authentication - -The script supports bearer tokens or username/password pairs for both systems. - -CLI arguments: - -- `--artifactory-token` -- `--artifactory-user` -- `--artifactory-password` -- `--pkgly-token` -- `--pkgly-user` -- `--pkgly-password` - -Environment variable fallbacks: - -- `ARTIFACTORY_TOKEN` -- `ARTIFACTORY_USER` -- `ARTIFACTORY_PASSWORD` -- `PKGLY_TOKEN` -- `PKGLY_USER` -- `PKGLY_PASSWORD` - -If a token is provided, it wins over username/password. - -## Usage - -Show help: - -```bash -python3 scripts/artifactory_to_pkgly.py --help -``` - -Migrate one repository into an existing Pkgly repository: - -```bash -python3 scripts/artifactory_to_pkgly.py \ - --artifactory-url "https://artifactory.example.com" \ - --pkgly-url "https://pkgly.example.com" \ - --pkgly-storage "primary" \ - --repo "npm-local" -``` - -Create missing target repositories automatically: - -```bash -python3 scripts/artifactory_to_pkgly.py \ - --artifactory-url "https://artifactory.example.com" \ - --pkgly-url "https://pkgly.example.com" \ - --pkgly-storage "primary" \ - --create-targets \ - --repo "cargo-local" -``` - -Dry-run every supported repository: - -```bash -python3 scripts/artifactory_to_pkgly.py \ - --artifactory-url "https://artifactory.example.com" \ - --pkgly-url "https://pkgly.example.com" \ - --pkgly-storage "primary" \ - --all-repos \ - --dry-run -``` - -Use explicit Debian defaults when migrating `deb` repositories: - -```bash -python3 scripts/artifactory_to_pkgly.py \ - --artifactory-url "https://artifactory.example.com" \ - --pkgly-url "https://pkgly.example.com" \ - --pkgly-storage "primary" \ - --repo "apt-local" \ - --deb-distribution "stable" \ - --deb-component "main" \ - --deb-architectures "amd64,all" -``` - -## Important flags - -- `--repo ` - Select a repository to migrate. Repeat to migrate multiple repositories. - -- `--all-repos` - Migrate every repository discovered in Artifactory. Unsupported package types still report as failed. - -- `--path-prefix ` - Restrict Artifactory file listing to a subpath. This affects file-list based package types only; Docker discovery uses the Docker API instead. - -- `--parallelism ` - Number of repositories migrated concurrently. - -- `--timeout ` - HTTP timeout used for requests and uploads. - -- `--retries ` - Number of retries for retryable failures. - -- `--retry-backoff-seconds ` - Exponential backoff base delay. Each retry doubles the delay. - -- `--create-targets` - Create missing hosted target repositories in Pkgly. - -- `--dry-run` - Do not push artifacts. The script still validates repository existence and still performs existence checks. - -- `--deb-distribution` - Default Debian suite used for uploads and target repository creation. Default: `stable`. - -- `--deb-component` - Default Debian component used for uploads and target repository creation. Default: `main`. - -- `--deb-architectures` - Comma-separated architectures for Debian target creation. Default: `amd64,all`. - -## Retry behavior - -Retryable HTTP status codes: - -- `408` -- `425` -- `429` -- `500` -- `502` -- `503` -- `504` - -Retryable exception classes include: - -- `URLError` -- `HTTPException` -- `TimeoutError` -- transient `OSError` - -Non-retryable failures, such as `404`, invalid credentials, or malformed package archives, fail immediately. - -## Recommended workflow - -1. Run a `--dry-run` for one repository. -2. Migrate a small repository of the same package type. -3. Verify the package with the native client against Pkgly. -4. Migrate larger repositories. -5. Review repositories with non-zero `skipped_noncanonical` or `skipped_unsupported_artifacts` before deleting the source. - -## Limitations - -- Only Artifactory `local` repositories are supported. -- The script still has no checkpoint or resume file. -- Artifact processing is still sequential inside a single repository. -- Re-runs still use `HEAD` checks before upload/publish. -- Docker pagination is not implemented for very large Artifactory catalogs. -- Docker schema1 manifests are skipped. -- Cargo metadata extraction relies on readable `Cargo.toml` content inside the `.crate` archive. -- Debian uploads use the explicit CLI defaults for suite/component instead of inferring them from generated source metadata. - -## Verification examples - -Maven: - -```bash -curl -I \ - "$PKGLY_URL/repositories/primary/maven-local/com/acme/demo/1.0.0/demo-1.0.0.jar" -``` - -npm: - -```bash -curl \ - "$PKGLY_URL/repositories/primary/npm-local/@acme/demo" -``` - -NuGet: - -```bash -curl \ - "$PKGLY_URL/repositories/primary/nuget-local/v3/index.json" -``` - -Cargo: - -```bash -curl \ - "$PKGLY_URL/repositories/primary/cargo-local/index/config.json" -``` - -Go: - -```bash -curl \ - "$PKGLY_URL/repositories/primary/go-local/github.com/acme/demo/@v/list" -``` - -Docker: - -```bash -curl -I \ - -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ - "$PKGLY_URL/v2/primary/docker-local/library/demo/manifests/latest" -``` diff --git a/scripts/artifactory_to_pkgly.py b/scripts/artifactory_to_pkgly.py deleted file mode 100755 index 864e5f6..0000000 --- a/scripts/artifactory_to_pkgly.py +++ /dev/null @@ -1,2452 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import argparse -import base64 -import concurrent.futures -import hashlib -import http.client -import io -import json -import mimetypes -import os -import re -import sys -import tarfile -import time -import xml.etree.ElementTree as ET -import zipfile -from contextlib import closing -from dataclasses import dataclass -from typing import BinaryIO -from urllib import error, parse, request -from uuid import uuid4 - -try: - import tomllib # type: ignore[attr-defined] -except ModuleNotFoundError: # pragma: no cover - exercised on Python < 3.11 - tomllib = None - - -CHECKSUM_SUFFIXES = (".sha1", ".sha256", ".sha512", ".md5") -DEFAULT_TIMEOUT_SECONDS = 60 -DEFAULT_RETRIES = 3 -DEFAULT_RETRY_BACKOFF_SECONDS = 1.0 -DEFAULT_CONTENT_TYPE = "application/octet-stream" -DEFAULT_DEB_ARCHITECTURES = ("amd64", "all") -RETRYABLE_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504} -USER_AGENT = "pkgly-artifactory-to-pkgly/0.3" -SUPPORTED_REPO_TYPES = {"local"} -RAW_COPY_PACKAGE_TYPES = {"composer", "helm", "maven", "pypi"} -SUPPORTED_PACKAGE_TYPES = { - "cargo", - "composer", - "deb", - "docker", - "gems", - "go", - "helm", - "maven", - "npm", - "nuget", - "pypi", -} -PACKAGE_TYPE_ALIASES = { - "composer": "composer", - "cargo": "cargo", - "deb": "deb", - "docker": "docker", - "gems": "gems", - "go": "go", - "gradle": "maven", - "helm": "helm", - "ivy": "maven", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "pypi": "pypi", - "sbt": "maven", -} -PACKAGE_TYPE_TO_PKGLY_TYPE = { - "cargo": "cargo", - "composer": "php", - "deb": "deb", - "docker": "docker", - "gems": "ruby", - "go": "go", - "helm": "helm", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "pypi": "python", -} -CONTENT_TYPE_OVERRIDES = { - ".crate": "application/x-tar", - ".deb": "application/vnd.debian.binary-package", - ".ear": "application/java-archive", - ".gem": "application/octet-stream", - ".jar": "application/java-archive", - ".mod": "text/plain", - ".nupkg": "application/octet-stream", - ".pom": "application/xml", - ".prov": "application/pgp-signature", - ".snupkg": "application/octet-stream", - ".war": "application/java-archive", - ".whl": "application/zip", - ".xml": "application/xml", -} -DOCKER_ACCEPT_MEDIA_TYPES = ( - "application/vnd.oci.image.index.v1+json", - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.docker.distribution.manifest.v2+json", -) -DOCKER_SCHEMA1_MEDIA_TYPES = { - "application/vnd.docker.distribution.manifest.v1+json", - "application/vnd.docker.distribution.manifest.v1+prettyjws", -} - - -class HttpStatusError(RuntimeError): - def __init__(self, status_code: int, body: str) -> None: - super().__init__(f"HTTP {status_code}: {body}") - self.status_code = status_code - self.body = body - - -@dataclass(frozen=True) -class RepositoryDescriptor: - key: str - package_type: str - repo_type: str - - -@dataclass(frozen=True) -class ArtifactEntry: - path: str - size: int - - -@dataclass(frozen=True) -class RepositoryMigrationResult: - repository_key: str - package_type: str - repo_type: str - status: str - discovered: int = 0 - skipped_filtered: int = 0 - skipped_existing: int = 0 - skipped_noncanonical: int = 0 - skipped_unsupported_artifacts: int = 0 - transferred: int = 0 - dry_run: int = 0 - error: str = "" - - -@dataclass(frozen=True) -class ParsedNpmPackage: - package_name: str - version: str - tarball_path: str - publish_payload: dict - - -@dataclass(frozen=True) -class ParsedNugetPackage: - package_id: str - version: str - target_path: str - - -@dataclass(frozen=True) -class ParsedCargoPackage: - crate_name: str - version: str - target_path: str - payload_bytes: bytes - - -@dataclass(frozen=True) -class ParsedDebPackage: - package_name: str - version: str - architecture: str - target_path: str - - -@dataclass -class GoArtifactGroup: - zip_entry: ArtifactEntry | None = None - mod_entry: ArtifactEntry | None = None - info_entry: ArtifactEntry | None = None - - -def normalize_base_url(url: str) -> str: - return url.rstrip("/") - - -def normalize_path(path: str) -> str: - return path.strip("/") - - -def quote_path(path: str) -> str: - return parse.quote(path, safe="/-._~:@+") - - -def build_auth_header(token: str | None, username: str | None, password: str | None) -> str | None: - if token: - return f"Bearer {token}" - if username is None and password is None: - return None - if username is None or password is None: - raise ValueError("username and password must be provided together") - credentials = f"{username}:{password}".encode("utf-8") - encoded = base64.b64encode(credentials).decode("ascii") - return f"Basic {encoded}" - - -def resolve_argument(value: str | None, env_name: str) -> str | None: - if value is not None: - return value - env_value = os.environ.get(env_name) - return env_value or None - - -def map_artifactory_package_type(package_type: str) -> str: - return PACKAGE_TYPE_ALIASES.get(package_type.lower(), package_type.lower()) - - -def is_retryable_exception(exc: Exception) -> bool: - if isinstance(exc, HttpStatusError): - return exc.status_code in RETRYABLE_STATUS_CODES - if isinstance(exc, error.HTTPError): - return exc.code in RETRYABLE_STATUS_CODES - return isinstance(exc, (TimeoutError, error.URLError, http.client.HTTPException, OSError)) - - -def retry_operation( - description: str, - operation, - *, - retries: int, - backoff_seconds: float, - sleep=time.sleep, -): - for attempt in range(retries + 1): - try: - return operation() - except Exception as exc: - if attempt >= retries or not is_retryable_exception(exc): - raise - delay = backoff_seconds * (2**attempt) - print( - f"Retrying {description} after {delay:.1f}s due to: {exc}", - file=sys.stderr, - ) - sleep(delay) - - -def guess_content_type(path: str) -> str: - lowered = path.lower() - for suffix, content_type in CONTENT_TYPE_OVERRIDES.items(): - if lowered.endswith(suffix): - return content_type - guessed, _ = mimetypes.guess_type(path) - return guessed or DEFAULT_CONTENT_TYPE - - -def _normalize_python_package_name(name: str) -> str: - return name.lower().replace("_", "-").replace(".", "-") - - -def _parse_python_filename(filename: str) -> tuple[str, str] | None: - if filename.endswith(".whl"): - stem = filename[:-4] - parts = stem.split("-") - if len(parts) not in {5, 6}: - return None - return parts[0], parts[1] - - suffixes = (".tar.gz", ".tar.bz2", ".tgz", ".zip", ".egg", ".tar") - for suffix in suffixes: - if filename.endswith(suffix): - stem = filename[: -len(suffix)] - if suffix == ".egg": - parts = stem.rsplit("-", 2) - if len(parts) != 3: - return None - return parts[0], parts[1] - parts = stem.rsplit("-", 1) - if len(parts) != 2: - return None - return parts[0], parts[1] - return None - - -def _resolve_python_target_path(path: str) -> str | None: - normalized = normalize_path(path) - if not normalized or normalized.lower().endswith(CHECKSUM_SUFFIXES): - return None - - parts = normalized.split("/") - filename = parts[-1] - parsed_filename = _parse_python_filename(filename) - if parsed_filename is None: - return None - package, version = parsed_filename - - if len(parts) >= 3: - path_package = parts[-3] - path_version = parts[-2] - if _normalize_python_package_name(path_package) == _normalize_python_package_name( - package - ) and path_version == version: - return "/".join([path_package, path_version, filename]) - - return "/".join([package, version, filename]) - - -def _resolve_helm_target_path(path: str) -> str | None: - normalized = normalize_path(path) - if not normalized or normalized.lower().endswith(CHECKSUM_SUFFIXES): - return None - filename = normalized.rsplit("/", 1)[-1] - if filename.endswith(".tgz.prov"): - stem = filename[: -len(".tgz.prov")] - suffix = ".tgz.prov" - elif filename.endswith(".tgz"): - stem = filename[: -len(".tgz")] - suffix = ".tgz" - else: - return None - name, sep, version = stem.rpartition("-") - if not sep or not name or not version: - return None - return f"charts/{name}/{name}-{version}{suffix}" - - -def _resolve_php_target_path(path: str) -> str | None: - normalized = normalize_path(path) - if not normalized or normalized.lower().endswith(CHECKSUM_SUFFIXES): - return None - parts = normalized.split("/") - if parts[0] == "dist": - parts = parts[1:] - if len(parts) < 3: - return None - vendor = parts[0] - package = parts[1] - filename = parts[-1] - if not filename.endswith(".zip"): - return None - return f"dist/{vendor}/{package}/{filename}" - - -def resolve_target_path(package_type: str, path: str) -> str | None: - normalized = normalize_path(path) - canonical_type = map_artifactory_package_type(package_type) - if not normalized: - return None - if canonical_type == "maven": - if normalized.lower().endswith(CHECKSUM_SUFFIXES): - return None - return normalized - if canonical_type == "helm": - return _resolve_helm_target_path(normalized) - if canonical_type == "composer": - return _resolve_php_target_path(normalized) - if canonical_type == "pypi": - return _resolve_python_target_path(normalized) - return None - - -def parse_repositories_response(payload: list[dict]) -> list[RepositoryDescriptor]: - repositories = [] - for item in payload: - repositories.append( - RepositoryDescriptor( - key=str(item["key"]), - package_type=str(item.get("packageType", "")).lower(), - repo_type=str(item.get("repoType", "")).lower(), - ) - ) - return repositories - - -def parse_artifactory_file_list(payload: dict) -> list[ArtifactEntry]: - entries = [] - for item in payload.get("files", []): - item_type = str(item.get("type", "file")).lower() - if item_type == "folder": - continue - path = str(item.get("path", item.get("uri", ""))).lstrip("/") - entries.append(ArtifactEntry(path=path, size=int(item.get("size", 0)))) - return entries - - -def select_repositories( - available: list[RepositoryDescriptor], - requested_names: list[str], - all_repositories: bool, -) -> tuple[list[RepositoryDescriptor], list[str]]: - by_key = {repo.key: repo for repo in available} - if all_repositories: - return list(available), [] - selected = [] - missing = [] - for name in requested_names: - repo = by_key.get(name) - if repo is None: - missing.append(name) - continue - selected.append(repo) - return selected, missing - - -def _request_bytes( - url: str, - *, - method: str, - auth_header: str | None, - timeout: int, - payload: bytes | None = None, - headers: dict[str, str] | None = None, -) -> tuple[int, dict[str, str], bytes]: - request_headers = {"User-Agent": USER_AGENT} - if auth_header: - request_headers["Authorization"] = auth_header - if headers: - request_headers.update(headers) - req = request.Request(url, data=payload, headers=request_headers, method=method) - try: - with request.urlopen(req, timeout=timeout) as response: - return response.status, dict(response.headers.items()), response.read() - except error.HTTPError as exc: - body = exc.read() - return exc.code, dict(exc.headers.items()), body - - -def _json_request( - url: str, - *, - method: str, - auth_header: str | None, - timeout: int, - payload: dict | None = None, - headers: dict[str, str] | None = None, -) -> dict | list: - request_headers = {"Accept": "application/json"} - if headers: - request_headers.update(headers) - data = None - if payload is not None: - request_headers["Content-Type"] = "application/json" - data = json.dumps(payload).encode("utf-8") - status, _, body = _request_bytes( - url, - method=method, - auth_header=auth_header, - timeout=timeout, - payload=data, - headers=request_headers, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, body.decode("utf-8", errors="replace")) - if not body: - return {} - return json.loads(body.decode("utf-8")) - - -def _head_request( - url: str, - *, - auth_header: str | None, - timeout: int, - headers: dict[str, str] | None = None, -) -> int: - status, _, _ = _request_bytes( - url, - method="HEAD", - auth_header=auth_header, - timeout=timeout, - headers=headers, - ) - return status - - -def _stream_put( - url: str, - *, - auth_header: str | None, - content_type: str, - timeout: int, - stream: BinaryIO, - size: int, - headers: dict[str, str] | None = None, -) -> None: - parsed = parse.urlsplit(url) - connection_class = ( - http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection - ) - connection = connection_class(parsed.hostname, parsed.port, timeout=timeout) - request_path = parsed.path or "/" - if parsed.query: - request_path = f"{request_path}?{parsed.query}" - - try: - connection.putrequest("PUT", request_path) - connection.putheader("User-Agent", USER_AGENT) - connection.putheader("Content-Length", str(size)) - connection.putheader("Content-Type", content_type) - if auth_header: - connection.putheader("Authorization", auth_header) - for key, value in (headers or {}).items(): - connection.putheader(key, value) - connection.endheaders() - - while True: - chunk = stream.read(64 * 1024) - if not chunk: - break - connection.send(chunk) - - response = connection.getresponse() - body = response.read().decode("utf-8", errors="replace") - finally: - connection.close() - - if response.status < 200 or response.status >= 300: - raise HttpStatusError(response.status, body) - - -def _encode_multipart(fields: dict[str, str], files: list[tuple[str, str, str, bytes]]) -> tuple[str, bytes]: - boundary = f"----pkgly-migration-{uuid4().hex}" - body = bytearray() - for name, value in fields.items(): - body.extend(f"--{boundary}\r\n".encode("ascii")) - body.extend( - f'Content-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode("utf-8") - ) - for field_name, filename, content_type, content in files: - body.extend(f"--{boundary}\r\n".encode("ascii")) - body.extend( - ( - f'Content-Disposition: form-data; name="{field_name}"; ' - f'filename="{filename}"\r\n' - ).encode("utf-8") - ) - body.extend(f"Content-Type: {content_type}\r\n\r\n".encode("ascii")) - body.extend(content) - body.extend(b"\r\n") - body.extend(f"--{boundary}--\r\n".encode("ascii")) - return f"multipart/form-data; boundary={boundary}", bytes(body) - - -def _join_location(base_url: str, location: str) -> str: - return parse.urljoin(f"{normalize_base_url(base_url)}/", location) - - -def _safe_json_loads(payload: bytes) -> dict: - return json.loads(payload.decode("utf-8")) - - -def _version_sort_key(value: str) -> tuple: - parts = re.split(r"([0-9]+)", value) - key = [] - for part in parts: - if not part: - continue - if part.isdigit(): - key.append((0, int(part))) - else: - key.append((1, part)) - return tuple(key) - - -class ArtifactoryClient: - def __init__(self, *, base_url: str, auth_header: str | None, timeout: int) -> None: - self.base_url = normalize_base_url(base_url) - self.auth_header = auth_header - self.timeout = timeout - - def list_repositories(self) -> list[RepositoryDescriptor]: - payload = _json_request( - f"{self.base_url}/artifactory/api/repositories", - method="GET", - auth_header=self.auth_header, - timeout=self.timeout, - ) - return parse_repositories_response(payload) - - def list_files(self, repository_key: str, path_prefix: str = "") -> list[ArtifactEntry]: - prefix = normalize_path(path_prefix) - base = f"{self.base_url}/artifactory/api/repo/{parse.quote(repository_key)}/list" - if prefix: - base = f"{base}/{quote_path(prefix)}" - else: - base = f"{base}/" - payload = _json_request( - f"{base}?deep=1&listFolders=0", - method="GET", - auth_header=self.auth_header, - timeout=self.timeout, - ) - return parse_artifactory_file_list(payload) - - def open_file(self, repository_key: str, path: str): - quoted = quote_path(normalize_path(path)) - url = f"{self.base_url}/artifactory/{parse.quote(repository_key)}/{quoted}" - headers = {"User-Agent": USER_AGENT} - if self.auth_header: - headers["Authorization"] = self.auth_header - req = request.Request(url, headers=headers, method="GET") - return request.urlopen(req, timeout=self.timeout) - - def list_docker_images(self, repository_key: str) -> list[str]: - payload = _json_request( - f"{self.base_url}/artifactory/api/docker/{parse.quote(repository_key)}/v2/_catalog", - method="GET", - auth_header=self.auth_header, - timeout=self.timeout, - ) - return list(payload.get("repositories", [])) - - def list_docker_tags(self, repository_key: str, image_name: str) -> list[str]: - payload = _json_request( - f"{self.base_url}/artifactory/api/docker/{parse.quote(repository_key)}/v2/" - f"{quote_path(image_name)}/tags/list", - method="GET", - auth_header=self.auth_header, - timeout=self.timeout, - ) - return list(payload.get("tags", []) or []) - - def get_docker_manifest(self, repository_key: str, image_name: str, reference: str) -> tuple[bytes, str]: - status, headers, body = _request_bytes( - f"{self.base_url}/artifactory/api/docker/{parse.quote(repository_key)}/v2/" - f"{quote_path(image_name)}/manifests/{parse.quote(reference, safe=':@')}", - method="GET", - auth_header=self.auth_header, - timeout=self.timeout, - headers={"Accept": ", ".join(DOCKER_ACCEPT_MEDIA_TYPES)}, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, body.decode("utf-8", errors="replace")) - return body, headers.get("Content-Type", "").split(";", 1)[0].strip() - - def open_docker_blob(self, repository_key: str, image_name: str, digest: str): - url = ( - f"{self.base_url}/artifactory/api/docker/{parse.quote(repository_key)}/v2/" - f"{quote_path(image_name)}/blobs/{parse.quote(digest, safe=':')}" - ) - headers = {"User-Agent": USER_AGENT} - if self.auth_header: - headers["Authorization"] = self.auth_header - req = request.Request(url, headers=headers, method="GET") - return request.urlopen(req, timeout=self.timeout) - - -class PkglyClient: - def __init__(self, *, base_url: str, auth_header: str | None, timeout: int) -> None: - self.base_url = normalize_base_url(base_url) - self.auth_header = auth_header - self.timeout = timeout - - def repository_exists(self, storage_name: str, repository_name: str) -> bool: - try: - _json_request( - ( - f"{self.base_url}/api/repository/find-id/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}" - ), - method="GET", - auth_header=self.auth_header, - timeout=self.timeout, - ) - return True - except HttpStatusError as exc: - if exc.status_code == 404: - return False - raise - - def create_repository( - self, - storage_name: str, - repository_name: str, - package_type: str, - package_options: dict | None = None, - ) -> None: - if package_type not in PACKAGE_TYPE_TO_PKGLY_TYPE.values(): - raise RuntimeError(f"unsupported package type for target creation: {package_type}") - config = {"type": "Hosted"} - if package_options: - config.update(package_options) - payload = { - "name": repository_name, - "storage_name": storage_name, - "configs": { - package_type: config, - }, - } - _json_request( - f"{self.base_url}/api/repository/new/{parse.quote(package_type)}", - method="POST", - auth_header=self.auth_header, - timeout=self.timeout, - payload=payload, - ) - - def artifact_exists(self, storage_name: str, repository_name: str, relative_path: str) -> bool: - status = _head_request( - ( - f"{self.base_url}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}/{quote_path(relative_path)}" - ), - auth_header=self.auth_header, - timeout=self.timeout, - ) - if status == 200: - return True - if status == 404: - return False - raise RuntimeError(f"Pkgly HEAD check failed with HTTP {status}") - - def upload_file( - self, - storage_name: str, - repository_name: str, - relative_path: str, - stream: BinaryIO, - size: int, - ) -> None: - _stream_put( - ( - f"{self.base_url}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}/{quote_path(relative_path)}" - ), - auth_header=self.auth_header, - content_type=guess_content_type(relative_path), - timeout=self.timeout, - stream=stream, - size=size, - ) - - def publish_npm_package( - self, - storage_name: str, - repository_name: str, - package_name: str, - version: str, - tarball_path: str, - payload: dict, - ) -> None: - del version, tarball_path - body = json.dumps(payload).encode("utf-8") - status, _, response_body = _request_bytes( - ( - f"{self.base_url}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}/{quote_path(package_name)}" - ), - method="PUT", - auth_header=self.auth_header, - timeout=self.timeout, - payload=body, - headers={ - "Content-Type": "application/json", - "npm-command": "publish", - }, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, response_body.decode("utf-8", errors="replace")) - - def publish_nuget_package( - self, - storage_name: str, - repository_name: str, - package_id: str, - version: str, - filename: str, - package_bytes: bytes, - ) -> None: - del package_id, version - content_type, body = _encode_multipart( - {}, - [("package", filename, guess_content_type(filename), package_bytes)], - ) - status, _, response_body = _request_bytes( - ( - f"{self.base_url}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}/api/v2/package" - ), - method="PUT", - auth_header=self.auth_header, - timeout=self.timeout, - payload=body, - headers={"Content-Type": content_type}, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, response_body.decode("utf-8", errors="replace")) - - def publish_ruby_gem( - self, - storage_name: str, - repository_name: str, - filename: str, - gem_bytes: bytes, - ) -> None: - del filename - status, _, response_body = _request_bytes( - ( - f"{self.base_url}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}/api/v1/gems" - ), - method="POST", - auth_header=self.auth_header, - timeout=self.timeout, - payload=gem_bytes, - headers={"Content-Type": "application/octet-stream"}, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, response_body.decode("utf-8", errors="replace")) - - def publish_cargo_package( - self, - storage_name: str, - repository_name: str, - crate_name: str, - version: str, - payload_bytes: bytes, - ) -> None: - del crate_name, version - status, _, response_body = _request_bytes( - ( - f"{self.base_url}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}/api/v1/crates/new" - ), - method="PUT", - auth_header=self.auth_header, - timeout=self.timeout, - payload=payload_bytes, - headers={"Content-Type": "application/octet-stream"}, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, response_body.decode("utf-8", errors="replace")) - - def upload_go_module( - self, - storage_name: str, - repository_name: str, - module_name: str, - version: str, - module_zip: bytes, - go_mod: bytes | None, - info_json: bytes | None, - ) -> None: - fields = { - "version": version, - "module_name": module_name, - } - files = [("module", f"{module_name.rsplit('/', 1)[-1]}-{version}.zip", "application/zip", module_zip)] - if go_mod is not None: - files.append(("gomod", "go.mod", "text/plain", go_mod)) - if info_json is not None: - files.append(("info", f"{version}.info", "application/json", info_json)) - content_type, body = _encode_multipart(fields, files) - status, _, response_body = _request_bytes( - ( - f"{self.base_url}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}/upload" - ), - method="POST", - auth_header=self.auth_header, - timeout=self.timeout, - payload=body, - headers={"Content-Type": content_type}, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, response_body.decode("utf-8", errors="replace")) - - def upload_deb_package( - self, - storage_name: str, - repository_name: str, - distribution: str, - component: str, - filename: str, - package_bytes: bytes, - ) -> None: - content_type, body = _encode_multipart( - { - "distribution": distribution, - "component": component, - }, - [("package", filename, guess_content_type(filename), package_bytes)], - ) - status, _, response_body = _request_bytes( - ( - f"{self.base_url}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}" - ), - method="POST", - auth_header=self.auth_header, - timeout=self.timeout, - payload=body, - headers={"Content-Type": content_type}, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, response_body.decode("utf-8", errors="replace")) - - def docker_blob_exists(self, storage_name: str, repository_name: str, image_name: str, digest: str) -> bool: - status = _head_request( - ( - f"{self.base_url}/v2/{parse.quote(storage_name)}/{parse.quote(repository_name)}/" - f"{quote_path(image_name)}/blobs/{parse.quote(digest, safe=':')}" - ), - auth_header=self.auth_header, - timeout=self.timeout, - ) - if status == 200: - return True - if status == 404: - return False - raise RuntimeError(f"Pkgly Docker blob lookup failed with HTTP {status}") - - def upload_docker_blob( - self, - storage_name: str, - repository_name: str, - image_name: str, - digest: str, - blob_bytes: bytes, - ) -> None: - start_url = ( - f"{self.base_url}/v2/{parse.quote(storage_name)}/{parse.quote(repository_name)}/" - f"{quote_path(image_name)}/blobs/uploads/" - ) - status, headers, body = _request_bytes( - start_url, - method="POST", - auth_header=self.auth_header, - timeout=self.timeout, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, body.decode("utf-8", errors="replace")) - upload_url = _join_location(self.base_url, headers.get("Location", "")) - status, headers, body = _request_bytes( - upload_url, - method="PATCH", - auth_header=self.auth_header, - timeout=self.timeout, - payload=blob_bytes, - headers={"Content-Type": "application/octet-stream"}, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, body.decode("utf-8", errors="replace")) - finalize_url = _join_location(self.base_url, headers.get("Location", upload_url)) - separator = "&" if parse.urlsplit(finalize_url).query else "?" - finalize_url = f"{finalize_url}{separator}digest={parse.quote(digest, safe=':')}" - status, _, body = _request_bytes( - finalize_url, - method="PUT", - auth_header=self.auth_header, - timeout=self.timeout, - payload=b"", - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, body.decode("utf-8", errors="replace")) - - def docker_manifest_exists( - self, - storage_name: str, - repository_name: str, - image_name: str, - reference: str, - ) -> bool: - status = _head_request( - ( - f"{self.base_url}/v2/{parse.quote(storage_name)}/{parse.quote(repository_name)}/" - f"{quote_path(image_name)}/manifests/{parse.quote(reference, safe=':@')}" - ), - auth_header=self.auth_header, - timeout=self.timeout, - headers={"Accept": ", ".join(DOCKER_ACCEPT_MEDIA_TYPES)}, - ) - if status == 200: - return True - if status == 404: - return False - raise RuntimeError(f"Pkgly Docker manifest lookup failed with HTTP {status}") - - def upload_docker_manifest( - self, - storage_name: str, - repository_name: str, - image_name: str, - reference: str, - media_type: str, - manifest_bytes: bytes, - ) -> None: - status, _, body = _request_bytes( - ( - f"{self.base_url}/v2/{parse.quote(storage_name)}/{parse.quote(repository_name)}/" - f"{quote_path(image_name)}/manifests/{parse.quote(reference, safe=':@')}" - ), - method="PUT", - auth_header=self.auth_header, - timeout=self.timeout, - payload=manifest_bytes, - headers={"Content-Type": media_type}, - ) - if status < 200 or status >= 300: - raise HttpStatusError(status, body.decode("utf-8", errors="replace")) - - -def _read_artifactory_file_bytes( - artifactory: ArtifactoryClient, - repository_key: str, - path: str, -) -> bytes: - with closing(artifactory.open_file(repository_key, path)) as stream: - return stream.read() - - -def _read_artifactory_docker_blob_bytes( - artifactory: ArtifactoryClient, - repository_key: str, - image_name: str, - digest: str, -) -> bytes: - with closing(artifactory.open_docker_blob(repository_key, image_name, digest)) as stream: - return stream.read() - - -def _find_archive_member( - data: bytes, - *, - suffixes: tuple[str, ...] | None = None, - exact_names: tuple[str, ...] | None = None, -) -> bytes: - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: - for member in archive.getmembers(): - member_name = member.name.lstrip("./") - if exact_names and member_name in exact_names: - extracted = archive.extractfile(member) - if extracted is None: - break - return extracted.read() - if suffixes and member_name.endswith(suffixes): - extracted = archive.extractfile(member) - if extracted is None: - break - return extracted.read() - raise RuntimeError("required archive member not found") - - -def _split_toml_items(value: str, delimiter: str = ",") -> list[str]: - items = [] - current = [] - depth = 0 - quote = None - for character in value: - if quote is not None: - current.append(character) - if character == quote: - quote = None - continue - if character in {"'", '"'}: - quote = character - current.append(character) - continue - if character in "[{": - depth += 1 - elif character in "]}": - depth -= 1 - if character == delimiter and depth == 0: - items.append("".join(current).strip()) - current = [] - continue - current.append(character) - if current: - items.append("".join(current).strip()) - return [item for item in items if item] - - -def _strip_toml_comment(line: str) -> str: - result = [] - quote = None - for character in line: - if quote is not None: - result.append(character) - if character == quote: - quote = None - continue - if character in {"'", '"'}: - quote = character - result.append(character) - continue - if character == "#": - break - result.append(character) - return "".join(result).strip() - - -def _parse_toml_key_path(header: str) -> list[str]: - keys = [] - current = [] - quote = None - for character in header: - if quote is not None: - if character == quote: - quote = None - else: - current.append(character) - continue - if character in {"'", '"'}: - quote = character - continue - if character == ".": - keys.append("".join(current).strip()) - current = [] - continue - current.append(character) - keys.append("".join(current).strip()) - return [key for key in keys if key] - - -def _parse_toml_value(raw_value: str): - raw_value = raw_value.strip() - if raw_value.startswith('"') and raw_value.endswith('"'): - return raw_value[1:-1] - if raw_value.startswith("'") and raw_value.endswith("'"): - return raw_value[1:-1] - if raw_value == "true": - return True - if raw_value == "false": - return False - if raw_value.startswith("[") and raw_value.endswith("]"): - inner = raw_value[1:-1].strip() - if not inner: - return [] - return [_parse_toml_value(item) for item in _split_toml_items(inner)] - if raw_value.startswith("{") and raw_value.endswith("}"): - inner = raw_value[1:-1].strip() - if not inner: - return {} - parsed = {} - for item in _split_toml_items(inner): - key, _, value = item.partition("=") - parsed[key.strip()] = _parse_toml_value(value) - return parsed - if raw_value.isdigit(): - return int(raw_value) - return raw_value - - -def _load_toml(payload: bytes) -> dict: - if tomllib is not None: - return tomllib.loads(payload.decode("utf-8")) - - root: dict = {} - current = root - for raw_line in payload.decode("utf-8").splitlines(): - line = _strip_toml_comment(raw_line) - if not line: - continue - if line.startswith("[") and line.endswith("]"): - current = root - for key in _parse_toml_key_path(line[1:-1].strip()): - current = current.setdefault(key, {}) - continue - key, separator, value = line.partition("=") - if not separator: - raise RuntimeError(f"unsupported TOML syntax: {raw_line}") - current[key.strip()] = _parse_toml_value(value) - return root - - -def parse_npm_package_bytes( - payload: bytes, - *, - pkgly_base_url: str, - storage_name: str, - repository_name: str, -) -> ParsedNpmPackage: - package_json_bytes = _find_archive_member( - payload, - exact_names=("package/package.json", "package.json"), - ) - package_json = json.loads(package_json_bytes.decode("utf-8")) - package_name = str(package_json["name"]) - version = str(package_json["version"]) - attachment_name = package_name.rsplit("/", 1)[-1] - tarball_filename = f"{attachment_name}-{version}.tgz" - tarball_path = f"{package_name}/-/{tarball_filename}" - tarball_url = ( - f"{normalize_base_url(pkgly_base_url)}/repositories/" - f"{parse.quote(storage_name)}/{parse.quote(repository_name)}/{quote_path(tarball_path)}" - ) - - publish_version = dict(package_json) - publish_version.update( - { - "name": package_name, - "version": version, - "_id": f"{package_name}@{version}", - "readme": package_json.get("readme", ""), - "readmeFilename": package_json.get("readmeFilename", ""), - "_nodeVersion": package_json.get("_nodeVersion", "migration"), - "_npmVersion": package_json.get("_npmVersion", "migration"), - "dist": { - "shasum": hashlib.sha1(payload).hexdigest(), - "integrity": "sha512-" - + base64.b64encode(hashlib.sha512(payload).digest()).decode("ascii"), - "tarball": tarball_url, - }, - } - ) - publish_payload = { - "name": package_name, - "versions": { - version: publish_version, - }, - "_attachments": { - tarball_path: { - "content_type": "application/octet-stream", - "data": base64.b64encode(payload).decode("ascii"), - "length": len(payload), - } - }, - } - return ParsedNpmPackage( - package_name=package_name, - version=version, - tarball_path=tarball_path, - publish_payload=publish_payload, - ) - - -def parse_nuget_package_bytes(filename: str, payload: bytes) -> ParsedNugetPackage: - with zipfile.ZipFile(io.BytesIO(payload)) as archive: - nuspec_name = next((name for name in archive.namelist() if name.endswith(".nuspec")), None) - if nuspec_name is None: - raise RuntimeError(f"{filename} does not contain a .nuspec file") - root = ET.fromstring(archive.read(nuspec_name)) - package_id = root.findtext(".//{*}id") - version = root.findtext(".//{*}version") - if not package_id or not version: - raise RuntimeError(f"{filename} is missing package id or version") - lower_id = package_id.lower() - lower_version = version.lower() - return ParsedNugetPackage( - package_id=package_id, - version=version, - target_path=f"v3/flatcontainer/{lower_id}/{lower_version}/{lower_id}.{lower_version}.nupkg", - ) - - -def _normalize_cargo_version_spec(value) -> str | None: - if value is None: - return None - if isinstance(value, str): - return value - if isinstance(value, dict): - version = value.get("version") - return str(version) if version is not None else None - return None - - -def _cargo_dependency_from_value(name: str, value, *, kind: str | None, target: str | None) -> dict: - if isinstance(value, str): - return { - "name": name, - "vers": value, - "optional": False, - "default_features": True, - "features": [], - "target": target, - "kind": kind, - "registry": None, - "package": None, - } - if not isinstance(value, dict): - raise RuntimeError(f"unsupported Cargo dependency declaration for {name}") - package_name = value.get("package") - return { - "name": name, - "vers": _normalize_cargo_version_spec(value), - "optional": bool(value.get("optional", False)), - "default_features": bool(value.get("default-features", True)), - "features": list(value.get("features", [])), - "target": target, - "kind": kind, - "registry": value.get("registry"), - "package": package_name, - } - - -def _extract_cargo_metadata(payload: bytes) -> dict: - cargo_toml = _find_archive_member(payload, suffixes=("/Cargo.toml",)) - manifest = _load_toml(cargo_toml) - package = manifest.get("package") - if not isinstance(package, dict): - raise RuntimeError("Cargo.toml is missing [package]") - - dependencies = [] - for table_name, kind in ( - ("dependencies", None), - ("dev-dependencies", "dev"), - ("build-dependencies", "build"), - ): - section = manifest.get(table_name, {}) - if isinstance(section, dict): - for name, value in section.items(): - dependencies.append( - _cargo_dependency_from_value(name, value, kind=kind, target=None) - ) - - target_tables = manifest.get("target", {}) - if isinstance(target_tables, dict): - for target_name, target_config in target_tables.items(): - if not isinstance(target_config, dict): - continue - for table_name, kind in ( - ("dependencies", None), - ("dev-dependencies", "dev"), - ("build-dependencies", "build"), - ): - section = target_config.get(table_name, {}) - if not isinstance(section, dict): - continue - for name, value in section.items(): - dependencies.append( - _cargo_dependency_from_value(name, value, kind=kind, target=target_name) - ) - - return { - "name": str(package["name"]), - "vers": str(package["version"]), - "deps": dependencies, - "features": manifest.get("features", {}), - "authors": list(package.get("authors", [])), - "description": package.get("description"), - "documentation": package.get("documentation"), - "homepage": package.get("homepage"), - "repository": package.get("repository"), - "keywords": list(package.get("keywords", [])), - "categories": list(package.get("categories", [])), - "license": package.get("license"), - "license_file": package.get("license-file"), - "readme": package.get("readme"), - "readme_file": package.get("readme-file"), - "badges": manifest.get("badges", {}), - "links": package.get("links"), - } - - -def build_cargo_publish_body(payload: bytes) -> bytes: - metadata = json.dumps(_extract_cargo_metadata(payload), separators=(",", ":")).encode("utf-8") - return ( - len(metadata).to_bytes(4, "little") - + metadata - + len(payload).to_bytes(4, "little") - + payload - ) - - -def parse_cargo_publish_body(payload: bytes) -> dict: - if len(payload) < 8: - raise RuntimeError("truncated Cargo publish body") - metadata_len = int.from_bytes(payload[:4], "little") - metadata_end = 4 + metadata_len - metadata = json.loads(payload[4:metadata_end].decode("utf-8")) - crate_len = int.from_bytes(payload[metadata_end : metadata_end + 4], "little") - crate_archive = payload[metadata_end + 4 : metadata_end + 4 + crate_len] - return {"metadata": metadata, "crate_archive": crate_archive} - - -def parse_cargo_package_bytes(filename: str, payload: bytes) -> ParsedCargoPackage: - metadata = _extract_cargo_metadata(payload) - crate_name = str(metadata["name"]) - version = str(metadata["vers"]) - normalized = crate_name.lower() - return ParsedCargoPackage( - crate_name=crate_name, - version=version, - target_path=f"crates/{normalized}/{version}/{normalized}-{version}.crate", - payload_bytes=build_cargo_publish_body(payload), - ) - - -def _split_go_artifact_path(path: str) -> tuple[str, str, str] | None: - match = re.match(r"^(.+)/@v/([^/]+)\.(zip|mod|info)$", normalize_path(path)) - if match is None: - return None - return match.group(1), match.group(2), match.group(3) - - -def group_go_artifacts(entries: list[ArtifactEntry]) -> tuple[dict[tuple[str, str], GoArtifactGroup], int]: - groups: dict[tuple[str, str], GoArtifactGroup] = {} - skipped_noncanonical = 0 - for entry in entries: - parsed = _split_go_artifact_path(entry.path) - if parsed is None: - skipped_noncanonical += 1 - continue - module_name, version, extension = parsed - group = groups.setdefault((module_name, version), GoArtifactGroup()) - if extension == "zip": - group.zip_entry = entry - elif extension == "mod": - group.mod_entry = entry - else: - group.info_entry = entry - return groups, skipped_noncanonical - - -def _parse_ar_members(payload: bytes) -> dict[str, bytes]: - if not payload.startswith(b"!\n"): - raise RuntimeError("invalid ar archive") - offset = 8 - members: dict[str, bytes] = {} - while offset + 60 <= len(payload): - header = payload[offset : offset + 60] - offset += 60 - name = header[:16].decode("utf-8").strip() - size = int(header[48:58].decode("ascii").strip()) - content = payload[offset : offset + size] - offset += size - if size % 2 == 1: - offset += 1 - members[name.rstrip("/")] = content - return members - - -def _parse_deb_control(payload: bytes) -> dict[str, str]: - members = _parse_ar_members(payload) - control_name = next((name for name in members if name.startswith("control.tar")), None) - if control_name is None: - raise RuntimeError("deb package is missing control.tar.*") - with tarfile.open(fileobj=io.BytesIO(members[control_name]), mode="r:*") as archive: - for member in archive.getmembers(): - member_name = member.name.lstrip("./") - if member_name != "control": - continue - extracted = archive.extractfile(member) - if extracted is None: - break - lines = extracted.read().decode("utf-8").splitlines() - fields: dict[str, str] = {} - current_key = None - for line in lines: - if not line: - continue - if line.startswith(" ") and current_key is not None: - fields[current_key] += f"\n{line[1:]}" - continue - key, _, value = line.partition(":") - current_key = key.strip() - fields[current_key] = value.strip() - return fields - raise RuntimeError("deb package is missing control metadata") - - -def parse_deb_package_bytes(filename: str, payload: bytes, *, component: str) -> ParsedDebPackage: - control = _parse_deb_control(payload) - package_name = control.get("Package") - version = control.get("Version") - architecture = control.get("Architecture") - if not package_name or not version or not architecture: - raise RuntimeError(f"{filename} is missing Debian control fields") - first_letter = package_name[0].lower() - return ParsedDebPackage( - package_name=package_name, - version=version, - architecture=architecture, - target_path=f"pool/{component}/{first_letter}/{package_name}/{filename}", - ) - - -def _extract_docker_blob_digests(manifest: dict) -> list[str]: - digests = [] - config = manifest.get("config") - if isinstance(config, dict) and config.get("digest"): - digests.append(str(config["digest"])) - for layer in manifest.get("layers", []): - digest = layer.get("digest") - if digest: - digests.append(str(digest)) - return digests - - -def _extract_docker_child_manifests(manifest: dict) -> list[str]: - digests = [] - for child in manifest.get("manifests", []): - digest = child.get("digest") - if digest: - digests.append(str(digest)) - return digests - - -def _resolve_package_options(package_options: dict | None) -> dict: - package_options = package_options or {} - deb_distribution = package_options.get("deb_distribution", "stable") - deb_component = package_options.get("deb_component", "main") - deb_architectures = package_options.get( - "deb_architectures", list(DEFAULT_DEB_ARCHITECTURES) - ) - return { - "deb_distribution": deb_distribution, - "deb_component": deb_component, - "deb_architectures": list(deb_architectures), - } - - -def _target_repository_options(package_type: str, package_options: dict | None) -> dict | None: - if package_type != "deb": - return None - resolved = _resolve_package_options(package_options) - return { - "distributions": [resolved["deb_distribution"]], - "components": [resolved["deb_component"]], - "architectures": list(resolved["deb_architectures"]), - } - - -def prepare_target_repository( - *, - pkgly: PkglyClient, - storage_name: str, - repository: RepositoryDescriptor, - create_targets: bool, - package_options: dict | None, -) -> None: - package_type = map_artifactory_package_type(repository.package_type) - if package_type not in SUPPORTED_PACKAGE_TYPES: - raise RuntimeError(f"unsupported package type: {repository.package_type}") - if pkgly.repository_exists(storage_name, repository.key): - return - if not create_targets: - raise RuntimeError(f"target repository {storage_name}/{repository.key} does not exist") - pkgly.create_repository( - storage_name, - repository.key, - PACKAGE_TYPE_TO_PKGLY_TYPE[package_type], - _target_repository_options(package_type, package_options), - ) - - -def _empty_counts() -> dict[str, int]: - return { - "skipped_filtered": 0, - "skipped_existing": 0, - "skipped_noncanonical": 0, - "skipped_unsupported_artifacts": 0, - "transferred": 0, - "dry_run": 0, - } - - -def _migrate_raw_copy_repository( - *, - repository: RepositoryDescriptor, - canonical_package_type: str, - entries: list[ArtifactEntry], - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, -) -> dict[str, int]: - counts = _empty_counts() - for entry in entries: - target_path = resolve_target_path(canonical_package_type, entry.path) - if target_path is None: - counts["skipped_filtered"] += 1 - continue - if retry_operation( - f"check target artifact {repository.key}/{target_path}", - lambda target_path=target_path: pkgly.artifact_exists( - target_storage_name, repository.key, target_path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ): - counts["skipped_existing"] += 1 - continue - if dry_run: - counts["dry_run"] += 1 - continue - - def copy_entry() -> None: - with closing(artifactory.open_file(repository.key, entry.path)) as stream: - pkgly.upload_file( - target_storage_name, - repository.key, - target_path, - stream, - entry.size, - ) - - retry_operation( - f"copy artifact {repository.key}/{entry.path}", - copy_entry, - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - counts["transferred"] += 1 - return counts - - -def _migrate_npm_repository( - *, - repository: RepositoryDescriptor, - entries: list[ArtifactEntry], - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, -) -> dict[str, int]: - counts = _empty_counts() - tarball_entries = [entry for entry in entries if entry.path.endswith(".tgz")] - counts["skipped_unsupported_artifacts"] += len(entries) - len(tarball_entries) - - parsed_candidates = [] - for entry in tarball_entries: - package = retry_operation( - f"read npm package {repository.key}/{entry.path}", - lambda entry=entry: parse_npm_package_bytes( - _read_artifactory_file_bytes(artifactory, repository.key, entry.path), - pkgly_base_url=pkgly.base_url, - storage_name=target_storage_name, - repository_name=repository.key, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - parsed_candidates.append((package.package_name, package.version, entry, package)) - parsed_candidates.sort(key=lambda item: (item[0], _version_sort_key(item[1]))) - - for _, _, entry, package in parsed_candidates: - if retry_operation( - f"check npm tarball {repository.key}/{package.tarball_path}", - lambda package=package: pkgly.artifact_exists( - target_storage_name, repository.key, package.tarball_path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ): - counts["skipped_existing"] += 1 - continue - if dry_run: - counts["dry_run"] += 1 - continue - retry_operation( - f"publish npm package {repository.key}/{entry.path}", - lambda package=package: pkgly.publish_npm_package( - target_storage_name, - repository.key, - package.package_name, - package.version, - package.tarball_path, - package.publish_payload, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - counts["transferred"] += 1 - return counts - - -def _migrate_nuget_repository( - *, - repository: RepositoryDescriptor, - entries: list[ArtifactEntry], - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, -) -> dict[str, int]: - counts = _empty_counts() - for entry in entries: - if entry.path.endswith(".snupkg") or entry.path.endswith(".symbols.nupkg"): - counts["skipped_unsupported_artifacts"] += 1 - continue - if not entry.path.endswith(".nupkg"): - counts["skipped_unsupported_artifacts"] += 1 - continue - package_bytes = retry_operation( - f"read NuGet package {repository.key}/{entry.path}", - lambda entry=entry: _read_artifactory_file_bytes( - artifactory, repository.key, entry.path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - parsed = parse_nuget_package_bytes(entry.path.rsplit("/", 1)[-1], package_bytes) - if retry_operation( - f"check NuGet package {repository.key}/{parsed.target_path}", - lambda parsed=parsed: pkgly.artifact_exists( - target_storage_name, repository.key, parsed.target_path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ): - counts["skipped_existing"] += 1 - continue - if dry_run: - counts["dry_run"] += 1 - continue - retry_operation( - f"publish NuGet package {repository.key}/{entry.path}", - lambda parsed=parsed, package_bytes=package_bytes, entry=entry: pkgly.publish_nuget_package( - target_storage_name, - repository.key, - parsed.package_id, - parsed.version, - entry.path.rsplit("/", 1)[-1], - package_bytes, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - counts["transferred"] += 1 - return counts - - -def _ruby_target_path(path: str) -> str | None: - filename = normalize_path(path).rsplit("/", 1)[-1] - if filename.endswith(".gem"): - return f"gems/{filename}" - return None - - -def _migrate_ruby_repository( - *, - repository: RepositoryDescriptor, - entries: list[ArtifactEntry], - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, -) -> dict[str, int]: - counts = _empty_counts() - for entry in entries: - target_path = _ruby_target_path(entry.path) - if target_path is None: - counts["skipped_unsupported_artifacts"] += 1 - continue - if retry_operation( - f"check Ruby gem {repository.key}/{target_path}", - lambda target_path=target_path: pkgly.artifact_exists( - target_storage_name, repository.key, target_path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ): - counts["skipped_existing"] += 1 - continue - if dry_run: - counts["dry_run"] += 1 - continue - gem_bytes = retry_operation( - f"read Ruby gem {repository.key}/{entry.path}", - lambda entry=entry: _read_artifactory_file_bytes( - artifactory, repository.key, entry.path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - retry_operation( - f"publish Ruby gem {repository.key}/{entry.path}", - lambda gem_bytes=gem_bytes, entry=entry: pkgly.publish_ruby_gem( - target_storage_name, - repository.key, - entry.path.rsplit("/", 1)[-1], - gem_bytes, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - counts["transferred"] += 1 - return counts - - -def _migrate_cargo_repository( - *, - repository: RepositoryDescriptor, - entries: list[ArtifactEntry], - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, -) -> dict[str, int]: - counts = _empty_counts() - for entry in entries: - if not entry.path.endswith(".crate"): - counts["skipped_unsupported_artifacts"] += 1 - continue - crate_bytes = retry_operation( - f"read Cargo crate {repository.key}/{entry.path}", - lambda entry=entry: _read_artifactory_file_bytes( - artifactory, repository.key, entry.path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - parsed = parse_cargo_package_bytes(entry.path.rsplit("/", 1)[-1], crate_bytes) - if retry_operation( - f"check Cargo crate {repository.key}/{parsed.target_path}", - lambda parsed=parsed: pkgly.artifact_exists( - target_storage_name, repository.key, parsed.target_path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ): - counts["skipped_existing"] += 1 - continue - if dry_run: - counts["dry_run"] += 1 - continue - retry_operation( - f"publish Cargo crate {repository.key}/{entry.path}", - lambda parsed=parsed: pkgly.publish_cargo_package( - target_storage_name, - repository.key, - parsed.crate_name, - parsed.version, - parsed.payload_bytes, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - counts["transferred"] += 1 - return counts - - -def _migrate_go_repository( - *, - repository: RepositoryDescriptor, - entries: list[ArtifactEntry], - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, -) -> dict[str, int]: - counts = _empty_counts() - groups, skipped_noncanonical = group_go_artifacts(entries) - counts["skipped_noncanonical"] += skipped_noncanonical - for (module_name, version), group in groups.items(): - if group.zip_entry is None: - counts["skipped_noncanonical"] += 1 - continue - target_path = f"{module_name}/@v/{version}.zip" - if retry_operation( - f"check Go module {repository.key}/{target_path}", - lambda target_path=target_path: pkgly.artifact_exists( - target_storage_name, repository.key, target_path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ): - counts["skipped_existing"] += 1 - continue - if dry_run: - counts["dry_run"] += 1 - continue - module_zip = retry_operation( - f"read Go zip {repository.key}/{group.zip_entry.path}", - lambda group=group: _read_artifactory_file_bytes( - artifactory, repository.key, group.zip_entry.path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - go_mod = None - if group.mod_entry is not None: - go_mod = retry_operation( - f"read Go mod {repository.key}/{group.mod_entry.path}", - lambda group=group: _read_artifactory_file_bytes( - artifactory, repository.key, group.mod_entry.path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - info_json = None - if group.info_entry is not None: - info_json = retry_operation( - f"read Go info {repository.key}/{group.info_entry.path}", - lambda group=group: _read_artifactory_file_bytes( - artifactory, repository.key, group.info_entry.path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - retry_operation( - f"publish Go module {repository.key}/{module_name}@{version}", - lambda module_zip=module_zip, go_mod=go_mod, info_json=info_json: pkgly.upload_go_module( - target_storage_name, - repository.key, - module_name, - version, - module_zip, - go_mod, - info_json, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - counts["transferred"] += 1 - return counts - - -def _migrate_deb_repository( - *, - repository: RepositoryDescriptor, - entries: list[ArtifactEntry], - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, - package_options: dict | None, -) -> dict[str, int]: - counts = _empty_counts() - resolved = _resolve_package_options(package_options) - for entry in entries: - if not entry.path.endswith(".deb"): - counts["skipped_unsupported_artifacts"] += 1 - continue - package_bytes = retry_operation( - f"read deb package {repository.key}/{entry.path}", - lambda entry=entry: _read_artifactory_file_bytes( - artifactory, repository.key, entry.path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - filename = entry.path.rsplit("/", 1)[-1] - parsed = parse_deb_package_bytes( - filename, - package_bytes, - component=resolved["deb_component"], - ) - if retry_operation( - f"check deb package {repository.key}/{parsed.target_path}", - lambda parsed=parsed: pkgly.artifact_exists( - target_storage_name, repository.key, parsed.target_path - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ): - counts["skipped_existing"] += 1 - continue - if dry_run: - counts["dry_run"] += 1 - continue - retry_operation( - f"publish deb package {repository.key}/{entry.path}", - lambda filename=filename, package_bytes=package_bytes, resolved=resolved: pkgly.upload_deb_package( - target_storage_name, - repository.key, - resolved["deb_distribution"], - resolved["deb_component"], - filename, - package_bytes, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - counts["transferred"] += 1 - return counts - - -def _migrate_docker_manifest_reference( - *, - repository: RepositoryDescriptor, - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - image_name: str, - reference: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, - counts: dict[str, int], - visited: set[tuple[str, str]], -) -> None: - key = (image_name, reference) - if key in visited: - return - visited.add(key) - - if retry_operation( - f"check Docker manifest {repository.key}/{image_name}:{reference}", - lambda: pkgly.docker_manifest_exists( - target_storage_name, repository.key, image_name, reference - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ): - counts["skipped_existing"] += 1 - return - - manifest_bytes, media_type = retry_operation( - f"read Docker manifest {repository.key}/{image_name}:{reference}", - lambda: artifactory.get_docker_manifest(repository.key, image_name, reference), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - if media_type in DOCKER_SCHEMA1_MEDIA_TYPES: - counts["skipped_unsupported_artifacts"] += 1 - return - manifest = _safe_json_loads(manifest_bytes) - for child_digest in _extract_docker_child_manifests(manifest): - _migrate_docker_manifest_reference( - repository=repository, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - image_name=image_name, - reference=child_digest, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - counts=counts, - visited=visited, - ) - for digest in _extract_docker_blob_digests(manifest): - exists = retry_operation( - f"check Docker blob {repository.key}/{image_name}@{digest}", - lambda digest=digest: pkgly.docker_blob_exists( - target_storage_name, repository.key, image_name, digest - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - if exists: - continue - if dry_run: - continue - blob_bytes = retry_operation( - f"read Docker blob {repository.key}/{image_name}@{digest}", - lambda digest=digest: _read_artifactory_docker_blob_bytes( - artifactory, repository.key, image_name, digest - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - retry_operation( - f"upload Docker blob {repository.key}/{image_name}@{digest}", - lambda digest=digest, blob_bytes=blob_bytes: pkgly.upload_docker_blob( - target_storage_name, repository.key, image_name, digest, blob_bytes - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - if dry_run: - counts["dry_run"] += 1 - return - retry_operation( - f"upload Docker manifest {repository.key}/{image_name}:{reference}", - lambda: pkgly.upload_docker_manifest( - target_storage_name, - repository.key, - image_name, - reference, - media_type, - manifest_bytes, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - counts["transferred"] += 1 - - -def _migrate_docker_repository( - *, - repository: RepositoryDescriptor, - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - dry_run: bool, - retries: int, - retry_backoff_seconds: float, -) -> tuple[int, dict[str, int]]: - counts = _empty_counts() - discovered = 0 - images = retry_operation( - f"list Docker catalog for {repository.key}", - lambda: artifactory.list_docker_images(repository.key), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - visited: set[tuple[str, str]] = set() - for image_name in images: - tags = retry_operation( - f"list Docker tags for {repository.key}/{image_name}", - lambda image_name=image_name: artifactory.list_docker_tags(repository.key, image_name), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - discovered += len(tags) - for tag in tags: - _migrate_docker_manifest_reference( - repository=repository, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - image_name=image_name, - reference=tag, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - counts=counts, - visited=visited, - ) - return discovered, counts - - -def migrate_repository( - *, - repository: RepositoryDescriptor, - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - target_storage_name: str, - create_targets: bool, - path_prefix: str, - dry_run: bool, - retries: int = DEFAULT_RETRIES, - retry_backoff_seconds: float = DEFAULT_RETRY_BACKOFF_SECONDS, - package_options: dict | None = None, -) -> RepositoryMigrationResult: - canonical_package_type = map_artifactory_package_type(repository.package_type) - if repository.repo_type not in SUPPORTED_REPO_TYPES: - return RepositoryMigrationResult( - repository_key=repository.key, - package_type=repository.package_type, - repo_type=repository.repo_type, - status="failed", - error=f"unsupported repository class: {repository.repo_type}", - ) - if canonical_package_type not in SUPPORTED_PACKAGE_TYPES: - return RepositoryMigrationResult( - repository_key=repository.key, - package_type=repository.package_type, - repo_type=repository.repo_type, - status="failed", - error=f"unsupported package type: {repository.package_type}", - ) - - try: - retry_operation( - f"prepare target repository {repository.key}", - lambda: prepare_target_repository( - pkgly=pkgly, - storage_name=target_storage_name, - repository=repository, - create_targets=create_targets, - package_options=package_options, - ), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - - if canonical_package_type == "docker": - discovered, counts = _migrate_docker_repository( - repository=repository, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - else: - entries = retry_operation( - f"list files for {repository.key}", - lambda: artifactory.list_files(repository.key, path_prefix), - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - discovered = len(entries) - if canonical_package_type in RAW_COPY_PACKAGE_TYPES: - counts = _migrate_raw_copy_repository( - repository=repository, - canonical_package_type=canonical_package_type, - entries=entries, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - elif canonical_package_type == "npm": - counts = _migrate_npm_repository( - repository=repository, - entries=entries, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - elif canonical_package_type == "nuget": - counts = _migrate_nuget_repository( - repository=repository, - entries=entries, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - elif canonical_package_type == "gems": - counts = _migrate_ruby_repository( - repository=repository, - entries=entries, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - elif canonical_package_type == "cargo": - counts = _migrate_cargo_repository( - repository=repository, - entries=entries, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - elif canonical_package_type == "go": - counts = _migrate_go_repository( - repository=repository, - entries=entries, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - ) - elif canonical_package_type == "deb": - counts = _migrate_deb_repository( - repository=repository, - entries=entries, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - package_options=package_options, - ) - else: - raise RuntimeError(f"unsupported package type: {repository.package_type}") - except (RuntimeError, HttpStatusError, error.HTTPError) as exc: - if isinstance(exc, error.HTTPError): - body = exc.read().decode("utf-8", errors="replace") - message = f"HTTP {exc.code}: {body}" - else: - message = str(exc) - return RepositoryMigrationResult( - repository_key=repository.key, - package_type=repository.package_type, - repo_type=repository.repo_type, - status="failed", - error=message, - ) - - return RepositoryMigrationResult( - repository_key=repository.key, - package_type=repository.package_type, - repo_type=repository.repo_type, - status="success", - discovered=discovered, - skipped_filtered=counts["skipped_filtered"], - skipped_existing=counts["skipped_existing"], - skipped_noncanonical=counts["skipped_noncanonical"], - skipped_unsupported_artifacts=counts["skipped_unsupported_artifacts"], - transferred=counts["transferred"], - dry_run=counts["dry_run"], - ) - - -def migrate_repositories( - *, - artifactory: ArtifactoryClient, - pkgly: PkglyClient, - requested_names: list[str], - all_repositories: bool, - target_storage_name: str, - create_targets: bool, - path_prefix: str, - dry_run: bool, - parallelism: int, - retries: int = DEFAULT_RETRIES, - retry_backoff_seconds: float = DEFAULT_RETRY_BACKOFF_SECONDS, - executor_class=concurrent.futures.ThreadPoolExecutor, - package_options: dict | None = None, -) -> list[RepositoryMigrationResult]: - available = retry_operation( - "list Artifactory repositories", - artifactory.list_repositories, - retries=retries, - backoff_seconds=retry_backoff_seconds, - ) - selected, missing = select_repositories(available, requested_names, all_repositories) - - results = [] - with executor_class(max_workers=parallelism) as executor: - futures = [ - executor.submit( - migrate_repository, - repository=repository, - artifactory=artifactory, - pkgly=pkgly, - target_storage_name=target_storage_name, - create_targets=create_targets, - path_prefix=path_prefix, - dry_run=dry_run, - retries=retries, - retry_backoff_seconds=retry_backoff_seconds, - package_options=package_options, - ) - for repository in selected - ] - for future in futures: - results.append(future.result()) - - for name in missing: - results.append( - RepositoryMigrationResult( - repository_key=name, - package_type="", - repo_type="", - status="failed", - error="repository not found in Artifactory", - ) - ) - return results - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Migrate supported Artifactory repositories into Pkgly." - ) - parser.add_argument("--artifactory-url", required=True) - parser.add_argument("--artifactory-token") - parser.add_argument("--artifactory-user") - parser.add_argument("--artifactory-password") - - parser.add_argument("--pkgly-url", required=True) - parser.add_argument("--pkgly-storage", required=True) - parser.add_argument("--pkgly-token") - parser.add_argument("--pkgly-user") - parser.add_argument("--pkgly-password") - - parser.add_argument("--repo", action="append", default=[]) - parser.add_argument("--all-repos", action="store_true") - parser.add_argument("--path-prefix", default="") - parser.add_argument("--parallelism", type=int, default=4) - parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT_SECONDS) - parser.add_argument("--retries", type=int, default=DEFAULT_RETRIES) - parser.add_argument( - "--retry-backoff-seconds", - type=float, - default=DEFAULT_RETRY_BACKOFF_SECONDS, - ) - parser.add_argument("--create-targets", action="store_true") - parser.add_argument("--dry-run", action="store_true") - parser.add_argument("--deb-distribution", default="stable") - parser.add_argument("--deb-component", default="main") - parser.add_argument("--deb-architectures", default="amd64,all") - return parser - - -def main(argv: list[str] | None = None) -> int: - parser = _build_parser() - args = parser.parse_args(argv) - - if not args.all_repos and not args.repo: - parser.error("pass --repo at least once or use --all-repos") - if args.parallelism < 1: - parser.error("--parallelism must be at least 1") - if args.retries < 0: - parser.error("--retries must be at least 0") - if args.retry_backoff_seconds < 0: - parser.error("--retry-backoff-seconds must be at least 0") - - deb_architectures = [item.strip() for item in args.deb_architectures.split(",") if item.strip()] - if not deb_architectures: - parser.error("--deb-architectures must include at least one architecture") - - try: - artifactory_auth = build_auth_header( - resolve_argument(args.artifactory_token, "ARTIFACTORY_TOKEN"), - resolve_argument(args.artifactory_user, "ARTIFACTORY_USER"), - resolve_argument(args.artifactory_password, "ARTIFACTORY_PASSWORD"), - ) - pkgly_auth = build_auth_header( - resolve_argument(args.pkgly_token, "PKGLY_TOKEN"), - resolve_argument(args.pkgly_user, "PKGLY_USER"), - resolve_argument(args.pkgly_password, "PKGLY_PASSWORD"), - ) - except ValueError as exc: - parser.error(str(exc)) - package_options = { - "deb_distribution": args.deb_distribution, - "deb_component": args.deb_component, - "deb_architectures": deb_architectures, - } - - artifactory = ArtifactoryClient( - base_url=args.artifactory_url, - auth_header=artifactory_auth, - timeout=args.timeout, - ) - pkgly = PkglyClient( - base_url=args.pkgly_url, - auth_header=pkgly_auth, - timeout=args.timeout, - ) - results = migrate_repositories( - artifactory=artifactory, - pkgly=pkgly, - requested_names=args.repo, - all_repositories=args.all_repos, - target_storage_name=args.pkgly_storage, - create_targets=args.create_targets, - path_prefix=args.path_prefix, - dry_run=args.dry_run, - parallelism=args.parallelism, - retries=args.retries, - retry_backoff_seconds=args.retry_backoff_seconds, - package_options=package_options, - ) - for result in results: - print(json.dumps(result.__dict__, sort_keys=True)) - return 0 if all(result.status == "success" for result in results) else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/site/src/components/admin/repository/RepositoryListInner.vue b/site/src/components/admin/repository/RepositoryListInner.vue deleted file mode 100644 index cac3681..0000000 --- a/site/src/components/admin/repository/RepositoryListInner.vue +++ /dev/null @@ -1,259 +0,0 @@ - - - - diff --git a/site/src/components/admin/repository/__tests__/RepositoryListInner.spec.ts b/site/src/components/admin/repository/__tests__/RepositoryListInner.spec.ts deleted file mode 100644 index 6933c4d..0000000 --- a/site/src/components/admin/repository/__tests__/RepositoryListInner.spec.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { mount } from "@vue/test-utils"; -import { describe, expect, it, vi } from "vitest"; -import RepositoryListInner from "@/components/admin/repository/RepositoryListInner.vue"; - -vi.mock("@/router", () => ({ - default: { - push: vi.fn(), - }, -})); - -describe("RepositoryListInner.vue", () => { - it("shows a clear button that resets the search value", async () => { - const wrapper = mount(RepositoryListInner, { - props: { - repositories: [ - { - id: 1, - name: "Alpha", - storage_name: "Primary", - repository_type: "npm", - auth_enabled: true, - storage_usage_bytes: 0, - active: true, - storage_usage_updated_at: "2024-01-01T00:00:00Z", - }, - ], - }, - }); - - const input = wrapper.get("input#nameSearch"); - await input.setValue("Alpha"); - expect(wrapper.vm.searchValue).toBe("Alpha"); - - const clearButton = wrapper.find("[data-testid='repository-search-clear']"); - expect(clearButton.exists()).toBe(true); - - await clearButton.trigger("click"); - expect(wrapper.vm.searchValue).toBe(""); - }); -}); diff --git a/site/src/components/admin/repository/configs/MarkdownEditor.vue b/site/src/components/admin/repository/configs/MarkdownEditor.vue deleted file mode 100644 index b949716..0000000 --- a/site/src/components/admin/repository/configs/MarkdownEditor.vue +++ /dev/null @@ -1,46 +0,0 @@ - - - - diff --git a/site/src/components/core/WorkInProgress.vue b/site/src/components/core/WorkInProgress.vue deleted file mode 100644 index a74a9e5..0000000 --- a/site/src/components/core/WorkInProgress.vue +++ /dev/null @@ -1,3 +0,0 @@ - diff --git a/site/src/components/form/DatePicker.vue b/site/src/components/form/DatePicker.vue deleted file mode 100644 index 02ec963..0000000 --- a/site/src/components/form/DatePicker.vue +++ /dev/null @@ -1,25 +0,0 @@ - - - - diff --git a/site/src/components/form/TextArea.vue b/site/src/components/form/TextArea.vue deleted file mode 100644 index 610bfb5..0000000 --- a/site/src/components/form/TextArea.vue +++ /dev/null @@ -1,25 +0,0 @@ - - - - diff --git a/site/src/components/form/text/EmailInput.vue b/site/src/components/form/text/EmailInput.vue deleted file mode 100644 index 55fe2c8..0000000 --- a/site/src/components/form/text/EmailInput.vue +++ /dev/null @@ -1,24 +0,0 @@ - - - diff --git a/site/src/components/form/text/FixedTextInput.vue b/site/src/components/form/text/FixedTextInput.vue deleted file mode 100644 index e8f788c..0000000 --- a/site/src/components/form/text/FixedTextInput.vue +++ /dev/null @@ -1,21 +0,0 @@ - - - diff --git a/site/src/components/nav/sideNav/ExpandingSideNav.vue b/site/src/components/nav/sideNav/ExpandingSideNav.vue deleted file mode 100644 index 5cab16f..0000000 --- a/site/src/components/nav/sideNav/ExpandingSideNav.vue +++ /dev/null @@ -1,45 +0,0 @@ - - - diff --git a/site/src/components/nr/project/ShowProject.vue b/site/src/components/nr/project/ShowProject.vue deleted file mode 100644 index e69de29..0000000 diff --git a/site/src/components/nr/storage/s3/S3StorageConfig.vue b/site/src/components/nr/storage/s3/S3StorageConfig.vue index 8d89d65..5ac8008 100644 --- a/site/src/components/nr/storage/s3/S3StorageConfig.vue +++ b/site/src/components/nr/storage/s3/S3StorageConfig.vue @@ -1,3 +1,5 @@ + +