From c0ec0e4cdfde36e23babae0ce4f873c9f11e5b14 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Fri, 31 Oct 2025 21:44:53 +0545 Subject: [PATCH 01/22] feat(crate): init soar-dl crate --- Cargo.lock | 278 ++++++++++++++-- Cargo.toml | 50 ++- crates/soar-dl/Cargo.toml | 22 ++ crates/soar-dl/src/download.rs | 252 +++++++++++++++ crates/soar-dl/src/error.rs | 66 ++++ crates/soar-dl/src/filter.rs | 57 ++++ crates/soar-dl/src/github.rs | 88 ++++++ crates/soar-dl/src/gitlab.rs | 95 ++++++ crates/soar-dl/src/http.rs | 33 ++ crates/soar-dl/src/http_client.rs | 130 ++++++++ crates/soar-dl/src/lib.rs | 14 + crates/soar-dl/src/oci.rs | 508 ++++++++++++++++++++++++++++++ crates/soar-dl/src/platform.rs | 157 +++++++++ crates/soar-dl/src/release.rs | 128 ++++++++ crates/soar-dl/src/traits.rs | 30 ++ crates/soar-dl/src/types.rs | 26 ++ crates/soar-dl/src/utils.rs | 60 ++++ crates/soar-dl/src/xattr.rs | 20 ++ crates/soar-utils/Cargo.toml | 1 + crates/soar-utils/src/fs.rs | 15 +- crates/soar-utils/src/lib.rs | 1 + crates/soar-utils/src/string.rs | 29 ++ 22 files changed, 2018 insertions(+), 42 deletions(-) create mode 100644 crates/soar-dl/Cargo.toml create mode 100644 crates/soar-dl/src/download.rs create mode 100644 crates/soar-dl/src/error.rs create mode 100644 crates/soar-dl/src/filter.rs create mode 100644 crates/soar-dl/src/github.rs create mode 100644 crates/soar-dl/src/gitlab.rs create mode 100644 crates/soar-dl/src/http.rs create mode 100644 crates/soar-dl/src/http_client.rs create mode 100644 crates/soar-dl/src/lib.rs create mode 100644 crates/soar-dl/src/oci.rs create mode 100644 crates/soar-dl/src/platform.rs create mode 100644 crates/soar-dl/src/release.rs create mode 100644 crates/soar-dl/src/traits.rs create mode 100644 crates/soar-dl/src/types.rs create mode 100644 crates/soar-dl/src/utils.rs create mode 100644 crates/soar-dl/src/xattr.rs create mode 100644 crates/soar-utils/src/string.rs diff --git a/Cargo.lock b/Cargo.lock index e45d62cd4..40c491369 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + [[package]] name = "base64" version = "0.22.1" @@ -342,7 +351,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width", + "unicode-width 0.2.1", "windows-sys 0.60.2", ] @@ -361,6 +370,35 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -597,6 +635,15 @@ dependencies = [ "syn", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs 1.0.0", +] + [[package]] name = "documented" version = "0.9.2" @@ -605,7 +652,7 @@ checksum = "ed6b3e31251e87acd1b74911aed84071c8364fc9087972748ade2f1094ccce34" dependencies = [ "documented-macros", "phf 0.12.1", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -692,6 +739,15 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "fast-glob" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d26eec0ae9682c457cb0f85de67ad417b716ae852736a5d94c2ad6e92a997c9" +dependencies = [ + "arrayvec", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -743,9 +799,9 @@ checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -1152,9 +1208,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", "smallvec", @@ -1221,7 +1277,7 @@ checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" dependencies = [ "console", "portable-atomic", - "unicode-width", + "unicode-width 0.2.1", "unit-prefix", "web-time", ] @@ -1262,6 +1318,12 @@ dependencies = [ "serde", ] +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -1355,6 +1417,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.13" @@ -1412,6 +1480,36 @@ dependencies = [ "libc", ] +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "migrations_internals" version = "2.3.0" @@ -1547,6 +1645,12 @@ dependencies = [ "syn", ] +[[package]] +name = "owo-colors" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" + [[package]] name = "parking_lot" version = "0.12.4" @@ -1582,9 +1686,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "phf" @@ -1775,7 +1879,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls", "socket2 0.5.10", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -1796,7 +1900,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.16", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -2019,7 +2123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddfb91de75df4c93dc85781b3bc7e5d5ce3c2de9eb23dc7c4e3b82ae48b22ef8" dependencies = [ "fallible-iterator", - "litrs", + "litrs 0.4.2", "sqlite3-parser", ] @@ -2060,6 +2164,7 @@ version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -2068,6 +2173,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.12.0" @@ -2307,7 +2421,7 @@ dependencies = [ "serde", "serde_json", "soar-core", - "soar-dl", + "soar-dl 0.6.3", "soar-utils", "tokio", "toml", @@ -2328,10 +2442,10 @@ dependencies = [ "rusqlite", "serde", "serde_json", - "soar-dl", + "soar-dl 0.6.3", "soar-utils", "squishy", - "thiserror 2.0.16", + "thiserror 2.0.17", "toml", "toml_edit 0.23.5", "tracing", @@ -2356,7 +2470,7 @@ checksum = "77b09e1dc64c524e404153ea68db8396b6aece8c23c2427cfd079af50ad2a75e" dependencies = [ "blake3", "bzip2", - "fast-glob", + "fast-glob 0.4.5", "flate2", "futures", "regex", @@ -2371,12 +2485,29 @@ dependencies = [ "zstd", ] +[[package]] +name = "soar-dl" +version = "0.7.0" +dependencies = [ + "fast-glob 1.0.0", + "miette", + "regex", + "serde", + "serde_json", + "soar-utils", + "thiserror 2.0.17", + "ureq", + "url", + "xattr", +] + [[package]] name = "soar-utils" version = "0.1.0" dependencies = [ "blake3", "nix", + "regex", "serial_test", "tempfile", ] @@ -2411,7 +2542,7 @@ dependencies = [ "js-sys", "once_cell", "parking_lot", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "wasm-array-cp", "wasm-bindgen", @@ -2446,7 +2577,7 @@ dependencies = [ "backhand", "goblin", "rayon", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -2488,6 +2619,27 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f44ed3c63152de6a9f90acbea1a110441de43006ea51bcce8f436196a288b" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + [[package]] name = "syn" version = "2.0.106" @@ -2549,6 +2701,26 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "terminal_size" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" +dependencies = [ + "rustix", + "windows-sys 0.60.2", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "unicode-linebreak", + "unicode-width 0.2.1", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2560,11 +2732,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.16", + "thiserror-impl 2.0.17", ] [[package]] @@ -2580,9 +2752,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", @@ -2908,12 +3080,24 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-segmentation" version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.1" @@ -2932,17 +3116,57 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ba1025f18a4a3fc3e9b48c868e9beb4f24f4b4b1a325bada26bd4119f46537" +dependencies = [ + "base64", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b4531c118335662134346048ddb0e54cc86bd7e81866757873055f0e38f5d2" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + [[package]] name = "url" -version = "2.5.4" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3317,9 +3541,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", "rustix", diff --git a/Cargo.toml b/Cargo.toml index 7256a049f..0dc3b98ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,35 +1,46 @@ [workspace] +resolver = "2" members = [ "crates/soar-db", + "crates/soar-dl", "crates/soar-utils", "soar-cli", "soar-core" ] -resolver = "2" [workspace.package] authors = ["Rabindra Dhakal "] -license = "MIT" edition = "2021" -repository = "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/pkgforge/soar" -keywords = ["package-manager", "portable", "binary", "appimage", "linux"] readme = "README.md" +repository = "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/pkgforge/soar" +license = "MIT" +keywords = ["appimage", "binary", "linux", "package-manager", "portable"] categories = ["command-line-utilities"] -[profile.release] -strip = true -opt-level = 3 -lto = true -codegen-units = 1 -panic = "abort" - [workspace.dependencies] -diesel = { version = "2.3.2", features = ["64-column-tables", "sqlite", "returning_clauses_for_sqlite_3_35"] } +diesel = { version = "2.3.2", features = [ + "64-column-tables", + "returning_clauses_for_sqlite_3_35", + "sqlite" +] } diesel_migrations = { version = "2.3.0", features = ["sqlite"] } +fast-glob = "1.0.0" futures = "0.3.31" +miette = { version = "7.6.0", features = ["fancy"] } rayon = "1.11.0" -regex = { version = "1.11.2", default-features = false, features = ["unicode-case", "unicode-perl", "std"] } -reqwest = { version = "0.12.23", default-features = false, features = ["rustls-tls", "blocking", "http2", "json", "stream", "gzip"] } +regex = { version = "1.11.2", default-features = false, features = [ + "std", + "unicode-case", + "unicode-perl" +] } +reqwest = { version = "0.12.23", default-features = false, features = [ + "blocking", + "gzip", + "http2", + "json", + "rustls-tls", + "stream" +] } rusqlite = { version = "0.37.0", features = ["bundled", "rusqlite-macros"] } serde = { version = "1.0.225", features = ["derive"] } serde_json = { version = "1.0.145", features = ["indexmap"] } @@ -37,4 +48,15 @@ serial_test = "3.2.0" soar-dl = { version = "0.6.3" } soar-utils = { path = "crates/soar-utils" } tempfile = "3.10.1" +thiserror = "2.0.17" tracing = { version = "0.1.41", default-features = false } +ureq = { version = "3.1.2", features = ["json"] } +url = "2.5.7" +xattr = "1.6.1" + +[profile.release] +opt-level = 3 +strip = true +lto = true +panic = "abort" +codegen-units = 1 diff --git a/crates/soar-dl/Cargo.toml b/crates/soar-dl/Cargo.toml new file mode 100644 index 000000000..bc000ade2 --- /dev/null +++ b/crates/soar-dl/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "soar-dl" +version = "0.7.0" +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true +keywords.workspace = true +readme.workspace = true +categories.workspace = true + +[dependencies] +fast-glob = { workspace = true } +miette = { workspace = true } +regex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +soar-utils = { path = "../soar-utils" } +thiserror = { workspace = true } +ureq = { workspace = true } +url = { workspace = true } +xattr = { workspace = true } diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs new file mode 100644 index 000000000..538172e23 --- /dev/null +++ b/crates/soar-dl/src/download.rs @@ -0,0 +1,252 @@ +use std::{ + fs::{self, File, OpenOptions, Permissions}, + io::{Read as _, Write as _}, + os::unix::fs::PermissionsExt as _, + path::{Path, PathBuf}, +}; + +use soar_utils::fs::is_elf; +use ureq::{ + http::{ + header::{CONTENT_DISPOSITION, CONTENT_LENGTH, CONTENT_RANGE, ETAG}, + Response, + }, + Body, +}; + +use crate::{ + error::DownloadError, + http::Http, + types::{OverwriteMode, Progress, ResumeInfo}, + utils::{filename_from_header, filename_from_url, resolve_output_path}, + xattr::{read_resume, remove_resume, write_resume}, +}; + +pub struct Download { + pub url: String, + pub output: Option, + pub overwrite: OverwriteMode, + pub extract: bool, + pub extract_to: Option, + pub on_progress: Option>, +} + +impl Download { + pub fn new(url: impl Into) -> Self { + Self { + url: url.into(), + output: None, + overwrite: OverwriteMode::Prompt, + extract: false, + extract_to: None, + on_progress: None, + } + } + + pub fn output(mut self, output: impl Into) -> Self { + self.output = Some(output.into()); + self + } + + pub fn overwrite(mut self, overwrite: OverwriteMode) -> Self { + self.overwrite = overwrite; + self + } + + pub fn extract(mut self, extract: bool) -> Self { + self.extract = extract; + self + } + + pub fn extract_to(mut self, extract_to: impl Into) -> Self { + self.extract_to = Some(extract_to.into()); + self + } + + pub fn progress(mut self, on_progress: F) -> Self + where + F: Fn(Progress) + Send + Sync + 'static, + { + self.on_progress = Some(Box::new(on_progress)); + self + } + + pub fn execute(self) -> Result { + if self.output.as_deref() == Some("-") { + return self.download_to_stdout(); + } + + let resp = Http::fetch(&self.url, None, None)?; + + let header_filename = resp + .headers() + .get(CONTENT_DISPOSITION) + .and_then(filename_from_header); + let url_filename = filename_from_url(&self.url); + + let output_path = + resolve_output_path(self.output.as_deref(), url_filename, header_filename)?; + + let resume_info = if output_path.is_file() { + match self.overwrite { + OverwriteMode::Skip => return Ok(output_path), + OverwriteMode::Force => { + fs::remove_file(&output_path)?; + None + } + OverwriteMode::Prompt => { + if !prompt_overwrite(&output_path)? { + return Ok(output_path); + } + fs::remove_file(&output_path)?; + None + } + } + } else { + read_resume(&output_path) + }; + + if let Some(parent) = output_path.parent() { + std::fs::create_dir_all(parent)?; + } + + self.download_to_file(&output_path, resume_info)?; + + if is_elf(&output_path) { + std::fs::set_permissions(&output_path, Permissions::from_mode(0o755))?; + } + + remove_resume(&output_path)?; + + if self.extract { + let extract_dir = self.extract_to.unwrap_or_else(|| { + output_path + .parent() + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".")) + }); + + extract_archive(&output_path, &extract_dir)?; + } + + Ok(output_path) + } + + fn download_to_stdout(&self) -> Result { + let resp = Http::fetch(&self.url, None, None)?; + let mut stdout = std::io::stdout(); + let mut reader = resp.into_body().into_reader(); + + std::io::copy(&mut reader, &mut stdout)?; + stdout.flush()?; + + Ok(PathBuf::from("-")) + } + + fn download_to_file( + &self, + path: &Path, + resume_info: Option, + ) -> Result<(), DownloadError> { + let (resume_from, etag) = resume_info + .as_ref() + .map(|r| (Some(r.downloaded), r.etag.as_deref())) + .unwrap_or((None, None)); + + let resp = Http::fetch(&self.url, resume_from, etag)?; + + let status = resp.status(); + if resume_from.is_some() && status != 206 { + // Server doesn't support resume, start from the beginning + return self.download_to_file(path, None); + } + + let total = Self::parse_content_length(&resp); + let new_etag = resp + .headers() + .get(ETAG) + .and_then(|h| h.to_str().ok()) + .map(String::from); + + if let Some(ref cb) = self.on_progress { + cb(Progress::Starting { + total, + }); + } + + let mut file = if resume_from.is_some() { + OpenOptions::new().append(true).open(path)? + } else { + File::create(path)? + }; + + let mut reader = resp.into_body().into_reader(); + let mut buffer = [0u8; 8192]; + let mut downloaded = resume_from.unwrap_or(0); + + loop { + let n = reader.read(&mut buffer)?; + if n == 0 { + break; + } + + file.write_all(&buffer[..n])?; + downloaded += n as u64; + + if downloaded % (1024 * 1024) == 0 { + write_resume( + path, + &ResumeInfo { + downloaded, + total, + etag: new_etag.clone(), + last_modified: None, + }, + )?; + } + + if let Some(ref cb) = self.on_progress { + cb(Progress::Chunk { + current: downloaded, + total, + }); + } + } + + if let Some(ref cb) = self.on_progress { + cb(Progress::Complete { + total, + }); + } + + Ok(()) + } + + fn parse_content_length(resp: &Response) -> u64 { + resp.headers() + .get(CONTENT_RANGE) + .and_then(|h| h.to_str().ok()) + .and_then(|range| range.rsplit_once('/').and_then(|(_, tot)| tot.parse().ok())) + .or_else(|| { + resp.headers() + .get(CONTENT_LENGTH) + .and_then(|h| h.to_str().ok()) + .and_then(|len| len.parse::().ok()) + }) + .unwrap_or(0) + } +} + +fn prompt_overwrite(path: &Path) -> std::io::Result { + print!("Overwrite {}? [y/N] ", path.display()); + std::io::stdout().flush()?; + + let mut line = String::new(); + std::io::stdin().read_line(&mut line)?; + + Ok(matches!(line.trim().to_lowercase().as_str(), "y" | "yes")) +} + +pub fn extract_archive(archive_path: &Path, output_dir: &Path) -> Result<(), DownloadError> { + Ok(()) +} diff --git a/crates/soar-dl/src/error.rs b/crates/soar-dl/src/error.rs new file mode 100644 index 000000000..121da1617 --- /dev/null +++ b/crates/soar-dl/src/error.rs @@ -0,0 +1,66 @@ +use miette::Diagnostic; +use thiserror::Error; + +#[derive(Error, Diagnostic, Debug)] +pub enum DownloadError { + #[error("Invalid URL: {url}")] + #[diagnostic(code(soar_dl::invalid_url))] + InvalidUrl { + url: String, + #[source] + source: url::ParseError, + }, + + #[error("Network request failed")] + #[diagnostic( + code(soar_dl::network), + help("Check your internet connection or try again later") + )] + Network(#[from] Box), + + #[error("HTTP {status}: {url}")] + #[diagnostic(code(soar_dl::http_error))] + HttpError { status: u16, url: String }, + + #[error("I/O error: {0}")] + #[diagnostic(code(soar_dl::io))] + Io(#[from] std::io::Error), + + #[error("No matching assets found")] + #[diagnostic( + code(soar_dl::no_match), + help("Available assets:\n{}", .available.join("\n")) + )] + NoMatch { available: Vec }, + + #[error("Layer not found")] + #[diagnostic(code(soar_dl::layer_not_found))] + LayerNotFound, + + #[error("Invalid response from server")] + #[diagnostic(code(soar_dl::invalid_response))] + InvalidResponse, + + #[error("File name could not be determined")] + #[diagnostic( + code(soar_dl::no_filename), + help("Try specifying an output path explicitly") + )] + NoFilename, + + #[error("Resume metadata mismatch")] + #[diagnostic(code(soar_dl::resume_mismatch))] + ResumeMismatch, + + #[error("Multiple download errors occurred")] + #[diagnostic(code(soar_dl::multiple_errors))] + Multiple { errors: Vec }, +} + +pub type Result = miette::Result; + +impl From for DownloadError { + fn from(e: ureq::Error) -> Self { + Self::Network(Box::new(e)) + } +} diff --git a/crates/soar-dl/src/filter.rs b/crates/soar-dl/src/filter.rs new file mode 100644 index 000000000..ad3b474e9 --- /dev/null +++ b/crates/soar-dl/src/filter.rs @@ -0,0 +1,57 @@ +use fast_glob::glob_match; +use regex::Regex; + +#[derive(Debug, Clone, Default)] +pub struct Filter { + pub regexes: Vec, + pub globs: Vec, + pub include: Vec, + pub exclude: Vec, + pub case_sensitive: bool, +} + +impl Filter { + pub fn matches(&self, name: &str) -> bool { + let matches_regex = self.regexes.iter().all(|r| r.is_match(name)); + let matches_glob = self.globs.iter().any(|g| glob_match(g, name)); + let matches_include = self.matches_keywords(name, &self.include, true); + let matches_exclude = !self.matches_keywords(name, &self.exclude, false); + + matches_regex && matches_glob && matches_include && matches_exclude + } + + fn matches_keywords(&self, name: &str, keywords: &[String], must_match: bool) -> bool { + if keywords.is_empty() { + return true; + } + + let haystack = if self.case_sensitive { + name.to_string() + } else { + name.to_lowercase() + }; + + keywords.iter().all(|kw| { + let parts: Vec<_> = kw + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + + let any_match = parts.iter().any(|&part| { + let needle = if self.case_sensitive { + part.to_string() + } else { + part.to_lowercase() + }; + haystack.contains(&needle) + }); + + if must_match { + any_match + } else { + !any_match + } + }) + } +} diff --git a/crates/soar-dl/src/github.rs b/crates/soar-dl/src/github.rs new file mode 100644 index 000000000..38e449d6d --- /dev/null +++ b/crates/soar-dl/src/github.rs @@ -0,0 +1,88 @@ +use serde::Deserialize; + +use crate::{ + error::DownloadError, + platform::fetch_with_fallback, + traits::{Asset, Platform, Release}, +}; + +pub struct Github; + +#[derive(Debug, Clone, Deserialize)] +pub struct GithubRelease { + name: Option, + tag_name: String, + prerelease: bool, + published_at: String, + assets: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct GithubAsset { + name: String, + size: u64, + browser_download_url: String, +} + +impl Platform for Github { + type Release = GithubRelease; + + const API_PKGFORGE: &'static str = "https://api.gh.pkgforge.dev"; + const API_UPSTREAM: &'static str = "https://api.github.com"; + const TOKEN_ENV: &'static str = "GITHUB_TOKEN"; + + fn fetch_releases( + project: &str, + tag: Option<&str>, + ) -> Result, DownloadError> { + let path = match tag { + Some(tag) => format!("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/repos/{project}/releases/tags/{tag}?per_page=100"), + None => format!("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/repos/{project}/releases?per_page=100"), + }; + + fetch_with_fallback::( + &path, + Self::API_UPSTREAM, + Self::API_PKGFORGE, + Self::TOKEN_ENV, + ) + } +} + +impl Release for GithubRelease { + type Asset = GithubAsset; + + fn name(&self) -> &str { + self.name.as_deref().unwrap_or("") + } + + fn tag(&self) -> &str { + &self.tag_name + } + + fn is_prerelease(&self) -> bool { + self.prerelease + } + + fn published_at(&self) -> &str { + &self.published_at + } + + fn assets(&self) -> &[Self::Asset] { + &self.assets + } +} + +impl Asset for GithubAsset { + fn name(&self) -> &str { + &self.name + } + + fn size(&self) -> Option { + Some(self.size) + } + + fn url(&self) -> &str { + &self.browser_download_url + } +} diff --git a/crates/soar-dl/src/gitlab.rs b/crates/soar-dl/src/gitlab.rs new file mode 100644 index 000000000..bcd78669f --- /dev/null +++ b/crates/soar-dl/src/gitlab.rs @@ -0,0 +1,95 @@ +use serde::Deserialize; + +use crate::{ + error::DownloadError, + platform::fetch_with_fallback, + traits::{Asset, Platform, Release}, +}; + +pub struct GitLab; + +#[derive(Debug, Clone, Deserialize)] +pub struct GitLabRelease { + name: String, + tag_name: String, + upcoming_release: bool, + released_at: String, + assets: GitLabAssets, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct GitLabAssets { + pub links: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct GitLabAsset { + pub name: String, + pub direct_asset_url: String, +} + +impl Platform for GitLab { + type Release = GitLabRelease; + + const API_PKGFORGE: &'static str = "https://api.gl.pkgforge.dev"; + const API_UPSTREAM: &'static str = "https://gitlab.com"; + const TOKEN_ENV: &'static str = "GITLAB_TOKEN"; + + fn fetch_releases( + project: &str, + tag: Option<&str>, + ) -> Result, DownloadError> { + let encoded = project.replace('/', "%2F"); + let path = match tag { + Some(t) if project.chars().all(char::is_numeric) => { + format!("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/api/v4/projects/{}/releases/{}", encoded, t) + } + _ => format!("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/api/v4/projects/{}/releases", encoded), + }; + + fetch_with_fallback::( + &path, + Self::API_UPSTREAM, + Self::API_PKGFORGE, + Self::TOKEN_ENV, + ) + } +} + +impl Release for GitLabRelease { + type Asset = GitLabAsset; + + fn name(&self) -> &str { + &self.name + } + + fn tag(&self) -> &str { + &self.tag_name + } + + fn is_prerelease(&self) -> bool { + self.upcoming_release + } + + fn published_at(&self) -> &str { + &self.released_at + } + + fn assets(&self) -> &[Self::Asset] { + &self.assets.links + } +} + +impl Asset for GitLabAsset { + fn name(&self) -> &str { + &self.name + } + + fn size(&self) -> Option { + None + } + + fn url(&self) -> &str { + &self.direct_asset_url + } +} diff --git a/crates/soar-dl/src/http.rs b/crates/soar-dl/src/http.rs new file mode 100644 index 000000000..7cdeb5db6 --- /dev/null +++ b/crates/soar-dl/src/http.rs @@ -0,0 +1,33 @@ +use ureq::{http::Response, Body}; + +use crate::{error::DownloadError, http_client::SHARED_AGENT}; + +pub struct Http; + +impl Http { + pub fn fetch( + url: &str, + resume_from: Option, + etag: Option<&str>, + ) -> Result, DownloadError> { + let mut req = SHARED_AGENT.get(url); + + if let Some(pos) = resume_from { + req = req.header("Range", &format!("bytes={}-", pos)); + if let Some(tag) = etag { + req = req.header("If-Range", tag); + } + } + + req.call().map_err(DownloadError::from) + } + + pub fn json(url: &str) -> Result { + SHARED_AGENT + .get(url) + .call()? + .body_mut() + .read_json() + .map_err(|_| DownloadError::InvalidResponse) + } +} diff --git a/crates/soar-dl/src/http_client.rs b/crates/soar-dl/src/http_client.rs new file mode 100644 index 000000000..f0375c6ce --- /dev/null +++ b/crates/soar-dl/src/http_client.rs @@ -0,0 +1,130 @@ +use std::{ + sync::{Arc, LazyLock, RwLock}, + time::Duration, +}; + +use ureq::{ + http::{self, HeaderMap, Uri}, + typestate::{WithBody, WithoutBody}, + Agent, Proxy, RequestBuilder, +}; + +#[derive(Clone, Debug)] +pub struct ClientConfig { + pub user_agent: Option, + pub headers: Option, + pub proxy: Option, + pub timeout: Option, +} + +impl Default for ClientConfig { + fn default() -> Self { + Self { + user_agent: Some("pkgforge/soar".into()), + proxy: None, + headers: None, + timeout: None, + } + } +} + +impl ClientConfig { + pub fn build(&self) -> Agent { + let mut config = ureq::Agent::config_builder() + .proxy(self.proxy.clone()) + .timeout_global(self.timeout); + + if let Some(user_agent) = &self.user_agent { + config = config.user_agent(user_agent); + } + + config.build().into() + } +} + +struct SharedClient { + agent: Agent, + config: ClientConfig, +} + +static SHARED_CLIENT_STATE: LazyLock>> = LazyLock::new(|| { + let config = ClientConfig::default(); + let agent = config.build(); + + Arc::new(RwLock::new(SharedClient { + agent, + config, + })) +}); + +#[derive(Clone, Default)] +pub struct SharedAgent; + +impl SharedAgent { + pub fn new() -> Self { + Self + } + + pub fn get(&self, uri: T) -> RequestBuilder + where + Uri: TryFrom, + >::Error: Into, + { + let state = SHARED_CLIENT_STATE.read().unwrap(); + let req = state.agent.get(uri); + apply_headers(req, &state.config.headers) + } + + pub fn post(&self, uri: T) -> RequestBuilder + where + Uri: TryFrom, + >::Error: Into, + { + let state = SHARED_CLIENT_STATE.read().unwrap(); + let req = state.agent.post(uri); + apply_headers(req, &state.config.headers) + } + + pub fn put(&self, uri: T) -> RequestBuilder + where + Uri: TryFrom, + >::Error: Into, + { + let state = SHARED_CLIENT_STATE.read().unwrap(); + let req = state.agent.put(uri); + apply_headers(req, &state.config.headers) + } + + pub fn delete(&self, uri: T) -> RequestBuilder + where + Uri: TryFrom, + >::Error: Into, + { + let state = SHARED_CLIENT_STATE.read().unwrap(); + let req = state.agent.delete(uri); + apply_headers(req, &state.config.headers) + } +} + +fn apply_headers(mut req: RequestBuilder, headers: &Option) -> RequestBuilder { + if let Some(headers) = headers { + for (key, value) in headers.iter() { + req = req.header(key, value); + } + } + req +} + +pub static SHARED_AGENT: LazyLock = LazyLock::new(SharedAgent::new); + +pub fn configure_http_client(updater: F) +where + F: FnOnce(&mut ClientConfig), +{ + let mut state = SHARED_CLIENT_STATE.write().unwrap(); + let mut new_config = state.config.clone(); + updater(&mut new_config); + let new_agent = new_config.build(); + state.agent = new_agent; + state.config = new_config; +} diff --git a/crates/soar-dl/src/lib.rs b/crates/soar-dl/src/lib.rs new file mode 100644 index 000000000..f95e46fb2 --- /dev/null +++ b/crates/soar-dl/src/lib.rs @@ -0,0 +1,14 @@ +pub mod download; +pub mod error; +pub mod filter; +pub mod github; +pub mod gitlab; +pub mod http; +pub mod http_client; +pub mod oci; +pub mod platform; +pub mod release; +pub mod traits; +pub mod types; +pub mod utils; +pub mod xattr; diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs new file mode 100644 index 000000000..ae8300080 --- /dev/null +++ b/crates/soar-dl/src/oci.rs @@ -0,0 +1,508 @@ +use std::{ + fs::{File, OpenOptions, Permissions}, + io::{Read as _, Write as _}, + os::unix::fs::PermissionsExt as _, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + thread, +}; + +use serde::Deserialize; +use soar_utils::fs::is_elf; +use ureq::http::header::{ACCEPT, AUTHORIZATION, ETAG, IF_RANGE, RANGE}; + +use crate::{ + download::{extract_archive, Download}, + error::DownloadError, + filter::Filter, + http_client::SHARED_AGENT, + types::{OverwriteMode, Progress, ResumeInfo}, + xattr::{read_resume, remove_resume, write_resume}, +}; + +#[derive(Debug, Clone)] +pub struct OciReference { + pub registry: String, + pub package: String, + pub tag: String, +} + +impl From<&str> for OciReference { + fn from(value: &str) -> Self { + let paths = value.trim_start_matches("ghcr.io/"); + + // @sha256: + if let Some((package, digest)) = paths.split_once('@') { + return Self { + registry: "ghcr.io".to_string(), + package: package.to_string(), + tag: digest.to_string(), + }; + } + + // : + if let Some((package, tag)) = paths.split_once(':') { + return Self { + registry: "ghcr.io".to_string(), + package: package.to_string(), + tag: tag.to_string(), + }; + } + + Self { + registry: "ghcr.io".to_string(), + package: paths.to_string(), + tag: "latest".to_string(), + } + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct OciManifest { + #[serde(rename = "mediaType")] + pub media_type: String, + pub config: OciConfig, + pub layers: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct OciConfig { + #[serde(rename = "mediaType")] + pub media_type: String, + pub digest: String, + pub size: u64, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct OciLayer { + #[serde(rename = "mediaType")] + pub media_type: String, + pub digest: String, + pub size: u64, + #[serde(default)] + pub annotations: std::collections::HashMap, +} + +impl OciLayer { + pub fn title(&self) -> Option<&str> { + self.annotations + .get("org.opencontainers.image.title") + .map(|s| s.as_str()) + } +} + +pub struct OciDownload { + reference: OciReference, + api: String, + filter: Filter, + output: Option, + overwrite: OverwriteMode, + extract: bool, + extract_to: Option, + parallel: usize, + on_progress: Option>, +} + +impl OciDownload { + pub fn new(reference: impl Into) -> Self { + Self { + reference: reference.into(), + api: "https://ghcr.io/v2".into(), + filter: Filter::default(), + output: None, + overwrite: OverwriteMode::Prompt, + extract: false, + extract_to: None, + parallel: 1, + on_progress: None, + } + } + + pub fn api(mut self, api: impl Into) -> Self { + self.api = api.into(); + self + } + + pub fn filter(mut self, filter: Filter) -> Self { + self.filter = filter; + self + } + + pub fn output(mut self, path: impl Into) -> Self { + self.output = Some(path.into()); + self + } + + pub fn overwrite(mut self, mode: OverwriteMode) -> Self { + self.overwrite = mode; + self + } + + pub fn extract(mut self, extract: bool) -> Self { + self.extract = extract; + self + } + + pub fn extract_to(mut self, path: impl Into) -> Self { + self.extract_to = Some(path.into()); + self + } + + pub fn parallel(mut self, n: usize) -> Self { + self.parallel = n.max(1); + self + } + + pub fn progress(mut self, f: F) -> Self + where + F: Fn(Progress) + Send + Sync + 'static, + { + self.on_progress = Some(Arc::new(f)); + self + } + + pub fn execute(self) -> Result, DownloadError> { + // If it's a blob digest, download directly + if self.reference.tag.starts_with("sha256:") { + return self.download_blob(); + } + + let manifest = self.fetch_manifest()?; + + let layers: Vec<_> = manifest + .layers + .iter() + .filter(|layer| { + layer + .title() + .map(|t| self.filter.matches(t)) + .unwrap_or(false) + }) + .collect(); + + if layers.is_empty() { + return Err(DownloadError::LayerNotFound); + } + + let total_size: u64 = layers.iter().map(|l| l.size).sum(); + + if let Some(cb) = &self.on_progress { + cb(Progress::Starting { + total: total_size, + }); + } + + let output_dir = self + .output + .as_deref() + .map(PathBuf::from) + .unwrap_or_default(); + if !output_dir.as_os_str().is_empty() { + std::fs::create_dir_all(&output_dir)?; + } + + let paths = if self.parallel > 1 { + self.download_layers_parallel(&layers, &output_dir)? + } else { + self.download_layers_sequential(&layers, &output_dir)? + }; + + if let Some(cb) = &self.on_progress { + cb(Progress::Complete { + total: total_size, + }); + } + + Ok(paths) + } + + fn download_layers_sequential( + &self, + layers: &[&OciLayer], + output_dir: &Path, + ) -> Result, DownloadError> { + let mut paths = Vec::new(); + let mut downloaded = 0u64; + + for layer in layers { + let filename = layer.title().unwrap(); + let path = output_dir.join(filename); + + self.download_layer(layer, &path, &mut downloaded)?; + + if self.extract { + let extract_dir = self + .extract_to + .clone() + .unwrap_or_else(|| output_dir.to_path_buf()); + extract_archive(&path, &extract_dir)?; + } + + paths.push(path); + } + + Ok(paths) + } + + fn download_layers_parallel( + &self, + layers: &[&OciLayer], + output_dir: &Path, + ) -> Result, DownloadError> { + let downloaded = Arc::new(Mutex::new(0u64)); + let paths = Arc::new(Mutex::new(Vec::new())); + let errors = Arc::new(Mutex::new(Vec::new())); + + let owned_layers: Vec = layers.iter().map(|&layer| layer.clone()).collect(); + let chunks: Vec<_> = owned_layers + .chunks(layers.len().div_ceil(self.parallel)) + .map(|chunk| chunk.to_vec()) + .collect(); + + let handles: Vec<_> = chunks + .into_iter() + .map(|chunk| { + let api = self.api.clone(); + let reference = self.reference.clone(); + let output_dir = output_dir.to_path_buf(); + let downloaded = Arc::clone(&downloaded); + let paths = Arc::clone(&paths); + let errors = Arc::clone(&errors); + let extract = self.extract; + let extract_to = self.extract_to.clone(); + let on_progress = self.on_progress.clone(); + + thread::spawn(move || { + for layer in chunk { + let filename = match layer.title() { + Some(f) => f, + None => continue, + }; + let path = output_dir.join(filename); + + let mut local_downloaded = 0u64; + let result = download_layer_impl( + &api, + &reference, + &layer, + &path, + &mut local_downloaded, + on_progress.as_ref(), + &downloaded, + ); + + match result { + Ok(()) => { + if extract { + let extract_dir = + extract_to.clone().unwrap_or_else(|| output_dir.clone()); + if let Err(e) = extract_archive(&path, &extract_dir) { + errors.lock().unwrap().push(format!("{e}")); + continue; + } + } + paths.lock().unwrap().push(path); + } + Err(e) => { + errors.lock().unwrap().push(format!("{e}")); + } + } + } + }) + }) + .collect(); + + for handle in handles { + handle.join().ok(); + } + + let errors = errors.lock().unwrap(); + if !errors.is_empty() { + return Err(DownloadError::Multiple { + errors: errors.clone(), + }); + } + + let paths = paths.lock().unwrap().clone(); + Ok(paths) + } + + fn fetch_manifest(&self) -> Result { + let url = format!( + "{}/{}/manifests/{}", + self.api.trim_end_matches('/'), + self.reference.package, + self.reference.tag + ); + + let mut resp = SHARED_AGENT + .get(&url) + .header( + ACCEPT, + "application/vnd.docker.distribution.manifest.v2+json, \ + application/vnd.docker.distribution.manifest.list.v2+json, \ + application/vnd.oci.image.manifest.v1+json, \ + application/vnd.oci.image.index.v1+json", + ) + .header(AUTHORIZATION, "Bearer QQ==") + .call()?; + + if !resp.status().is_success() { + return Err(DownloadError::HttpError { + status: resp.status().as_u16(), + url, + }); + } + + resp.body_mut() + .read_json() + .map_err(|_| DownloadError::InvalidResponse) + } + + fn download_blob(&self) -> Result, DownloadError> { + let filename = self + .reference + .package + .rsplit_once('/') + .map(|(_, name)| name) + .unwrap_or(&self.reference.tag); + + let output = self.output.as_deref().unwrap_or(filename); + let path = PathBuf::from(output); + + let url = format!( + "{}/{}/blobs/{}", + self.api.trim_end_matches('/'), + self.reference.package, + self.reference.tag + ); + + let dl = Download::new(url).output(output).overwrite(self.overwrite); + + let dl = if let Some(ref cb) = self.on_progress { + let cb = cb.clone(); + dl.progress(move |p| cb(p)) + } else { + dl + }; + + dl.execute()?; + + Ok(vec![path]) + } + + fn download_layer( + &self, + layer: &OciLayer, + path: &Path, + downloaded: &mut u64, + ) -> Result<(), DownloadError> { + download_layer_impl( + &self.api, + &self.reference, + layer, + path, + downloaded, + self.on_progress.as_ref(), + &std::sync::Arc::new(std::sync::Mutex::new(0u64)), + ) + } +} + +fn download_layer_impl( + api: &str, + reference: &OciReference, + layer: &OciLayer, + path: &Path, + local_downloaded: &mut u64, + on_progress: Option<&Arc>, + shared_downloaded: &std::sync::Arc>, +) -> Result<(), DownloadError> { + let url = format!( + "{}/{}/blobs/{}", + api.trim_end_matches('/'), + reference.package, + layer.digest + ); + + let resume_info = read_resume(path); + let (resume_from, etag) = resume_info + .as_ref() + .map(|r| (Some(r.downloaded), r.etag.as_deref())) + .unwrap_or((None, None)); + + let mut req = SHARED_AGENT.get(&url).header(AUTHORIZATION, "Bearer QQ=="); + + if let Some(pos) = resume_from { + req = req.header(RANGE, &format!("bytes={}-", pos)); + if let Some(tag) = etag { + req = req.header(IF_RANGE, tag); + } + } + + let resp = req.call()?; + + if !resp.status().is_success() { + return Err(DownloadError::HttpError { + status: resp.status().as_u16(), + url, + }); + } + + let mut file = if resume_from.is_some() && resp.status() == 206 { + OpenOptions::new().append(true).open(path)? + } else { + File::create(path)? + }; + + let new_etag = resp + .headers() + .get(ETAG) + .and_then(|h| h.to_str().ok()) + .map(String::from); + let mut reader = resp.into_body().into_reader(); + let mut buffer = [0u8; 8192]; + *local_downloaded = resume_from.unwrap_or(0); + + loop { + let n = reader.read(&mut buffer)?; + if n == 0 { + break; + } + + file.write_all(&buffer[..n])?; + *local_downloaded += n as u64; + + { + let mut shared = shared_downloaded.lock().unwrap(); + *shared += n as u64; + } + + if (*local_downloaded).is_multiple_of(1024 * 1024) { + write_resume( + path, + &ResumeInfo { + downloaded: *local_downloaded, + total: layer.size, + etag: new_etag.clone(), + last_modified: None, + }, + )?; + } + + if let Some(cb) = on_progress { + let shared = shared_downloaded.lock().unwrap(); + cb(Progress::Chunk { + current: *shared, + total: layer.size, + }); + } + } + + if is_elf(path) { + std::fs::set_permissions(path, Permissions::from_mode(0o755))?; + } + + remove_resume(path)?; + Ok(()) +} diff --git a/crates/soar-dl/src/platform.rs b/crates/soar-dl/src/platform.rs new file mode 100644 index 000000000..1223e5973 --- /dev/null +++ b/crates/soar-dl/src/platform.rs @@ -0,0 +1,157 @@ +use std::{env, sync::LazyLock}; + +use regex::Regex; +use soar_utils::string::decode_uri; +use ureq::http::header::AUTHORIZATION; +use url::Url; + +use crate::{error::DownloadError, http_client::SHARED_AGENT}; + +pub enum ApiKind { + Pkgforge, + Primary, +} + +#[derive(Debug)] +pub enum PlatformUrl { + Github { + project: String, + tag: Option, + }, + Gitlab { + project: String, + tag: Option, + }, + Oci { + reference: String, + }, + Direct { + url: String, + }, +} + +static GITHUB_RE: LazyLock = LazyLock::new(|| { + Regex::new( + r"^(?i)(?:https?://)?(?:github(?:\.com)?[:/])([^/@]+/[^/@]+)(?:@([^/\s]+(?:/[^/\s]*)*)?)?$", + ) + .expect("unable to compile github release regex") +}); +static GITLAB_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"^(?i)(?:https?://)?(?:gitlab(?:\.com)?[:/])((?:\d+)|(?:[^/@]+(?:/[^/@]+)*))(?:@([^/\s]+(?:/[^/\s]*)*)?)?$") + .expect("unable to compile gitlab release regex") +}); + +impl PlatformUrl { + pub fn parse(url: impl AsRef) -> Option { + let url = url.as_ref(); + + if url.starts_with("ghcr.io") { + return Some(Self::Oci { + reference: url.to_string(), + }); + } + + if let Some((project, tag)) = Self::parse_repo(&GITHUB_RE, url) { + return Some(Self::Github { + project, + tag, + }); + } + + if let Some((project, tag)) = Self::parse_repo(&GITLAB_RE, url) { + if project.starts_with("api") || project.contains("/-/") { + return Url::parse(url).ok().map(|_| { + Self::Direct { + url: url.to_string(), + } + }); + } + return Some(Self::Gitlab { + project, + tag, + }); + } + + Url::parse(url) + .ok() + .filter(|u| !u.scheme().is_empty() && u.host().is_some()) + .map(|_| { + Self::Direct { + url: url.to_string(), + } + }) + } + + fn parse_repo(re: &Regex, url: &str) -> Option<(String, Option)> { + let caps = re.captures(url)?; + let project = caps.get(1)?.as_str().to_string(); + let tag = caps + .get(2) + .map(|m| m.as_str().trim_matches(&['\'', '"', ' '][..])) + .filter(|s| !s.is_empty()) + .map(decode_uri); + + Some((project, tag)) + } +} + +pub fn fetch_with_fallback( + path: &str, + primary: &str, + fallback: &str, + token_env: &str, +) -> Result, DownloadError> +where + T: serde::de::DeserializeOwned, +{ + let try_fetch = |base: &str, use_token: bool| -> Result, DownloadError> { + let url = format!("{}{}", base, path); + let mut req = SHARED_AGENT.get(&url); + + if use_token { + if let Ok(token) = env::var(token_env) { + req = req.header(AUTHORIZATION, &format!("Bearer {}", token)); + } + } + + let mut resp = req.call()?; + let status = resp.status(); + + if !status.is_success() { + return Err(DownloadError::HttpError { + status: status.as_u16(), + url: url.clone(), + }); + } + + let json: serde_json::Value = resp + .body_mut() + .read_json() + .map_err(|_| DownloadError::InvalidResponse)?; + + match json { + serde_json::Value::Array(_) => { + serde_json::from_value(json).map_err(|_| DownloadError::InvalidResponse) + } + serde_json::Value::Object(_) => { + let single: T = + serde_json::from_value(json).map_err(|_| DownloadError::InvalidResponse)?; + Ok(vec![single]) + } + _ => Err(DownloadError::InvalidResponse), + } + }; + + try_fetch(fallback, false).or_else(|e| { + if should_fallback_status(&e) { + try_fetch(primary, true) + } else { + Err(e) + } + }) +} + +fn should_fallback_status(e: &DownloadError) -> bool { + matches!(e, DownloadError::HttpError { status, .. } + if *status == 429 || *status == 401 || *status == 403 || *status >= 500) +} diff --git a/crates/soar-dl/src/release.rs b/crates/soar-dl/src/release.rs new file mode 100644 index 000000000..ad1539c49 --- /dev/null +++ b/crates/soar-dl/src/release.rs @@ -0,0 +1,128 @@ +use std::{path::PathBuf, sync::Arc}; + +use crate::{ + download::Download, + error::DownloadError, + filter::Filter, + traits::{Asset as _, Platform, Release as _}, + types::{OverwriteMode, Progress}, +}; + +pub struct ReleaseDownload { + project: String, + tag: Option, + filter: Filter, + output: Option, + overwrite: OverwriteMode, + extract: bool, + extract_to: Option, + on_progress: Option>, + _platform: std::marker::PhantomData

, +} + +impl ReleaseDownload

{ + pub fn new(project: impl Into) -> Self { + Self { + project: project.into(), + tag: None, + filter: Filter::default(), + output: None, + overwrite: OverwriteMode::Prompt, + extract: false, + extract_to: None, + on_progress: None, + _platform: std::marker::PhantomData, + } + } + + pub fn tag(mut self, tag: impl Into) -> Self { + self.tag = Some(tag.into()); + self + } + + pub fn filter(mut self, filter: Filter) -> Self { + self.filter = filter; + self + } + + pub fn output(mut self, path: impl Into) -> Self { + self.output = Some(path.into()); + self + } + + pub fn overwrite(mut self, mode: OverwriteMode) -> Self { + self.overwrite = mode; + self + } + + pub fn extract(mut self, extract: bool) -> Self { + self.extract = extract; + self + } + + pub fn extract_to(mut self, path: impl Into) -> Self { + self.extract_to = Some(path.into()); + self + } + + pub fn progress(mut self, f: F) -> Self + where + F: Fn(Progress) + Send + Sync + 'static, + { + self.on_progress = Some(Arc::new(f)); + self + } + + pub fn execute(self) -> Result, DownloadError> { + let releases = P::fetch_releases(&self.project, self.tag.as_deref())?; + + let release = if let Some(ref tag) = self.tag { + releases.iter().find(|r| r.tag() == tag) + } else { + releases + .iter() + .find(|r| !r.is_prerelease()) + .or_else(|| releases.first()) + }; + + let release = release.ok_or_else(|| DownloadError::InvalidResponse)?; + + let assets: Vec<_> = release + .assets() + .iter() + .filter(|a| self.filter.matches(a.name())) + .collect(); + + if assets.is_empty() { + return Err(DownloadError::NoMatch { + available: release + .assets() + .iter() + .map(|a| a.name().to_string()) + .collect(), + }); + } + + let mut paths = Vec::new(); + for asset in assets { + let dl = Download::new(asset.url()) + .output(self.output.clone().unwrap_or_default()) + .overwrite(self.overwrite) + .extract(self.extract) + .extract_to(self.extract_to.clone().unwrap_or_default()); + + let dl = if let Some(ref cb) = self.on_progress { + let cb = cb.clone(); + dl.progress(move |p| cb(p)) + } else { + dl + }; + + let path = dl.execute()?; + + paths.push(path); + } + + Ok(paths) + } +} diff --git a/crates/soar-dl/src/traits.rs b/crates/soar-dl/src/traits.rs new file mode 100644 index 000000000..ab6b4a823 --- /dev/null +++ b/crates/soar-dl/src/traits.rs @@ -0,0 +1,30 @@ +use crate::error::DownloadError; + +pub trait Asset: Clone { + fn name(&self) -> &str; + fn size(&self) -> Option; + fn url(&self) -> &str; +} + +pub trait Release { + type Asset: Asset; + + fn name(&self) -> &str; + fn tag(&self) -> &str; + fn is_prerelease(&self) -> bool; + fn published_at(&self) -> &str; + fn assets(&self) -> &[Self::Asset]; +} + +pub trait Platform { + type Release: Release; + + const API_UPSTREAM: &'static str; + const API_PKGFORGE: &'static str; + const TOKEN_ENV: &'static str; + + fn fetch_releases( + project: &str, + tag: Option<&str>, + ) -> Result, DownloadError>; +} diff --git a/crates/soar-dl/src/types.rs b/crates/soar-dl/src/types.rs new file mode 100644 index 000000000..18b0f52e7 --- /dev/null +++ b/crates/soar-dl/src/types.rs @@ -0,0 +1,26 @@ +use serde::{Deserialize, Serialize}; + +/// Download progress events +#[derive(Debug, Clone, Copy)] +pub enum Progress { + Starting { total: u64 }, + Chunk { current: u64, total: u64 }, + Complete { total: u64 }, +} + +/// How to handle existing files +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OverwriteMode { + Skip, + Force, + Prompt, +} + +/// Resume information stored in xattrs +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResumeInfo { + pub downloaded: u64, + pub total: u64, + pub etag: Option, + pub last_modified: Option, +} diff --git a/crates/soar-dl/src/utils.rs b/crates/soar-dl/src/utils.rs new file mode 100644 index 000000000..e27327dd2 --- /dev/null +++ b/crates/soar-dl/src/utils.rs @@ -0,0 +1,60 @@ +use std::path::PathBuf; + +use ureq::http::HeaderValue; +use url::Url; + +use crate::error::DownloadError; + +/// Extract filename from URL path +pub fn filename_from_url(url: &str) -> Option { + Url::parse(url).ok().and_then(|u| { + u.path_segments() + .and_then(|mut s| s.next_back()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + }) +} + +/// Extract filename from Content-Disposition header +pub fn filename_from_header(value: &HeaderValue) -> Option { + value + .to_str() + .ok()? + .split(',') + .find_map(|p| p.trim().strip_prefix("filename=")) + .map(|s| s.trim_matches('"').to_string()) +} + +/// Determine output path +pub fn resolve_output_path( + output: Option<&str>, + url_filename: Option, + header_filename: Option, +) -> Result { + match output { + Some("-") => Ok(PathBuf::from("-")), + Some(p) if p.ends_with('/') => { + let filename = header_filename + .or(url_filename) + .ok_or(DownloadError::NoFilename)?; + Ok(PathBuf::from(p).join(filename)) + } + Some(p) => { + let path = PathBuf::from(p); + if path.is_dir() { + let filename = header_filename + .or(url_filename) + .ok_or(DownloadError::NoFilename)?; + Ok(path.join(filename)) + } else { + Ok(path) + } + } + None => { + header_filename + .or(url_filename) + .map(PathBuf::from) + .ok_or(DownloadError::NoFilename) + } + } +} diff --git a/crates/soar-dl/src/xattr.rs b/crates/soar-dl/src/xattr.rs new file mode 100644 index 000000000..c974b41aa --- /dev/null +++ b/crates/soar-dl/src/xattr.rs @@ -0,0 +1,20 @@ +use std::path::Path; + +use crate::types::ResumeInfo; + +const XATTR_RESUME_KEY: &str = "user.soar.resume"; + +pub fn read_resume>(path: P) -> Option { + xattr::get(path, XATTR_RESUME_KEY) + .ok() + .flatten() + .and_then(|v| serde_json::from_slice(&v).ok()) +} + +pub fn write_resume>(path: P, info: &ResumeInfo) -> std::io::Result<()> { + xattr::set(path, XATTR_RESUME_KEY, &serde_json::to_vec(info)?) +} + +pub fn remove_resume>(path: P) -> std::io::Result<()> { + xattr::remove(path, XATTR_RESUME_KEY) +} diff --git a/crates/soar-utils/Cargo.toml b/crates/soar-utils/Cargo.toml index 2ab1751e6..58ecbcb6a 100644 --- a/crates/soar-utils/Cargo.toml +++ b/crates/soar-utils/Cargo.toml @@ -13,6 +13,7 @@ categories.workspace = true [dependencies] blake3 = { version = "1.8.2", features = ["mmap"] } nix = { version = "0.30.1", features = ["ioctl", "term", "user"] } +regex = { workspace = true } serial_test = { workspace = true } [dev-dependencies] diff --git a/crates/soar-utils/src/fs.rs b/crates/soar-utils/src/fs.rs index 6a3ccb9fc..7adddda7c 100644 --- a/crates/soar-utils/src/fs.rs +++ b/crates/soar-utils/src/fs.rs @@ -271,13 +271,26 @@ pub fn dir_size>(path: P) -> FileSystemResult { Ok(total_size) } +/// Check if file is ELF binary +pub fn is_elf>(path: P) -> bool { + std::fs::read(path.as_ref()) + .ok() + .and_then(|bytes| { + bytes + .get(..4) + .map(|magic| magic == [0x7f, 0x45, 0x4c, 0x46]) + }) + .unwrap_or(false) +} + #[cfg(test)] mod tests { use std::{fs::Permissions, os::unix::fs::PermissionsExt}; - use super::*; use tempfile::tempdir; + use super::*; + #[test] fn test_safe_remove_file() { let dir = tempdir().unwrap(); diff --git a/crates/soar-utils/src/lib.rs b/crates/soar-utils/src/lib.rs index 72d85f9b8..0dc23fd41 100644 --- a/crates/soar-utils/src/lib.rs +++ b/crates/soar-utils/src/lib.rs @@ -4,5 +4,6 @@ pub mod fs; pub mod hash; pub mod path; pub mod pattern; +pub mod string; pub mod system; pub mod time; diff --git a/crates/soar-utils/src/string.rs b/crates/soar-utils/src/string.rs new file mode 100644 index 000000000..f967a37e0 --- /dev/null +++ b/crates/soar-utils/src/string.rs @@ -0,0 +1,29 @@ +use std::sync::LazyLock; + +use regex::Regex; + +static ENCODED_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"(%[A-Fa-f0-9]{2})+").expect("unable to compile encoded url regex") +}); + +/// https://users.rust-lang.org/t/encode-decode-uri/90017/16 +/// Decode URI-encoded string +pub fn decode_uri(s: &str) -> String { + ENCODED_RE + .replace_all(s, |caps: ®ex::Captures| { + let seq = &caps[0]; + let bytes: Vec = seq + .as_bytes() + .chunks(3) + .filter_map(|chunk| { + if chunk.len() == 3 && chunk[0] == b'%' { + u8::from_str_radix(std::str::from_utf8(&chunk[1..3]).ok()?, 16).ok() + } else { + None + } + }) + .collect(); + String::from_utf8_lossy(&bytes).into_owned() + }) + .into_owned() +} From 9e6c9de07199133d18c377dee0cdc29d06a711b8 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Fri, 31 Oct 2025 22:19:08 +0545 Subject: [PATCH 02/22] Update crates/soar-dl/src/utils.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Rabindra Dhakal --- crates/soar-dl/src/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/soar-dl/src/utils.rs b/crates/soar-dl/src/utils.rs index e27327dd2..6f4f3d3f4 100644 --- a/crates/soar-dl/src/utils.rs +++ b/crates/soar-dl/src/utils.rs @@ -20,7 +20,7 @@ pub fn filename_from_header(value: &HeaderValue) -> Option { value .to_str() .ok()? - .split(',') + .split(';') .find_map(|p| p.trim().strip_prefix("filename=")) .map(|s| s.trim_matches('"').to_string()) } From a1cc2d37e217bb5fd9a3b170c5949fefef59d264 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Fri, 31 Oct 2025 22:20:57 +0545 Subject: [PATCH 03/22] Update crates/soar-dl/src/platform.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Rabindra Dhakal --- crates/soar-dl/src/platform.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/soar-dl/src/platform.rs b/crates/soar-dl/src/platform.rs index 1223e5973..20cab996a 100644 --- a/crates/soar-dl/src/platform.rs +++ b/crates/soar-dl/src/platform.rs @@ -59,7 +59,7 @@ impl PlatformUrl { } if let Some((project, tag)) = Self::parse_repo(&GITLAB_RE, url) { - if project.starts_with("api") || project.contains("/-/") { + if project.starts_with("api/") || project.contains("/-/") { return Url::parse(url).ok().map(|_| { Self::Direct { url: url.to_string(), From 876ebaafee08f280f97febf48921c8863739bd8b Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Fri, 31 Oct 2025 22:21:15 +0545 Subject: [PATCH 04/22] Update crates/soar-dl/src/platform.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Rabindra Dhakal --- crates/soar-dl/src/platform.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/soar-dl/src/platform.rs b/crates/soar-dl/src/platform.rs index 20cab996a..5899dfab1 100644 --- a/crates/soar-dl/src/platform.rs +++ b/crates/soar-dl/src/platform.rs @@ -45,9 +45,12 @@ impl PlatformUrl { pub fn parse(url: impl AsRef) -> Option { let url = url.as_ref(); - if url.starts_with("ghcr.io") { + let normalized = url + .trim_start_matches("https://") + .trim_start_matches("http://"); + if normalized.starts_with("ghcr.io/") { return Some(Self::Oci { - reference: url.to_string(), + reference: normalized.to_string(), }); } From 82fc7a8e3d0a273c148b443adea523741aaf4aa4 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Fri, 31 Oct 2025 22:22:13 +0545 Subject: [PATCH 05/22] Update Cargo.toml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: Rabindra Dhakal --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 0dc3b98ff..c19d3cf84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ rusqlite = { version = "0.37.0", features = ["bundled", "rusqlite-macros"] } serde = { version = "1.0.225", features = ["derive"] } serde_json = { version = "1.0.145", features = ["indexmap"] } serial_test = "3.2.0" -soar-dl = { version = "0.6.3" } +soar-dl = { path = "crates/soar-dl" } soar-utils = { path = "crates/soar-utils" } tempfile = "3.10.1" thiserror = "2.0.17" From 65bdb083a40a95c09300cd86b2f233ba18e61cf6 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Sat, 1 Nov 2025 10:45:01 +0545 Subject: [PATCH 06/22] fix major issues --- Cargo.lock | 150 +++++++++++------- Cargo.toml | 2 + crates/soar-dl/Cargo.toml | 2 + crates/soar-dl/src/download.rs | 8 +- crates/soar-dl/src/error.rs | 4 + crates/soar-dl/src/filter.rs | 5 +- crates/soar-dl/src/github.rs | 9 +- crates/soar-dl/src/http.rs | 4 + crates/soar-dl/src/http_client.rs | 10 ++ crates/soar-dl/src/oci.rs | 6 +- crates/soar-dl/src/platform.rs | 12 +- crates/soar-dl/src/release.rs | 22 +-- crates/soar-dl/src/utils.rs | 14 +- crates/soar-utils/Cargo.toml | 1 - crates/soar-utils/src/bytes.rs | 8 +- crates/soar-utils/src/error.rs | 242 +++++++++++++++++++++--------- crates/soar-utils/src/fs.rs | 8 +- crates/soar-utils/src/hash.rs | 12 +- crates/soar-utils/src/lib.rs | 1 - crates/soar-utils/src/path.rs | 17 ++- crates/soar-utils/src/string.rs | 29 ---- crates/soar-utils/src/system.rs | 1 + 22 files changed, 365 insertions(+), 202 deletions(-) delete mode 100644 crates/soar-utils/src/string.rs diff --git a/Cargo.lock b/Cargo.lock index 40c491369..a8f550323 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,6 +220,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -246,21 +255,20 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "bzip2" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ - "bzip2-sys", + "libbz2-rs-sys", ] [[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" +name = "cbc" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cc", - "pkg-config", + "cipher", ] [[package]] @@ -342,6 +350,21 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "compak" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4808bea93a8c344da5097d534975f848342ba85c6e322e268e3650c84ca7451d" +dependencies = [ + "bzip2", + "flate2", + "sevenz-rust2", + "tar", + "xz2", + "zip", + "zstd", +] + [[package]] name = "console" version = "0.16.0" @@ -730,15 +753,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" -[[package]] -name = "fast-glob" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ea3f6bbcf4dbe2076b372186fc7aeecd5f6f84754582e56ee7db262b15a6f0" -dependencies = [ - "arrayvec", -] - [[package]] name = "fast-glob" version = "1.0.0" @@ -765,14 +779,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" dependencies = [ "cfg-if", "libc", "libredox", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -909,9 +923,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -1288,6 +1302,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ + "block-padding", "generic-array", ] @@ -1371,6 +1386,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" version = "0.2.175" @@ -1379,9 +1400,9 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libredox" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags", "libc", @@ -1445,6 +1466,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rust2" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd57dc1513725e598b9609a9505d6fe28ee280861e5b4d792b3db9d00b1a315" + [[package]] name = "lzma-sys" version = "0.1.20" @@ -1829,6 +1856,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2352,6 +2385,24 @@ dependencies = [ "syn", ] +[[package]] +name = "sevenz-rust2" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a0dc37cf54d88d6da2ccff560e53118682134fbdb5fdad78f7d41ea620efb3c" +dependencies = [ + "aes", + "bzip2", + "cbc", + "crc32fast", + "getrandom 0.3.3", + "js-sys", + "lzma-rust2", + "ppmd-rust", + "sha2", + "wasm-bindgen", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2363,6 +2414,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2421,7 +2483,7 @@ dependencies = [ "serde", "serde_json", "soar-core", - "soar-dl 0.6.3", + "soar-dl", "soar-utils", "tokio", "toml", @@ -2442,7 +2504,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", - "soar-dl 0.6.3", + "soar-dl", "soar-utils", "squishy", "thiserror 2.0.17", @@ -2462,35 +2524,14 @@ dependencies = [ "serde_json", ] -[[package]] -name = "soar-dl" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b09e1dc64c524e404153ea68db8396b6aece8c23c2427cfd079af50ad2a75e" -dependencies = [ - "blake3", - "bzip2", - "fast-glob 0.4.5", - "flate2", - "futures", - "regex", - "reqwest", - "serde", - "serde_json", - "tar", - "tokio", - "url", - "xz2", - "zip", - "zstd", -] - [[package]] name = "soar-dl" version = "0.7.0" dependencies = [ - "fast-glob 1.0.0", + "compak", + "fast-glob", "miette", + "percent-encoding", "regex", "serde", "serde_json", @@ -2507,7 +2548,6 @@ version = "0.1.0" dependencies = [ "blake3", "nix", - "regex", "serial_test", "tempfile", ] @@ -3061,9 +3101,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "uncased" @@ -3678,9 +3718,9 @@ dependencies = [ [[package]] name = "zip" -version = "4.3.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aed4ac33e8eb078c89e6cbb1d5c4c7703ec6d299fc3e7c3695af8f8b423468b" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" dependencies = [ "aes", "arbitrary", diff --git a/Cargo.toml b/Cargo.toml index c19d3cf84..f85bce0da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ keywords = ["appimage", "binary", "linux", "package-manager", "portable"] categories = ["command-line-utilities"] [workspace.dependencies] +compak = "0.1.0" diesel = { version = "2.3.2", features = [ "64-column-tables", "returning_clauses_for_sqlite_3_35", @@ -27,6 +28,7 @@ diesel_migrations = { version = "2.3.0", features = ["sqlite"] } fast-glob = "1.0.0" futures = "0.3.31" miette = { version = "7.6.0", features = ["fancy"] } +percent-encoding = "2.3.2" rayon = "1.11.0" regex = { version = "1.11.2", default-features = false, features = [ "std", diff --git a/crates/soar-dl/Cargo.toml b/crates/soar-dl/Cargo.toml index bc000ade2..3d9d39e25 100644 --- a/crates/soar-dl/Cargo.toml +++ b/crates/soar-dl/Cargo.toml @@ -10,8 +10,10 @@ readme.workspace = true categories.workspace = true [dependencies] +compak = { workspace = true } fast-glob = { workspace = true } miette = { workspace = true } +percent-encoding = { workspace = true } regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs index 538172e23..cad05ac4e 100644 --- a/crates/soar-dl/src/download.rs +++ b/crates/soar-dl/src/download.rs @@ -76,7 +76,7 @@ impl Download { return self.download_to_stdout(); } - let resp = Http::fetch(&self.url, None, None)?; + let resp = Http::head(&self.url)?; let header_filename = resp .headers() @@ -126,7 +126,7 @@ impl Download { .unwrap_or_else(|| PathBuf::from(".")) }); - extract_archive(&output_path, &extract_dir)?; + compak::extract_archive(&output_path, &extract_dir)?; } Ok(output_path) @@ -246,7 +246,3 @@ fn prompt_overwrite(path: &Path) -> std::io::Result { Ok(matches!(line.trim().to_lowercase().as_str(), "y" | "yes")) } - -pub fn extract_archive(archive_path: &Path, output_dir: &Path) -> Result<(), DownloadError> { - Ok(()) -} diff --git a/crates/soar-dl/src/error.rs b/crates/soar-dl/src/error.rs index 121da1617..215fa7621 100644 --- a/crates/soar-dl/src/error.rs +++ b/crates/soar-dl/src/error.rs @@ -11,6 +11,10 @@ pub enum DownloadError { source: url::ParseError, }, + #[error("Error extracting archive")] + #[diagnostic(code(soar_dl::extract_error))] + ExtractError(#[from] compak::error::ArchiveError), + #[error("Network request failed")] #[diagnostic( code(soar_dl::network), diff --git a/crates/soar-dl/src/filter.rs b/crates/soar-dl/src/filter.rs index ad3b474e9..5147102b1 100644 --- a/crates/soar-dl/src/filter.rs +++ b/crates/soar-dl/src/filter.rs @@ -12,8 +12,9 @@ pub struct Filter { impl Filter { pub fn matches(&self, name: &str) -> bool { - let matches_regex = self.regexes.iter().all(|r| r.is_match(name)); - let matches_glob = self.globs.iter().any(|g| glob_match(g, name)); + let matches_regex = + self.regexes.is_empty() || self.regexes.iter().all(|r| r.is_match(name)); + let matches_glob = self.globs.is_empty() || self.globs.iter().any(|g| glob_match(g, name)); let matches_include = self.matches_keywords(name, &self.include, true); let matches_exclude = !self.matches_keywords(name, &self.exclude, false); diff --git a/crates/soar-dl/src/github.rs b/crates/soar-dl/src/github.rs index 38e449d6d..98669a8a7 100644 --- a/crates/soar-dl/src/github.rs +++ b/crates/soar-dl/src/github.rs @@ -36,7 +36,14 @@ impl Platform for Github { tag: Option<&str>, ) -> Result, DownloadError> { let path = match tag { - Some(tag) => format!("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/repos/{project}/releases/tags/{tag}?per_page=100"), + Some(tag) => { + let encoded_tag = + url::form_urlencoded::byte_serialize(tag.as_bytes()).collect::(); + format!( + "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/repos/{project}/releases/tags/{}?per_page=100", + encoded_tag + ) + } None => format!("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/repos/{project}/releases?per_page=100"), }; diff --git a/crates/soar-dl/src/http.rs b/crates/soar-dl/src/http.rs index 7cdeb5db6..70a572074 100644 --- a/crates/soar-dl/src/http.rs +++ b/crates/soar-dl/src/http.rs @@ -5,6 +5,10 @@ use crate::{error::DownloadError, http_client::SHARED_AGENT}; pub struct Http; impl Http { + pub fn head(url: &str) -> Result, DownloadError> { + SHARED_AGENT.head(url).call().map_err(DownloadError::from) + } + pub fn fetch( url: &str, resume_from: Option, diff --git a/crates/soar-dl/src/http_client.rs b/crates/soar-dl/src/http_client.rs index f0375c6ce..b8bf173e7 100644 --- a/crates/soar-dl/src/http_client.rs +++ b/crates/soar-dl/src/http_client.rs @@ -65,6 +65,16 @@ impl SharedAgent { Self } + pub fn head(&self, uri: T) -> RequestBuilder + where + Uri: TryFrom, + >::Error: Into, + { + let state = SHARED_CLIENT_STATE.read().unwrap(); + let req = state.agent.head(uri); + apply_headers(req, &state.config.headers) + } + pub fn get(&self, uri: T) -> RequestBuilder where Uri: TryFrom, diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs index ae8300080..675256871 100644 --- a/crates/soar-dl/src/oci.rs +++ b/crates/soar-dl/src/oci.rs @@ -12,7 +12,7 @@ use soar_utils::fs::is_elf; use ureq::http::header::{ACCEPT, AUTHORIZATION, ETAG, IF_RANGE, RANGE}; use crate::{ - download::{extract_archive, Download}, + download::Download, error::DownloadError, filter::Filter, http_client::SHARED_AGENT, @@ -235,7 +235,7 @@ impl OciDownload { .extract_to .clone() .unwrap_or_else(|| output_dir.to_path_buf()); - extract_archive(&path, &extract_dir)?; + compak::extract_archive(&path, &extract_dir)?; } paths.push(path); @@ -296,7 +296,7 @@ impl OciDownload { if extract { let extract_dir = extract_to.clone().unwrap_or_else(|| output_dir.clone()); - if let Err(e) = extract_archive(&path, &extract_dir) { + if let Err(e) = compak::extract_archive(&path, &extract_dir) { errors.lock().unwrap().push(format!("{e}")); continue; } diff --git a/crates/soar-dl/src/platform.rs b/crates/soar-dl/src/platform.rs index 5899dfab1..fb3a20e05 100644 --- a/crates/soar-dl/src/platform.rs +++ b/crates/soar-dl/src/platform.rs @@ -1,12 +1,13 @@ use std::{env, sync::LazyLock}; +use percent_encoding::percent_decode_str; use regex::Regex; -use soar_utils::string::decode_uri; use ureq::http::header::AUTHORIZATION; use url::Url; use crate::{error::DownloadError, http_client::SHARED_AGENT}; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ApiKind { Pkgforge, Primary, @@ -92,7 +93,12 @@ impl PlatformUrl { .get(2) .map(|m| m.as_str().trim_matches(&['\'', '"', ' '][..])) .filter(|s| !s.is_empty()) - .map(decode_uri); + .and_then(|s| { + percent_decode_str(s) + .decode_utf8() + .ok() + .map(|cow| cow.into_owned()) + }); Some((project, tag)) } @@ -113,7 +119,7 @@ where if use_token { if let Ok(token) = env::var(token_env) { - req = req.header(AUTHORIZATION, &format!("Bearer {}", token)); + req = req.header(AUTHORIZATION, &format!("Bearer {}", token.trim())); } } diff --git a/crates/soar-dl/src/release.rs b/crates/soar-dl/src/release.rs index ad1539c49..77bf85131 100644 --- a/crates/soar-dl/src/release.rs +++ b/crates/soar-dl/src/release.rs @@ -105,18 +105,22 @@ impl ReleaseDownload

{ let mut paths = Vec::new(); for asset in assets { - let dl = Download::new(asset.url()) - .output(self.output.clone().unwrap_or_default()) + let mut dl = Download::new(asset.url()) .overwrite(self.overwrite) - .extract(self.extract) - .extract_to(self.extract_to.clone().unwrap_or_default()); + .extract(self.extract); - let dl = if let Some(ref cb) = self.on_progress { + if let Some(ref output) = self.output { + dl = dl.output(output); + } + + if let Some(ref extract_to) = self.extract_to { + dl = dl.extract_to(extract_to); + } + + if let Some(ref cb) = self.on_progress { let cb = cb.clone(); - dl.progress(move |p| cb(p)) - } else { - dl - }; + dl = dl.progress(move |p| cb(p)); + } let path = dl.execute()?; diff --git a/crates/soar-dl/src/utils.rs b/crates/soar-dl/src/utils.rs index 6f4f3d3f4..910f51c00 100644 --- a/crates/soar-dl/src/utils.rs +++ b/crates/soar-dl/src/utils.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use percent_encoding::percent_decode_str; use ureq::http::HeaderValue; use url::Url; @@ -11,7 +12,12 @@ pub fn filename_from_url(url: &str) -> Option { u.path_segments() .and_then(|mut s| s.next_back()) .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) + .and_then(|s| { + percent_decode_str(s) + .decode_utf8() + .ok() + .map(|cow| cow.into_owned()) + }) }) } @@ -23,6 +29,12 @@ pub fn filename_from_header(value: &HeaderValue) -> Option { .split(';') .find_map(|p| p.trim().strip_prefix("filename=")) .map(|s| s.trim_matches('"').to_string()) + .map(|s| { + s.split(['/', '\\']) + .next_back() + .map(String::from) + .unwrap_or(s) + }) } /// Determine output path diff --git a/crates/soar-utils/Cargo.toml b/crates/soar-utils/Cargo.toml index 58ecbcb6a..2ab1751e6 100644 --- a/crates/soar-utils/Cargo.toml +++ b/crates/soar-utils/Cargo.toml @@ -13,7 +13,6 @@ categories.workspace = true [dependencies] blake3 = { version = "1.8.2", features = ["mmap"] } nix = { version = "0.30.1", features = ["ioctl", "term", "user"] } -regex = { workspace = true } serial_test = { workspace = true } [dev-dependencies] diff --git a/crates/soar-utils/src/bytes.rs b/crates/soar-utils/src/bytes.rs index 6c724163c..ecea102e5 100644 --- a/crates/soar-utils/src/bytes.rs +++ b/crates/soar-utils/src/bytes.rs @@ -101,9 +101,11 @@ pub fn parse_bytes(s: &str) -> BytesResult { .map(|n| n.round() as u64) }) }) - .ok_or_else(|| BytesError::ParseFailed { - input: s.to_string(), - reason: "Unrecognized size format".into(), + .ok_or_else(|| { + BytesError::ParseFailed { + input: s.to_string(), + reason: "Unrecognized size format".into(), + } }) } diff --git a/crates/soar-utils/src/error.rs b/crates/soar-utils/src/error.rs index 5b0b5b32c..fa90a10e8 100644 --- a/crates/soar-utils/src/error.rs +++ b/crates/soar-utils/src/error.rs @@ -8,7 +8,10 @@ pub enum BytesError { impl fmt::Display for BytesError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - BytesError::ParseFailed { input, reason } => { + BytesError::ParseFailed { + input, + reason, + } => { write!(f, "Failed to parse `{input}` as bytes: {reason}") } } @@ -26,7 +29,10 @@ pub enum HashError { impl fmt::Display for HashError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - HashError::ReadFailed { path, source } => { + HashError::ReadFailed { + path, + source, + } => { write!(f, "Failed to read file `{}`: {source}", path.display()) } } @@ -36,7 +42,9 @@ impl fmt::Display for HashError { impl Error for HashError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - HashError::ReadFailed { source, .. } => Some(source), + HashError::ReadFailed { + source, .. + } => Some(source), } } } @@ -58,13 +66,20 @@ impl fmt::Display for PathError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { PathError::Empty => write!(f, "Path is empty"), - PathError::FailedToGetCurrentDir { source } => { + PathError::FailedToGetCurrentDir { + source, + } => { write!(f, "Failed to get current directory: {source}") } - PathError::UnclosedVariable { input } => { + PathError::UnclosedVariable { + input, + } => { write!(f, "Unclosed variable expression starting at `{input}`") } - PathError::MissingEnvVar { var, input } => { + PathError::MissingEnvVar { + var, + input, + } => { write!(f, "Environment variable `{var}` not set in `{input}`") } } @@ -74,7 +89,9 @@ impl fmt::Display for PathError { impl Error for PathError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - PathError::FailedToGetCurrentDir { source } => Some(source), + PathError::FailedToGetCurrentDir { + source, + } => Some(source), _ => None, } } @@ -153,29 +170,50 @@ pub enum FileSystemError { impl fmt::Display for FileSystemError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - FileSystemError::ReadFile { path, source } => { + FileSystemError::ReadFile { + path, + source, + } => { write!(f, "Failed to read file `{}`: {source}", path.display()) } - FileSystemError::WriteFile { path, source } => { + FileSystemError::WriteFile { + path, + source, + } => { write!(f, "Failed to write file `{}`: {source}", path.display()) } - FileSystemError::CreateFile { path, source } => { + FileSystemError::CreateFile { + path, + source, + } => { write!(f, "Failed to create file `{}`: {source}", path.display()) } - FileSystemError::RemoveFile { path, source } => { + FileSystemError::RemoveFile { + path, + source, + } => { write!(f, "Failed to remove file `{}`: {source}", path.display()) } - FileSystemError::ReadDirectory { path, source } => { + FileSystemError::ReadDirectory { + path, + source, + } => { write!(f, "Failed to read directory `{}`: {source}", path.display()) } - FileSystemError::CreateDirectory { path, source } => { + FileSystemError::CreateDirectory { + path, + source, + } => { write!( f, "Failed to create directory `{}`: {source}", path.display() ) } - FileSystemError::RemoveDirectory { path, source } => { + FileSystemError::RemoveDirectory { + path, + source, + } => { write!( f, "Failed to remove directory `{}`: {source}", @@ -194,19 +232,31 @@ impl fmt::Display for FileSystemError { target.display() ) } - FileSystemError::RemoveSymlink { path, source } => { + FileSystemError::RemoveSymlink { + path, + source, + } => { write!(f, "Failed to remove symlink `{}`: {source}", path.display()) } - FileSystemError::ReadSymlink { path, source } => { + FileSystemError::ReadSymlink { + path, + source, + } => { write!(f, "Failed to read symlink `{}`: {source}", path.display()) } - FileSystemError::NotFound { path } => { + FileSystemError::NotFound { + path, + } => { write!(f, "Path `{}` not found", path.display()) } - FileSystemError::NotADirectory { path } => { + FileSystemError::NotADirectory { + path, + } => { write!(f, "`{}` is not a directory", path.display()) } - FileSystemError::NotAFile { path } => { + FileSystemError::NotAFile { + path, + } => { write!(f, "`{}` is not a file", path.display()) } } @@ -216,16 +266,36 @@ impl fmt::Display for FileSystemError { impl Error for FileSystemError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - FileSystemError::ReadFile { source, .. } => Some(source), - FileSystemError::WriteFile { source, .. } => Some(source), - FileSystemError::CreateFile { source, .. } => Some(source), - FileSystemError::RemoveFile { source, .. } => Some(source), - FileSystemError::ReadDirectory { source, .. } => Some(source), - FileSystemError::CreateDirectory { source, .. } => Some(source), - FileSystemError::RemoveDirectory { source, .. } => Some(source), - FileSystemError::CreateSymlink { source, .. } => Some(source), - FileSystemError::RemoveSymlink { source, .. } => Some(source), - FileSystemError::ReadSymlink { source, .. } => Some(source), + FileSystemError::ReadFile { + source, .. + } => Some(source), + FileSystemError::WriteFile { + source, .. + } => Some(source), + FileSystemError::CreateFile { + source, .. + } => Some(source), + FileSystemError::RemoveFile { + source, .. + } => Some(source), + FileSystemError::ReadDirectory { + source, .. + } => Some(source), + FileSystemError::CreateDirectory { + source, .. + } => Some(source), + FileSystemError::RemoveDirectory { + source, .. + } => Some(source), + FileSystemError::CreateSymlink { + source, .. + } => Some(source), + FileSystemError::RemoveSymlink { + source, .. + } => Some(source), + FileSystemError::ReadSymlink { + source, .. + } => Some(source), _ => None, } } @@ -252,7 +322,10 @@ pub enum IoOperation { impl IoContext { pub fn new(path: PathBuf, operation: IoOperation) -> Self { - Self { path, operation } + Self { + path, + operation, + } } pub fn read_file>(path: P) -> Self { @@ -308,47 +381,69 @@ impl IoContext { impl From<(IoContext, std::io::Error)> for FileSystemError { fn from((ctx, source): (IoContext, std::io::Error)) -> Self { match ctx.operation { - IoOperation::ReadFile => FileSystemError::ReadFile { - path: ctx.path, - source, - }, - IoOperation::WriteFile => FileSystemError::WriteFile { - path: ctx.path, - source, - }, - IoOperation::CreateFile => FileSystemError::CreateFile { - path: ctx.path, - source, - }, - IoOperation::RemoveFile => FileSystemError::RemoveFile { - path: ctx.path, - source, - }, - IoOperation::CreateDirectory => FileSystemError::CreateDirectory { - path: ctx.path, - source, - }, - IoOperation::RemoveDirectory => FileSystemError::RemoveDirectory { - path: ctx.path, - source, - }, - IoOperation::ReadDirectory => FileSystemError::ReadDirectory { - path: ctx.path, - source, - }, - IoOperation::CreateSymlink { target } => FileSystemError::CreateSymlink { - from: ctx.path, + IoOperation::ReadFile => { + FileSystemError::ReadFile { + path: ctx.path, + source, + } + } + IoOperation::WriteFile => { + FileSystemError::WriteFile { + path: ctx.path, + source, + } + } + IoOperation::CreateFile => { + FileSystemError::CreateFile { + path: ctx.path, + source, + } + } + IoOperation::RemoveFile => { + FileSystemError::RemoveFile { + path: ctx.path, + source, + } + } + IoOperation::CreateDirectory => { + FileSystemError::CreateDirectory { + path: ctx.path, + source, + } + } + IoOperation::RemoveDirectory => { + FileSystemError::RemoveDirectory { + path: ctx.path, + source, + } + } + IoOperation::ReadDirectory => { + FileSystemError::ReadDirectory { + path: ctx.path, + source, + } + } + IoOperation::CreateSymlink { target, - source, - }, - IoOperation::RemoveSymlink => FileSystemError::RemoveSymlink { - path: ctx.path, - source, - }, - IoOperation::ReadSymlink => FileSystemError::ReadSymlink { - path: ctx.path, - source, - }, + } => { + FileSystemError::CreateSymlink { + from: ctx.path, + target, + source, + } + } + IoOperation::RemoveSymlink => { + FileSystemError::RemoveSymlink { + path: ctx.path, + source, + } + } + IoOperation::ReadSymlink => { + FileSystemError::ReadSymlink { + path: ctx.path, + source, + } + } } } } @@ -420,9 +515,10 @@ pub type UtilsResult = std::result::Result; #[cfg(test)] mod tests { - use super::*; use std::io; + use super::*; + #[test] fn test_bytes_error_display() { let error = BytesError::ParseFailed { @@ -452,7 +548,9 @@ mod tests { #[test] fn test_path_error_display_and_source() { let io_error = io::Error::other("some error"); - let current_dir_error = PathError::FailedToGetCurrentDir { source: io_error }; + let current_dir_error = PathError::FailedToGetCurrentDir { + source: io_error, + }; assert_eq!( current_dir_error.to_string(), "Failed to get current directory: some error" diff --git a/crates/soar-utils/src/fs.rs b/crates/soar-utils/src/fs.rs index 7adddda7c..743384fdf 100644 --- a/crates/soar-utils/src/fs.rs +++ b/crates/soar-utils/src/fs.rs @@ -273,13 +273,9 @@ pub fn dir_size>(path: P) -> FileSystemResult { /// Check if file is ELF binary pub fn is_elf>(path: P) -> bool { - std::fs::read(path.as_ref()) + read_file_signature(path, 4) .ok() - .and_then(|bytes| { - bytes - .get(..4) - .map(|magic| magic == [0x7f, 0x45, 0x4c, 0x46]) - }) + .map(|magic| magic == [0x7f, 0x45, 0x4c, 0x46]) .unwrap_or(false) } diff --git a/crates/soar-utils/src/hash.rs b/crates/soar-utils/src/hash.rs index 60160f34c..64c23dcbf 100644 --- a/crates/soar-utils/src/hash.rs +++ b/crates/soar-utils/src/hash.rs @@ -31,12 +31,12 @@ use crate::error::{HashError, HashResult}; pub fn calculate_checksum>(file_path: P) -> HashResult { let file_path = file_path.as_ref(); let mut hasher = blake3::Hasher::new(); - hasher - .update_mmap(file_path) - .map_err(|err| HashError::ReadFailed { + hasher.update_mmap(file_path).map_err(|err| { + HashError::ReadFailed { path: file_path.to_path_buf(), source: err, - })?; + } + })?; Ok(hasher.finalize().to_hex().to_string()) } @@ -74,10 +74,12 @@ pub fn verify_checksum>(file_path: P, expected: &str) -> HashResu #[cfg(test)] mod tests { - use super::{calculate_checksum, verify_checksum}; use std::io::Write; + use tempfile::NamedTempFile; + use super::{calculate_checksum, verify_checksum}; + #[test] fn test_calculate_checksum() { let mut file = NamedTempFile::new().unwrap(); diff --git a/crates/soar-utils/src/lib.rs b/crates/soar-utils/src/lib.rs index 0dc23fd41..72d85f9b8 100644 --- a/crates/soar-utils/src/lib.rs +++ b/crates/soar-utils/src/lib.rs @@ -4,6 +4,5 @@ pub mod fs; pub mod hash; pub mod path; pub mod pattern; -pub mod string; pub mod system; pub mod time; diff --git a/crates/soar-utils/src/path.rs b/crates/soar-utils/src/path.rs index 1f83bec80..fab366e10 100644 --- a/crates/soar-utils/src/path.rs +++ b/crates/soar-utils/src/path.rs @@ -53,7 +53,11 @@ pub fn resolve_path(path: &str) -> PathResult { } else { env::current_dir() .map(|cwd| cwd.join(path_buf)) - .map_err(|err| PathError::FailedToGetCurrentDir { source: err }) + .map_err(|err| { + PathError::FailedToGetCurrentDir { + source: err, + } + }) } } @@ -210,9 +214,11 @@ fn expand_env_var(var_name: &str, result: &mut String, original: &str) -> PathRe "XDG_DATA_HOME" => result.push_str(&xdg_data_home().to_string_lossy()), "XDG_CACHE_HOME" => result.push_str(&xdg_cache_home().to_string_lossy()), _ => { - let value = env::var(var_name).map_err(|_| PathError::MissingEnvVar { - input: original.into(), - var: var_name.into(), + let value = env::var(var_name).map_err(|_| { + PathError::MissingEnvVar { + input: original.into(), + var: var_name.into(), + } })?; result.push_str(&value); } @@ -222,11 +228,12 @@ fn expand_env_var(var_name: &str, result: &mut String, original: &str) -> PathRe #[cfg(test)] mod tests { - use super::*; use std::env; use serial_test::serial; + use super::*; + #[test] fn test_expand_variables_simple() { env::set_var("TEST_VAR", "test_value"); diff --git a/crates/soar-utils/src/string.rs b/crates/soar-utils/src/string.rs deleted file mode 100644 index f967a37e0..000000000 --- a/crates/soar-utils/src/string.rs +++ /dev/null @@ -1,29 +0,0 @@ -use std::sync::LazyLock; - -use regex::Regex; - -static ENCODED_RE: LazyLock = LazyLock::new(|| { - Regex::new(r"(%[A-Fa-f0-9]{2})+").expect("unable to compile encoded url regex") -}); - -/// https://users.rust-lang.org/t/encode-decode-uri/90017/16 -/// Decode URI-encoded string -pub fn decode_uri(s: &str) -> String { - ENCODED_RE - .replace_all(s, |caps: ®ex::Captures| { - let seq = &caps[0]; - let bytes: Vec = seq - .as_bytes() - .chunks(3) - .filter_map(|chunk| { - if chunk.len() == 3 && chunk[0] == b'%' { - u8::from_str_radix(std::str::from_utf8(&chunk[1..3]).ok()?, 16).ok() - } else { - None - } - }) - .collect(); - String::from_utf8_lossy(&bytes).into_owned() - }) - .into_owned() -} diff --git a/crates/soar-utils/src/system.rs b/crates/soar-utils/src/system.rs index 26ffae644..3d4acb092 100644 --- a/crates/soar-utils/src/system.rs +++ b/crates/soar-utils/src/system.rs @@ -62,6 +62,7 @@ mod tests { fn env_var(&self, _: &str) -> Option { None } + fn uid_name(&self) -> Option { None } From 5a9830fcc036b52b1132c3a4650cb8f4d1bb40df Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Sat, 1 Nov 2025 10:51:39 +0545 Subject: [PATCH 07/22] fix inverted exclude logic --- crates/soar-dl/src/filter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/soar-dl/src/filter.rs b/crates/soar-dl/src/filter.rs index 5147102b1..25304317d 100644 --- a/crates/soar-dl/src/filter.rs +++ b/crates/soar-dl/src/filter.rs @@ -16,7 +16,7 @@ impl Filter { self.regexes.is_empty() || self.regexes.iter().all(|r| r.is_match(name)); let matches_glob = self.globs.is_empty() || self.globs.iter().any(|g| glob_match(g, name)); let matches_include = self.matches_keywords(name, &self.include, true); - let matches_exclude = !self.matches_keywords(name, &self.exclude, false); + let matches_exclude = self.matches_keywords(name, &self.exclude, false); matches_regex && matches_glob && matches_include && matches_exclude } From 93a3ae2cc0e99063fc1e50cf128a69e170e473f1 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:58:12 +0545 Subject: [PATCH 08/22] =?UTF-8?q?=F0=9F=93=9D=20Add=20docstrings=20to=20`c?= =?UTF-8?q?rates/soar-dl`=20(#103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Rabindra Dhakal --- crates/soar-dl/src/download.rs | 188 ++++++++++++++++ crates/soar-dl/src/error.rs | 17 +- crates/soar-dl/src/filter.rs | 54 ++++- crates/soar-dl/src/github.rs | 159 +++++++++++++- crates/soar-dl/src/gitlab.rs | 133 +++++++++++- crates/soar-dl/src/http.rs | 39 ++++ crates/soar-dl/src/http_client.rs | 117 ++++++++++ crates/soar-dl/src/oci.rs | 342 +++++++++++++++++++++++++++++- crates/soar-dl/src/platform.rs | 65 +++++- crates/soar-dl/src/release.rs | 127 ++++++++++- crates/soar-dl/src/utils.rs | 85 +++++++- crates/soar-dl/src/xattr.rs | 44 +++- crates/soar-utils/src/fs.rs | 39 +++- 13 files changed, 1388 insertions(+), 21 deletions(-) diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs index cad05ac4e..2e3fdc096 100644 --- a/crates/soar-dl/src/download.rs +++ b/crates/soar-dl/src/download.rs @@ -32,6 +32,22 @@ pub struct Download { } impl Download { + /// Creates a new `Download` configured for the given URL with sensible defaults. + /// + /// The returned builder defaults to: + /// - no explicit output path (downloaded filename will be resolved automatically), + /// - `OverwriteMode::Prompt` for existing files, + /// - extraction disabled, + /// - no extraction destination, + /// - no progress callback. + /// + /// # Examples + /// + /// ``` + /// let dl = Download::new("https://example.com/archive.tar.gz") + /// .output("archive.tar.gz"); + /// // `dl` is ready to call `execute()` + /// ``` pub fn new(url: impl Into) -> Self { Self { url: url.into(), @@ -43,26 +59,95 @@ impl Download { } } + /// Sets the download output destination. + /// + /// The `output` value may be a filesystem path or `"-"` to write to stdout. + /// + /// # Returns + /// + /// The modified `Download` builder. + /// + /// # Examples + /// + /// ``` + /// use crate::download::Download; + /// let _ = Download::new("https://example.com/file").output("path/to/file"); + /// ``` pub fn output(mut self, output: impl Into) -> Self { self.output = Some(output.into()); self } + /// Sets how existing destination files are handled when performing the download. + /// + /// Sets the overwrite mode that determines what to do if the resolved output path already exists. The method returns the modified `Download` builder to allow chaining. + /// + /// # Examples + /// + /// ``` + /// use crate::download::Download; + /// use crate::types::OverwriteMode; + /// + /// let d = Download::new("https://example.com/file") + /// .overwrite(OverwriteMode::Force) + /// .output("file.bin"); + /// ``` pub fn overwrite(mut self, overwrite: OverwriteMode) -> Self { self.overwrite = overwrite; self } + /// Enable or disable extraction of the downloaded archive after a successful download. + /// + /// When set to `true`, the downloader will extract the downloaded archive to the configured + /// extraction directory (or to the output file's parent directory if no extraction target was set). + /// + /// # Examples + /// + /// ``` + /// let dl = crate::download::Download::new("https://example.com/archive.tar.gz") + /// .extract(true); + /// ``` pub fn extract(mut self, extract: bool) -> Self { self.extract = extract; self } + /// Set the directory to extract a downloaded archive into. + /// + /// When `extract` is enabled on the downloader, the downloaded archive will be + /// extracted into this path instead of the default (the downloaded file's + /// parent directory). + /// + /// # Examples + /// + /// ```no_run + /// let dl = Download::new("https://example.com/archive.tar.gz") + /// .extract(true) + /// .extract_to("/tmp/my-extract-dir"); + /// ``` pub fn extract_to(mut self, extract_to: impl Into) -> Self { self.extract_to = Some(extract_to.into()); self } + /// Registers a progress callback that will be invoked with `Progress` events during the download lifecycle. + /// + /// The provided closure is stored and called for events such as `Progress::Starting`, `Progress::Chunk`, and `Progress::Complete`. + /// + /// # Examples + /// + /// ``` + /// use crate::download::Download; + /// use crate::types::Progress; + /// + /// let _dl = Download::new("https://example.com/file") + /// .progress(|event: Progress| match event { + /// Progress::Starting { total } => eprintln!("starting, total={}", total), + /// Progress::Chunk { downloaded, chunk } => eprintln!("downloaded {} (+{})", downloaded, chunk), + /// Progress::Complete { total } => eprintln!("complete, total={}", total), + /// }); + /// ``` pub fn progress(mut self, on_progress: F) -> Self where F: Fn(Progress) + Send + Sync + 'static, @@ -71,6 +156,30 @@ impl Download { self } + /// Performs the configured download and returns the final output path. + /// + /// The method downloads the URL configured in this `Download` instance to the resolved + /// output location (or to stdout when the configured output is `"-"`). It creates parent + /// directories as needed, respects the configured overwrite mode (skip, force, or prompt), + /// supports resuming interrupted downloads when resume metadata is available, and persists + /// resume state during an active download. After a successful download, it clears any + /// stored resume metadata, sets the executable bit on ELF binaries, and—if extraction was + /// requested—extracts the archive into the configured destination directory. + /// + /// # Returns + /// + /// `Ok(PathBuf)` containing the filesystem path to the downloaded file (or `PathBuf::from("-")` + /// when written to stdout), or `Err(DownloadError)` on failure. + /// + /// # Examples + /// + /// ``` + /// let dl = Download::new("https://example.com/archive.tar.gz") + /// .output("archive.tar.gz") + /// .extract(true); + /// let path = dl.execute().expect("download failed"); + /// assert!(path.ends_with("archive.tar.gz")); + /// ``` pub fn execute(self) -> Result { if self.output.as_deref() == Some("-") { return self.download_to_stdout(); @@ -132,6 +241,21 @@ impl Download { Ok(output_path) } + /// Streams the HTTP response body for this download's URL to standard output. + /// + /// # Examples + /// + /// ``` + /// use std::path::PathBuf; + /// + /// // The function returns a PathBuf `"-"` to indicate stdout was used. + /// let stdout_path = PathBuf::from("-"); + /// assert_eq!(stdout_path.to_str(), Some("-")); + /// ``` + /// + /// # Returns + /// + /// `PathBuf::from("-")` on success. fn download_to_stdout(&self) -> Result { let resp = Http::fetch(&self.url, None, None)?; let mut stdout = std::io::stdout(); @@ -143,6 +267,31 @@ impl Download { Ok(PathBuf::from("-")) } + /// Download the HTTP response body into the given file path, using resume metadata when available and emitting progress events. + /// + /// When `resume_info` is provided the method attempts to resume the download from the recorded byte offset and validates server support + /// for ranged requests; if the server does not return a partial-content (206) response, the download restarts from the beginning. + /// Progress callbacks (if configured on `self`) are invoked with `Progress::Starting`, `Progress::Chunk`, and `Progress::Complete`. + /// + /// # Parameters + /// + /// - `path`: destination filesystem path to write the downloaded bytes to. If resuming, the file is opened for append; otherwise it is (re)created. + /// - `resume_info`: optional resume metadata describing previously downloaded bytes and a prior `ETag`; when present the method will request a ranged response starting at `resume_info.downloaded`. + /// + /// # Returns + /// + /// `Ok(())` on successful completion of the download and file write, or `Err(DownloadError)` on IO, HTTP, or resume-state persistence failures. + /// + /// # Examples + /// + /// ```no_run + /// use std::path::Path; + /// + /// // Illustrative usage (no network interaction in this example). + /// let download = crate::download::Download::new("https://example.com/file.tar.gz").output("file.tar.gz"); + /// // Attempt a fresh download (no resume info). + /// let _ = download.download_to_file(Path::new("file.tar.gz"), None); + /// ``` fn download_to_file( &self, path: &Path, @@ -222,6 +371,24 @@ impl Download { Ok(()) } + /// Determine the total size of the response body from HTTP headers. + /// + /// Checks the `Content-Range` header first (parsing the value after the final '/'), + /// and falls back to the `Content-Length` header. If neither header yields a valid + /// size, returns 0. + /// + /// # Returns + /// + /// `u64` total size in bytes if present in the response headers, `0` otherwise. + /// + /// # Examples + /// + /// ```rust,no_run + /// use ureq::{Response, Body}; + /// + /// // Given a `Response` named `resp`, obtain the total size: + /// // let total = parse_content_length(&resp); + /// ``` fn parse_content_length(resp: &Response) -> u64 { resp.headers() .get(CONTENT_RANGE) @@ -237,6 +404,27 @@ impl Download { } } +/// Prompts the user to confirm overwriting the specified path. +/// +/// Reads a line from stdin after printing "Overwrite ? [y/N] " and interprets +/// a case-insensitive `"y"` or `"yes"` as confirmation. +/// +/// # Returns +/// +/// `true` if the user entered `"y"` or `"yes"` (case-insensitive), `false` otherwise. +/// +/// # Examples +/// +/// ```no_run +/// use std::path::Path; +/// +/// let confirmed = prompt_overwrite(Path::new("output.bin")).unwrap(); +/// if confirmed { +/// println!("User confirmed overwrite"); +/// } else { +/// println!("User declined overwrite"); +/// } +/// ``` fn prompt_overwrite(path: &Path) -> std::io::Result { print!("Overwrite {}? [y/N] ", path.display()); std::io::stdout().flush()?; diff --git a/crates/soar-dl/src/error.rs b/crates/soar-dl/src/error.rs index 215fa7621..85b479649 100644 --- a/crates/soar-dl/src/error.rs +++ b/crates/soar-dl/src/error.rs @@ -64,7 +64,22 @@ pub enum DownloadError { pub type Result = miette::Result; impl From for DownloadError { + /// Converts a `ureq::Error` into a `DownloadError::Network` variant. + /// + /// # Examples + /// + /// ```no_run + /// use soar_dl::error::DownloadError; + /// + /// // Given a `ureq::Error` `e`, convert it into a `DownloadError` + /// let e: ureq::Error = /* obtained from a ureq request */ unimplemented!(); + /// let err: DownloadError = DownloadError::from(e); + /// match err { + /// DownloadError::Network(_) => (), + /// _ => panic!("expected DownloadError::Network"), + /// } + /// ``` fn from(e: ureq::Error) -> Self { Self::Network(Box::new(e)) } -} +} \ No newline at end of file diff --git a/crates/soar-dl/src/filter.rs b/crates/soar-dl/src/filter.rs index 25304317d..5bc2caebc 100644 --- a/crates/soar-dl/src/filter.rs +++ b/crates/soar-dl/src/filter.rs @@ -11,6 +11,29 @@ pub struct Filter { } impl Filter { + /// Determines whether a name satisfies this filter's combined criteria. + /// + /// The name must match every regex in `self.regexes`, match at least one glob in + /// `self.globs`, satisfy all include keyword groups in `self.include`, and must + /// not match any exclude keyword groups in `self.exclude`. + /// + /// # Returns + /// + /// `true` if the name matches all regexes, at least one glob, all include groups, + /// and no exclude groups; `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// let f = Filter { + /// regexes: Vec::new(), + /// globs: vec!["*".into()], + /// include: Vec::new(), + /// exclude: Vec::new(), + /// case_sensitive: true, + /// }; + /// assert!(f.matches("anything")); + /// ``` pub fn matches(&self, name: &str) -> bool { let matches_regex = self.regexes.is_empty() || self.regexes.iter().all(|r| r.is_match(name)); @@ -21,6 +44,35 @@ impl Filter { matches_regex && matches_glob && matches_include && matches_exclude } + /// Determines whether every keyword group in `keywords` satisfies the required presence or absence + /// against `name` according to `must_match`. + /// + /// - If `keywords` is empty, returns `true`. + /// - Splits each keyword string on commas, trims parts, and ignores empty parts. + /// - Respects `case_sensitive`: comparisons use the original case when `true`, otherwise both + /// haystack and needles are lowercased. + /// - For each keyword (a group of comma-separated alternatives), any one alternative matching + /// `name` counts as a match for that keyword. + /// - If `must_match` is `true`, each keyword group must have at least one matching alternative. + /// If `must_match` is `false`, each keyword group must have no matching alternatives. + /// + /// # Examples + /// + /// ``` + /// use regex::Regex; + /// use crate::filter::Filter; + /// + /// let filter = Filter { + /// regexes: vec![], + /// globs: vec![], + /// include: vec!["foo,bar".to_string()], + /// exclude: vec![], + /// case_sensitive: false, + /// }; + /// + /// // "barbaz" contains "bar", one of the alternatives in the include group. + /// assert!(filter.matches("barbaz")); + /// ``` fn matches_keywords(&self, name: &str, keywords: &[String], must_match: bool) -> bool { if keywords.is_empty() { return true; @@ -55,4 +107,4 @@ impl Filter { } }) } -} +} \ No newline at end of file diff --git a/crates/soar-dl/src/github.rs b/crates/soar-dl/src/github.rs index 98669a8a7..a212953ce 100644 --- a/crates/soar-dl/src/github.rs +++ b/crates/soar-dl/src/github.rs @@ -31,6 +31,25 @@ impl Platform for Github { const API_UPSTREAM: &'static str = "https://api.github.com"; const TOKEN_ENV: &'static str = "GITHUB_TOKEN"; + /// Fetches releases for the given GitHub repository, optionally filtered by a specific tag. + /// + /// If `tag` is provided, fetches the release that matches that tag; otherwise fetches the repository's releases (up to 100 per page). + /// + /// # Arguments + /// + /// * `project` — repository identifier in the form "owner/repo". + /// * `tag` — optional release tag to filter the results. + /// + /// # Returns + /// + /// `Ok` with a vector of releases on success, or `Err(DownloadError)` on failure. + /// + /// # Examples + /// + /// ``` + /// let releases = Github::fetch_releases("rust-lang/rust", None).unwrap(); + /// assert!(releases.iter().all(|r| r.tag().len() > 0)); + /// ``` fn fetch_releases( project: &str, tag: Option<&str>, @@ -59,37 +78,175 @@ impl Platform for Github { impl Release for GithubRelease { type Asset = GithubAsset; + /// The release's name, or an empty string if the release has no name. + /// + /// # Examples + /// + /// ``` + /// let r = GithubRelease { + /// name: Some("v1.0".into()), + /// tag_name: "v1.0".into(), + /// prerelease: false, + /// published_at: "".into(), + /// assets: vec![], + /// }; + /// assert_eq!(r.name(), "v1.0"); + /// + /// let unnamed = GithubRelease { + /// name: None, + /// tag_name: "v1.1".into(), + /// prerelease: false, + /// published_at: "".into(), + /// assets: vec![], + /// }; + /// assert_eq!(unnamed.name(), ""); + /// ``` fn name(&self) -> &str { self.name.as_deref().unwrap_or("") } + /// Get the release tag as a string slice. + /// + /// # Examples + /// + /// ``` + /// let release = crate::github::GithubRelease { + /// name: None, + /// tag_name: "v1.0.0".into(), + /// prerelease: false, + /// published_at: "".into(), + /// assets: vec![], + /// }; + /// assert_eq!(release.tag(), "v1.0.0"); + /// ``` + /// + /// # Returns + /// + /// `&str` containing the release tag. fn tag(&self) -> &str { &self.tag_name } + /// Indicates whether the release is marked as a prerelease. + /// + /// # Returns + /// + /// `true` if the release is marked as a prerelease, `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// let r = GithubRelease { + /// name: None, + /// tag_name: "v1.0.0".to_string(), + /// prerelease: true, + /// published_at: "".to_string(), + /// assets: vec![], + /// }; + /// assert!(r.is_prerelease()); + /// ``` fn is_prerelease(&self) -> bool { self.prerelease } + /// Returns the release's publication timestamp as an RFC 3339 formatted string. + /// + /// # Examples + /// + /// ``` + /// let r = GithubRelease { + /// name: None, + /// tag_name: "v1.0.0".into(), + /// prerelease: false, + /// published_at: "2021-01-01T00:00:00Z".into(), + /// assets: vec![], + /// }; + /// assert_eq!(r.published_at(), "2021-01-01T00:00:00Z"); + /// ``` fn published_at(&self) -> &str { &self.published_at } + /// Get a slice of assets associated with the release. + /// + /// The slice contains the release's assets in declaration order. + /// + /// # Examples + /// + /// ``` + /// use crate::github::{GithubRelease, GithubAsset}; + /// + /// let asset = GithubAsset { + /// name: "example.zip".into(), + /// size: 1024, + /// browser_download_url: "https://example.com/example.zip".into(), + /// }; + /// + /// let release = GithubRelease { + /// name: Some("v1.0".into()), + /// tag_name: "v1.0".into(), + /// prerelease: false, + /// published_at: "2025-01-01T00:00:00Z".into(), + /// assets: vec![asset], + /// }; + /// + /// assert_eq!(release.assets().len(), 1); + /// ``` fn assets(&self) -> &[Self::Asset] { &self.assets } } impl Asset for GithubAsset { + /// Retrieves the asset's name. + /// + /// # Examples + /// + /// ``` + /// let asset = crate::github::GithubAsset { + /// name: "file.zip".to_string(), + /// size: 123, + /// browser_download_url: "https://example.com/file.zip".to_string(), + /// }; + /// assert_eq!(asset.name(), "file.zip"); + /// ``` + /// + /// # Returns + /// + /// A `&str` containing the asset's name. fn name(&self) -> &str { &self.name } + /// Asset size in bytes. + /// + /// # Returns + /// + /// `Some(size)` containing the asset size in bytes. + /// + /// # Examples + /// + /// ``` + /// let asset = crate::github::GithubAsset { name: "file".into(), size: 12345, browser_download_url: "https://example.com".into() }; + /// assert_eq!(asset.size(), Some(12345)); + /// ``` fn size(&self) -> Option { Some(self.size) } + /// Returns the asset's browser download URL. + /// + /// # Examples + /// + /// ``` + /// let asset = crate::github::GithubAsset { + /// name: "example".into(), + /// size: 123, + /// browser_download_url: "https://example.com/download".into(), + /// }; + /// assert_eq!(asset.url(), "https://example.com/download"); + /// ``` fn url(&self) -> &str { &self.browser_download_url } -} +} \ No newline at end of file diff --git a/crates/soar-dl/src/gitlab.rs b/crates/soar-dl/src/gitlab.rs index bcd78669f..34dd55d9b 100644 --- a/crates/soar-dl/src/gitlab.rs +++ b/crates/soar-dl/src/gitlab.rs @@ -35,6 +35,29 @@ impl Platform for GitLab { const API_UPSTREAM: &'static str = "https://gitlab.com"; const TOKEN_ENV: &'static str = "GITLAB_TOKEN"; + /// Fetches releases for a GitLab project, optionally narrowing to a specific tag. + /// + /// The `project` is the repository identifier (for example `"group/name"` or a numeric project ID). + /// If `tag` is provided and the `project` consists only of digits, the fetch targets that single release; otherwise the fetch returns the project's release list. + /// + /// # Parameters + /// + /// - `project`: repository identifier or numeric project ID. + /// - `tag`: optional release tag to narrow the request. + /// + /// # Returns + /// + /// `Ok(Vec)` with the fetched releases on success, or a `DownloadError` on failure. + /// + /// # Examples + /// + /// ``` + /// // Fetch all releases for a namespaced project + /// let _ = GitLab::fetch_releases("group/project", None); + /// + /// // Fetch a specific release when using a numeric project ID + /// let _ = GitLab::fetch_releases("123456", Some("v1.0.0")); + /// ``` fn fetch_releases( project: &str, tag: Option<&str>, @@ -59,37 +82,145 @@ impl Platform for GitLab { impl Release for GitLabRelease { type Asset = GitLabAsset; + /// Gets the asset's name. + /// + /// # Examples + /// + /// ``` + /// let asset = GitLabAsset { name: String::from("v1.0.0"), direct_asset_url: String::from("https://example") }; + /// assert_eq!(asset.name(), "v1.0.0"); + /// ``` fn name(&self) -> &str { &self.name } + /// Get the release's tag name. + /// + /// # Examples + /// + /// ``` + /// let r = GitLabRelease { + /// name: "Release".into(), + /// tag_name: "v1.0.0".into(), + /// upcoming_release: false, + /// released_at: "2025-01-01T00:00:00Z".into(), + /// assets: GitLabAssets { links: vec![] }, + /// }; + /// assert_eq!(r.tag(), "v1.0.0"); + /// ``` fn tag(&self) -> &str { &self.tag_name } + /// Indicates whether the release is marked as upcoming. + /// + /// # Returns + /// + /// `true` if the release is marked as upcoming, `false` otherwise. + /// + /// # Examples + /// + /// ``` + /// let rel = GitLabRelease { + /// name: "v1".to_string(), + /// tag_name: "v1".to_string(), + /// upcoming_release: true, + /// released_at: "".to_string(), + /// assets: GitLabAssets { links: vec![] }, + /// }; + /// assert!(rel.is_prerelease()); + /// ``` fn is_prerelease(&self) -> bool { self.upcoming_release } + /// Get the release's published date/time string. + /// + /// # Examples + /// + /// ``` + /// let r = GitLabRelease { + /// name: String::from("v1"), + /// tag_name: String::from("v1"), + /// upcoming_release: false, + /// released_at: String::from("2020-01-01T00:00:00Z"), + /// assets: GitLabAssets { links: vec![] }, + /// }; + /// assert_eq!(r.published_at(), "2020-01-01T00:00:00Z"); + /// ``` fn published_at(&self) -> &str { &self.released_at } + /// A slice of assets associated with the release. + /// + /// # Examples + /// + /// ``` + /// let asset = GitLabAsset { name: "file.tar.gz".into(), direct_asset_url: "https://example.com/file.tar.gz".into() }; + /// let assets = GitLabAssets { links: vec![asset.clone()] }; + /// let release = GitLabRelease { + /// name: "v1.0".into(), + /// tag_name: "v1.0".into(), + /// upcoming_release: false, + /// released_at: "2025-10-31T00:00:00Z".into(), + /// assets, + /// }; + /// let slice = release.assets(); + /// assert_eq!(slice.len(), 1); + /// assert_eq!(slice[0].name(), "file.tar.gz"); + /// ``` + /// + /// # Returns + /// + /// A slice of the release's assets. fn assets(&self) -> &[Self::Asset] { &self.assets.links } } impl Asset for GitLabAsset { + /// Gets the asset's name. + /// + /// # Examples + /// + /// ``` + /// let asset = GitLabAsset { name: String::from("v1.0.0"), direct_asset_url: String::from("https://example") }; + /// assert_eq!(asset.name(), "v1.0.0"); + /// ``` fn name(&self) -> &str { &self.name } + /// Returns the asset size when available; for GitLab assets this is not provided. + /// + /// This implementation always reports that size information is unavailable. + /// + /// # Examples + /// + /// ``` + /// let asset = GitLabAsset { + /// name: "example".into(), + /// direct_asset_url: "https://gitlab.com/example".into(), + /// }; + /// assert_eq!(asset.size(), None); + /// ``` fn size(&self) -> Option { None } + /// Returns the direct URL of the asset. + /// + /// # Examples + /// + /// ``` + /// let asset = GitLabAsset { + /// name: String::from("example"), + /// direct_asset_url: String::from("https://example.com/download"), + /// }; + /// assert_eq!(asset.url(), "https://example.com/download"); + /// ``` fn url(&self) -> &str { &self.direct_asset_url } -} +} \ No newline at end of file diff --git a/crates/soar-dl/src/http.rs b/crates/soar-dl/src/http.rs index 70a572074..1595f6e69 100644 --- a/crates/soar-dl/src/http.rs +++ b/crates/soar-dl/src/http.rs @@ -9,6 +9,28 @@ impl Http { SHARED_AGENT.head(url).call().map_err(DownloadError::from) } + /// Fetches a GET response for the given URL, optionally requesting a byte range and using an ETag for conditional requests. + /// + /// If `resume_from` is `Some(pos)`, the request includes a `Range: bytes={pos}-` header. If `etag` is `Some(tag)` and a range is requested, + /// the request also includes an `If-Range: {tag}` header. + /// + /// # Returns + /// + /// `Ok(Response)` with the HTTP response on success, `Err(DownloadError)` on failure. + /// + /// # Examples + /// + /// ``` + /// # use soar_dl::http::Http; + /// # use ureq::http::Status; + /// let resp = Http::fetch("https://example.com/resource", Some(1024), Some("\"etag-value\"")); + /// match resp { + /// Ok(r) => { + /// assert!(r.status() < 600); // got a response + /// } + /// Err(e) => panic!("request failed: {:?}", e), + /// } + /// ``` pub fn fetch( url: &str, resume_from: Option, @@ -26,6 +48,22 @@ impl Http { req.call().map_err(DownloadError::from) } + /// Fetches JSON from the given URL and deserializes it into `T`. + /// + /// Performs an HTTP GET request to `url` and deserializes the response body into the requested type. + /// + /// # Returns + /// + /// `T` parsed from the response body on success, `DownloadError` on failure. + /// + /// # Examples + /// + /// ```no_run + /// use serde_json::Value; + /// use soar_dl::http::Http; + /// + /// let json: Value = Http::json("https://example.com/data.json").unwrap(); + /// ``` pub fn json(url: &str) -> Result { SHARED_AGENT .get(url) @@ -35,3 +73,4 @@ impl Http { .map_err(|_| DownloadError::InvalidResponse) } } + diff --git a/crates/soar-dl/src/http_client.rs b/crates/soar-dl/src/http_client.rs index b8bf173e7..7eaeee6b9 100644 --- a/crates/soar-dl/src/http_client.rs +++ b/crates/soar-dl/src/http_client.rs @@ -18,6 +18,19 @@ pub struct ClientConfig { } impl Default for ClientConfig { + /// Creates a default ClientConfig populated with sensible defaults for HTTP requests. + /// + /// The default sets a user agent of "pkgforge/soar" and leaves proxy, headers, and timeout unset. + /// + /// # Examples + /// + /// ``` + /// let cfg = crates::soar_dl::http_client::ClientConfig::default(); + /// assert_eq!(cfg.user_agent.as_deref(), Some("pkgforge/soar")); + /// assert!(cfg.proxy.is_none()); + /// assert!(cfg.headers.is_none()); + /// assert!(cfg.timeout.is_none()); + /// ``` fn default() -> Self { Self { user_agent: Some("pkgforge/soar".into()), @@ -29,6 +42,19 @@ impl Default for ClientConfig { } impl ClientConfig { + /// Builds an HTTP `Agent` configured from this `ClientConfig`. + /// + /// The returned `Agent` will incorporate the configured proxy, global timeout, + /// and user agent header (if present). + /// + /// # Examples + /// + /// ``` + /// let config = ClientConfig::default(); + /// let agent = config.build(); + /// // create a request builder using the configured agent + /// let _req = agent.get("http://example.com"); + /// ``` pub fn build(&self) -> Agent { let mut config = ureq::Agent::config_builder() .proxy(self.proxy.clone()) @@ -61,6 +87,15 @@ static SHARED_CLIENT_STATE: LazyLock>> = LazyLock::new( pub struct SharedAgent; impl SharedAgent { + /// Create a new `SharedAgent` instance. + /// + /// # Examples + /// + /// ``` + /// use crate::http_client::SharedAgent; + /// + /// let _agent = SharedAgent::new(); + /// ``` pub fn new() -> Self { Self } @@ -75,6 +110,19 @@ impl SharedAgent { apply_headers(req, &state.config.headers) } + /// Create a GET request builder for the given URI using the shared agent. + /// + /// The returned `RequestBuilder` does not contain a body; any global headers + /// configured in the shared client are applied to the request. + /// + /// # Examples + /// + /// ```no_run + /// use soar_dl::http_client::SHARED_AGENT; + /// + /// // Create and send a GET request to the specified URI. + /// let response = SHARED_AGENT.get("https://example.com").call(); + /// ``` pub fn get(&self, uri: T) -> RequestBuilder where Uri: TryFrom, @@ -85,6 +133,16 @@ impl SharedAgent { apply_headers(req, &state.config.headers) } + /// Starts a POST request to the given URI using the shared agent and applies any globally configured headers. + /// + /// The returned `RequestBuilder` is ready to accept a request body and further per-request modifications. + /// + /// # Examples + /// + /// ```no_run + /// let req = crate::SHARED_AGENT.post("https://example.com/"); + /// let req = req.send_string("payload"); // sends the request with a string body + /// ``` pub fn post(&self, uri: T) -> RequestBuilder where Uri: TryFrom, @@ -95,6 +153,19 @@ impl SharedAgent { apply_headers(req, &state.config.headers) } + /// Creates a PUT request builder for the specified URI using the shared agent and applies any configured global headers. + /// + /// # Examples + /// + /// ``` + /// let req = SHARED_AGENT.put("https://example.com/resource"); + /// // `req` is a `RequestBuilder` ready to have a body set and be sent: + /// // let resp = req.send_string("payload"); + /// ``` + pub fn put(&self, uri: T) -> RequestBuilder + where + Uri: TryFrom, + >::Error: Into, pub fn put(&self, uri: T) -> RequestBuilder where Uri: TryFrom, @@ -105,6 +176,18 @@ impl SharedAgent { apply_headers(req, &state.config.headers) } + /// Creates a DELETE request for the given URI using the shared agent and applies configured global headers. + /// + /// # Returns + /// + /// A `RequestBuilder` for the DELETE request with any configured global headers applied. + /// + /// # Examples + /// + /// ``` + /// let agent = SharedAgent::new(); + /// let _req = agent.delete("https://example.com/resource"); + /// ``` pub fn delete(&self, uri: T) -> RequestBuilder where Uri: TryFrom, @@ -116,6 +199,27 @@ impl SharedAgent { } } +/// Apply headers from an optional `HeaderMap` to a `RequestBuilder`. +/// +/// If `headers` is `Some`, each header key/value pair is added to the provided request +/// and the modified `RequestBuilder` is returned. If `headers` is `None`, the original +/// request is returned unchanged. +/// +/// # Examples +/// +/// ``` +/// use http::HeaderMap; +/// use ureq::Agent; +/// +/// // create headers +/// let mut headers = HeaderMap::new(); +/// headers.insert("x-test".parse().unwrap(), "v".parse().unwrap()); +/// +/// let agent = Agent::new(); +/// let req = agent.get("http://example.com"); +/// let req = apply_headers(req, &Some(headers)); +/// // `req` now contains the `x-test: v` header +/// ``` fn apply_headers(mut req: RequestBuilder, headers: &Option) -> RequestBuilder { if let Some(headers) = headers { for (key, value) in headers.iter() { @@ -127,6 +231,19 @@ fn apply_headers(mut req: RequestBuilder, headers: &Option) -> pub static SHARED_AGENT: LazyLock = LazyLock::new(SharedAgent::new); +/// Updates the global shared HTTP client configuration by applying the provided updater and rebuilding the shared Agent. +/// +/// The `updater` closure receives a mutable reference to a `ClientConfig` that will replace the current shared configuration. +/// After the updater runs, a new `Agent` is built from the updated config and atomically replaces the shared agent and config. +/// +/// # Examples +/// +/// ``` +/// // Change the global user agent string used by the shared HTTP client. +/// configure_http_client(|cfg| { +/// cfg.user_agent = Some("my-app/1.0".to_string()); +/// }); +/// ``` pub fn configure_http_client(updater: F) where F: FnOnce(&mut ClientConfig), diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs index 675256871..79031cb8d 100644 --- a/crates/soar-dl/src/oci.rs +++ b/crates/soar-dl/src/oci.rs @@ -28,6 +28,31 @@ pub struct OciReference { } impl From<&str> for OciReference { + /// Parses an OCI/GHCR reference string into an `OciReference`. + /// + /// Accepts strings with optional `ghcr.io/` prefix and extracts the package and tag or digest: + /// - `package@sha256:` → digest used as `tag` + /// - `package:` → tag used as `tag` + /// - otherwise the full path is treated as `package` and `tag` is set to `"latest"`. + /// + /// # Examples + /// + /// ``` + /// let r = OciReference::from("ghcr.io/org/repo@sha256:deadbeef"); + /// assert_eq!(r.registry, "ghcr.io"); + /// assert_eq!(r.package, "org/repo"); + /// assert_eq!(r.tag, "sha256:deadbeef"); + /// + /// let r2 = OciReference::from("org/repo:1.2.3"); + /// assert_eq!(r2.registry, "ghcr.io"); + /// assert_eq!(r2.package, "org/repo"); + /// assert_eq!(r2.tag, "1.2.3"); + /// + /// let r3 = OciReference::from("org/repo"); + /// assert_eq!(r3.registry, "ghcr.io"); + /// assert_eq!(r3.package, "org/repo"); + /// assert_eq!(r3.tag, "latest"); + /// ``` fn from(value: &str) -> Self { let paths = value.trim_start_matches("ghcr.io/"); @@ -84,6 +109,31 @@ pub struct OciLayer { } impl OciLayer { + /// Extracts the human-friendly title from the layer's annotations. + /// + /// Returns `Some(&str)` containing the value of the `"org.opencontainers.image.title"` + /// annotation if present, or `None` if that annotation is absent. + /// + /// # Examples + /// + /// ``` + /// use std::collections::HashMap; + /// + /// let mut annotations = HashMap::new(); + /// annotations.insert( + /// "org.opencontainers.image.title".to_string(), + /// "example.txt".to_string(), + /// ); + /// + /// let layer = crate::OciLayer { + /// media_type: String::new(), + /// digest: String::new(), + /// size: 0, + /// annotations, + /// }; + /// + /// assert_eq!(layer.title(), Some("example.txt")); + /// ``` pub fn title(&self) -> Option<&str> { self.annotations .get("org.opencontainers.image.title") @@ -104,6 +154,24 @@ pub struct OciDownload { } impl OciDownload { + /// Creates a new `OciDownload` for the given OCI reference using sensible defaults. + /// + /// Defaults: + /// - `api` = "https://ghcr.io/v2" + /// - `filter` = `Filter::default()` + /// - no output path (downloads to current working directory unless `output` is set) + /// - `overwrite` = `OverwriteMode::Prompt` + /// - `extract` = `false` + /// - `parallel` = 1 (sequential downloads) + /// - no progress callback + /// + /// # Examples + /// + /// ``` + /// let dl = OciDownload::new("ghcr.io/myorg/myrepo:latest"); + /// // configure and run: + /// // let result = dl.output("out").execute(); + /// ``` pub fn new(reference: impl Into) -> Self { Self { reference: reference.into(), @@ -118,41 +186,145 @@ impl OciDownload { } } + /// Sets the base API URL used for manifest and blob requests and returns the downloader for chaining. + /// + /// # Examples + /// + /// ``` + /// let dl = OciDownload::new("ghcr.io/foo/bar:latest").api("https://ghcr.example.com/v2"); + /// ``` pub fn api(mut self, api: impl Into) -> Self { self.api = api.into(); self } + /// Sets the layer filter used to select which OCI image layers will be downloaded. + /// + /// The provided `filter` is applied when a manifest is inspected to decide which layers are included + /// in the download operation. Returns the downloader with the filter updated for further chaining. + /// + /// # Examples + /// + /// ``` + /// let downloader = OciDownload::new("owner/repo:tag") + /// .filter(Filter::SomeVariant) + /// .output("out/dir"); + /// ``` pub fn filter(mut self, filter: Filter) -> Self { self.filter = filter; self } + /// Sets the output directory path where downloaded files will be written. + + /// + + /// The provided path is stored and used as the destination for downloaded blobs and extracted contents. + + /// + + /// # Examples + + /// + + /// ``` + + /// let dl = OciDownload::new("ghcr.io/org/repo:tag").output("downloads/"); + + /// ``` pub fn output(mut self, path: impl Into) -> Self { self.output = Some(path.into()); self } + /// Sets how existing files are handled when writing downloads. + /// + /// This configures the downloader's overwrite behavior to the provided `mode`. + /// + /// # Examples + /// + /// ``` + /// let dl = OciDownload::new("ghcr.io/example/repo:tag").overwrite(OverwriteMode::Prompt); + /// ``` + — Returns the modified downloader with the given overwrite mode. pub fn overwrite(mut self, mode: OverwriteMode) -> Self { self.overwrite = mode; self } + /// Enable or disable extraction of downloaded layers. + + /// + + /// When `true`, downloaded archive layers will be extracted after download into the extraction + + /// destination (if set via `extract_to`) or into the configured output directory. When `false`, + + /// archives are left as downloaded. + + /// + + /// # Examples + + /// + + /// ``` + + /// let dl = OciDownload::new("ghcr.io/org/pkg:1.0") + + /// .extract(true) + + /// .extract_to("/tmp/out"); + + /// ``` pub fn extract(mut self, extract: bool) -> Self { self.extract = extract; self } + /// Sets the destination directory for extracting downloaded archives. + /// + /// When extraction is enabled, extracted files will be written to this path instead of the download output directory. + /// + /// # Examples + /// + /// ``` + /// let downloader = OciDownload::new("ghcr.io/org/pkg:1.0") + /// .extract(true) + /// .extract_to("/tmp/extracted"); + /// ``` pub fn extract_to(mut self, path: impl Into) -> Self { self.extract_to = Some(path.into()); self } + /// Set the number of parallel download workers (at least 1). + /// + /// If `n` is less than 1, the value will be clamped to 1. Returns the updated downloader to allow chaining. + /// + /// # Examples + /// + /// ``` + /// let _ = OciDownload::new("owner/repo:tag").parallel(4); + /// ``` pub fn parallel(mut self, n: usize) -> Self { self.parallel = n.max(1); self } + /// Registers a progress callback to receive download `Progress` events. + /// + /// The provided callback will be invoked with progress updates (e.g., Starting, Chunk, Complete) + /// from the download workers and must be thread-safe (`Send + Sync`) and `'static`. + /// + /// # Examples + /// + /// ``` + /// let downloader = OciDownload::new("ghcr.io/owner/repo:tag") + /// .progress(|progress| { + /// println!("progress: {:?}", progress); + /// }); + /// ``` pub fn progress(mut self, f: F) -> Self where F: Fn(Progress) + Send + Sync + 'static, @@ -161,6 +333,32 @@ impl OciDownload { self } + /// Downloads the OCI reference according to the configured options and returns the downloaded file paths. + /// + /// Attempts a direct blob download if the reference tag is a digest (e.g., `sha256:...`); otherwise it + /// fetches the image manifest, selects layers whose titles match the configured filter, creates the + /// output directory if provided, and downloads the selected layers either sequentially or in parallel + /// according to the `parallel` setting. Emits `Progress::Starting` and `Progress::Complete` events + /// via the registered progress callback when present. + /// + /// # Returns + /// + /// A `Vec` containing the filesystem paths of the downloaded files on success. Returns a + /// `DownloadError::LayerNotFound` if no manifest layers match the configured filter, or other + /// `DownloadError` variants for network, IO, or extraction failures. + /// + /// # Examples + /// + /// ``` + /// # use std::path::PathBuf; + /// # use soar_dl::oci::OciDownload; + /// // Download layers from an OCI reference into "out" directory. + /// let paths: Vec = OciDownload::new("ghcr.io/owner/repo:latest") + /// .output("out") + /// .execute() + /// .unwrap(); + /// assert!(!paths.is_empty()); + /// ``` pub fn execute(self) -> Result, DownloadError> { // If it's a blob digest, download directly if self.reference.tag.starts_with("sha256:") { @@ -216,6 +414,24 @@ impl OciDownload { Ok(paths) } + /// Downloads the provided OCI layers one after another into `output_dir` and returns the paths of the saved files. + /// + /// Each layer is written using its `title()` as the filename. If `self.extract` is enabled, each downloaded archive + /// is extracted into `self.extract_to` (if set) or into `output_dir`. The method updates an internal downloaded byte + /// counter while performing transfers. + /// + /// # Errors + /// + /// Returns a `DownloadError` if any individual layer download or extraction fails. + /// + /// # Parameters + /// + /// - `layers`: slice of layer references to download (order is preserved). + /// - `output_dir`: destination directory for downloaded files. + /// + /// # Returns + /// + /// A `Vec` containing the full paths to the downloaded (and optionally extracted) files. fn download_layers_sequential( &self, layers: &[&OciLayer], @@ -244,6 +460,38 @@ impl OciDownload { Ok(paths) } + /// Download multiple OCI layers concurrently and return the downloaded file paths. + /// + /// This method downloads the provided layers using up to `self.parallel` worker threads, + /// writes each layer into `output_dir` (using the layer title as filename), optionally + /// extracts archives when extraction is enabled, and aggregates any errors that occur + /// across worker threads into a single `DownloadError::Multiple`. + /// + /// - If a layer has no title, it is skipped. + /// - On success, returns a `Vec` containing the paths to successfully downloaded files. + /// - On failure, returns `DownloadError::Multiple` with stringified error messages from workers. + /// + /// # Parameters + /// + /// - `layers`: slice of layer references to download. Each layer's title is used as the filename. + /// - `output_dir`: directory where downloaded files will be written. + /// + /// # Returns + /// + /// `Ok(Vec)` with the paths of successfully downloaded files, or `Err(DownloadError::Multiple)` + /// if one or more worker threads report errors. + /// + /// # Examples + /// + /// ``` + /// // High-level example: download all layers from a reference in parallel. + /// // (Types and setup omitted for brevity.) + /// let dl = OciDownload::new("ghcr.io/owner/repo:tag") + /// .output("out") + /// .parallel(4); + /// let result = dl.execute(); // invokes the parallel download path when appropriate + /// assert!(result.is_ok()); + /// ``` fn download_layers_parallel( &self, layers: &[&OciLayer], @@ -327,6 +575,19 @@ impl OciDownload { Ok(paths) } + /// Fetches the OCI/Docker manifest for the configured reference and returns it deserialized as an `OciManifest`. + /// + /// The request is made against the download instance's `api` base and the reference's `package`/`tag`. On + /// non-success HTTP status codes this returns `DownloadError::HttpError`; if the response body cannot be + /// parsed as a manifest JSON this returns `DownloadError::InvalidResponse`. + /// + /// # Examples + /// + /// ```no_run + /// let dl = OciDownload::new("ghcr.io/example/repo:latest"); + /// let manifest = dl.fetch_manifest().unwrap(); + /// assert!(manifest.layers.len() >= 0); + /// ``` fn fetch_manifest(&self) -> Result { let url = format!( "{}/{}/manifests/{}", @@ -359,6 +620,20 @@ impl OciDownload { .map_err(|_| DownloadError::InvalidResponse) } + /// Downloads the single blob identified by the downloader's reference into the configured output location. + /// + /// The resulting file is written using the configured output path (or a name derived from the reference if no output is set), and download options such as overwrite mode and the registered progress callback are respected. + /// + /// # Examples + /// + /// ``` + /// // Constructing and using `OciDownload` in this example assumes the builder API is available. + /// let dl = OciDownload::new("ghcr.io/org/package:tag"); + /// let paths = dl.download_blob().unwrap(); + /// assert_eq!(paths.len(), 1); + /// let downloaded = &paths[0]; + /// println!("Downloaded to: {:?}", downloaded); + /// ``` fn download_blob(&self) -> Result, DownloadError> { let filename = self .reference @@ -391,6 +666,31 @@ impl OciDownload { Ok(vec![path]) } + /// Downloads a single OCI layer blob to the given file path, updating the provided + /// cumulative `downloaded` byte counter and emitting progress events if configured. + /// + /// # Parameters + /// + /// - `layer`: The manifest layer to download. + /// - `path`: Destination filesystem path for the downloaded blob. + /// - `downloaded`: Mutable cumulative byte counter; this function increments it by the + /// number of bytes written for this layer. + /// + /// # Returns + /// + /// `Ok(())` on success, or a `DownloadError` describing the failure. + /// + /// # Examples + /// + /// ```no_run + /// use std::path::Path; + /// let downloader = OciDownload::new("ghcr.io/owner/repo:tag"); + /// let layer = /* obtain OciLayer from a manifest */; + /// let mut downloaded = 0u64; + /// let dest = Path::new("/tmp/layer.tar"); + /// downloader.download_layer(&layer, dest, &mut downloaded).unwrap(); + /// assert!(downloaded > 0); + /// ``` fn download_layer( &self, layer: &OciLayer, @@ -409,6 +709,46 @@ impl OciDownload { } } +/// Downloads a single OCI layer blob to the given file path with resume support, progress reporting, and post-download handling. +/// +/// This function: +/// - Resumes partially downloaded blobs when resume metadata exists (uses Range and If-Range headers). +/// - Appends to the existing file when a partial response (206) is returned, otherwise creates a new file. +/// - Periodically persists resume metadata while downloading (every 1 MiB). +/// - Emits `Progress::Chunk` updates via the optional `on_progress` callback using the shared downloaded counter. +/// - Marks the file executable (0o755) if it appears to be an ELF binary and removes any resume metadata on success. +/// Returns `Ok(())` on success or a `DownloadError` on failure. +/// +/// # Examples +/// +/// ``` +/// use std::sync::{Arc, Mutex}; +/// use std::path::Path; +/// +/// let reference = OciReference::from("ghcr.io/owner/repo:tag"); +/// let layer = OciLayer { +/// media_type: "application/vnd.oci.image.layer.v1.tar".to_string(), +/// digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(), +/// size: 1024, +/// annotations: Default::default(), +/// }; +/// +/// let mut local_downloaded = 0u64; +/// let shared_downloaded = Arc::new(Mutex::new(0u64)); +/// let progress_cb = Arc::new(|_p: Progress| {}); +/// let out_path = Path::new("/tmp/layer.blob"); +/// +/// // Note: in real use, `api` should be the registry v2 API base and the types must be in scope. +/// let _ = download_layer_impl( +/// "https://ghcr.io/v2", +/// &reference, +/// &layer, +/// out_path, +/// &mut local_downloaded, +/// Some(&progress_cb), +/// &shared_downloaded, +/// ); +/// ``` fn download_layer_impl( api: &str, reference: &OciReference, @@ -505,4 +845,4 @@ fn download_layer_impl( remove_resume(path)?; Ok(()) -} +} \ No newline at end of file diff --git a/crates/soar-dl/src/platform.rs b/crates/soar-dl/src/platform.rs index fb3a20e05..ea03e9f64 100644 --- a/crates/soar-dl/src/platform.rs +++ b/crates/soar-dl/src/platform.rs @@ -43,6 +43,33 @@ static GITLAB_RE: LazyLock = LazyLock::new(|| { }); impl PlatformUrl { + /// Classifies an input string as a platform URL and returns the corresponding `PlatformUrl` variant. + /// + /// This inspects the input URL (or reference) and returns: + /// - `Oci` when the normalized string starts with `ghcr.io/` (treated as an OCI reference), + /// - `Github` when it matches the GitHub repository pattern, extracting project and optional tag, + /// - `Gitlab` when it matches the GitLab repository pattern, extracting project and optional tag + /// (except when the project looks like an API path or contains `/-/`, which is treated as `Direct`), + /// - `Direct` when the input parses as a valid URL with a scheme and host. + /// Returns `None` if the input cannot be classified or parsed as a valid URL. + /// + /// # Examples + /// + /// ``` + /// use crate::platform::PlatformUrl; + /// + /// // OCI reference + /// let p = PlatformUrl::parse("ghcr.io/myorg/myimage:latest").unwrap(); + /// assert!(matches!(p, PlatformUrl::Oci { .. })); + /// + /// // GitHub repo + /// let p = PlatformUrl::parse("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/owner/repo/releases/tag/v1.0").unwrap(); + /// assert!(matches!(p, PlatformUrl::Github { .. })); + /// + /// // Direct URL + /// let p = PlatformUrl::parse("https://example.com/resource").unwrap(); + /// assert!(matches!(p, PlatformUrl::Direct { .. })); + /// ``` pub fn parse(url: impl AsRef) -> Option { let url = url.as_ref(); @@ -86,6 +113,23 @@ impl PlatformUrl { }) } + /// Extracts a repository project path and an optional tag from `url` using `re`. + /// + /// The returned `project` is the first capture group as a `String`. The optional `tag` + /// is taken from the second capture group (if present), with surrounding quotes and + /// spaces removed and URI-decoded. Returns `None` if the regex does not match. + /// + /// # Examples + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"^/?([^@#]+)(?:[@#](.+))?$").unwrap(); + /// let url = "owner/repo@v1.2.3"; + /// let (project, tag) = super::parse_repo(&re, url).unwrap(); + /// assert_eq!(project, "owner/repo"); + /// assert_eq!(tag.as_deref(), Some("v1.2.3")); + /// ``` fn parse_repo(re: &Regex, url: &str) -> Option<(String, Option)> { let caps = re.captures(url)?; let project = caps.get(1)?.as_str().to_string(); @@ -104,6 +148,20 @@ impl PlatformUrl { } } +/// Fetches JSON from a fallback base URL then, on retryable HTTP failures, from a primary base URL using an optional Bearer token, and returns the deserialized items as a Vec. +/// +/// The function first requests `fallback + path` without a token. If that request fails with a status that qualifies for retry (429, 401, 403, or any 5xx), it retries `primary + path` and, if the environment variable named by `token_env` is set, includes it as a `Authorization: Bearer ` header. The response body must be either a JSON array (mapped to `Vec`) or a single JSON object (mapped to a one-element `Vec`); other shapes produce `DownloadError::InvalidResponse`. Non-success HTTP statuses produce `DownloadError::HttpError`. +/// +/// # Examples +/// +/// ```no_run +/// use serde::Deserialize; +/// #[derive(Deserialize)] +/// struct Item { id: u32 } +/// +/// // Attempts fallback first, then primary with token from "API_TOKEN" if needed. +/// let result: Result, _> = fetch_with_fallback("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/api/items", "https://primary.example.com", "https://fallback.example.com", "API_TOKEN"); +/// ``` pub fn fetch_with_fallback( path: &str, primary: &str, @@ -160,7 +218,12 @@ where }) } +/// Determines whether a download `HttpError` status should cause a fallback attempt. +/// +/// # Returns +/// +/// `true` if the error is an HTTP status of 429, 401, 403, or any status >= 500; `false` otherwise. fn should_fallback_status(e: &DownloadError) -> bool { matches!(e, DownloadError::HttpError { status, .. } if *status == 429 || *status == 401 || *status == 403 || *status >= 500) -} +} \ No newline at end of file diff --git a/crates/soar-dl/src/release.rs b/crates/soar-dl/src/release.rs index 77bf85131..86209939c 100644 --- a/crates/soar-dl/src/release.rs +++ b/crates/soar-dl/src/release.rs @@ -21,6 +21,26 @@ pub struct ReleaseDownload { } impl ReleaseDownload

{ + /// Creates a new `ReleaseDownload` configured for the given project with sensible defaults. + /// + /// The returned builder is initialized with: + /// - `tag = None` + /// - a default `Filter` + /// - no explicit output path + /// - `overwrite = OverwriteMode::Prompt` + /// - extraction disabled + /// - no extraction path + /// - no progress callback + /// + /// # Examples + /// + /// ``` + /// // `MyPlatform` must implement the `Platform` trait for your environment. + /// // Replace `MyPlatform` with the concrete platform type you use (e.g., `GitHub`). + /// let dl = ReleaseDownload::::new("owner/repo"); + /// // You can then chain further configuration: + /// // let dl = dl.tag("v1.2.3").output("downloads/").extract(true); + /// ``` pub fn new(project: impl Into) -> Self { Self { project: project.into(), @@ -35,36 +55,113 @@ impl ReleaseDownload

{ } } + /// Sets the release tag to target when selecting a release. + /// + /// The provided tag will be used by `execute` to find a release with a matching tag. + /// Returns the updated builder to allow method chaining. + /// + /// # Examples + /// + /// ``` + /// let builder = ReleaseDownload::::new("owner/repo").tag("v1.2.3"); + /// ``` pub fn tag(mut self, tag: impl Into) -> Self { self.tag = Some(tag.into()); self } + /// Sets the asset filter used to select which release assets will be downloaded. + /// + /// The provided `filter` will be used to match asset names when executing the download. + /// + /// # Examples + /// + /// ``` + /// let rd = ReleaseDownload::new("owner/repo").filter(Filter::default()); + /// ``` pub fn filter(mut self, filter: Filter) -> Self { self.filter = filter; self } + /// Sets the base output path for downloaded assets. + /// + /// The provided path will be used as the destination directory or base file path when downloads are written. + /// + /// # Returns + /// + /// The modified `ReleaseDownload` builder with the output path set. + /// + /// # Examples + /// + /// ``` + /// let dl = ReleaseDownload::::new("owner/repo").output("downloads"); + /// ``` pub fn output(mut self, path: impl Into) -> Self { self.output = Some(path.into()); self } + /// Set the overwrite behavior for downloaded files. + /// + /// `mode` determines how existing files are handled when downloading (for example, overwrite, skip, or prompt). + /// + /// # Examples + /// + /// ``` + /// let dl = ReleaseDownload::new("owner/repo").overwrite(OverwriteMode::Overwrite); + /// ``` pub fn overwrite(mut self, mode: OverwriteMode) -> Self { self.overwrite = mode; self } + /// Enables or disables extraction of downloaded assets. + /// + /// When set to `true`, assets that are archives will be extracted after they are downloaded. + /// + /// # Examples + /// + /// ``` + /// let rd = ReleaseDownload::::new("owner/repo").extract(true); + /// ``` pub fn extract(mut self, extract: bool) -> Self { self.extract = extract; self } + /// Sets the destination directory where downloaded archives will be extracted. + /// + /// # Arguments + /// + /// * `path` - Destination path to extract downloaded assets into. + /// + /// # Examples + /// + /// ``` + /// let rd = ReleaseDownload::new("owner/repo").extract_to("out/artifacts"); + /// ``` pub fn extract_to(mut self, path: impl Into) -> Self { self.extract_to = Some(path.into()); self } + /// Registers a callback that will be invoked with progress updates for each download. + /// + /// The provided callback is stored and called with `Progress` events as assets are downloaded. + /// + /// # Examples + /// + /// ``` + /// use std::sync::Arc; + /// use soar_dl::{ReleaseDownload, Progress}; + /// + /// let _rd = ReleaseDownload::new("owner/repo") + /// .progress(|progress: Progress| { + /// // handle progress (e.g., log or update UI) + /// println!("{:?}", progress); + /// }); + /// ``` pub fn progress(mut self, f: F) -> Self where F: Fn(Progress) + Send + Sync + 'static, @@ -73,6 +170,34 @@ impl ReleaseDownload

{ self } + /// Downloads matched assets for a project's release and returns their local file paths. + /// + /// Selects a release by the configured tag if provided; otherwise prefers the first non-prerelease + /// release or falls back to the first release. Filters the release's assets using the configured + /// `Filter`, downloads each matching asset with the configured output, overwrite, and extraction + /// options, and returns a vector of the resulting local `PathBuf`s. Returns an error if no release + /// is found or if no assets match the filter. + /// + /// # Returns + /// + /// A `Vec` containing the local paths of the downloaded assets on success, or a + /// `DownloadError` on failure. + /// + /// # Examples + /// + /// ```no_run + /// use std::path::PathBuf; + /// + /// // Example usage (replace `MyPlatform` with an implementation of `Platform`). + /// let paths: Vec = ReleaseDownload::::new("owner/repo") + /// .tag("v1.0") + /// .filter(Filter::default()) + /// .output("downloads") + /// .execute() + /// .unwrap(); + /// + /// assert!(!paths.is_empty()); + /// ``` pub fn execute(self) -> Result, DownloadError> { let releases = P::fetch_releases(&self.project, self.tag.as_deref())?; @@ -129,4 +254,4 @@ impl ReleaseDownload

{ Ok(paths) } -} +} \ No newline at end of file diff --git a/crates/soar-dl/src/utils.rs b/crates/soar-dl/src/utils.rs index 910f51c00..db26006f5 100644 --- a/crates/soar-dl/src/utils.rs +++ b/crates/soar-dl/src/utils.rs @@ -6,7 +6,39 @@ use url::Url; use crate::error::DownloadError; -/// Extract filename from URL path +/// Extracts the final path segment from a URL as a filename. + +/// + +/// Returns `Some(String)` containing the last non-empty path segment if the URL + +/// parses and its path has a non-empty final segment, otherwise `None`. + +/// + +/// # Examples + +/// + +/// ``` + +/// let name = filename_from_url("https://example.com/path/to/file.txt"); + +/// assert_eq!(name.as_deref(), Some("file.txt")); + +/// + +/// let no_name = filename_from_url("https://example.com/path/to/"); + +/// assert_eq!(no_name, None); + +/// + +/// let invalid = filename_from_url("not a url"); + +/// assert_eq!(invalid, None); + +/// ``` pub fn filename_from_url(url: &str) -> Option { Url::parse(url).ok().and_then(|u| { u.path_segments() @@ -21,7 +53,18 @@ pub fn filename_from_url(url: &str) -> Option { }) } -/// Extract filename from Content-Disposition header +/// Extracts a filename from a `Content-Disposition` header value. +/// +/// Parses the header string for a `filename=` parameter and returns its value +/// with surrounding double quotes removed. +/// +/// # Examples +/// +/// ``` +/// use ureq::http::HeaderValue; +/// let header = HeaderValue::from_static("attachment; filename=\"example.txt\""); +/// assert_eq!(crate::utils::filename_from_header(&header), Some("example.txt".to_string())); +/// ``` pub fn filename_from_header(value: &HeaderValue) -> Option { value .to_str() @@ -37,7 +80,41 @@ pub fn filename_from_header(value: &HeaderValue) -> Option { }) } -/// Determine output path +/// Compute the final file system path for a download based on an explicit output and inferred filenames. +/// +/// The `output` argument can be: +/// - `Some("-")` to represent writing to stdout (path is `"-"`). +/// - A string ending with `'/'` to indicate a directory; a filename is chosen from `header_filename` or `url_filename`. +/// - A path that is an existing directory, in which case a filename is chosen from `header_filename` or `url_filename`. +/// - A file path (returned as-is). +/// If `output` is `None`, a filename is required from `header_filename` or `url_filename`. +/// +/// # Returns +/// +/// `Ok(PathBuf)` with the resolved path, or `Err(DownloadError::NoFilename)` if a filename is required but neither `header_filename` nor `url_filename` is available. +/// +/// # Examples +/// +/// ``` +/// use std::path::PathBuf; +/// use crate::error::DownloadError; +/// +/// // explicit directory with trailing slash prefers header filename +/// let out = super::resolve_output_path(Some("downloads/"), Some("from_url.txt".into()), Some("from_header.txt".into())).unwrap(); +/// assert_eq!(out, PathBuf::from("downloads").join("from_header.txt")); +/// +/// // stdout shortcut +/// let out = super::resolve_output_path(Some("-"), None, None).unwrap(); +/// assert_eq!(out, PathBuf::from("-")); +/// +/// // no output given uses header or url filename +/// let out = super::resolve_output_path(None, Some("a.txt".into()), None).unwrap(); +/// assert_eq!(out, PathBuf::from("a.txt")); +/// +/// // error when no filename available +/// let err = super::resolve_output_path(None, None, None).unwrap_err(); +/// assert_eq!(err, DownloadError::NoFilename); +/// ``` pub fn resolve_output_path( output: Option<&str>, url_filename: Option, @@ -69,4 +146,4 @@ pub fn resolve_output_path( .ok_or(DownloadError::NoFilename) } } -} +} \ No newline at end of file diff --git a/crates/soar-dl/src/xattr.rs b/crates/soar-dl/src/xattr.rs index c974b41aa..f8d5f23ee 100644 --- a/crates/soar-dl/src/xattr.rs +++ b/crates/soar-dl/src/xattr.rs @@ -4,6 +4,21 @@ use crate::types::ResumeInfo; const XATTR_RESUME_KEY: &str = "user.soar.resume"; +/// Reads the `user.soar.resume` extended attribute from the given path and deserializes it into a `ResumeInfo`. +/// +/// Returns `Some(ResumeInfo)` if the attribute exists and contains valid JSON; returns `None` if the attribute is missing, cannot be read, or fails to deserialize. +/// +/// # Examples +/// +/// ``` +/// // Attempt to read resume info from a file path. +/// if let Some(info) = read_resume("/path/to/file") { +/// // use `info: ResumeInfo` +/// let _ = info; +/// } else { +/// // no resume info available or it could not be parsed +/// } +/// ``` pub fn read_resume>(path: P) -> Option { xattr::get(path, XATTR_RESUME_KEY) .ok() @@ -11,10 +26,37 @@ pub fn read_resume>(path: P) -> Option { .and_then(|v| serde_json::from_slice(&v).ok()) } +/// Writes the provided `ResumeInfo` into the file's extended attribute `user.soar.resume`. +/// +/// Serializes `info` to JSON and stores the resulting bytes in the extended attribute for `path`. +/// +/// # Returns +/// +/// `Ok(())` on success, or an `std::io::Error` if serialization or the xattr write fails. +/// +/// # Examples +/// +/// ``` +/// // Construct a ResumeInfo value (fields omitted for brevity) +/// let info = /* ResumeInfo { ... } */ ; +/// let path = std::path::Path::new("/tmp/download.partial"); +/// write_resume(path, &info).unwrap(); +/// ``` pub fn write_resume>(path: P, info: &ResumeInfo) -> std::io::Result<()> { xattr::set(path, XATTR_RESUME_KEY, &serde_json::to_vec(info)?) } +/// Removes the stored resume extended attribute from the given path. +/// +/// Returns `Ok(())` if the attribute was removed successfully, or an `Err` with the I/O error encountered. +/// +/// # Examples +/// +/// ``` +/// use std::path::Path; +/// // Call with any path; result indicates whether the xattr removal succeeded. +/// let _ = remove_resume(Path::new("/tmp/some_file")); +/// ``` pub fn remove_resume>(path: P) -> std::io::Result<()> { xattr::remove(path, XATTR_RESUME_KEY) -} +} \ No newline at end of file diff --git a/crates/soar-utils/src/fs.rs b/crates/soar-utils/src/fs.rs index 743384fdf..09a1a143c 100644 --- a/crates/soar-utils/src/fs.rs +++ b/crates/soar-utils/src/fs.rs @@ -226,19 +226,22 @@ pub fn read_file_signature>(path: P, bytes: usize) -> FileSystemR Ok(buffer) } -/// Returns the total size of a directory and its contents. +/// Calculate the total size in bytes of a directory and all files contained within it. /// -/// # Arguments -/// * `path` - The path to the directory +/// Skips entries whose directory entry or metadata cannot be read. Recurses into subdirectories +/// and accumulates file sizes. /// /// # Returns -/// Returns the total size of the directory and its contents. +/// +/// The total size in bytes of the directory and its contents. /// /// # Errors -/// Returns a [`FileSystemError::Directory`] if the directory could not be read. /// -/// # Example -/// ```no_run +/// Returns a [`FileSystemError::Directory`] if the directory itself cannot be read. +/// +/// # Examples +/// +/// ``` /// use soar_utils::fs::dir_size; /// use soar_utils::error::FileSystemResult; /// @@ -271,7 +274,25 @@ pub fn dir_size>(path: P) -> FileSystemResult { Ok(total_size) } -/// Check if file is ELF binary +/// Determine whether the file at the given path is an ELF binary. +/// +/// Checks the file's first four bytes for the ELF magic sequence (0x7F, 'E', 'L', 'F') and +/// returns `true` if they match, `false` otherwise. +/// +/// # Examples +/// +/// ``` +/// use std::fs::File; +/// use std::io::Write; +/// use tempfile::tempdir; +/// +/// let dir = tempdir().unwrap(); +/// let path = dir.path().join("example_elf"); +/// let mut f = File::create(&path).unwrap(); +/// f.write_all(&[0x7f, b'E', b'L', b'F', 0x00]).unwrap(); +/// +/// assert!(is_elf(&path)); +/// ``` pub fn is_elf>(path: P) -> bool { read_file_signature(path, 4) .ok() @@ -684,4 +705,4 @@ mod tests { perms.set_readonly(false); fs::set_permissions(tempdir.path(), perms).unwrap(); } -} +} \ No newline at end of file From 7a9ea016f7215b50ac8adab02ab7397699e323ed Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Sat, 1 Nov 2025 12:29:43 +0545 Subject: [PATCH 09/22] fix docs --- crates/soar-dl/src/download.rs | 60 +++++---------- crates/soar-dl/src/error.rs | 2 +- crates/soar-dl/src/filter.rs | 15 +++- crates/soar-dl/src/github.rs | 53 +++++++++---- crates/soar-dl/src/gitlab.rs | 50 +++++++++--- crates/soar-dl/src/http.rs | 7 +- crates/soar-dl/src/http_client.rs | 41 ++++------ crates/soar-dl/src/oci.rs | 124 +++++++++++++----------------- crates/soar-dl/src/platform.rs | 34 +++----- crates/soar-dl/src/release.rs | 63 ++++++++++----- crates/soar-dl/src/utils.rs | 39 ++++------ crates/soar-dl/src/xattr.rs | 25 ++++-- 12 files changed, 271 insertions(+), 242 deletions(-) diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs index 2e3fdc096..c6d0068ee 100644 --- a/crates/soar-dl/src/download.rs +++ b/crates/soar-dl/src/download.rs @@ -44,6 +44,8 @@ impl Download { /// # Examples /// /// ``` + /// use soar_dl::download::Download; + /// /// let dl = Download::new("https://example.com/archive.tar.gz") /// .output("archive.tar.gz"); /// // `dl` is ready to call `execute()` @@ -70,7 +72,8 @@ impl Download { /// # Examples /// /// ``` - /// use crate::download::Download; + /// use soar_dl::download::Download; + /// /// let _ = Download::new("https://example.com/file").output("path/to/file"); /// ``` pub fn output(mut self, output: impl Into) -> Self { @@ -85,8 +88,8 @@ impl Download { /// # Examples /// /// ``` - /// use crate::download::Download; - /// use crate::types::OverwriteMode; + /// use soar_dl::download::Download; + /// use soar_dl::types::OverwriteMode; /// /// let d = Download::new("https://example.com/file") /// .overwrite(OverwriteMode::Force) @@ -105,7 +108,9 @@ impl Download { /// # Examples /// /// ``` - /// let dl = crate::download::Download::new("https://example.com/archive.tar.gz") + /// use soar_dl::download::Download; + /// + /// let dl = Download::new("https://example.com/archive.tar.gz") /// .extract(true); /// ``` pub fn extract(mut self, extract: bool) -> Self { @@ -122,6 +127,8 @@ impl Download { /// # Examples /// /// ```no_run + /// use soar_dl::download::Download; + /// /// let dl = Download::new("https://example.com/archive.tar.gz") /// .extract(true) /// .extract_to("/tmp/my-extract-dir"); @@ -137,14 +144,14 @@ impl Download { /// /// # Examples /// - /// ``` - /// use crate::download::Download; - /// use crate::types::Progress; + /// ```no_run + /// use soar_dl::download::Download; + /// use soar_dl::types::Progress; /// /// let _dl = Download::new("https://example.com/file") /// .progress(|event: Progress| match event { /// Progress::Starting { total } => eprintln!("starting, total={}", total), - /// Progress::Chunk { downloaded, chunk } => eprintln!("downloaded {} (+{})", downloaded, chunk), + /// Progress::Chunk { total, current } => eprintln!("downloaded {} (+{})", total, current), /// Progress::Complete { total } => eprintln!("complete, total={}", total), /// }); /// ``` @@ -173,7 +180,9 @@ impl Download { /// /// # Examples /// - /// ``` + /// ```no_run + /// use soar_dl::download::Download; + /// /// let dl = Download::new("https://example.com/archive.tar.gz") /// .output("archive.tar.gz") /// .extract(true); @@ -281,17 +290,6 @@ impl Download { /// # Returns /// /// `Ok(())` on successful completion of the download and file write, or `Err(DownloadError)` on IO, HTTP, or resume-state persistence failures. - /// - /// # Examples - /// - /// ```no_run - /// use std::path::Path; - /// - /// // Illustrative usage (no network interaction in this example). - /// let download = crate::download::Download::new("https://example.com/file.tar.gz").output("file.tar.gz"); - /// // Attempt a fresh download (no resume info). - /// let _ = download.download_to_file(Path::new("file.tar.gz"), None); - /// ``` fn download_to_file( &self, path: &Path, @@ -380,15 +378,6 @@ impl Download { /// # Returns /// /// `u64` total size in bytes if present in the response headers, `0` otherwise. - /// - /// # Examples - /// - /// ```rust,no_run - /// use ureq::{Response, Body}; - /// - /// // Given a `Response` named `resp`, obtain the total size: - /// // let total = parse_content_length(&resp); - /// ``` fn parse_content_length(resp: &Response) -> u64 { resp.headers() .get(CONTENT_RANGE) @@ -412,19 +401,6 @@ impl Download { /// # Returns /// /// `true` if the user entered `"y"` or `"yes"` (case-insensitive), `false` otherwise. -/// -/// # Examples -/// -/// ```no_run -/// use std::path::Path; -/// -/// let confirmed = prompt_overwrite(Path::new("output.bin")).unwrap(); -/// if confirmed { -/// println!("User confirmed overwrite"); -/// } else { -/// println!("User declined overwrite"); -/// } -/// ``` fn prompt_overwrite(path: &Path) -> std::io::Result { print!("Overwrite {}? [y/N] ", path.display()); std::io::stdout().flush()?; diff --git a/crates/soar-dl/src/error.rs b/crates/soar-dl/src/error.rs index 85b479649..1fd712981 100644 --- a/crates/soar-dl/src/error.rs +++ b/crates/soar-dl/src/error.rs @@ -82,4 +82,4 @@ impl From for DownloadError { fn from(e: ureq::Error) -> Self { Self::Network(Box::new(e)) } -} \ No newline at end of file +} diff --git a/crates/soar-dl/src/filter.rs b/crates/soar-dl/src/filter.rs index 5bc2caebc..166626ae6 100644 --- a/crates/soar-dl/src/filter.rs +++ b/crates/soar-dl/src/filter.rs @@ -25,6 +25,8 @@ impl Filter { /// # Examples /// /// ``` + /// use soar_dl::filter::Filter; + /// /// let f = Filter { /// regexes: Vec::new(), /// globs: vec!["*".into()], @@ -37,7 +39,14 @@ impl Filter { pub fn matches(&self, name: &str) -> bool { let matches_regex = self.regexes.is_empty() || self.regexes.iter().all(|r| r.is_match(name)); - let matches_glob = self.globs.is_empty() || self.globs.iter().any(|g| glob_match(g, name)); + let matches_glob = self.globs.is_empty() + || if self.case_sensitive { + self.globs.iter().any(|g| glob_match(g, name)) + } else { + self.globs + .iter() + .any(|g| glob_match(g.to_lowercase(), name.to_lowercase())) + }; let matches_include = self.matches_keywords(name, &self.include, true); let matches_exclude = self.matches_keywords(name, &self.exclude, false); @@ -60,7 +69,7 @@ impl Filter { /// /// ``` /// use regex::Regex; - /// use crate::filter::Filter; + /// use soar_dl::filter::Filter; /// /// let filter = Filter { /// regexes: vec![], @@ -107,4 +116,4 @@ impl Filter { } }) } -} \ No newline at end of file +} diff --git a/crates/soar-dl/src/github.rs b/crates/soar-dl/src/github.rs index a212953ce..f49c62bf7 100644 --- a/crates/soar-dl/src/github.rs +++ b/crates/soar-dl/src/github.rs @@ -10,18 +10,18 @@ pub struct Github; #[derive(Debug, Clone, Deserialize)] pub struct GithubRelease { - name: Option, - tag_name: String, - prerelease: bool, - published_at: String, - assets: Vec, + pub name: Option, + pub tag_name: String, + pub prerelease: bool, + pub published_at: String, + pub assets: Vec, } #[derive(Debug, Clone, Deserialize)] pub struct GithubAsset { - name: String, - size: u64, - browser_download_url: String, + pub name: String, + pub size: u64, + pub browser_download_url: String, } impl Platform for Github { @@ -47,6 +47,9 @@ impl Platform for Github { /// # Examples /// /// ``` + /// use soar_dl::github::Github; + /// use soar_dl::traits::{Platform, Release}; + /// /// let releases = Github::fetch_releases("rust-lang/rust", None).unwrap(); /// assert!(releases.iter().all(|r| r.tag().len() > 0)); /// ``` @@ -83,6 +86,9 @@ impl Release for GithubRelease { /// # Examples /// /// ``` + /// use soar_dl::github::GithubRelease; + /// use soar_dl::traits::Release; + /// /// let r = GithubRelease { /// name: Some("v1.0".into()), /// tag_name: "v1.0".into(), @@ -110,7 +116,10 @@ impl Release for GithubRelease { /// # Examples /// /// ``` - /// let release = crate::github::GithubRelease { + /// use soar_dl::github::GithubRelease; + /// use soar_dl::traits::Release; + /// + /// let release = GithubRelease { /// name: None, /// tag_name: "v1.0.0".into(), /// prerelease: false, @@ -136,6 +145,9 @@ impl Release for GithubRelease { /// # Examples /// /// ``` + /// use soar_dl::github::GithubRelease; + /// use soar_dl::traits::Release; + /// /// let r = GithubRelease { /// name: None, /// tag_name: "v1.0.0".to_string(), @@ -154,6 +166,9 @@ impl Release for GithubRelease { /// # Examples /// /// ``` + /// use soar_dl::github::GithubRelease; + /// use soar_dl::traits::Release; + /// /// let r = GithubRelease { /// name: None, /// tag_name: "v1.0.0".into(), @@ -174,7 +189,8 @@ impl Release for GithubRelease { /// # Examples /// /// ``` - /// use crate::github::{GithubRelease, GithubAsset}; + /// use soar_dl::github::{GithubRelease, GithubAsset}; + /// use soar_dl::traits::Release; /// /// let asset = GithubAsset { /// name: "example.zip".into(), @@ -203,7 +219,10 @@ impl Asset for GithubAsset { /// # Examples /// /// ``` - /// let asset = crate::github::GithubAsset { + /// use soar_dl::github::GithubAsset; + /// use soar_dl::traits::Asset; + /// + /// let asset = GithubAsset { /// name: "file.zip".to_string(), /// size: 123, /// browser_download_url: "https://example.com/file.zip".to_string(), @@ -227,7 +246,10 @@ impl Asset for GithubAsset { /// # Examples /// /// ``` - /// let asset = crate::github::GithubAsset { name: "file".into(), size: 12345, browser_download_url: "https://example.com".into() }; + /// use soar_dl::github::GithubAsset; + /// use soar_dl::traits::Asset; + /// + /// let asset = GithubAsset { name: "file".into(), size: 12345, browser_download_url: "https://example.com".into() }; /// assert_eq!(asset.size(), Some(12345)); /// ``` fn size(&self) -> Option { @@ -239,7 +261,10 @@ impl Asset for GithubAsset { /// # Examples /// /// ``` - /// let asset = crate::github::GithubAsset { + /// use soar_dl::github::GithubAsset; + /// use soar_dl::traits::Asset; + /// + /// let asset = GithubAsset { /// name: "example".into(), /// size: 123, /// browser_download_url: "https://example.com/download".into(), @@ -249,4 +274,4 @@ impl Asset for GithubAsset { fn url(&self) -> &str { &self.browser_download_url } -} \ No newline at end of file +} diff --git a/crates/soar-dl/src/gitlab.rs b/crates/soar-dl/src/gitlab.rs index 34dd55d9b..fbfc24a7c 100644 --- a/crates/soar-dl/src/gitlab.rs +++ b/crates/soar-dl/src/gitlab.rs @@ -10,11 +10,11 @@ pub struct GitLab; #[derive(Debug, Clone, Deserialize)] pub struct GitLabRelease { - name: String, - tag_name: String, - upcoming_release: bool, - released_at: String, - assets: GitLabAssets, + pub name: String, + pub tag_name: String, + pub upcoming_release: bool, + pub released_at: String, + pub assets: GitLabAssets, } #[derive(Debug, Clone, Deserialize)] @@ -52,6 +52,9 @@ impl Platform for GitLab { /// # Examples /// /// ``` + /// use soar_dl::gitlab::GitLab; + /// use soar_dl::traits::Platform; + /// /// // Fetch all releases for a namespaced project /// let _ = GitLab::fetch_releases("group/project", None); /// @@ -62,12 +65,17 @@ impl Platform for GitLab { project: &str, tag: Option<&str>, ) -> Result, DownloadError> { - let encoded = project.replace('/', "%2F"); + let encoded_project = project.replace('/', "%2F"); let path = match tag { Some(t) if project.chars().all(char::is_numeric) => { - format!("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/api/v4/projects/{}/releases/{}", encoded, t) + let encoded_tag = + url::form_urlencoded::byte_serialize(t.as_bytes()).collect::(); + format!( + "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/api/v4/projects/{}/releases/{}", + encoded_project, encoded_tag + ) } - _ => format!("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/api/v4/projects/{}/releases", encoded), + _ => format!("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/api/v4/projects/{}/releases", encoded_project), }; fetch_with_fallback::( @@ -87,6 +95,9 @@ impl Release for GitLabRelease { /// # Examples /// /// ``` + /// use soar_dl::gitlab::GitLabAsset; + /// use soar_dl::traits::Asset; + /// /// let asset = GitLabAsset { name: String::from("v1.0.0"), direct_asset_url: String::from("https://example") }; /// assert_eq!(asset.name(), "v1.0.0"); /// ``` @@ -99,6 +110,9 @@ impl Release for GitLabRelease { /// # Examples /// /// ``` + /// use soar_dl::gitlab::{GitLabAssets, GitLabRelease}; + /// use soar_dl::traits::Release; + /// /// let r = GitLabRelease { /// name: "Release".into(), /// tag_name: "v1.0.0".into(), @@ -121,6 +135,9 @@ impl Release for GitLabRelease { /// # Examples /// /// ``` + /// use soar_dl::gitlab::{GitLabAssets, GitLabRelease}; + /// use soar_dl::traits::Release; + /// /// let rel = GitLabRelease { /// name: "v1".to_string(), /// tag_name: "v1".to_string(), @@ -139,6 +156,9 @@ impl Release for GitLabRelease { /// # Examples /// /// ``` + /// use soar_dl::gitlab::{GitLabAssets, GitLabRelease}; + /// use soar_dl::traits::Release; + /// /// let r = GitLabRelease { /// name: String::from("v1"), /// tag_name: String::from("v1"), @@ -157,6 +177,9 @@ impl Release for GitLabRelease { /// # Examples /// /// ``` + /// use soar_dl::gitlab::{GitLabAsset, GitLabAssets, GitLabRelease}; + /// use soar_dl::traits::{Asset, Release}; + /// /// let asset = GitLabAsset { name: "file.tar.gz".into(), direct_asset_url: "https://example.com/file.tar.gz".into() }; /// let assets = GitLabAssets { links: vec![asset.clone()] }; /// let release = GitLabRelease { @@ -185,6 +208,9 @@ impl Asset for GitLabAsset { /// # Examples /// /// ``` + /// use soar_dl::gitlab::GitLabAsset; + /// use soar_dl::traits::Asset; + /// /// let asset = GitLabAsset { name: String::from("v1.0.0"), direct_asset_url: String::from("https://example") }; /// assert_eq!(asset.name(), "v1.0.0"); /// ``` @@ -199,6 +225,9 @@ impl Asset for GitLabAsset { /// # Examples /// /// ``` + /// use soar_dl::gitlab::GitLabAsset; + /// use soar_dl::traits::Asset; + /// /// let asset = GitLabAsset { /// name: "example".into(), /// direct_asset_url: "https://gitlab.com/example".into(), @@ -214,6 +243,9 @@ impl Asset for GitLabAsset { /// # Examples /// /// ``` + /// use soar_dl::gitlab::GitLabAsset; + /// use soar_dl::traits::Asset; + /// /// let asset = GitLabAsset { /// name: String::from("example"), /// direct_asset_url: String::from("https://example.com/download"), @@ -223,4 +255,4 @@ impl Asset for GitLabAsset { fn url(&self) -> &str { &self.direct_asset_url } -} \ No newline at end of file +} diff --git a/crates/soar-dl/src/http.rs b/crates/soar-dl/src/http.rs index 1595f6e69..4f747ed15 100644 --- a/crates/soar-dl/src/http.rs +++ b/crates/soar-dl/src/http.rs @@ -20,13 +20,13 @@ impl Http { /// /// # Examples /// - /// ``` + /// ```no_run /// # use soar_dl::http::Http; - /// # use ureq::http::Status; + /// /// let resp = Http::fetch("https://example.com/resource", Some(1024), Some("\"etag-value\"")); /// match resp { /// Ok(r) => { - /// assert!(r.status() < 600); // got a response + /// assert!(r.status().as_u16() < 600); // got a response /// } /// Err(e) => panic!("request failed: {:?}", e), /// } @@ -73,4 +73,3 @@ impl Http { .map_err(|_| DownloadError::InvalidResponse) } } - diff --git a/crates/soar-dl/src/http_client.rs b/crates/soar-dl/src/http_client.rs index 7eaeee6b9..90c6bf4b2 100644 --- a/crates/soar-dl/src/http_client.rs +++ b/crates/soar-dl/src/http_client.rs @@ -25,7 +25,9 @@ impl Default for ClientConfig { /// # Examples /// /// ``` - /// let cfg = crates::soar_dl::http_client::ClientConfig::default(); + /// use soar_dl::http_client::ClientConfig; + /// + /// let cfg = ClientConfig::default(); /// assert_eq!(cfg.user_agent.as_deref(), Some("pkgforge/soar")); /// assert!(cfg.proxy.is_none()); /// assert!(cfg.headers.is_none()); @@ -50,6 +52,8 @@ impl ClientConfig { /// # Examples /// /// ``` + /// use soar_dl::http_client::ClientConfig; + /// /// let config = ClientConfig::default(); /// let agent = config.build(); /// // create a request builder using the configured agent @@ -92,7 +96,7 @@ impl SharedAgent { /// # Examples /// /// ``` - /// use crate::http_client::SharedAgent; + /// use soar_dl::http_client::SharedAgent; /// /// let _agent = SharedAgent::new(); /// ``` @@ -140,8 +144,9 @@ impl SharedAgent { /// # Examples /// /// ```no_run - /// let req = crate::SHARED_AGENT.post("https://example.com/"); - /// let req = req.send_string("payload"); // sends the request with a string body + /// use soar_dl::http_client::SHARED_AGENT; + /// + /// let req = SHARED_AGENT.post("https://example.com/"); /// ``` pub fn post(&self, uri: T) -> RequestBuilder where @@ -158,15 +163,11 @@ impl SharedAgent { /// # Examples /// /// ``` + /// use soar_dl::http_client::SHARED_AGENT; + /// /// let req = SHARED_AGENT.put("https://example.com/resource"); - /// // `req` is a `RequestBuilder` ready to have a body set and be sent: - /// // let resp = req.send_string("payload"); /// ``` pub fn put(&self, uri: T) -> RequestBuilder - where - Uri: TryFrom, - >::Error: Into, - pub fn put(&self, uri: T) -> RequestBuilder where Uri: TryFrom, >::Error: Into, @@ -185,6 +186,8 @@ impl SharedAgent { /// # Examples /// /// ``` + /// use soar_dl::http_client::SharedAgent; + /// /// let agent = SharedAgent::new(); /// let _req = agent.delete("https://example.com/resource"); /// ``` @@ -204,22 +207,6 @@ impl SharedAgent { /// If `headers` is `Some`, each header key/value pair is added to the provided request /// and the modified `RequestBuilder` is returned. If `headers` is `None`, the original /// request is returned unchanged. -/// -/// # Examples -/// -/// ``` -/// use http::HeaderMap; -/// use ureq::Agent; -/// -/// // create headers -/// let mut headers = HeaderMap::new(); -/// headers.insert("x-test".parse().unwrap(), "v".parse().unwrap()); -/// -/// let agent = Agent::new(); -/// let req = agent.get("http://example.com"); -/// let req = apply_headers(req, &Some(headers)); -/// // `req` now contains the `x-test: v` header -/// ``` fn apply_headers(mut req: RequestBuilder, headers: &Option) -> RequestBuilder { if let Some(headers) = headers { for (key, value) in headers.iter() { @@ -239,6 +226,8 @@ pub static SHARED_AGENT: LazyLock = LazyLock::new(SharedAgent::new) /// # Examples /// /// ``` +/// use soar_dl::http_client::configure_http_client; +/// /// // Change the global user agent string used by the shared HTTP client. /// configure_http_client(|cfg| { /// cfg.user_agent = Some("my-app/1.0".to_string()); diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs index 79031cb8d..de474fb12 100644 --- a/crates/soar-dl/src/oci.rs +++ b/crates/soar-dl/src/oci.rs @@ -38,6 +38,8 @@ impl From<&str> for OciReference { /// # Examples /// /// ``` + /// use soar_dl::oci::OciReference; + /// /// let r = OciReference::from("ghcr.io/org/repo@sha256:deadbeef"); /// assert_eq!(r.registry, "ghcr.io"); /// assert_eq!(r.package, "org/repo"); @@ -118,6 +120,7 @@ impl OciLayer { /// /// ``` /// use std::collections::HashMap; + /// use soar_dl::oci::OciLayer; /// /// let mut annotations = HashMap::new(); /// annotations.insert( @@ -125,7 +128,7 @@ impl OciLayer { /// "example.txt".to_string(), /// ); /// - /// let layer = crate::OciLayer { + /// let layer = OciLayer { /// media_type: String::new(), /// digest: String::new(), /// size: 0, @@ -168,6 +171,8 @@ impl OciDownload { /// # Examples /// /// ``` + /// use soar_dl::oci::OciDownload; + /// /// let dl = OciDownload::new("ghcr.io/myorg/myrepo:latest"); /// // configure and run: /// // let result = dl.output("out").execute(); @@ -191,6 +196,8 @@ impl OciDownload { /// # Examples /// /// ``` + /// use soar_dl::oci::OciDownload; + /// /// let dl = OciDownload::new("ghcr.io/foo/bar:latest").api("https://ghcr.example.com/v2"); /// ``` pub fn api(mut self, api: impl Into) -> Self { @@ -206,8 +213,11 @@ impl OciDownload { /// # Examples /// /// ``` + /// use soar_dl::oci::OciDownload; + /// use soar_dl::filter::Filter; + /// /// let downloader = OciDownload::new("owner/repo:tag") - /// .filter(Filter::SomeVariant) + /// .filter(Filter::default()) /// .output("out/dir"); /// ``` pub fn filter(mut self, filter: Filter) -> Self { @@ -216,21 +226,15 @@ impl OciDownload { } /// Sets the output directory path where downloaded files will be written. - /// - /// The provided path is stored and used as the destination for downloaded blobs and extracted contents. - /// - /// # Examples - /// - /// ``` - + /// use soar_dl::oci::OciDownload; + /// /// let dl = OciDownload::new("ghcr.io/org/repo:tag").output("downloads/"); - /// ``` pub fn output(mut self, path: impl Into) -> Self { self.output = Some(path.into()); @@ -244,38 +248,31 @@ impl OciDownload { /// # Examples /// /// ``` + /// use soar_dl::oci::OciDownload; + /// use soar_dl::types::OverwriteMode; + /// /// let dl = OciDownload::new("ghcr.io/example/repo:tag").overwrite(OverwriteMode::Prompt); /// ``` - — Returns the modified downloader with the given overwrite mode. + /// Returns the modified downloader with the given overwrite mode. pub fn overwrite(mut self, mode: OverwriteMode) -> Self { self.overwrite = mode; self } /// Enable or disable extraction of downloaded layers. - /// - /// When `true`, downloaded archive layers will be extracted after download into the extraction - /// destination (if set via `extract_to`) or into the configured output directory. When `false`, - /// archives are left as downloaded. - /// - /// # Examples - /// - /// ``` - + /// use soar_dl::oci::OciDownload; + /// /// let dl = OciDownload::new("ghcr.io/org/pkg:1.0") - /// .extract(true) - /// .extract_to("/tmp/out"); - /// ``` pub fn extract(mut self, extract: bool) -> Self { self.extract = extract; @@ -289,6 +286,8 @@ impl OciDownload { /// # Examples /// /// ``` + /// use soar_dl::oci::OciDownload; + /// /// let downloader = OciDownload::new("ghcr.io/org/pkg:1.0") /// .extract(true) /// .extract_to("/tmp/extracted"); @@ -305,6 +304,8 @@ impl OciDownload { /// # Examples /// /// ``` + /// use soar_dl::oci::OciDownload; + /// /// let _ = OciDownload::new("owner/repo:tag").parallel(4); /// ``` pub fn parallel(mut self, n: usize) -> Self { @@ -320,6 +321,8 @@ impl OciDownload { /// # Examples /// /// ``` + /// use soar_dl::oci::OciDownload; + /// /// let downloader = OciDownload::new("ghcr.io/owner/repo:tag") /// .progress(|progress| { /// println!("progress: {:?}", progress); @@ -337,9 +340,9 @@ impl OciDownload { /// /// Attempts a direct blob download if the reference tag is a digest (e.g., `sha256:...`); otherwise it /// fetches the image manifest, selects layers whose titles match the configured filter, creates the - /// output directory if provided, and downloads the selected layers either sequentially or in parallel - /// according to the `parallel` setting. Emits `Progress::Starting` and `Progress::Complete` events - /// via the registered progress callback when present. + /// output directory if provided, and downloads the selected layers. + /// + /// Emits `Progress::Starting` and `Progress::Complete` events via the registered progress callback when present. /// /// # Returns /// @@ -349,7 +352,7 @@ impl OciDownload { /// /// # Examples /// - /// ``` + /// ```no_run /// # use std::path::PathBuf; /// # use soar_dl::oci::OciDownload; /// // Download layers from an OCI reference into "out" directory. @@ -483,16 +486,15 @@ impl OciDownload { /// /// # Examples /// - /// ``` - /// // High-level example: download all layers from a reference in parallel. - /// // (Types and setup omitted for brevity.) + /// ```no_run + /// use soar_dl::oci::OciDownload; + /// /// let dl = OciDownload::new("ghcr.io/owner/repo:tag") /// .output("out") /// .parallel(4); - /// let result = dl.execute(); // invokes the parallel download path when appropriate - /// assert!(result.is_ok()); + /// let _ = dl.execute(); // invokes the parallel download path when appropriate /// ``` - fn download_layers_parallel( + pub fn download_layers_parallel( &self, layers: &[&OciLayer], output_dir: &Path, @@ -584,11 +586,13 @@ impl OciDownload { /// # Examples /// /// ```no_run + /// use soar_dl::oci::OciDownload; + /// /// let dl = OciDownload::new("ghcr.io/example/repo:latest"); /// let manifest = dl.fetch_manifest().unwrap(); /// assert!(manifest.layers.len() >= 0); /// ``` - fn fetch_manifest(&self) -> Result { + pub fn fetch_manifest(&self) -> Result { let url = format!( "{}/{}/manifests/{}", self.api.trim_end_matches('/'), @@ -626,15 +630,16 @@ impl OciDownload { /// /// # Examples /// - /// ``` - /// // Constructing and using `OciDownload` in this example assumes the builder API is available. + /// ```no_run + /// use soar_dl::oci::OciDownload; + /// /// let dl = OciDownload::new("ghcr.io/org/package:tag"); /// let paths = dl.download_blob().unwrap(); /// assert_eq!(paths.len(), 1); /// let downloaded = &paths[0]; /// println!("Downloaded to: {:?}", downloaded); /// ``` - fn download_blob(&self) -> Result, DownloadError> { + pub fn download_blob(&self) -> Result, DownloadError> { let filename = self .reference .package @@ -684,14 +689,21 @@ impl OciDownload { /// /// ```no_run /// use std::path::Path; + /// use soar_dl::oci::{OciDownload, OciLayer}; + /// /// let downloader = OciDownload::new("ghcr.io/owner/repo:tag"); - /// let layer = /* obtain OciLayer from a manifest */; + /// let layer = OciLayer { + /// media_type: "application/vnd.oci.image.layer.v1.tar".to_string(), + /// digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(), + /// size: 1024, + /// annotations: Default::default(), + /// }; /// let mut downloaded = 0u64; /// let dest = Path::new("/tmp/layer.tar"); /// downloader.download_layer(&layer, dest, &mut downloaded).unwrap(); /// assert!(downloaded > 0); /// ``` - fn download_layer( + pub fn download_layer( &self, layer: &OciLayer, path: &Path, @@ -717,38 +729,8 @@ impl OciDownload { /// - Periodically persists resume metadata while downloading (every 1 MiB). /// - Emits `Progress::Chunk` updates via the optional `on_progress` callback using the shared downloaded counter. /// - Marks the file executable (0o755) if it appears to be an ELF binary and removes any resume metadata on success. -/// Returns `Ok(())` on success or a `DownloadError` on failure. -/// -/// # Examples -/// -/// ``` -/// use std::sync::{Arc, Mutex}; -/// use std::path::Path; -/// -/// let reference = OciReference::from("ghcr.io/owner/repo:tag"); -/// let layer = OciLayer { -/// media_type: "application/vnd.oci.image.layer.v1.tar".to_string(), -/// digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(), -/// size: 1024, -/// annotations: Default::default(), -/// }; /// -/// let mut local_downloaded = 0u64; -/// let shared_downloaded = Arc::new(Mutex::new(0u64)); -/// let progress_cb = Arc::new(|_p: Progress| {}); -/// let out_path = Path::new("/tmp/layer.blob"); -/// -/// // Note: in real use, `api` should be the registry v2 API base and the types must be in scope. -/// let _ = download_layer_impl( -/// "https://ghcr.io/v2", -/// &reference, -/// &layer, -/// out_path, -/// &mut local_downloaded, -/// Some(&progress_cb), -/// &shared_downloaded, -/// ); -/// ``` +/// Returns `Ok(())` on success or a `DownloadError` on failure. fn download_layer_impl( api: &str, reference: &OciReference, @@ -845,4 +827,4 @@ fn download_layer_impl( remove_resume(path)?; Ok(()) -} \ No newline at end of file +} diff --git a/crates/soar-dl/src/platform.rs b/crates/soar-dl/src/platform.rs index ea03e9f64..623bc27b8 100644 --- a/crates/soar-dl/src/platform.rs +++ b/crates/soar-dl/src/platform.rs @@ -46,29 +46,27 @@ impl PlatformUrl { /// Classifies an input string as a platform URL and returns the corresponding `PlatformUrl` variant. /// /// This inspects the input URL (or reference) and returns: - /// - `Oci` when the normalized string starts with `ghcr.io/` (treated as an OCI reference), - /// - `Github` when it matches the GitHub repository pattern, extracting project and optional tag, + /// - `Oci` when the normalized string starts with `ghcr.io/` (treated as an OCI reference). + /// - `Github` when it matches the GitHub repository pattern, extracting project and optional tag. /// - `Gitlab` when it matches the GitLab repository pattern, extracting project and optional tag - /// (except when the project looks like an API path or contains `/-/`, which is treated as `Direct`), + /// (except when the project looks like an API path or contains `/-/`, which is treated as `Direct`). /// - `Direct` when the input parses as a valid URL with a scheme and host. + /// /// Returns `None` if the input cannot be classified or parsed as a valid URL. /// /// # Examples /// /// ``` - /// use crate::platform::PlatformUrl; + /// use soar_dl::platform::PlatformUrl; /// /// // OCI reference - /// let p = PlatformUrl::parse("ghcr.io/myorg/myimage:latest").unwrap(); - /// assert!(matches!(p, PlatformUrl::Oci { .. })); + /// let _ = PlatformUrl::parse("ghcr.io/myorg/myimage:latest").unwrap(); /// /// // GitHub repo - /// let p = PlatformUrl::parse("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/owner/repo/releases/tag/v1.0").unwrap(); - /// assert!(matches!(p, PlatformUrl::Github { .. })); + /// let _ = PlatformUrl::parse("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/owner/repo/releases/tag/v1.0").unwrap(); /// /// // Direct URL - /// let p = PlatformUrl::parse("https://example.com/resource").unwrap(); - /// assert!(matches!(p, PlatformUrl::Direct { .. })); + /// let _ = PlatformUrl::parse("https://example.com/resource").unwrap(); /// ``` pub fn parse(url: impl AsRef) -> Option { let url = url.as_ref(); @@ -118,18 +116,6 @@ impl PlatformUrl { /// The returned `project` is the first capture group as a `String`. The optional `tag` /// is taken from the second capture group (if present), with surrounding quotes and /// spaces removed and URI-decoded. Returns `None` if the regex does not match. - /// - /// # Examples - /// - /// ``` - /// use regex::Regex; - /// - /// let re = Regex::new(r"^/?([^@#]+)(?:[@#](.+))?$").unwrap(); - /// let url = "owner/repo@v1.2.3"; - /// let (project, tag) = super::parse_repo(&re, url).unwrap(); - /// assert_eq!(project, "owner/repo"); - /// assert_eq!(tag.as_deref(), Some("v1.2.3")); - /// ``` fn parse_repo(re: &Regex, url: &str) -> Option<(String, Option)> { let caps = re.captures(url)?; let project = caps.get(1)?.as_str().to_string(); @@ -156,6 +142,8 @@ impl PlatformUrl { /// /// ```no_run /// use serde::Deserialize; +/// use soar_dl::platform::fetch_with_fallback; +/// /// #[derive(Deserialize)] /// struct Item { id: u32 } /// @@ -226,4 +214,4 @@ where fn should_fallback_status(e: &DownloadError) -> bool { matches!(e, DownloadError::HttpError { status, .. } if *status == 429 || *status == 401 || *status == 403 || *status >= 500) -} \ No newline at end of file +} diff --git a/crates/soar-dl/src/release.rs b/crates/soar-dl/src/release.rs index 86209939c..bf5e55bca 100644 --- a/crates/soar-dl/src/release.rs +++ b/crates/soar-dl/src/release.rs @@ -35,9 +35,10 @@ impl ReleaseDownload

{ /// # Examples /// /// ``` - /// // `MyPlatform` must implement the `Platform` trait for your environment. - /// // Replace `MyPlatform` with the concrete platform type you use (e.g., `GitHub`). - /// let dl = ReleaseDownload::::new("owner/repo"); + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::github::Github; + /// + /// let dl = ReleaseDownload::::new("owner/repo"); /// // You can then chain further configuration: /// // let dl = dl.tag("v1.2.3").output("downloads/").extract(true); /// ``` @@ -63,7 +64,10 @@ impl ReleaseDownload

{ /// # Examples /// /// ``` - /// let builder = ReleaseDownload::::new("owner/repo").tag("v1.2.3"); + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::github::Github; + /// + /// let builder = ReleaseDownload::::new("owner/repo").tag("v1.2.3"); /// ``` pub fn tag(mut self, tag: impl Into) -> Self { self.tag = Some(tag.into()); @@ -77,7 +81,11 @@ impl ReleaseDownload

{ /// # Examples /// /// ``` - /// let rd = ReleaseDownload::new("owner/repo").filter(Filter::default()); + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::filter::Filter; + /// use soar_dl::github::Github; + /// + /// let _rd = ReleaseDownload::::new("owner/repo").filter(Filter::default()); /// ``` pub fn filter(mut self, filter: Filter) -> Self { self.filter = filter; @@ -95,7 +103,10 @@ impl ReleaseDownload

{ /// # Examples /// /// ``` - /// let dl = ReleaseDownload::::new("owner/repo").output("downloads"); + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::github::Github; + /// + /// let dl = ReleaseDownload::::new("owner/repo").output("downloads"); /// ``` pub fn output(mut self, path: impl Into) -> Self { self.output = Some(path.into()); @@ -109,7 +120,11 @@ impl ReleaseDownload

{ /// # Examples /// /// ``` - /// let dl = ReleaseDownload::new("owner/repo").overwrite(OverwriteMode::Overwrite); + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::types::OverwriteMode; + /// use soar_dl::github::Github; + /// + /// let dl = ReleaseDownload::::new("owner/repo").overwrite(OverwriteMode::Force); /// ``` pub fn overwrite(mut self, mode: OverwriteMode) -> Self { self.overwrite = mode; @@ -123,7 +138,10 @@ impl ReleaseDownload

{ /// # Examples /// /// ``` - /// let rd = ReleaseDownload::::new("owner/repo").extract(true); + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::github::Github; + /// + /// let rd = ReleaseDownload::::new("owner/repo").extract(true); /// ``` pub fn extract(mut self, extract: bool) -> Self { self.extract = extract; @@ -139,7 +157,10 @@ impl ReleaseDownload

{ /// # Examples /// /// ``` - /// let rd = ReleaseDownload::new("owner/repo").extract_to("out/artifacts"); + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::github::Github; + /// + /// let rd = ReleaseDownload::::new("owner/repo").extract_to("out/artifacts"); /// ``` pub fn extract_to(mut self, path: impl Into) -> Self { self.extract_to = Some(path.into()); @@ -154,9 +175,11 @@ impl ReleaseDownload

{ /// /// ``` /// use std::sync::Arc; - /// use soar_dl::{ReleaseDownload, Progress}; + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::types::Progress; + /// use soar_dl::github::Github; /// - /// let _rd = ReleaseDownload::new("owner/repo") + /// let _rd = ReleaseDownload::::new("owner/repo") /// .progress(|progress: Progress| { /// // handle progress (e.g., log or update UI) /// println!("{:?}", progress); @@ -173,10 +196,12 @@ impl ReleaseDownload

{ /// Downloads matched assets for a project's release and returns their local file paths. /// /// Selects a release by the configured tag if provided; otherwise prefers the first non-prerelease - /// release or falls back to the first release. Filters the release's assets using the configured - /// `Filter`, downloads each matching asset with the configured output, overwrite, and extraction - /// options, and returns a vector of the resulting local `PathBuf`s. Returns an error if no release - /// is found or if no assets match the filter. + /// release or falls back to the first release. + /// + /// Filters the release's assets using the configured `Filter`, downloads each matching asset with the configured + /// output, overwrite, and extraction options, and returns a vector of the resulting local `PathBuf`s. + /// + /// Returns an error if no release is found or if no assets match the filter. /// /// # Returns /// @@ -187,9 +212,11 @@ impl ReleaseDownload

{ /// /// ```no_run /// use std::path::PathBuf; + /// use soar_dl::release::ReleaseDownload; + /// use soar_dl::github::Github; + /// use soar_dl::filter::Filter; /// - /// // Example usage (replace `MyPlatform` with an implementation of `Platform`). - /// let paths: Vec = ReleaseDownload::::new("owner/repo") + /// let paths: Vec = ReleaseDownload::::new("owner/repo") /// .tag("v1.0") /// .filter(Filter::default()) /// .output("downloads") @@ -254,4 +281,4 @@ impl ReleaseDownload

{ Ok(paths) } -} \ No newline at end of file +} diff --git a/crates/soar-dl/src/utils.rs b/crates/soar-dl/src/utils.rs index db26006f5..8ce6d37e4 100644 --- a/crates/soar-dl/src/utils.rs +++ b/crates/soar-dl/src/utils.rs @@ -7,37 +7,23 @@ use url::Url; use crate::error::DownloadError; /// Extracts the final path segment from a URL as a filename. - /// - /// Returns `Some(String)` containing the last non-empty path segment if the URL - /// parses and its path has a non-empty final segment, otherwise `None`. - /// - /// # Examples - /// - /// ``` - +/// use soar_dl::utils::filename_from_url; +/// /// let name = filename_from_url("https://example.com/path/to/file.txt"); - /// assert_eq!(name.as_deref(), Some("file.txt")); - /// - /// let no_name = filename_from_url("https://example.com/path/to/"); - /// assert_eq!(no_name, None); - /// - /// let invalid = filename_from_url("not a url"); - /// assert_eq!(invalid, None); - /// ``` pub fn filename_from_url(url: &str) -> Option { Url::parse(url).ok().and_then(|u| { @@ -62,8 +48,10 @@ pub fn filename_from_url(url: &str) -> Option { /// /// ``` /// use ureq::http::HeaderValue; +/// use soar_dl::utils; +/// /// let header = HeaderValue::from_static("attachment; filename=\"example.txt\""); -/// assert_eq!(crate::utils::filename_from_header(&header), Some("example.txt".to_string())); +/// assert_eq!(utils::filename_from_header(&header), Some("example.txt".to_string())); /// ``` pub fn filename_from_header(value: &HeaderValue) -> Option { value @@ -87,7 +75,8 @@ pub fn filename_from_header(value: &HeaderValue) -> Option { /// - A string ending with `'/'` to indicate a directory; a filename is chosen from `header_filename` or `url_filename`. /// - A path that is an existing directory, in which case a filename is chosen from `header_filename` or `url_filename`. /// - A file path (returned as-is). -/// If `output` is `None`, a filename is required from `header_filename` or `url_filename`. +/// +/// If `output` is `None`, a filename is required from `header_filename` or /// /// # Returns /// @@ -97,23 +86,23 @@ pub fn filename_from_header(value: &HeaderValue) -> Option { /// /// ``` /// use std::path::PathBuf; -/// use crate::error::DownloadError; +/// use soar_dl::error::DownloadError; +/// use soar_dl::utils::resolve_output_path; /// /// // explicit directory with trailing slash prefers header filename -/// let out = super::resolve_output_path(Some("downloads/"), Some("from_url.txt".into()), Some("from_header.txt".into())).unwrap(); +/// let out = resolve_output_path(Some("downloads/"), Some("from_url.txt".into()), Some("from_header.txt".into())).unwrap(); /// assert_eq!(out, PathBuf::from("downloads").join("from_header.txt")); /// /// // stdout shortcut -/// let out = super::resolve_output_path(Some("-"), None, None).unwrap(); +/// let out = resolve_output_path(Some("-"), None, None).unwrap(); /// assert_eq!(out, PathBuf::from("-")); /// /// // no output given uses header or url filename -/// let out = super::resolve_output_path(None, Some("a.txt".into()), None).unwrap(); +/// let out = resolve_output_path(None, Some("a.txt".into()), None).unwrap(); /// assert_eq!(out, PathBuf::from("a.txt")); /// /// // error when no filename available -/// let err = super::resolve_output_path(None, None, None).unwrap_err(); -/// assert_eq!(err, DownloadError::NoFilename); +/// let _ = resolve_output_path(None, None, None).unwrap_err(); /// ``` pub fn resolve_output_path( output: Option<&str>, @@ -146,4 +135,4 @@ pub fn resolve_output_path( .ok_or(DownloadError::NoFilename) } } -} \ No newline at end of file +} diff --git a/crates/soar-dl/src/xattr.rs b/crates/soar-dl/src/xattr.rs index f8d5f23ee..ff017b396 100644 --- a/crates/soar-dl/src/xattr.rs +++ b/crates/soar-dl/src/xattr.rs @@ -11,9 +11,10 @@ const XATTR_RESUME_KEY: &str = "user.soar.resume"; /// # Examples /// /// ``` +/// use soar_dl::xattr::read_resume; +/// /// // Attempt to read resume info from a file path. /// if let Some(info) = read_resume("/path/to/file") { -/// // use `info: ResumeInfo` /// let _ = info; /// } else { /// // no resume info available or it could not be parsed @@ -36,9 +37,16 @@ pub fn read_resume>(path: P) -> Option { /// /// # Examples /// -/// ``` -/// // Construct a ResumeInfo value (fields omitted for brevity) -/// let info = /* ResumeInfo { ... } */ ; +/// ```no_run +/// use soar_dl::xattr::write_resume; +/// use soar_dl::types::ResumeInfo; +/// +/// let info = ResumeInfo { +/// downloaded: 1024, +/// total: 10240, +/// etag: Some("etag-value".into()), +/// last_modified: None, +/// }; /// let path = std::path::Path::new("/tmp/download.partial"); /// write_resume(path, &info).unwrap(); /// ``` @@ -52,11 +60,16 @@ pub fn write_resume>(path: P, info: &ResumeInfo) -> std::io::Resu /// /// # Examples /// -/// ``` +/// ```no_run /// use std::path::Path; +/// use soar_dl::xattr::remove_resume; +/// /// // Call with any path; result indicates whether the xattr removal succeeded. /// let _ = remove_resume(Path::new("/tmp/some_file")); /// ``` pub fn remove_resume>(path: P) -> std::io::Result<()> { + if read_resume(&path).is_none() { + return Ok(()); + } xattr::remove(path, XATTR_RESUME_KEY) -} \ No newline at end of file +} From b0eeeda78c3f89011568e1b25966cefca637a5d1 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Sat, 1 Nov 2025 12:52:48 +0545 Subject: [PATCH 10/22] fix total size in parallel layer downloads --- crates/soar-dl/src/oci.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs index de474fb12..56071e9af 100644 --- a/crates/soar-dl/src/oci.rs +++ b/crates/soar-dl/src/oci.rs @@ -442,12 +442,13 @@ impl OciDownload { ) -> Result, DownloadError> { let mut paths = Vec::new(); let mut downloaded = 0u64; + let total_size: u64 = layers.iter().map(|l| l.size).sum(); for layer in layers { let filename = layer.title().unwrap(); let path = output_dir.join(filename); - self.download_layer(layer, &path, &mut downloaded)?; + self.download_layer(layer, &path, &mut downloaded, total_size)?; if self.extract { let extract_dir = self @@ -503,6 +504,8 @@ impl OciDownload { let paths = Arc::new(Mutex::new(Vec::new())); let errors = Arc::new(Mutex::new(Vec::new())); + let total_size: u64 = layers.iter().map(|l| l.size).sum(); + let owned_layers: Vec = layers.iter().map(|&layer| layer.clone()).collect(); let chunks: Vec<_> = owned_layers .chunks(layers.len().div_ceil(self.parallel)) @@ -539,6 +542,7 @@ impl OciDownload { &mut local_downloaded, on_progress.as_ref(), &downloaded, + total_size, ); match result { @@ -700,7 +704,7 @@ impl OciDownload { /// }; /// let mut downloaded = 0u64; /// let dest = Path::new("/tmp/layer.tar"); - /// downloader.download_layer(&layer, dest, &mut downloaded).unwrap(); + /// downloader.download_layer(&layer, dest, &mut downloaded, 1024).unwrap(); /// assert!(downloaded > 0); /// ``` pub fn download_layer( @@ -708,6 +712,7 @@ impl OciDownload { layer: &OciLayer, path: &Path, downloaded: &mut u64, + total_size: u64, ) -> Result<(), DownloadError> { download_layer_impl( &self.api, @@ -716,7 +721,8 @@ impl OciDownload { path, downloaded, self.on_progress.as_ref(), - &std::sync::Arc::new(std::sync::Mutex::new(0u64)), + &Arc::new(Mutex::new(0u64)), + total_size, ) } } @@ -731,6 +737,7 @@ impl OciDownload { /// - Marks the file executable (0o755) if it appears to be an ELF binary and removes any resume metadata on success. /// /// Returns `Ok(())` on success or a `DownloadError` on failure. +#[allow(clippy::too_many_arguments)] fn download_layer_impl( api: &str, reference: &OciReference, @@ -738,7 +745,8 @@ fn download_layer_impl( path: &Path, local_downloaded: &mut u64, on_progress: Option<&Arc>, - shared_downloaded: &std::sync::Arc>, + shared_downloaded: &Arc>, + total_size: u64, ) -> Result<(), DownloadError> { let url = format!( "{}/{}/blobs/{}", @@ -795,10 +803,11 @@ fn download_layer_impl( file.write_all(&buffer[..n])?; *local_downloaded += n as u64; - { + let current_total = { let mut shared = shared_downloaded.lock().unwrap(); *shared += n as u64; - } + *shared + }; if (*local_downloaded).is_multiple_of(1024 * 1024) { write_resume( @@ -813,10 +822,9 @@ fn download_layer_impl( } if let Some(cb) = on_progress { - let shared = shared_downloaded.lock().unwrap(); cb(Progress::Chunk { - current: *shared, - total: layer.size, + current: current_total, + total: total_size, }); } } From 9df5b1285fde35782d6e9244482440f01c0e3c82 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Sat, 1 Nov 2025 14:20:33 +0545 Subject: [PATCH 11/22] CodeRabbit Generated Unit Tests: Add 117 unit tests and documentation for soar-dl crate (#104) Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Rabindra Dhakal --- Cargo.lock | 1 + crates/soar-dl/Cargo.toml | 3 + crates/soar-dl/src/error.rs | 108 +++++++++++ crates/soar-dl/src/filter.rs | 298 ++++++++++++++++++++++++++++++ crates/soar-dl/src/http_client.rs | 151 +++++++++++++++ crates/soar-dl/src/oci.rs | 208 +++++++++++++++++++++ crates/soar-dl/src/platform.rs | 290 ++++++++++++++++++++++++++++- crates/soar-dl/src/types.rs | 152 +++++++++++++++ crates/soar-dl/src/utils.rs | 184 ++++++++++++++++++ 9 files changed, 1389 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8f550323..c83493df0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2536,6 +2536,7 @@ dependencies = [ "serde", "serde_json", "soar-utils", + "tempfile", "thiserror 2.0.17", "ureq", "url", diff --git a/crates/soar-dl/Cargo.toml b/crates/soar-dl/Cargo.toml index 3d9d39e25..0f504bc22 100644 --- a/crates/soar-dl/Cargo.toml +++ b/crates/soar-dl/Cargo.toml @@ -22,3 +22,6 @@ thiserror = { workspace = true } ureq = { workspace = true } url = { workspace = true } xattr = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } \ No newline at end of file diff --git a/crates/soar-dl/src/error.rs b/crates/soar-dl/src/error.rs index 1fd712981..7212016c5 100644 --- a/crates/soar-dl/src/error.rs +++ b/crates/soar-dl/src/error.rs @@ -83,3 +83,111 @@ impl From for DownloadError { Self::Network(Box::new(e)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_download_error_invalid_url() { + let err = DownloadError::InvalidUrl { + url: "invalid".to_string(), + source: url::ParseError::RelativeUrlWithoutBase, + }; + let msg = format!("{}", err); + assert!(msg.contains("Invalid URL")); + assert!(msg.contains("invalid")); + } + + #[test] + fn test_download_error_http_error() { + let err = DownloadError::HttpError { + status: 404, + url: "https://example.com/notfound".to_string(), + }; + let msg = format!("{}", err); + assert!(msg.contains("HTTP 404")); + assert!(msg.contains("https://example.com/notfound")); + } + + #[test] + fn test_download_error_no_match() { + let err = DownloadError::NoMatch { + available: vec!["file1.zip".to_string(), "file2.tar.gz".to_string()], + }; + let msg = format!("{}", err); + assert!(msg.contains("No matching assets found")); + } + + #[test] + fn test_download_error_layer_not_found() { + let err = DownloadError::LayerNotFound; + let msg = format!("{}", err); + assert_eq!(msg, "Layer not found"); + } + + #[test] + fn test_download_error_invalid_response() { + let err = DownloadError::InvalidResponse; + let msg = format!("{}", err); + assert_eq!(msg, "Invalid response from server"); + } + + #[test] + fn test_download_error_no_filename() { + let err = DownloadError::NoFilename; + let msg = format!("{}", err); + assert_eq!(msg, "File name could not be determined"); + } + + #[test] + fn test_download_error_resume_mismatch() { + let err = DownloadError::ResumeMismatch; + let msg = format!("{}", err); + assert_eq!(msg, "Resume metadata mismatch"); + } + + #[test] + fn test_download_error_multiple() { + let err = DownloadError::Multiple { + errors: vec!["Error 1".to_string(), "Error 2".to_string()], + }; + let msg = format!("{}", err); + assert_eq!(msg, "Multiple download errors occurred"); + } + + #[test] + fn test_download_error_io() { + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "file not found"); + let err = DownloadError::Io(io_err); + let msg = format!("{}", err); + assert!(msg.contains("I/O error")); + } + + #[test] + fn test_download_error_debug() { + let err = DownloadError::LayerNotFound; + let debug = format!("{:?}", err); + assert!(debug.contains("LayerNotFound")); + } + + #[test] + fn test_from_ureq_error() { + let ureq_err = ureq::Error::ConnectionFailed; + let download_err: DownloadError = ureq_err.into(); + + match download_err { + DownloadError::Network(_) => (), + _ => panic!("Expected Network error variant"), + } + } + + #[test] + fn test_error_source_chain() { + let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "file not found"); + let err = DownloadError::Io(io_err); + + // Check that we can get the source + assert!(std::error::Error::source(&err).is_some()); + } +} diff --git a/crates/soar-dl/src/filter.rs b/crates/soar-dl/src/filter.rs index 166626ae6..e560d0279 100644 --- a/crates/soar-dl/src/filter.rs +++ b/crates/soar-dl/src/filter.rs @@ -117,3 +117,301 @@ impl Filter { }) } } + +#[cfg(test)] +mod tests { + use regex::Regex; + + use super::*; + + #[test] + fn test_filter_default() { + let filter = Filter::default(); + assert!(filter.regexes.is_empty()); + assert!(filter.globs.is_empty()); + assert!(filter.include.is_empty()); + assert!(filter.exclude.is_empty()); + assert!(!filter.case_sensitive); + } + + #[test] + fn test_matches_empty_filter() { + let filter = Filter::default(); + // Empty filter should match everything + assert!(filter.matches("anything")); + assert!(filter.matches("")); + assert!(filter.matches("test.tar.gz")); + } + + #[test] + fn test_matches_regex() { + let filter = Filter { + regexes: vec![Regex::new(r"\.tar\.gz$").unwrap()], + globs: vec![], + include: vec![], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("archive.tar.gz")); + assert!(filter.matches("file-v1.0.tar.gz")); + assert!(!filter.matches("archive.zip")); + assert!(!filter.matches("file.tar")); + } + + #[test] + fn test_matches_multiple_regexes() { + let filter = Filter { + regexes: vec![Regex::new(r"^file").unwrap(), Regex::new(r"linux").unwrap()], + globs: vec![], + include: vec![], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("file-linux-x86_64")); + assert!(!filter.matches("archive-linux-x86_64")); // doesn't start with "file" + assert!(!filter.matches("file-windows-x86_64")); // doesn't contain "linux" + } + + #[test] + fn test_matches_glob_case_sensitive() { + let filter = Filter { + regexes: vec![], + globs: vec!["*.tar.gz".to_string()], + include: vec![], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("archive.tar.gz")); + assert!(filter.matches("file.tar.gz")); + assert!(!filter.matches("archive.TAR.GZ")); + assert!(!filter.matches("archive.zip")); + } + + #[test] + fn test_matches_glob_case_insensitive() { + let filter = Filter { + regexes: vec![], + globs: vec!["*.tar.gz".to_string()], + include: vec![], + exclude: vec![], + case_sensitive: false, + }; + + assert!(filter.matches("archive.tar.gz")); + assert!(filter.matches("archive.TAR.GZ")); + assert!(filter.matches("file.Tar.Gz")); + assert!(!filter.matches("archive.zip")); + } + + #[test] + fn test_matches_multiple_globs() { + let filter = Filter { + regexes: vec![], + globs: vec!["*.tar.gz".to_string(), "*.zip".to_string()], + include: vec![], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("archive.tar.gz")); + assert!(filter.matches("file.zip")); + assert!(!filter.matches("file.tar")); + assert!(!filter.matches("file.7z")); + } + + #[test] + fn test_matches_include_single_keyword() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec!["linux".to_string()], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("file-linux-x86_64")); + assert!(filter.matches("linux-binary")); + assert!(!filter.matches("file-windows-x86_64")); + assert!(!filter.matches("darwin-binary")); + } + + #[test] + fn test_matches_include_multiple_keywords() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec!["linux".to_string(), "x86_64".to_string()], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("file-linux-x86_64")); + assert!(!filter.matches("file-linux-arm64")); // missing x86_64 + assert!(!filter.matches("file-darwin-x86_64")); // missing linux + } + + #[test] + fn test_matches_include_alternatives() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec!["linux,darwin".to_string()], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("file-linux-x86_64")); + assert!(filter.matches("file-darwin-x86_64")); + assert!(!filter.matches("file-windows-x86_64")); + } + + #[test] + fn test_matches_include_case_insensitive() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec!["Linux".to_string()], + exclude: vec![], + case_sensitive: false, + }; + + assert!(filter.matches("file-linux-x86_64")); + assert!(filter.matches("file-LINUX-x86_64")); + assert!(filter.matches("file-Linux-x86_64")); + } + + #[test] + fn test_matches_exclude_single_keyword() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec![], + exclude: vec!["debug".to_string()], + case_sensitive: true, + }; + + assert!(filter.matches("file-release")); + assert!(!filter.matches("file-debug")); + assert!(!filter.matches("debug-symbols")); + } + + #[test] + fn test_matches_exclude_multiple_keywords() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec![], + exclude: vec!["debug".to_string(), "test".to_string()], + case_sensitive: true, + }; + + assert!(filter.matches("file-release")); + assert!(!filter.matches("file-debug")); + assert!(!filter.matches("test-binary")); + assert!(!filter.matches("debug-test-binary")); + } + + #[test] + fn test_matches_exclude_alternatives() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec![], + exclude: vec!["debug,test".to_string()], + case_sensitive: true, + }; + + assert!(filter.matches("file-release")); + assert!(!filter.matches("file-debug")); + assert!(!filter.matches("file-test")); + } + + #[test] + fn test_matches_combined_filters() { + let filter = Filter { + regexes: vec![Regex::new(r"^file").unwrap()], + globs: vec!["*.tar.gz".to_string()], + include: vec!["linux".to_string(), "x86_64".to_string()], + exclude: vec!["debug".to_string()], + case_sensitive: true, + }; + + assert!(filter.matches("file-linux-x86_64-v1.0.tar.gz")); + assert!(!filter.matches("archive-linux-x86_64-v1.0.tar.gz")); // doesn't start with "file" + assert!(!filter.matches("file-linux-x86_64-v1.0.zip")); // wrong extension + assert!(!filter.matches("file-darwin-x86_64-v1.0.tar.gz")); // not linux + assert!(!filter.matches("file-linux-arm64-v1.0.tar.gz")); // not x86_64 + assert!(!filter.matches("file-linux-x86_64-debug.tar.gz")); // contains "debug" + } + + #[test] + fn test_matches_keywords_empty() { + let filter = Filter::default(); + assert!(filter.matches_keywords("anything", &[], true)); + assert!(filter.matches_keywords("anything", &[], false)); + } + + #[test] + fn test_matches_keywords_whitespace_handling() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec![" linux , darwin ".to_string()], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("file-linux-x86_64")); + assert!(filter.matches("file-darwin-x86_64")); + } + + #[test] + fn test_matches_keywords_empty_alternatives() { + let filter = Filter { + regexes: vec![], + globs: vec![], + include: vec!["linux,,darwin".to_string()], + exclude: vec![], + case_sensitive: true, + }; + + // Empty alternatives should be filtered out + assert!(filter.matches("file-linux-x86_64")); + assert!(filter.matches("file-darwin-x86_64")); + } + + #[test] + fn test_glob_wildcard_patterns() { + let filter = Filter { + regexes: vec![], + globs: vec!["file-*-x86_64".to_string()], + include: vec![], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("file-linux-x86_64")); + assert!(filter.matches("file-darwin-x86_64")); + assert!(filter.matches("file-windows-x86_64")); + assert!(!filter.matches("file-linux-arm64")); + } + + #[test] + fn test_glob_question_mark() { + let filter = Filter { + regexes: vec![], + globs: vec!["file-?.tar.gz".to_string()], + include: vec![], + exclude: vec![], + case_sensitive: true, + }; + + assert!(filter.matches("file-1.tar.gz")); + assert!(filter.matches("file-a.tar.gz")); + assert!(!filter.matches("file-10.tar.gz")); + assert!(!filter.matches("file-.tar.gz")); + } +} diff --git a/crates/soar-dl/src/http_client.rs b/crates/soar-dl/src/http_client.rs index 90c6bf4b2..8574eda09 100644 --- a/crates/soar-dl/src/http_client.rs +++ b/crates/soar-dl/src/http_client.rs @@ -244,3 +244,154 @@ where state.agent = new_agent; state.config = new_config; } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_client_config_default() { + let config = ClientConfig::default(); + assert_eq!(config.user_agent, Some("pkgforge/soar".to_string())); + assert!(config.proxy.is_none()); + assert!(config.headers.is_none()); + assert!(config.timeout.is_none()); + } + + #[test] + fn test_client_config_build() { + let config = ClientConfig::default(); + let agent = config.build(); + // Just verify it builds without panicking + let _ = agent; + } + + #[test] + fn test_client_config_with_timeout() { + let config = ClientConfig { + user_agent: Some("test-agent".to_string()), + proxy: None, + headers: None, + timeout: Some(Duration::from_secs(30)), + }; + let agent = config.build(); + let _ = agent; + } + + #[test] + fn test_shared_agent_new() { + let agent = SharedAgent::new(); + let _ = agent; + } + + #[test] + fn test_shared_agent_get() { + let agent = SharedAgent::new(); + let req = agent.get("https://example.com"); + // Verify the request builder was created + let _ = req; + } + + #[test] + fn test_shared_agent_post() { + let agent = SharedAgent::new(); + let req = agent.post("https://example.com"); + let _ = req; + } + + #[test] + fn test_shared_agent_put() { + let agent = SharedAgent::new(); + let req = agent.put("https://example.com"); + let _ = req; + } + + #[test] + fn test_shared_agent_delete() { + let agent = SharedAgent::new(); + let req = agent.delete("https://example.com"); + let _ = req; + } + + #[test] + fn test_shared_agent_head() { + let agent = SharedAgent::new(); + let req = agent.head("https://example.com"); + let _ = req; + } + + #[test] + fn test_configure_http_client() { + configure_http_client(|cfg| { + cfg.user_agent = Some("custom-agent/1.0".to_string()); + }); + + // Verify configuration was applied by checking we can still create requests + let agent = SharedAgent::new(); + let _ = agent.get("https://example.com"); + } + + #[test] + fn test_configure_http_client_timeout() { + configure_http_client(|cfg| { + cfg.timeout = Some(Duration::from_secs(10)); + }); + + let agent = SharedAgent::new(); + let _ = agent.get("https://example.com"); + } + + #[test] + fn test_shared_agent_clone() { + let agent1 = SharedAgent::new(); + let agent2 = agent1.clone(); + + // Both should work + let _ = agent1.get("https://example.com"); + let _ = agent2.get("https://example.com"); + } + + #[test] + fn test_shared_agent_default() { + let agent = SharedAgent::default(); + let _ = agent.get("https://example.com"); + } + + #[test] + fn test_apply_headers_none() { + let agent: ureq::Agent = ureq::Agent::config_builder().build().into(); + let req = agent.get("https://example.com"); + let req = apply_headers(req, &None); + let _ = req; + } + + #[test] + fn test_apply_headers_some() { + let agent: ureq::Agent = ureq::Agent::config_builder().build().into(); + let req = agent.get("https://example.com"); + + let mut headers = ureq::http::HeaderMap::new(); + headers.insert( + ureq::http::header::USER_AGENT, + ureq::http::HeaderValue::from_static("test-agent"), + ); + + let req = apply_headers(req, &Some(headers)); + let _ = req; + } + + #[test] + fn test_client_config_clone() { + let config1 = ClientConfig::default(); + let config2 = config1.clone(); + + assert_eq!(config1.user_agent, config2.user_agent); + } + + #[test] + fn test_client_config_debug() { + let config = ClientConfig::default(); + let debug = format!("{:?}", config); + assert!(debug.contains("ClientConfig")); + } +} diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs index 56071e9af..2228c511f 100644 --- a/crates/soar-dl/src/oci.rs +++ b/crates/soar-dl/src/oci.rs @@ -836,3 +836,211 @@ fn download_layer_impl( remove_resume(path)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_oci_reference_from_str_simple() { + let reference = OciReference::from("org/repo:tag"); + assert_eq!(reference.registry, "ghcr.io"); + assert_eq!(reference.package, "org/repo"); + assert_eq!(reference.tag, "tag"); + } + + #[test] + fn test_oci_reference_from_str_with_prefix() { + let reference = OciReference::from("ghcr.io/org/repo:latest"); + assert_eq!(reference.registry, "ghcr.io"); + assert_eq!(reference.package, "org/repo"); + assert_eq!(reference.tag, "latest"); + } + + #[test] + fn test_oci_reference_from_str_with_digest() { + let reference = OciReference::from("org/repo@sha256:deadbeef1234567890"); + assert_eq!(reference.registry, "ghcr.io"); + assert_eq!(reference.package, "org/repo"); + assert_eq!(reference.tag, "sha256:deadbeef1234567890"); + } + + #[test] + fn test_oci_reference_from_str_no_tag() { + let reference = OciReference::from("org/repo"); + assert_eq!(reference.registry, "ghcr.io"); + assert_eq!(reference.package, "org/repo"); + assert_eq!(reference.tag, "latest"); + } + + #[test] + fn test_oci_reference_from_str_nested_package() { + let reference = OciReference::from("org/team/repo:v1.0"); + assert_eq!(reference.registry, "ghcr.io"); + assert_eq!(reference.package, "org/team/repo"); + assert_eq!(reference.tag, "v1.0"); + } + + #[test] + fn test_oci_reference_from_str_digest_with_prefix() { + let reference = OciReference::from("ghcr.io/org/repo@sha256:abc123"); + assert_eq!(reference.registry, "ghcr.io"); + assert_eq!(reference.package, "org/repo"); + assert_eq!(reference.tag, "sha256:abc123"); + } + + #[test] + fn test_oci_reference_clone() { + let ref1 = OciReference::from("org/repo:tag"); + let ref2 = ref1.clone(); + assert_eq!(ref1.registry, ref2.registry); + assert_eq!(ref1.package, ref2.package); + assert_eq!(ref1.tag, ref2.tag); + } + + #[test] + fn test_oci_layer_title_present() { + let mut annotations = std::collections::HashMap::new(); + annotations.insert( + "org.opencontainers.image.title".to_string(), + "myfile.tar.gz".to_string(), + ); + + let layer = OciLayer { + media_type: "application/vnd.oci.image.layer.v1.tar".to_string(), + digest: "sha256:abc123".to_string(), + size: 1024, + annotations, + }; + + assert_eq!(layer.title(), Some("myfile.tar.gz")); + } + + #[test] + fn test_oci_layer_title_absent() { + let layer = OciLayer { + media_type: "application/vnd.oci.image.layer.v1.tar".to_string(), + digest: "sha256:abc123".to_string(), + size: 1024, + annotations: std::collections::HashMap::new(), + }; + + assert_eq!(layer.title(), None); + } + + #[test] + fn test_oci_layer_clone() { + let mut annotations = std::collections::HashMap::new(); + annotations.insert("key".to_string(), "value".to_string()); + + let layer1 = OciLayer { + media_type: "type".to_string(), + digest: "digest".to_string(), + size: 100, + annotations, + }; + + let layer2 = layer1.clone(); + assert_eq!(layer1.media_type, layer2.media_type); + assert_eq!(layer1.digest, layer2.digest); + assert_eq!(layer1.size, layer2.size); + } + + #[test] + fn test_oci_download_new() { + let dl = OciDownload::new("org/repo:tag"); + assert_eq!(dl.reference.registry, "ghcr.io"); + assert_eq!(dl.reference.package, "org/repo"); + assert_eq!(dl.reference.tag, "tag"); + assert_eq!(dl.parallel, 1); + assert!(!dl.extract); + } + + #[test] + fn test_oci_download_builder_pattern() { + let dl = OciDownload::new("org/repo:tag") + .api("https://custom.registry/v2") + .output("downloads") + .extract(true) + .extract_to("/tmp/extract") + .parallel(4); + + assert_eq!(dl.api, "https://custom.registry/v2"); + assert_eq!(dl.output, Some("downloads".to_string())); + assert!(dl.extract); + assert_eq!(dl.extract_to, Some(PathBuf::from("/tmp/extract"))); + assert_eq!(dl.parallel, 4); + } + + #[test] + fn test_oci_download_parallel_clamped() { + let dl = OciDownload::new("org/repo:tag").parallel(0); + assert_eq!(dl.parallel, 1); + + let dl = OciDownload::new("org/repo:tag").parallel(100); + assert_eq!(dl.parallel, 100); + } + + #[test] + fn test_oci_manifest_deserialize() { + let json = r#"{ + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:config123", + "size": 512 + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "digest": "sha256:layer123", + "size": 1024, + "annotations": { + "org.opencontainers.image.title": "file.tar.gz" + } + } + ] + }"#; + + let manifest: OciManifest = serde_json::from_str(json).unwrap(); + assert_eq!( + manifest.media_type, + "application/vnd.oci.image.manifest.v1+json" + ); + assert_eq!(manifest.config.digest, "sha256:config123"); + assert_eq!(manifest.layers.len(), 1); + assert_eq!(manifest.layers[0].title(), Some("file.tar.gz")); + } + + #[test] + fn test_oci_layer_deserialize_without_annotations() { + let json = r#"{ + "mediaType": "application/vnd.oci.image.layer.v1.tar", + "digest": "sha256:abc", + "size": 2048 + }"#; + + let layer: OciLayer = serde_json::from_str(json).unwrap(); + assert_eq!(layer.media_type, "application/vnd.oci.image.layer.v1.tar"); + assert_eq!(layer.digest, "sha256:abc"); + assert_eq!(layer.size, 2048); + assert!(layer.annotations.is_empty()); + } + + #[test] + fn test_oci_config_deserialize() { + let json = r#"{ + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:xyz789", + "size": 256 + }"#; + + let config: OciConfig = serde_json::from_str(json).unwrap(); + assert_eq!( + config.media_type, + "application/vnd.oci.image.config.v1+json" + ); + assert_eq!(config.digest, "sha256:xyz789"); + assert_eq!(config.size, 256); + } +} diff --git a/crates/soar-dl/src/platform.rs b/crates/soar-dl/src/platform.rs index 623bc27b8..47f1af023 100644 --- a/crates/soar-dl/src/platform.rs +++ b/crates/soar-dl/src/platform.rs @@ -32,14 +32,15 @@ pub enum PlatformUrl { } static GITHUB_RE: LazyLock = LazyLock::new(|| { - Regex::new( - r"^(?i)(?:https?://)?(?:github(?:\.com)?[:/])([^/@]+/[^/@]+)(?:@([^/\s]+(?:/[^/\s]*)*)?)?$", - ) - .expect("unable to compile github release regex") + Regex::new(r"^(?i)(?:https?://)?(?:github(?:\.com)?[:/])([^/@]+/[^/@]+)(?:@([^\r\n]+))?$") + .expect("unable to compile github release regex") }); + static GITLAB_RE: LazyLock = LazyLock::new(|| { - Regex::new(r"^(?i)(?:https?://)?(?:gitlab(?:\.com)?[:/])((?:\d+)|(?:[^/@]+(?:/[^/@]+)*))(?:@([^/\s]+(?:/[^/\s]*)*)?)?$") - .expect("unable to compile gitlab release regex") + Regex::new( + r"^(?i)(?:https?://)?(?:gitlab(?:\.com)?[:/])((?:\d+)|(?:[^/@]+(?:/[^/@]+)*))(?:@([^\r\n]+))?$", + ) + .expect("unable to compile gitlab release regex") }); impl PlatformUrl { @@ -215,3 +216,280 @@ fn should_fallback_status(e: &DownloadError) -> bool { matches!(e, DownloadError::HttpError { status, .. } if *status == 429 || *status == 401 || *status == 403 || *status >= 500) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_platform_url_parse_oci() { + let result = PlatformUrl::parse("ghcr.io/owner/repo:latest"); + match result { + Some(PlatformUrl::Oci { + reference, + }) => { + assert_eq!(reference, "ghcr.io/owner/repo:latest"); + } + _ => panic!("Expected OCI variant"), + } + } + + #[test] + fn test_platform_url_parse_oci_with_prefix() { + let result = PlatformUrl::parse("https://ghcr.io/owner/repo:v1.0"); + match result { + Some(PlatformUrl::Oci { + reference, + }) => { + assert_eq!(reference, "ghcr.io/owner/repo:v1.0"); + } + _ => panic!("Expected OCI variant"), + } + } + + #[test] + fn test_platform_url_parse_github_https() { + let result = PlatformUrl::parse("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/owner/repo"); + match result { + Some(PlatformUrl::Github { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, None); + } + _ => panic!("Expected Github variant"), + } + } + + #[test] + fn test_platform_url_parse_github_with_tag() { + let result = PlatformUrl::parse("/owner/repo@v1.0.0"); + match result { + Some(PlatformUrl::Github { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, Some("v1.0.0".to_string())); + } + _ => panic!("Expected Github variant with tag"), + } + } + + #[test] + fn test_platform_url_parse_github_shorthand() { + let result = PlatformUrl::parse("github:owner/repo"); + match result { + Some(PlatformUrl::Github { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, None); + } + _ => panic!("Expected Github variant"), + } + } + + #[test] + fn test_platform_url_parse_github_case_insensitive() { + let result = PlatformUrl::parse("GITHUB.COM/owner/repo"); + match result { + Some(PlatformUrl::Github { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, None); + } + _ => panic!("Expected Github variant"), + } + } + + #[test] + fn test_platform_url_parse_gitlab_https() { + let result = PlatformUrl::parse("https://gitlab.com/owner/repo"); + match result { + Some(PlatformUrl::Gitlab { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, None); + } + _ => panic!("Expected Gitlab variant"), + } + } + + #[test] + fn test_platform_url_parse_gitlab_with_tag() { + let result = PlatformUrl::parse("https://gitlab.com/owner/repo@v2.0"); + match result { + Some(PlatformUrl::Gitlab { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, Some("v2.0".to_string())); + } + _ => panic!("Expected Gitlab variant with tag"), + } + } + + #[test] + fn test_platform_url_parse_gitlab_numeric_project() { + let result = PlatformUrl::parse("https://gitlab.com/12345@v1.0"); + match result { + Some(PlatformUrl::Gitlab { + project, + tag, + }) => { + assert_eq!(project, "12345"); + assert_eq!(tag, Some("v1.0".to_string())); + } + _ => panic!("Expected Gitlab variant with numeric project"), + } + } + + #[test] + fn test_platform_url_parse_gitlab_nested_groups() { + let result = PlatformUrl::parse("https://gitlab.com/group/subgroup/repo"); + match result { + Some(PlatformUrl::Gitlab { + project, + tag, + }) => { + assert_eq!(project, "group/subgroup/repo"); + assert_eq!(tag, None); + } + _ => panic!("Expected Gitlab variant with nested groups"), + } + } + + #[test] + fn test_platform_url_parse_gitlab_api_path_as_direct() { + let result = PlatformUrl::parse("https://gitlab.com/api/v4/projects/123"); + match result { + Some(PlatformUrl::Direct { + url, + }) => { + assert_eq!(url, "https://gitlab.com/api/v4/projects/123"); + } + _ => panic!("Expected Direct variant for API path"), + } + } + + #[test] + fn test_platform_url_parse_gitlab_special_path_as_direct() { + let result = PlatformUrl::parse("https://gitlab.com/owner/repo/-/releases"); + match result { + Some(PlatformUrl::Direct { + url, + }) => { + assert_eq!(url, "https://gitlab.com/owner/repo/-/releases"); + } + _ => panic!("Expected Direct variant for special path"), + } + } + + #[test] + fn test_platform_url_parse_direct_url() { + let result = PlatformUrl::parse("https://example.com/download/file.tar.gz"); + match result { + Some(PlatformUrl::Direct { + url, + }) => { + assert_eq!(url, "https://example.com/download/file.tar.gz"); + } + _ => panic!("Expected Direct variant"), + } + } + + #[test] + fn test_platform_url_parse_direct_http() { + let result = PlatformUrl::parse("http://example.com/file.zip"); + match result { + Some(PlatformUrl::Direct { + url, + }) => { + assert_eq!(url, "http://example.com/file.zip"); + } + _ => panic!("Expected Direct variant"), + } + } + + #[test] + fn test_platform_url_parse_invalid() { + assert!(PlatformUrl::parse("not a valid url").is_none()); + assert!(PlatformUrl::parse("").is_none()); + assert!(PlatformUrl::parse("/not/a/url").is_none()); + } + + #[test] + fn test_platform_url_parse_github_with_spaces_in_tag() { + let result = PlatformUrl::parse("github.com/owner/repo@v1.0 beta"); + match result { + Some(PlatformUrl::Github { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, Some("v1.0 beta".to_string())); + } + _ => panic!("Expected Github variant with tag containing spaces"), + } + } + + #[test] + fn test_platform_url_parse_tag_with_special_chars() { + let result = PlatformUrl::parse("github.com/owner/repo@v1.0-rc.1+build.123"); + match result { + Some(PlatformUrl::Github { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, Some("v1.0-rc.1+build.123".to_string())); + } + _ => panic!("Expected Github variant with complex tag"), + } + } + + #[test] + fn test_api_kind_equality() { + assert_eq!(ApiKind::Pkgforge, ApiKind::Pkgforge); + assert_eq!(ApiKind::Primary, ApiKind::Primary); + assert_ne!(ApiKind::Pkgforge, ApiKind::Primary); + } + + #[test] + fn test_parse_repo_with_quotes() { + let result = PlatformUrl::parse("github.com/owner/repo@'v1.0'"); + match result { + Some(PlatformUrl::Github { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, Some("v1.0".to_string())); + } + _ => panic!("Expected quotes to be stripped from tag"), + } + } + + #[test] + fn test_parse_repo_percent_encoded_tag() { + let result = PlatformUrl::parse("github.com/owner/repo@v1.0%2Bbuild"); + match result { + Some(PlatformUrl::Github { + project, + tag, + }) => { + assert_eq!(project, "owner/repo"); + assert_eq!(tag, Some("v1.0+build".to_string())); + } + _ => panic!("Expected percent-encoded tag to be decoded"), + } + } +} diff --git a/crates/soar-dl/src/types.rs b/crates/soar-dl/src/types.rs index 18b0f52e7..87fdafc17 100644 --- a/crates/soar-dl/src/types.rs +++ b/crates/soar-dl/src/types.rs @@ -24,3 +24,155 @@ pub struct ResumeInfo { pub etag: Option, pub last_modified: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_progress_starting() { + let progress = Progress::Starting { + total: 1024, + }; + match progress { + Progress::Starting { + total, + } => assert_eq!(total, 1024), + _ => panic!("Expected Progress::Starting"), + } + } + + #[test] + fn test_progress_chunk() { + let progress = Progress::Chunk { + current: 512, + total: 1024, + }; + match progress { + Progress::Chunk { + current, + total, + } => { + assert_eq!(current, 512); + assert_eq!(total, 1024); + } + _ => panic!("Expected Progress::Chunk"), + } + } + + #[test] + fn test_progress_complete() { + let progress = Progress::Complete { + total: 1024, + }; + match progress { + Progress::Complete { + total, + } => assert_eq!(total, 1024), + _ => panic!("Expected Progress::Complete"), + } + } + + #[test] + fn test_progress_clone() { + let p1 = Progress::Starting { + total: 100, + }; + let p2 = p1; + match (p1, p2) { + ( + Progress::Starting { + total: t1, + }, + Progress::Starting { + total: t2, + }, + ) => { + assert_eq!(t1, t2); + } + _ => panic!("Clone failed"), + } + } + + #[test] + fn test_overwrite_mode_equality() { + assert_eq!(OverwriteMode::Skip, OverwriteMode::Skip); + assert_eq!(OverwriteMode::Force, OverwriteMode::Force); + assert_eq!(OverwriteMode::Prompt, OverwriteMode::Prompt); + + assert_ne!(OverwriteMode::Skip, OverwriteMode::Force); + assert_ne!(OverwriteMode::Force, OverwriteMode::Prompt); + assert_ne!(OverwriteMode::Skip, OverwriteMode::Prompt); + } + + #[test] + fn test_overwrite_mode_clone() { + let mode1 = OverwriteMode::Force; + let mode2 = mode1; + assert_eq!(mode1, mode2); + } + + #[test] + fn test_resume_info_with_etag() { + let info = ResumeInfo { + downloaded: 512, + total: 1024, + etag: Some("\"abc123\"".to_string()), + last_modified: None, + }; + + assert_eq!(info.downloaded, 512); + assert_eq!(info.total, 1024); + assert_eq!(info.etag, Some("\"abc123\"".to_string())); + assert_eq!(info.last_modified, None); + } + + #[test] + fn test_resume_info_with_last_modified() { + let info = ResumeInfo { + downloaded: 256, + total: 1024, + etag: None, + last_modified: Some("Wed, 21 Oct 2015 07:28:00 GMT".to_string()), + }; + + assert_eq!(info.downloaded, 256); + assert_eq!( + info.last_modified, + Some("Wed, 21 Oct 2015 07:28:00 GMT".to_string()) + ); + } + + #[test] + fn test_resume_info_serialize_deserialize() { + let info = ResumeInfo { + downloaded: 1024, + total: 2048, + etag: Some("etag-value".to_string()), + last_modified: Some("date".to_string()), + }; + + let json = serde_json::to_string(&info).unwrap(); + let deserialized: ResumeInfo = serde_json::from_str(&json).unwrap(); + + assert_eq!(deserialized.downloaded, info.downloaded); + assert_eq!(deserialized.total, info.total); + assert_eq!(deserialized.etag, info.etag); + assert_eq!(deserialized.last_modified, info.last_modified); + } + + #[test] + fn test_resume_info_clone() { + let info1 = ResumeInfo { + downloaded: 100, + total: 200, + etag: Some("tag".to_string()), + last_modified: None, + }; + + let info2 = info1.clone(); + assert_eq!(info1.downloaded, info2.downloaded); + assert_eq!(info1.total, info2.total); + assert_eq!(info1.etag, info2.etag); + } +} diff --git a/crates/soar-dl/src/utils.rs b/crates/soar-dl/src/utils.rs index 8ce6d37e4..909bbf865 100644 --- a/crates/soar-dl/src/utils.rs +++ b/crates/soar-dl/src/utils.rs @@ -136,3 +136,187 @@ pub fn resolve_output_path( } } } + +#[cfg(test)] +mod tests { + use ureq::http::HeaderValue; + + use super::*; + + #[test] + fn test_filename_from_url_simple() { + assert_eq!( + filename_from_url("https://example.com/file.txt"), + Some("file.txt".to_string()) + ); + assert_eq!( + filename_from_url("https://example.com/path/to/archive.tar.gz"), + Some("archive.tar.gz".to_string()) + ); + } + + #[test] + fn test_filename_from_url_trailing_slash() { + assert_eq!(filename_from_url("https://example.com/path/"), None); + assert_eq!(filename_from_url("https://example.com/"), None); + } + + #[test] + fn test_filename_from_url_no_path() { + assert_eq!(filename_from_url("https://example.com"), None); + } + + #[test] + fn test_filename_from_url_invalid() { + assert_eq!(filename_from_url("not a url"), None); + assert_eq!(filename_from_url(""), None); + } + + #[test] + fn test_filename_from_url_percent_encoded() { + assert_eq!( + filename_from_url("https://example.com/hello%20world.txt"), + Some("hello world.txt".to_string()) + ); + assert_eq!( + filename_from_url("https://example.com/file%2Bname.tar.gz"), + Some("file+name.tar.gz".to_string()) + ); + } + + #[test] + fn test_filename_from_url_query_params() { + assert_eq!( + filename_from_url("https://example.com/file.txt?version=1"), + Some("file.txt".to_string()) + ); + } + + #[test] + fn test_filename_from_url_fragment() { + assert_eq!( + filename_from_url("https://example.com/file.txt#section"), + Some("file.txt".to_string()) + ); + } + + #[test] + fn test_filename_from_header_simple() { + let header = HeaderValue::from_static("attachment; filename=\"example.txt\""); + assert_eq!( + filename_from_header(&header), + Some("example.txt".to_string()) + ); + } + + #[test] + fn test_filename_from_header_no_quotes() { + let header = HeaderValue::from_static("attachment; filename=example.txt"); + assert_eq!( + filename_from_header(&header), + Some("example.txt".to_string()) + ); + } + + #[test] + fn test_filename_from_header_with_path() { + let header = HeaderValue::from_static("attachment; filename=\"/path/to/file.txt\""); + assert_eq!(filename_from_header(&header), Some("file.txt".to_string())); + + let header = HeaderValue::from_static("attachment; filename=\"path\\to\\file.txt\""); + assert_eq!(filename_from_header(&header), Some("file.txt".to_string())); + } + + #[test] + fn test_filename_from_header_multiple_params() { + let header = + HeaderValue::from_static("inline; name=value; filename=\"test.pdf\"; size=1024"); + assert_eq!(filename_from_header(&header), Some("test.pdf".to_string())); + } + + #[test] + fn test_filename_from_header_no_filename() { + let header = HeaderValue::from_static("attachment"); + assert_eq!(filename_from_header(&header), None); + + let header = HeaderValue::from_static("inline; name=value"); + assert_eq!(filename_from_header(&header), None); + } + + #[test] + fn test_filename_from_header_empty_filename() { + let header = HeaderValue::from_static("attachment; filename=\"\""); + assert_eq!(filename_from_header(&header), Some("".to_string())); + } + + #[test] + fn test_resolve_output_path_stdout() { + let result = resolve_output_path(Some("-"), None, None).unwrap(); + assert_eq!(result, PathBuf::from("-")); + } + + #[test] + fn test_resolve_output_path_trailing_slash() { + let result = resolve_output_path( + Some("downloads/"), + Some("from_url.txt".into()), + Some("from_header.txt".into()), + ) + .unwrap(); + // Should prefer header filename + assert_eq!(result, PathBuf::from("downloads/from_header.txt")); + } + + #[test] + fn test_resolve_output_path_trailing_slash_no_header() { + let result = + resolve_output_path(Some("downloads/"), Some("from_url.txt".into()), None).unwrap(); + assert_eq!(result, PathBuf::from("downloads/from_url.txt")); + } + + #[test] + fn test_resolve_output_path_trailing_slash_no_filenames() { + let result = resolve_output_path(Some("downloads/"), None, None); + assert!(matches!(result, Err(DownloadError::NoFilename))); + } + + #[test] + fn test_resolve_output_path_explicit_file() { + let result = resolve_output_path( + Some("output.txt"), + Some("url.txt".into()), + Some("header.txt".into()), + ) + .unwrap(); + assert_eq!(result, PathBuf::from("output.txt")); + } + + #[test] + fn test_resolve_output_path_none_uses_header() { + let result = resolve_output_path( + None, + Some("from_url.txt".into()), + Some("from_header.txt".into()), + ) + .unwrap(); + assert_eq!(result, PathBuf::from("from_header.txt")); + } + + #[test] + fn test_resolve_output_path_none_uses_url() { + let result = resolve_output_path(None, Some("from_url.txt".into()), None).unwrap(); + assert_eq!(result, PathBuf::from("from_url.txt")); + } + + #[test] + fn test_resolve_output_path_none_no_filenames() { + let result = resolve_output_path(None, None, None); + assert!(matches!(result, Err(DownloadError::NoFilename))); + } + + #[test] + fn test_resolve_output_path_with_subdirectories() { + let result = resolve_output_path(Some("path/to/"), Some("file.txt".into()), None).unwrap(); + assert_eq!(result, PathBuf::from("path/to/file.txt")); + } +} From 57f3a356a80edf412054b5a0eea2245606ca49b8 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Sat, 1 Nov 2025 14:51:55 +0545 Subject: [PATCH 12/22] fix some issues --- crates/soar-dl/src/download.rs | 5 ++++- crates/soar-dl/src/github.rs | 2 +- crates/soar-dl/src/gitlab.rs | 2 +- crates/soar-dl/src/oci.rs | 16 +++++++++++----- crates/soar-dl/src/utils.rs | 18 +++++------------- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs index c6d0068ee..abb70e959 100644 --- a/crates/soar-dl/src/download.rs +++ b/crates/soar-dl/src/download.rs @@ -330,6 +330,7 @@ impl Download { let mut reader = resp.into_body().into_reader(); let mut buffer = [0u8; 8192]; let mut downloaded = resume_from.unwrap_or(0); + let mut last_checkpoint = downloaded / (1024 * 1024); loop { let n = reader.read(&mut buffer)?; @@ -340,7 +341,9 @@ impl Download { file.write_all(&buffer[..n])?; downloaded += n as u64; - if downloaded % (1024 * 1024) == 0 { + let checkpoint = downloaded / (1024 * 1024); + if checkpoint > last_checkpoint { + last_checkpoint = checkpoint; write_resume( path, &ResumeInfo { diff --git a/crates/soar-dl/src/github.rs b/crates/soar-dl/src/github.rs index f49c62bf7..16b169d1a 100644 --- a/crates/soar-dl/src/github.rs +++ b/crates/soar-dl/src/github.rs @@ -46,7 +46,7 @@ impl Platform for Github { /// /// # Examples /// - /// ``` + /// ```no_run /// use soar_dl::github::Github; /// use soar_dl::traits::{Platform, Release}; /// diff --git a/crates/soar-dl/src/gitlab.rs b/crates/soar-dl/src/gitlab.rs index fbfc24a7c..c44da51e7 100644 --- a/crates/soar-dl/src/gitlab.rs +++ b/crates/soar-dl/src/gitlab.rs @@ -51,7 +51,7 @@ impl Platform for GitLab { /// /// # Examples /// - /// ``` + /// ```no_run /// use soar_dl::gitlab::GitLab; /// use soar_dl::traits::Platform; /// diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs index 2228c511f..f33e4a0b0 100644 --- a/crates/soar-dl/src/oci.rs +++ b/crates/soar-dl/src/oci.rs @@ -567,7 +567,12 @@ impl OciDownload { .collect(); for handle in handles { - handle.join().ok(); + if let Err(err) = handle.join() { + errors + .lock() + .unwrap() + .push(format!("Worker thread panicked: {err:?}")); + } } let errors = errors.lock().unwrap(); @@ -594,7 +599,6 @@ impl OciDownload { /// /// let dl = OciDownload::new("ghcr.io/example/repo:latest"); /// let manifest = dl.fetch_manifest().unwrap(); - /// assert!(manifest.layers.len() >= 0); /// ``` pub fn fetch_manifest(&self) -> Result { let url = format!( @@ -652,7 +656,6 @@ impl OciDownload { .unwrap_or(&self.reference.tag); let output = self.output.as_deref().unwrap_or(filename); - let path = PathBuf::from(output); let url = format!( "{}/{}/blobs/{}", @@ -670,7 +673,7 @@ impl OciDownload { dl }; - dl.execute()?; + let path = dl.execute()?; Ok(vec![path]) } @@ -793,6 +796,7 @@ fn download_layer_impl( let mut reader = resp.into_body().into_reader(); let mut buffer = [0u8; 8192]; *local_downloaded = resume_from.unwrap_or(0); + let mut last_checkpoint = *local_downloaded / (1024 * 1024); loop { let n = reader.read(&mut buffer)?; @@ -809,7 +813,9 @@ fn download_layer_impl( *shared }; - if (*local_downloaded).is_multiple_of(1024 * 1024) { + let checkpoint = *local_downloaded / (1024 * 1024); + if checkpoint > last_checkpoint { + last_checkpoint = checkpoint; write_resume( path, &ResumeInfo { diff --git a/crates/soar-dl/src/utils.rs b/crates/soar-dl/src/utils.rs index 909bbf865..9cfb4cd4f 100644 --- a/crates/soar-dl/src/utils.rs +++ b/crates/soar-dl/src/utils.rs @@ -109,28 +109,20 @@ pub fn resolve_output_path( url_filename: Option, header_filename: Option, ) -> Result { + let filename = || header_filename.or(url_filename); + match output { Some("-") => Ok(PathBuf::from("-")), - Some(p) if p.ends_with('/') => { - let filename = header_filename - .or(url_filename) - .ok_or(DownloadError::NoFilename)?; - Ok(PathBuf::from(p).join(filename)) - } Some(p) => { let path = PathBuf::from(p); - if path.is_dir() { - let filename = header_filename - .or(url_filename) - .ok_or(DownloadError::NoFilename)?; - Ok(path.join(filename)) + if p.ends_with('/') || path.is_dir() { + Ok(path.join(filename().ok_or(DownloadError::NoFilename)?)) } else { Ok(path) } } None => { - header_filename - .or(url_filename) + filename() .map(PathBuf::from) .ok_or(DownloadError::NoFilename) } From 8665a6f15ff0a8b2818c526895392be04e6d9928 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Sat, 1 Nov 2025 18:22:06 +0545 Subject: [PATCH 13/22] migrate to new soar-dl --- Cargo.lock | 1 + crates/soar-dl/src/download.rs | 6 +- crates/soar-dl/src/oci.rs | 1 + crates/soar-dl/src/types.rs | 3 + crates/soar-utils/src/fs.rs | 2 +- soar-cli/Cargo.toml | 1 + soar-cli/src/download.rs | 442 +++++++++++++-------- soar-cli/src/install.rs | 24 +- soar-cli/src/list.rs | 20 +- soar-cli/src/main.rs | 97 +++-- soar-cli/src/nest.rs | 9 +- soar-cli/src/progress.rs | 40 +- soar-cli/src/run.rs | 68 ++-- soar-cli/src/self_actions.rs | 86 ++-- soar-cli/src/utils.rs | 15 +- soar-core/src/config.rs | 14 +- soar-core/src/database/connection.rs | 11 +- soar-core/src/database/migration.rs | 9 +- soar-core/src/database/models.rs | 22 +- soar-core/src/database/nests/repository.rs | 15 +- soar-core/src/database/packages/query.rs | 27 +- soar-core/src/error.rs | 44 +- soar-core/src/package/formats/appimage.rs | 3 +- soar-core/src/package/formats/common.rs | 66 ++- soar-core/src/package/formats/wrappe.rs | 3 +- soar-core/src/package/install.rs | 116 +++--- soar-core/src/package/remove.rs | 5 +- soar-core/src/toml.rs | 34 +- 28 files changed, 675 insertions(+), 509 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c83493df0..4bd0f52fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2489,6 +2489,7 @@ dependencies = [ "toml", "tracing", "tracing-subscriber", + "ureq", ] [[package]] diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs index abb70e959..5e34a1c74 100644 --- a/crates/soar-dl/src/download.rs +++ b/crates/soar-dl/src/download.rs @@ -3,6 +3,7 @@ use std::{ io::{Read as _, Write as _}, os::unix::fs::PermissionsExt as _, path::{Path, PathBuf}, + sync::Arc, }; use soar_utils::fs::is_elf; @@ -22,13 +23,14 @@ use crate::{ xattr::{read_resume, remove_resume, write_resume}, }; +#[derive(Clone)] pub struct Download { pub url: String, pub output: Option, pub overwrite: OverwriteMode, pub extract: bool, pub extract_to: Option, - pub on_progress: Option>, + pub on_progress: Option>, } impl Download { @@ -159,7 +161,7 @@ impl Download { where F: Fn(Progress) + Send + Sync + 'static, { - self.on_progress = Some(Box::new(on_progress)); + self.on_progress = Some(Arc::new(on_progress)); self } diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs index f33e4a0b0..55708c653 100644 --- a/crates/soar-dl/src/oci.rs +++ b/crates/soar-dl/src/oci.rs @@ -144,6 +144,7 @@ impl OciLayer { } } +#[derive(Clone)] pub struct OciDownload { reference: OciReference, api: String, diff --git a/crates/soar-dl/src/types.rs b/crates/soar-dl/src/types.rs index 87fdafc17..9e3ffeec9 100644 --- a/crates/soar-dl/src/types.rs +++ b/crates/soar-dl/src/types.rs @@ -6,6 +6,9 @@ pub enum Progress { Starting { total: u64 }, Chunk { current: u64, total: u64 }, Complete { total: u64 }, + Error, + Aborted, + Recovered, } /// How to handle existing files diff --git a/crates/soar-utils/src/fs.rs b/crates/soar-utils/src/fs.rs index 09a1a143c..2019f5c40 100644 --- a/crates/soar-utils/src/fs.rs +++ b/crates/soar-utils/src/fs.rs @@ -705,4 +705,4 @@ mod tests { perms.set_readonly(false); fs::set_permissions(tempdir.path(), perms).unwrap(); } -} \ No newline at end of file +} diff --git a/soar-cli/Cargo.toml b/soar-cli/Cargo.toml index 4433e9ea4..427528498 100644 --- a/soar-cli/Cargo.toml +++ b/soar-cli/Cargo.toml @@ -40,3 +40,4 @@ tokio = { version = "1.47.1", features = ["macros", "rt-multi-thread"] } toml = "0.9.6" tracing = { workspace = true } tracing-subscriber = { version = "0.3.20", default-features = false, features = ["env-filter", "fmt", "json", "nu-ansi-term"] } +ureq = { workspace = true } diff --git a/soar-cli/src/download.rs b/soar-cli/src/download.rs index 9f73bb05a..d5be03afc 100644 --- a/soar-cli/src/download.rs +++ b/soar-cli/src/download.rs @@ -2,8 +2,6 @@ use std::{sync::Arc, thread::sleep, time::Duration}; use indicatif::HumanBytes; use regex::Regex; -use reqwest::StatusCode; -use serde::Deserialize; use soar_core::{ config::get_config, database::{models::Package, packages::PackageQueryBuilder}, @@ -11,21 +9,22 @@ use soar_core::{ SoarResult, }; use soar_dl::{ - downloader::{DownloadOptions, DownloadState, Downloader, OciDownloadOptions, OciDownloader}, + download::Download, error::DownloadError, - github::{Github, GithubAsset, GithubRelease}, - gitlab::{Gitlab, GitlabAsset, GitlabRelease}, - platform::{ - PlatformDownloadOptions, PlatformUrl, Release, ReleaseAsset, ReleaseHandler, - ReleasePlatform, - }, - utils::FileMode, + filter::Filter, + github::Github, + gitlab::GitLab, + oci::OciDownload, + platform::PlatformUrl, + release::ReleaseDownload, + traits::{Asset, Platform as _, Release as _}, + types::{OverwriteMode, Progress}, }; use tracing::{error, info}; use crate::{ state::AppState, - utils::{get_file_mode, interactive_ask, select_package_interactively}, + utils::{interactive_ask, select_package_interactively}, }; pub struct DownloadContext { @@ -35,7 +34,7 @@ pub struct DownloadContext { pub exclude_keywords: Vec, pub output: Option, pub yes: bool, - pub progress_callback: Arc, + pub progress_callback: Arc, pub exact_case: bool, pub extract: bool, pub extract_dir: Option, @@ -43,13 +42,35 @@ pub struct DownloadContext { pub force_overwrite: bool, } +impl DownloadContext { + fn get_overwrite_mode(&self) -> OverwriteMode { + if self.force_overwrite || self.yes { + OverwriteMode::Force + } else if self.skip_existing { + OverwriteMode::Skip + } else { + OverwriteMode::Prompt + } + } + + fn create_filter(&self) -> Filter { + Filter { + regexes: self.regexes.clone(), + globs: self.globs.clone(), + include: self.match_keywords.clone(), + exclude: self.exclude_keywords.clone(), + case_sensitive: self.exact_case, + } + } +} + pub async fn download( ctx: DownloadContext, links: Vec, github: Vec, gitlab: Vec, ghcr: Vec, - progress_callback: Arc, + progress_callback: Arc, ) -> SoarResult<()> { handle_direct_downloads(&ctx, links, ctx.output.clone(), progress_callback.clone()).await?; @@ -72,57 +93,62 @@ pub async fn handle_direct_downloads( ctx: &DownloadContext, links: Vec, output: Option, - progress_callback: Arc, + progress_callback: Arc, ) -> SoarResult<()> { - let downloader = Downloader::default(); - for link in &links { match PlatformUrl::parse(link) { - Ok(PlatformUrl::DirectUrl(url)) => { + Some(PlatformUrl::Direct { + url, + }) => { info!("Downloading using direct link: {}", url); - let options = DownloadOptions { - url: link.clone(), - output_path: output.clone(), - progress_callback: Some(progress_callback.clone()), - extract_archive: ctx.extract, - extract_dir: ctx.extract_dir.clone(), - file_mode: get_file_mode(ctx.skip_existing, ctx.yes || ctx.force_overwrite), - prompt: None, - }; - let _ = downloader - .download(options) - .await - .map_err(|e| error!("{}", e)); + let mut dl = Download::new(url) + .overwrite(ctx.get_overwrite_mode()) + .extract(ctx.extract); + + if let Some(ref out) = output { + dl = dl.output(out); + } + + if let Some(extract_dir) = ctx.extract_dir.clone() { + dl = dl.extract_to(extract_dir); + } + + let cb = ctx.progress_callback.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + + if let Err(err) = dl.execute() { + error!("{}", err); + } } - Ok(PlatformUrl::Github(project)) => { + Some(PlatformUrl::Github { + project, + tag, + }) => { info!("Detected GitHub URL, processing as GitHub release"); - let handler = ReleaseHandler::::new(); - if let Err(e) = handle_platform_download::( - ctx, &handler, &project, - ) - .await - { - error!("{}", e); + if let Err(err) = handle_github_release(ctx, &project, tag.as_deref()) { + error!("{}", err); } } - Ok(PlatformUrl::Gitlab(project)) => { + Some(PlatformUrl::Gitlab { + project, + tag, + }) => { info!("Detected GitLab URL, processing as GitLab release"); - let handler = ReleaseHandler::::new(); - if let Err(e) = handle_platform_download::( - ctx, &handler, &project, - ) - .await - { - error!("{}", e); + if let Err(err) = handle_gitlab_release(ctx, &project, tag.as_deref()) { + error!("{}", err); } } - Ok(PlatformUrl::Oci(url)) => { - if let Err(e) = handle_oci_download(ctx, &url).await { - error!("{}", e); + Some(PlatformUrl::Oci { + reference, + }) => { + if let Err(err) = handle_oci_download(ctx, &reference).await { + error!("{}", err); }; } - Err(_) => { + None => { // if it's not a url, try to parse it as package let state = AppState::new(); let repo_db = state.repo_db().await?; @@ -147,36 +173,34 @@ pub async fn handle_direct_downloads( package.pkg_name, package.pkg_id ); if let Some(ref url) = package.ghcr_blob { - let options = OciDownloadOptions { - url: url.to_string(), - output_path: output.clone(), - progress_callback: Some(progress_callback.clone()), - api: None, - concurrency: Some(1), - regexes: Vec::new(), - globs: Vec::new(), - exclude_keywords: Vec::new(), - match_keywords: Vec::new(), - exact_case: false, - file_mode: FileMode::ForceOverwrite, - }; - - let mut downloader = OciDownloader::new(options); - - downloader.download_oci().await?; + let mut dl = OciDownload::new(url.as_str()).overwrite(OverwriteMode::Force); + + if let Some(ref out) = output { + dl = dl.output(out); + } + + let cb = progress_callback.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + + if let Err(err) = dl.execute() { + error!("{}", err); + } } else { - let downloader = Downloader::default(); - let options = DownloadOptions { - url: package.download_url.clone(), - output_path: output.clone(), - progress_callback: Some(progress_callback.clone()), - extract_archive: false, - extract_dir: Some("SOAR_AUTO_EXTRACT".into()), - file_mode: FileMode::ForceOverwrite, - prompt: None, - }; - - downloader.download(options).await?; + let mut dl = + Download::new(&package.download_url).overwrite(OverwriteMode::Force); + + if let Some(ref out) = output { + dl = dl.output(out); + } + + let cb = progress_callback.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + + dl.execute()?; } } }; @@ -188,42 +212,49 @@ pub async fn handle_direct_downloads( async fn handle_oci_download(ctx: &DownloadContext, reference: &str) -> SoarResult<()> { info!("Downloading using OCI reference: {}", reference); - let options = OciDownloadOptions { - url: reference.to_string(), - output_path: ctx.output.clone(), - progress_callback: Some(ctx.progress_callback.clone()), - api: None, - regexes: ctx.regexes.clone(), - concurrency: get_config().ghcr_concurrency, - match_keywords: ctx.match_keywords.clone(), - exclude_keywords: ctx.exclude_keywords.clone(), - exact_case: ctx.exact_case, - globs: ctx.globs.clone(), - file_mode: get_file_mode(ctx.skip_existing, ctx.yes || ctx.force_overwrite), - }; + let mut dl = OciDownload::new(reference) + .filter(ctx.create_filter()) + .parallel(get_config().ghcr_concurrency.unwrap_or(8)) + .overwrite(ctx.get_overwrite_mode()); + + if let Some(ref output) = ctx.output { + dl = dl.output(output); + } + + let cb = ctx.progress_callback.clone(); + dl = dl.progress(move |p| { + cb(p); + }); - let mut downloader = OciDownloader::new(options); let mut retries = 0; + let max_retries = 5; + loop { - if retries > 5 { - error!("Max retries exhausted. Aborting."); - break; - } - match downloader.download_oci().await { - Ok(_) => break, - Err( - DownloadError::ResourceError { - status: StatusCode::TOO_MANY_REQUESTS, - .. - } - | DownloadError::ChunkError, - ) => sleep(Duration::from_secs(5)), - Err(err) => { - error!("{}", err); + match dl.clone().execute() { + Ok(_) => { + info!("Download completed successfully"); break; } - }; - retries += 1; + Err(err) + if matches!( + err, + DownloadError::HttpError { + status: 429, + .. + } | DownloadError::Network(_) + ) && retries < max_retries => + { + retries += 1; + info!("Retrying... ({}/{})", retries, max_retries); + ctx.progress_callback.clone()(Progress::Recovered); + sleep(Duration::from_secs(5)); + } + Err(err) => { + ctx.progress_callback.clone()(Progress::Error); + error!("Download failed: {}", err); + return Err(err)?; + } + } } Ok(()) @@ -239,6 +270,114 @@ pub async fn handle_oci_downloads( Ok(()) } +fn handle_github_release( + ctx: &DownloadContext, + project: &str, + tag: Option<&str>, +) -> SoarResult<()> { + let releases = Github::fetch_releases(project, tag)?; + + let release = if let Some(tag) = tag { + releases.iter().find(|r| r.tag() == tag) + } else { + releases + .iter() + .find(|r| !r.is_prerelease()) + .or_else(|| releases.first()) + }; + + let release = release.ok_or_else(|| DownloadError::InvalidResponse)?; + + info!("Found release: {}", release.tag()); + let filter = ctx.create_filter(); + + let assets: Vec<_> = release + .assets() + .iter() + .filter(|a| filter.matches(a.name())) + .collect(); + + if assets.is_empty() { + let available = release + .assets() + .iter() + .map(|a| a.name().to_string()) + .collect::>(); + + Err(DownloadError::NoMatch { + available, + })? + } + + let selected_asset = if assets.len() == 1 || ctx.yes { + assets[0] + } else { + &select_asset_interactively(assets)? + }; + + info!("Downloading asset: {}", selected_asset.name()); + + let mut dl = Download::new(selected_asset.url()) + .overwrite(ctx.get_overwrite_mode()) + .extract(ctx.extract); + + if let Some(ref out) = ctx.output { + dl = dl.output(out); + } + + if let Some(ref extract_dir) = ctx.extract_dir { + dl = dl.extract_to(extract_dir); + } + + let cb = ctx.progress_callback.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + + dl.execute()?; + + Ok(()) +} + +fn handle_gitlab_release( + ctx: &DownloadContext, + project: &str, + tag: Option<&str>, +) -> SoarResult<()> { + let mut dl = ReleaseDownload::::new(project) + .filter(ctx.create_filter()) + .overwrite(ctx.get_overwrite_mode()) + .extract(ctx.extract); + + if let Some(tag) = tag { + dl = dl.tag(tag); + } + + if let Some(ref out) = ctx.output { + dl = dl.output(out); + } + + if let Some(ref extract_dir) = ctx.extract_dir { + dl = dl.extract_to(extract_dir); + } + + let cb = ctx.progress_callback.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + + let paths = dl.execute()?; + + if paths.len() > 1 && !ctx.yes { + info!("Multiple assets found, please select one:"); + for (i, path) in paths.iter().enumerate() { + info!("{}. {}", i + 1, path.display()); + } + } + + Ok(()) +} + pub fn create_regex_patterns(regex_patterns: Option>) -> Vec { regex_patterns .clone() @@ -253,63 +392,20 @@ pub fn create_regex_patterns(regex_patterns: Option>) -> Vec .unwrap_or_default() } -fn create_platform_options(ctx: &DownloadContext, tag: Option) -> PlatformDownloadOptions { - PlatformDownloadOptions { - output_path: ctx.output.clone(), - progress_callback: Some(ctx.progress_callback.clone()), - tag, - regexes: ctx.regexes.clone(), - globs: ctx.globs.clone(), - match_keywords: ctx.match_keywords.clone(), - exclude_keywords: ctx.exclude_keywords.clone(), - exact_case: ctx.exact_case, - extract_archive: ctx.extract, - extract_dir: ctx.extract_dir.clone(), - file_mode: get_file_mode(ctx.skip_existing, ctx.yes || ctx.force_overwrite), - prompt: None, - } -} - -async fn handle_platform_download( - ctx: &DownloadContext, - handler: &ReleaseHandler<'_, P>, - project: &str, -) -> SoarResult<()> -where - R: Release + for<'de> Deserialize<'de>, - A: ReleaseAsset + Clone, -{ - let (project, tag) = match project.trim().split_once('@') { - Some((proj, tag)) if !tag.trim().is_empty() => (proj, Some(tag.trim())), - _ => (project.trim_end_matches('@'), None), - }; - - let options = create_platform_options(ctx, tag.map(String::from)); - let releases = handler.fetch_releases::(project, tag).await?; - let assets = handler.filter_releases(&releases, &options).await?; - - let selected_asset = if assets.len() == 1 || ctx.yes { - assets[0].clone() - } else { - select_asset(&assets)? - }; - - info!("Downloading asset from {}", selected_asset.download_url()); - handler.download(&selected_asset, options.clone()).await?; - Ok(()) -} - pub async fn handle_github_downloads( ctx: &DownloadContext, projects: Vec, ) -> SoarResult<()> { - let handler = ReleaseHandler::::new(); for project in &projects { info!("Fetching releases from GitHub: {}", project); - if let Err(e) = - handle_platform_download::<_, GithubRelease, _>(ctx, &handler, project).await - { - error!("{}", e); + + let (project, tag) = match project.trim().split_once('@') { + Some((proj, tag)) if !tag.trim().is_empty() => (proj, Some(tag.trim())), + _ => (project.trim_end_matches('@'), None), + }; + + if let Err(err) = handle_github_release(ctx, project, tag) { + error!("{}", err); } } Ok(()) @@ -319,22 +415,24 @@ pub async fn handle_gitlab_downloads( ctx: &DownloadContext, projects: Vec, ) -> SoarResult<()> { - let handler = ReleaseHandler::::new(); for project in &projects { info!("Fetching releases from GitLab: {}", project); - if let Err(e) = - handle_platform_download::<_, GitlabRelease, _>(ctx, &handler, project).await - { - error!("{}", e); + + let (project, tag) = match project.trim().split_once('@') { + Some((proj, tag)) if !tag.trim().is_empty() => (proj, Some(tag.trim())), + _ => (project.trim_end_matches('@'), None), + }; + + if let Err(err) = handle_gitlab_release(ctx, project, tag) { + error!("{}", err); } } Ok(()) } -fn select_asset(assets: &[A]) -> SoarResult +fn select_asset_interactively(assets: Vec<&A>) -> SoarResult where - A: Clone, - A: ReleaseAsset, + A: Asset + Clone, { info!("\nAvailable assets:"); for (i, asset) in assets.iter().enumerate() { @@ -342,13 +440,13 @@ where .size() .map(|s| format!(" ({})", HumanBytes(s))) .unwrap_or_default(); - info!("{}. {}{}", i + 1, asset.name(), size); + info!(" {}. {}{}", i + 1, asset.name(), size); } loop { let max = assets.len(); - let response = interactive_ask(&format!("Select an asset (1-{max}): "))?; - match response.parse::() { + let response = interactive_ask(&format!("Select an asset (1-{}): ", max))?; + match response.trim().parse::() { Ok(n) if n > 0 && n <= max => return Ok(assets[n - 1].clone()), _ => error!("Invalid selection, please try again."), } diff --git a/soar-cli/src/install.rs b/soar-cli/src/install.rs index 31ab3e17e..60fdf4fcb 100644 --- a/soar-cli/src/install.rs +++ b/soar-cli/src/install.rs @@ -29,7 +29,7 @@ use soar_core::{ utils::default_install_patterns, SoarResult, }; -use soar_dl::downloader::DownloadState; +use soar_dl::types::Progress; use soar_utils::{hash::calculate_checksum, pattern::apply_sig_variants}; use tokio::sync::Semaphore; use tracing::{error, info, warn}; @@ -412,16 +412,18 @@ async fn spawn_installation_task( installed_count.fetch_add(1, Ordering::Relaxed); total_pb.inc(1); } - Err(err) => match err { - SoarError::Warning(err) => { - let mut warnings = ctx.warnings.lock().unwrap(); - warnings.push(err); - } - _ => { - let mut errors = ctx.errors.lock().unwrap(); - errors.push(err.to_string()); + Err(err) => { + match err { + SoarError::Warning(err) => { + let mut warnings = ctx.warnings.lock().unwrap(); + warnings.push(err); + } + _ => { + let mut errors = ctx.errors.lock().unwrap(); + errors.push(err.to_string()); + } } - }, + } } drop(permit); @@ -431,7 +433,7 @@ async fn spawn_installation_task( pub async fn install_single_package( ctx: &InstallContext, target: &InstallTarget, - progress_callback: Arc, + progress_callback: Arc, core_db: Arc>, ) -> SoarResult<(PathBuf, Vec<(PathBuf, PathBuf)>)> { let bin_dir = get_config().get_bin_path()?; diff --git a/soar-cli/src/list.rs b/soar-cli/src/list.rs index 152ebafa7..725001df3 100644 --- a/soar-cli/src/list.rs +++ b/soar-cli/src/list.rs @@ -100,10 +100,12 @@ pub async fn search_packages( package.pkg_name.clone(), ); let install_state = match installed_pkgs.get(&key) { - Some(is_installed) => match is_installed { - true => "+", - false => "?", - }, + Some(is_installed) => { + match is_installed { + true => "+", + false => "?", + } + } None => "-", }; @@ -471,10 +473,12 @@ pub async fn list_packages(repo_name: Option) -> SoarResult<()> { package.pkg_name.clone(), ); let install_state = match installed_pkgs.get(&key) { - Some(is_installed) => match is_installed { - true => "+", - false => "?", - }, + Some(is_installed) => { + match is_installed { + true => "+", + false => "?", + } + } None => "-", }; diff --git a/soar-cli/src/main.rs b/soar-cli/src/main.rs index 7b0b978d6..e9198ff74 100644 --- a/soar-cli/src/main.rs +++ b/soar-cli/src/main.rs @@ -1,7 +1,6 @@ -use std::{env, error::Error, fs, io::Read, process::Command, sync::Arc}; +use std::{env, fs, io::Read, process::Command, sync::Arc}; use clap::Parser; - use cli::Args; use download::{create_regex_patterns, download, DownloadContext}; use health::{display_health, remove_broken_packages}; @@ -19,11 +18,12 @@ use soar_core::{ utils::{cleanup_cache, remove_broken_symlinks, setup_required_paths}, SoarResult, }; -use soar_dl::http_client::{configure_http_client, create_http_header_map}; +use soar_dl::http_client::configure_http_client; use soar_utils::path::resolve_path; use state::AppState; use tracing::{error, info, warn}; use update::update_packages; +use ureq::Proxy; use use_package::use_alternate_package; use utils::COLOR; @@ -98,23 +98,26 @@ async fn handle_cli() -> SoarResult<()> { let user_agent = args.user_agent.clone(); let header = args.header.clone(); - if let Err(err) = configure_http_client(|config| { - config.proxy = proxy; + configure_http_client(|config| { + if let Some(proxy) = proxy.as_deref() { + config.proxy = Some(Proxy::new(proxy).unwrap()); + } if let Some(user_agent) = user_agent { config.user_agent = Some(user_agent); } if let Some(headers) = header { - config.headers = Some(create_http_header_map(headers)); - } - }) { - error!("Error configuring HTTP client: {}", err); - if let Some(source) = err.source() { - error!(" Caused by: {}", source); + let headers = headers + .into_iter() + .filter_map(|header| { + let (key, value) = header.split_once(':')?; + Some((key.parse().unwrap(), value.parse().unwrap())) + }) + .collect(); + config.headers = Some(headers); } - std::process::exit(1); - }; + }); match args.command { cli::Commands::DefConfig { @@ -174,10 +177,14 @@ async fn handle_cli() -> SoarResult<()> { } => { search_packages(query, case_sensitive, limit).await?; } - cli::Commands::Query { query } => { + cli::Commands::Query { + query, + } => { query_package(query).await?; } - cli::Commands::Remove { packages } => { + cli::Commands::Remove { + packages, + } => { remove_packages(&packages).await?; } cli::Commands::Sync => { @@ -193,18 +200,23 @@ async fn handle_cli() -> SoarResult<()> { } => { update_packages(packages, keep, ask, no_verify).await?; } - cli::Commands::ListInstalledPackages { repo_name, count } => { + cli::Commands::ListInstalledPackages { + repo_name, + count, + } => { list_installed_packages(repo_name, count).await?; } - cli::Commands::ListPackages { repo_name } => { + cli::Commands::ListPackages { + repo_name, + } => { list_packages(repo_name).await?; } - cli::Commands::Log { package } => { - inspect_log(&package, InspectType::BuildLog).await? - } - cli::Commands::Inspect { package } => { - inspect_log(&package, InspectType::BuildScript).await? - } + cli::Commands::Log { + package, + } => inspect_log(&package, InspectType::BuildLog).await?, + cli::Commands::Inspect { + package, + } => inspect_log(&package, InspectType::BuildScript).await?, cli::Commands::Run { yes, command, @@ -219,7 +231,9 @@ async fn handle_cli() -> SoarResult<()> { ) .await?; } - cli::Commands::Use { package_name } => { + cli::Commands::Use { + package_name, + } => { use_alternate_package(&package_name).await?; } cli::Commands::Download { @@ -282,7 +296,9 @@ async fn handle_cli() -> SoarResult<()> { ); } #[cfg(feature = "self")] - cli::Commands::SelfCmd { action } => { + cli::Commands::SelfCmd { + action, + } => { process_self_action(&action).await?; } cli::Commands::Clean { @@ -301,7 +317,9 @@ async fn handle_cli() -> SoarResult<()> { remove_broken_packages().await?; } } - cli::Commands::Config { edit } => { + cli::Commands::Config { + edit, + } => { let config_path = CONFIG_PATH.read().unwrap(); match edit { Some(editor) => { @@ -339,17 +357,24 @@ async fn handle_cli() -> SoarResult<()> { } }; } - cli::Commands::Nest(nest_command) => match nest_command { - cli::NestCommands::Add { name, url } => { - add_nest(&name, &url).await?; - } - cli::NestCommands::Remove { name } => { - remove_nest(&name).await?; - } - cli::NestCommands::List => { - list_nests().await?; + cli::Commands::Nest(nest_command) => { + match nest_command { + cli::NestCommands::Add { + name, + url, + } => { + add_nest(&name, &url).await?; + } + cli::NestCommands::Remove { + name, + } => { + remove_nest(&name).await?; + } + cli::NestCommands::List => { + list_nests().await?; + } } - }, + } _ => unreachable!(), } } diff --git a/soar-cli/src/nest.rs b/soar-cli/src/nest.rs index 468d0848e..19927d4ad 100644 --- a/soar-cli/src/nest.rs +++ b/soar-cli/src/nest.rs @@ -1,7 +1,8 @@ -use soar_core::config::get_config; -use soar_core::database::nests::models::Nest; -use soar_core::database::nests::repository; -use soar_core::SoarResult; +use soar_core::{ + config::get_config, + database::nests::{models::Nest, repository}, + SoarResult, +}; pub async fn add_nest(name: &str, url: &str) -> SoarResult<()> { let name = format!("nest-{name}"); diff --git a/soar-cli/src/progress.rs b/soar-cli/src/progress.rs index b40e8aa7e..4e265862f 100644 --- a/soar-cli/src/progress.rs +++ b/soar-cli/src/progress.rs @@ -3,7 +3,7 @@ use std::sync::atomic::Ordering; use indicatif::{HumanBytes, ProgressBar, ProgressState, ProgressStyle}; use nu_ansi_term::Color::Red; use soar_core::database::models::Package; -use soar_dl::downloader::DownloadState; +use soar_dl::types::Progress; use crate::{install::InstallContext, utils::Colored}; @@ -29,21 +29,27 @@ fn format_bytes(state: &ProgressState, w: &mut dyn std::fmt::Write) { .unwrap(); } -pub fn handle_progress(state: DownloadState, progress_bar: &ProgressBar) { +pub fn handle_progress(state: Progress, progress_bar: &ProgressBar) { match state { - DownloadState::Preparing(total) => { + Progress::Starting { + total, + } => { progress_bar.set_length(total); } - DownloadState::Progress(progress) => { - progress_bar.set_position(progress); + Progress::Chunk { + current, .. + } => { + progress_bar.set_position(current); } - DownloadState::Complete => progress_bar.finish(), + Progress::Complete { + .. + } => progress_bar.finish(), _ => {} } } pub fn handle_install_progress( - state: DownloadState, + state: Progress, progress_bar: &mut Option, ctx: &InstallContext, package: &Package, @@ -73,22 +79,28 @@ pub fn handle_install_progress( } match state { - DownloadState::Preparing(total) => { + Progress::Starting { + total, + } => { if let Some(pb) = progress_bar { pb.set_length(total); } } - DownloadState::Progress(progress) => { + Progress::Chunk { + current, .. + } => { if let Some(pb) = progress_bar { - pb.set_position(progress); + pb.set_position(current); } } - DownloadState::Complete => { + Progress::Complete { + .. + } => { if let Some(pb) = progress_bar.take() { pb.finish(); } } - DownloadState::Error => { + Progress::Error => { let count = ctx.retrying.fetch_add(1, Ordering::Relaxed); let failed_count = ctx.failed.load(Ordering::Relaxed); ctx.total_progress_bar.set_message(format!( @@ -101,7 +113,7 @@ pub fn handle_install_progress( }, )); } - DownloadState::Aborted => { + Progress::Aborted => { let failed_count = ctx.failed.fetch_add(1, Ordering::Relaxed); if let Some(pb) = progress_bar { let prefix = format!( @@ -130,7 +142,7 @@ pub fn handle_install_progress( } } } - DownloadState::Recovered => { + Progress::Recovered => { let count = ctx.retrying.fetch_sub(1, Ordering::Relaxed); let failed_count = ctx.failed.load(Ordering::Relaxed); if count > 1 || failed_count > 0 { diff --git a/soar-cli/src/run.rs b/soar-cli/src/run.rs index 35f03d31c..5ab3f4c35 100644 --- a/soar-cli/src/run.rs +++ b/soar-cli/src/run.rs @@ -1,4 +1,4 @@ -use std::{fs, path::PathBuf, process::Command, sync::Arc}; +use std::{fs, process::Command, sync::Arc}; use soar_core::{ database::{ @@ -10,10 +10,7 @@ use soar_core::{ utils::get_extract_dir, SoarResult, }; -use soar_dl::{ - downloader::{DownloadOptions, Downloader, OciDownloadOptions, OciDownloader}, - utils::FileMode, -}; +use soar_dl::{download::Download, oci::OciDownload, types::OverwriteMode}; use soar_utils::hash::calculate_checksum; use crate::{ @@ -83,46 +80,37 @@ pub async fn run_package( }); if let Some(url) = package.ghcr_blob { - let options = OciDownloadOptions { - url: url.to_string(), - output_path: Some(output_path.to_string_lossy().to_string()), - progress_callback: Some(progress_callback.clone()), - api: None, - concurrency: Some(1), - regexes: Vec::new(), - globs: Vec::new(), - exclude_keywords: Vec::new(), - match_keywords: Vec::new(), - exact_case: false, - file_mode: FileMode::ForceOverwrite, - }; - - let mut downloader = OciDownloader::new(options); - - downloader.download_oci().await?; + let mut dl = OciDownload::new(url.as_str()) + .output(output_path.to_string_lossy()) + .overwrite(OverwriteMode::Force); + let cb = progress_callback.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + + dl.execute()?; } else { - let downloader = Downloader::default(); let extract_dir = get_extract_dir(&cache_bin); - let options = DownloadOptions { - url: package.download_url.clone(), - output_path: Some(output_path.to_string_lossy().to_string()), - progress_callback: Some(progress_callback), - extract_archive: true, - extract_dir: Some(extract_dir.to_string_lossy().to_string()), - file_mode: FileMode::ForceOverwrite, - prompt: None, - }; - - let file_name = downloader.download(options).await?; - let extract_path = PathBuf::from(&extract_dir); - if extract_path.exists() { + let mut dl = Download::new(&package.download_url) + .output(output_path.to_string_lossy()) + .overwrite(OverwriteMode::Force) + .extract(true) + .extract_to(&extract_dir); + + let cb = progress_callback.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + + let file_name = dl.execute()?; + if extract_dir.exists() { fs::remove_file(file_name).ok(); - for entry in fs::read_dir(&extract_path) - .with_context(|| format!("reading {} directory", extract_path.display()))? + for entry in fs::read_dir(&extract_dir) + .with_context(|| format!("reading {} directory", extract_dir.display()))? { let entry = entry.with_context(|| { - format!("reading entry from directory {}", extract_path.display()) + format!("reading entry from directory {}", extract_dir.display()) })?; let from = entry.path(); let to = cache_bin.join(entry.file_name()); @@ -131,7 +119,7 @@ pub async fn run_package( })?; } - fs::remove_dir_all(&extract_path).ok(); + fs::remove_dir_all(&extract_dir).ok(); } } diff --git a/soar-cli/src/self_actions.rs b/soar-cli/src/self_actions.rs index a7c1f6cc8..a2f09ea0c 100644 --- a/soar-cli/src/self_actions.rs +++ b/soar-cli/src/self_actions.rs @@ -4,12 +4,15 @@ use std::{ }; use semver::Version; -use soar_core::{error::ErrorContext, SoarResult}; +use soar_core::{ + error::{ErrorContext, SoarError}, + SoarResult, +}; use soar_dl::{ - downloader::{DownloadOptions, Downloader}, - github::{Github, GithubRelease}, - platform::{Release, ReleaseAsset, ReleaseHandler}, - utils::FileMode, + download::Download, + github::Github, + traits::{Asset as _, Platform as _, Release as _}, + types::{OverwriteMode, Progress}, }; use tracing::{debug, error, info}; @@ -33,28 +36,24 @@ pub async fn process_self_action(action: &SelfAction) -> SoarResult<()> { _ => is_nightly, }; - let handler = ReleaseHandler::::new(); - let releases = handler - .fetch_releases::("pkgforge/soar", None) - .await?; - - let release = releases.iter().find(|rel| { - let is_nightly_release = rel.tag_name().starts_with("nightly"); + let releases = Github::fetch_releases("pkgforge/soar", None)?; + let release = releases.iter().find(|release| { + let is_nightly_release = release.tag().starts_with("nightly"); debug!( "Checking release: {}, Release Channel: {}", - rel.tag_name(), + release.tag(), if is_nightly_release { "nightly" } else { "stable" } ); + if target_nightly { - is_nightly_release && rel.name() != self_version + is_nightly_release && release.name() != self_version } else { - let release_version = rel.tag_name().trim_start_matches("v"); - + let release_version = release.tag().trim_start_matches("v"); let parsed_release_version = Version::parse(release_version).ok(); let parsed_self_version = Version::parse(self_version).ok(); @@ -87,7 +86,7 @@ pub async fn process_self_action(action: &SelfAction) -> SoarResult<()> { if target_nightly { "nightly" } else { "stable" } ); } else { - info!("Found new update: {}", release.tag_name()); + info!("Found new update: {}", release.tag()); } let assets = release.assets(); let asset = assets @@ -95,24 +94,43 @@ pub async fn process_self_action(action: &SelfAction) -> SoarResult<()> { .find(|a| { a.name.contains(ARCH) && !a.name.contains("tar") && !a.name.contains("sum") }) - .unwrap(); - - debug!("Selected asset: {}", asset.name); - - let downloader = Downloader::default(); - let options = DownloadOptions { - url: asset.download_url().to_string(), - output_path: Some(self_bin.to_string_lossy().to_string()), - progress_callback: None, - extract_archive: false, - extract_dir: None, - file_mode: FileMode::ForceOverwrite, - prompt: None, - }; + .ok_or_else(|| { + SoarError::Custom(format!("No matching asset fund for {}", ARCH)) + })?; + + debug!("Selected asset: {}", asset.name()); + + let dl = Download::new(asset.url()) + .output(self_bin.to_string_lossy()) + .overwrite(OverwriteMode::Force) + .progress(|p| { + match p { + Progress::Starting { + total, + } => { + info!("Downloading update ({} bytes)...", total); + } + Progress::Chunk { + current, + total, + } => { + if current % (1024 * 1024) == 0 { + let pct = (current as f64 / total as f64 * 100.0) as u8; + debug!("Progress: {}%", pct); + } + } + Progress::Complete { + .. + } => { + debug!("Download complete"); + } + _ => {} + } + }); - debug!("Downloading update from: {}", options.url); - downloader.download(options).await?; - info!("Soar updated to {}", release.tag_name()); + debug!("Downloading update from: {}", asset.url()); + dl.execute()?; + info!("Soar updated to {}", release.tag()); } else { eprintln!("No updates found."); } diff --git a/soar-cli/src/utils.rs b/soar-cli/src/utils.rs index 3cb73cb8a..319a37904 100644 --- a/soar-cli/src/utils.rs +++ b/soar-cli/src/utils.rs @@ -22,8 +22,7 @@ use soar_core::{ repositories::get_platform_repositories, SoarResult, }; -use soar_dl::utils::{is_elf, FileMode}; -use soar_utils::system::platform; +use soar_utils::{fs::is_elf, system::platform}; use tracing::{error, info}; pub static COLOR: LazyLock> = LazyLock::new(|| RwLock::new(true)); @@ -159,16 +158,6 @@ pub fn ask_target_action(targets: &[InstallTarget], action: &str) -> SoarResult< Ok(()) } -pub fn get_file_mode(skip_existing: bool, force_overwrite: bool) -> FileMode { - if force_overwrite { - FileMode::ForceOverwrite - } else if skip_existing { - FileMode::SkipExisting - } else { - FileMode::PromptOverwrite - } -} - pub async fn mangle_package_symlinks( install_dir: &Path, bin_dir: &Path, @@ -240,7 +229,7 @@ pub async fn mangle_package_symlinks( ) })? .path(); - if path.is_file() && (is_syms || is_elf(&path).await) { + if path.is_file() && (is_syms || is_elf(&path)) { if let Some(file_name) = path.file_name() { let symlink_target_path = bin_dir.join(file_name); if symlink_target_path.is_symlink() || symlink_target_path.is_file() { diff --git a/soar-core/src/config.rs b/soar-core/src/config.rs index 6ae4df12e..94bd073ce 100644 --- a/soar-core/src/config.rs +++ b/soar-core/src/config.rs @@ -6,6 +6,7 @@ use std::{ }; use documented::{Documented, DocumentedFields}; +use rusqlite::Connection; use serde::{de::Error, Deserialize, Serialize}; use soar_utils::{ path::{resolve_path, xdg_config_home, xdg_data_home}, @@ -23,7 +24,6 @@ use crate::{ utils::default_install_patterns, SoarResult, }; -use rusqlite::Connection; type Result = std::result::Result; @@ -184,7 +184,7 @@ pub struct Config { /// Maximum number of concurrent requests for GHCR (GitHub Container Registry). /// Default: 8 - pub ghcr_concurrency: Option, + pub ghcr_concurrency: Option, /// Limits the number of results returned by a search. /// Default: 20 @@ -357,10 +357,12 @@ impl Config { let config_path = CONFIG_PATH.read().unwrap().to_path_buf(); let mut config = match fs::read_to_string(&config_path) { - Ok(content) => match toml::from_str(&content) { - Ok(c) => Ok(c), - Err(err) => Err(ConfigError::TomlDeError(err)), - }, + Ok(content) => { + match toml::from_str(&content) { + Ok(c) => Ok(c), + Err(err) => Err(ConfigError::TomlDeError(err)), + } + } Err(e) if e.kind() == std::io::ErrorKind::NotFound => { Ok(Self::default_config::<&str>(false, &[])) } diff --git a/soar-core/src/database/connection.rs b/soar-core/src/database/connection.rs index 149c77317..aaa97de12 100644 --- a/soar-core/src/database/connection.rs +++ b/soar-core/src/database/connection.rs @@ -5,9 +5,8 @@ use std::{ use rusqlite::Connection; -use crate::error::SoarError; - use super::{models::RemotePackage, repository::PackageRepository, statements::DbStatements}; +use crate::error::SoarError; type Result = std::result::Result; @@ -20,7 +19,9 @@ impl Database { let path = path.as_ref(); let conn = Connection::open(path)?; let conn = Arc::new(Mutex::new(conn)); - Ok(Database { conn }) + Ok(Database { + conn, + }) } pub fn new_multi>(paths: &[P]) -> Result { @@ -36,7 +37,9 @@ impl Database { conn.execute(&format!("PRAGMA shard{idx}.case_sensitive_like = ON;"), [])?; } let conn = Arc::new(Mutex::new(conn)); - Ok(Database { conn }) + Ok(Database { + conn, + }) } pub fn from_remote_metadata(&self, metadata: &[RemotePackage], repo_name: &str) -> Result<()> { diff --git a/soar-core/src/database/migration.rs b/soar-core/src/database/migration.rs index ebf808227..e23ef6a95 100644 --- a/soar-core/src/database/migration.rs +++ b/soar-core/src/database/migration.rs @@ -21,7 +21,9 @@ pub enum DbKind { impl MigrationManager { pub fn new(conn: Connection) -> rusqlite::Result { - Ok(Self { conn }) + Ok(Self { + conn, + }) } fn get_current_version(&self) -> rusqlite::Result { @@ -69,7 +71,10 @@ impl MigrationManager { let sql = entry.contents_utf8().unwrap().to_string(); - migrations.push(Migration { version, sql }); + migrations.push(Migration { + version, + sql, + }); } } diff --git a/soar-core/src/database/models.rs b/soar-core/src/database/models.rs index 2d73afeeb..f589569bf 100644 --- a/soar-core/src/database/models.rs +++ b/soar-core/src/database/models.rs @@ -285,15 +285,19 @@ where { match Option::::deserialize(deserializer)? { Some(FlexiBool::Bool(b)) => Ok(Some(b)), - Some(FlexiBool::String(s)) => match s.to_lowercase().as_str() { - "true" | "yes" | "1" => Ok(Some(true)), - "false" | "no" | "0" => Ok(Some(false)), - "" => Ok(None), - _ => Err(de::Error::invalid_value( - de::Unexpected::Str(&s), - &"a valid boolean (true/false, yes/no, 1/0)", - )), - }, + Some(FlexiBool::String(s)) => { + match s.to_lowercase().as_str() { + "true" | "yes" | "1" => Ok(Some(true)), + "false" | "no" | "0" => Ok(Some(false)), + "" => Ok(None), + _ => { + Err(de::Error::invalid_value( + de::Unexpected::Str(&s), + &"a valid boolean (true/false, yes/no, 1/0)", + )) + } + } + } None => Ok(None), } } diff --git a/soar-core/src/database/nests/repository.rs b/soar-core/src/database/nests/repository.rs index efd4e20de..7aa2002c3 100644 --- a/soar-core/src/database/nests/repository.rs +++ b/soar-core/src/database/nests/repository.rs @@ -1,8 +1,7 @@ use rusqlite::{params, Result}; -use crate::{database::models::FromRow, error::SoarError, SoarResult}; - use super::models::Nest; +use crate::{database::models::FromRow, error::SoarError, SoarResult}; pub fn add(tx: &rusqlite::Transaction, nest: &Nest) -> Result<()> { tx.execute( @@ -16,11 +15,13 @@ pub fn list(tx: &rusqlite::Transaction) -> Result> { let mut stmt = tx.prepare("SELECT id, name, url FROM nests")?; let nests = stmt .query_map([], Nest::from_row)? - .filter_map(|n| match n { - Ok(nest) => Some(nest), - Err(err) => { - eprintln!("Nest map error: {err:#?}"); - None + .filter_map(|n| { + match n { + Ok(nest) => Some(nest), + Err(err) => { + eprintln!("Nest map error: {err:#?}"); + None + } } }) .collect(); diff --git a/soar-core/src/database/packages/query.rs b/soar-core/src/database/packages/query.rs index 03b8e499f..136760b95 100644 --- a/soar-core/src/database/packages/query.rs +++ b/soar-core/src/database/packages/query.rs @@ -2,14 +2,13 @@ use std::sync::{Arc, Mutex}; use rusqlite::{Connection, ToSql}; +use super::{FilterCondition, LogicalOp, PaginatedResponse, QueryFilter, SortDirection}; use crate::{ database::models::{FromRow, InstalledPackage}, error::SoarError, SoarResult, }; -use super::{FilterCondition, LogicalOp, PaginatedResponse, QueryFilter, SortDirection}; - #[derive(Debug, Clone)] pub struct PackageQueryBuilder { db: Arc>, @@ -147,11 +146,13 @@ impl PackageQueryBuilder { let items = stmt .query_map(params_ref.as_slice(), T::from_row)? - .filter_map(|r| match r { - Ok(pkg) => Some(pkg), - Err(err) => { - eprintln!("Package map error: {err:#?}"); - None + .filter_map(|r| { + match r { + Ok(pkg) => Some(pkg), + Err(err) => { + eprintln!("Package map error: {err:#?}"); + None + } } }) .collect(); @@ -355,11 +356,13 @@ impl PackageQueryBuilder { .collect(); let items = stmt .query_map(params_ref.as_slice(), InstalledPackage::from_row)? - .filter_map(|r| match r { - Ok(pkg) => Some(pkg), - Err(err) => { - eprintln!("Installed package map error: {err:#?}"); - None + .filter_map(|r| { + match r { + Ok(pkg) => Some(pkg), + Err(err) => { + eprintln!("Installed package map error: {err:#?}"); + None + } } }) .collect(); diff --git a/soar-core/src/error.rs b/soar-core/src/error.rs index 08acfb434..757839aaa 100644 --- a/soar-core/src/error.rs +++ b/soar-core/src/error.rs @@ -1,5 +1,6 @@ -use soar_utils::error::{FileSystemError, HashError, PathError}; use std::error::Error; + +use soar_utils::error::{FileSystemError, HashError, PathError}; use thiserror::Error; #[derive(Error, Debug)] @@ -82,9 +83,6 @@ pub enum SoarError { #[error("Download failed: {0}")] DownloadError(#[from] soar_dl::error::DownloadError), - #[error("{0}")] - PlatformError(soar_dl::error::PlatformError), - #[error("Squashy Error: {0}")] SquishyError(#[from] squishy::error::SquishyError), @@ -132,16 +130,20 @@ impl SoarError { pub fn root_cause(&self) -> String { match self { - Self::ReqwestError(e) => format!( - "Root cause: {}", - e.source() - .map_or_else(|| e.to_string(), |source| source.to_string()) - ), - Self::RusqliteError(e) => format!( - "Root cause: {}", - e.source() - .map_or_else(|| e.to_string(), |source| source.to_string()) - ), + Self::ReqwestError(e) => { + format!( + "Root cause: {}", + e.source() + .map_or_else(|| e.to_string(), |source| source.to_string()) + ) + } + Self::RusqliteError(e) => { + format!( + "Root cause: {}", + e.source() + .map_or_else(|| e.to_string(), |source| source.to_string()) + ) + } Self::Config(err) => err.to_string(), _ => self.to_string(), } @@ -154,12 +156,6 @@ impl From> for SoarError { } } -impl From for SoarError { - fn from(value: soar_dl::error::PlatformError) -> Self { - Self::PlatformError(value) - } -} - pub trait ErrorContext { fn with_context(self, context: C) -> Result where @@ -171,9 +167,11 @@ impl ErrorContext for std::io::Result { where C: FnOnce() -> String, { - self.map_err(|err| SoarError::IoError { - action: context(), - source: err, + self.map_err(|err| { + SoarError::IoError { + action: context(), + source: err, + } }) } } diff --git a/soar-core/src/package/formats/appimage.rs b/soar-core/src/package/formats/appimage.rs index f875f10c1..02d03d2e5 100644 --- a/soar-core/src/package/formats/appimage.rs +++ b/soar-core/src/package/formats/appimage.rs @@ -3,12 +3,11 @@ use std::{fs, path::Path}; use soar_utils::fs::read_file_signature; use squishy::{appimage::AppImage, EntryKind}; +use super::common::{symlink_desktop, symlink_icon}; use crate::{ constants::PNG_MAGIC_BYTES, database::models::PackageExt, error::ErrorContext, SoarResult, }; -use super::common::{symlink_desktop, symlink_icon}; - pub async fn integrate_appimage, T: PackageExt>( install_dir: P, file_path: P, diff --git a/soar-core/src/package/formats/common.rs b/soar-core/src/package/formats/common.rs index 13da0080c..da69dbb6c 100644 --- a/soar-core/src/package/formats/common.rs +++ b/soar-core/src/package/formats/common.rs @@ -8,14 +8,16 @@ use std::{ use image::{imageops::FilterType, DynamicImage, GenericImageView}; use regex::Regex; -use soar_dl::downloader::{DownloadOptions, Downloader}; -use soar_dl::utils::FileMode; +use soar_dl::{download::Download, types::OverwriteMode}; use soar_utils::{ error::FileSystemResult, fs::{create_symlink, read_file_signature, walk_dir}, path::{desktop_dir, icons_dir}, }; +use super::{ + appimage::integrate_appimage, get_file_type, wrappe::setup_wrappe_portable_dir, PackageFormat, +}; use crate::{ config::get_config, constants::PNG_MAGIC_BYTES, @@ -24,10 +26,6 @@ use crate::{ SoarResult, }; -use super::{ - appimage::integrate_appimage, get_file_type, wrappe::setup_wrappe_portable_dir, PackageFormat, -}; - const SUPPORTED_DIMENSIONS: &[(u32, u32)] = &[ (16, 16), (24, 24), @@ -113,20 +111,22 @@ pub fn symlink_desktop, T: PackageExt>( let final_content = { let re = Regex::new(r"(?m)^(Icon|Exec|TryExec)=(.*)").unwrap(); - re.replace_all(&content, |caps: ®ex::Captures| match &caps[1] { - "Icon" => format!("Icon={}-soar", file_name.to_string_lossy()), - "Exec" | "TryExec" => { - let value = &caps[0]; - let bin_path = get_config().get_bin_path().unwrap(); - let new_value = format!("{}/{}", &bin_path.display(), pkg_name); - - if value.contains("{{pkg_path}}") { - value.replace("{{pkg_path}}", &new_value) - } else { - format!("{}={}", &caps[1], new_value) + re.replace_all(&content, |caps: ®ex::Captures| { + match &caps[1] { + "Icon" => format!("Icon={}-soar", file_name.to_string_lossy()), + "Exec" | "TryExec" => { + let value = &caps[0]; + let bin_path = get_config().get_bin_path().unwrap(); + let new_value = format!("{}/{}", &bin_path.display(), pkg_name); + + if value.contains("{{pkg_path}}") { + value.replace("{{pkg_path}}", &new_value) + } else { + format!("{}={}", &caps[1], new_value) + } } + _ => unreachable!(), } - _ => unreachable!(), }) .to_string() }; @@ -156,19 +156,11 @@ pub async fn integrate_remote>( let mut icon_output_path = package_path.join(".DirIcon"); let desktop_output_path = package_path.join(format!("{}.desktop", package.pkg_name)); - let downloader = Downloader::default(); - if let Some(icon_url) = icon_url { - let options = DownloadOptions { - url: icon_url.clone(), - output_path: Some(icon_output_path.to_string_lossy().to_string()), - progress_callback: None, - extract_archive: false, - extract_dir: None, - file_mode: FileMode::SkipExisting, - prompt: None, - }; - downloader.download(options).await?; + Download::new(icon_url) + .output(icon_output_path.to_string_lossy()) + .overwrite(OverwriteMode::Skip) + .execute()?; let ext = if read_file_signature(icon_output_path, 8)? == PNG_MAGIC_BYTES { "png" @@ -179,16 +171,10 @@ pub async fn integrate_remote>( } if let Some(desktop_url) = desktop_url { - let options = DownloadOptions { - url: desktop_url.clone(), - output_path: Some(desktop_output_path.to_string_lossy().to_string()), - progress_callback: None, - extract_archive: false, - extract_dir: None, - file_mode: FileMode::SkipExisting, - prompt: None, - }; - downloader.download(options).await?; + Download::new(desktop_url) + .output(desktop_output_path.to_string_lossy()) + .overwrite(OverwriteMode::Skip) + .execute()?; } else { let content = create_default_desktop_entry(&package.pkg_name, "Utility"); fs::write(&desktop_output_path, &content).with_context(|| { diff --git a/soar-core/src/package/formats/wrappe.rs b/soar-core/src/package/formats/wrappe.rs index abc69d25b..42a2a31be 100644 --- a/soar-core/src/package/formats/wrappe.rs +++ b/soar-core/src/package/formats/wrappe.rs @@ -1,8 +1,7 @@ use std::path::{Path, PathBuf}; -use crate::SoarResult; - use super::common::create_portable_link; +use crate::SoarResult; pub fn setup_wrappe_portable_dir>( bin_path: P, diff --git a/soar-core/src/package/install.rs b/soar-core/src/package/install.rs index 092ef1512..851de4dbb 100644 --- a/soar-core/src/package/install.rs +++ b/soar-core/src/package/install.rs @@ -6,12 +6,13 @@ use std::{ time::Duration, }; -use reqwest::StatusCode; use rusqlite::{params, prepare_and_bind, Connection}; use soar_dl::{ - downloader::{DownloadOptions, DownloadState, Downloader, OciDownloadOptions, OciDownloader}, + download::Download, error::DownloadError, - utils::FileMode, + filter::Filter, + oci::OciDownload, + types::{OverwriteMode, Progress}, }; use soar_utils::{ error::FileSystemResult, @@ -34,7 +35,7 @@ use crate::{ pub struct PackageInstaller { package: Package, install_dir: PathBuf, - progress_callback: Option>, + progress_callback: Option>, db: Arc>, with_pkg_id: bool, globs: Vec, @@ -57,7 +58,7 @@ impl PackageInstaller { pub async fn new>( target: &InstallTarget, install_dir: P, - progress_callback: Option>, + progress_callback: Option>, db: Arc>, with_pkg_id: bool, globs: Vec, @@ -119,74 +120,87 @@ impl PackageInstaller { }; if self.package.ghcr_pkg.is_some() { - let progress_callback = &self.progress_callback.clone(); - let options = OciDownloadOptions { - url: url.to_string(), - output_path: Some(output_path.to_string_lossy().to_string()), - progress_callback: self.progress_callback.clone(), - api: None, - concurrency: Some(get_config().ghcr_concurrency.unwrap_or(8)), - regexes: vec![], - exclude_keywords: vec![], - match_keywords: vec![], - exact_case: true, - globs: self.globs.clone(), - file_mode: FileMode::SkipExisting, - }; - let mut downloader = OciDownloader::new(options); + let mut dl = OciDownload::new(url.as_str()) + .output(output_path.to_string_lossy()) + .parallel(get_config().ghcr_concurrency.unwrap_or(8)) + .overwrite(OverwriteMode::Skip); + + if let Some(ref cb) = self.progress_callback { + let cb = cb.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + } + + if !self.globs.is_empty() { + dl = dl.filter(Filter { + globs: self.globs.clone(), + ..Default::default() + }); + } + let mut retries = 0; loop { if retries > 5 { - if let Some(ref callback) = progress_callback { - callback(DownloadState::Aborted); + if let Some(ref callback) = self.progress_callback { + callback(Progress::Complete { + total: 0, + }); } break; } - match downloader.download_oci().await { + match dl.clone().execute() { Ok(_) => break, - Err( - DownloadError::ResourceError { - status: StatusCode::TOO_MANY_REQUESTS, - .. + Err(err) => { + if matches!( + err, + DownloadError::HttpError { + status: 429, + .. + } | DownloadError::Network(_) + ) { + sleep(Duration::from_secs(5)); + retries += 1; + if retries > 1 { + if let Some(ref callback) = self.progress_callback { + callback(Progress::Error); + } + } + } else { + return Err(err)?; } - | DownloadError::ChunkError, - ) => sleep(Duration::from_secs(5)), - Err(err) => return Err(err)?, - }; - retries += 1; - if retries > 1 { - continue; - } - if let Some(ref callback) = progress_callback { - callback(DownloadState::Error); + } } } Ok(None) } else { - let downloader = Downloader::default(); let extract_dir = get_extract_dir(&self.install_dir); - let options = DownloadOptions { - url: url.to_string(), - output_path: Some(output_path.to_string_lossy().to_string()), - progress_callback: self.progress_callback.clone(), - extract_archive: true, - file_mode: FileMode::SkipExisting, - extract_dir: Some(extract_dir.to_string_lossy().to_string()), - prompt: None, - }; - let file_name = downloader.download(options).await?; + let mut dl = Download::new(url.as_str()) + .output(output_path.to_string_lossy()) + .overwrite(OverwriteMode::Skip) + .extract(true) + .extract_to(&extract_dir); + + if let Some(ref cb) = self.progress_callback { + let cb = cb.clone(); + dl = dl.progress(move |p| { + cb(p); + }); + } + + let file_path = dl.execute()?; - let checksum = if PathBuf::from(&file_name).exists() { - Some(calculate_checksum(&file_name)?) + let checksum = if PathBuf::from(&file_path).exists() { + Some(calculate_checksum(&file_path)?) } else { None }; let extract_path = PathBuf::from(&extract_dir); if extract_path.exists() { - fs::remove_file(file_name).ok(); + fs::remove_file(file_path).ok(); for entry in fs::read_dir(&extract_path) .with_context(|| format!("reading {} directory", extract_path.display()))? diff --git a/soar-core/src/package/remove.rs b/soar-core/src/package/remove.rs index 7972b51b0..60e3b287d 100644 --- a/soar-core/src/package/remove.rs +++ b/soar-core/src/package/remove.rs @@ -22,7 +22,10 @@ pub struct PackageRemover { impl PackageRemover { pub async fn new(package: InstalledPackage, db: Arc>) -> Self { - Self { package, db } + Self { + package, + db, + } } pub async fn remove(&self) -> SoarResult<()> { diff --git a/soar-core/src/toml.rs b/soar-core/src/toml.rs index dd304402d..0dd9e3d75 100644 --- a/soar-core/src/toml.rs +++ b/soar-core/src/toml.rs @@ -63,22 +63,26 @@ where for (mut key_mut, value_item) in table.iter_mut() { let key_str = key_mut.get(); match T::get_field_docs(key_str) { - Ok(docs) => match value_item { - Item::None => { - return Err(ConfigError::Custom(format!( - "Encountered TomlEditItem::None for key '{key_str}' unexpectedly", - ))) + Ok(docs) => { + match value_item { + Item::None => { + return Err(ConfigError::Custom(format!( + "Encountered TomlEditItem::None for key '{key_str}' unexpectedly", + ))) + } + Item::Value(_) => append_docs_as_toml_comments(key_mut.leaf_decor_mut(), docs), + Item::Table(sub_table) => { + append_docs_as_toml_comments(sub_table.decor_mut(), docs) + } + Item::ArrayOfTables(array) => { + let first_table = array + .iter_mut() + .next() + .expect("Array of table should not be empty"); + append_docs_as_toml_comments(first_table.decor_mut(), docs); + } } - Item::Value(_) => append_docs_as_toml_comments(key_mut.leaf_decor_mut(), docs), - Item::Table(sub_table) => append_docs_as_toml_comments(sub_table.decor_mut(), docs), - Item::ArrayOfTables(array) => { - let first_table = array - .iter_mut() - .next() - .expect("Array of table should not be empty"); - append_docs_as_toml_comments(first_table.decor_mut(), docs); - } - }, + } Err(_) => { warn!( "Field '{}' found in TOML but not in struct '{}' for documentation lookup, or it's a complex case like flatten.", From 7b9a25981e2b132c08d0e0a5a29c73fa84244cc3 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Sat, 1 Nov 2025 18:24:59 +0545 Subject: [PATCH 14/22] fix gitlab download --- soar-cli/src/download.rs | 59 ++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/soar-cli/src/download.rs b/soar-cli/src/download.rs index d5be03afc..a0631e801 100644 --- a/soar-cli/src/download.rs +++ b/soar-cli/src/download.rs @@ -16,7 +16,6 @@ use soar_dl::{ gitlab::GitLab, oci::OciDownload, platform::PlatformUrl, - release::ReleaseDownload, traits::{Asset, Platform as _, Release as _}, types::{OverwriteMode, Progress}, }; @@ -344,15 +343,52 @@ fn handle_gitlab_release( project: &str, tag: Option<&str>, ) -> SoarResult<()> { - let mut dl = ReleaseDownload::::new(project) - .filter(ctx.create_filter()) - .overwrite(ctx.get_overwrite_mode()) - .extract(ctx.extract); + let releases = GitLab::fetch_releases(project, tag)?; - if let Some(tag) = tag { - dl = dl.tag(tag); + let release = if let Some(tag) = tag { + releases.iter().find(|r| r.tag() == tag) + } else { + releases + .iter() + .find(|r| !r.is_prerelease()) + .or_else(|| releases.first()) + }; + + let release = release.ok_or_else(|| DownloadError::InvalidResponse)?; + + info!("Found release: {}", release.tag()); + let filter = ctx.create_filter(); + + let assets: Vec<_> = release + .assets() + .iter() + .filter(|a| filter.matches(a.name())) + .collect(); + + if assets.is_empty() { + let available = release + .assets() + .iter() + .map(|a| a.name().to_string()) + .collect::>(); + + Err(DownloadError::NoMatch { + available, + })? } + let selected_asset = if assets.len() == 1 || ctx.yes { + assets[0] + } else { + &select_asset_interactively(assets)? + }; + + info!("Downloading asset: {}", selected_asset.name()); + + let mut dl = Download::new(selected_asset.url()) + .overwrite(ctx.get_overwrite_mode()) + .extract(ctx.extract); + if let Some(ref out) = ctx.output { dl = dl.output(out); } @@ -366,14 +402,7 @@ fn handle_gitlab_release( cb(p); }); - let paths = dl.execute()?; - - if paths.len() > 1 && !ctx.yes { - info!("Multiple assets found, please select one:"); - for (i, path) in paths.iter().enumerate() { - info!("{}. {}", i + 1, path.display()); - } - } + dl.execute()?; Ok(()) } From 1ca6499225c5e936cd2c50f09c0785da60aaa5ee Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Sat, 1 Nov 2025 20:17:03 +0545 Subject: [PATCH 15/22] fix issues --- crates/soar-dl/src/download.rs | 18 ++++--- crates/soar-dl/src/gitlab.rs | 25 ++++++++-- soar-cli/src/download.rs | 25 +++++----- soar-cli/src/main.rs | 2 +- soar-core/src/error.rs | 3 ++ soar-core/src/package/formats/common.rs | 62 +------------------------ soar-core/src/package/install.rs | 4 +- 7 files changed, 50 insertions(+), 89 deletions(-) diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs index 5e34a1c74..67b6db3f6 100644 --- a/crates/soar-dl/src/download.rs +++ b/crates/soar-dl/src/download.rs @@ -196,13 +196,17 @@ impl Download { return self.download_to_stdout(); } - let resp = Http::head(&self.url)?; - - let header_filename = resp - .headers() - .get(CONTENT_DISPOSITION) - .and_then(filename_from_header); - let url_filename = filename_from_url(&self.url); + let (header_filename, url_filename) = if self.output.as_deref().is_none() { + let resp = Http::head(&self.url)?; + ( + resp.headers() + .get(CONTENT_DISPOSITION) + .and_then(filename_from_header), + filename_from_url(&self.url), + ) + } else { + (None, filename_from_url(&self.url)) + }; let output_path = resolve_output_path(self.output.as_deref(), url_filename, header_filename)?; diff --git a/crates/soar-dl/src/gitlab.rs b/crates/soar-dl/src/gitlab.rs index c44da51e7..6ec1dc5e9 100644 --- a/crates/soar-dl/src/gitlab.rs +++ b/crates/soar-dl/src/gitlab.rs @@ -90,16 +90,31 @@ impl Platform for GitLab { impl Release for GitLabRelease { type Asset = GitLabAsset; - /// Gets the asset's name. + /// The release's name, or an empty string if the release has no name. /// /// # Examples /// /// ``` - /// use soar_dl::gitlab::GitLabAsset; - /// use soar_dl::traits::Asset; + /// use soar_dl::gitlab::{GitlabAssets, GitLabRelease}; + /// use soar_dl::traits::Release; /// - /// let asset = GitLabAsset { name: String::from("v1.0.0"), direct_asset_url: String::from("https://example") }; - /// assert_eq!(asset.name(), "v1.0.0"); + /// let r = GitLabRelease { + /// name: Some("v1.0".into()), + /// tag_name: "v1.0".into(), + /// upcoming_release: false, + /// released_at: "".into(), + /// assets: GitlabAssets { links: vec![] }, + /// }; + /// assert_eq!(r.name(), "v1.0"); + /// + /// let unnamed = GitLabRelease { + /// name: None, + /// tag_name: "v1.1".into(), + /// upcoming_release: false, + /// released_at: "".into(), + /// assets: GitlabAssets { links: vec![] }, + /// }; + /// assert_eq!(unnamed.name(), ""); /// ``` fn name(&self) -> &str { &self.name diff --git a/soar-cli/src/download.rs b/soar-cli/src/download.rs index a0631e801..91600b92e 100644 --- a/soar-cli/src/download.rs +++ b/soar-cli/src/download.rs @@ -1,4 +1,4 @@ -use std::{sync::Arc, thread::sleep, time::Duration}; +use std::{sync::Arc, time::Duration}; use indicatif::HumanBytes; use regex::Regex; @@ -19,6 +19,7 @@ use soar_dl::{ traits::{Asset, Platform as _, Release as _}, types::{OverwriteMode, Progress}, }; +use tokio::time::sleep; use tracing::{error, info}; use crate::{ @@ -246,12 +247,12 @@ async fn handle_oci_download(ctx: &DownloadContext, reference: &str) -> SoarResu retries += 1; info!("Retrying... ({}/{})", retries, max_retries); ctx.progress_callback.clone()(Progress::Recovered); - sleep(Duration::from_secs(5)); + sleep(Duration::from_secs(5)).await; } Err(err) => { ctx.progress_callback.clone()(Progress::Error); error!("Download failed: {}", err); - return Err(err)?; + return Err(err.into()); } } } @@ -407,18 +408,16 @@ fn handle_gitlab_release( Ok(()) } -pub fn create_regex_patterns(regex_patterns: Option>) -> Vec { - regex_patterns - .clone() - .map(|patterns| { +pub fn create_regex_patterns(regex_patterns: Option>) -> SoarResult> { + match regex_patterns { + Some(patterns) => { patterns .iter() - .map(|pattern| Regex::new(pattern)) - .collect::, regex::Error>>() - }) - .transpose() - .unwrap() - .unwrap_or_default() + .map(|pattern| Regex::new(pattern).map_err(|err| err.into())) + .collect() + } + None => Ok(Vec::new()), + } } pub async fn handle_github_downloads( diff --git a/soar-cli/src/main.rs b/soar-cli/src/main.rs index e9198ff74..662bb4a29 100644 --- a/soar-cli/src/main.rs +++ b/soar-cli/src/main.rs @@ -256,7 +256,7 @@ async fn handle_cli() -> SoarResult<()> { let progress_bar = create_progress_bar(); let progress_callback = Arc::new(move |state| progress::handle_progress(state, &progress_bar)); - let regexes = create_regex_patterns(regexes); + let regexes = create_regex_patterns(regexes)?; let globs = globs.unwrap_or_default(); let match_keywords = match_keywords.unwrap_or_default(); let exclude_keywords = exclude_keywords.unwrap_or_default(); diff --git a/soar-core/src/error.rs b/soar-core/src/error.rs index 757839aaa..e8b5380ce 100644 --- a/soar-core/src/error.rs +++ b/soar-core/src/error.rs @@ -121,6 +121,9 @@ pub enum SoarError { #[error("{0}")] Warning(String), + + #[error("Regex compilation error: {0}")] + RegexError(#[from] regex::Error), } impl SoarError { diff --git a/soar-core/src/package/formats/common.rs b/soar-core/src/package/formats/common.rs index da69dbb6c..7aa1cc5e7 100644 --- a/soar-core/src/package/formats/common.rs +++ b/soar-core/src/package/formats/common.rs @@ -8,10 +8,9 @@ use std::{ use image::{imageops::FilterType, DynamicImage, GenericImageView}; use regex::Regex; -use soar_dl::{download::Download, types::OverwriteMode}; use soar_utils::{ error::FileSystemResult, - fs::{create_symlink, read_file_signature, walk_dir}, + fs::{create_symlink, walk_dir}, path::{desktop_dir, icons_dir}, }; @@ -20,8 +19,7 @@ use super::{ }; use crate::{ config::get_config, - constants::PNG_MAGIC_BYTES, - database::models::{Package, PackageExt}, + database::models::PackageExt, error::{ErrorContext, SoarError}, SoarResult, }; @@ -145,49 +143,6 @@ pub fn symlink_desktop, T: PackageExt>( Ok(final_path) } -pub async fn integrate_remote>( - package_path: P, - package: &Package, -) -> SoarResult<()> { - let package_path = package_path.as_ref(); - let icon_url = &package.icon; - let desktop_url = &package.desktop; - - let mut icon_output_path = package_path.join(".DirIcon"); - let desktop_output_path = package_path.join(format!("{}.desktop", package.pkg_name)); - - if let Some(icon_url) = icon_url { - Download::new(icon_url) - .output(icon_output_path.to_string_lossy()) - .overwrite(OverwriteMode::Skip) - .execute()?; - - let ext = if read_file_signature(icon_output_path, 8)? == PNG_MAGIC_BYTES { - "png" - } else { - "svg" - }; - icon_output_path = package_path.join(format!("{}.{}", package.pkg_name, ext)); - } - - if let Some(desktop_url) = desktop_url { - Download::new(desktop_url) - .output(desktop_output_path.to_string_lossy()) - .overwrite(OverwriteMode::Skip) - .execute()?; - } else { - let content = create_default_desktop_entry(&package.pkg_name, "Utility"); - fs::write(&desktop_output_path, &content).with_context(|| { - format!("writing to desktop file {}", desktop_output_path.display()) - })?; - } - - symlink_icon(&icon_output_path)?; - symlink_desktop(&desktop_output_path, package)?; - - Ok(()) -} - pub fn create_portable_link>( portable_path: P, real_path: P, @@ -268,19 +223,6 @@ pub fn setup_portable_dir, T: PackageExt>( Ok(()) } -fn create_default_desktop_entry(name: &str, categories: &str) -> Vec { - format!( - "[Desktop Entry]\n\ - Type=Application\n\ - Name={name}\n\ - Icon={name}\n\ - Exec={name}\n\ - Categories={categories};\n", - ) - .as_bytes() - .to_vec() -} - pub async fn integrate_package, T: PackageExt>( install_dir: P, package: &T, diff --git a/soar-core/src/package/install.rs b/soar-core/src/package/install.rs index 851de4dbb..a64f1ae7e 100644 --- a/soar-core/src/package/install.rs +++ b/soar-core/src/package/install.rs @@ -143,9 +143,7 @@ impl PackageInstaller { loop { if retries > 5 { if let Some(ref callback) = self.progress_callback { - callback(Progress::Complete { - total: 0, - }); + callback(Progress::Aborted); } break; } From acf92ee174294d9e1ee7fb16354d65417a3afdc0 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Tue, 11 Nov 2025 17:24:12 +0545 Subject: [PATCH 16/22] fix ghcr blob download --- crates/soar-dl/src/download.rs | 23 +++++++++++++++++++++-- crates/soar-dl/src/http.rs | 5 +++++ crates/soar-dl/src/oci.rs | 5 ++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs index 67b6db3f6..a235e4522 100644 --- a/crates/soar-dl/src/download.rs +++ b/crates/soar-dl/src/download.rs @@ -31,6 +31,7 @@ pub struct Download { pub extract: bool, pub extract_to: Option, pub on_progress: Option>, + pub ghcr_blob: bool, } impl Download { @@ -60,9 +61,27 @@ impl Download { extract: false, extract_to: None, on_progress: None, + ghcr_blob: false, } } + /// Turns on GHCR blob support. + /// + /// When enabled, the `Authorization` header is set to `Bearer QQ==` + /// + /// # Examples + /// + /// ``` + /// use soar_dl::download::Download; + /// + /// let dl = Download::new("https://example.com/archive.tar.gz") + /// .ghcr_blob(); + /// ``` + pub fn ghcr_blob(mut self) -> Self { + self.ghcr_blob = true; + self + } + /// Sets the download output destination. /// /// The `output` value may be a filesystem path or `"-"` to write to stdout. @@ -272,7 +291,7 @@ impl Download { /// /// `PathBuf::from("-")` on success. fn download_to_stdout(&self) -> Result { - let resp = Http::fetch(&self.url, None, None)?; + let resp = Http::fetch(&self.url, None, None, self.ghcr_blob)?; let mut stdout = std::io::stdout(); let mut reader = resp.into_body().into_reader(); @@ -306,7 +325,7 @@ impl Download { .map(|r| (Some(r.downloaded), r.etag.as_deref())) .unwrap_or((None, None)); - let resp = Http::fetch(&self.url, resume_from, etag)?; + let resp = Http::fetch(&self.url, resume_from, etag, self.ghcr_blob)?; let status = resp.status(); if resume_from.is_some() && status != 206 { diff --git a/crates/soar-dl/src/http.rs b/crates/soar-dl/src/http.rs index 4f747ed15..e33beba07 100644 --- a/crates/soar-dl/src/http.rs +++ b/crates/soar-dl/src/http.rs @@ -35,9 +35,14 @@ impl Http { url: &str, resume_from: Option, etag: Option<&str>, + ghcr_blob: bool, ) -> Result, DownloadError> { let mut req = SHARED_AGENT.get(url); + if ghcr_blob { + req = req.header("Authorization", "Bearer QQ=="); + } + if let Some(pos) = resume_from { req = req.header("Range", &format!("bytes={}-", pos)); if let Some(tag) = etag { diff --git a/crates/soar-dl/src/oci.rs b/crates/soar-dl/src/oci.rs index 55708c653..203e587f2 100644 --- a/crates/soar-dl/src/oci.rs +++ b/crates/soar-dl/src/oci.rs @@ -665,7 +665,10 @@ impl OciDownload { self.reference.tag ); - let dl = Download::new(url).output(output).overwrite(self.overwrite); + let dl = Download::new(url) + .output(output) + .overwrite(self.overwrite) + .ghcr_blob(); let dl = if let Some(ref cb) = self.on_progress { let cb = cb.clone(); From b54be4be0e9ce7a9847d707f05337da27acf679b Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Tue, 11 Nov 2025 17:56:27 +0545 Subject: [PATCH 17/22] remove reqwest --- Cargo.lock | 415 +-------- Cargo.toml | 9 +- soar-cli/Cargo.lock | 7 - soar-cli/Cargo.toml | 5 +- soar-cli/src/inspect.rs | 21 +- soar-core/Cargo.lock | 1686 ------------------------------------- soar-core/Cargo.toml | 3 +- soar-core/src/error.rs | 4 +- soar-core/src/metadata.rs | 91 +- 9 files changed, 67 insertions(+), 2174 deletions(-) delete mode 100644 soar-cli/Cargo.lock delete mode 100644 soar-core/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index 4bd0f52fc..4b5ab615b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,25 +108,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "async-compression" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" -dependencies = [ - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.5.0" @@ -141,7 +122,7 @@ checksum = "8f2fc1bc7bb7fd449e02000cc1592cc63dcdcd61710f8b9efe32bab2d1784603" dependencies = [ "deku", "flate2", - "rustc-hash 1.1.0", + "rustc-hash", "thiserror 1.0.69", "tracing", "xz2", @@ -938,10 +919,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", - "wasm-bindgen", ] [[package]] @@ -951,11 +930,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", - "wasm-bindgen", ] [[package]] @@ -975,25 +952,6 @@ dependencies = [ "scroll", ] -[[package]] -name = "h2" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.15.5" @@ -1038,96 +996,12 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - [[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "hyper" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-util" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2 0.6.0", - "tokio", - "tower-service", - "tracing", -] - [[package]] name = "icu_collections" version = "2.0.0" @@ -1317,22 +1191,6 @@ dependencies = [ "libc", ] -[[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_ci" version = "1.2.0" @@ -1460,12 +1318,6 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - [[package]] name = "lzma-rust2" version = "0.15.1" @@ -1898,61 +1750,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "quinn" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash 2.1.1", - "rustls", - "socket2 0.5.10", - "thiserror 2.0.17", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" -dependencies = [ - "bytes", - "getrandom 0.3.3", - "lru-slab", - "rand 0.9.2", - "ring", - "rustc-hash 2.1.1", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.17", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2 0.5.10", - "tracing", - "windows-sys 0.59.0", -] - [[package]] name = "quote" version = "1.0.40" @@ -2076,50 +1873,6 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" -[[package]] -name = "reqwest" -version = "0.12.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" -dependencies = [ - "async-compression", - "base64", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "webpki-roots", -] - [[package]] name = "ring" version = "0.17.14" @@ -2172,12 +1925,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustc-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" - [[package]] name = "rustix" version = "1.0.8" @@ -2221,7 +1968,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ - "web-time", "zeroize", ] @@ -2348,18 +2094,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "serial_test" version = "3.2.0" @@ -2477,7 +2211,6 @@ dependencies = [ "rand 0.9.2", "rayon", "regex", - "reqwest", "rusqlite", "semver", "serde", @@ -2501,7 +2234,6 @@ dependencies = [ "include_dir", "nix", "regex", - "reqwest", "rusqlite", "serde", "serde_json", @@ -2512,6 +2244,8 @@ dependencies = [ "toml", "toml_edit 0.23.5", "tracing", + "ureq", + "url", "zstd", ] @@ -2554,26 +2288,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "socket2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - [[package]] name = "sqlite-wasm-rs" version = "0.4.3" @@ -2693,15 +2407,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - [[package]] name = "synstructure" version = "0.13.2" @@ -2853,21 +2558,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinyvec" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.47.1" @@ -2875,15 +2565,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", - "bytes", "io-uring", "libc", "mio", "pin-project-lite", "slab", - "socket2 0.6.0", "tokio-macros", - "windows-sys 0.59.0", ] [[package]] @@ -2897,29 +2584,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-rustls" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - [[package]] name = "toml" version = "0.9.6" @@ -2989,51 +2653,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "iri-string", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - [[package]] name = "tracing" version = "0.1.41" @@ -3095,12 +2714,6 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "typenum" version = "1.19.0" @@ -3239,15 +2852,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3344,19 +2948,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "web-sys" version = "0.3.77" diff --git a/Cargo.toml b/Cargo.toml index f85bce0da..46f7540fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,18 +35,11 @@ regex = { version = "1.11.2", default-features = false, features = [ "unicode-case", "unicode-perl" ] } -reqwest = { version = "0.12.23", default-features = false, features = [ - "blocking", - "gzip", - "http2", - "json", - "rustls-tls", - "stream" -] } rusqlite = { version = "0.37.0", features = ["bundled", "rusqlite-macros"] } serde = { version = "1.0.225", features = ["derive"] } serde_json = { version = "1.0.145", features = ["indexmap"] } serial_test = "3.2.0" +soar-core = { path = "soar-core" } soar-dl = { path = "crates/soar-dl" } soar-utils = { path = "crates/soar-utils" } tempfile = "3.10.1" diff --git a/soar-cli/Cargo.lock b/soar-cli/Cargo.lock deleted file mode 100644 index 7f5bcefdf..000000000 --- a/soar-cli/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "soar-cli" -version = "0.1.0" diff --git a/soar-cli/Cargo.toml b/soar-cli/Cargo.toml index 427528498..ba420f83f 100644 --- a/soar-cli/Cargo.toml +++ b/soar-cli/Cargo.toml @@ -28,15 +28,14 @@ once_cell = "1.21.3" rand = "0.9.2" rayon = { workspace = true } regex = { workspace = true } -reqwest = { workspace = true } rusqlite = { workspace = true } semver = "1.0.27" serde = { workspace = true } serde_json = { workspace = true } -soar-core = { version = "0.8.1", path = "../soar-core" } +soar-core = { workspace = true } soar-dl = { workspace = true } soar-utils = { workspace = true } -tokio = { version = "1.47.1", features = ["macros", "rt-multi-thread"] } +tokio = { version = "1.47.1", features = ["macros", "rt-multi-thread", "sync", "time"] } toml = "0.9.6" tracing = { workspace = true } tracing-subscriber = { version = "0.3.20", default-features = false, features = ["env-filter", "fmt", "json", "nu-ansi-term"] } diff --git a/soar-cli/src/inspect.rs b/soar-cli/src/inspect.rs index 8bd6946f1..b2bbce535 100644 --- a/soar-cli/src/inspect.rs +++ b/soar-cli/src/inspect.rs @@ -5,7 +5,6 @@ use std::{ sync::{Arc, Mutex}, }; -use futures::StreamExt; use indicatif::HumanBytes; use rusqlite::Connection; use soar_core::{ @@ -17,7 +16,9 @@ use soar_core::{ package::query::PackageQuery, SoarResult, }; +use soar_dl::http_client::SHARED_AGENT; use tracing::{error, info}; +use ureq::http::header::CONTENT_LENGTH; use crate::{ state::AppState, @@ -131,7 +132,8 @@ pub async fn inspect_log(package: &str, inspect_type: InspectType) -> SoarResult url }; - let resp = reqwest::get(url).await?; + let resp = SHARED_AGENT.get(url).call()?; + if !resp.status().is_success() { error!( "Error fetching build {inspect_type} from {} [{}]", @@ -141,7 +143,13 @@ pub async fn inspect_log(package: &str, inspect_type: InspectType) -> SoarResult return Ok(()); } - let content_length = resp.content_length().unwrap_or_default(); + let content_length = resp + .headers() + .get(CONTENT_LENGTH) + .and_then(|h| h.to_str().ok()) + .and_then(|len| len.parse::().ok()) + .unwrap_or(0); + if content_length > 1_048_576 { let response = interactive_ask( "The {inspect_type} file is too large. Do you really want to view it (y/N)?", @@ -157,12 +165,7 @@ pub async fn inspect_log(package: &str, inspect_type: InspectType) -> SoarResult HumanBytes(content_length) ); - let mut stream = resp.bytes_stream(); - let mut content = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - content.extend_from_slice(&chunk); - } + let content = resp.into_body().read_to_vec()?; let output = String::from_utf8_lossy(&content).replace("\r", "\n"); info!("\n{}", output); diff --git a/soar-core/Cargo.lock b/soar-core/Cargo.lock deleted file mode 100644 index ce45168c2..000000000 --- a/soar-core/Cargo.lock +++ /dev/null @@ -1,1686 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "backtrace" -version = "0.3.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets", -] - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bitflags" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" - -[[package]] -name = "blake3" -version = "1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "memmap2", -] - -[[package]] -name = "bumpalo" -version = "3.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" - -[[package]] -name = "cc" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "h2" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashbrown" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" - -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" -dependencies = [ - "hashbrown 0.14.5", -] - -[[package]] -name = "http" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" -dependencies = [ - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" - -[[package]] -name = "hyper" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" -dependencies = [ - "futures-util", - "http", - "hyper", - "hyper-util", - "rustls", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-util" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "icu_collections" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - -[[package]] -name = "icu_normalizer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "utf16_iter", - "utf8_iter", - "write16", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" - -[[package]] -name = "icu_properties" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locid_transform", - "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" - -[[package]] -name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "idna" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" -dependencies = [ - "equivalent", - "hashbrown 0.15.2", -] - -[[package]] -name = "ipnet" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" - -[[package]] -name = "itoa" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" - -[[package]] -name = "js-sys" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "libc" -version = "0.2.167" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" - -[[package]] -name = "libsqlite3-sys" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "litemap" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "memmap2" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" -dependencies = [ - "libc", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "miniz_oxide" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.52.0", -] - -[[package]] -name = "nix" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "object" -version = "0.36.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pin-project-lite" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" - -[[package]] -name = "ppv-lite86" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quinn" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" -dependencies = [ - "bytes", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "quinn-proto" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" -dependencies = [ - "bytes", - "getrandom", - "rand", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.59.0", -] - -[[package]] -name = "quote" -version = "1.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "reqwest" -version = "0.12.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pemfile", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "webpki-roots", - "windows-registry", -] - -[[package]] -name = "ring" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" -dependencies = [ - "cc", - "cfg-if", - "getrandom", - "libc", - "spin", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rusqlite" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" - -[[package]] -name = "rustc-hash" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" - -[[package]] -name = "rustls" -version = "0.23.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" -dependencies = [ - "web-time", -] - -[[package]] -name = "rustls-webpki" -version = "0.102.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "serde" -version = "1.0.215" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.215" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.133" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "soar-core" -version = "0.1.0" -dependencies = [ - "blake3", - "nix", - "once_cell", - "rayon", - "reqwest", - "rusqlite", - "serde", - "toml", -] - -[[package]] -name = "socket2" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "2.0.90" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thiserror" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "pin-project-lite", - "socket2", - "windows-sys 0.52.0", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "unicode-ident" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" -dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" - -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "0.26.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "windows-registry" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" -dependencies = [ - "windows-result", - "windows-strings", - "windows-targets", -] - -[[package]] -name = "windows-result" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result", - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "winnow" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" -dependencies = [ - "memchr", -] - -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - -[[package]] -name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - -[[package]] -name = "yoke" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" - -[[package]] -name = "zerovec" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/soar-core/Cargo.toml b/soar-core/Cargo.toml index 5a6301e3d..2d1a5c907 100644 --- a/soar-core/Cargo.toml +++ b/soar-core/Cargo.toml @@ -16,7 +16,6 @@ image = { version = "0.25.8", default-features = false, features = ["png"] } include_dir = "0.7.4" nix = { version = "0.30.1", features = ["ioctl", "term", "user"] } regex = { workspace = true } -reqwest = { workspace = true } rusqlite = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -27,4 +26,6 @@ thiserror = "2.0.16" toml = "0.9.6" toml_edit = "0.23.5" tracing = { workspace = true } +ureq = { workspace = true } +url = { workspace = true } zstd = "0.13.3" diff --git a/soar-core/src/error.rs b/soar-core/src/error.rs index e8b5380ce..a697ab53b 100644 --- a/soar-core/src/error.rs +++ b/soar-core/src/error.rs @@ -78,7 +78,7 @@ pub enum SoarError { DatabaseError(String), #[error("HTTP request error: {0:?}")] - ReqwestError(#[from] reqwest::Error), + UreqError(#[from] ureq::Error), #[error("Download failed: {0}")] DownloadError(#[from] soar_dl::error::DownloadError), @@ -133,7 +133,7 @@ impl SoarError { pub fn root_cause(&self) -> String { match self { - Self::ReqwestError(e) => { + Self::UreqError(e) => { format!( "Root cause: {}", e.source() diff --git a/soar-core/src/metadata.rs b/soar-core/src/metadata.rs index af3514029..7f48a8775 100644 --- a/soar-core/src/metadata.rs +++ b/soar-core/src/metadata.rs @@ -4,10 +4,15 @@ use std::{ path::Path, }; -use reqwest::header::{self, HeaderMap}; use rusqlite::Connection; +use soar_dl::{download::Download, http_client::SHARED_AGENT, types::OverwriteMode}; use soar_utils::{fs::read_file_signature, system::platform}; use tracing::info; +use ureq::http::{ + header::{CACHE_CONTROL, ETAG, IF_NONE_MATCH, PRAGMA}, + StatusCode, +}; +use url::Url; use crate::{ config::{self, Repository}, @@ -22,7 +27,7 @@ use crate::{ SoarResult, }; -fn construct_nest_url(url: &str) -> SoarResult { +fn construct_nest_url(url: &str) -> SoarResult { let url = if let Some(repo) = url.strip_prefix("github:") { format!( "/{}/releases/download/soar-nest/{}.json", @@ -32,7 +37,8 @@ fn construct_nest_url(url: &str) -> SoarResult { } else { url.to_string() }; - reqwest::Url::parse(&url).map_err(|err| SoarError::Custom(err.to_string())) + Url::parse(&url).map_err(|err| SoarError::Custom(err.to_string()))?; + Ok(url) } pub async fn fetch_nest_metadata( @@ -73,17 +79,20 @@ pub async fn fetch_nest_metadata( let url = construct_nest_url(&nest.url)?; - let client = reqwest::Client::new(); - let mut headers = HeaderMap::new(); - headers.insert(header::CACHE_CONTROL, "no-cache".parse().unwrap()); - headers.insert(header::PRAGMA, "no-cache".parse().unwrap()); + let mut req = SHARED_AGENT + .get(&nest.url) + .header(CACHE_CONTROL, "no-cache") + .header(PRAGMA, "no-cache"); + if !etag.is_empty() { - headers.insert(header::IF_NONE_MATCH, etag.parse().unwrap()); + req = req.header(IF_NONE_MATCH, etag); } - let resp = client.get(url.clone()).headers(headers).send().await?; + let resp = req + .call() + .map_err(|err| SoarError::FailedToFetchRemote(err.to_string()))?; - if resp.status() == reqwest::StatusCode::NOT_MODIFIED { + if resp.status() == StatusCode::NOT_MODIFIED { return Ok(None); } @@ -94,14 +103,14 @@ pub async fn fetch_nest_metadata( let etag = resp .headers() - .get(header::ETAG) - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()) + .get(ETAG) + .and_then(|h| h.to_str().ok()) + .map(String::from) .ok_or_else(|| SoarError::Custom("etag not found in metadata response".to_string()))?; info!("Fetching nest from {}", url); - let content = resp.bytes().await?.to_vec(); + let content = resp.into_body().read_to_vec()?; let nest_name = format!("nest-{}", nest.name); process_metadata_content(content, &metadata_db, &nest_name, &nest.url)?; @@ -190,11 +199,7 @@ fn handle_json_metadata>( Ok(()) } -pub async fn fetch_public_key>( - client: &reqwest::Client, - repo_path: P, - pubkey_url: &str, -) -> SoarResult<()> { +pub async fn fetch_public_key>(repo_path: P, pubkey_url: &str) -> SoarResult<()> { let repo_path = repo_path.as_ref(); let pubkey_file = repo_path.join("minisign.pub"); @@ -203,18 +208,12 @@ pub async fn fetch_public_key>( return Ok(()); } - let resp = client.get(pubkey_url).send().await?; - info!("Fetching public key from {}", pubkey_url); - if !resp.status().is_success() { - let msg = format!("{} [{}]", pubkey_url, resp.status()); - return Err(SoarError::FailedToFetchRemote(msg)); - } - - let content = resp.bytes().await?; - fs::write(&pubkey_file, content) - .with_context(|| format!("writing minisign key {}", pubkey_file.display()))?; + Download::new(pubkey_url) + .output(pubkey_file.to_string_lossy().to_string()) + .overwrite(OverwriteMode::Force) + .execute()?; Ok(()) } @@ -250,24 +249,26 @@ pub async fn fetch_metadata(repo: Repository, force: bool) -> SoarResult SoarResult Date: Tue, 11 Nov 2025 18:22:18 +0545 Subject: [PATCH 18/22] fix resume handling --- crates/soar-dl/src/download.rs | 34 ++++++++++++++++++++++++---------- crates/soar-dl/src/gitlab.rs | 6 +++--- soar-cli/src/download.rs | 4 ++-- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/crates/soar-dl/src/download.rs b/crates/soar-dl/src/download.rs index a235e4522..2fcb979fa 100644 --- a/crates/soar-dl/src/download.rs +++ b/crates/soar-dl/src/download.rs @@ -174,6 +174,7 @@ impl Download { /// Progress::Starting { total } => eprintln!("starting, total={}", total), /// Progress::Chunk { total, current } => eprintln!("downloaded {} (+{})", total, current), /// Progress::Complete { total } => eprintln!("complete, total={}", total), + /// _ => {} /// }); /// ``` pub fn progress(mut self, on_progress: F) -> Self @@ -215,7 +216,16 @@ impl Download { return self.download_to_stdout(); } - let (header_filename, url_filename) = if self.output.as_deref().is_none() { + let needs_head = match self.output.as_deref() { + None => true, + Some("-") => false, + Some(p) => { + let path = Path::new(p); + p.ends_with("/") || path.is_dir() + } + }; + + let (header_filename, url_filename) = if needs_head { let resp = Http::head(&self.url)?; ( resp.headers() @@ -230,23 +240,27 @@ impl Download { let output_path = resolve_output_path(self.output.as_deref(), url_filename, header_filename)?; - let resume_info = if output_path.is_file() { + let mut resume_info = read_resume(&output_path); + + if output_path.is_file() { match self.overwrite { - OverwriteMode::Skip => return Ok(output_path), + OverwriteMode::Skip => { + return Ok(output_path); + } OverwriteMode::Force => { fs::remove_file(&output_path)?; - None + resume_info = None; } OverwriteMode::Prompt => { - if !prompt_overwrite(&output_path)? { - return Ok(output_path); + if resume_info.is_none() { + if !prompt_overwrite(&output_path)? { + return Ok(output_path); + } + fs::remove_file(&output_path)?; + resume_info = None; } - fs::remove_file(&output_path)?; - None } } - } else { - read_resume(&output_path) }; if let Some(parent) = output_path.parent() { diff --git a/crates/soar-dl/src/gitlab.rs b/crates/soar-dl/src/gitlab.rs index 6ec1dc5e9..ab323c4a9 100644 --- a/crates/soar-dl/src/gitlab.rs +++ b/crates/soar-dl/src/gitlab.rs @@ -95,7 +95,7 @@ impl Release for GitLabRelease { /// # Examples /// /// ``` - /// use soar_dl::gitlab::{GitlabAssets, GitLabRelease}; + /// use soar_dl::gitlab::{GitLabAssets, GitLabRelease}; /// use soar_dl::traits::Release; /// /// let r = GitLabRelease { @@ -103,7 +103,7 @@ impl Release for GitLabRelease { /// tag_name: "v1.0".into(), /// upcoming_release: false, /// released_at: "".into(), - /// assets: GitlabAssets { links: vec![] }, + /// assets: GitLabAssets { links: vec![] }, /// }; /// assert_eq!(r.name(), "v1.0"); /// @@ -112,7 +112,7 @@ impl Release for GitLabRelease { /// tag_name: "v1.1".into(), /// upcoming_release: false, /// released_at: "".into(), - /// assets: GitlabAssets { links: vec![] }, + /// assets: GitLabAssets { links: vec![] }, /// }; /// assert_eq!(unnamed.name(), ""); /// ``` diff --git a/soar-cli/src/download.rs b/soar-cli/src/download.rs index 91600b92e..f759396fc 100644 --- a/soar-cli/src/download.rs +++ b/soar-cli/src/download.rs @@ -173,7 +173,7 @@ pub async fn handle_direct_downloads( package.pkg_name, package.pkg_id ); if let Some(ref url) = package.ghcr_blob { - let mut dl = OciDownload::new(url.as_str()).overwrite(OverwriteMode::Force); + let mut dl = OciDownload::new(url.as_str()).overwrite(ctx.get_overwrite_mode()); if let Some(ref out) = output { dl = dl.output(out); @@ -189,7 +189,7 @@ pub async fn handle_direct_downloads( } } else { let mut dl = - Download::new(&package.download_url).overwrite(OverwriteMode::Force); + Download::new(&package.download_url).overwrite(ctx.get_overwrite_mode()); if let Some(ref out) = output { dl = dl.output(out); From 9cc51480a7bfc853f2b24bc59e35e607fd62ec99 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Tue, 11 Nov 2025 18:23:32 +0545 Subject: [PATCH 19/22] fix nest get --- soar-core/src/metadata.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soar-core/src/metadata.rs b/soar-core/src/metadata.rs index 7f48a8775..49f6620f1 100644 --- a/soar-core/src/metadata.rs +++ b/soar-core/src/metadata.rs @@ -80,7 +80,7 @@ pub async fn fetch_nest_metadata( let url = construct_nest_url(&nest.url)?; let mut req = SHARED_AGENT - .get(&nest.url) + .get(&url) .header(CACHE_CONTROL, "no-cache") .header(PRAGMA, "no-cache"); From f3be0d7d53c6dc3d1c1658c62a0d86628d06dc43 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Tue, 11 Nov 2025 18:26:21 +0545 Subject: [PATCH 20/22] remove unused dep --- Cargo.lock | 13 ------------- Cargo.toml | 1 - soar-cli/Cargo.toml | 1 - 3 files changed, 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b5ab615b..2bb749ed1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -861,17 +861,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "futures-sink" version = "0.3.31" @@ -893,7 +882,6 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", - "futures-macro", "futures-sink", "futures-task", "memchr", @@ -2203,7 +2191,6 @@ name = "soar-cli" version = "0.8.1" dependencies = [ "clap", - "futures", "indicatif", "minisign-verify", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index 46f7540fd..1ebd24c4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,6 @@ diesel = { version = "2.3.2", features = [ ] } diesel_migrations = { version = "2.3.0", features = ["sqlite"] } fast-glob = "1.0.0" -futures = "0.3.31" miette = { version = "7.6.0", features = ["fancy"] } percent-encoding = "2.3.2" rayon = "1.11.0" diff --git a/soar-cli/Cargo.toml b/soar-cli/Cargo.toml index ba420f83f..65bfb9d7e 100644 --- a/soar-cli/Cargo.toml +++ b/soar-cli/Cargo.toml @@ -20,7 +20,6 @@ self = [] [dependencies] clap = { version = "4.5.47", features = ["cargo", "derive"] } -futures = { workspace = true } indicatif = "0.18.0" minisign-verify = "0.2.4" nu-ansi-term = "0.50.1" From d8331e36d5f0571a1635bdb70920a9bd7623b151 Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Tue, 11 Nov 2025 18:27:29 +0545 Subject: [PATCH 21/22] add rustfmt --- rustfmt.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 rustfmt.toml diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 000000000..36b389414 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,9 @@ +edition = "2024" +force_multiline_blocks = true +imports_granularity = "Crate" +group_imports = "StdExternalCrate" +newline_style = "Unix" +reorder_impl_items = true +reorder_imports = true +struct_lit_single_line = false +use_field_init_shorthand = true From 32c0b7823fa2cd7bdec3ecda52633065c33b806b Mon Sep 17 00:00:00 2001 From: Rabindra Dhakal Date: Tue, 11 Nov 2025 18:37:20 +0545 Subject: [PATCH 22/22] remove unnecessary param --- crates/soar-dl/src/gitlab.rs | 13 ++----------- crates/soar-dl/src/http.rs | 2 +- soar-cli/src/download.rs | 8 +++----- soar-cli/src/main.rs | 2 +- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/crates/soar-dl/src/gitlab.rs b/crates/soar-dl/src/gitlab.rs index ab323c4a9..f4e94bf39 100644 --- a/crates/soar-dl/src/gitlab.rs +++ b/crates/soar-dl/src/gitlab.rs @@ -90,7 +90,7 @@ impl Platform for GitLab { impl Release for GitLabRelease { type Asset = GitLabAsset; - /// The release's name, or an empty string if the release has no name. + /// The release's name /// /// # Examples /// @@ -99,22 +99,13 @@ impl Release for GitLabRelease { /// use soar_dl::traits::Release; /// /// let r = GitLabRelease { - /// name: Some("v1.0".into()), + /// name: "v1.0".into(), /// tag_name: "v1.0".into(), /// upcoming_release: false, /// released_at: "".into(), /// assets: GitLabAssets { links: vec![] }, /// }; /// assert_eq!(r.name(), "v1.0"); - /// - /// let unnamed = GitLabRelease { - /// name: None, - /// tag_name: "v1.1".into(), - /// upcoming_release: false, - /// released_at: "".into(), - /// assets: GitLabAssets { links: vec![] }, - /// }; - /// assert_eq!(unnamed.name(), ""); /// ``` fn name(&self) -> &str { &self.name diff --git a/crates/soar-dl/src/http.rs b/crates/soar-dl/src/http.rs index e33beba07..2f12def55 100644 --- a/crates/soar-dl/src/http.rs +++ b/crates/soar-dl/src/http.rs @@ -23,7 +23,7 @@ impl Http { /// ```no_run /// # use soar_dl::http::Http; /// - /// let resp = Http::fetch("https://example.com/resource", Some(1024), Some("\"etag-value\"")); + /// let resp = Http::fetch("https://example.com/resource", Some(1024), Some("\"etag-value\""), false); /// match resp { /// Ok(r) => { /// assert!(r.status().as_u16() < 600); // got a response diff --git a/soar-cli/src/download.rs b/soar-cli/src/download.rs index f759396fc..ff9b9e64b 100644 --- a/soar-cli/src/download.rs +++ b/soar-cli/src/download.rs @@ -70,9 +70,8 @@ pub async fn download( github: Vec, gitlab: Vec, ghcr: Vec, - progress_callback: Arc, ) -> SoarResult<()> { - handle_direct_downloads(&ctx, links, ctx.output.clone(), progress_callback.clone()).await?; + handle_direct_downloads(&ctx, links, ctx.output.clone()).await?; if !github.is_empty() { handle_github_downloads(&ctx, github).await?; @@ -93,7 +92,6 @@ pub async fn handle_direct_downloads( ctx: &DownloadContext, links: Vec, output: Option, - progress_callback: Arc, ) -> SoarResult<()> { for link in &links { match PlatformUrl::parse(link) { @@ -179,7 +177,7 @@ pub async fn handle_direct_downloads( dl = dl.output(out); } - let cb = progress_callback.clone(); + let cb = ctx.progress_callback.clone(); dl = dl.progress(move |p| { cb(p); }); @@ -195,7 +193,7 @@ pub async fn handle_direct_downloads( dl = dl.output(out); } - let cb = progress_callback.clone(); + let cb = ctx.progress_callback.clone(); dl = dl.progress(move |p| { cb(p); }); diff --git a/soar-cli/src/main.rs b/soar-cli/src/main.rs index 662bb4a29..c5a73e4da 100644 --- a/soar-cli/src/main.rs +++ b/soar-cli/src/main.rs @@ -276,7 +276,7 @@ async fn handle_cli() -> SoarResult<()> { force_overwrite, }; - download(context, links, github, gitlab, ghcr, progress_callback).await?; + download(context, links, github, gitlab, ghcr).await?; } cli::Commands::Health => display_health().await?, cli::Commands::Env => {