diff --git a/Cargo.lock b/Cargo.lock
index dbbfcde..ab96620 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -67,35 +67,6 @@ version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
-[[package]]
-name = "apache-avro"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
-dependencies = [
- "bigdecimal",
- "bon",
- "bzip2",
- "crc32fast",
- "digest",
- "liblzma",
- "log",
- "miniz_oxide",
- "num-bigint",
- "quad-rand",
- "rand 0.9.4",
- "regex-lite",
- "serde",
- "serde_bytes",
- "serde_json",
- "snap",
- "strum",
- "strum_macros",
- "thiserror 2.0.18",
- "uuid",
- "zstd",
-]
-
[[package]]
name = "ar_archive_writer"
version = "0.5.2"
@@ -171,6 +142,30 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "arrow-avro"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-schema",
+ "bytes",
+ "bzip2",
+ "crc",
+ "flate2",
+ "indexmap",
+ "liblzma",
+ "rand 0.9.4",
+ "serde",
+ "serde_json",
+ "snap",
+ "strum_macros",
+ "uuid",
+ "zstd",
+]
+
[[package]]
name = "arrow-buffer"
version = "58.3.0"
@@ -414,7 +409,6 @@ dependencies = [
"num-bigint",
"num-integer",
"num-traits",
- "serde",
]
[[package]]
@@ -429,7 +423,7 @@ version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
dependencies = [
- "digest",
+ "digest 0.10.7",
]
[[package]]
@@ -443,7 +437,7 @@ dependencies = [
"cc",
"cfg-if",
"constant_time_eq",
- "cpufeatures 0.3.0",
+ "cpufeatures",
]
[[package]]
@@ -456,28 +450,12 @@ dependencies = [
]
[[package]]
-name = "bon"
-version = "3.9.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3"
-dependencies = [
- "bon-macros",
- "rustversion",
-]
-
-[[package]]
-name = "bon-macros"
-version = "3.9.2"
+name = "block-buffer"
+version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
dependencies = [
- "darling",
- "ident_case",
- "prettyplease",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn",
+ "hybrid-array",
]
[[package]]
@@ -565,7 +543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
dependencies = [
"cfg-if",
- "cpufeatures 0.3.0",
+ "cpufeatures",
"rand_core 0.10.1",
]
@@ -641,6 +619,12 @@ version = "0.4.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
+[[package]]
+name = "const-oid"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
+
[[package]]
name = "const-random"
version = "0.1.18"
@@ -685,22 +669,28 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "cpufeatures"
-version = "0.2.17"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
dependencies = [
"libc",
]
[[package]]
-name = "cpufeatures"
-version = "0.3.0"
+name = "crc"
+version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
dependencies = [
- "libc",
+ "crc-catalog",
]
+[[package]]
+name = "crc-catalog"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
+
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -732,6 +722,15 @@ dependencies = [
"typenum",
]
+[[package]]
+name = "crypto-common"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
+dependencies = [
+ "hybrid-array",
+]
+
[[package]]
name = "csv"
version = "1.4.0"
@@ -753,40 +752,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "darling"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
-dependencies = [
- "darling_core",
- "darling_macro",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
-dependencies = [
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim",
- "syn",
-]
-
-[[package]]
-name = "darling_macro"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
-dependencies = [
- "darling_core",
- "quote",
- "syn",
-]
-
[[package]]
name = "dashmap"
version = "6.2.1"
@@ -803,14 +768,13 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
dependencies = [
"arrow",
"arrow-schema",
"async-trait",
- "bytes",
"bzip2",
"chrono",
"datafusion-catalog",
@@ -841,14 +805,13 @@ dependencies = [
"datafusion-sql",
"flate2",
"futures",
+ "indexmap",
"itertools",
"liblzma",
"log",
"object_store",
"parking_lot",
"parquet",
- "rand 0.9.4",
- "regex",
"sqlparser",
"tempfile",
"tokio",
@@ -859,9 +822,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
dependencies = [
"arrow",
"async-trait",
@@ -884,9 +847,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
dependencies = [
"arrow",
"async-trait",
@@ -907,35 +870,35 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
dependencies = [
- "ahash",
- "apache-avro",
"arrow",
"arrow-ipc",
+ "arrow-schema",
"chrono",
+ "foldhash 0.2.0",
"half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"indexmap",
"itertools",
"libc",
"log",
"object_store",
"parquet",
- "paste",
"recursive",
"sqlparser",
"tokio",
+ "uuid",
"web-time",
]
[[package]]
name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
dependencies = [
"futures",
"log",
@@ -944,9 +907,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
dependencies = [
"arrow",
"async-compression",
@@ -970,6 +933,7 @@ dependencies = [
"liblzma",
"log",
"object_store",
+ "parking_lot",
"rand 0.9.4",
"tokio",
"tokio-util",
@@ -979,9 +943,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
dependencies = [
"arrow",
"arrow-ipc",
@@ -1003,29 +967,28 @@ dependencies = [
[[package]]
name = "datafusion-datasource-avro"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4"
+checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d"
dependencies = [
- "apache-avro",
"arrow",
+ "arrow-avro",
"async-trait",
"bytes",
"datafusion-common",
"datafusion-datasource",
- "datafusion-physical-expr-common",
+ "datafusion-physical-expr-adapter",
"datafusion-physical-plan",
"datafusion-session",
"futures",
- "num-traits",
"object_store",
]
[[package]]
name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
dependencies = [
"arrow",
"async-trait",
@@ -1046,9 +1009,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
dependencies = [
"arrow",
"async-trait",
@@ -1063,16 +1026,15 @@ dependencies = [
"datafusion-session",
"futures",
"object_store",
- "serde_json",
"tokio",
"tokio-stream",
]
[[package]]
name = "datafusion-datasource-parquet"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
+checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a"
dependencies = [
"arrow",
"async-trait",
@@ -1082,6 +1044,7 @@ dependencies = [
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-functions",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
@@ -1100,20 +1063,19 @@ dependencies = [
[[package]]
name = "datafusion-doc"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"
[[package]]
name = "datafusion-execution"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
dependencies = [
"arrow",
"arrow-buffer",
"async-trait",
- "chrono",
"dashmap",
"datafusion-common",
"datafusion-expr",
@@ -1129,11 +1091,12 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
dependencies = [
"arrow",
+ "arrow-schema",
"async-trait",
"chrono",
"datafusion-common",
@@ -1144,7 +1107,6 @@ dependencies = [
"datafusion-physical-expr-common",
"indexmap",
"itertools",
- "paste",
"recursive",
"serde_json",
"sqlparser",
@@ -1152,22 +1114,21 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
dependencies = [
"arrow",
"datafusion-common",
"indexmap",
"itertools",
- "paste",
]
[[package]]
name = "datafusion-functions"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
dependencies = [
"arrow",
"arrow-buffer",
@@ -1182,26 +1143,25 @@ dependencies = [
"datafusion-expr",
"datafusion-expr-common",
"datafusion-macros",
+ "datafusion-physical-expr-common",
"hex",
"itertools",
"log",
- "md-5",
+ "md-5 0.11.0",
"memchr",
"num-traits",
"rand 0.9.4",
"regex",
"sha2",
- "unicode-segmentation",
"uuid",
]
[[package]]
name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
dependencies = [
- "ahash",
"arrow",
"datafusion-common",
"datafusion-doc",
@@ -1211,19 +1171,18 @@ dependencies = [
"datafusion-macros",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "foldhash 0.2.0",
"half",
"log",
"num-traits",
- "paste",
]
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
dependencies = [
- "ahash",
"arrow",
"datafusion-common",
"datafusion-expr-common",
@@ -1232,9 +1191,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
dependencies = [
"arrow",
"arrow-ord",
@@ -1248,34 +1207,34 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-macros",
"datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"itertools",
"itoa",
"log",
- "paste",
+ "memchr",
]
[[package]]
name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
dependencies = [
"arrow",
"async-trait",
"datafusion-catalog",
"datafusion-common",
"datafusion-expr",
+ "datafusion-physical-expr",
"datafusion-physical-plan",
"parking_lot",
- "paste",
]
[[package]]
name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
dependencies = [
"arrow",
"datafusion-common",
@@ -1286,14 +1245,13 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"log",
- "paste",
]
[[package]]
name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -1333,9 +1291,9 @@ dependencies = [
[[package]]
name = "datafusion-macros"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
dependencies = [
"datafusion-doc",
"quote",
@@ -1344,9 +1302,9 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
dependencies = [
"arrow",
"chrono",
@@ -1364,11 +1322,10 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
dependencies = [
- "ahash",
"arrow",
"datafusion-common",
"datafusion-expr",
@@ -1376,11 +1333,10 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
"half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"indexmap",
"itertools",
"parking_lot",
- "paste",
"petgraph",
"recursive",
"tokio",
@@ -1388,9 +1344,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
dependencies = [
"arrow",
"datafusion-common",
@@ -1403,26 +1359,26 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
dependencies = [
- "ahash",
"arrow",
"chrono",
"datafusion-common",
"datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"indexmap",
"itertools",
"parking_lot",
+ "pin-project",
]
[[package]]
name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
dependencies = [
"arrow",
"datafusion-common",
@@ -1439,12 +1395,13 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
dependencies = [
- "ahash",
"arrow",
+ "arrow-data",
+ "arrow-ipc",
"arrow-ord",
"arrow-schema",
"async-trait",
@@ -1459,7 +1416,7 @@ dependencies = [
"datafusion-physical-expr-common",
"futures",
"half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"indexmap",
"itertools",
"log",
@@ -1471,9 +1428,9 @@ dependencies = [
[[package]]
name = "datafusion-proto"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738"
+checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e"
dependencies = [
"arrow",
"chrono",
@@ -1494,14 +1451,13 @@ dependencies = [
"datafusion-proto-common",
"object_store",
"prost",
- "rand 0.9.4",
]
[[package]]
name = "datafusion-proto-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9"
+checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb"
dependencies = [
"arrow",
"datafusion-common",
@@ -1510,9 +1466,9 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
dependencies = [
"arrow",
"datafusion-common",
@@ -1521,15 +1477,14 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
- "itertools",
"log",
]
[[package]]
name = "datafusion-session"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
dependencies = [
"async-trait",
"datafusion-common",
@@ -1541,9 +1496,9 @@ dependencies = [
[[package]]
name = "datafusion-spark"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5"
+checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e"
dependencies = [
"arrow",
"bigdecimal",
@@ -1556,21 +1511,24 @@ dependencies = [
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
"datafusion-functions-nested",
"log",
+ "num-traits",
"percent-encoding",
"rand 0.9.4",
"serde_json",
"sha1",
"sha2",
+ "twox-hash",
"url",
]
[[package]]
name = "datafusion-sql"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
dependencies = [
"arrow",
"bigdecimal",
@@ -1587,9 +1545,9 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7"
dependencies = [
"async-recursion",
"async-trait",
@@ -1611,11 +1569,22 @@ version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
- "block-buffer",
- "crypto-common",
+ "block-buffer 0.10.4",
+ "crypto-common 0.1.7",
"subtle",
]
+[[package]]
+name = "digest"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
+dependencies = [
+ "block-buffer 0.12.1",
+ "const-oid",
+ "crypto-common 0.2.2",
+]
+
[[package]]
name = "displaydoc"
version = "0.2.6"
@@ -1928,6 +1897,11 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
[[package]]
name = "heck"
@@ -1986,6 +1960,15 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+[[package]]
+name = "hybrid-array"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b"
+dependencies = [
+ "typenum",
+]
+
[[package]]
name = "hyper"
version = "1.10.1"
@@ -2158,12 +2141,6 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
-[[package]]
-name = "ident_case"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
-
[[package]]
name = "idna"
version = "1.1.0"
@@ -2439,7 +2416,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
- "digest",
+ "digest 0.10.7",
+]
+
+[[package]]
+name = "md-5"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"
+dependencies = [
+ "cfg-if",
+ "digest 0.11.3",
]
[[package]]
@@ -2483,7 +2470,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
"num-integer",
"num-traits",
- "serde",
]
[[package]]
@@ -2542,7 +2528,7 @@ dependencies = [
"humantime",
"hyper",
"itertools",
- "md-5",
+ "md-5 0.10.6",
"parking_lot",
"percent-encoding",
"quick-xml",
@@ -2721,6 +2707,26 @@ dependencies = [
"siphasher",
]
+[[package]]
+name = "pin-project"
+version = "1.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -2904,12 +2910,6 @@ dependencies = [
"cc",
]
-[[package]]
-name = "quad-rand"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
-
[[package]]
name = "quick-xml"
version = "0.39.4"
@@ -3094,12 +3094,6 @@ dependencies = [
"regex-syntax",
]
-[[package]]
-name = "regex-lite"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
-
[[package]]
name = "regex-syntax"
version = "0.8.11"
@@ -3356,16 +3350,6 @@ dependencies = [
"serde_derive",
]
-[[package]]
-name = "serde_bytes"
-version = "0.11.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
-dependencies = [
- "serde",
- "serde_core",
-]
-
[[package]]
name = "serde_core"
version = "1.0.228"
@@ -3403,6 +3387,7 @@ version = "1.0.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
dependencies = [
+ "indexmap",
"itoa",
"memchr",
"serde",
@@ -3449,24 +3434,24 @@ dependencies = [
[[package]]
name = "sha1"
-version = "0.10.6"
+version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
+checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
dependencies = [
"cfg-if",
- "cpufeatures 0.2.17",
- "digest",
+ "cpufeatures",
+ "digest 0.11.3",
]
[[package]]
name = "sha2"
-version = "0.10.9"
+version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
dependencies = [
"cfg-if",
- "cpufeatures 0.2.17",
- "digest",
+ "cpufeatures",
+ "digest 0.11.3",
]
[[package]]
@@ -3523,9 +3508,9 @@ dependencies = [
[[package]]
name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
dependencies = [
"log",
"recursive",
@@ -3562,23 +3547,11 @@ dependencies = [
"windows-sys 0.61.2",
]
-[[package]]
-name = "strsim"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
-[[package]]
-name = "strum"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
-
[[package]]
name = "strum_macros"
-version = "0.27.2"
+version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
+checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
dependencies = [
"heck",
"proc-macro2",
@@ -3588,11 +3561,12 @@ dependencies = [
[[package]]
name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
dependencies = [
"heck",
+ "indexmap",
"pbjson",
"pbjson-build",
"pbjson-types",
@@ -3907,6 +3881,9 @@ name = "twox-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+dependencies = [
+ "rand 0.9.4",
+]
[[package]]
name = "typenum"
@@ -4023,7 +4000,6 @@ checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7"
dependencies = [
"getrandom 0.4.2",
"js-sys",
- "serde_core",
"wasm-bindgen",
]
diff --git a/Cargo.toml b/Cargo.toml
index fd1971a..57256e6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -39,13 +39,13 @@ repository = "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/apache/datafusion-java"
[workspace.dependencies]
arrow = { version = "58", features = ["ffi"] }
async-trait = "0.1"
-datafusion = { version = "53.1.0" }
-datafusion-proto = "53.1.0"
-datafusion-spark = "53.1.0"
-datafusion-substrait = "53.1.0"
+datafusion = { version = "54.1.0" }
+datafusion-proto = "54.1.0"
+datafusion-spark = "54.1.0"
+datafusion-substrait = "54.1.0"
futures = "0.3"
jni = "0.21"
-# Pinned to the major DataFusion 53.1 pulls in transitively (0.13.x) so we
+# Pinned to the major DataFusion 54.1 pulls in transitively (0.13.x) so we
# share the same `dyn ObjectStore` vtable and don't double-link.
object_store = { version = "0.13", default-features = false }
prost = "0.14"
diff --git a/core/pom.xml b/core/pom.xml
index 1e25736..ee2cb65 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -129,7 +129,7 @@ under the License.
https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto-common/proto/datafusion_common.proto
${project.build.directory}/proto/datafusion/proto-common/proto
datafusion_common.proto
- d6f3368372ea277cc23e26f196994b81616d38599357bb374cbd7eb1760e649a789e4c133d86a395ac701049a500348da2ec039d3f978ac5d8112c2876dded1f
+ e6a20115badec14f641b9237b0faa3f553162992444208322558ea3fcfaa24ca15334e4945827259dcb7c1de7f9aed659d496974eb8eeb1cfcf54ad4d7f6d20f
@@ -140,7 +140,7 @@ under the License.
https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto/proto/datafusion.proto
${project.build.directory}/proto/datafusion/proto/proto
datafusion.proto
- c3d162b8e2a418e03f74caceaccfd934af89bb95a12ede13d4cc1701d24c734d74b1e96372142b173db05938dab7f965ad60d476363308c441677a63ea5fbcf7
+ bdae5361d32b4d4f0418677ce650ee8cc212fddfe9390b2c033ee4ab3bb3c969a6148b7b318b9bb268ee9384f18d4254ebe6e0701f15639059a8f2800d6e7847
diff --git a/core/src/main/java/org/apache/datafusion/IoException.java b/core/src/main/java/org/apache/datafusion/IoException.java
index 79f196e..42cdad7 100644
--- a/core/src/main/java/org/apache/datafusion/IoException.java
+++ b/core/src/main/java/org/apache/datafusion/IoException.java
@@ -21,9 +21,8 @@
/**
* IO-shaped failure: a local filesystem read failed, an object store request failed, or a parquet /
- * arrow / avro decoder reported a malformed file. Surfaces upstream {@code
- * DataFusionError::IoError}, {@code ObjectStore}, {@code ArrowError}, {@code ParquetError}, and
- * {@code AvroError}.
+ * arrow decoder reported a malformed file. Surfaces upstream {@code DataFusionError::IoError},
+ * {@code ObjectStore}, {@code ParquetError}, and the IO-shaped {@code ArrowError} variants.
*
* Note: this is {@code org.apache.datafusion.IoException} (lowercase {@code o}), distinct from
* {@code java.io.IOException}. The {@code IoException} spelling matches the orthography of the
diff --git a/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java b/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java
index 91cf1e7..ede96d9 100644
--- a/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java
+++ b/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java
@@ -195,7 +195,7 @@ void describeReturnsSummaryStats() throws Exception {
}
}
}
- // DataFusion 53.1 reports these seven labels.
+ // DataFusion 54.1 reports these seven labels.
assertTrue(seen.contains("count"), () -> "labels=" + seen);
assertTrue(seen.contains("null_count"), () -> "labels=" + seen);
assertTrue(seen.contains("mean"), () -> "labels=" + seen);
diff --git a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java
index a2cfb0a..2877fcf 100644
--- a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java
+++ b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java
@@ -73,7 +73,14 @@ static void checkFeatureEnabled() {
/**
* Build a minimal Substrait {@code Plan} that scans a registered named table {@code tableName}
- * with two columns {@code (id int32, v int32)} and projects them through unchanged.
+ * with two columns {@code (id int64, v int64)} and projects them through unchanged.
+ *
+ *
The columns are declared {@code NULLABILITY_NULLABLE} to match the schema DataFusion infers
+ * for the CSV these tests register — CSV inference always yields nullable fields. DataFusion's
+ * Substrait consumer verifies that a field the plan declares non-nullable really is non-nullable
+ * in the table, and rejects the plan otherwise: a plan built around a "never null" assumption
+ * must not silently run against data that can contain nulls. The enclosing struct stays {@code
+ * REQUIRED} — the row itself is always present.
*/
private static Plan namedTableScanPlan(String tableName) {
NamedStruct schema =
@@ -87,13 +94,13 @@ private static Plan namedTableScanPlan(String tableName) {
.setI64(
Type.I64
.newBuilder()
- .setNullability(Type.Nullability.NULLABILITY_REQUIRED)))
+ .setNullability(Type.Nullability.NULLABILITY_NULLABLE)))
.addTypes(
Type.newBuilder()
.setI64(
Type.I64
.newBuilder()
- .setNullability(Type.Nullability.NULLABILITY_REQUIRED)))
+ .setNullability(Type.Nullability.NULLABILITY_NULLABLE)))
.setNullability(Type.Nullability.NULLABILITY_REQUIRED))
.build();
ReadRel read =
diff --git a/native-common/Cargo.toml b/native-common/Cargo.toml
index 21a2296..d136dcd 100644
--- a/native-common/Cargo.toml
+++ b/native-common/Cargo.toml
@@ -27,13 +27,6 @@ publish = false
readme = "README.md"
description = "Shared JNI plumbing for DataFusion Java native crates: error-to-exception mapping, the per-cdylib Tokio runtime singleton, and the async-stream-to-FFI_ArrowArrayStream bridge."
-[features]
-# `datafusion-jni` builds DataFusion with `avro`, which adds the
-# `DataFusionError::AvroError` variant our classifier maps to IoException.
-# Feature-forwarded so consumers that don't read Avro (the Spark helper)
-# don't pull the apache-avro stack into their cdylib.
-avro = ["datafusion/avro"]
-
[dependencies]
datafusion = { workspace = true }
futures = { workspace = true }
diff --git a/native-common/src/errors.rs b/native-common/src/errors.rs
index f9dbb03..b3e438d 100644
--- a/native-common/src/errors.rs
+++ b/native-common/src/errors.rs
@@ -97,10 +97,6 @@ fn classify(err: &DataFusionError) -> &'static str {
DataFusionError::IoError(_)
| DataFusionError::ObjectStore(_)
| DataFusionError::ParquetError(_) => "org/apache/datafusion/IoException",
- // The AvroError variant only exists when DataFusion is built with its
- // `avro` feature, forwarded by this crate's own `avro` feature.
- #[cfg(feature = "avro")]
- DataFusionError::AvroError(_) => "org/apache/datafusion/IoException",
// ArrowError is a 21-variant grab bag -- only some of those variants
// are actually IO-shaped. DivideByZero / ArithmeticOverflow / Compute
// / Cast / InvalidArgument / Memory etc. are execution-time failures
@@ -125,9 +121,15 @@ fn classify(err: &DataFusionError) -> &'static str {
/// Map an [`ArrowError`] variant onto the Java exception class to throw.
/// Only the genuinely IO-shaped variants (`IoError`, `IpcError`) land on
/// `IoException`; everything else is execution-time and routes through
-/// `ExecutionException`. Schema/parse-shaped variants route through
-/// `PlanException` so a malformed IPC schema or a parse error surfaces as a
-/// query problem rather than an execution failure.
+/// `ExecutionException` -- including `AvroError`, which is where Avro decode
+/// failures land now that DataFusion 54 reads Avro through `arrow-avro`
+/// instead of `apache-avro` (the old `DataFusionError::AvroError` variant is
+/// gone). That puts Avro alongside the `CsvError` / `JsonError` decoder
+/// variants, which this arm already classified as execution failures.
+///
+/// Schema/parse-shaped variants route through `PlanException` so a malformed
+/// IPC schema or a parse error surfaces as a query problem rather than an
+/// execution failure.
///
/// Variants without a clean caller-facing category (`CDataInterface`, the
/// various overflow/index-overflow markers) fall through to the parent.
diff --git a/native/Cargo.toml b/native/Cargo.toml
index c040448..8f4add5 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -82,9 +82,8 @@ arrow = { workspace = true }
async-trait = { workspace = true }
datafusion = { workspace = true, features = ["avro"] }
# Shared JNI plumbing (error->exception mapping, runtime singleton,
-# StreamingReader). `avro` keeps the classifier's AvroError->IoException arm
-# in sync with the `avro` feature on `datafusion` above.
-datafusion-jni-common = { path = "../native-common", features = ["avro"] }
+# StreamingReader).
+datafusion-jni-common = { path = "../native-common" }
datafusion-proto = { workspace = true }
# Apache Spark-compatible functions + expression planners. Optional and
# gated behind the `spark` feature (in the default set). The `core` feature
diff --git a/native/src/cache_manager.rs b/native/src/cache_manager.rs
index ec38dc8..4be7070 100644
--- a/native/src/cache_manager.rs
+++ b/native/src/cache_manager.rs
@@ -29,10 +29,8 @@ use std::sync::Arc;
use std::time::Duration;
use datafusion::execution::cache::cache_manager::CacheManagerConfig;
-use datafusion::execution::cache::cache_unit::{
- DefaultFileStatisticsCache, DefaultFilesMetadataCache,
-};
-use datafusion::execution::cache::DefaultListFilesCache;
+use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache;
+use datafusion::execution::cache::{DefaultFilesMetadataCache, DefaultListFilesCache};
use crate::proto_gen::CacheManagerOptionsProto;
use datafusion_jni_common::errors::JniResult;
@@ -76,8 +74,20 @@ pub(crate) fn build_config(
config.list_files_cache_ttl = ttl;
}
- if opts.file_statistics_cache_enabled.unwrap_or(false) {
- config.table_files_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default()));
+ // DataFusion 54 renamed `table_files_statistics_cache` to
+ // `file_statistics_cache` and made the *limit* the on/off switch:
+ // `CacheManager::try_new` now builds a `DefaultFileStatisticsCache`
+ // whenever `file_statistics_cache_limit > 0`, even when the cache slot is
+ // `None`. The default limit is non-zero, so an explicit `false` from the
+ // Java surface has to zero the limit -- otherwise upstream would install a
+ // stats cache the caller just asked us not to.
+ if let Some(enabled) = opts.file_statistics_cache_enabled {
+ if enabled {
+ config.file_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default()));
+ } else {
+ config.file_statistics_cache = None;
+ config.file_statistics_cache_limit = 0;
+ }
}
Ok(Some(config))
diff --git a/native/src/memory.rs b/native/src/memory.rs
index e105299..8068aba 100644
--- a/native/src/memory.rs
+++ b/native/src/memory.rs
@@ -120,7 +120,27 @@ impl TrackingMemoryPool {
}
}
+/// DataFusion 54 added `Display` as a supertrait of [`MemoryPool`] so pools
+/// can render themselves in resource-exhausted messages. Mirror upstream's
+/// wrapper convention: name the wrapper, then defer to the inner pool for the
+/// limit/usage detail, and add the counters this wrapper exists to expose.
+impl std::fmt::Display for TrackingMemoryPool {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let (current, peak) = self.snapshot();
+ write!(
+ f,
+ "{}(current: {current}, peak: {peak}, inner: {})",
+ self.name(),
+ self.inner
+ )
+ }
+}
+
impl MemoryPool for TrackingMemoryPool {
+ fn name(&self) -> &str {
+ "tracking"
+ }
+
fn register(&self, consumer: &datafusion::execution::memory_pool::MemoryConsumer) {
self.inner.register(consumer);
}
diff --git a/native/src/table_provider.rs b/native/src/table_provider.rs
index 70eaac2..51c892b 100644
--- a/native/src/table_provider.rs
+++ b/native/src/table_provider.rs
@@ -22,7 +22,6 @@
//! `TableProvider` trait; it currently only supports a single-partition, no-pushdown scan,
//! with future pushdown and partitioning support tracked as follow-ups.
-use std::any::Any;
use std::fmt;
use std::sync::Arc;
@@ -74,10 +73,6 @@ impl fmt::Debug for JavaTableProvider {
#[async_trait]
impl TableProvider for JavaTableProvider {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn schema(&self) -> SchemaRef {
Arc::clone(&self.schema)
}
@@ -152,10 +147,6 @@ impl ExecutionPlan for JavaScanExec {
"JavaScanExec"
}
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn properties(&self) -> &Arc {
&self.plan_properties
}
diff --git a/native/src/udf.rs b/native/src/udf.rs
index da7b260..3c3c7da 100644
--- a/native/src/udf.rs
+++ b/native/src/udf.rs
@@ -17,7 +17,6 @@
//! Java-backed scalar UDF support.
-use std::any::Any;
use std::fmt;
use datafusion::arrow::array::{make_array, Array, ArrayRef, StructArray};
@@ -80,10 +79,6 @@ impl std::hash::Hash for JavaScalarUdf {
}
impl ScalarUDFImpl for JavaScalarUdf {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn name(&self) -> &str {
&self.name
}
diff --git a/pom.xml b/pom.xml
index 7ceec07..d9f3212 100644
--- a/pom.xml
+++ b/pom.xml
@@ -40,7 +40,7 @@ under the License.
17
UTF-8
5.11.3
- 53.1.0
+ 54.1.0
3.25.5
19.0.0
1.12.0