This is an automated email from the ASF dual-hosted git repository.
comphead pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new e3363e2cb6 deps: bump to datafusion 54.1.0 and latest iceberg-rust
(#4996)
e3363e2cb6 is described below
commit e3363e2cb64174e7caf8d70f6b1534047e26c760
Author: Matt Butrovich <[email protected]>
AuthorDate: Tue Jul 21 20:48:15 2026 -0400
deps: bump to datafusion 54.1.0 and latest iceberg-rust (#4996)
---
native/Cargo.lock | 132 ++++++++++-----------
native/Cargo.toml | 12 +-
native/core/Cargo.toml | 2 +-
.../core/src/execution/operators/iceberg_scan.rs | 2 +
native/core/src/execution/planner.rs | 2 +
native/core/src/parquet/parquet_exec.rs | 32 ++---
native/shuffle/src/schema_align.rs | 27 ++---
7 files changed, 108 insertions(+), 101 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index fc04499b51..c9ae0b0520 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -1829,9 +1829,9 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
+checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
dependencies = [
"arrow",
"arrow-schema",
@@ -1878,9 +1878,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
+checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
dependencies = [
"arrow",
"async-trait",
@@ -1903,9 +1903,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
+checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
dependencies = [
"arrow",
"async-trait",
@@ -2070,9 +2070,9 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
+checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2096,9 +2096,9 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
+checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
dependencies = [
"futures",
"log",
@@ -2107,9 +2107,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
+checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
dependencies = [
"arrow",
"async-compression",
@@ -2143,9 +2143,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
+checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2167,9 +2167,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
+checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
dependencies = [
"arrow",
"async-trait",
@@ -2190,9 +2190,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
+checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
dependencies = [
"arrow",
"async-trait",
@@ -2213,9 +2213,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-parquet"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450"
+checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a"
dependencies = [
"arrow",
"async-trait",
@@ -2244,15 +2244,15 @@ dependencies = [
[[package]]
name = "datafusion-doc"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
+checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"
[[package]]
name = "datafusion-execution"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
+checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2273,9 +2273,9 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
+checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
dependencies = [
"arrow",
"arrow-schema",
@@ -2295,9 +2295,9 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
+checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
dependencies = [
"arrow",
"datafusion-common",
@@ -2307,9 +2307,9 @@ dependencies = [
[[package]]
name = "datafusion-functions"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
+checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2339,9 +2339,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
+checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
dependencies = [
"arrow",
"datafusion-common",
@@ -2360,9 +2360,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
+checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
dependencies = [
"arrow",
"datafusion-common",
@@ -2372,9 +2372,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
+checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
dependencies = [
"arrow",
"arrow-ord",
@@ -2397,9 +2397,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
+checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
dependencies = [
"arrow",
"async-trait",
@@ -2413,9 +2413,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
+checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
dependencies = [
"arrow",
"datafusion-common",
@@ -2430,9 +2430,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
+checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2440,9 +2440,9 @@ dependencies = [
[[package]]
name = "datafusion-macros"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
+checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
dependencies = [
"datafusion-doc",
"quote",
@@ -2451,9 +2451,9 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
+checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
dependencies = [
"arrow",
"chrono",
@@ -2470,9 +2470,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
+checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
dependencies = [
"arrow",
"datafusion-common",
@@ -2491,9 +2491,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
+checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
dependencies = [
"arrow",
"datafusion-common",
@@ -2506,9 +2506,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
+checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
dependencies = [
"arrow",
"chrono",
@@ -2523,9 +2523,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
+checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
dependencies = [
"arrow",
"datafusion-common",
@@ -2541,9 +2541,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
+checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
dependencies = [
"arrow",
"arrow-data",
@@ -2574,9 +2574,9 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
+checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
dependencies = [
"arrow",
"datafusion-common",
@@ -2590,9 +2590,9 @@ dependencies = [
[[package]]
name = "datafusion-session"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
+checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
dependencies = [
"async-trait",
"datafusion-common",
@@ -2604,9 +2604,9 @@ dependencies = [
[[package]]
name = "datafusion-spark"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "390bb0bf37cb2b95ffd65eacd66f60df50793d1f94097799e416f39477a51957"
+checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e"
dependencies = [
"arrow",
"bigdecimal",
@@ -2634,9 +2634,9 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "54.0.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
+checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
dependencies = [
"arrow",
"bigdecimal",
@@ -3487,7 +3487,7 @@ dependencies = [
[[package]]
name = "iceberg"
version = "0.10.0"
-source =
"git+https://github.com/apache/iceberg-rust?rev=ff678ca#ff678ca6d1123d9d82902b34379ccba2caf172b6";
+source =
"git+https://github.com/apache/iceberg-rust?rev=e8460ee#e8460eee8725c7f66d53e26b89b3fef578f90ce5";
dependencies = [
"aes-gcm",
"anyhow",
@@ -3543,7 +3543,7 @@ dependencies = [
[[package]]
name = "iceberg-storage-opendal"
version = "0.10.0"
-source =
"git+https://github.com/apache/iceberg-rust?rev=ff678ca#ff678ca6d1123d9d82902b34379ccba2caf172b6";
+source =
"git+https://github.com/apache/iceberg-rust?rev=e8460ee#e8460eee8725c7f66d53e26b89b3fef578f90ce5";
dependencies = [
"anyhow",
"async-trait",
diff --git a/native/Cargo.toml b/native/Cargo.toml
index 7e52ec0282..313a0b43be 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -38,10 +38,10 @@ arrow = { version = "58.3.0", features = ["prettyprint",
"ffi", "chrono-tz"] }
async-trait = { version = "0.1" }
bytes = { version = "1.11.1" }
parquet = { version = "58.3.0", default-features = false, features =
["experimental"] }
-datafusion = { version = "54.0.0", default-features = false, features =
["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] }
-datafusion-datasource = { version = "54.0.0" }
-datafusion-physical-expr-adapter = { version = "54.0.0" }
-datafusion-spark = { version = "54.0.0", features = ["core"] }
+datafusion = { version = "54.1.0", default-features = false, features =
["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] }
+datafusion-datasource = { version = "54.1.0" }
+datafusion-physical-expr-adapter = { version = "54.1.0" }
+datafusion-spark = { version = "54.1.0", features = ["core"] }
datafusion-comet-spark-expr = { path = "spark-expr" }
datafusion-comet-common = { path = "common" }
datafusion-comet-jni-bridge = { path = "jni-bridge" }
@@ -58,8 +58,8 @@ object_store = { version = "0.13.2", features = ["gcp",
"azure", "aws", "http"]
url = "2.2"
aws-config = "1.8.18"
aws-credential-types = "1.2.13"
-iceberg = { git = "https://github.com/apache/iceberg-rust";, rev = "ff678ca" }
-iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust";,
rev = "ff678ca", features = ["opendal-memory", "opendal-fs", "opendal-s3",
"opendal-gcs", "opendal-oss", "opendal-azdls"] }
+iceberg = { git = "https://github.com/apache/iceberg-rust";, rev = "e8460ee" }
+iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust";,
rev = "e8460ee", features = ["opendal-memory", "opendal-fs", "opendal-s3",
"opendal-gcs", "opendal-oss", "opendal-azdls"] }
reqsign-core = "3"
[profile.release]
diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml
index 7be91683b8..2b51b92182 100644
--- a/native/core/Cargo.toml
+++ b/native/core/Cargo.toml
@@ -90,7 +90,7 @@ jni = { version = "0.22.4", features = ["invocation"] }
lazy_static = "1.4"
assertables = "10"
hex = "0.4.3"
-datafusion-functions-nested = { version = "54.0.0" }
+datafusion-functions-nested = { version = "54.1.0" }
[features]
backtrace = ["datafusion/backtrace"]
diff --git a/native/core/src/execution/operators/iceberg_scan.rs
b/native/core/src/execution/operators/iceberg_scan.rs
index d89a25c879..7c95c6a53b 100644
--- a/native/core/src/execution/operators/iceberg_scan.rs
+++ b/native/core/src/execution/operators/iceberg_scan.rs
@@ -660,6 +660,7 @@ mod tests {
partition_spec: None,
name_mapping: None,
case_sensitive: false,
+ key_metadata: None,
}
}
@@ -670,6 +671,7 @@ mod tests {
file_size_in_bytes: 0,
partition_spec_id: 0,
equality_ids: None,
+ key_metadata: None,
}
}
diff --git a/native/core/src/execution/planner.rs
b/native/core/src/execution/planner.rs
index a23a7b1f67..33e7df9387 100644
--- a/native/core/src/execution/planner.rs
+++ b/native/core/src/execution/planner.rs
@@ -3703,6 +3703,7 @@ fn parse_file_scan_tasks_from_common(
} else {
Some(del.equality_ids.clone())
},
+ key_metadata: None,
})
})
.collect::<Result<Vec<_>, ExecutionError>>()
@@ -3827,6 +3828,7 @@ fn parse_file_scan_tasks_from_common(
partition_spec,
name_mapping,
case_sensitive: false,
+ key_metadata: None,
})
})
.collect();
diff --git a/native/core/src/parquet/parquet_exec.rs
b/native/core/src/parquet/parquet_exec.rs
index d8704d80c0..5a527a19b8 100644
--- a/native/core/src/parquet/parquet_exec.rs
+++ b/native/core/src/parquet/parquet_exec.rs
@@ -142,9 +142,11 @@ pub(crate) fn init_datasource_exec(
);
}
- // DataFusion's metadata-caching reader factory: loads each file's full
metadata (including the
- // page index) once into the per-task RuntimeEnv cache (bounded LRU,
`metadata_cache_limit`) and
- // reuses it across that file's row-group splits, so the opener does not
re-fetch the page index.
+ // DataFusion's metadata-caching reader factory: loads each file's footer
metadata once into the
+ // per-task RuntimeEnv cache (bounded LRU, `metadata_cache_limit`) and
reuses it across that
+ // file's row-group splits. The page index is loaded lazily by the opener,
only when row-group
+ // statistics cannot already prove no further pruning is possible
(apache/datafusion#22857), and
+ // that load bypasses this factory, so only footer metadata is cached.
//
// TODO: metadata I/O is invisible in metrics. `fetch_metadata` reads via
`ObjectStore::get_ranges`,
// bypassing the `get_bytes` path where `bytes_scanned` is counted. A
byte-counting ObjectStore
@@ -257,14 +259,16 @@ mod tests {
use parquet::file::properties::{EnabledStatistics, WriterProperties};
use std::fs::File;
- // Regression test for #3978: the scan's reader factory must load the full
- // Parquet metadata, including the page index, into the per-task RuntimeEnv
- // metadata cache. The previous hand-rolled factory cached only the footer,
- // so DataFusion re-fetched the page index on every open of the same file.
+ // The scan's reader factory caches each file's footer metadata in the
per-task RuntimeEnv
+ // metadata cache, reused across the file's row-group splits (#3978). The
page index is loaded
+ // lazily and only when row-group statistics cannot already prune the
surviving row groups
+ // (apache/datafusion#22857); that load bypasses the caching factory, so
the cache holds footer
+ // metadata only. Without a pruning predicate here the page index is never
loaded.
#[tokio::test]
- async fn caches_full_metadata_with_page_index() {
- // Write a file with a page index: page-level statistics and a small
data
- // page row limit so the column and offset indexes span multiple pages.
+ async fn caches_footer_metadata() {
+ // A file with a page index (page-level statistics and a small data
page row limit) so we can
+ // assert the cached metadata deliberately excludes the page index
rather than lacking it
+ // because none was written.
let schema = Arc::new(Schema::new(vec![Field::new("a",
DataType::Int32, false)]));
let batch = RecordBatch::try_new(
Arc::clone(&schema),
@@ -318,8 +322,8 @@ mod tests {
batch.unwrap();
}
- // The per-task RuntimeEnv metadata cache must now hold this file's
- // metadata with the page index (column + offset index) loaded.
+ // The per-task RuntimeEnv metadata cache holds this file's footer
metadata. The page index
+ // is not loaded (no pruning predicate), so the cached metadata
excludes it.
let cache = session_ctx
.runtime_env()
.cache_manager
@@ -334,8 +338,8 @@ mod tests {
.expect("cached entry should hold Parquet metadata")
.parquet_metadata();
assert!(
- parquet_meta.column_index().is_some() &&
parquet_meta.offset_index().is_some(),
- "cached metadata must include the page index"
+ parquet_meta.column_index().is_none() &&
parquet_meta.offset_index().is_none(),
+ "footer-only cache must not include the page index"
);
}
}
diff --git a/native/shuffle/src/schema_align.rs
b/native/shuffle/src/schema_align.rs
index b5566becef..6c8d0bb97b 100644
--- a/native/shuffle/src/schema_align.rs
+++ b/native/shuffle/src/schema_align.rs
@@ -307,28 +307,27 @@ mod tests {
batches
}
- /// `width_bucket`: catalyst declares `Int64`, DataFusion produces
`Int32`. A top-level cast.
+ /// `width_bucket`: since DataFusion 54.1.0 (apache/datafusion#23087)
`SparkWidthBucket`
+ /// returns `Int64`, matching the `Int64` catalyst declares. The former
Int32->Int64 drift is
+ /// gone, so the writer sees identical types and `try_new_or_passthrough`
hands the child back
+ /// unwrapped. This passthrough assertion records that the width_bucket
workaround has retired.
#[tokio::test]
- async fn aligns_width_bucket_int32_to_int64() {
- let child_schema = Arc::new(Schema::new(vec![Field::new("a",
DataType::Int32, false)]));
+ async fn width_bucket_no_longer_drifts() {
+ let child_schema = Arc::new(Schema::new(vec![Field::new("a",
DataType::Int64, false)]));
let batch = RecordBatch::try_new(
child_schema,
- vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+ vec![Arc::new(Int64Array::from(vec![1, 2, 3]))],
)
.unwrap();
let expected = Arc::new(Schema::new(vec![Field::new("a",
DataType::Int64, false)]));
+ let child = memory_child(batch);
let aligned =
- SchemaAlignExec::try_new_or_passthrough(memory_child(batch),
&expected).unwrap();
- assert_eq!(aligned.schema().field(0).data_type(), &DataType::Int64);
-
- let out = collect_one_partition(aligned).await;
- let col = out[0]
- .column(0)
- .as_any()
- .downcast_ref::<Int64Array>()
- .unwrap();
- assert_eq!(col.values(), &[1, 2, 3]);
+ SchemaAlignExec::try_new_or_passthrough(Arc::clone(&child),
&expected).unwrap();
+ assert!(
+ Arc::ptr_eq(&child, &aligned),
+ "identical schemas must pass through without a SchemaAlignExec
wrapper"
+ );
}
/// `date_trunc`: catalyst declares `Timestamp(us, "UTC")`, DataFusion
produces `Timestamp(us)`
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
