This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new acf0f78987 Upgrade to object_store `0.9.0` and arrow `50.0.0` (#8758)
acf0f78987 is described below
commit acf0f78987c2fae2315893d43923a33080df9402
Author: Raphael Taylor-Davies <[email protected]>
AuthorDate: Sun Jan 14 16:42:51 2024 +0000
Upgrade to object_store `0.9.0` and arrow `50.0.0` (#8758)
* Prepare object_store 0.9.0
* Update test
* Update to arrow 50.0.0
* Update sqllogictest
* Update sqllogictests
* Format
* Use nullif
* Use released version of arrow-rs
* Update README to remove references to SIMD
* unpatch datafusion-cli
* Adjust memory sizes in tests
* Restore test without explicit region
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
Cargo.toml | 20 +-
README.md | 1 -
benchmarks/Cargo.toml | 1 -
benchmarks/README.md | 6 +-
datafusion-cli/Cargo.lock | 280 +++++++++++----------
datafusion-cli/Cargo.toml | 6 +-
datafusion-cli/src/exec.rs | 7 +-
datafusion/core/Cargo.toml | 1 -
.../core/src/datasource/physical_plan/csv.rs | 2 +-
.../core/src/datasource/physical_plan/json.rs | 2 +-
.../core/src/datasource/physical_plan/mod.rs | 6 +-
datafusion/core/tests/dataframe/describe.rs | 4 +-
.../user_defined/user_defined_scalar_functions.rs | 4 +-
.../physical-expr/src/conditional_expressions.rs | 7 +-
datafusion/physical-expr/src/expressions/case.rs | 42 ++--
datafusion/physical-plan/src/aggregates/mod.rs | 5 +-
datafusion/sql/tests/sql_integration.rs | 4 -
datafusion/sqllogictest/test_files/clickbench.slt | 2 +-
datafusion/sqllogictest/test_files/expr.slt | 2 +-
.../sqllogictest/test_files/repartition_scan.slt | 8 +-
docs/source/user-guide/example-usage.md | 2 +-
21 files changed, 207 insertions(+), 205 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index a87923b6a1..cc18616774 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,13 +32,13 @@ rust-version = "1.70"
version = "34.0.0"
[workspace.dependencies]
-arrow = { version = "49.0.0", features = ["prettyprint"] }
-arrow-array = { version = "49.0.0", default-features = false, features =
["chrono-tz"] }
-arrow-buffer = { version = "49.0.0", default-features = false }
-arrow-flight = { version = "49.0.0", features = ["flight-sql-experimental"] }
-arrow-ipc = { version = "49.0.0", default-features = false, features = ["lz4"]
}
-arrow-ord = { version = "49.0.0", default-features = false }
-arrow-schema = { version = "49.0.0", default-features = false }
+arrow = { version = "50.0.0", features = ["prettyprint"] }
+arrow-array = { version = "50.0.0", default-features = false, features =
["chrono-tz"] }
+arrow-buffer = { version = "50.0.0", default-features = false }
+arrow-flight = { version = "50.0.0", features = ["flight-sql-experimental"] }
+arrow-ipc = { version = "50.0.0", default-features = false, features = ["lz4"]
}
+arrow-ord = { version = "50.0.0", default-features = false }
+arrow-schema = { version = "50.0.0", default-features = false }
async-trait = "0.1.73"
bigdecimal = "0.4.1"
bytes = "1.4"
@@ -64,9 +64,9 @@ indexmap = "2.0.0"
itertools = "0.12"
log = "^0.4"
num_cpus = "1.13.0"
-object_store = { version = "0.8.0", default-features = false }
+object_store = { version = "0.9.0", default-features = false }
parking_lot = "0.12"
-parquet = { version = "49.0.0", default-features = false, features = ["arrow",
"async", "object_store"] }
+parquet = { version = "50.0.0", default-features = false, features = ["arrow",
"async", "object_store"] }
rand = "0.8"
rstest = "0.18.0"
serde_json = "1"
@@ -91,4 +91,4 @@ lto = false
opt-level = 3
overflow-checks = false
panic = 'unwind'
-rpath = false
+rpath = false
\ No newline at end of file
diff --git a/README.md b/README.md
index 883700a393..81ae30ab68 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,6 @@ Optional features:
- `backtrace`: include backtrace information in error messages
- `pyarrow`: conversions between PyArrow and DataFusion types
- `serde`: enable arrow-schema's `serde` feature
-- `simd`: enable arrow-rs's manual `SIMD` kernels (requires Rust `nightly`)
[apache avro]: https://avro.apache.org/
[apache parquet]: https://parquet.apache.org/
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
index 4ce46968e1..94c1ebe7ee 100644
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -29,7 +29,6 @@ rust-version = "1.70"
[features]
ci = []
default = ["mimalloc"]
-simd = ["datafusion/simd"]
snmalloc = ["snmalloc-rs"]
[dependencies]
diff --git a/benchmarks/README.md b/benchmarks/README.md
index c0baa43ab8..4b4d9eabd4 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -163,10 +163,10 @@ See the help for more details
### Different features
-You can enable the features `simd` (to use SIMD instructions, `cargo nightly`
is required.) and/or `mimalloc` or `snmalloc` (to use either the mimalloc or
snmalloc allocator) as features by passing them in as `--features`:
+You can enable `mimalloc` or `snmalloc` (to use either the mimalloc or
snmalloc allocator) as features by passing them in as `--features`. For example
-```
-cargo run --release --features "simd mimalloc" --bin tpch -- benchmark
datafusion --iterations 3 --path ./data --format tbl --query 1 --batch-size 4096
+```shell
+cargo run --release --features "mimalloc" --bin tpch -- benchmark datafusion
--iterations 3 --path ./data --format tbl --query 1 --batch-size 4096
```
The benchmark program also supports CSV and Parquet input file formats and a
utility is provided to convert from `tbl`
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index 252b00ca0a..5663e736db 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -25,9 +25,9 @@ checksum =
"aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "ahash"
-version = "0.8.6"
+version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
+checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01"
dependencies = [
"cfg-if",
"const-random",
@@ -130,11 +130,10 @@ checksum =
"96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
name = "arrow"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5bc25126d18a012146a888a0298f2c22e1150327bd2765fc76d710a556b2d614"
+checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c"
dependencies = [
- "ahash",
"arrow-arith",
"arrow-array",
"arrow-buffer",
@@ -152,9 +151,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "34ccd45e217ffa6e53bbb0080990e77113bdd4e91ddb84e97b77649810bcf1a7"
+checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -167,9 +166,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6bda9acea48b25123c08340f3a8ac361aa0f74469bb36f5ee9acf923fce23e9d"
+checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609"
dependencies = [
"ahash",
"arrow-buffer",
@@ -184,9 +183,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "01a0fc21915b00fc6c2667b069c1b64bdd920982f426079bc4a7cab86822886c"
+checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4"
dependencies = [
"bytes",
"half",
@@ -195,9 +194,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5dc0368ed618d509636c1e3cc20db1281148190a78f43519487b2daf07b63b4a"
+checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -214,9 +213,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2e09aa6246a1d6459b3f14baeaa49606cfdbca34435c46320e14054d244987ca"
+checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -233,9 +232,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "907fafe280a3874474678c1858b9ca4cb7fd83fb8034ff5b6d6376205a08c634"
+checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -245,9 +244,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "79a43d6808411886b8c7d4f6f7dd477029c1e77ffffffb7923555cc6579639cd"
+checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -260,9 +259,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d82565c91fd627922ebfe2810ee4e8346841b6f9361b87505a9acea38b614fee"
+checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -280,9 +279,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9b23b0e53c0db57c6749997fd343d4c0354c994be7eca67152dd2bdb9a3e1bb4"
+checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -295,9 +294,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "361249898d2d6d4a6eeb7484be6ac74977e48da12a4dd81a708d620cc558117a"
+checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf"
dependencies = [
"ahash",
"arrow-array",
@@ -310,15 +309,15 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "09e28a5e781bf1b0f981333684ad13f5901f4cd2f20589eab7cf1797da8fc167"
+checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029"
[[package]]
name = "arrow-select"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4f6208466590960efc1d2a7172bc4ff18a67d6e25c529381d7f96ddaf0dc4036"
+checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c"
dependencies = [
"ahash",
"arrow-array",
@@ -330,9 +329,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a4a48149c63c11c9ff571e50ab8f017d2a7cb71037a882b42f6354ed2da9acc7"
+checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -346,9 +345,9 @@ dependencies = [
[[package]]
name = "assert_cmd"
-version = "2.0.12"
+version = "2.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6"
+checksum = "00ad3f3a942eee60335ab4342358c161ee296829e0d16ff42fc1d6cb07815467"
dependencies = [
"anstyle",
"bstr",
@@ -379,13 +378,13 @@ dependencies = [
[[package]]
name = "async-trait"
-version = "0.1.75"
+version = "0.1.77"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98"
+checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -712,9 +711,9 @@ dependencies = [
[[package]]
name = "base64"
-version = "0.21.5"
+version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "base64-simd"
@@ -792,9 +791,9 @@ dependencies = [
[[package]]
name = "bstr"
-version = "1.8.0"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c"
+checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc"
dependencies = [
"memchr",
"regex-automata",
@@ -881,9 +880,9 @@ dependencies = [
[[package]]
name = "chrono-tz"
-version = "0.8.4"
+version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76"
+checksum = "91d7b79e99bfaa0d47da0687c43aa3b7381938a62ad3a6498599039321f660b7"
dependencies = [
"chrono",
"chrono-tz-build",
@@ -1015,9 +1014,9 @@ dependencies = [
[[package]]
name = "cpufeatures"
-version = "0.2.11"
+version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
+checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
dependencies = [
"libc",
]
@@ -1075,7 +1074,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e"
dependencies = [
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -1316,9 +1315,9 @@ dependencies = [
[[package]]
name = "deranged"
-version = "0.3.10"
+version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
+checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
dependencies = [
"powerfmt",
]
@@ -1579,7 +1578,7 @@ checksum =
"87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -1630,9 +1629,9 @@ dependencies = [
[[package]]
name = "getrandom"
-version = "0.2.11"
+version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
+checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
dependencies = [
"cfg-if",
"libc",
@@ -1653,9 +1652,9 @@ checksum =
"d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "h2"
-version = "0.3.22"
+version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178"
+checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7"
dependencies = [
"bytes",
"fnv",
@@ -1837,9 +1836,9 @@ dependencies = [
[[package]]
name = "iana-time-zone"
-version = "0.1.58"
+version = "0.1.59"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
+checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539"
dependencies = [
"android_system_properties",
"core-foundation-sys",
@@ -1944,9 +1943,9 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.66"
+version = "0.3.67"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca"
+checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1"
dependencies = [
"wasm-bindgen",
]
@@ -2023,9 +2022,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.151"
+version = "0.2.152"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
[[package]]
name = "libflate"
@@ -2102,9 +2101,9 @@ checksum =
"b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "lz4_flex"
-version = "0.11.1"
+version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8"
+checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15"
dependencies = [
"twox-hash",
]
@@ -2132,9 +2131,9 @@ dependencies = [
[[package]]
name = "memchr"
-version = "2.6.4"
+version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
+checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
[[package]]
name = "mimalloc"
@@ -2295,9 +2294,9 @@ dependencies = [
[[package]]
name = "object_store"
-version = "0.8.0"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050"
+checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000"
dependencies = [
"async-trait",
"base64",
@@ -2306,14 +2305,14 @@ dependencies = [
"futures",
"humantime",
"hyper",
- "itertools 0.11.0",
+ "itertools 0.12.0",
"parking_lot",
"percent-encoding",
"quick-xml",
"rand",
"reqwest",
"ring 0.17.7",
- "rustls-pemfile",
+ "rustls-pemfile 2.0.0",
"serde",
"serde_json",
"snafu",
@@ -2381,9 +2380,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "49.0.0"
+version = "50.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "af88740a842787da39b3d69ce5fbf6fce97d20211d3b299fee0a0da6430c74d4"
+checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750"
dependencies = [
"ahash",
"arrow-array",
@@ -2399,6 +2398,7 @@ dependencies = [
"chrono",
"flate2",
"futures",
+ "half",
"hashbrown 0.14.3",
"lz4_flex",
"num",
@@ -2499,7 +2499,7 @@ checksum =
"4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -2589,9 +2589,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.71"
+version = "1.0.76"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8"
+checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
dependencies = [
"unicode-ident",
]
@@ -2614,9 +2614,9 @@ dependencies = [
[[package]]
name = "quote"
-version = "1.0.33"
+version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
dependencies = [
"proc-macro2",
]
@@ -2740,7 +2740,8 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"rustls 0.21.10",
- "rustls-pemfile",
+ "rustls-native-certs",
+ "rustls-pemfile 1.0.4",
"serde",
"serde_json",
"serde_urlencoded",
@@ -2754,7 +2755,6 @@ dependencies = [
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
- "webpki-roots",
"winreg",
]
@@ -2836,9 +2836,9 @@ dependencies = [
[[package]]
name = "rustix"
-version = "0.38.28"
+version = "0.38.29"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
+checksum = "0a1a81a2478639a14e68937903356dbac62cf52171148924f754bb8a8cd7a96c"
dependencies = [
"bitflags 2.4.1",
"errno",
@@ -2878,7 +2878,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
dependencies = [
"openssl-probe",
- "rustls-pemfile",
+ "rustls-pemfile 1.0.4",
"schannel",
"security-framework",
]
@@ -2892,6 +2892,22 @@ dependencies = [
"base64",
]
+[[package]]
+name = "rustls-pemfile"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4"
+dependencies = [
+ "base64",
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a"
+
[[package]]
name = "rustls-webpki"
version = "0.101.7"
@@ -2948,11 +2964,11 @@ dependencies = [
[[package]]
name = "schannel"
-version = "0.1.22"
+version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
+checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -2996,9 +3012,9 @@ dependencies = [
[[package]]
name = "semver"
-version = "1.0.20"
+version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
+checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0"
[[package]]
name = "seq-macro"
@@ -3008,29 +3024,29 @@ checksum =
"a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
-version = "1.0.193"
+version = "1.0.195"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
+checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
-version = "1.0.193"
+version = "1.0.195"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
+checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
name = "serde_json"
-version = "1.0.108"
+version = "1.0.111"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
+checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4"
dependencies = [
"itoa",
"ryu",
@@ -3152,13 +3168,13 @@ dependencies = [
[[package]]
name = "sqlparser_derive"
-version = "0.2.1"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3e9c2e1dde0efa87003e7923d94a90f46e3274ad1649f51de96812be561f041f"
+checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
dependencies = [
"proc-macro2",
"quote",
- "syn 1.0.109",
+ "syn 2.0.48",
]
[[package]]
@@ -3198,7 +3214,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -3220,9 +3236,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.43"
+version = "2.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53"
+checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
dependencies = [
"proc-macro2",
"quote",
@@ -3252,22 +3268,22 @@ dependencies = [
[[package]]
name = "tempfile"
-version = "3.8.1"
+version = "3.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5"
+checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa"
dependencies = [
"cfg-if",
"fastrand 2.0.1",
"redox_syscall",
"rustix",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
]
[[package]]
name = "termcolor"
-version = "1.4.0"
+version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
dependencies = [
"winapi-util",
]
@@ -3286,22 +3302,22 @@ checksum =
"222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
[[package]]
name = "thiserror"
-version = "1.0.52"
+version = "1.0.56"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d"
+checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.52"
+version = "1.0.56"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3"
+checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -3394,7 +3410,7 @@ checksum =
"5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -3491,7 +3507,7 @@ checksum =
"34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -3536,7 +3552,7 @@ checksum =
"f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
@@ -3671,9 +3687,9 @@ checksum =
"9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
-version = "0.2.89"
+version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e"
+checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@@ -3681,24 +3697,24 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
-version = "0.2.89"
+version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826"
+checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.39"
+version = "0.4.40"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12"
+checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461"
dependencies = [
"cfg-if",
"js-sys",
@@ -3708,9 +3724,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.89"
+version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2"
+checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -3718,22 +3734,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.89"
+version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283"
+checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.89"
+version = "0.2.90"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f"
+checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b"
[[package]]
name = "wasm-streams"
@@ -3750,9 +3766,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.66"
+version = "0.3.67"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f"
+checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -3768,12 +3784,6 @@ dependencies = [
"untrusted 0.9.0",
]
-[[package]]
-name = "webpki-roots"
-version = "0.25.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"
-
[[package]]
name = "winapi"
version = "0.3.9"
@@ -3807,11 +3817,11 @@ checksum =
"712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
-version = "0.51.1"
+version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
+checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
- "windows-targets 0.48.5",
+ "windows-targets 0.52.0",
]
[[package]]
@@ -3988,7 +3998,7 @@ checksum =
"9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.43",
+ "syn 2.0.48",
]
[[package]]
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index eab7c8e0d1..d084938030 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.70"
readme = "README.md"
[dependencies]
-arrow = "49.0.0"
+arrow = "50.0.0"
async-trait = "0.1.41"
aws-config = "0.55"
aws-credential-types = "0.55"
@@ -40,9 +40,9 @@ dirs = "4.0.0"
env_logger = "0.9"
futures = "0.3"
mimalloc = { version = "0.1", default-features = false }
-object_store = { version = "0.8.0", features = ["aws", "gcp"] }
+object_store = { version = "0.9.0", features = ["aws", "gcp"] }
parking_lot = { version = "0.12" }
-parquet = { version = "49.0.0", default-features = false }
+parquet = { version = "50.0.0", default-features = false }
regex = "1.8"
rustyline = "11.0"
tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread",
"sync", "parking_lot", "signal"] }
diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs
index 6598437830..637fc7e4d9 100644
--- a/datafusion-cli/src/exec.rs
+++ b/datafusion-cli/src/exec.rs
@@ -340,13 +340,10 @@ mod tests {
let session_token = "fake_session_token";
let location = "s3://bucket/path/file.parquet";
- // Missing region
+ // Missing region, use object_store defaults
let sql = format!("CREATE EXTERNAL TABLE test STORED AS PARQUET
OPTIONS('access_key_id' '{access_key_id}', 'secret_access_key'
'{secret_access_key}') LOCATION '{location}'");
- let err = create_external_table_test(location, &sql)
- .await
- .unwrap_err();
- assert!(err.to_string().contains("Missing region"));
+ create_external_table_test(location, &sql).await?;
// Should be OK
let sql = format!("CREATE EXTERNAL TABLE test STORED AS PARQUET
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index c2e8c2b445..f5496d4c47 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -47,7 +47,6 @@ parquet = ["datafusion-common/parquet", "dep:parquet"]
pyarrow = ["datafusion-common/pyarrow", "parquet"]
regex_expressions = ["datafusion-physical-expr/regex_expressions",
"datafusion-optimizer/regex_expressions"]
serde = ["arrow-schema/serde"]
-simd = ["arrow/simd"]
unicode_expressions = ["datafusion-physical-expr/unicode_expressions",
"datafusion-optimizer/unicode_expressions",
"datafusion-sql/unicode_expressions"]
[dependencies]
diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs
b/datafusion/core/src/datasource/physical_plan/csv.rs
index b28bc7d566..a818c572f7 100644
--- a/datafusion/core/src/datasource/physical_plan/csv.rs
+++ b/datafusion/core/src/datasource/physical_plan/csv.rs
@@ -375,7 +375,7 @@ impl FileOpener for CsvOpener {
let range = match calculated_range {
RangeCalculation::Range(None) => None,
- RangeCalculation::Range(Some(range)) => Some(range),
+ RangeCalculation::Range(Some(range)) => Some(range.into()),
RangeCalculation::TerminateEarly => {
return Ok(
futures::stream::poll_fn(move |_|
Poll::Ready(None)).boxed()
diff --git a/datafusion/core/src/datasource/physical_plan/json.rs
b/datafusion/core/src/datasource/physical_plan/json.rs
index 529632dab8..a8a371fed9 100644
--- a/datafusion/core/src/datasource/physical_plan/json.rs
+++ b/datafusion/core/src/datasource/physical_plan/json.rs
@@ -239,7 +239,7 @@ impl FileOpener for JsonOpener {
let range = match calculated_range {
RangeCalculation::Range(None) => None,
- RangeCalculation::Range(Some(range)) => Some(range),
+ RangeCalculation::Range(Some(range)) => Some(range.into()),
RangeCalculation::TerminateEarly => {
return Ok(
futures::stream::poll_fn(move |_|
Poll::Ready(None)).boxed()
diff --git a/datafusion/core/src/datasource/physical_plan/mod.rs
b/datafusion/core/src/datasource/physical_plan/mod.rs
index d7be017a18..24155d3fd1 100644
--- a/datafusion/core/src/datasource/physical_plan/mod.rs
+++ b/datafusion/core/src/datasource/physical_plan/mod.rs
@@ -75,7 +75,7 @@ use datafusion_physical_plan::ExecutionPlan;
use log::debug;
use object_store::ObjectMeta;
-use object_store::{path::Path, GetOptions, ObjectStore};
+use object_store::{path::Path, GetOptions, GetRange, ObjectStore};
/// The base configurations to provide when creating a physical plan for
/// writing to any given file format.
@@ -604,10 +604,8 @@ async fn find_first_newline(
start: usize,
end: usize,
) -> Result<usize> {
- let range = Some(Range { start, end });
-
let options = GetOptions {
- range,
+ range: Some(GetRange::Bounded(start..end)),
..Default::default()
};
diff --git a/datafusion/core/tests/dataframe/describe.rs
b/datafusion/core/tests/dataframe/describe.rs
index da7589072b..e82c06efd6 100644
--- a/datafusion/core/tests/dataframe/describe.rs
+++ b/datafusion/core/tests/dataframe/describe.rs
@@ -40,12 +40,12 @@ async fn describe() -> Result<()> {
"+------------+-------------------+----------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-----------------+------------+-------------------------+--------------------+-------------------+",
"| count | 7300.0 | 7300 | 7300.0 |
7300.0 | 7300.0 | 7300.0 | 7300.0
| 7300.0 | 7300 | 7300 | 7300
| 7300.0 | 7300.0 |",
"| null_count | 7300.0 | 7300 | 7300.0 |
7300.0 | 7300.0 | 7300.0 | 7300.0
| 7300.0 | 7300 | 7300 | 7300
| 7300.0 | 7300.0 |",
- "| mean | 3649.5 | null | 4.5 |
4.5 | 4.5 | 45.0 |
4.949999964237213 | 45.45000000000001 | null | null | null
| 2009.5 | 6.526027397260274 |",
+ "| mean | 3649.5 | null | 4.5 |
4.5 | 4.5 | 45.0 |
4.949999964237213 | 45.45 | null | null | null
| 2009.5 | 6.526027397260274 |",
"| std | 2107.472815166704 | null | 2.8724780750809518 |
2.8724780750809518 | 2.8724780750809518 | 28.724780750809533 |
3.1597258182544645 | 29.012028558317645 | null | null | null
| 0.5000342500942125 | 3.44808750051728 |",
"| min | 0.0 | null | 0.0 |
0.0 | 0.0 | 0.0 | 0.0
| 0.0 | 01/01/09 | 0 | 2008-12-31T23:00:00
| 2009.0 | 1.0 |",
"| max | 7299.0 | null | 9.0 |
9.0 | 9.0 | 90.0 |
9.899999618530273 | 90.89999999999999 | 12/31/10 | 9 |
2010-12-31T04:09:13.860 | 2010.0 | 12.0 |",
"| median | 3649.0 | null | 4.0 |
4.0 | 4.0 | 45.0 |
4.949999809265137 | 45.45 | null | null | null
| 2009.0 | 7.0 |",
-
"+------------+-------------------+----------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-----------------+------------+-------------------------+--------------------+-------------------+"
+
"+------------+-------------------+----------+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+-----------------+------------+-------------------------+--------------------+-------------------+",
];
assert_batches_eq!(expected, &describe_record_batch);
Ok(())
diff --git
a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
index 4f39f2374e..fe88ea6cf1 100644
--- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
+++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
@@ -43,7 +43,7 @@ async fn csv_query_custom_udf_with_cast() -> Result<()> {
"+------------------------------------------+",
"| AVG(custom_sqrt(aggregate_test_100.c11)) |",
"+------------------------------------------+",
- "| 0.6584408483418833 |",
+ "| 0.6584408483418835 |",
"+------------------------------------------+",
];
assert_batches_eq!(&expected, &actual);
@@ -61,7 +61,7 @@ async fn csv_query_avg_sqrt() -> Result<()> {
"+------------------------------------------+",
"| AVG(custom_sqrt(aggregate_test_100.c12)) |",
"+------------------------------------------+",
- "| 0.6706002946036462 |",
+ "| 0.6706002946036459 |",
"+------------------------------------------+",
];
assert_batches_eq!(&expected, &actual);
diff --git a/datafusion/physical-expr/src/conditional_expressions.rs
b/datafusion/physical-expr/src/conditional_expressions.rs
index a9a25ffe2e..782897d463 100644
--- a/datafusion/physical-expr/src/conditional_expressions.rs
+++ b/datafusion/physical-expr/src/conditional_expressions.rs
@@ -47,16 +47,15 @@ pub fn coalesce(args: &[ColumnarValue]) ->
Result<ColumnarValue> {
match arg {
ColumnarValue::Array(ref array) => {
let to_apply = and(&remainder,
&is_not_null(array.as_ref())?)?;
- current_value = zip(&to_apply, array,
current_value.as_ref())?;
+ current_value = zip(&to_apply, array, ¤t_value)?;
remainder = and(&remainder, &is_null(array)?)?;
}
ColumnarValue::Scalar(value) => {
if value.is_null() {
continue;
} else {
- let last_value = value.to_array_of_size(size)?;
- current_value =
- zip(&remainder, &last_value,
current_value.as_ref())?;
+ let last_value = value.to_scalar()?;
+ current_value = zip(&remainder, &last_value,
¤t_value)?;
break;
}
}
diff --git a/datafusion/physical-expr/src/expressions/case.rs
b/datafusion/physical-expr/src/expressions/case.rs
index 52fb85657f..414ddd0921 100644
--- a/datafusion/physical-expr/src/expressions/case.rs
+++ b/datafusion/physical-expr/src/expressions/case.rs
@@ -26,11 +26,11 @@ use crate::PhysicalExpr;
use arrow::array::*;
use arrow::compute::kernels::cmp::eq;
use arrow::compute::kernels::zip::zip;
-use arrow::compute::{and, is_null, not, or, prep_null_mask_filter};
+use arrow::compute::{and, is_null, not, nullif, or, prep_null_mask_filter};
use arrow::datatypes::{DataType, Schema};
use arrow::record_batch::RecordBatch;
-use datafusion_common::exec_err;
use datafusion_common::{cast::as_boolean_array, internal_err, DataFusionError,
Result};
+use datafusion_common::{exec_err, ScalarValue};
use datafusion_expr::ColumnarValue;
use itertools::Itertools;
@@ -151,16 +151,19 @@ impl CaseExpr {
let then_value = self.when_then_expr[i]
.1
.evaluate_selection(batch, &when_match)?;
- let then_value = match then_value {
- ColumnarValue::Scalar(value) if value.is_null() => {
- new_null_array(&return_type, batch.num_rows())
+
+ current_value = match then_value {
+ ColumnarValue::Scalar(ScalarValue::Null) => {
+ nullif(current_value.as_ref(), &when_match)?
+ }
+ ColumnarValue::Scalar(then_value) => {
+ zip(&when_match, &then_value.to_scalar()?, ¤t_value)?
+ }
+ ColumnarValue::Array(then_value) => {
+ zip(&when_match, &then_value, ¤t_value)?
}
- _ => then_value.into_array(batch.num_rows())?,
};
- current_value =
- zip(&when_match, then_value.as_ref(), current_value.as_ref())?;
-
remainder = and(&remainder, ¬(&when_match)?)?;
}
@@ -173,7 +176,7 @@ impl CaseExpr {
let else_ = expr
.evaluate_selection(batch, &remainder)?
.into_array(batch.num_rows())?;
- current_value = zip(&remainder, else_.as_ref(),
current_value.as_ref())?;
+ current_value = zip(&remainder, &else_, ¤t_value)?;
}
Ok(ColumnarValue::Array(current_value))
@@ -214,16 +217,19 @@ impl CaseExpr {
let then_value = self.when_then_expr[i]
.1
.evaluate_selection(batch, &when_value)?;
- let then_value = match then_value {
- ColumnarValue::Scalar(value) if value.is_null() => {
- new_null_array(&return_type, batch.num_rows())
+
+ current_value = match then_value {
+ ColumnarValue::Scalar(ScalarValue::Null) => {
+ nullif(current_value.as_ref(), &when_value)?
+ }
+ ColumnarValue::Scalar(then_value) => {
+ zip(&when_value, &then_value.to_scalar()?, ¤t_value)?
+ }
+ ColumnarValue::Array(then_value) => {
+ zip(&when_value, &then_value, ¤t_value)?
}
- _ => then_value.into_array(batch.num_rows())?,
};
- current_value =
- zip(&when_value, then_value.as_ref(), current_value.as_ref())?;
-
// Succeed tuples should be filtered out for short-circuit
evaluation,
// null values for the current when expr should be kept
remainder = and(&remainder, ¬(&when_value)?)?;
@@ -236,7 +242,7 @@ impl CaseExpr {
let else_ = expr
.evaluate_selection(batch, &remainder)?
.into_array(batch.num_rows())?;
- current_value = zip(&remainder, else_.as_ref(),
current_value.as_ref())?;
+ current_value = zip(&remainder, &else_, ¤t_value)?;
}
Ok(ColumnarValue::Array(current_value))
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs
b/datafusion/physical-plan/src/aggregates/mod.rs
index 4f37be7263..facd601955 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -1482,7 +1482,7 @@ mod tests {
))];
let task_ctx = if spill {
- new_spill_ctx(2, 1500)
+ new_spill_ctx(2, 1600)
} else {
Arc::new(TaskContext::default())
};
@@ -1738,7 +1738,6 @@ mod tests {
}
#[tokio::test]
- #[ignore]
async fn aggregate_source_with_yielding_with_spill() -> Result<()> {
let input: Arc<dyn ExecutionPlan> =
Arc::new(TestYieldingExec { yield_first: true });
@@ -1949,7 +1948,7 @@ mod tests {
spill: bool,
) -> Result<()> {
let task_ctx = if spill {
- new_spill_ctx(2, 2886)
+ new_spill_ctx(2, 3200)
} else {
Arc::new(TaskContext::default())
};
diff --git a/datafusion/sql/tests/sql_integration.rs
b/datafusion/sql/tests/sql_integration.rs
index 4de08a7124..44da4cd4d8 100644
--- a/datafusion/sql/tests/sql_integration.rs
+++ b/datafusion/sql/tests/sql_integration.rs
@@ -451,10 +451,6 @@ Dml: op=[Insert Into] table=[test_decimal]
"INSERT INTO test_decimal (nonexistent, price) VALUES (1, 2), (4, 5)",
"Schema error: No field named nonexistent. Valid fields are id, price."
)]
-#[case::type_mismatch(
- "INSERT INTO test_decimal SELECT '2022-01-01',
to_timestamp('2022-01-01T12:00:00')",
- "Error during planning: Cannot automatically convert Timestamp(Nanosecond,
None) to Decimal128(10, 2)"
-)]
#[case::target_column_count_mismatch(
"INSERT INTO person (id, first_name, last_name) VALUES ($1, $2)",
"Error during planning: Column count doesn't match insert query!"
diff --git a/datafusion/sqllogictest/test_files/clickbench.slt
b/datafusion/sqllogictest/test_files/clickbench.slt
index f6afa525ad..21befd7822 100644
--- a/datafusion/sqllogictest/test_files/clickbench.slt
+++ b/datafusion/sqllogictest/test_files/clickbench.slt
@@ -52,7 +52,7 @@ SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth")
FROM hits;
query R
SELECT AVG("UserID") FROM hits;
----
--304548765855551600
+-304548765855551740
query I
SELECT COUNT(DISTINCT "UserID") FROM hits;
diff --git a/datafusion/sqllogictest/test_files/expr.slt
b/datafusion/sqllogictest/test_files/expr.slt
index a2a8d9c647..b5b50eca81 100644
--- a/datafusion/sqllogictest/test_files/expr.slt
+++ b/datafusion/sqllogictest/test_files/expr.slt
@@ -63,7 +63,7 @@ SELECT
NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL
NULL NULL NULL NULL
# test_array_cast_invalid_timezone_will_panic
-statement error Parser error: Invalid timezone "Foo": 'Foo' is not a valid
timezone
+statement error Parser error: Invalid timezone "Foo": 'Foo' is not a valid
timezone
SELECT arrow_cast('2021-01-02T03:04:00', 'Timestamp(Nanosecond, Some("Foo"))')
# test_array_index
diff --git a/datafusion/sqllogictest/test_files/repartition_scan.slt
b/datafusion/sqllogictest/test_files/repartition_scan.slt
index 73487635e9..4b8c8f2f08 100644
--- a/datafusion/sqllogictest/test_files/repartition_scan.slt
+++ b/datafusion/sqllogictest/test_files/repartition_scan.slt
@@ -61,7 +61,7 @@ Filter: parquet_table.column1 != Int32(42)
physical_plan
CoalesceBatchesExec: target_batch_size=8192
--FilterExec: column1@0 != 42
-----ParquetExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:0..101],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:101..202],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:202..303],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:303..403]]},
project [...]
+----ParquetExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:0..104],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:104..208],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:208..312],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:312..414]]},
project [...]
# disable round robin repartitioning
statement ok
@@ -77,7 +77,7 @@ Filter: parquet_table.column1 != Int32(42)
physical_plan
CoalesceBatchesExec: target_batch_size=8192
--FilterExec: column1@0 != 42
-----ParquetExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:0..101],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:101..202],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:202..303],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:303..403]]},
project [...]
+----ParquetExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:0..104],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:104..208],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:208..312],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:312..414]]},
project [...]
# enable round robin repartitioning again
statement ok
@@ -102,7 +102,7 @@ SortPreservingMergeExec: [column1@0 ASC NULLS LAST]
--SortExec: expr=[column1@0 ASC NULLS LAST]
----CoalesceBatchesExec: target_batch_size=8192
------FilterExec: column1@0 != 42
---------ParquetExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/1.parquet:0..200],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/1.parquet:200..394,
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:0..6],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:6..206],
[WORKSPACE_RO [...]
+--------ParquetExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/1.parquet:0..205],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/1.parquet:205..405,
WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:0..5],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:5..210],
[WORKSPACE_RO [...]
## Read the files as though they are ordered
@@ -138,7 +138,7 @@ physical_plan
SortPreservingMergeExec: [column1@0 ASC NULLS LAST]
--CoalesceBatchesExec: target_batch_size=8192
----FilterExec: column1@0 != 42
-------ParquetExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/1.parquet:0..197],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:0..201],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:201..403],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/1.parquet:197..394]]},
project [...]
+------ParquetExec: file_groups={4 groups:
[[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/1.parquet:0..202],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:0..207],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/2.parquet:207..414],
[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_scan/parquet_table/1.parquet:202..405]]},
project [...]
# Cleanup
statement ok
diff --git a/docs/source/user-guide/example-usage.md
b/docs/source/user-guide/example-usage.md
index a7557f9b0b..77b196c124 100644
--- a/docs/source/user-guide/example-usage.md
+++ b/docs/source/user-guide/example-usage.md
@@ -194,7 +194,7 @@ worth noting that using the settings in the
`[profile.release]` section will sig
```toml
[dependencies]
-datafusion = { version = "22.0" , features = ["simd"]}
+datafusion = { version = "22.0" }
tokio = { version = "^1.0", features = ["rt-multi-thread"] }
snmalloc-rs = "0.3"
