This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
The following commit(s) were added to refs/heads/main by this push:
new 4e073470 chore: Update to Arrow 57, DataFusion 50 (#466)
4e073470 is described below
commit 4e073470831c3f9b968d3d425ec3f084c24f5957
Author: Dewey Dunnington <[email protected]>
AuthorDate: Mon Jan 5 13:01:45 2026 -0600
chore: Update to Arrow 57, DataFusion 50 (#466)
Co-authored-by: Copilot <[email protected]>
---
.github/workflows/examples.yml | 2 +-
.github/workflows/packaging.yml | 2 +-
.github/workflows/python-wheels.yml | 2 +-
.github/workflows/python.yml | 4 +-
.github/workflows/r.yml | 2 +-
.github/workflows/rust-gpu.yml | 4 +-
.github/workflows/rust.yml | 4 +-
Cargo.lock | 735 +++++++++++++--------
Cargo.toml | 58 +-
c/sedona-libgpuspatial/build.rs | 4 +-
c/sedona-s2geography/s2geography | 2 +-
python/sedonadb/tests/test_sjoin.py | 2 +-
r/sedonadb/src/Makevars.win.in | 2 +-
r/sedonadb/src/rust/src/dataframe.rs | 5 +-
rust/sedona-datasource/src/format.rs | 17 +-
rust/sedona-functions/Cargo.toml | 2 +-
rust/sedona-functions/src/barrier.rs | 2 +-
rust/sedona-functions/src/st_dump.rs | 2 +-
rust/sedona-geoparquet/src/file_opener.rs | 16 +-
rust/sedona-geoparquet/src/format.rs | 11 +-
rust/sedona-raster-functions/src/rs_srid.rs | 8 +-
rust/sedona-schema/src/crs.rs | 2 +-
rust/sedona-spatial-join/Cargo.toml | 3 +-
.../bench/partitioning/common.rs | 8 +-
rust/sedona-spatial-join/bench/partitioning/kdb.rs | 8 +-
rust/sedona-spatial-join/src/exec.rs | 4 +-
rust/sedona-spatial-join/src/utils/bbox_sampler.rs | 28 +-
rust/sedona-testing/src/benchmark_util.rs | 13 +-
rust/sedona-testing/src/datagen.rs | 222 ++++---
rust/sedona-testing/src/testers.rs | 1 +
rust/sedona/Cargo.toml | 3 +-
rust/sedona/src/context.rs | 20 +-
rust/sedona/src/random_geometry_provider.rs | 2 +-
sedona-cli/src/exec.rs | 2 +-
sedona-cli/src/helper.rs | 21 +-
35 files changed, 706 insertions(+), 517 deletions(-)
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
index 1d801b13..95ac4914 100644
--- a/.github/workflows/examples.yml
+++ b/.github/workflows/examples.yml
@@ -66,7 +66,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
# Update this key to force a new cache
- prefix-key: "rust-${{ matrix.name }}-v4"
+ prefix-key: "rust-${{ matrix.name }}-v5"
- name: Install dependencies
shell: bash
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
index dacee800..f85f97a6 100644
--- a/.github/workflows/packaging.yml
+++ b/.github/workflows/packaging.yml
@@ -113,7 +113,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
# Update this key to force a new cache (but share with the
python.yml workflow)
- prefix-key: "python-v2"
+ prefix-key: "python-v3"
- name: Install docs requirements
run: |
diff --git a/.github/workflows/python-wheels.yml
b/.github/workflows/python-wheels.yml
index 229348fb..277404cf 100644
--- a/.github/workflows/python-wheels.yml
+++ b/.github/workflows/python-wheels.yml
@@ -60,7 +60,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
# Update this key to force a new cache
- prefix-key: "python-wheel-windows-latest-v2"
+ prefix-key: "python-wheel-windows-latest-v3"
- name: Clone vcpkg
uses: actions/checkout@v6
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index bbd09c56..c82030ec 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -92,7 +92,7 @@ jobs:
with:
path: vcpkg/packages
# Bump the number at the end of this line to force a new dependency
build
- key: vcpkg-installed-${{ runner.os }}-${{ runner.arch }}-${{
env.VCPKG_REF }}-2
+ key: vcpkg-installed-${{ runner.os }}-${{ runner.arch }}-${{
env.VCPKG_REF }}-3
- name: Install vcpkg dependencies
if: steps.cache-vcpkg.outputs.cache-hit != 'true'
@@ -112,7 +112,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
# Update this key to force a new cache (sync with packaging.yml)
- prefix-key: "python-v2"
+ prefix-key: "python-v3"
- name: Install
run: |
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
index 04cbc638..0f0ee3a4 100644
--- a/.github/workflows/r.yml
+++ b/.github/workflows/r.yml
@@ -86,7 +86,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
# Update this key to force a new cache
- prefix-key: "r-v3"
+ prefix-key: "r-v4"
# The R package uses its own target directory
workspaces: ". -> r/sedonadb/src/rust/target"
diff --git a/.github/workflows/rust-gpu.yml b/.github/workflows/rust-gpu.yml
index 523873f1..fc54e4d3 100644
--- a/.github/workflows/rust-gpu.yml
+++ b/.github/workflows/rust-gpu.yml
@@ -179,7 +179,7 @@ jobs:
with:
path: vcpkg/packages
# Bump the number at the end of this line to force a new dependency
build
- key: vcpkg-installed-${{ runner.os }}-${{ runner.arch }}-${{
env.VCPKG_REF }}-2
+ key: vcpkg-installed-${{ runner.os }}-${{ runner.arch }}-${{
env.VCPKG_REF }}-3
# Install vcpkg dependencies from vcpkg.json manifest
- name: Install vcpkg dependencies
@@ -198,7 +198,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
- prefix-key: "rust-gpu-v3"
+ prefix-key: "rust-gpu-v4"
# Build WITH GPU feature to compile CUDA code
# CUDA compilation (nvcc) works without GPU hardware
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index bf9c2ceb..f2c4e547 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -92,7 +92,7 @@ jobs:
with:
path: vcpkg/packages
# Bump the number at the end of this line to force a new dependency
build
- key: vcpkg-installed-${{ runner.os }}-${{ runner.arch }}-${{
env.VCPKG_REF }}-2
+ key: vcpkg-installed-${{ runner.os }}-${{ runner.arch }}-${{
env.VCPKG_REF }}-3
- name: Install vcpkg dependencies
if: steps.cache-vcpkg.outputs.cache-hit != 'true'
@@ -111,7 +111,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
# Update this key to force a new cache
- prefix-key: "rust-${{ matrix.name }}-v4"
+ prefix-key: "rust-${{ matrix.name }}-v5"
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
diff --git a/Cargo.lock b/Cargo.lock
index a79e1c52..f946c71c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -216,7 +216,7 @@ dependencies = [
"miniz_oxide",
"num-bigint",
"quad-rand",
- "rand 0.9.2",
+ "rand",
"regex-lite",
"serde",
"serde_bytes",
@@ -262,9 +262,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fd798aea3553913a5986813e9c6ad31a2d2b04e931fe8ea4a37155eb541cebb5"
+checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -283,23 +283,23 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "508dafb53e5804a238cab7fd97a59ddcbfab20cc4d9814b1ab5465b9fa147f2e"
+checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
- "num",
+ "num-traits",
]
[[package]]
name = "arrow-array"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e2730bc045d62bb2e53ef8395b7d4242f5c8102f41ceac15e8395b9ac3d08461"
+checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31"
dependencies = [
"ahash",
"arrow-buffer",
@@ -308,26 +308,29 @@ dependencies = [
"chrono",
"chrono-tz",
"half",
- "hashbrown 0.15.5",
- "num",
+ "hashbrown 0.16.1",
+ "num-complex",
+ "num-integer",
+ "num-traits",
]
[[package]]
name = "arrow-buffer"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "54295b93beb702ee9a6f6fbced08ad7f4d76ec1c297952d4b83cf68755421d1d"
+checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27"
dependencies = [
"bytes",
"half",
- "num",
+ "num-bigint",
+ "num-traits",
]
[[package]]
name = "arrow-cast"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "67e8bcb7dc971d779a7280593a1bf0c2743533b8028909073e804552e85e75b5"
+checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -340,15 +343,15 @@ dependencies = [
"comfy-table",
"half",
"lexical-core",
- "num",
+ "num-traits",
"ryu",
]
[[package]]
name = "arrow-csv"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "673fd2b5fb57a1754fdbfac425efd7cf54c947ac9950c1cce86b14e248f1c458"
+checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -361,26 +364,28 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "97c22fe3da840039c69e9f61f81e78092ea36d57037b4900151f063615a2f6b4"
+checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b"
dependencies = [
"arrow-buffer",
"arrow-schema",
"half",
- "num",
+ "num-integer",
+ "num-traits",
]
[[package]]
name = "arrow-ipc"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "778de14c5a69aedb27359e3dd06dd5f9c481d5f6ee9fbae912dba332fd64636b"
+checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
+ "arrow-select",
"flatbuffers",
"lz4_flex",
"zstd",
@@ -388,9 +393,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3860db334fe7b19fcf81f6b56f8d9d95053f3839ffe443d56b5436f7a29a1794"
+checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -400,19 +405,21 @@ dependencies = [
"chrono",
"half",
"indexmap",
+ "itoa",
"lexical-core",
"memchr",
- "num",
- "serde",
+ "num-traits",
+ "ryu",
+ "serde_core",
"serde_json",
"simdutf8",
]
[[package]]
name = "arrow-ord"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "425fa0b42a39d3ff55160832e7c25553e7f012c3f187def3d70313e7a29ba5d9"
+checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -423,9 +430,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "df9c9423c9e71abd1b08a7f788fcd203ba2698ac8e72a1f236f1faa1a06a7414"
+checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -436,34 +443,34 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "85fa1babc4a45fdc64a92175ef51ff00eba5ebbc0007962fecf8022ac1c6ce28"
+checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5"
dependencies = [
"bitflags",
- "serde",
+ "serde_core",
"serde_json",
]
[[package]]
name = "arrow-select"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d8854d15f1cf5005b4b358abeb60adea17091ff5bdd094dca5d3f73787d81170"
+checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47"
dependencies = [
"ahash",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
- "num",
+ "num-traits",
]
[[package]]
name = "arrow-string"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2c477e8b89e1213d5927a2a84a72c384a9bf4dd0dbf15f9fd66d821aafd9e95e"
+checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -471,11 +478,23 @@ dependencies = [
"arrow-schema",
"arrow-select",
"memchr",
- "num",
+ "num-traits",
"regex",
"regex-syntax",
]
+[[package]]
+name = "as-slice"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "45403b49e3954a4b8428a0ac21a4b7afadccf92bfd96273f1a58cd4812496ae0"
+dependencies = [
+ "generic-array 0.12.4",
+ "generic-array 0.13.3",
+ "generic-array 0.14.7",
+ "stable_deref_trait",
+]
+
[[package]]
name = "as_derive_utils"
version = "0.11.0"
@@ -534,6 +553,15 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "atomic-polyfill"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4"
+dependencies = [
+ "critical-section",
+]
+
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -559,9 +587,9 @@ checksum =
"c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-config"
-version = "1.8.11"
+version = "1.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a0149602eeaf915158e14029ba0c78dedb8c08d554b024d54c8f239aab46511d"
+checksum = "96571e6996817bf3d58f6b569e4b9fd2e9d2fcf9f7424eed07b2ce9bb87535e5"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -589,9 +617,9 @@ dependencies = [
[[package]]
name = "aws-credential-types"
-version = "1.2.10"
+version = "1.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b01c9521fa01558f750d183c8c68c81b0155b9d193a4ba7f84c36bd1b6d04a06"
+checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -601,9 +629,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.15.1"
+version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6b5ce75405893cd713f9ab8e297d8e438f624dde7d706108285f7e17a25a180f"
+checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -611,9 +639,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.34.0"
+version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "179c3777a8b5e70e90ea426114ffc565b2c1a9f82f6c4a0c5a34aa6ef5e781b6"
+checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1"
dependencies = [
"cc",
"cmake",
@@ -623,9 +651,9 @@ dependencies = [
[[package]]
name = "aws-runtime"
-version = "1.5.16"
+version = "1.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7ce527fb7e53ba9626fc47824f25e256250556c40d8f81d27dd92aa38239d632"
+checksum = "d81b5b2898f6798ad58f484856768bca817e3cd9de0974c24ae0f1113fe88f1b"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -647,9 +675,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
-version = "1.90.0"
+version = "1.91.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4f18e53542c522459e757f81e274783a78f8c81acdfc8d1522ee8a18b5fb1c66"
+checksum = "8ee6402a36f27b52fe67661c6732d684b2635152b676aa2babbfb5204f99115d"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -669,9 +697,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
-version = "1.92.0"
+version = "1.93.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "532f4d866012ffa724a4385c82e8dd0e59f0ca0e600f3f22d4c03b6824b34e4a"
+checksum = "a45a7f750bbd170ee3677671ad782d90b894548f4e4ae168302c57ec9de5cb3e"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -691,9 +719,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
-version = "1.94.0"
+version = "1.95.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1be6fbbfa1a57724788853a623378223fe828fc4c09b146c992f0c95b6256174"
+checksum = "55542378e419558e6b1f398ca70adb0b2088077e79ad9f14eb09441f2f7b2164"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -714,9 +742,9 @@ dependencies = [
[[package]]
name = "aws-sigv4"
-version = "1.3.6"
+version = "1.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11"
+checksum = "69e523e1c4e8e7e8ff219d732988e22bfeae8a1cafdbe6d9eca1546fa080be7c"
dependencies = [
"aws-credential-types",
"aws-smithy-http",
@@ -736,9 +764,9 @@ dependencies = [
[[package]]
name = "aws-smithy-async"
-version = "1.2.6"
+version = "1.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "127fcfad33b7dfc531141fda7e1c402ac65f88aca5511a4d31e2e3d2cd01ce9c"
+checksum = "9ee19095c7c4dda59f1697d028ce704c24b2d33c6718790c7f1d5a3015b4107c"
dependencies = [
"futures-util",
"pin-project-lite",
@@ -747,9 +775,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http"
-version = "0.62.5"
+version = "0.62.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca"
+checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
@@ -768,9 +796,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http-client"
-version = "1.1.4"
+version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "623254723e8dfd535f566ee7b2381645f8981da086b5c4aa26c0c41582bb1d2c"
+checksum = "59e62db736db19c488966c8d787f52e6270be565727236fd5579eaa301e7bc4a"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -792,27 +820,27 @@ dependencies = [
[[package]]
name = "aws-smithy-json"
-version = "0.61.7"
+version = "0.61.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2db31f727935fc63c6eeae8b37b438847639ec330a9161ece694efba257e0c54"
+checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551"
dependencies = [
"aws-smithy-types",
]
[[package]]
name = "aws-smithy-observability"
-version = "0.1.4"
+version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2d1881b1ea6d313f9890710d65c158bdab6fb08c91ea825f74c1c8c357baf4cc"
+checksum = "17f616c3f2260612fe44cede278bafa18e73e6479c4e393e2c4518cf2a9a228a"
dependencies = [
"aws-smithy-runtime-api",
]
[[package]]
name = "aws-smithy-query"
-version = "0.60.8"
+version = "0.60.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d28a63441360c477465f80c7abac3b9c4d075ca638f982e605b7dc2a2c7156c9"
+checksum = "ae5d689cf437eae90460e944a58b5668530d433b4ff85789e69d2f2a556e057d"
dependencies = [
"aws-smithy-types",
"urlencoding",
@@ -820,9 +848,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime"
-version = "1.9.4"
+version = "1.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0bbe9d018d646b96c7be063dd07987849862b0e6d07c778aad7d93d1be6c1ef0"
+checksum = "65fda37911905ea4d3141a01364bc5509a0f32ae3f3b22d6e330c0abfb62d247"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
@@ -844,9 +872,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
-version = "1.9.2"
+version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ec7204f9fd94749a7c53b26da1b961b4ac36bf070ef1e0b94bb09f79d4f6c193"
+checksum = "ab0d43d899f9e508300e587bf582ba54c27a452dd0a9ea294690669138ae14a2"
dependencies = [
"aws-smithy-async",
"aws-smithy-types",
@@ -861,9 +889,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
-version = "1.3.4"
+version = "1.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "25f535879a207fce0db74b679cfc3e91a3159c8144d717d55f5832aea9eef46e"
+checksum = "905cb13a9895626d49cf2ced759b062d913834c7482c38e49557eac4e6193f01"
dependencies = [
"base64-simd",
"bytes",
@@ -884,18 +912,18 @@ dependencies = [
[[package]]
name = "aws-smithy-xml"
-version = "0.60.12"
+version = "0.60.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "eab77cdd036b11056d2a30a7af7b775789fb024bf216acc13884c6c97752ae56"
+checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57"
dependencies = [
"xmlparser",
]
[[package]]
name = "aws-types"
-version = "1.3.10"
+version = "1.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d79fb68e3d7fe5d4833ea34dc87d2e97d26d3086cb3da660bb6b1f76d98680b6"
+checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
@@ -989,7 +1017,7 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
- "generic-array",
+ "generic-array 0.14.7",
]
[[package]]
@@ -1040,9 +1068,9 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.19.0"
+version = "3.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
+checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510"
[[package]]
name = "bytemuck"
@@ -1108,9 +1136,9 @@ checksum =
"37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.2.48"
+version = "1.2.50"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a"
+checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1250,9 +1278,9 @@ dependencies = [
[[package]]
name = "cmake"
-version = "0.1.54"
+version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
+checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"
dependencies = [
"cc",
]
@@ -1393,6 +1421,12 @@ dependencies = [
"itertools 0.13.0",
]
+[[package]]
+name = "critical-section"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
+
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
@@ -1462,7 +1496,7 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
- "generic-array",
+ "generic-array 0.14.7",
"typenum",
]
@@ -1573,12 +1607,11 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a"
+checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8"
dependencies = [
"arrow",
- "arrow-ipc",
"arrow-schema",
"async-trait",
"bytes",
@@ -1589,6 +1622,7 @@ dependencies = [
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
+ "datafusion-datasource-arrow",
"datafusion-datasource-avro",
"datafusion-datasource-csv",
"datafusion-datasource-json",
@@ -1616,8 +1650,9 @@ dependencies = [
"object_store",
"parking_lot",
"parquet",
- "rand 0.9.2",
+ "rand",
"regex",
+ "rstest",
"sqlparser",
"tempfile",
"tokio",
@@ -1629,9 +1664,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621"
+checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d"
dependencies = [
"arrow",
"async-trait",
@@ -1644,7 +1679,6 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-plan",
"datafusion-session",
- "datafusion-sql",
"futures",
"itertools 0.14.0",
"log",
@@ -1655,9 +1689,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83"
+checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0"
dependencies = [
"arrow",
"async-trait",
@@ -1667,10 +1701,11 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-expr",
+ "datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
- "datafusion-session",
"futures",
+ "itertools 0.14.0",
"log",
"object_store",
"tokio",
@@ -1678,15 +1713,14 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1"
+checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0"
dependencies = [
"ahash",
"apache-avro",
"arrow",
"arrow-ipc",
- "base64",
"chrono",
"half",
"hashbrown 0.14.5",
@@ -1704,9 +1738,9 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e"
+checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95"
dependencies = [
"futures",
"log",
@@ -1715,9 +1749,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86"
+checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6"
dependencies = [
"arrow",
"async-compression",
@@ -1740,9 +1774,7 @@ dependencies = [
"itertools 0.14.0",
"log",
"object_store",
- "parquet",
- "rand 0.9.2",
- "tempfile",
+ "rand",
"tokio",
"tokio-util",
"url",
@@ -1750,47 +1782,64 @@ dependencies = [
"zstd",
]
+[[package]]
+name = "datafusion-datasource-arrow"
+version = "51.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea"
+dependencies = [
+ "arrow",
+ "arrow-ipc",
+ "async-trait",
+ "bytes",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "futures",
+ "itertools 0.14.0",
+ "object_store",
+ "tokio",
+]
+
[[package]]
name = "datafusion-datasource-avro"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "10d40b6953ebc9099b37adfd12fde97eb73ff0cee44355c6dea64b8a4537d561"
+checksum = "388ed8be535f562cc655b9c3d22edbfb0f1a50a25c242647a98b6d92a75b55a1"
dependencies = [
"apache-avro",
"arrow",
"async-trait",
"bytes",
- "chrono",
- "datafusion-catalog",
"datafusion-common",
"datafusion-datasource",
- "datafusion-execution",
- "datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"num-traits",
"object_store",
- "tokio",
]
[[package]]
name = "datafusion-datasource-csv"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825"
+checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568"
dependencies = [
"arrow",
"async-trait",
"bytes",
- "datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
- "datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
@@ -1802,49 +1851,44 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6"
+checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85"
dependencies = [
"arrow",
"async-trait",
"bytes",
- "datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
- "datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"object_store",
- "serde_json",
"tokio",
]
[[package]]
name = "datafusion-datasource-parquet"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133"
+checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9"
dependencies = [
"arrow",
"async-trait",
"bytes",
- "datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
- "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
- "datafusion-physical-optimizer",
"datafusion-physical-plan",
"datafusion-pruning",
"datafusion-session",
@@ -1854,21 +1898,20 @@ dependencies = [
"object_store",
"parking_lot",
"parquet",
- "rand 0.9.2",
"tokio",
]
[[package]]
name = "datafusion-doc"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429"
+checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07"
[[package]]
name = "datafusion-execution"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531"
+checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178"
dependencies = [
"arrow",
"async-trait",
@@ -1879,16 +1922,16 @@ dependencies = [
"log",
"object_store",
"parking_lot",
- "rand 0.9.2",
+ "rand",
"tempfile",
"url",
]
[[package]]
name = "datafusion-expr"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe"
+checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91"
dependencies = [
"arrow",
"async-trait",
@@ -1900,6 +1943,7 @@ dependencies = [
"datafusion-functions-window-common",
"datafusion-physical-expr-common",
"indexmap",
+ "itertools 0.14.0",
"paste",
"recursive",
"serde_json",
@@ -1908,9 +1952,9 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6"
+checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc"
dependencies = [
"arrow",
"datafusion-common",
@@ -1921,9 +1965,9 @@ dependencies = [
[[package]]
name = "datafusion-ffi"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "25ddb7c4e645df080c27dad13a198d191da328dd1c98e198664a7a0f64b335cc"
+checksum = "ec510e7787641279b0336e8b79e4b7bd1385d5976875ff9b97f4269ce5231a67"
dependencies = [
"abi_stable",
"arrow",
@@ -1931,6 +1975,7 @@ dependencies = [
"async-ffi",
"async-trait",
"datafusion",
+ "datafusion-common",
"datafusion-functions-aggregate-common",
"datafusion-proto",
"datafusion-proto-common",
@@ -1943,9 +1988,9 @@ dependencies = [
[[package]]
name = "datafusion-functions"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977"
+checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f"
dependencies = [
"arrow",
"arrow-buffer",
@@ -1963,7 +2008,8 @@ dependencies = [
"itertools 0.14.0",
"log",
"md-5",
- "rand 0.9.2",
+ "num-traits",
+ "rand",
"regex",
"sha2",
"unicode-segmentation",
@@ -1972,9 +2018,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf"
+checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305"
dependencies = [
"ahash",
"arrow",
@@ -1993,9 +2039,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17"
+checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d"
dependencies = [
"ahash",
"arrow",
@@ -2006,9 +2052,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3"
+checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4"
dependencies = [
"arrow",
"arrow-ord",
@@ -2016,6 +2062,7 @@ dependencies = [
"datafusion-doc",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-expr-common",
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-aggregate-common",
@@ -2028,9 +2075,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981"
+checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5"
dependencies = [
"arrow",
"async-trait",
@@ -2044,9 +2091,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b"
+checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1"
dependencies = [
"arrow",
"datafusion-common",
@@ -2062,9 +2109,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f"
+checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2072,20 +2119,20 @@ dependencies = [
[[package]]
name = "datafusion-macros"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730"
+checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848"
dependencies = [
- "datafusion-expr",
+ "datafusion-doc",
"quote",
"syn 2.0.111",
]
[[package]]
name = "datafusion-optimizer"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a"
+checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a"
dependencies = [
"arrow",
"chrono",
@@ -2103,9 +2150,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6"
+checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d"
dependencies = [
"ahash",
"arrow",
@@ -2118,7 +2165,6 @@ dependencies = [
"hashbrown 0.14.5",
"indexmap",
"itertools 0.14.0",
- "log",
"parking_lot",
"paste",
"petgraph",
@@ -2126,9 +2172,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0"
+checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768"
dependencies = [
"arrow",
"datafusion-common",
@@ -2141,9 +2187,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3"
+checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03"
dependencies = [
"ahash",
"arrow",
@@ -2155,9 +2201,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c"
+checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6"
dependencies = [
"arrow",
"datafusion-common",
@@ -2169,15 +2215,14 @@ dependencies = [
"datafusion-physical-plan",
"datafusion-pruning",
"itertools 0.14.0",
- "log",
"recursive",
]
[[package]]
name = "datafusion-physical-plan"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c"
+checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac"
dependencies = [
"ahash",
"arrow",
@@ -2206,15 +2251,26 @@ dependencies = [
[[package]]
name = "datafusion-proto"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a7df9f606892e6af45763d94d210634eec69b9bb6ced5353381682ff090028a3"
+checksum = "d368093a98a17d1449b1083ac22ed16b7128e4c67789991869480d8c4a40ecb9"
dependencies = [
"arrow",
"chrono",
- "datafusion",
+ "datafusion-catalog",
+ "datafusion-catalog-listing",
"datafusion-common",
+ "datafusion-datasource",
+ "datafusion-datasource-arrow",
+ "datafusion-datasource-csv",
+ "datafusion-datasource-json",
+ "datafusion-datasource-parquet",
+ "datafusion-execution",
"datafusion-expr",
+ "datafusion-functions-table",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
"datafusion-proto-common",
"object_store",
"prost",
@@ -2222,9 +2278,9 @@ dependencies = [
[[package]]
name = "datafusion-proto-common"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b4b14f288ca4ef77743d9672cafecf3adfffff0b9b04af9af79ecbeaaf736901"
+checksum = "3b6aef3d5e5c1d2bc3114c4876730cb76a9bdc5a8df31ef1b6db48f0c1671895"
dependencies = [
"arrow",
"datafusion-common",
@@ -2233,12 +2289,11 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65"
+checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db"
dependencies = [
"arrow",
- "arrow-schema",
"datafusion-common",
"datafusion-datasource",
"datafusion-expr-common",
@@ -2251,36 +2306,27 @@ dependencies = [
[[package]]
name = "datafusion-session"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d"
+checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776"
dependencies = [
- "arrow",
"async-trait",
- "dashmap",
"datafusion-common",
- "datafusion-common-runtime",
"datafusion-execution",
"datafusion-expr",
- "datafusion-physical-expr",
"datafusion-physical-plan",
- "datafusion-sql",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
"parking_lot",
- "tokio",
]
[[package]]
name = "datafusion-sql"
-version = "50.3.0"
+version = "51.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b"
+checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f"
dependencies = [
"arrow",
"bigdecimal",
+ "chrono",
"datafusion-common",
"datafusion-expr",
"indexmap",
@@ -2642,6 +2688,24 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "generic-array"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd"
+dependencies = [
+ "typenum",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f797e67af32588215eaaab8327027ee8e71b9dd0b2b26996aedf20c030fce309"
+dependencies = [
+ "typenum",
+]
+
[[package]]
name = "generic-array"
version = "0.14.7"
@@ -2666,7 +2730,7 @@ dependencies = [
"log",
"num-traits",
"robust",
- "rstar",
+ "rstar 0.12.2",
"spade",
]
@@ -2703,7 +2767,11 @@ dependencies = [
"approx",
"num-traits",
"rayon",
- "rstar",
+ "rstar 0.10.0",
+ "rstar 0.11.0",
+ "rstar 0.12.2",
+ "rstar 0.8.4",
+ "rstar 0.9.3",
"serde",
]
@@ -2716,6 +2784,19 @@ dependencies = [
"libm",
]
+[[package]]
+name = "geojson"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e26f3c45b36fccc9cf2805e61d4da6bc4bbd5a3a9589b01afa3a40eff703bd79"
+dependencies = [
+ "geo-types",
+ "log",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.17",
+]
+
[[package]]
name = "geos"
version = "10.0.0"
@@ -2802,6 +2883,24 @@ dependencies = [
"zerocopy",
]
+[[package]]
+name = "hash32"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d4041af86e63ac4298ce40e5cca669066e75b6f1aa3390fe2561ffa5e1d9f4cc"
+dependencies = [
+ "byteorder",
+]
+
+[[package]]
+name = "hash32"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67"
+dependencies = [
+ "byteorder",
+]
+
[[package]]
name = "hash32"
version = "0.3.1"
@@ -2838,13 +2937,38 @@ version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+[[package]]
+name = "heapless"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422"
+dependencies = [
+ "as-slice",
+ "generic-array 0.14.7",
+ "hash32 0.1.1",
+ "stable_deref_trait",
+]
+
+[[package]]
+name = "heapless"
+version = "0.7.17"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f"
+dependencies = [
+ "atomic-polyfill",
+ "hash32 0.2.1",
+ "rustc_version",
+ "spin",
+ "stable_deref_trait",
+]
+
[[package]]
name = "heapless"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad"
dependencies = [
- "hash32",
+ "hash32 0.3.1",
"stable_deref_trait",
]
@@ -3141,9 +3265,9 @@ checksum =
"7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
[[package]]
name = "icu_properties"
-version = "2.1.1"
+version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99"
+checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
dependencies = [
"icu_collections",
"icu_locale_core",
@@ -3155,9 +3279,9 @@ dependencies = [
[[package]]
name = "icu_properties_data"
-version = "2.1.1"
+version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899"
+checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
[[package]]
name = "icu_provider"
@@ -3432,13 +3556,13 @@ dependencies = [
[[package]]
name = "libredox"
-version = "0.1.10"
+version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb"
+checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50"
dependencies = [
"bitflags",
"libc",
- "redox_syscall",
+ "redox_syscall 0.6.0",
]
[[package]]
@@ -3453,9 +3577,9 @@ dependencies = [
[[package]]
name = "libz-rs-sys"
-version = "0.5.3"
+version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8b484ba8d4f775eeca644c452a56650e544bf7e617f1d170fe7298122ead5222"
+checksum = "15413ef615ad868d4d65dce091cb233b229419c7c0c4bcaa746c0901c49ff39c"
dependencies = [
"zlib-rs",
]
@@ -3762,7 +3886,7 @@ dependencies = [
"parking_lot",
"percent-encoding",
"quick-xml",
- "rand 0.9.2",
+ "rand",
"reqwest",
"ring",
"rustls-pemfile",
@@ -3851,16 +3975,16 @@ checksum =
"2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall",
+ "redox_syscall 0.5.18",
"smallvec",
"windows-link",
]
[[package]]
name = "parquet"
-version = "56.0.0"
+version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c7288a07ed5d25939a90f9cb1ca5afa6855faa08ec7700613511ae64bdb0620c"
+checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a"
dependencies = [
"ahash",
"arrow-array",
@@ -3877,13 +4001,13 @@ dependencies = [
"flate2",
"futures",
"half",
- "hashbrown 0.15.5",
+ "hashbrown 0.16.1",
"lz4_flex",
- "num",
"num-bigint",
+ "num-integer",
+ "num-traits",
"object_store",
"paste",
- "ring",
"seq-macro",
"simdutf8",
"snap",
@@ -3899,6 +4023,12 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+[[package]]
+name = "pdqselect"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4ec91767ecc0a0bbe558ce8c9da33c068066c57ecc8bb8477ef8c1ad3ef77c27"
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -4074,9 +4204,9 @@ dependencies = [
[[package]]
name = "prost"
-version = "0.13.5"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
+checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d"
dependencies = [
"bytes",
"prost-derive",
@@ -4084,9 +4214,9 @@ dependencies = [
[[package]]
name = "prost-derive"
-version = "0.13.5"
+version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
+checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -4218,7 +4348,7 @@ dependencies = [
"bytes",
"getrandom 0.3.4",
"lru-slab",
- "rand 0.9.2",
+ "rand",
"ring",
"rustc-hash",
"rustls",
@@ -4269,35 +4399,14 @@ dependencies = [
"nibble_vec",
]
-[[package]]
-name = "rand"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
-dependencies = [
- "libc",
- "rand_chacha 0.3.1",
- "rand_core 0.6.4",
-]
-
[[package]]
name = "rand"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
dependencies = [
- "rand_chacha 0.9.0",
- "rand_core 0.9.3",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.6.4",
+ "rand_chacha",
+ "rand_core",
]
[[package]]
@@ -4307,16 +4416,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core 0.9.3",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom 0.2.16",
+ "rand_core",
]
[[package]]
@@ -4335,7 +4435,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
dependencies = [
"num-traits",
- "rand 0.9.2",
+ "rand",
]
[[package]]
@@ -4387,6 +4487,15 @@ dependencies = [
"bitflags",
]
+[[package]]
+name = "redox_syscall"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ec96166dafa0886eb81fe1c0a388bece180fbef2135f97c1e2cf8302e74b43b5"
+dependencies = [
+ "bitflags",
+]
+
[[package]]
name = "redox_users"
version = "0.5.2"
@@ -4450,9 +4559,9 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.12.24"
+version = "0.12.26"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f"
+checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f"
dependencies = [
"base64",
"bytes",
@@ -4510,14 +4619,64 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839"
+[[package]]
+name = "rstar"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3a45c0e8804d37e4d97e55c6f258bc9ad9c5ee7b07437009dd152d764949a27c"
+dependencies = [
+ "heapless 0.6.1",
+ "num-traits",
+ "pdqselect",
+ "serde",
+ "smallvec",
+]
+
+[[package]]
+name = "rstar"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b40f1bfe5acdab44bc63e6699c28b74f75ec43afb59f3eda01e145aff86a25fa"
+dependencies = [
+ "heapless 0.7.17",
+ "num-traits",
+ "serde",
+ "smallvec",
+]
+
+[[package]]
+name = "rstar"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1f39465655a1e3d8ae79c6d9e007f4953bfc5d55297602df9dc38f9ae9f1359a"
+dependencies = [
+ "heapless 0.7.17",
+ "num-traits",
+ "serde",
+ "smallvec",
+]
+
+[[package]]
+name = "rstar"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "73111312eb7a2287d229f06c00ff35b51ddee180f017ab6dec1f69d62ac098d6"
+dependencies = [
+ "heapless 0.7.17",
+ "num-traits",
+ "serde",
+ "smallvec",
+]
+
[[package]]
name = "rstar"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb"
dependencies = [
- "heapless",
+ "heapless 0.8.0",
"num-traits",
+ "serde",
"smallvec",
]
@@ -4616,9 +4775,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
-version = "1.13.1"
+version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c"
+checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282"
dependencies = [
"web-time",
"zeroize",
@@ -4761,7 +4920,6 @@ dependencies = [
name = "sedona"
version = "0.3.0"
dependencies = [
- "abi_stable",
"arrow-array",
"arrow-schema",
"async-trait",
@@ -4939,6 +5097,7 @@ dependencies = [
"geo",
"geo-traits",
"geo-types",
+ "geojson",
"rstest",
"sedona-expr",
"sedona-functions",
@@ -4946,6 +5105,7 @@ dependencies = [
"sedona-geometry",
"sedona-schema",
"sedona-testing",
+ "serde_json",
"wkb",
"wkt 0.14.0",
]
@@ -4964,10 +5124,10 @@ dependencies = [
"log",
"num-traits",
"pretty_env_logger",
- "rand 0.8.5",
+ "rand",
"rand_distr",
"robust",
- "rstar",
+ "rstar 0.12.2",
"sedona-geo-traits-ext",
"sedona-testing",
"wkb",
@@ -5220,7 +5380,7 @@ dependencies = [
"geos",
"once_cell",
"parking_lot",
- "rand 0.8.5",
+ "rand",
"rstest",
"sedona-common",
"sedona-expr",
@@ -5254,7 +5414,7 @@ dependencies = [
"geo-traits",
"geo-types",
"parquet",
- "rand 0.8.5",
+ "rand",
"rstest",
"sedona-common",
"sedona-expr",
@@ -5464,9 +5624,9 @@ dependencies = [
[[package]]
name = "simd-adler32"
-version = "0.3.7"
+version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
+checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
[[package]]
name = "simdutf8"
@@ -5520,11 +5680,20 @@ dependencies = [
"smallvec",
]
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+dependencies = [
+ "lock_api",
+]
+
[[package]]
name = "sqlparser"
-version = "0.58.0"
+version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c"
+checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f"
dependencies = [
"log",
"recursive",
@@ -5652,9 +5821,9 @@ dependencies = [
[[package]]
name = "target-lexicon"
-version = "0.13.3"
+version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c"
+checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba"
[[package]]
name = "tempfile"
@@ -5856,18 +6025,18 @@ dependencies = [
[[package]]
name = "toml_datetime"
-version = "0.7.3"
+version = "0.7.5+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533"
+checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
dependencies = [
"serde_core",
]
[[package]]
name = "toml_edit"
-version = "0.23.7"
+version = "0.23.10+spec-1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d"
+checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
dependencies = [
"indexmap",
"toml_datetime",
@@ -5877,9 +6046,9 @@ dependencies = [
[[package]]
name = "toml_parser"
-version = "1.0.4"
+version = "1.0.6+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e"
+checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44"
dependencies = [
"winnow",
]
@@ -5901,9 +6070,9 @@ dependencies = [
[[package]]
name = "tower-http"
-version = "0.6.7"
+version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456"
+checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
dependencies = [
"bitflags",
"bytes",
@@ -5931,9 +6100,9 @@ checksum =
"8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
-version = "0.1.43"
+version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647"
+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"pin-project-lite",
"tracing-attributes",
@@ -5953,9 +6122,9 @@ dependencies = [
[[package]]
name = "tracing-core"
-version = "0.1.35"
+version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c"
+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
dependencies = [
"once_cell",
]
@@ -6680,9 +6849,9 @@ dependencies = [
[[package]]
name = "zlib-rs"
-version = "0.5.3"
+version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7"
+checksum = "51f936044d677be1a1168fae1d03b583a285a5dd9d8cbf7b24c23aa1fc775235"
[[package]]
name = "zstd"
diff --git a/Cargo.toml b/Cargo.toml
index a84e149c..7a0d999c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -54,46 +54,44 @@ description = "Apache SedonaDB Rust API"
readme = "README.md"
edition = "2021"
-# Same rust-version, arrow, and datafusion dependencies as datafusion-comet
-# https://github.com/apache/datafusion-comet/blob/main/native/Cargo.toml
-rust-version = "1.86"
+rust-version = "1.88.0"
keywords = ["geospatial", "gis", "spatial", "datafusion", "arrow"]
categories = ["science::geo", "database"]
[workspace.dependencies]
approx = "0.5"
-abi_stable = "0.11.3"
adbc_core = ">=0.21.0"
adbc_ffi = ">=0.21.0"
lru = "0.12"
-arrow = { version = "56.0.0", features = ["prettyprint", "ffi", "chrono-tz"] }
-arrow-array = { version = "56.0.0" }
-arrow-cast = { version = "56.0.0" }
-arrow-data = { version = "56.0.0" }
-arrow-json = { version = "56.0.0" }
-arrow-schema = { version = "56.0.0" }
-arrow-buffer = { version = "56.0.0" }
+arrow = { version = "57.0.0", features = ["prettyprint", "ffi", "chrono-tz"] }
+arrow-array = { version = "57.0.0" }
+arrow-cast = { version = "57.0.0" }
+arrow-data = { version = "57.0.0" }
+arrow-json = { version = "57.0.0" }
+arrow-schema = { version = "57.0.0" }
+arrow-buffer = { version = "57.0.0" }
async-trait = { version = "0.1.87" }
bytes = "1.10"
byteorder = "1"
-chrono = { version = "0.4.38", default-features = false }
-comfy-table = { version = "7.1.4" }
+chrono = { version = "0.4.41", default-features = false }
+comfy-table = { version = "7" }
criterion = { version = "0.8", features = ["html_reports"] }
-datafusion = { version = "50.2.0", default-features = false }
-datafusion-catalog = { version = "50.2.0" }
-datafusion-common = { version = "50.2.0", default-features = false }
-datafusion-common-runtime = { version = "50.2.0", default-features = false }
-datafusion-datasource-parquet = { version = "50.2.0" }
-datafusion-execution = { version = "50.2.0", default-features = false }
-datafusion-expr = { version = "50.2.0" }
-datafusion-ffi = { version = "50.2.0" }
-datafusion-physical-expr = { version = "50.2.0" }
-datafusion-physical-plan = { version = "50.2.0" }
+datafusion = { version = "51.0.0", default-features = false }
+datafusion-catalog = { version = "51.0.0" }
+datafusion-common = { version = "51.0.0", default-features = false }
+datafusion-common-runtime = { version = "51.0.0", default-features = false }
+datafusion-datasource-parquet = { version = "51.0.0" }
+datafusion-execution = { version = "51.0.0", default-features = false }
+datafusion-expr = { version = "51.0.0" }
+datafusion-ffi = { version = "51.0.0" }
+datafusion-functions-nested = { version = "51.0.0" }
+datafusion-physical-expr = { version = "51.0.0" }
+datafusion-physical-plan = { version = "51.0.0" }
dirs = "6.0.0"
env_logger = "0.11"
fastrand = "2.0"
-futures = { version = "0.3" }
-object_store = { version = "0.12.0", default-features = false }
+futures = "0.3"
+object_store = { version = "0.12.4", default-features = false }
float_next_after = "1"
num-traits = { version = "0.2", default-features = false, features = ["libm"] }
mimalloc = { version = "0.1", default-features = false }
@@ -113,21 +111,21 @@ wkb = "0.9.2"
wkt = "0.14.0"
parking_lot = "0.12"
-parquet = { version = "56.0.0", default-features = false, features = [
+parquet = { version = "57.0.0", default-features = false, features = [
"arrow",
"async",
"object_store",
] }
-rand = "0.8"
-regex = "1.8"
+rand = "0.9"
+regex = "1.12"
rstest = "0.26.1"
serde = { version = "1" }
serde_json = { version = "1" }
serde_with = { version = "1" }
tempfile = { version = "3"}
thiserror = { version = "2" }
-tokio = { version = "1.44" }
-url = "2.5.4"
+tokio = { version = "1.48", features = ["macros", "rt", "sync"] }
+url = "2.5.7"
# Workspace path dependencies for internal crates
sedona = { version = "0.3.0", path = "rust/sedona" }
diff --git a/c/sedona-libgpuspatial/build.rs b/c/sedona-libgpuspatial/build.rs
index 06eac921..6bf5f3f8 100644
--- a/c/sedona-libgpuspatial/build.rs
+++ b/c/sedona-libgpuspatial/build.rs
@@ -135,14 +135,14 @@ fn main() {
// Detect CUDA library path from CUDA_HOME or default locations
let cuda_lib_path = if let Ok(cuda_home) = env::var("CUDA_HOME") {
- format!("{}/lib64", cuda_home)
+ format!("{cuda_home}/lib64")
} else if std::path::Path::new("/usr/local/cuda/lib64").exists() {
"/usr/local/cuda/lib64".to_string()
} else {
panic!("CUDA lib is not found. Neither CUDA_HOME is set nor the
default path /usr/local/cuda/lib64 exists.");
};
- println!("cargo:rustc-link-search=native={}", cuda_lib_path); // CUDA
runtime
+ println!("cargo:rustc-link-search=native={cuda_lib_path}"); // CUDA
runtime
if let Some(driver_lib_path) = find_cuda_driver_path() {
println!(
diff --git a/c/sedona-s2geography/s2geography b/c/sedona-s2geography/s2geography
index e63647cb..c4d4e5f7 160000
--- a/c/sedona-s2geography/s2geography
+++ b/c/sedona-s2geography/s2geography
@@ -1 +1 @@
-Subproject commit e63647cb4536648b06cc5aa6b97ce1777a5f6163
+Subproject commit c4d4e5f7416dc203d3cb0485d56f5d72e9ccb6dd
diff --git a/python/sedonadb/tests/test_sjoin.py
b/python/sedonadb/tests/test_sjoin.py
index acf2c41f..beb412ce 100644
--- a/python/sedonadb/tests/test_sjoin.py
+++ b/python/sedonadb/tests/test_sjoin.py
@@ -106,7 +106,7 @@ def test_spatial_join_geography(join_type, on):
"vertices_per_linestring_range": [2, 10],
"bounds": west_most_bound,
"size_range": [0.1, 5],
- "seed": 43,
+ "seed": 958,
}
)
df_point = eng_sedonadb.execute_and_collect(
diff --git a/r/sedonadb/src/Makevars.win.in b/r/sedonadb/src/Makevars.win.in
index 6ee99c74..24cb6eb6 100644
--- a/r/sedonadb/src/Makevars.win.in
+++ b/r/sedonadb/src/Makevars.win.in
@@ -27,7 +27,7 @@ RUSTFLAGS =
TARGET_DIR = @TARGET_DIR@
LIBDIR = $(TARGET_DIR)/$(TARGET)/$(subst dev,debug,$(PROFILE))
STATLIB = $(LIBDIR)/libsedonadbr.a
-PKG_LIBS = -L$(LIBDIR) -lsedonadbr -lws2_32 -ladvapi32 -luserenv -lbcrypt
-lntdll @PKG_LIBS@
+PKG_LIBS = -L$(LIBDIR) -lsedonadbr -lws2_32 -ladvapi32 -luserenv -lbcrypt
-lntdll -lole32 @PKG_LIBS@
# Rtools doesn't have the linker in the location that cargo expects, so we need
# to overwrite it via configuration.
diff --git a/r/sedonadb/src/rust/src/dataframe.rs
b/r/sedonadb/src/rust/src/dataframe.rs
index 45a3cf97..e34cee82 100644
--- a/r/sedonadb/src/rust/src/dataframe.rs
+++ b/r/sedonadb/src/rust/src/dataframe.rs
@@ -139,7 +139,10 @@ impl InternalDataFrame {
let schema = self.inner.schema();
let batches =
wait_for_future_captured_r(&self.runtime,
self.inner.clone().collect_partitioned())??;
- let provider = Arc::new(MemTable::try_new(schema.clone().into(),
batches)?);
+ let provider = Arc::new(MemTable::try_new(
+ schema.as_arrow().clone().into(),
+ batches,
+ )?);
let inner = ctx.inner.ctx.read_table(provider)?;
Ok(new_data_frame(inner, self.runtime.clone()))
}
diff --git a/rust/sedona-datasource/src/format.rs
b/rust/sedona-datasource/src/format.rs
index 69a2bc9f..005a6d6a 100644
--- a/rust/sedona-datasource/src/format.rs
+++ b/rust/sedona-datasource/src/format.rs
@@ -25,9 +25,10 @@ use datafusion::{
file_format::{file_compression_type::FileCompressionType, FileFormat,
FileFormatFactory},
listing::PartitionedFile,
physical_plan::{
- FileGroupPartitioner, FileMeta, FileOpenFuture, FileOpener,
FileScanConfig,
- FileSinkConfig, FileSource,
+ FileGroupPartitioner, FileOpenFuture, FileOpener, FileScanConfig,
FileSinkConfig,
+ FileSource,
},
+ table_schema::TableSchema,
},
};
use datafusion_catalog::{memory::DataSourceExec, Session};
@@ -204,7 +205,7 @@ impl FileFormat for ExternalFileFormat {
struct ExternalFileSource {
spec: Arc<dyn ExternalFormatSpec>,
batch_size: Option<usize>,
- file_schema: Option<SchemaRef>,
+ file_schema: Option<TableSchema>,
file_projection: Option<Vec<usize>>,
filters: Vec<Arc<dyn PhysicalExpr>>,
metrics: ExecutionPlanMetricsSet,
@@ -240,7 +241,7 @@ impl FileSource for ExternalFileSource {
range: None,
},
batch_size: self.batch_size,
- file_schema: self.file_schema.clone(),
+ file_schema: self.file_schema.as_ref().map(|s|
s.file_schema().clone()),
file_projection: self.file_projection.clone(),
filters: self.filters.clone(),
};
@@ -285,7 +286,7 @@ impl FileSource for ExternalFileSource {
})
}
- fn with_schema(&self, schema: SchemaRef) -> Arc<dyn FileSource> {
+ fn with_schema(&self, schema: TableSchema) -> Arc<dyn FileSource> {
Arc::new(Self {
file_schema: Some(schema),
..self.clone()
@@ -360,11 +361,11 @@ struct ExternalFileOpener {
}
impl FileOpener for ExternalFileOpener {
- fn open(&self, file_meta: FileMeta, _file: PartitionedFile) ->
Result<FileOpenFuture> {
+ fn open(&self, file: PartitionedFile) -> Result<FileOpenFuture> {
let mut self_clone = self.clone();
Ok(Box::pin(async move {
- self_clone.args.src.meta.replace(file_meta.object_meta);
- self_clone.args.src.range = file_meta.range;
+ self_clone.args.src.meta.replace(file.object_meta);
+ self_clone.args.src.range = file.range;
let reader = self_clone.spec.open_reader(&self_clone.args).await?;
let stream =
futures::stream::iter(reader.into_iter().map(|batch|
batch.map_err(Into::into)));
diff --git a/rust/sedona-functions/Cargo.toml b/rust/sedona-functions/Cargo.toml
index 9fb7f35f..4d0cedbe 100644
--- a/rust/sedona-functions/Cargo.toml
+++ b/rust/sedona-functions/Cargo.toml
@@ -35,7 +35,7 @@ arrow-json = { workspace = true }
criterion = { workspace = true }
rstest = { workspace = true }
sedona-testing = { workspace = true, features = ["criterion"] }
-datafusion = { workspace = true }
+datafusion = { workspace = true, features = ["sql"] }
tokio = { workspace = true, features = ["rt", "macros"] }
[dependencies]
diff --git a/rust/sedona-functions/src/barrier.rs
b/rust/sedona-functions/src/barrier.rs
index 83d7fefd..6cd7e9bd 100644
--- a/rust/sedona-functions/src/barrier.rs
+++ b/rust/sedona-functions/src/barrier.rs
@@ -89,7 +89,7 @@ impl SedonaScalarKernel for Barrier {
// Remaining arguments should be pairs of (string, any_type)
let remaining_args = &args[1..];
- if remaining_args.len() % 2 != 0 {
+ if !remaining_args.len().is_multiple_of(2) {
return exec_err!(
"Arguments after expression must be pairs of (column_name,
column_value)"
);
diff --git a/rust/sedona-functions/src/st_dump.rs
b/rust/sedona-functions/src/st_dump.rs
index 3b404105..aafbc4f6 100644
--- a/rust/sedona-functions/src/st_dump.rs
+++ b/rust/sedona-functions/src/st_dump.rs
@@ -195,7 +195,7 @@ impl STDumpBuilder {
DataType::List(Arc::new(Field::new("item", DataType::UInt32,
true))),
true,
),
- Field::new("geom", DataType::Binary, true),
+ WKB_GEOMETRY.to_storage_field("geom", true).unwrap(),
]);
let struct_array = StructArray::try_new(
fields.clone(),
diff --git a/rust/sedona-geoparquet/src/file_opener.rs
b/rust/sedona-geoparquet/src/file_opener.rs
index d634b92a..e0aa5274 100644
--- a/rust/sedona-geoparquet/src/file_opener.rs
+++ b/rust/sedona-geoparquet/src/file_opener.rs
@@ -19,7 +19,7 @@ use std::{collections::HashMap, sync::Arc};
use arrow_schema::SchemaRef;
use datafusion::datasource::{
listing::PartitionedFile,
- physical_plan::{parquet::ParquetAccessPlan, FileMeta, FileOpenFuture,
FileOpener},
+ physical_plan::{parquet::ParquetAccessPlan, FileOpenFuture, FileOpener},
};
use datafusion_common::Result;
use datafusion_datasource_parquet::metadata::DFParquetMetadata;
@@ -112,12 +112,12 @@ impl GeoParquetFileOpener {
}
impl FileOpener for GeoParquetFileOpener {
- fn open(&self, file_meta: FileMeta, file: PartitionedFile) ->
Result<FileOpenFuture> {
+ fn open(&self, file: PartitionedFile) -> Result<FileOpenFuture> {
let self_clone = self.clone();
Ok(Box::pin(async move {
let parquet_metadata =
- DFParquetMetadata::new(&self_clone.object_store,
&file_meta.object_meta)
+ DFParquetMetadata::new(&self_clone.object_store,
&file.object_meta)
.with_metadata_size_hint(self_clone.metadata_size_hint)
.fetch_metadata()
.await?;
@@ -154,15 +154,9 @@ impl FileOpener for GeoParquetFileOpener {
// We could also consider filtering using null_count here in the
future (i.e.,
// skip row groups that are all null)
+ let file = file.with_extensions(Arc::new(access_plan));
- let file_meta = FileMeta {
- object_meta: file_meta.object_meta,
- range: file_meta.range,
- extensions: Some(Arc::new(access_plan)),
- metadata_size_hint: self_clone.metadata_size_hint,
- };
-
- self_clone.inner.open(file_meta, file)?.await
+ self_clone.inner.open(file)?.await
}))
}
}
diff --git a/rust/sedona-geoparquet/src/format.rs
b/rust/sedona-geoparquet/src/format.rs
index e20adb1c..c31b2d25 100644
--- a/rust/sedona-geoparquet/src/format.rs
+++ b/rust/sedona-geoparquet/src/format.rs
@@ -30,6 +30,7 @@ use datafusion::{
physical_plan::{
FileOpener, FileScanConfig, FileScanConfigBuilder, FileSinkConfig,
FileSource,
},
+ table_schema::TableSchema,
},
};
use datafusion_catalog::{memory::DataSourceExec, Session};
@@ -363,7 +364,7 @@ impl GeoParquetFileSource {
if let Some(parquet_source) =
inner.as_any().downcast_ref::<ParquetSource>() {
let parquet_source = parquet_source.clone();
// Extract the predicate from the existing source if it exists so
we can keep a copy of it
- let new_predicate = match (parquet_source.predicate().cloned(),
predicate) {
+ let new_predicate = match (parquet_source.filter(), predicate) {
(None, None) => None,
(None, Some(specified_predicate)) => Some(specified_predicate),
(Some(inner_predicate), None) => Some(inner_predicate),
@@ -444,7 +445,7 @@ impl FileSource for GeoParquetFileSource {
.create_file_opener(object_store.clone(), base_config,
partition);
// If there are no geo columns or no pruning predicate, just return
the inner opener
- if self.predicate.is_none() ||
!storage_schema_contains_geo(&base_config.file_schema) {
+ if self.predicate.is_none() ||
!storage_schema_contains_geo(base_config.file_schema()) {
return inner_opener;
}
@@ -453,7 +454,7 @@ impl FileSource for GeoParquetFileSource {
object_store,
self.metadata_size_hint,
self.predicate.clone().unwrap(),
- base_config.file_schema.clone(),
+ base_config.file_schema().clone(),
self.inner.table_parquet_options().global.pruning,
// HACK: Since there is no public API to set inner's metrics, so
we use
// inner's metrics as the ExecutionPlan-global metrics
@@ -492,7 +493,7 @@ impl FileSource for GeoParquetFileSource {
))
}
- fn with_schema(&self, schema: SchemaRef) -> Arc<dyn FileSource> {
+ fn with_schema(&self, schema: TableSchema) -> Arc<dyn FileSource> {
Arc::new(Self::from_file_source(
self.inner.with_schema(schema),
self.metadata_size_hint,
@@ -818,7 +819,7 @@ mod test {
GeoParquetFileSource::try_from_file_source(Arc::new(parquet_source), None, None)
.unwrap();
let geo_source_with_predicate = geo_source.with_predicate(predicate);
- assert!(geo_source_with_predicate.inner.predicate().is_some());
+ assert!(geo_source_with_predicate.inner.filter().is_some());
}
#[tokio::test]
diff --git a/rust/sedona-raster-functions/src/rs_srid.rs
b/rust/sedona-raster-functions/src/rs_srid.rs
index b3e3709c..1697fe33 100644
--- a/rust/sedona-raster-functions/src/rs_srid.rs
+++ b/rust/sedona-raster-functions/src/rs_srid.rs
@@ -109,8 +109,7 @@ impl SedonaScalarKernel for RsSrid {
Some(crs_str) => {
let crs = deserialize_crs(crs_str).map_err(|e| {
DataFusionError::Execution(format!(
- "Failed to deserialize CRS: {}",
- e
+ "Failed to deserialize CRS: {e}"
))
})?;
@@ -118,8 +117,7 @@ impl SedonaScalarKernel for RsSrid {
Some(crs_ref) => {
let srid = crs_ref.srid().map_err(|e| {
DataFusionError::Execution(format!(
- "Failed to get SRID from CRS: {}",
- e
+ "Failed to get SRID from CRS: {e}"
))
})?;
@@ -175,7 +173,7 @@ impl SedonaScalarKernel for RsCrs {
None => builder.append_null(),
Some(crs_str) => {
let crs = deserialize_crs(crs_str).map_err(|e| {
- DataFusionError::Execution(format!("Failed to
deserialize CRS: {}", e))
+ DataFusionError::Execution(format!("Failed to
deserialize CRS: {e}"))
})?;
let crs_string = crs
diff --git a/rust/sedona-schema/src/crs.rs b/rust/sedona-schema/src/crs.rs
index 295d1d84..2a77ad60 100644
--- a/rust/sedona-schema/src/crs.rs
+++ b/rust/sedona-schema/src/crs.rs
@@ -221,7 +221,7 @@ impl AuthorityCode {
/// Example: "EPSG:4269"
pub fn crs(auth_code: &str) -> Crs {
let ac = if Self::validate_epsg_code(auth_code) {
- format!("EPSG:{}", auth_code)
+ format!("EPSG:{auth_code}")
} else {
auth_code.to_string()
};
diff --git a/rust/sedona-spatial-join/Cargo.toml
b/rust/sedona-spatial-join/Cargo.toml
index 178b3178..b4c8f630 100644
--- a/rust/sedona-spatial-join/Cargo.toml
+++ b/rust/sedona-spatial-join/Cargo.toml
@@ -68,8 +68,9 @@ fastrand = { workspace = true }
[dev-dependencies]
criterion = { workspace = true }
+datafusion = { workspace = true, features = ["sql"] }
rstest = { workspace = true }
-sedona-testing = { workspace = true }
+sedona-testing = { workspace = true}
wkt = { workspace = true }
tokio = { workspace = true, features = ["macros"] }
rand = { workspace = true }
diff --git a/rust/sedona-spatial-join/bench/partitioning/common.rs
b/rust/sedona-spatial-join/bench/partitioning/common.rs
index 1b0e1c71..dd77f973 100644
--- a/rust/sedona-spatial-join/bench/partitioning/common.rs
+++ b/rust/sedona-spatial-join/bench/partitioning/common.rs
@@ -70,11 +70,11 @@ fn random_bbox(extent: &BoundingBox, rng: &mut StdRng,
max_span: f64) -> Boundin
let (min_x, max_x) = (extent.x().lo(), extent.x().hi());
let (min_y, max_y) = (extent.y().lo(), extent.y().hi());
- let span_x = rng.gen_range(0.01..max_span).min(max_x - min_x);
- let span_y = rng.gen_range(0.01..max_span).min(max_y - min_y);
+ let span_x = rng.random_range(0.01..max_span).min(max_x - min_x);
+ let span_y = rng.random_range(0.01..max_span).min(max_y - min_y);
- let start_x = rng.gen_range(min_x..=max_x - span_x);
- let start_y = rng.gen_range(min_y..=max_y - span_y);
+ let start_x = rng.random_range(min_x..=max_x - span_x);
+ let start_y = rng.random_range(min_y..=max_y - span_y);
BoundingBox::xy((start_x, start_x + span_x), (start_y, start_y + span_y))
}
diff --git a/rust/sedona-spatial-join/bench/partitioning/kdb.rs
b/rust/sedona-spatial-join/bench/partitioning/kdb.rs
index f38e8300..1edc4bb6 100644
--- a/rust/sedona-spatial-join/bench/partitioning/kdb.rs
+++ b/rust/sedona-spatial-join/bench/partitioning/kdb.rs
@@ -106,11 +106,11 @@ fn random_bbox(extent: &BoundingBox, rng: &mut StdRng,
max_span: f64) -> Boundin
let (min_x, max_x) = (extent.x().lo(), extent.x().hi());
let (min_y, max_y) = (extent.y().lo(), extent.y().hi());
- let span_x = rng.gen_range(0.01..max_span).min(max_x - min_x);
- let span_y = rng.gen_range(0.01..max_span).min(max_y - min_y);
+ let span_x = rng.random_range(0.01..max_span).min(max_x - min_x);
+ let span_y = rng.random_range(0.01..max_span).min(max_y - min_y);
- let start_x = rng.gen_range(min_x..=max_x - span_x);
- let start_y = rng.gen_range(min_y..=max_y - span_y);
+ let start_x = rng.random_range(min_x..=max_x - span_x);
+ let start_y = rng.random_range(min_y..=max_y - span_y);
BoundingBox::xy((start_x, start_x + span_x), (start_y, start_y + span_y))
}
diff --git a/rust/sedona-spatial-join/src/exec.rs
b/rust/sedona-spatial-join/src/exec.rs
index 1644f069..804a5d88 100644
--- a/rust/sedona-spatial-join/src/exec.rs
+++ b/rust/sedona-spatial-join/src/exec.rs
@@ -652,7 +652,7 @@ mod tests {
let bounds = Rect::new(Coord { x: 0.0, y: 0.0 }, Coord { x: 100.0, y:
100.0 });
let left_data = RandomPartitionedDataBuilder::new()
- .seed(1)
+ .seed(11584)
.num_partitions(2)
.batches_per_partition(2)
.rows_per_batch(30)
@@ -664,7 +664,7 @@ mod tests {
.build()?;
let right_data = RandomPartitionedDataBuilder::new()
- .seed(2)
+ .seed(54843)
.num_partitions(4)
.batches_per_partition(4)
.rows_per_batch(30)
diff --git a/rust/sedona-spatial-join/src/utils/bbox_sampler.rs
b/rust/sedona-spatial-join/src/utils/bbox_sampler.rs
index d823b506..498f3863 100644
--- a/rust/sedona-spatial-join/src/utils/bbox_sampler.rs
+++ b/rust/sedona-spatial-join/src/utils/bbox_sampler.rs
@@ -303,8 +303,8 @@ mod tests {
// Stage 1
for _ in 0..100 {
let bbox = BoundingBox::xy(
- (rng.gen::<f64>(), rng.gen::<f64>()),
- (rng.gen::<f64>(), rng.gen::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
);
sampler.add_bbox(&bbox);
count += 1;
@@ -314,8 +314,8 @@ mod tests {
// Stage 2
for _ in 0..9900 {
let bbox = BoundingBox::xy(
- (rng.gen::<f64>(), rng.gen::<f64>()),
- (rng.gen::<f64>(), rng.gen::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
);
sampler.add_bbox(&bbox);
count += 1;
@@ -325,8 +325,8 @@ mod tests {
// Stage 3
for _ in 0..5000 {
let bbox = BoundingBox::xy(
- (rng.gen::<f64>(), rng.gen::<f64>()),
- (rng.gen::<f64>(), rng.gen::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
);
sampler.add_bbox(&bbox);
count += 1;
@@ -336,8 +336,8 @@ mod tests {
for _ in 0..5000 {
let bbox = BoundingBox::xy(
- (rng.gen::<f64>(), rng.gen::<f64>()),
- (rng.gen::<f64>(), rng.gen::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
);
sampler.add_bbox(&bbox);
count += 1;
@@ -347,8 +347,8 @@ mod tests {
// Stage 4
for _ in 0..20000 {
let bbox = BoundingBox::xy(
- (-rng.gen::<f64>(), -rng.gen::<f64>()),
- (rng.gen::<f64>(), rng.gen::<f64>()),
+ (-rng.random::<f64>(), -rng.random::<f64>()),
+ (rng.random::<f64>(), rng.random::<f64>()),
);
sampler.add_bbox(&bbox);
count += 1;
@@ -374,8 +374,8 @@ mod tests {
// Add 10000 points uniformly distributed
for _ in 0..10000 {
- let x = rng.gen::<f64>() * 100.0;
- let y = rng.gen::<f64>() * 100.0;
+ let x = rng.random::<f64>() * 100.0;
+ let y = rng.random::<f64>() * 100.0;
let bbox = BoundingBox::xy((x, x), (y, y));
sampler.add_bbox(&bbox);
}
@@ -444,8 +444,8 @@ mod tests {
for _ in 0..50000 {
let bbox = BoundingBox::xy(
- (rng.gen::<f64>() * 100.0, rng.gen::<f64>() * 100.0),
- (rng.gen::<f64>() * 100.0, rng.gen::<f64>() * 100.0),
+ (rng.random::<f64>() * 100.0, rng.random::<f64>() * 100.0),
+ (rng.random::<f64>() * 100.0, rng.random::<f64>() * 100.0),
);
sampler.add_bbox(&bbox);
}
diff --git a/rust/sedona-testing/src/benchmark_util.rs
b/rust/sedona-testing/src/benchmark_util.rs
index ba86db9c..78504b83 100644
--- a/rust/sedona-testing/src/benchmark_util.rs
+++ b/rust/sedona-testing/src/benchmark_util.rs
@@ -19,10 +19,10 @@ use std::{fmt::Debug, sync::Arc, vec};
use arrow_array::{ArrayRef, Float64Array, Int64Array};
use arrow_schema::DataType;
-use datafusion_common::{Result, ScalarValue};
+use datafusion_common::{DataFusionError, Result, ScalarValue};
use datafusion_expr::{AggregateUDF, ScalarUDF};
use geo_types::Rect;
-use rand::{distributions::Uniform, rngs::StdRng, Rng, SeedableRng};
+use rand::{distr::Uniform, rngs::StdRng, Rng, SeedableRng};
use sedona_common::sedona_internal_err;
use sedona_geometry::types::GeometryTypeId;
@@ -402,7 +402,8 @@ impl BenchmarkArgSpec {
),
BenchmarkArgSpec::Int64(lo, hi) => {
let mut rng = self.rng(i);
- let dist = Uniform::new(lo, hi);
+ let dist =
+ Uniform::new(lo, hi).map_err(|e|
DataFusionError::External(Box::new(e)))?;
(0..num_batches)
.map(|_| -> Result<ArrayRef> {
let int64_array: Int64Array =
@@ -413,7 +414,8 @@ impl BenchmarkArgSpec {
}
BenchmarkArgSpec::Float64(lo, hi) => {
let mut rng = self.rng(i);
- let dist = Uniform::new(lo, hi);
+ let dist =
+ Uniform::new(lo, hi).map_err(|e|
DataFusionError::External(Box::new(e)))?;
(0..num_batches)
.map(|_| -> Result<ArrayRef> {
let float64_array: Float64Array =
@@ -424,7 +426,8 @@ impl BenchmarkArgSpec {
}
BenchmarkArgSpec::Int32(lo, hi) => {
let mut rng = self.rng(i);
- let dist = Uniform::new(lo, hi);
+ let dist =
+ Uniform::new(lo, hi).map_err(|e|
DataFusionError::External(Box::new(e)))?;
(0..num_batches)
.map(|_| -> Result<ArrayRef> {
let int32_array: arrow_array::Int32Array =
diff --git a/rust/sedona-testing/src/datagen.rs
b/rust/sedona-testing/src/datagen.rs
index 44818603..088bf220 100644
--- a/rust/sedona-testing/src/datagen.rs
+++ b/rust/sedona-testing/src/datagen.rs
@@ -27,14 +27,12 @@ use arrow_array::{ArrayRef, RecordBatch, RecordBatchReader};
use arrow_array::{BinaryArray, BinaryViewArray};
use arrow_array::{Float64Array, Int32Array};
use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
-use datafusion_common::Result;
+use datafusion_common::{DataFusionError, Result};
use geo_types::{
Coord, Geometry, GeometryCollection, LineString, MultiLineString,
MultiPoint, MultiPolygon,
Point, Polygon, Rect,
};
-use rand::distributions::Uniform;
-use rand::rngs::StdRng;
-use rand::{Rng, SeedableRng};
+use rand::{distr::Uniform, rngs::StdRng, Rng, SeedableRng};
use sedona_common::sedona_internal_err;
use sedona_geometry::types::GeometryTypeId;
use sedona_schema::datatypes::{SedonaType, WKB_GEOMETRY};
@@ -389,21 +387,23 @@ impl RandomPartitionedDataBuilder {
.collect();
// Generate random distances between 0.0 and 100.0
- let distance_dist = Uniform::new(0.0, 100.0);
+ let distance_dist = Uniform::new(0.0, 100.0).expect("valid input to
Uniform::new()");
let distances: Vec<f64> = (0..self.rows_per_batch)
.map(|_| rng.sample(distance_dist))
.collect();
// Generate random geometries based on the geometry type
- let wkb_geometries: Vec<Option<Vec<u8>>> = (0..self.rows_per_batch)
- .map(|_| {
- if rng.sample(Uniform::new(0.0, 1.0)) < self.null_rate {
- None
+ let wkb_geometries = (0..self.rows_per_batch)
+ .map(|_| -> Result<Option<Vec<u8>>> {
+ if rng.sample(Uniform::new(0.0, 1.0).expect("valid input to
Uniform::new()"))
+ < self.null_rate
+ {
+ Ok(None)
} else {
- Some(generate_random_wkb(rng, &self.options))
+ Ok(Some(generate_random_wkb(rng, &self.options)?))
}
})
- .collect();
+ .collect::<Result<Vec<Option<Vec<u8>>>>>()?;
// Create Arrow arrays
let id_array = Arc::new(Int32Array::from(ids));
@@ -499,8 +499,11 @@ impl Default for RandomGeometryOptions {
}
/// Generate random geometry WKB bytes based on the geometry type
-fn generate_random_wkb<R: rand::Rng>(rng: &mut R, options:
&RandomGeometryOptions) -> Vec<u8> {
- let geometry = generate_random_geometry(rng, options);
+fn generate_random_wkb<R: rand::Rng>(
+ rng: &mut R,
+ options: &RandomGeometryOptions,
+) -> Result<Vec<u8>> {
+ let geometry = generate_random_geometry(rng, options)?;
// Convert geometry to WKB
let mut out: Vec<u8> = vec![];
@@ -511,167 +514,178 @@ fn generate_random_wkb<R: rand::Rng>(rng: &mut R,
options: &RandomGeometryOption
endianness: Endianness::LittleEndian,
},
)
- .unwrap();
- out
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
+ Ok(out)
}
fn generate_random_geometry<R: rand::Rng>(
rng: &mut R,
options: &RandomGeometryOptions,
-) -> Geometry {
- match options.geom_type {
- GeometryTypeId::Point => Geometry::Point(generate_random_point(rng,
options)),
+) -> Result<Geometry> {
+ Ok(match options.geom_type {
+ GeometryTypeId::Point => Geometry::Point(generate_random_point(rng,
options)?),
GeometryTypeId::LineString => {
- Geometry::LineString(generate_random_linestring(rng, options))
+ Geometry::LineString(generate_random_linestring(rng, options)?)
}
- GeometryTypeId::Polygon =>
Geometry::Polygon(generate_random_polygon(rng, options)),
+ GeometryTypeId::Polygon =>
Geometry::Polygon(generate_random_polygon(rng, options)?),
GeometryTypeId::MultiPoint => {
- Geometry::MultiPoint(generate_random_multipoint(rng, options))
+ Geometry::MultiPoint(generate_random_multipoint(rng, options)?)
}
GeometryTypeId::MultiLineString => {
- Geometry::MultiLineString(generate_random_multilinestring(rng,
options))
+ Geometry::MultiLineString(generate_random_multilinestring(rng,
options)?)
}
GeometryTypeId::MultiPolygon => {
- Geometry::MultiPolygon(generate_random_multipolygon(rng, options))
+ Geometry::MultiPolygon(generate_random_multipolygon(rng, options)?)
}
GeometryTypeId::GeometryCollection => {
-
Geometry::GeometryCollection(generate_random_geometrycollection(rng, options))
+
Geometry::GeometryCollection(generate_random_geometrycollection(rng, options)?)
}
GeometryTypeId::Geometry => {
let mut copy_options = options.clone();
copy_options.geom_type = pick_random_geometry_type(rng);
- generate_random_geometry(rng, ©_options)
+ generate_random_geometry(rng, ©_options)?
}
- }
+ })
}
-fn generate_random_point<R: rand::Rng>(rng: &mut R, options:
&RandomGeometryOptions) -> Point {
- if rng.gen_bool(options.empty_rate) {
+fn generate_random_point<R: rand::Rng>(
+ rng: &mut R,
+ options: &RandomGeometryOptions,
+) -> Result<Point> {
+ if rng.random_bool(options.empty_rate) {
// This is a bit of a hack because geo-types doesn't support empty
point; however,
// this does work with respect to sending this directly to the WKB
reader and getting
// the WKB result we want
- Point::new(f64::NAN, f64::NAN)
+ Ok(Point::new(f64::NAN, f64::NAN))
} else {
// Generate random points within the specified bounds
- let x_dist = Uniform::new(options.bounds.min().x,
options.bounds.max().x);
- let y_dist = Uniform::new(options.bounds.min().y,
options.bounds.max().y);
+ let x_dist = Uniform::new(options.bounds.min().x,
options.bounds.max().x)
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
+ let y_dist = Uniform::new(options.bounds.min().y,
options.bounds.max().y)
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
let x = rng.sample(x_dist);
let y = rng.sample(y_dist);
- Point::new(x, y)
+ Ok(Point::new(x, y))
}
}
fn generate_random_linestring<R: rand::Rng>(
rng: &mut R,
options: &RandomGeometryOptions,
-) -> LineString {
- if rng.gen_bool(options.empty_rate) {
- LineString::new(vec![])
+) -> Result<LineString> {
+ if rng.random_bool(options.empty_rate) {
+ Ok(LineString::new(vec![]))
} else {
- let (center_x, center_y, half_size) = generate_random_circle(rng,
options);
+ let (center_x, center_y, half_size) = generate_random_circle(rng,
options)?;
let vertices_dist = Uniform::new_inclusive(
options.vertices_per_linestring_range.0,
options.vertices_per_linestring_range.1,
- );
+ )
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
// Always sample in such a way that we end up with a valid linestring
let num_vertices = rng.sample(vertices_dist).max(2);
let coords =
- generate_circular_vertices(rng, center_x, center_y, half_size,
num_vertices, false);
- LineString::from(coords)
+ generate_circular_vertices(rng, center_x, center_y, half_size,
num_vertices, false)?;
+ Ok(LineString::from(coords))
}
}
-fn generate_random_polygon<R: rand::Rng>(rng: &mut R, options:
&RandomGeometryOptions) -> Polygon {
- if rng.gen_bool(options.empty_rate) {
- Polygon::new(LineString::new(vec![]), vec![])
+fn generate_random_polygon<R: rand::Rng>(
+ rng: &mut R,
+ options: &RandomGeometryOptions,
+) -> Result<Polygon> {
+ if rng.random_bool(options.empty_rate) {
+ Ok(Polygon::new(LineString::new(vec![]), vec![]))
} else {
- let (center_x, center_y, half_size) = generate_random_circle(rng,
options);
+ let (center_x, center_y, half_size) = generate_random_circle(rng,
options)?;
let vertices_dist = Uniform::new_inclusive(
options.vertices_per_linestring_range.0,
options.vertices_per_linestring_range.1,
- );
+ )
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
// Always sample in such a way that we end up with a valid Polygon
let num_vertices = rng.sample(vertices_dist).max(3);
let coords =
- generate_circular_vertices(rng, center_x, center_y, half_size,
num_vertices, true);
+ generate_circular_vertices(rng, center_x, center_y, half_size,
num_vertices, true)?;
let shell = LineString::from(coords);
let mut holes = Vec::new();
// Potentially add a hole based on probability
- let add_hole = rng.gen_bool(options.polygon_hole_rate);
- let hole_scale_factor_dist = Uniform::new(0.1, 0.5);
+ let add_hole = rng.random_bool(options.polygon_hole_rate);
+ let hole_scale_factor_dist = Uniform::new(0.1, 0.5).expect("Valid
input range");
let hole_scale_factor = rng.sample(hole_scale_factor_dist);
if add_hole {
let new_size = half_size * hole_scale_factor;
let mut coords =
- generate_circular_vertices(rng, center_x, center_y, new_size,
num_vertices, true);
+ generate_circular_vertices(rng, center_x, center_y, new_size,
num_vertices, true)?;
coords.reverse();
holes.push(LineString::from(coords));
}
- Polygon::new(shell, holes)
+ Ok(Polygon::new(shell, holes))
}
}
fn generate_random_multipoint<R: rand::Rng>(
rng: &mut R,
options: &RandomGeometryOptions,
-) -> MultiPoint {
- if rng.gen_bool(options.empty_rate) {
- MultiPoint::new(vec![])
+) -> Result<MultiPoint> {
+ if rng.random_bool(options.empty_rate) {
+ Ok(MultiPoint::new(vec![]))
} else {
- let children = generate_random_children(rng, options,
generate_random_point);
- MultiPoint::new(children)
+ let children = generate_random_children(rng, options,
generate_random_point)?;
+ Ok(MultiPoint::new(children))
}
}
fn generate_random_multilinestring<R: rand::Rng>(
rng: &mut R,
options: &RandomGeometryOptions,
-) -> MultiLineString {
- if rng.gen_bool(options.empty_rate) {
- MultiLineString::new(vec![])
+) -> Result<MultiLineString> {
+ if rng.random_bool(options.empty_rate) {
+ Ok(MultiLineString::new(vec![]))
} else {
- let children = generate_random_children(rng, options,
generate_random_linestring);
- MultiLineString::new(children)
+ let children = generate_random_children(rng, options,
generate_random_linestring)?;
+ Ok(MultiLineString::new(children))
}
}
fn generate_random_multipolygon<R: rand::Rng>(
rng: &mut R,
options: &RandomGeometryOptions,
-) -> MultiPolygon {
- if rng.gen_bool(options.empty_rate) {
- MultiPolygon::new(vec![])
+) -> Result<MultiPolygon> {
+ if rng.random_bool(options.empty_rate) {
+ Ok(MultiPolygon::new(vec![]))
} else {
- let children = generate_random_children(rng, options,
generate_random_polygon);
- MultiPolygon::new(children)
+ let children = generate_random_children(rng, options,
generate_random_polygon)?;
+ Ok(MultiPolygon::new(children))
}
}
fn generate_random_geometrycollection<R: rand::Rng>(
rng: &mut R,
options: &RandomGeometryOptions,
-) -> GeometryCollection {
- if rng.gen_bool(options.empty_rate) {
- GeometryCollection::new_from(vec![])
+) -> Result<GeometryCollection> {
+ if rng.random_bool(options.empty_rate) {
+ Ok(GeometryCollection::new_from(vec![]))
} else {
- let children = generate_random_children(rng, options,
generate_random_geometry);
- GeometryCollection::new_from(children)
+ let children = generate_random_children(rng, options,
generate_random_geometry)?;
+ Ok(GeometryCollection::new_from(children))
}
}
-fn generate_random_children<R: Rng, T, F: Fn(&mut R, &RandomGeometryOptions)
-> T>(
+fn generate_random_children<R: Rng, T, F: Fn(&mut R, &RandomGeometryOptions)
-> Result<T>>(
rng: &mut R,
options: &RandomGeometryOptions,
func: F,
-) -> Vec<T> {
+) -> Result<Vec<T>> {
let num_parts_dist =
- Uniform::new_inclusive(options.num_parts_range.0,
options.num_parts_range.1);
+ Uniform::new_inclusive(options.num_parts_range.0,
options.num_parts_range.1)
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
let num_parts = rng.sample(num_parts_dist);
// Constrain this feature to the size range indicated in the option
- let (center_x, center_y, half_width) = generate_random_circle(rng,
options);
+ let (center_x, center_y, half_width) = generate_random_circle(rng,
options)?;
let feature_bounds = Rect::new(
Coord {
x: center_x - half_width,
@@ -698,10 +712,10 @@ fn generate_random_children<R: Rng, T, F: Fn(&mut R,
&RandomGeometryOptions) ->
if options.geom_type == GeometryTypeId::GeometryCollection {
child_options.geom_type = pick_random_geometry_type(rng);
}
- children.push(func(rng, &child_options));
+ children.push(func(rng, &child_options)?);
}
- children
+ Ok(children)
}
fn pick_random_geometry_type<R: Rng>(rng: &mut R) -> GeometryTypeId {
@@ -712,30 +726,33 @@ fn pick_random_geometry_type<R: Rng>(rng: &mut R) ->
GeometryTypeId {
GeometryTypeId::MultiPoint,
GeometryTypeId::MultiLineString,
GeometryTypeId::MultiPolygon,
- ][rng.gen_range(0..6)]
+ ][rng.random_range(0..6)]
}
fn generate_random_circle<R: rand::Rng>(
rng: &mut R,
options: &RandomGeometryOptions,
-) -> (f64, f64, f64) {
+) -> Result<(f64, f64, f64)> {
// Generate random diamond polygons (rotated squares)
- let size_dist = Uniform::new(options.size_range.0, options.size_range.1);
+ let size_dist = Uniform::new(options.size_range.0, options.size_range.1)
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
let half_size = rng.sample(size_dist) / 2.0;
// Ensure diamond fits within bounds by constraining center position
let center_x_dist = Uniform::new(
options.bounds.min().x + half_size,
options.bounds.max().x - half_size,
- );
+ )
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
let center_y_dist = Uniform::new(
options.bounds.min().y + half_size,
options.bounds.max().y - half_size,
- );
+ )
+ .map_err(|e| DataFusionError::External(Box::new(e)))?;
let center_x = rng.sample(center_x_dist);
let center_y = rng.sample(center_y_dist);
- (center_x, center_y, half_size)
+ Ok((center_x, center_y, half_size))
}
fn generate_non_overlapping_sub_rectangles(num_parts: usize, bounds: &Rect) ->
Vec<Rect> {
@@ -774,11 +791,12 @@ fn generate_circular_vertices<R: rand::Rng>(
radius: f64,
num_vertices: usize,
closed: bool,
-) -> Vec<Coord> {
+) -> Result<Vec<Coord>> {
let mut out = Vec::new();
// Randomize starting angle (0 to 2 * PI)
- let start_angle_dist = Uniform::new(0.0, 2.0 * PI);
+ let start_angle_dist =
+ Uniform::new(0.0, 2.0 * PI).map_err(|e|
DataFusionError::External(Box::new(e)))?;
let mut angle: f64 = rng.sample(start_angle_dist);
let dangle = 2.0 * PI / (num_vertices as f64).max(3.0);
@@ -794,7 +812,7 @@ fn generate_circular_vertices<R: rand::Rng>(
out.push(out[0]);
}
- out
+ Ok(out)
}
#[cfg(test)]
@@ -831,7 +849,7 @@ mod tests {
};
for _ in 0..iterations {
- let wkb_bytes = generate_random_wkb(&mut rng, &options);
+ let wkb_bytes = generate_random_wkb(&mut rng,
&options).unwrap();
// Verify WKB is not empty and has reasonable size
assert!(!wkb_bytes.is_empty());
@@ -874,8 +892,8 @@ mod tests {
size_range,
..Default::default()
};
- let wkb1 = generate_random_wkb(&mut rng1, &options);
- let wkb2 = generate_random_wkb(&mut rng2, &options);
+ let wkb1 = generate_random_wkb(&mut rng1, &options).unwrap();
+ let wkb2 = generate_random_wkb(&mut rng2, &options).unwrap();
// Should generate identical results
assert_eq!(wkb1, wkb2);
@@ -1085,13 +1103,13 @@ mod tests {
let mut options = RandomGeometryOptions::new();
options.vertices_per_linestring_range = (3, 3);
for _ in 0..100 {
- let geom = generate_random_linestring(&mut rng, &options);
+ let geom = generate_random_linestring(&mut rng, &options).unwrap();
assert_eq!(geom.coords().count(), 3);
}
options.vertices_per_linestring_range = (50, 50);
for _ in 0..100 {
- let geom = generate_random_linestring(&mut rng, &options);
+ let geom = generate_random_linestring(&mut rng, &options).unwrap();
assert_eq!(geom.coords().count(), 50);
}
}
@@ -1103,13 +1121,13 @@ mod tests {
options.polygon_hole_rate = 0.0;
for _ in 0..100 {
- let geom = generate_random_polygon(&mut rng, &options);
+ let geom = generate_random_polygon(&mut rng, &options).unwrap();
assert_eq!(geom.interiors().len(), 0);
}
options.polygon_hole_rate = 1.0;
for _ in 0..100 {
- let geom = generate_random_polygon(&mut rng, &options);
+ let geom = generate_random_polygon(&mut rng, &options).unwrap();
assert!(!geom.interiors().is_empty());
}
}
@@ -1121,13 +1139,13 @@ mod tests {
options.num_parts_range = (3, 3);
for _ in 0..100 {
- let geom = generate_random_multipoint(&mut rng, &options);
+ let geom = generate_random_multipoint(&mut rng, &options).unwrap();
assert_eq!(geom.len(), 3);
}
options.num_parts_range = (10, 10);
for _ in 0..100 {
- let geom = generate_random_multipoint(&mut rng, &options);
+ let geom = generate_random_multipoint(&mut rng, &options).unwrap();
assert_eq!(geom.len(), 10);
}
}
@@ -1139,13 +1157,13 @@ mod tests {
options.num_parts_range = (3, 3);
for _ in 0..100 {
- let geom = generate_random_multilinestring(&mut rng, &options);
+ let geom = generate_random_multilinestring(&mut rng,
&options).unwrap();
assert_eq!(geom.num_line_strings(), 3);
}
options.num_parts_range = (10, 10);
for _ in 0..100 {
- let geom = generate_random_multilinestring(&mut rng, &options);
+ let geom = generate_random_multilinestring(&mut rng,
&options).unwrap();
assert_eq!(geom.num_line_strings(), 10);
}
}
@@ -1157,13 +1175,13 @@ mod tests {
options.num_parts_range = (3, 3);
for _ in 0..100 {
- let geom = generate_random_multipolygon(&mut rng, &options);
+ let geom = generate_random_multipolygon(&mut rng,
&options).unwrap();
assert_eq!(geom.num_polygons(), 3);
}
options.num_parts_range = (10, 10);
for _ in 0..100 {
- let geom = generate_random_multipolygon(&mut rng, &options);
+ let geom = generate_random_multipolygon(&mut rng,
&options).unwrap();
assert_eq!(geom.num_polygons(), 10);
}
}
@@ -1175,13 +1193,13 @@ mod tests {
options.num_parts_range = (3, 3);
for _ in 0..100 {
- let geom = generate_random_geometrycollection(&mut rng, &options);
+ let geom = generate_random_geometrycollection(&mut rng,
&options).unwrap();
assert_eq!(geom.len(), 3);
}
options.num_parts_range = (10, 10);
for _ in 0..100 {
- let geom = generate_random_geometrycollection(&mut rng, &options);
+ let geom = generate_random_geometrycollection(&mut rng,
&options).unwrap();
assert_eq!(geom.len(), 10);
}
}
@@ -1205,7 +1223,7 @@ mod tests {
options.empty_rate = 0.0;
for _ in 0..100 {
- let geom = generate_random_wkb(&mut rng, &options);
+ let geom = generate_random_wkb(&mut rng, &options).unwrap();
let wkb = wkb::reader::read_wkb(&geom).unwrap();
let analysis = analyze_geometry(&wkb).unwrap();
assert_eq!(analysis.geometry_type.geometry_type(), geom_type);
@@ -1231,7 +1249,7 @@ mod tests {
options.empty_rate = 0.0;
for _ in 0..100 {
- let geom = generate_random_wkb(&mut rng, &options);
+ let geom = generate_random_wkb(&mut rng, &options).unwrap();
let bounds = wkb_bounds_xy(&geom).unwrap();
assert!(!bounds.x().is_empty());
assert!(!bounds.y().is_empty());
@@ -1246,7 +1264,7 @@ mod tests {
options.empty_rate = 1.0;
for _ in 0..100 {
- let geom = generate_random_wkb(&mut rng, &options);
+ let geom = generate_random_wkb(&mut rng, &options).unwrap();
let bounds = wkb_bounds_xy(&geom).unwrap();
assert!(bounds.x().is_empty());
assert!(bounds.y().is_empty());
diff --git a/rust/sedona-testing/src/testers.rs
b/rust/sedona-testing/src/testers.rs
index 89cdd27f..cb56bbb3 100644
--- a/rust/sedona-testing/src/testers.rs
+++ b/rust/sedona-testing/src/testers.rs
@@ -119,6 +119,7 @@ impl AggregateUdfTester {
name: "",
is_distinct: false,
exprs: &exprs,
+ expr_fields: &[],
};
self.udf.accumulator(accumulator_args)
diff --git a/rust/sedona/Cargo.toml b/rust/sedona/Cargo.toml
index 9e031a93..aab77f2e 100644
--- a/rust/sedona/Cargo.toml
+++ b/rust/sedona/Cargo.toml
@@ -48,14 +48,13 @@ tokio = { workspace = true, features = ["rt-multi-thread"] }
rstest = { workspace = true }
[dependencies]
-abi_stable = { workspace = true }
arrow-schema = { workspace = true }
arrow-array = { workspace = true }
async-trait = { workspace = true }
aws-config = { version = "1.5.17", optional = true }
aws-credential-types = { version = "1.2.0", optional = true }
comfy-table = { workspace = true }
-datafusion = { workspace = true, default_features = false, features =
["parquet"] }
+datafusion = { workspace = true, default_features = false, features = ["sql",
"parquet"] }
datafusion-common = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-ffi = { workspace = true }
diff --git a/rust/sedona/src/context.rs b/rust/sedona/src/context.rs
index 82947460..998881df 100644
--- a/rust/sedona/src/context.rs
+++ b/rust/sedona/src/context.rs
@@ -31,7 +31,7 @@ use async_trait::async_trait;
use datafusion::dataframe::DataFrameWriteOptions;
use datafusion::datasource::file_format::format_as_file_type;
use datafusion::{
- common::{plan_datafusion_err, plan_err},
+ common::plan_err,
error::{DataFusionError, Result},
execution::{context::DataFilePaths, runtime_env::RuntimeEnvBuilder,
SessionStateBuilder},
prelude::{DataFrame, SessionConfig, SessionContext},
@@ -222,8 +222,8 @@ impl SedonaContext {
/// statements
pub async fn multi_sql(&self, sql: &str) -> Result<Vec<DataFrame>> {
let task_ctx = self.ctx.task_ctx();
- let dialect_str =
&task_ctx.session_config().options().sql_parser.dialect;
- let dialect = ThreadSafeDialect::try_new(dialect_str)?;
+ let dialect = &task_ctx.session_config().options().sql_parser.dialect;
+ let dialect = ThreadSafeDialect::try_new(dialect)?;
let statements = dialect.parse(sql)?;
let mut results = Vec::with_capacity(statements.len());
@@ -509,12 +509,14 @@ struct ThreadSafeDialect {
unsafe impl Send for ThreadSafeDialect {}
impl ThreadSafeDialect {
- pub fn try_new(dialect_str: &str) -> Result<Self> {
- let dialect = dialect_from_str(dialect_str)
- .ok_or_else(|| plan_datafusion_err!("Unsupported SQL dialect:
{dialect_str}"))?;
- Ok(Self {
- inner: dialect.into(),
- })
+ pub fn try_new(dialect: &datafusion::config::Dialect) -> Result<Self> {
+ if let Some(box_dialect) = dialect_from_str(dialect) {
+ Ok(Self {
+ inner: box_dialect.into(),
+ })
+ } else {
+ plan_err!("Unsupported SQL dialect: {dialect}")
+ }
}
pub fn parse(&self, sql: &str) -> Result<VecDeque<Statement>> {
diff --git a/rust/sedona/src/random_geometry_provider.rs
b/rust/sedona/src/random_geometry_provider.rs
index 00b67dc9..82d1edba 100644
--- a/rust/sedona/src/random_geometry_provider.rs
+++ b/rust/sedona/src/random_geometry_provider.rs
@@ -314,7 +314,7 @@ fn builder_with_partition_sizes(
target_rows: usize,
) -> RandomPartitionedDataBuilder {
let rows_for_one_batch_per_partition = batch_size * partitions;
- let batches_per_partition = if target_rows %
rows_for_one_batch_per_partition == 0 {
+ let batches_per_partition = if
target_rows.is_multiple_of(rows_for_one_batch_per_partition) {
target_rows / rows_for_one_batch_per_partition
} else {
target_rows / rows_for_one_batch_per_partition + 1
diff --git a/sedona-cli/src/exec.rs b/sedona-cli/src/exec.rs
index f280516e..246baf2f 100644
--- a/sedona-cli/src/exec.rs
+++ b/sedona-cli/src/exec.rs
@@ -125,7 +125,7 @@ pub async fn exec_from_repl(
) -> rustyline::Result<()> {
let mut rl = Editor::new()?;
rl.set_helper(Some(CliHelper::new(
- &ctx.ctx
+ ctx.ctx
.task_ctx()
.session_config()
.options()
diff --git a/sedona-cli/src/helper.rs b/sedona-cli/src/helper.rs
index 4c19f7d5..f3b69ded 100644
--- a/sedona-cli/src/helper.rs
+++ b/sedona-cli/src/helper.rs
@@ -22,6 +22,7 @@ use std::borrow::Cow;
use crate::highlighter::{NoSyntaxHighlighter, SyntaxHighlighter};
+use datafusion::config::Dialect;
use datafusion::sql::parser::{DFParser, Statement};
use datafusion::sql::sqlparser::dialect::dialect_from_str;
@@ -34,33 +35,33 @@ use rustyline::{Context, Helper, Result};
pub struct CliHelper {
completer: FilenameCompleter,
- dialect: String,
+ dialect: datafusion::config::Dialect,
highlighter: Box<dyn Highlighter>,
}
impl CliHelper {
- pub fn new(dialect: &str, color: bool) -> Self {
+ pub fn new(dialect: Dialect, color: bool) -> Self {
let highlighter: Box<dyn Highlighter> = if !color {
Box::new(NoSyntaxHighlighter {})
} else {
- Box::new(SyntaxHighlighter::new(dialect))
+ Box::new(SyntaxHighlighter::new(dialect.as_ref()))
};
Self {
completer: FilenameCompleter::new(),
- dialect: dialect.into(),
+ dialect,
highlighter,
}
}
- pub fn set_dialect(&mut self, dialect: &str) {
- if dialect != self.dialect {
- self.dialect = dialect.to_string();
+ pub fn set_dialect(&mut self, dialect: &datafusion::config::Dialect) {
+ if dialect != &self.dialect {
+ self.dialect = *dialect;
}
}
fn validate_input(&self, input: &str) -> Result<ValidationResult> {
if let Some(sql) = input.strip_suffix(';') {
- let dialect = match dialect_from_str(&self.dialect) {
+ let dialect = match dialect_from_str(self.dialect) {
Some(dialect) => dialect,
None => {
return Ok(ValidationResult::Invalid(Some(format!(
@@ -97,7 +98,7 @@ impl CliHelper {
impl Default for CliHelper {
fn default() -> Self {
- Self::new("generic", false)
+ Self::new(Dialect::Generic, false)
}
}
@@ -286,7 +287,7 @@ mod tests {
);
// valid in postgresql dialect
- validator.set_dialect("postgresql");
+ validator.set_dialect(&Dialect::PostgreSQL);
let result = readline_direct(Cursor::new(r"select 1 # 2;".as_bytes()),
&validator)?;
assert!(matches!(result, ValidationResult::Valid(None)));
