This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion-python.git
The following commit(s) were added to refs/heads/main by this push:
new f7e1c21 Use latest DataFusion rev in preparation for DF 18 release
(#174)
f7e1c21 is described below
commit f7e1c21a1be6f84c8873919be94a8240e6c00d15
Author: Andy Grove <[email protected]>
AuthorDate: Sat Feb 11 05:31:58 2023 -0700
Use latest DataFusion rev in preparation for DF 18 release (#174)
---
Cargo.lock | 100 ++++++++++++++++++++++++++++++++-----------------------
Cargo.toml | 10 +++---
src/substrait.rs | 6 ++--
3 files changed, 67 insertions(+), 49 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index bbd9f74..6ca4adb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -358,12 +358,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-[[package]]
-name = "base64"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
-
[[package]]
name = "base64"
version = "0.21.0"
@@ -502,9 +496,12 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
dependencies = [
"iana-time-zone",
+ "js-sys",
"num-integer",
"num-traits",
"serde",
+ "time",
+ "wasm-bindgen",
"winapi",
]
@@ -679,7 +676,7 @@ dependencies = [
[[package]]
name = "datafusion"
version = "17.0.0"
-source =
"git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d";
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20";
dependencies = [
"ahash",
"apache-avro",
@@ -727,7 +724,7 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "17.0.0"
-source =
"git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d";
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20";
dependencies = [
"apache-avro",
"arrow",
@@ -742,7 +739,7 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "17.0.0"
-source =
"git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d";
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20";
dependencies = [
"ahash",
"arrow",
@@ -754,7 +751,7 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "17.0.0"
-source =
"git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d";
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20";
dependencies = [
"arrow",
"async-trait",
@@ -770,7 +767,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "17.0.0"
-source =
"git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d";
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20";
dependencies = [
"ahash",
"arrow",
@@ -821,7 +818,7 @@ dependencies = [
[[package]]
name = "datafusion-row"
version = "17.0.0"
-source =
"git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d";
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20";
dependencies = [
"arrow",
"datafusion-common",
@@ -832,7 +829,7 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "17.0.0"
-source =
"git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d";
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20";
dependencies = [
"arrow-schema",
"datafusion-common",
@@ -844,11 +841,13 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
version = "17.0.0"
-source =
"git+https://github.com/apache/arrow-datafusion.git?rev=a265509b272084e758b9ec21f2133856f56e617d#a265509b272084e758b9ec21f2133856f56e617d";
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=8a262c3ec0a100607ae97641d7a374c847a71f20#8a262c3ec0a100607ae97641d7a374c847a71f20";
dependencies = [
"async-recursion",
+ "chrono",
"datafusion",
"itertools",
+ "object_store",
"prost 0.11.6",
"prost-build 0.9.0",
"prost-types 0.11.6",
@@ -1051,7 +1050,7 @@ checksum =
"c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if",
"libc",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
@@ -1545,7 +1544,7 @@ checksum =
"e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
dependencies = [
"libc",
"log",
- "wasi",
+ "wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.42.0",
]
@@ -1644,12 +1643,12 @@ dependencies = [
[[package]]
name = "object_store"
-version = "0.5.3"
+version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b4201837dc4c27a8670f0363b1255cd3845a4f0c521211cced1ed14c1d0cc6d2"
+checksum = "1f344e51ec9584d2f51199c0c29c6f73dddd04ade986497875bf8fa2f178caf0"
dependencies = [
"async-trait",
- "base64 0.20.0",
+ "base64",
"bytes",
"chrono",
"futures",
@@ -1722,7 +1721,7 @@ dependencies = [
"arrow-ipc",
"arrow-schema",
"arrow-select",
- "base64 0.21.0",
+ "base64",
"brotli",
"bytes",
"chrono",
@@ -1755,9 +1754,9 @@ checksum =
"478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]]
name = "pest"
-version = "2.5.4"
+version = "2.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4ab62d2fa33726dbe6321cc97ef96d8cde531e3eeaf858a058de53a8a6d40d8f"
+checksum = "028accff104c4e513bad663bbcd2ad7cfd5304144404c31ed0a77ac103d00660"
dependencies = [
"thiserror",
"ucd-trie",
@@ -1920,9 +1919,9 @@ dependencies = [
[[package]]
name = "pyo3"
-version = "0.18.0"
+version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ccd4149c8c3975099622b4e1962dac27565cf5663b76452c3e2b66e0b6824277"
+checksum = "06a3d8e8a46ab2738109347433cb7b96dffda2e4a218b03ef27090238886b147"
dependencies = [
"cfg-if",
"indoc",
@@ -1937,9 +1936,9 @@ dependencies = [
[[package]]
name = "pyo3-build-config"
-version = "0.18.0"
+version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9cd09fe469834db21ee60e0051030339e5d361293d8cb5ec02facf7fdcf52dbf"
+checksum = "75439f995d07ddfad42b192dfcf3bc66a7ecfd8b4a1f5f6f046aa5c2c5d7677d"
dependencies = [
"once_cell",
"target-lexicon",
@@ -1947,9 +1946,9 @@ dependencies = [
[[package]]
name = "pyo3-ffi"
-version = "0.18.0"
+version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0c427c9a96b9c5b12156dbc11f76b14f49e9aae8905ca783ea87c249044ef137"
+checksum = "839526a5c07a17ff44823679b68add4a58004de00512a95b6c1c98a6dcac0ee5"
dependencies = [
"libc",
"pyo3-build-config",
@@ -1957,9 +1956,9 @@ dependencies = [
[[package]]
name = "pyo3-macros"
-version = "0.18.0"
+version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "16b822bbba9d60630a44d2109bc410489bb2f439b33e3a14ddeb8a40b378a7c4"
+checksum = "bd44cf207476c6a9760c4653559be4f206efafb924d3e4cbf2721475fc0d6cc5"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
@@ -1969,9 +1968,9 @@ dependencies = [
[[package]]
name = "pyo3-macros-backend"
-version = "0.18.0"
+version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "84ae898104f7c99db06231160770f3e40dad6eb9021daddc0fedfa3e41dff10a"
+checksum = "dc1f43d8e30460f36350d18631ccf85ded64c059829208fe680904c65bcd0a4c"
dependencies = [
"proc-macro2",
"quote",
@@ -2089,7 +2088,7 @@ version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9"
dependencies = [
- "base64 0.21.0",
+ "base64",
"bytes",
"encoding_rs",
"futures-core",
@@ -2185,7 +2184,7 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"
dependencies = [
- "base64 0.21.0",
+ "base64",
]
[[package]]
@@ -2318,9 +2317,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.92"
+version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a"
+checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
dependencies = [
"itoa 1.0.5",
"ryu",
@@ -2574,6 +2573,17 @@ dependencies = [
"ordered-float",
]
+[[package]]
+name = "time"
+version = "0.1.45"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
+dependencies = [
+ "libc",
+ "wasi 0.10.0+wasi-snapshot-preview1",
+ "winapi",
+]
+
[[package]]
name = "tiny-keccak"
version = "2.0.2"
@@ -2652,9 +2662,9 @@ dependencies = [
[[package]]
name = "tokio-util"
-version = "0.7.4"
+version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"
+checksum = "bc6a3b08b64e6dfad376fa2432c7b1f01522e37a623c3050bc95db2d3ff21583"
dependencies = [
"bytes",
"futures-core",
@@ -2907,6 +2917,12 @@ dependencies = [
"try-lock",
]
+[[package]]
+name = "wasi"
+version = "0.10.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
+
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
@@ -3194,9 +3210,9 @@ dependencies = [
[[package]]
name = "zstd-safe"
-version = "6.0.3+zstd.1.5.2"
+version = "6.0.4+zstd.1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68e4a3f57d13d0ab7e478665c60f35e2a613dcd527851c2c7287ce5c787e134a"
+checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543"
dependencies = [
"libc",
"zstd-sys",
@@ -3204,9 +3220,9 @@ dependencies = [
[[package]]
name = "zstd-sys"
-version = "2.0.6+zstd.1.5.2"
+version = "2.0.7+zstd.1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68a3f9792c0c3dc6c165840a75f47ae1f4da402c2d006881129579f6597e801b"
+checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5"
dependencies = [
"cc",
"libc",
diff --git a/Cargo.toml b/Cargo.toml
index d1657f9..30e0454 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,11 +34,11 @@ default = ["mimalloc"]
tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread",
"sync"] }
rand = "0.8"
pyo3 = { version = "0.18.0", features = ["extension-module", "abi3",
"abi3-py37"] }
-datafusion = { git="https://github.com/apache/arrow-datafusion.git";,
rev="a265509b272084e758b9ec21f2133856f56e617d", features = ["pyarrow", "avro"] }
-datafusion-expr = { git="https://github.com/apache/arrow-datafusion.git";,
rev="a265509b272084e758b9ec21f2133856f56e617d" }
-datafusion-optimizer = { git="https://github.com/apache/arrow-datafusion.git";,
rev="a265509b272084e758b9ec21f2133856f56e617d" }
-datafusion-common = { git="https://github.com/apache/arrow-datafusion.git";,
rev="a265509b272084e758b9ec21f2133856f56e617d", features = ["pyarrow"] }
-datafusion-substrait = { git="https://github.com/apache/arrow-datafusion.git";,
rev="a265509b272084e758b9ec21f2133856f56e617d" }
+datafusion = { git="https://github.com/apache/arrow-datafusion.git";,
rev="8a262c3ec0a100607ae97641d7a374c847a71f20", features = ["pyarrow", "avro"] }
+datafusion-expr = { git="https://github.com/apache/arrow-datafusion.git";,
rev="8a262c3ec0a100607ae97641d7a374c847a71f20" }
+datafusion-optimizer = { git="https://github.com/apache/arrow-datafusion.git";,
rev="8a262c3ec0a100607ae97641d7a374c847a71f20" }
+datafusion-common = { git="https://github.com/apache/arrow-datafusion.git";,
rev="8a262c3ec0a100607ae97641d7a374c847a71f20", features = ["pyarrow"] }
+datafusion-substrait = { git="https://github.com/apache/arrow-datafusion.git";,
rev="8a262c3ec0a100607ae97641d7a374c847a71f20" }
uuid = { version = "1.2", features = ["v4"] }
mimalloc = { version = "*", optional = true, default-features = false }
async-trait = "0.1"
diff --git a/src/substrait.rs b/src/substrait.rs
index 0c42e21..b9e5c92 100644
--- a/src/substrait.rs
+++ b/src/substrait.rs
@@ -19,10 +19,12 @@ use pyo3::prelude::*;
use crate::context::PySessionContext;
use crate::errors::py_datafusion_err;
+use crate::errors::DataFusionError;
use crate::{logical::PyLogicalPlan, utils::wait_for_future};
-use crate::errors::DataFusionError;
-use datafusion_substrait::{consumer, producer, serializer,
substrait::proto::Plan};
+use datafusion_substrait::logical_plan::{consumer, producer};
+use datafusion_substrait::serializer;
+use datafusion_substrait::substrait::proto::Plan;
#[pyclass(name = "plan", module = "datafusion.substrait", subclass,
unsendable)]
#[derive(Debug, Clone)]
