This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion-python.git
The following commit(s) were added to refs/heads/main by this push:
new 9b2c3da Extension bindings (#266)
9b2c3da is described below
commit 9b2c3da9351529fe645ffe36b2d4633a082748b5
Author: Jeremy Dyer <[email protected]>
AuthorDate: Mon Mar 13 13:35:50 2023 -0400
Extension bindings (#266)
* Introduce to_variant trait function to LogicalNode and create Explain
LogicalNode bindings
* Cargo fmt
* bindings for Extension LogicalNode
* Add missing classes to list of exports so test_imports will pass
* Update to point to proper repo
* Update pytest to adhere to aggregate calls being wrapped in projections
* Address linter change which causes a pytest to fail
---
Cargo.lock | 268 +++++++++++++++++++++----------------
Cargo.toml | 12 +-
datafusion/__init__.py | 2 +
datafusion/tests/test_dataframe.py | 12 +-
datafusion/tests/test_imports.py | 4 +
src/context.rs | 8 +-
src/expr.rs | 2 +
src/expr/extension.rs | 52 +++++++
src/sql/logical.rs | 2 +
9 files changed, 228 insertions(+), 134 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 4d5941b..33156d6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -107,9 +107,9 @@ checksum =
"8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "arrow"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f3724c874f1517cf898cd1c3ad18ab5071edf893c48e73139ab1e16cf0f2affe"
+checksum = "f410d3907b6b3647b9e7bca4551274b2e3d716aa940afb67b7287257401da921"
dependencies = [
"ahash",
"arrow-arith",
@@ -131,9 +131,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e958823b8383ca14d0a2e973de478dd7674cd9f72837f8c41c132a0fda6a4e5e"
+checksum = "f87391cf46473c9bc53dab68cb8872c3a81d4dfd1703f1c8aa397dba9880a043"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -146,9 +146,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "db670eab50e76654065b5aed930f4367101fcddcb2223802007d1e0b4d5a2579"
+checksum = "d35d5475e65c57cffba06d0022e3006b677515f99b54af33a7cd54f6cdd4a5b5"
dependencies = [
"ahash",
"arrow-buffer",
@@ -162,9 +162,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9f0e01c931882448c0407bd32311a624b9f099739e94e786af68adc97016b5f2"
+checksum = "68b4ec72eda7c0207727df96cf200f539749d736b21f3e782ece113e18c1a0a7"
dependencies = [
"half",
"num",
@@ -172,9 +172,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4bf35d78836c93f80d9362f3ccb47ff5e2c5ecfc270ff42cdf1ef80334961d44"
+checksum = "0a7285272c9897321dfdba59de29f5b05aeafd3cdedf104a941256d155f6d304"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -188,9 +188,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0a6aa7c2531d89d01fed8c469a9b1bf97132a0bdf70b4724fe4bbb4537a50880"
+checksum = "981ee4e7f6a120da04e00d0b39182e1eeacccb59c8da74511de753c56b7fddf7"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -207,9 +207,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ea50db4d1e1e4c2da2bfdea7b6d2722eef64267d5ab680d815f7ae42428057f5"
+checksum = "27cc673ee6989ea6e4b4e8c7d461f7e06026a096c8f0b1a7288885ff71ae1e56"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -219,9 +219,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a4042fe6585155d1ec28a8e4937ec901a3ca7a19a22b9f6cd3f551b935cd84f5"
+checksum = "e37b8b69d9e59116b6b538e8514e0ec63a30f08b617ce800d31cb44e3ef64c1a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -233,9 +233,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7c907c4ab4f26970a3719dc06e78e8054a01d0c96da3664d23b941e201b33d2b"
+checksum = "80c3fa0bed7cfebf6d18e46b733f9cb8a1cb43ce8e6539055ca3e1e48a426266"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -252,9 +252,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e131b447242a32129efc7932f58ed8931b42f35d8701c1a08f9f524da13b1d3c"
+checksum = "d247dce7bed6a8d6a3c6debfa707a3a2f694383f0c692a39d736a593eae5ef94"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -266,9 +266,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b591ef70d76f4ac28dd7666093295fece0e5f9298f49af51ea49c001e1635bb6"
+checksum = "8d609c0181f963cea5c70fddf9a388595b5be441f3aa1d1cdbf728ca834bbd3a"
dependencies = [
"ahash",
"arrow-array",
@@ -281,18 +281,18 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "eb327717d87eb94be5eff3b0cb8987f54059d343ee5235abf7f143c85f54cfc8"
+checksum = "64951898473bfb8e22293e83a44f02874d2257514d49cd95f9aa4afcff183fbc"
dependencies = [
"bitflags",
]
[[package]]
name = "arrow-select"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "79d3c389d1cea86793934f31594f914c8547d82e91e3411d4833ad0aac3266a7"
+checksum = "2a513d89c2e1ac22b28380900036cf1f3992c6443efc5e079de631dcf83c6888"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -303,9 +303,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "30ee67790496dd310ddbf5096870324431e89aa76453e010020ac29b1184d356"
+checksum = "5288979b2705dae1114c864d73150629add9153b9b8f1d7ee3963db94c372ba5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -330,6 +330,8 @@ dependencies = [
"pin-project-lite",
"tokio",
"xz2",
+ "zstd 0.11.2+zstd.1.5.2",
+ "zstd-safe 5.0.2+zstd.1.5.2",
]
[[package]]
@@ -397,9 +399,9 @@ dependencies = [
[[package]]
name = "block-buffer"
-version = "0.10.3"
+version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
@@ -481,9 +483,9 @@ checksum =
"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
-version = "0.4.23"
+version = "0.4.24"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
+checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b"
dependencies = [
"iana-time-zone",
"js-sys",
@@ -540,9 +542,9 @@ dependencies = [
[[package]]
name = "constant_time_eq"
-version = "0.2.4"
+version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279"
+checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b"
[[package]]
name = "core-foundation-sys"
@@ -664,9 +666,8 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "19.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "12d462c103bd1cfd24f8e8a199986d89582af6280528e085c393c4be2ff25da7"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
dependencies = [
"ahash",
"apache-avro",
@@ -678,6 +679,7 @@ dependencies = [
"chrono",
"dashmap",
"datafusion-common",
+ "datafusion-execution",
"datafusion-expr",
"datafusion-optimizer",
"datafusion-physical-expr",
@@ -709,13 +711,13 @@ dependencies = [
"url",
"uuid",
"xz2",
+ "zstd 0.12.3+zstd.1.5.2",
]
[[package]]
name = "datafusion-common"
-version = "19.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b5babdbcf102862b1f1828c1ab41094e39ba881d5ece4cee2d481d528148f592"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
dependencies = [
"apache-avro",
"arrow",
@@ -727,11 +729,27 @@ dependencies = [
"sqlparser",
]
+[[package]]
+name = "datafusion-execution"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
+dependencies = [
+ "dashmap",
+ "datafusion-common",
+ "datafusion-expr",
+ "hashbrown 0.13.2",
+ "log",
+ "object_store",
+ "parking_lot",
+ "rand",
+ "tempfile",
+ "url",
+]
+
[[package]]
name = "datafusion-expr"
-version = "19.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "90f0c34e87fa541a59d378dc7ee7c9c3dd1fcfa793eab09561b8b4cb35e1827a"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
dependencies = [
"ahash",
"arrow",
@@ -742,9 +760,8 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
-version = "19.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7d0c6d912b7b7e4637d85947222455cd948ea193ca454ebf649e7265fd10b048"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
dependencies = [
"arrow",
"async-trait",
@@ -753,15 +770,15 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr",
"hashbrown 0.13.2",
+ "itertools",
"log",
"regex-syntax",
]
[[package]]
name = "datafusion-physical-expr"
-version = "19.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8000e8f8efafb810ff2943323bb48bd722ac5bb919fe302a66b832ed9c25245f"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
dependencies = [
"ahash",
"arrow",
@@ -781,6 +798,7 @@ dependencies = [
"md-5",
"num-traits",
"paste",
+ "petgraph",
"rand",
"regex",
"sha2",
@@ -812,9 +830,8 @@ dependencies = [
[[package]]
name = "datafusion-row"
-version = "19.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4e900f05d7e5666e8ab714a96a28cb6f143e62aa1d501ba1199024f8635c726c"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
dependencies = [
"arrow",
"datafusion-common",
@@ -824,9 +841,8 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "19.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "096f293799e8ae883e0f79f8ebaa51e4292e690ba45e0269b48ca9bd79f57094"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
dependencies = [
"arrow-schema",
"datafusion-common",
@@ -837,9 +853,8 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
-version = "19.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c5a1d24d6db620a9d18c69289bcf8ed413feb6e33b4f69902dbc415bce05d28e"
+version = "20.0.0"
+source =
"git+https://github.com/apache/arrow-datafusion.git?rev=dd98aab#dd98aabdaebcfc30ec4c370be93f6663de50e02f";
dependencies = [
"async-recursion",
"chrono",
@@ -964,9 +979,9 @@ dependencies = [
[[package]]
name = "futures"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
+checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549"
dependencies = [
"futures-channel",
"futures-core",
@@ -979,9 +994,9 @@ dependencies = [
[[package]]
name = "futures-channel"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
+checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac"
dependencies = [
"futures-core",
"futures-sink",
@@ -989,15 +1004,15 @@ dependencies = [
[[package]]
name = "futures-core"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
+checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd"
[[package]]
name = "futures-executor"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
+checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83"
dependencies = [
"futures-core",
"futures-task",
@@ -1006,15 +1021,15 @@ dependencies = [
[[package]]
name = "futures-io"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"
+checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91"
[[package]]
name = "futures-macro"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
+checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6"
dependencies = [
"proc-macro2",
"quote",
@@ -1023,21 +1038,21 @@ dependencies = [
[[package]]
name = "futures-sink"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
+checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2"
[[package]]
name = "futures-task"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
+checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879"
[[package]]
name = "futures-util"
-version = "0.3.26"
+version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
+checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab"
dependencies = [
"futures-channel",
"futures-core",
@@ -1191,9 +1206,9 @@ checksum =
"c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
[[package]]
name = "hyper"
-version = "0.14.24"
+version = "0.14.25"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c"
+checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899"
dependencies = [
"bytes",
"futures-channel",
@@ -1412,9 +1427,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.139"
+version = "0.2.140"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
[[package]]
name = "libflate"
@@ -1738,9 +1753,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "33.0.0"
+version = "34.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b1b076829801167d889795cd1957989055543430fa1469cb1f6e32b789bfc764"
+checksum = "7ac135ecf63ebb5f53dda0921b0b76d6048b3ef631a5f4760b9e8f863ff00cfa"
dependencies = [
"ahash",
"arrow-array",
@@ -1766,7 +1781,7 @@ dependencies = [
"thrift",
"tokio",
"twox-hash",
- "zstd",
+ "zstd 0.12.3+zstd.1.5.2",
]
[[package]]
@@ -1833,9 +1848,9 @@ checksum =
"dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
[[package]]
name = "proc-macro2"
-version = "1.0.51"
+version = "1.0.52"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
+checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
dependencies = [
"unicode-ident",
]
@@ -2023,9 +2038,9 @@ dependencies = [
[[package]]
name = "quote"
-version = "1.0.23"
+version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+checksum = "50686e0021c4136d1d453b2dfe059902278681512a34d4248435dc34b6b5c8ec"
dependencies = [
"proc-macro2",
]
@@ -2163,7 +2178,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
- "semver 1.0.16",
+ "semver 1.0.17",
]
[[package]]
@@ -2292,9 +2307,9 @@ dependencies = [
[[package]]
name = "semver"
-version = "1.0.16"
+version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
+checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
[[package]]
name = "semver-parser"
@@ -2313,18 +2328,18 @@ checksum =
"e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc"
[[package]]
name = "serde"
-version = "1.0.153"
+version = "1.0.155"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3a382c72b4ba118526e187430bb4963cd6d55051ebf13d9b25574d379cc98d20"
+checksum = "71f2b4817415c6d4210bfe1c7bfcf4801b2d904cb4d0e1a8fdb651013c9e86b8"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
-version = "1.0.153"
+version = "1.0.155"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1ef476a5790f0f6decbc66726b6e5d63680ed518283e64c7df415989d880954f"
+checksum = "d071a94a3fac4aff69d023a7f411e33f40f3483f8c5190b1953822b6b76d7630"
dependencies = [
"proc-macro2",
"quote",
@@ -2461,9 +2476,9 @@ checksum =
"6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "sqlparser"
-version = "0.30.0"
+version = "0.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "db67dc6ef36edb658196c3fef0464a80b53dbbc194a904e81f9bd4190f9ecc5b"
+checksum = "0366f270dbabb5cc2e4c88427dc4c08bba144f81e32fbd459a013f26a4d16aa0"
dependencies = [
"log",
"sqlparser_derive",
@@ -2507,9 +2522,9 @@ dependencies = [
[[package]]
name = "substrait"
-version = "0.4.0"
+version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e2feb96a6a106e21161551af32dc4e0fdab3aceb926b940d7e92a086b640fc7c"
+checksum = "3108bf99c703e39728847cce9becff451f8a94cbc72fb5918b4e7f0543d7b06a"
dependencies = [
"heck 0.4.1",
"prost 0.11.8",
@@ -2846,9 +2861,9 @@ checksum =
"9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
[[package]]
name = "unicode-bidi"
-version = "0.3.10"
+version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58"
+checksum = "524b68aca1d05e03fdf03fcdce2c6c94b6daf6d16861ddaa7e4f2b6638a9052c"
[[package]]
name = "unicode-ident"
@@ -3131,9 +3146,9 @@ dependencies = [
[[package]]
name = "windows-targets"
-version = "0.42.1"
+version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
@@ -3146,45 +3161,45 @@ dependencies = [
[[package]]
name = "windows_aarch64_gnullvm"
-version = "0.42.1"
+version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
[[package]]
name = "windows_aarch64_msvc"
-version = "0.42.1"
+version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
[[package]]
name = "windows_i686_gnu"
-version = "0.42.1"
+version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
[[package]]
name = "windows_i686_msvc"
-version = "0.42.1"
+version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
[[package]]
name = "windows_x86_64_gnu"
-version = "0.42.1"
+version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
[[package]]
name = "windows_x86_64_gnullvm"
-version = "0.42.1"
+version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
[[package]]
name = "windows_x86_64_msvc"
-version = "0.42.1"
+version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]]
name = "winreg"
@@ -3225,13 +3240,32 @@ dependencies = [
"syn",
]
+[[package]]
+name = "zstd"
+version = "0.11.2+zstd.1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
+dependencies = [
+ "zstd-safe 5.0.2+zstd.1.5.2",
+]
+
[[package]]
name = "zstd"
version = "0.12.3+zstd.1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806"
dependencies = [
- "zstd-safe",
+ "zstd-safe 6.0.4+zstd.1.5.4",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "5.0.2+zstd.1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"
+dependencies = [
+ "libc",
+ "zstd-sys",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index d84b2b6..f7c19e9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -34,12 +34,12 @@ default = ["mimalloc"]
tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread",
"sync"] }
rand = "0.8"
pyo3 = { version = "0.18.0", features = ["extension-module", "abi3",
"abi3-py37"] }
-datafusion = { version = "19.0.0", features = ["pyarrow", "avro"] }
-datafusion-expr = "19.0.0"
-datafusion-optimizer = "19.0.0"
-datafusion-common = { version = "19.0.0", features = ["pyarrow"] }
-datafusion-sql = "19.0.0"
-datafusion-substrait = "19.0.0"
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git";, rev =
"dd98aab", features = ["pyarrow", "avro"]}
+datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git";,
rev = "dd98aab" }
+datafusion-optimizer = { git =
"https://github.com/apache/arrow-datafusion.git";, rev = "dd98aab" }
+datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git";,
rev = "dd98aab", features = ["pyarrow"]}
+datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git";, rev
= "dd98aab" }
+datafusion-substrait = { git =
"https://github.com/apache/arrow-datafusion.git";, rev = "dd98aab" }
uuid = { version = "1.2", features = ["v4"] }
mimalloc = { version = "*", optional = true, default-features = false }
async-trait = "0.1"
diff --git a/datafusion/__init__.py b/datafusion/__init__.py
index 4956d9d..578e7dc 100644
--- a/datafusion/__init__.py
+++ b/datafusion/__init__.py
@@ -77,6 +77,7 @@ from .expr import (
TryCast,
Between,
Explain,
+ Extension,
)
__version__ = importlib_metadata.version(__name__)
@@ -129,6 +130,7 @@ __all__ = [
"TryCast",
"Between",
"Explain",
+ "Extension",
]
diff --git a/datafusion/tests/test_dataframe.py
b/datafusion/tests/test_dataframe.py
index b87b1f1..611bcab 100644
--- a/datafusion/tests/test_dataframe.py
+++ b/datafusion/tests/test_dataframe.py
@@ -350,14 +350,13 @@ def test_logical_plan(aggregate_df):
def test_optimized_logical_plan(aggregate_df):
plan = aggregate_df.optimized_logical_plan()
- expected = "Projection: test.c1, SUM(test.c2)"
+ expected = "Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]"
assert expected == plan.display()
expected = (
- "Projection: test.c1, SUM(test.c2)\n"
- " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
- " TableScan: test projection=[c1, c2]"
+ "Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
+ " TableScan: test projection=[c1, c2]"
)
assert expected == plan.display_indent()
@@ -366,9 +365,7 @@ def test_optimized_logical_plan(aggregate_df):
def test_execution_plan(aggregate_df):
plan = aggregate_df.execution_plan()
- expected = (
- "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
- )
+ expected = "AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1],
aggr=[SUM(test.c2)]\n" # noqa: E501
assert expected == plan.display()
@@ -382,7 +379,6 @@ def test_execution_plan(aggregate_df):
# indent plan will be different for everyone due to absolute path
# to filename, so we just check for some expected content
- assert "ProjectionExec:" in indent
assert "AggregateExec:" in indent
assert "CoalesceBatchesExec:" in indent
assert "RepartitionExec:" in indent
diff --git a/datafusion/tests/test_imports.py b/datafusion/tests/test_imports.py
index 77309e2..9075446 100644
--- a/datafusion/tests/test_imports.py
+++ b/datafusion/tests/test_imports.py
@@ -77,6 +77,8 @@ from datafusion.expr import (
Cast,
TryCast,
Between,
+ Explain,
+ Extension,
)
@@ -143,6 +145,8 @@ def test_class_module_is_datafusion():
Cast,
TryCast,
Between,
+ Explain,
+ Extension,
]:
assert klass.__module__ == "datafusion.expr"
diff --git a/src/context.rs b/src/context.rs
index e77a3b3..e2c509a 100644
--- a/src/context.rs
+++ b/src/context.rs
@@ -49,6 +49,7 @@ use datafusion::physical_plan::SendableRecordBatchStream;
use datafusion::prelude::{
AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions,
ParquetReadOptions,
};
+use datafusion_common::config::Extensions;
use datafusion_common::ScalarValue;
use pyo3::types::PyTuple;
use tokio::runtime::Runtime;
@@ -698,19 +699,20 @@ impl PySessionContext {
part: usize,
py: Python,
) -> PyResult<PyRecordBatchStream> {
- let ctx = Arc::new(TaskContext::new(
+ let ctx = TaskContext::try_new(
"task_id".to_string(),
"session_id".to_string(),
HashMap::new(),
HashMap::new(),
HashMap::new(),
Arc::new(RuntimeEnv::default()),
- ));
+ Extensions::default(),
+ );
// create a Tokio runtime to run the async code
let rt = Runtime::new().unwrap();
let plan = plan.plan.clone();
let fut:
JoinHandle<datafusion_common::Result<SendableRecordBatchStream>> =
- rt.spawn(async move { plan.execute(part, ctx) });
+ rt.spawn(async move { plan.execute(part, Arc::new(ctx?)) });
let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?;
Ok(PyRecordBatchStream::new(stream?))
}
diff --git a/src/expr.rs b/src/expr.rs
index 33f36f5..1d6c999 100644
--- a/src/expr.rs
+++ b/src/expr.rs
@@ -51,6 +51,7 @@ pub mod cross_join;
pub mod empty_relation;
pub mod exists;
pub mod explain;
+pub mod extension;
pub mod filter;
pub mod grouping_set;
pub mod in_list;
@@ -272,6 +273,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> {
m.add_class::<join::PyJoinConstraint>()?;
m.add_class::<cross_join::PyCrossJoin>()?;
m.add_class::<union::PyUnion>()?;
+ m.add_class::<extension::PyExtension>()?;
m.add_class::<filter::PyFilter>()?;
m.add_class::<projection::PyProjection>()?;
m.add_class::<table_scan::PyTableScan>()?;
diff --git a/src/expr/extension.rs b/src/expr/extension.rs
new file mode 100644
index 0000000..81a435c
--- /dev/null
+++ b/src/expr/extension.rs
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use datafusion_expr::Extension;
+use pyo3::prelude::*;
+
+use crate::sql::logical::PyLogicalPlan;
+
+use super::logical_node::LogicalNode;
+
+#[pyclass(name = "Extension", module = "datafusion.expr", subclass)]
+#[derive(Clone)]
+pub struct PyExtension {
+ pub node: Extension,
+}
+
+impl From<Extension> for PyExtension {
+ fn from(node: Extension) -> PyExtension {
+ PyExtension { node }
+ }
+}
+
+#[pymethods]
+impl PyExtension {
+ fn name(&self) -> PyResult<String> {
+ Ok(self.node.node.name().to_string())
+ }
+}
+
+impl LogicalNode for PyExtension {
+ fn inputs(&self) -> Vec<PyLogicalPlan> {
+ vec![]
+ }
+
+ fn to_variant(&self, py: Python) -> PyResult<PyObject> {
+ Ok(self.clone().into_py(py))
+ }
+}
diff --git a/src/sql/logical.rs b/src/sql/logical.rs
index 95ff3a2..a22f269 100644
--- a/src/sql/logical.rs
+++ b/src/sql/logical.rs
@@ -22,6 +22,7 @@ use crate::expr::aggregate::PyAggregate;
use crate::expr::analyze::PyAnalyze;
use crate::expr::empty_relation::PyEmptyRelation;
use crate::expr::explain::PyExplain;
+use crate::expr::extension::PyExtension;
use crate::expr::filter::PyFilter;
use crate::expr::limit::PyLimit;
use crate::expr::projection::PyProjection;
@@ -60,6 +61,7 @@ impl PyLogicalPlan {
LogicalPlan::Analyze(plan) =>
PyAnalyze::from(plan.clone()).to_variant(py),
LogicalPlan::EmptyRelation(plan) =>
PyEmptyRelation::from(plan.clone()).to_variant(py),
LogicalPlan::Explain(plan) =>
PyExplain::from(plan.clone()).to_variant(py),
+ LogicalPlan::Extension(plan) =>
PyExtension::from(plan.clone()).to_variant(py),
LogicalPlan::Filter(plan) =>
PyFilter::from(plan.clone()).to_variant(py),
LogicalPlan::Limit(plan) =>
PyLimit::from(plan.clone()).to_variant(py),
LogicalPlan::Projection(plan) =>
PyProjection::from(plan.clone()).to_variant(py),
