This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git
The following commit(s) were added to refs/heads/main by this push:
new b5446ef Upgrade datafusion 39 (#728)
b5446ef is described below
commit b5446efbc2e50fb017454173f6b11e4fad53fe5a
Author: Michael J Ward <[email protected]>
AuthorDate: Fri Jun 14 13:19:03 2024 -0500
Upgrade datafusion 39 (#728)
* deps: update datafusion to 39.0.0, pyo3 to 0.21, and object_store to
0.10.1
`datafusion-common` also depends on `pyo3`, so they need to be upgraded
together.
* feat: remove GetIndexField
datafusion replaced Expr::GetIndexField with a FieldAccessor trait.
Ref https://github.com/apache/datafusion/pull/10568
Ref https://github.com/apache/datafusion/pull/10769
* feat: update ScalarFunction
The field `func_name` was changed to `func` as part of removing
`ScalarFunctionDefinition` upstream.
Ref https://github.com/apache/datafusion/pull/10325
* feat: incorporate upstream array_slice fixes
Fixes #670
* update ExectionPlan::children impl for DatasetExec
Ref https://github.com/apache/datafusion/pull/10543
* update value_interval_daytime
Ref https://github.com/apache/arrow-rs/pull/5769
* update regexp_replace and regexp_match
Fixes #677
* add gil-refs feature to pyo3
This silences pyo3's deprecation warnings for its new Bounds api.
It's the 1st step of the migration, and should be removed before merge.
Ref https://pyo3.rs/v0.21.0/migration#from-020-to-021
* fix signature for octet_length
Ref https://github.com/apache/datafusion/pull/10726
* update signature for covar_samp
AggregateUDF expressions now have a builder API design, which removes
arguments like filter and order_by
Ref https://github.com/apache/datafusion/issues/10545
Ref https://github.com/apache/datafusion/pull/10492
* convert covar_pop to expr_fn api
Ref: https://github.com/apache/datafusion/pull/10418/files
* convert median to expr_fn api
Ref https://github.com/apache/datafusion/pull/10644
* convert variance sample to UDF
Ref https://github.com/apache/datafusion/issues/10667
* convert first_value and last_value to UDFs
Ref https://github.com/apache/datafusion/pull/10648
* checkpointing with a few todos to fix remaining compile errors
* impl PyExpr::python_value for IntervalDayTime and IntervalMonthDayNano
* convert sum aggregate function to UDF
* remove unnecessary clone on double reference
* apply cargo fmt
* remove duplicate allow-dead-code annotation
* update tpch examples for new pyarrow interval
Fixes #665
* marked q11 tpch example as expected fail
Ref #730
* add default stride of None back to array_slice
---
Cargo.lock | 540 ++++++++++++---------
Cargo.toml | 22 +-
.../user-guide/common-operations/functions.rst | 3 +-
examples/tpch/_tests.py | 5 +-
examples/tpch/q01_pricing_summary_report.py | 4 +-
examples/tpch/q04_order_priority_checking.py | 4 +-
examples/tpch/q05_local_supplier_volume.py | 4 +-
examples/tpch/q06_forecasting_revenue_change.py | 4 +-
examples/tpch/q10_returned_item_reporting.py | 4 +-
examples/tpch/q12_ship_mode_order_priority.py | 4 +-
examples/tpch/q14_promotion_effect.py | 5 +-
examples/tpch/q15_top_supplier.py | 5 +-
examples/tpch/q20_potential_part_promotion.py | 4 +-
python/datafusion/__init__.py | 2 -
python/datafusion/tests/test_functions.py | 1 -
python/datafusion/tests/test_imports.py | 2 -
src/common/data_type.rs | 1 +
src/dataset_exec.rs | 2 +-
src/expr.rs | 32 +-
src/expr/literal.rs | 7 +-
src/expr/signature.rs | 1 -
src/functions.rs | 136 ++++--
src/physical_plan.rs | 2 +-
23 files changed, 467 insertions(+), 327 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index ed1ee3b..eb674d0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -130,9 +130,9 @@ checksum =
"96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
name = "arrow"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658"
+checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -152,9 +152,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0"
+checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -167,9 +167,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea"
+checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97"
dependencies = [
"ahash",
"arrow-buffer",
@@ -184,9 +184,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27"
+checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1"
dependencies = [
"bytes",
"half",
@@ -195,9 +195,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba"
+checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -216,9 +216,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2"
+checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -235,9 +235,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5"
+checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -247,9 +247,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d"
+checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -262,9 +262,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49"
+checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -282,9 +282,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412"
+checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -297,9 +297,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228"
+checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e"
dependencies = [
"ahash",
"arrow-array",
@@ -312,18 +312,18 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528"
+checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393"
dependencies = [
"bitflags 2.5.0",
]
[[package]]
name = "arrow-select"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830"
+checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102"
dependencies = [
"ahash",
"arrow-array",
@@ -335,9 +335,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb"
+checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -364,8 +364,8 @@ dependencies = [
"pin-project-lite",
"tokio",
"xz2",
- "zstd 0.13.1",
- "zstd-safe 7.1.0",
+ "zstd 0.13.0",
+ "zstd-safe 7.0.0",
]
[[package]]
@@ -400,19 +400,16 @@ dependencies = [
]
[[package]]
-name = "autocfg"
-version = "1.3.0"
+name = "atomic-waker"
+version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
-name = "autotools"
-version = "0.2.7"
+name = "autocfg"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf"
-dependencies = [
- "cc",
-]
+checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
[[package]]
name = "backtrace"
@@ -486,9 +483,9 @@ dependencies = [
[[package]]
name = "brotli"
-version = "3.5.0"
+version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391"
+checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -497,9 +494,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
-version = "2.5.1"
+version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
+checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -576,9 +573,9 @@ dependencies = [
[[package]]
name = "chrono-tz"
-version = "0.8.6"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e"
+checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb"
dependencies = [
"chrono",
"chrono-tz-build",
@@ -587,15 +584,24 @@ dependencies = [
[[package]]
name = "chrono-tz-build"
-version = "0.2.1"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f"
+checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1"
dependencies = [
"parse-zoneinfo",
"phf",
"phf_codegen",
]
+[[package]]
+name = "cmake"
+version = "0.1.50"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "comfy-table"
version = "7.1.1"
@@ -734,9 +740,9 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "05fb4eeeb7109393a0739ac5b8fd892f95ccef691421491c85544f7997366f68"
+checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068"
dependencies = [
"ahash",
"apache-avro",
@@ -759,6 +765,7 @@ dependencies = [
"datafusion-functions-array",
"datafusion-optimizer",
"datafusion-physical-expr",
+ "datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-sql",
"flate2",
@@ -767,13 +774,14 @@ dependencies = [
"half",
"hashbrown",
"indexmap",
- "itertools",
+ "itertools 0.12.1",
"log",
"num-traits",
"num_cpus",
"object_store",
"parking_lot",
"parquet",
+ "paste",
"pin-project-lite",
"rand",
"sqlparser",
@@ -783,14 +791,14 @@ dependencies = [
"url",
"uuid",
"xz2",
- "zstd 0.13.1",
+ "zstd 0.13.0",
]
[[package]]
name = "datafusion-common"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "741aeac15c82f239f2fc17deccaab19873abbd62987be20023689b15fa72fa09"
+checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971"
dependencies = [
"ahash",
"apache-avro",
@@ -800,6 +808,7 @@ dependencies = [
"arrow-schema",
"chrono",
"half",
+ "hashbrown",
"instant",
"libc",
"num_cpus",
@@ -811,18 +820,18 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6e8ddfb8d8cb51646a30da0122ecfffb81ca16919ae9a3495a9e7468bdcd52b8"
+checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894"
dependencies = [
"tokio",
]
[[package]]
name = "datafusion-execution"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "282122f90b20e8f98ebfa101e4bf20e718fd2684cf81bef4e8c6366571c64404"
+checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1"
dependencies = [
"arrow",
"chrono",
@@ -841,13 +850,14 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5478588f733df0dfd87a62671c7478f590952c95fa2fa5c137e3ff2929491e22"
+checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3"
dependencies = [
"ahash",
"arrow",
"arrow-array",
+ "arrow-buffer",
"chrono",
"datafusion-common",
"paste",
@@ -859,9 +869,9 @@ dependencies = [
[[package]]
name = "datafusion-functions"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f4afd261cea6ac9c3ca1192fd5e9f940596d8e9208c5b1333f4961405db53185"
+checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b"
dependencies = [
"arrow",
"base64 0.22.1",
@@ -874,7 +884,7 @@ dependencies = [
"datafusion-physical-expr",
"hashbrown",
"hex",
- "itertools",
+ "itertools 0.12.1",
"log",
"md-5",
"rand",
@@ -886,11 +896,13 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1b36a6c4838ab94b5bf8f7a96ce6ce059d805c5d1dcaa6ace49e034eb65cd999"
+checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78"
dependencies = [
+ "ahash",
"arrow",
+ "arrow-schema",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
@@ -902,9 +914,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-array"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d5fdd200a6233f48d3362e7ccb784f926f759100e44ae2137a5e2dcb986a59c4"
+checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a"
dependencies = [
"arrow",
"arrow-array",
@@ -915,16 +927,16 @@ dependencies = [
"datafusion-execution",
"datafusion-expr",
"datafusion-functions",
- "itertools",
+ "itertools 0.12.1",
"log",
"paste",
]
[[package]]
name = "datafusion-optimizer"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "54f2820938810e8a2d71228fd6f59f33396aebc5f5f687fcbf14de5aab6a7e1a"
+checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76"
dependencies = [
"arrow",
"async-trait",
@@ -934,16 +946,16 @@ dependencies = [
"datafusion-physical-expr",
"hashbrown",
"indexmap",
- "itertools",
+ "itertools 0.12.1",
"log",
"regex-syntax",
]
[[package]]
name = "datafusion-physical-expr"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9adf8eb12716f52ddf01e09eb6c94d3c9b291e062c05c91b839a448bddba2ff8"
+checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e"
dependencies = [
"ahash",
"arrow",
@@ -963,7 +975,7 @@ dependencies = [
"hashbrown",
"hex",
"indexmap",
- "itertools",
+ "itertools 0.12.1",
"log",
"paste",
"petgraph",
@@ -972,20 +984,21 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8d5472c3230584c150197b3f2c23f2392b9dc54dbfb62ad41e7e36447cfce4be"
+checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77"
dependencies = [
"arrow",
"datafusion-common",
"datafusion-expr",
+ "rand",
]
[[package]]
name = "datafusion-physical-plan"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "18ae750c38389685a8b62e5b899bbbec488950755ad6d218f3662d35b800c4fe"
+checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff"
dependencies = [
"ahash",
"arrow",
@@ -1006,7 +1019,7 @@ dependencies = [
"half",
"hashbrown",
"indexmap",
- "itertools",
+ "itertools 0.12.1",
"log",
"once_cell",
"parking_lot",
@@ -1017,7 +1030,7 @@ dependencies = [
[[package]]
name = "datafusion-python"
-version = "38.0.1"
+version = "39.0.0"
dependencies = [
"async-trait",
"datafusion",
@@ -1045,9 +1058,9 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "befc67a3cdfbfa76853f43b10ac27337821bb98e519ab6baf431fcc0bcfcafdb"
+checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e"
dependencies = [
"arrow",
"arrow-array",
@@ -1055,23 +1068,25 @@ dependencies = [
"datafusion-common",
"datafusion-expr",
"log",
+ "regex",
"sqlparser",
"strum 0.26.2",
]
[[package]]
name = "datafusion-substrait"
-version = "38.0.0"
+version = "39.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1f62542caa77df003e23a8bc2f1b8a1ffc682fe447c7fcb4905d109e3d7a5b9d"
+checksum = "8249d69665c1cd32e07789ed6dd1da6528a23019ef16d3483db52952b6f9f68a"
dependencies = [
+ "arrow-buffer",
"async-recursion",
"chrono",
"datafusion",
- "itertools",
+ "itertools 0.12.1",
"object_store",
+ "pbjson-types",
"prost",
- "prost-types",
"substrait",
]
@@ -1104,15 +1119,6 @@ version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
-[[package]]
-name = "encoding_rs"
-version = "0.8.34"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
-dependencies = [
- "cfg-if",
-]
-
[[package]]
name = "equivalent"
version = "1.0.1"
@@ -1143,9 +1149,9 @@ checksum =
"0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flatbuffers"
-version = "23.5.26"
+version = "24.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640"
+checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f"
dependencies = [
"bitflags 1.3.2",
"rustc_version",
@@ -1300,15 +1306,15 @@ checksum =
"d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "h2"
-version = "0.3.26"
+version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
+checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab"
dependencies = [
+ "atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
- "futures-util",
"http",
"indexmap",
"slab",
@@ -1364,9 +1370,9 @@ checksum =
"7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "http"
-version = "0.2.12"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
+checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
dependencies = [
"bytes",
"fnv",
@@ -1375,12 +1381,24 @@ dependencies = [
[[package]]
name = "http-body"
-version = "0.4.6"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
+dependencies = [
+ "bytes",
+ "http",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
+checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d"
dependencies = [
"bytes",
+ "futures-core",
"http",
+ "http-body",
"pin-project-lite",
]
@@ -1390,12 +1408,6 @@ version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
-[[package]]
-name = "httpdate"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
-
[[package]]
name = "humantime"
version = "2.1.0"
@@ -1404,40 +1416,59 @@ checksum =
"9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
-version = "0.14.29"
+version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33"
+checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d"
dependencies = [
"bytes",
"futures-channel",
- "futures-core",
"futures-util",
"h2",
"http",
"http-body",
"httparse",
- "httpdate",
"itoa",
"pin-project-lite",
- "socket2",
+ "smallvec",
"tokio",
- "tower-service",
- "tracing",
"want",
]
[[package]]
name = "hyper-rustls"
-version = "0.24.2"
+version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
+checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c"
dependencies = [
"futures-util",
"http",
"hyper",
+ "hyper-util",
"rustls",
+ "rustls-pki-types",
"tokio",
"tokio-rustls",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower",
+ "tower-service",
+ "tracing",
]
[[package]]
@@ -1513,6 +1544,15 @@ version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+[[package]]
+name = "itertools"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itertools"
version = "0.12.1"
@@ -1865,18 +1905,18 @@ dependencies = [
[[package]]
name = "object_store"
-version = "0.9.1"
+version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3"
+checksum = "fbebfd32c213ba1907fa7a9c9138015a8de2b43e30c5aa45b18f7deb46786ad6"
dependencies = [
"async-trait",
- "base64 0.21.7",
+ "base64 0.22.1",
"bytes",
"chrono",
"futures",
"humantime",
"hyper",
- "itertools",
+ "itertools 0.12.1",
"md-5",
"parking_lot",
"percent-encoding",
@@ -1884,7 +1924,7 @@ dependencies = [
"rand",
"reqwest",
"ring",
- "rustls-pemfile 2.1.2",
+ "rustls-pemfile",
"serde",
"serde_json",
"snafu",
@@ -1940,9 +1980,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "51.0.0"
+version = "52.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32"
+checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab"
dependencies = [
"ahash",
"arrow-array",
@@ -1970,7 +2010,8 @@ dependencies = [
"thrift",
"tokio",
"twox-hash",
- "zstd 0.13.1",
+ "zstd 0.13.0",
+ "zstd-sys",
]
[[package]]
@@ -1988,6 +2029,43 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+[[package]]
+name = "pbjson"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90"
+dependencies = [
+ "base64 0.21.7",
+ "serde",
+]
+
+[[package]]
+name = "pbjson-build"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735"
+dependencies = [
+ "heck 0.4.1",
+ "itertools 0.11.0",
+ "prost",
+ "prost-types",
+]
+
+[[package]]
+name = "pbjson-types"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12"
+dependencies = [
+ "bytes",
+ "chrono",
+ "pbjson",
+ "pbjson-build",
+ "prost",
+ "prost-build",
+ "serde",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.1"
@@ -2042,6 +2120,26 @@ dependencies = [
"siphasher",
]
+[[package]]
+name = "pin-project"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.66",
+]
+
[[package]]
name = "pin-project-lite"
version = "0.2.14"
@@ -2109,14 +2207,16 @@ checksum =
"22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
dependencies = [
"bytes",
"heck 0.5.0",
- "itertools",
+ "itertools 0.12.1",
"log",
"multimap",
"once_cell",
"petgraph",
+ "prettyplease",
"prost",
"prost-types",
"regex",
+ "syn 2.0.66",
"tempfile",
]
@@ -2127,7 +2227,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
dependencies = [
"anyhow",
- "itertools",
+ "itertools 0.12.1",
"proc-macro2",
"quote",
"syn 2.0.66",
@@ -2144,18 +2244,18 @@ dependencies = [
[[package]]
name = "protobuf-src"
-version = "1.1.0+21.5"
+version = "2.0.1+26.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1"
+checksum = "f8ba1cfa4b9dc098926b8cce388bf434b93516db3ecf6e8b1a37eb643d733ee7"
dependencies = [
- "autotools",
+ "cmake",
]
[[package]]
name = "pyo3"
-version = "0.20.3"
+version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233"
+checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8"
dependencies = [
"cfg-if",
"indoc",
@@ -2171,9 +2271,9 @@ dependencies = [
[[package]]
name = "pyo3-build-config"
-version = "0.20.3"
+version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7"
+checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50"
dependencies = [
"once_cell",
"target-lexicon",
@@ -2181,9 +2281,9 @@ dependencies = [
[[package]]
name = "pyo3-ffi"
-version = "0.20.3"
+version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa"
+checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403"
dependencies = [
"libc",
"pyo3-build-config",
@@ -2191,9 +2291,9 @@ dependencies = [
[[package]]
name = "pyo3-macros"
-version = "0.20.3"
+version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158"
+checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
@@ -2203,9 +2303,9 @@ dependencies = [
[[package]]
name = "pyo3-macros-backend"
-version = "0.20.3"
+version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185"
+checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c"
dependencies = [
"heck 0.4.1",
"proc-macro2",
@@ -2315,9 +2415,9 @@ checksum =
"adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
[[package]]
name = "regress"
-version = "0.8.0"
+version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4f5f39ba4513916c1b2657b72af6ec671f091cd637992f58d0ede5cae4e5dea0"
+checksum = "0eae2a1ebfecc58aff952ef8ccd364329abe627762f5bf09ff42eb9d98522479"
dependencies = [
"hashbrown",
"memchr",
@@ -2325,20 +2425,21 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.11.27"
+version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
+checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10"
dependencies = [
- "base64 0.21.7",
+ "base64 0.22.1",
"bytes",
- "encoding_rs",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
+ "http-body-util",
"hyper",
"hyper-rustls",
+ "hyper-util",
"ipnet",
"js-sys",
"log",
@@ -2348,12 +2449,12 @@ dependencies = [
"pin-project-lite",
"rustls",
"rustls-native-certs",
- "rustls-pemfile 1.0.4",
+ "rustls-pemfile",
+ "rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
- "system-configuration",
"tokio",
"tokio-rustls",
"tokio-util",
@@ -2417,37 +2518,31 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.21.12"
+version = "0.22.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
+checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432"
dependencies = [
"log",
"ring",
+ "rustls-pki-types",
"rustls-webpki",
- "sct",
+ "subtle",
+ "zeroize",
]
[[package]]
name = "rustls-native-certs"
-version = "0.6.3"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
+checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792"
dependencies = [
"openssl-probe",
- "rustls-pemfile 1.0.4",
+ "rustls-pemfile",
+ "rustls-pki-types",
"schannel",
"security-framework",
]
-[[package]]
-name = "rustls-pemfile"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
-dependencies = [
- "base64 0.21.7",
-]
-
[[package]]
name = "rustls-pemfile"
version = "2.1.2"
@@ -2466,11 +2561,12 @@ checksum =
"976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
[[package]]
name = "rustls-webpki"
-version = "0.101.7"
+version = "0.102.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
+checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"
dependencies = [
"ring",
+ "rustls-pki-types",
"untrusted",
]
@@ -2534,16 +2630,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-[[package]]
-name = "sct"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
-dependencies = [
- "ring",
- "untrusted",
-]
-
[[package]]
name = "security-framework"
version = "2.11.0"
@@ -2572,6 +2658,9 @@ name = "semver"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
+dependencies = [
+ "serde",
+]
[[package]]
name = "seq-macro"
@@ -2736,9 +2825,9 @@ checksum =
"6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "sqlparser"
-version = "0.45.0"
+version = "0.47.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0"
+checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25"
dependencies = [
"log",
"sqlparser_derive",
@@ -2804,11 +2893,14 @@ dependencies = [
[[package]]
name = "substrait"
-version = "0.32.0"
+version = "0.34.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f01344023c2614171a9ffd6e387eea14e12f7387c5b6adb33f1563187d65e376"
+checksum = "04c77dec9b6c4e48ac828937bbe7cf473b0933168c5d76d51a5816ace7046be9"
dependencies = [
"heck 0.5.0",
+ "pbjson",
+ "pbjson-build",
+ "pbjson-types",
"prettyplease",
"prost",
"prost-build",
@@ -2858,27 +2950,6 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
-[[package]]
-name = "system-configuration"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
-dependencies = [
- "bitflags 1.3.2",
- "core-foundation",
- "system-configuration-sys",
-]
-
-[[package]]
-name = "system-configuration-sys"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
-
[[package]]
name = "target-lexicon"
version = "0.12.14"
@@ -2982,11 +3053,12 @@ dependencies = [
[[package]]
name = "tokio-rustls"
-version = "0.24.1"
+version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
+checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f"
dependencies = [
"rustls",
+ "rustls-pki-types",
"tokio",
]
@@ -3003,6 +3075,27 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project",
+ "pin-project-lite",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+
[[package]]
name = "tower-service"
version = "0.3.2"
@@ -3084,9 +3177,9 @@ checksum =
"42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "typify"
-version = "0.0.16"
+version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5c61e9db210bbff218e6535c664b37ec47da449169b98e7866d0580d0db75529"
+checksum = "adb6beec125971dda80a086f90b4a70f60f222990ce4d63ad0fc140492f53444"
dependencies = [
"typify-impl",
"typify-macro",
@@ -3094,16 +3187,18 @@ dependencies = [
[[package]]
name = "typify-impl"
-version = "0.0.16"
+version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "95e32f38493804f88e2dc7a5412eccd872ea5452b4db9b0a77de4df180f2a87e"
+checksum = "93bbb24e990654aff858d80fee8114f4322f7d7a1b1ecb45129e2fcb0d0ad5ae"
dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
"log",
"proc-macro2",
"quote",
"regress",
"schemars",
+ "semver",
+ "serde",
"serde_json",
"syn 2.0.66",
"thiserror",
@@ -3112,13 +3207,14 @@ dependencies = [
[[package]]
name = "typify-macro"
-version = "0.0.16"
+version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cc09508b72f63d521d68e42c7f172c7416d67986df44b3c7d1f7f9963948ed32"
+checksum = "f8e6491896e955692d68361c68db2b263e3bec317ec0b684e0e2fa882fb6e31e"
dependencies = [
"proc-macro2",
"quote",
"schemars",
+ "semver",
"serde",
"serde_json",
"serde_tokenstream",
@@ -3477,9 +3573,9 @@ checksum =
"bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
[[package]]
name = "winreg"
-version = "0.50.0"
+version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
+checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5"
dependencies = [
"cfg-if",
"windows-sys 0.48.0",
@@ -3514,6 +3610,12 @@ dependencies = [
"syn 2.0.66",
]
+[[package]]
+name = "zeroize"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
+
[[package]]
name = "zstd"
version = "0.12.4"
@@ -3525,11 +3627,11 @@ dependencies = [
[[package]]
name = "zstd"
-version = "0.13.1"
+version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a"
+checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110"
dependencies = [
- "zstd-safe 7.1.0",
+ "zstd-safe 7.0.0",
]
[[package]]
@@ -3544,18 +3646,18 @@ dependencies = [
[[package]]
name = "zstd-safe"
-version = "7.1.0"
+version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a"
+checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
-version = "2.0.10+zstd.1.5.6"
+version = "2.0.9+zstd.1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa"
+checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
dependencies = [
"cc",
"pkg-config",
diff --git a/Cargo.toml b/Cargo.toml
index fd6f4bb..85a19d1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,7 +17,7 @@
[package]
name = "datafusion-python"
-version = "38.0.1"
+version = "39.0.0"
homepage = "https://datafusion.apache.org/python";
repository = "https://github.com/apache/datafusion-python";
authors = ["Apache DataFusion <[email protected]>"]
@@ -36,28 +36,28 @@ substrait = ["dep:datafusion-substrait"]
[dependencies]
tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread",
"sync"] }
rand = "0.8"
-pyo3 = { version = "0.20", features = ["extension-module", "abi3",
"abi3-py38"] }
-datafusion = { version = "38.0.0", features = ["pyarrow", "avro",
"unicode_expressions"] }
-datafusion-common = { version = "38.0.0", features = ["pyarrow"] }
-datafusion-expr = "38.0.0"
-datafusion-functions-array = "38.0.0"
-datafusion-optimizer = "38.0.0"
-datafusion-sql = "38.0.0"
-datafusion-substrait = { version = "38.0.0", optional = true }
+pyo3 = { version = "0.21", features = ["extension-module", "abi3",
"abi3-py38", "gil-refs"] }
+datafusion = { version = "39.0.0", features = ["pyarrow", "avro",
"unicode_expressions"] }
+datafusion-common = { version = "39.0.0", features = ["pyarrow"] }
+datafusion-expr = "39.0.0"
+datafusion-functions-array = "39.0.0"
+datafusion-optimizer = "39.0.0"
+datafusion-sql = "39.0.0"
+datafusion-substrait = { version = "39.0.0", optional = true }
prost = "0.12"
prost-types = "0.12"
uuid = { version = "1.8", features = ["v4"] }
mimalloc = { version = "0.1", optional = true, default-features = false,
features = ["local_dynamic_tls"] }
async-trait = "0.1"
futures = "0.3"
-object_store = { version = "0.9.1", features = ["aws", "gcp", "azure"] }
+object_store = { version = "0.10.1", features = ["aws", "gcp", "azure"] }
parking_lot = "0.12"
regex-syntax = "0.8.1"
syn = "2.0.43"
url = "2.2"
[build-dependencies]
-pyo3-build-config = "0.20.0"
+pyo3-build-config = "0.21"
[lib]
name = "datafusion_python"
diff --git a/docs/source/user-guide/common-operations/functions.rst
b/docs/source/user-guide/common-operations/functions.rst
index 50b4930..d793314 100644
--- a/docs/source/user-guide/common-operations/functions.rst
+++ b/docs/source/user-guide/common-operations/functions.rst
@@ -92,12 +92,13 @@ DataFusion offers a range of helpful options.
f.left(col('"Name"'), literal(4)).alias("code")
)
-This also includes the functions for regular expressions like
:func:`.regexp_match`
+This also includes the functions for regular expressions like
:func:`.regexp_replace` and :func:`.regexp_match`
.. ipython:: python
df.select(
f.regexp_match(col('"Name"'), literal("Char")).alias("dragons"),
+ f.regexp_replace(col('"Name"'), literal("saur"),
literal("fleur")).alias("flowers")
)
diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py
index 049b43d..aa9491b 100644
--- a/examples/tpch/_tests.py
+++ b/examples/tpch/_tests.py
@@ -72,7 +72,10 @@ def check_q17(df):
("q08_market_share", "q8"),
("q09_product_type_profit_measure", "q9"),
("q10_returned_item_reporting", "q10"),
- ("q11_important_stock_identification", "q11"),
+ pytest.param(
+ "q11_important_stock_identification", "q11",
+ marks=pytest.mark.xfail #
https://github.com/apache/datafusion-python/issues/730
+ ),
("q12_ship_mode_order_priority", "q12"),
("q13_customer_distribution", "q13"),
("q14_promotion_effect", "q14"),
diff --git a/examples/tpch/q01_pricing_summary_report.py
b/examples/tpch/q01_pricing_summary_report.py
index 7e86055..cb9485a 100644
--- a/examples/tpch/q01_pricing_summary_report.py
+++ b/examples/tpch/q01_pricing_summary_report.py
@@ -48,9 +48,7 @@ greatest_ship_date = df.aggregate(
# want to report results for. It should be between 60-120 days before the end.
DAYS_BEFORE_FINAL = 90
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval = pa.scalar((0, 0, DAYS_BEFORE_FINAL),
type=pa.month_day_nano_interval())
+interval = pa.scalar((0, DAYS_BEFORE_FINAL, 0),
type=pa.month_day_nano_interval())
print("Final date in database:", greatest_ship_date)
diff --git a/examples/tpch/q04_order_priority_checking.py
b/examples/tpch/q04_order_priority_checking.py
index 40eab69..9dbd816 100644
--- a/examples/tpch/q04_order_priority_checking.py
+++ b/examples/tpch/q04_order_priority_checking.py
@@ -49,9 +49,7 @@ df_lineitem =
ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns
# Create a date object from the string
date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date()
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval())
+interval = pa.scalar((0, INTERVAL_DAYS, 0), type=pa.month_day_nano_interval())
# Limit results to cases where commitment date before receipt date
# Aggregate the results so we only get one row to join with the order table.
diff --git a/examples/tpch/q05_local_supplier_volume.py
b/examples/tpch/q05_local_supplier_volume.py
index 27b4b84..f17f600 100644
--- a/examples/tpch/q05_local_supplier_volume.py
+++ b/examples/tpch/q05_local_supplier_volume.py
@@ -41,9 +41,7 @@ REGION_OF_INTEREST = "ASIA"
date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date()
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval())
+interval = pa.scalar((0, INTERVAL_DAYS, 0), type=pa.month_day_nano_interval())
# Load the dataframes we need
diff --git a/examples/tpch/q06_forecasting_revenue_change.py
b/examples/tpch/q06_forecasting_revenue_change.py
index 3f58c5e..ec98aaf 100644
--- a/examples/tpch/q06_forecasting_revenue_change.py
+++ b/examples/tpch/q06_forecasting_revenue_change.py
@@ -45,9 +45,7 @@ INTERVAL_DAYS = 365
date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date()
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval())
+interval = pa.scalar((0, INTERVAL_DAYS, 0), type=pa.month_day_nano_interval())
# Load the dataframes we need
diff --git a/examples/tpch/q10_returned_item_reporting.py
b/examples/tpch/q10_returned_item_reporting.py
index ed88c29..78327c3 100644
--- a/examples/tpch/q10_returned_item_reporting.py
+++ b/examples/tpch/q10_returned_item_reporting.py
@@ -38,9 +38,7 @@ DATE_START_OF_QUARTER = "1993-10-01"
date_start_of_quarter = lit(datetime.strptime(DATE_START_OF_QUARTER,
"%Y-%m-%d").date())
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval_one_quarter = lit(pa.scalar((0, 0, 92),
type=pa.month_day_nano_interval()))
+interval_one_quarter = lit(pa.scalar((0, 92, 0),
type=pa.month_day_nano_interval()))
# Load the dataframes we need
diff --git a/examples/tpch/q12_ship_mode_order_priority.py
b/examples/tpch/q12_ship_mode_order_priority.py
index d3dd7d2..150870c 100644
--- a/examples/tpch/q12_ship_mode_order_priority.py
+++ b/examples/tpch/q12_ship_mode_order_priority.py
@@ -51,9 +51,7 @@ df_lineitem =
ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns
date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date()
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval = pa.scalar((0, 0, 365), type=pa.month_day_nano_interval())
+interval = pa.scalar((0, 365, 0), type=pa.month_day_nano_interval())
df = df_lineitem.filter(col("l_receiptdate") >= lit(date)).filter(
diff --git a/examples/tpch/q14_promotion_effect.py
b/examples/tpch/q14_promotion_effect.py
index 333398c..75fa363 100644
--- a/examples/tpch/q14_promotion_effect.py
+++ b/examples/tpch/q14_promotion_effect.py
@@ -34,9 +34,8 @@ from util import get_data_path
DATE = "1995-09-01"
date_of_interest = lit(datetime.strptime(DATE, "%Y-%m-%d").date())
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval_one_month = lit(pa.scalar((0, 0, 30),
type=pa.month_day_nano_interval()))
+
+interval_one_month = lit(pa.scalar((0, 30, 0),
type=pa.month_day_nano_interval()))
# Load the dataframes we need
diff --git a/examples/tpch/q15_top_supplier.py
b/examples/tpch/q15_top_supplier.py
index 91af34a..4b9e4c1 100644
--- a/examples/tpch/q15_top_supplier.py
+++ b/examples/tpch/q15_top_supplier.py
@@ -34,9 +34,8 @@ from util import get_data_path
DATE = "1996-01-01"
date_of_interest = lit(datetime.strptime(DATE, "%Y-%m-%d").date())
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval_3_months = lit(pa.scalar((0, 0, 91),
type=pa.month_day_nano_interval()))
+
+interval_3_months = lit(pa.scalar((0, 91, 0),
type=pa.month_day_nano_interval()))
# Load the dataframes we need
diff --git a/examples/tpch/q20_potential_part_promotion.py
b/examples/tpch/q20_potential_part_promotion.py
index 4a60284..85e7226 100644
--- a/examples/tpch/q20_potential_part_promotion.py
+++ b/examples/tpch/q20_potential_part_promotion.py
@@ -56,9 +56,7 @@ df_nation =
ctx.read_parquet(get_data_path("nation.parquet")).select_columns(
date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date()
-# Note: this is a hack on setting the values. It should be set differently once
-# https://github.com/apache/datafusion-python/issues/665 is resolved.
-interval = pa.scalar((0, 0, 365), type=pa.month_day_nano_interval())
+interval = pa.scalar((0, 365, 0), type=pa.month_day_nano_interval())
# Filter down dataframes
df_nation = df_nation.filter(col("n_name") == lit(NATION_OF_INTEREST))
diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py
index d0b823b..846b1a4 100644
--- a/python/datafusion/__init__.py
+++ b/python/datafusion/__init__.py
@@ -53,7 +53,6 @@ from .expr import (
ScalarVariable,
Sort,
TableScan,
- GetIndexedField,
Not,
IsNotNull,
IsTrue,
@@ -116,7 +115,6 @@ __all__ = [
"SimilarTo",
"ScalarVariable",
"Alias",
- "GetIndexedField",
"Not",
"IsNotNull",
"IsTrue",
diff --git a/python/datafusion/tests/test_functions.py
b/python/datafusion/tests/test_functions.py
index 493b6b6..449f706 100644
--- a/python/datafusion/tests/test_functions.py
+++ b/python/datafusion/tests/test_functions.py
@@ -461,7 +461,6 @@ def py_flatten(arr):
pytest.param(
lambda col: f.list_slice(col, literal(-1), literal(2)),
lambda data: [arr[-1:2] for arr in data],
- marks=pytest.mark.xfail,
),
[
lambda col: f.array_intersect(col, literal([3.0, 4.0])),
diff --git a/python/datafusion/tests/test_imports.py
b/python/datafusion/tests/test_imports.py
index 2a8a3de..bd4e7c3 100644
--- a/python/datafusion/tests/test_imports.py
+++ b/python/datafusion/tests/test_imports.py
@@ -53,7 +53,6 @@ from datafusion.expr import (
SimilarTo,
ScalarVariable,
Alias,
- GetIndexedField,
Not,
IsNotNull,
IsTrue,
@@ -126,7 +125,6 @@ def test_class_module_is_datafusion():
SimilarTo,
ScalarVariable,
Alias,
- GetIndexedField,
Not,
IsNotNull,
IsTrue,
diff --git a/src/common/data_type.rs b/src/common/data_type.rs
index 49959c4..cd4f864 100644
--- a/src/common/data_type.rs
+++ b/src/common/data_type.rs
@@ -251,6 +251,7 @@ impl DataTypeMap {
pub fn map_from_scalar_to_arrow(scalar_val: &ScalarValue) ->
Result<DataType, PyErr> {
match scalar_val {
ScalarValue::Boolean(_) => Ok(DataType::Boolean),
+ ScalarValue::Float16(_) => Ok(DataType::Float16),
ScalarValue::Float32(_) => Ok(DataType::Float32),
ScalarValue::Float64(_) => Ok(DataType::Float64),
ScalarValue::Decimal128(_, precision, scale) => {
diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs
index 4c9e2f3..8ef3a56 100644
--- a/src/dataset_exec.rs
+++ b/src/dataset_exec.rs
@@ -164,7 +164,7 @@ impl ExecutionPlan for DatasetExec {
self.schema.clone()
}
- fn children(&self) -> Vec<Arc<dyn ExecutionPlan>> {
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
// this is a leaf node and has no children
vec![]
}
diff --git a/src/expr.rs b/src/expr.rs
index 2f14774..9fd29a5 100644
--- a/src/expr.rs
+++ b/src/expr.rs
@@ -23,12 +23,12 @@ use std::sync::Arc;
use datafusion::arrow::datatypes::{DataType, Field};
use datafusion::arrow::pyarrow::PyArrowType;
+use datafusion::functions::core::expr_ext::FieldAccessor;
use datafusion::scalar::ScalarValue;
use datafusion_expr::{
col,
expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort,
WindowFunction},
- lit, Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess,
GetIndexedField, Like, Operator,
- TryCast,
+ lit, Between, BinaryExpr, Case, Cast, Expr, Like, Operator, TryCast,
};
use crate::common::data_type::{DataTypeMap, RexType};
@@ -71,7 +71,6 @@ pub mod filter;
pub mod grouping_set;
pub mod in_list;
pub mod in_subquery;
-pub mod indexed_field;
pub mod join;
pub mod like;
pub mod limit;
@@ -216,13 +215,7 @@ impl PyExpr {
}
fn __getitem__(&self, key: &str) -> PyResult<PyExpr> {
- Ok(Expr::GetIndexedField(GetIndexedField::new(
- Box::new(self.expr.clone()),
- GetFieldAccess::NamedStructField {
- name: ScalarValue::Utf8(Some(key.to_string())),
- },
- ))
- .into())
+ Ok(self.expr.clone().field(key).into())
}
#[staticmethod]
@@ -263,7 +256,7 @@ impl PyExpr {
pub fn rex_type(&self) -> PyResult<RexType> {
Ok(match self.expr {
Expr::Alias(..) => RexType::Alias,
- Expr::Column(..) | Expr::GetIndexedField { .. } =>
RexType::Reference,
+ Expr::Column(..) => RexType::Reference,
Expr::ScalarVariable(..) | Expr::Literal(..) => RexType::Literal,
Expr::BinaryExpr { .. }
| Expr::Not(..)
@@ -314,6 +307,11 @@ impl PyExpr {
),
)),
ScalarValue::Boolean(v) => Ok(v.into_py(py)),
+ ScalarValue::Float16(_) => Err(py_datafusion_err(
+ datafusion_common::DataFusionError::NotImplemented(
+ "ScalarValue::Float16".to_string(),
+ ),
+ )),
ScalarValue::Float32(v) => Ok(v.into_py(py)),
ScalarValue::Float64(v) => Ok(v.into_py(py)),
ScalarValue::Decimal128(v, _, _) => Ok(v.into_py(py)),
@@ -355,8 +353,10 @@ impl PyExpr {
ScalarValue::TimestampMicrosecond(v, _) => Ok(v.into_py(py)),
ScalarValue::TimestampNanosecond(v, _) => Ok(v.into_py(py)),
ScalarValue::IntervalYearMonth(v) => Ok(v.into_py(py)),
- ScalarValue::IntervalDayTime(v) => Ok(v.into_py(py)),
- ScalarValue::IntervalMonthDayNano(v) => Ok(v.into_py(py)),
+ ScalarValue::IntervalDayTime(v) =>
Ok(ScalarValue::IntervalDayTime(*v).into_py(py)),
+ ScalarValue::IntervalMonthDayNano(v) => {
+ Ok(ScalarValue::IntervalMonthDayNano(*v).into_py(py))
+ }
ScalarValue::DurationSecond(v) => Ok(v.into_py(py)),
ScalarValue::DurationMicrosecond(v) => Ok(v.into_py(py)),
ScalarValue::DurationNanosecond(v) => Ok(v.into_py(py)),
@@ -417,7 +417,6 @@ impl PyExpr {
| Expr::IsNotFalse(expr)
| Expr::IsNotUnknown(expr)
| Expr::Negative(expr)
- | Expr::GetIndexedField(GetIndexedField { expr, .. })
| Expr::Cast(Cast { expr, .. })
| Expr::TryCast(TryCast { expr, .. })
| Expr::Sort(Sort { expr, .. })
@@ -513,9 +512,7 @@ impl PyExpr {
op,
right: _,
}) => format!("{op}"),
- Expr::ScalarFunction(ScalarFunction { func_def, args: _ }) => {
- func_def.name().to_string()
- }
+ Expr::ScalarFunction(ScalarFunction { func, args: _ }) =>
func.name().to_string(),
Expr::Cast { .. } => "cast".to_string(),
Expr::Between { .. } => "between".to_string(),
Expr::Case { .. } => "case".to_string(),
@@ -674,7 +671,6 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> {
m.add_class::<cast::PyCast>()?;
m.add_class::<cast::PyTryCast>()?;
m.add_class::<between::PyBetween>()?;
- m.add_class::<indexed_field::PyGetIndexedField>()?;
m.add_class::<explain::PyExplain>()?;
m.add_class::<limit::PyLimit>()?;
m.add_class::<aggregate::PyAggregate>()?;
diff --git a/src/expr/literal.rs b/src/expr/literal.rs
index bce987f..0333432 100644
--- a/src/expr/literal.rs
+++ b/src/expr/literal.rs
@@ -137,12 +137,7 @@ impl PyLiteral {
pub fn value_interval_day_time(&self) -> PyResult<Option<(i32, i32)>> {
match &self.value {
- ScalarValue::IntervalDayTime(Some(iv)) => {
- let interval = *iv as u64;
- let days = (interval >> 32) as i32;
- let ms = interval as i32;
- Ok(Some((days, ms)))
- }
+ ScalarValue::IntervalDayTime(Some(iv)) => Ok(Some((iv.days,
iv.milliseconds))),
ScalarValue::IntervalDayTime(None) => Ok(None),
other => Err(unexpected_literal_value(other)),
}
diff --git a/src/expr/signature.rs b/src/expr/signature.rs
index 2f19498..7882ceb 100644
--- a/src/expr/signature.rs
+++ b/src/expr/signature.rs
@@ -20,7 +20,6 @@ use pyo3::prelude::*;
#[allow(dead_code)]
#[pyclass(name = "Signature", module = "datafusion.expr", subclass)]
-#[allow(dead_code)]
#[derive(Clone)]
pub struct PySignature {
type_signature: TypeSignature,
diff --git a/src/functions.rs b/src/functions.rs
index a4bd986..09cdee6 100644
--- a/src/functions.rs
+++ b/src/functions.rs
@@ -36,32 +36,84 @@ use datafusion_expr::{
};
#[pyfunction]
-#[pyo3(signature = (y, x, distinct = false, filter = None, order_by = None))]
-pub fn covar_samp(
- y: PyExpr,
- x: PyExpr,
+pub fn sum(args: PyExpr) -> PyExpr {
+ functions_aggregate::expr_fn::sum(args.expr).into()
+}
+
+#[pyfunction]
+pub fn covar_samp(y: PyExpr, x: PyExpr) -> PyExpr {
+ functions_aggregate::expr_fn::covar_samp(y.expr, x.expr).into()
+}
+
+#[pyfunction]
+pub fn covar_pop(y: PyExpr, x: PyExpr) -> PyExpr {
+ functions_aggregate::expr_fn::covar_pop(y.expr, x.expr).into()
+}
+
+#[pyfunction]
+pub fn median(arg: PyExpr) -> PyExpr {
+ functions_aggregate::expr_fn::median(arg.expr).into()
+}
+
+#[pyfunction]
+pub fn covar(y: PyExpr, x: PyExpr) -> PyExpr {
+ // alias for covar_samp
+ covar_samp(y, x)
+}
+
+#[pyfunction]
+pub fn var_samp(expression: PyExpr) -> PyExpr {
+ functions_aggregate::expr_fn::var_sample(expression.expr).into()
+}
+
+#[pyfunction]
+/// Alias for [`var_samp`]
+pub fn var(y: PyExpr) -> PyExpr {
+ var_samp(y)
+}
+
+#[pyfunction]
+#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None))]
+pub fn first_value(
+ args: Vec<PyExpr>,
distinct: bool,
filter: Option<PyExpr>,
order_by: Option<Vec<PyExpr>>,
- // null_treatment: Option<sqlparser::ast::NullTreatment>,
) -> PyExpr {
- let filter = filter.map(|x| Box::new(x.expr));
+ // TODO: allow user to select null_treatment
+ let null_treatment = None;
+ let args = args.into_iter().map(|x| x.expr).collect::<Vec<_>>();
let order_by = order_by.map(|x| x.into_iter().map(|x|
x.expr).collect::<Vec<_>>());
- functions_aggregate::expr_fn::covar_samp(y.expr, x.expr, distinct, filter,
order_by, None)
- .into()
+ functions_aggregate::expr_fn::first_value(
+ args,
+ distinct,
+ filter.map(|x| Box::new(x.expr)),
+ order_by,
+ null_treatment,
+ )
+ .into()
}
#[pyfunction]
-#[pyo3(signature = (y, x, distinct = false, filter = None, order_by = None))]
-pub fn covar(
- y: PyExpr,
- x: PyExpr,
+#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None))]
+pub fn last_value(
+ args: Vec<PyExpr>,
distinct: bool,
filter: Option<PyExpr>,
order_by: Option<Vec<PyExpr>>,
) -> PyExpr {
- // alias for covar_samp
- covar_samp(y, x, distinct, filter, order_by)
+ // TODO: allow user to select null_treatment
+ let null_treatment = None;
+ let args = args.into_iter().map(|x| x.expr).collect::<Vec<_>>();
+ let order_by = order_by.map(|x| x.into_iter().map(|x|
x.expr).collect::<Vec<_>>());
+ functions_aggregate::expr_fn::last_value(
+ args,
+ distinct,
+ filter.map(|x| Box::new(x.expr)),
+ order_by,
+ null_treatment,
+ )
+ .into()
}
#[pyfunction]
@@ -131,17 +183,20 @@ fn list_indexof(array: PyExpr, element: PyExpr, index:
Option<i64>) -> PyExpr {
}
#[pyfunction]
-#[pyo3(signature = (array, begin, end, stride = 1))]
-fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option<i64>)
-> PyExpr {
- let stride = ScalarValue::Int64(stride);
- let stride = Expr::Literal(stride);
- datafusion_functions_array::expr_fn::array_slice(array.into(),
begin.into(), end.into(), stride)
- .into()
+#[pyo3(signature = (array, begin, end, stride = None))]
+fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride:
Option<PyExpr>) -> PyExpr {
+ datafusion_functions_array::expr_fn::array_slice(
+ array.into(),
+ begin.into(),
+ end.into(),
+ stride.map(Into::into),
+ )
+ .into()
}
#[pyfunction]
-#[pyo3(signature = (array, begin, end, stride = 1))]
-fn list_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option<i64>)
-> PyExpr {
+#[pyo3(signature = (array, begin, end, stride = None))]
+fn list_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride:
Option<PyExpr>) -> PyExpr {
// alias of array_slice
array_slice(array, begin, end, stride)
}
@@ -176,6 +231,28 @@ fn concat_ws(sep: String, args: Vec<PyExpr>) ->
PyResult<PyExpr> {
Ok(functions::string::expr_fn::concat_ws(lit(sep), args).into())
}
+#[pyfunction]
+#[pyo3(signature = (values, regex, flags = None))]
+fn regexp_match(values: PyExpr, regex: PyExpr, flags: Option<PyExpr>) ->
PyResult<PyExpr> {
+ Ok(functions::expr_fn::regexp_match(values.expr, regex.expr, flags.map(|x|
x.expr)).into())
+}
+
+#[pyfunction]
+/// Replaces substring(s) matching a POSIX regular expression.
+fn regexp_replace(
+ string: PyExpr,
+ pattern: PyExpr,
+ replacement: PyExpr,
+ flags: Option<PyExpr>,
+) -> PyResult<PyExpr> {
+ Ok(functions::expr_fn::regexp_replace(
+ string.into(),
+ pattern.into(),
+ replacement.into(),
+ flags.map(|x| x.expr),
+ )
+ .into())
+}
/// Creates a new Sort Expr
#[pyfunction]
fn order_by(expr: PyExpr, asc: Option<bool>, nulls_first: Option<bool>) ->
PyResult<PyExpr> {
@@ -435,17 +512,11 @@ expr_fn!(
"Returns x if x is not NaN otherwise returns y."
);
expr_fn!(nullif, arg_1 arg_2);
-expr_fn_vec!(octet_length, "Returns number of bytes in the string. Since this
version of the function accepts type character directly, it will not strip
trailing spaces.");
+expr_fn!(octet_length, args, "Returns number of bytes in the string. Since
this version of the function accepts type character directly, it will not strip
trailing spaces.");
expr_fn!(pi);
expr_fn!(power, base exponent);
expr_fn!(pow, power, base exponent);
expr_fn!(radians, num);
-expr_fn!(regexp_match, input_arg1 input_arg2);
-expr_fn!(
- regexp_replace,
- arg1 arg2 arg3 arg4,
- "Replaces substring(s) matching a POSIX regular expression."
-);
expr_fn!(repeat, string n, "Repeats string the specified number of times.");
expr_fn!(
replace,
@@ -576,19 +647,14 @@ aggregate_function!(array_agg, ArrayAgg);
aggregate_function!(avg, Avg);
aggregate_function!(corr, Correlation);
aggregate_function!(count, Count);
-aggregate_function!(covar_pop, CovariancePop);
aggregate_function!(grouping, Grouping);
aggregate_function!(max, Max);
aggregate_function!(mean, Avg);
-aggregate_function!(median, Median);
aggregate_function!(min, Min);
-aggregate_function!(sum, Sum);
aggregate_function!(stddev, Stddev);
aggregate_function!(stddev_pop, StddevPop);
aggregate_function!(stddev_samp, Stddev);
-aggregate_function!(var, Variance);
aggregate_function!(var_pop, VariancePop);
-aggregate_function!(var_samp, Variance);
aggregate_function!(regr_avgx, RegrAvgx);
aggregate_function!(regr_avgy, RegrAvgy);
aggregate_function!(regr_count, RegrCount);
@@ -598,8 +664,6 @@ aggregate_function!(regr_slope, RegrSlope);
aggregate_function!(regr_sxx, RegrSXX);
aggregate_function!(regr_sxy, RegrSXY);
aggregate_function!(regr_syy, RegrSYY);
-aggregate_function!(first_value, FirstValue);
-aggregate_function!(last_value, LastValue);
aggregate_function!(bit_and, BitAnd);
aggregate_function!(bit_or, BitOr);
aggregate_function!(bit_xor, BitXor);
diff --git a/src/physical_plan.rs b/src/physical_plan.rs
index 51032f2..c97c1a9 100644
--- a/src/physical_plan.rs
+++ b/src/physical_plan.rs
@@ -40,7 +40,7 @@ impl PyExecutionPlan {
self.plan
.children()
.iter()
- .map(|p| p.to_owned().into())
+ .map(|&p| p.to_owned().into())
.collect()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
