This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 765dbd7df Update to arrow 25.0.0 (#3844)
765dbd7df is described below
commit 765dbd7df6893e70f1c2635a2883dcf23b1d4d22
Author: Raphael Taylor-Davies <[email protected]>
AuthorDate: Tue Oct 18 22:13:06 2022 +1300
Update to arrow 25.0.0 (#3844)
---
benchmarks/Cargo.toml | 4 +-
datafusion-cli/Cargo.lock | 25 +++---
datafusion-cli/Cargo.toml | 2 +-
datafusion-examples/Cargo.toml | 4 +-
datafusion/common/Cargo.toml | 4 +-
datafusion/common/src/scalar.rs | 15 ++--
datafusion/core/Cargo.toml | 6 +-
datafusion/core/fuzz-utils/Cargo.toml | 2 +-
datafusion/core/src/physical_plan/hash_utils.rs | 14 +---
datafusion/core/src/test/mod.rs | 11 ++-
datafusion/core/tests/sql/mod.rs | 35 ++++++--
datafusion/expr/Cargo.toml | 2 +-
datafusion/jit/Cargo.toml | 2 +-
datafusion/optimizer/Cargo.toml | 2 +-
datafusion/physical-expr/Cargo.toml | 2 +-
datafusion/physical-expr/src/aggregate/min_max.rs | 6 +-
datafusion/physical-expr/src/aggregate/sum.rs | 2 +-
datafusion/physical-expr/src/expressions/binary.rs | 19 ++---
.../src/expressions/binary/kernels_arrow.rs | 28 +++----
datafusion/physical-expr/src/expressions/cast.rs | 89 ++++----------------
.../physical-expr/src/expressions/in_list.rs | 8 +-
.../physical-expr/src/expressions/try_cast.rs | 97 +++++-----------------
datafusion/proto/Cargo.toml | 2 +-
datafusion/row/Cargo.toml | 2 +-
datafusion/sql/Cargo.toml | 2 +-
25 files changed, 140 insertions(+), 245 deletions(-)
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
index c07344508..1b891ab83 100644
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -32,14 +32,14 @@ simd = ["datafusion/simd"]
snmalloc = ["snmalloc-rs"]
[dependencies]
-arrow = "24.0.0"
+arrow = "25.0.0"
datafusion = { path = "../datafusion/core" }
env_logger = "0.9"
futures = "0.3"
mimalloc = { version = "0.1", optional = true, default-features = false }
num_cpus = "1.13.0"
object_store = "0.5.0"
-parquet = "24.0.0"
+parquet = "25.0.0"
rand = "0.8.4"
serde = { version = "1.0.136", features = ["derive"] }
serde_json = "1.0.78"
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index 29ddffd6c..364c412fb 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -79,9 +79,9 @@ checksum =
"8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "arrow"
-version = "24.0.0"
+version = "25.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d68391300d5237f6725f0f869ae7cb65d45fcf8a6d18f6ceecd328fb803bef93"
+checksum = "76312eb67808c67341f4234861c4fcd2f9868f55e88fa2186ab3b357a6c5830b"
dependencies = [
"ahash 0.8.0",
"arrow-array",
@@ -107,9 +107,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "24.0.0"
+version = "25.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f0bb00c5862b5eea683812083c495bef01a9a5149da46ad2f4c0e4aa8800f64d"
+checksum = "69dd2c257fa76de0bcc63cabe8c81d34c46ef6fa7651e3e497922c3c9878bd67"
dependencies = [
"ahash 0.8.0",
"arrow-buffer",
@@ -123,18 +123,19 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "24.0.0"
+version = "25.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3e594d0fe0026a8bc2459bdc5ac9623e5fb666724a715e0acbc96ba30c5d4cc7"
+checksum = "af963e71bdbbf928231d521083ddc8e8068cf5c8d45d4edcfeaf7eb5cdd779a9"
dependencies = [
"half",
+ "num",
]
[[package]]
name = "arrow-data"
-version = "24.0.0"
+version = "25.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8500df05060d86fdc53e9b5cb32e51bfeaacc040fdeced3eb99ac0d59200ff45"
+checksum = "52554ffff560c366d7210c2621a3cf1dc408f9969a0c7688a3ba0a62248a945d"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -144,9 +145,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "24.0.0"
+version = "25.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "86d1fef01f25e1452c86fa6887f078de8e0aaeeb828370feab205944cfc30e27"
+checksum = "1a5518f2bd7775057391f88257627cbb760ba3e1c2f2444a005ba79158624654"
[[package]]
name = "async-compression"
@@ -1619,9 +1620,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "24.0.0"
+version = "25.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "74fd590f0672998df84503d1bcbebc69732583d03cc3495c7dd8d3e5a1d8437f"
+checksum = "f7758803135c32b243e52832473fc8f7c768a0a170b0851fb1bb37904c6b3550"
dependencies = [
"ahash 0.8.0",
"arrow",
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index 61328a8bf..7367f2dc0 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.62"
readme = "README.md"
[dependencies]
-arrow = "24.0.0"
+arrow = "25.0.0"
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = { path = "../datafusion/core", version = "13.0.0" }
dirs = "4.0.0"
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index 2530f890f..9abf76b25 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -34,8 +34,8 @@ path = "examples/avro_sql.rs"
required-features = ["datafusion/avro"]
[dev-dependencies]
-arrow = "24.0.0"
-arrow-flight = "24.0.0"
+arrow = "25.0.0"
+arrow-flight = "25.0.0"
async-trait = "0.1.41"
datafusion = { path = "../datafusion/core" }
datafusion-common = { path = "../datafusion/common" }
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index 382f66e5d..c8ed7a034 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -40,10 +40,10 @@ pyarrow = ["pyo3", "arrow/pyarrow"]
[dependencies]
apache-avro = { version = "0.14", default-features = false, features =
["snappy"], optional = true }
-arrow = { version = "24.0.0", default-features = false }
+arrow = { version = "25.0.0", default-features = false }
cranelift-module = { version = "0.88.0", optional = true }
object_store = { version = "0.5.0", default-features = false, optional = true }
ordered-float = "3.0"
-parquet = { version = "24.0.0", default-features = false, optional = true }
+parquet = { version = "25.0.0", default-features = false, optional = true }
pyo3 = { version = "0.17.1", optional = true }
sqlparser = "0.25"
diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs
index c3f91dd9b..680bd3f66 100644
--- a/datafusion/common/src/scalar.rs
+++ b/datafusion/common/src/scalar.rs
@@ -34,7 +34,6 @@ use arrow::{
TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
DECIMAL128_MAX_PRECISION,
},
- util::decimal::Decimal128,
};
use ordered_float::OrderedFloat;
@@ -1696,7 +1695,7 @@ impl ScalarValue {
if array.is_null(index) {
ScalarValue::Decimal128(None, precision, scale)
} else {
- ScalarValue::Decimal128(Some(array.value(index).as_i128()),
precision, scale)
+ ScalarValue::Decimal128(Some(array.value(index)), precision, scale)
}
}
@@ -1881,11 +1880,7 @@ impl ScalarValue {
}
match value {
None => array.is_null(index),
- Some(v) => {
- !array.is_null(index)
- && array.value(index)
- == Decimal128::new(precision, scale, &v.to_le_bytes())
- }
+ Some(v) => !array.is_null(index) && array.value(index) == *v,
}
}
@@ -2501,15 +2496,15 @@ mod tests {
let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
assert_eq!(1, array.len());
assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
- assert_eq!(123i128, array.value(0).as_i128());
+ assert_eq!(123i128, array.value(0));
// decimal scalar to array with size
let array = decimal_value.to_array_of_size(10);
let array_decimal =
array.as_any().downcast_ref::<Decimal128Array>().unwrap();
assert_eq!(10, array.len());
assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
- assert_eq!(123i128, array_decimal.value(0).as_i128());
- assert_eq!(123i128, array_decimal.value(9).as_i128());
+ assert_eq!(123i128, array_decimal.value(0));
+ assert_eq!(123i128, array_decimal.value(9));
// test eq array
assert!(decimal_value.eq_array(&array, 1));
assert!(decimal_value.eq_array(&array, 5));
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index 44a5cbe1a..6512db0ae 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -56,7 +56,7 @@ unicode_expressions =
["datafusion-physical-expr/regex_expressions", "datafusion
[dependencies]
ahash = { version = "0.8", default-features = false, features =
["runtime-rng"] }
apache-avro = { version = "0.14", optional = true }
-arrow = { version = "24.0.0", features = ["prettyprint"] }
+arrow = { version = "25.0.0", features = ["prettyprint"] }
async-compression = { version = "0.3.14", features = ["bzip2", "gzip",
"futures-io", "tokio"] }
async-trait = "0.1.41"
bytes = "1.1"
@@ -81,7 +81,7 @@ num_cpus = "1.13.0"
object_store = "0.5.0"
ordered-float = "3.0"
parking_lot = "0.12"
-parquet = { version = "24.0.0", features = ["arrow", "async"] }
+parquet = { version = "25.0.0", features = ["arrow", "async"] }
paste = "^1.0"
percent-encoding = "2.2.0"
pin-project-lite = "^0.2.7"
@@ -98,7 +98,7 @@ url = "2.2"
uuid = { version = "1.0", features = ["v4"] }
[dev-dependencies]
-arrow = { version = "24.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
+arrow = { version = "25.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
async-trait = "0.1.53"
criterion = "0.4"
csv = "1.1.6"
diff --git a/datafusion/core/fuzz-utils/Cargo.toml
b/datafusion/core/fuzz-utils/Cargo.toml
index 3f1f2ed76..ba876d4ff 100644
--- a/datafusion/core/fuzz-utils/Cargo.toml
+++ b/datafusion/core/fuzz-utils/Cargo.toml
@@ -23,6 +23,6 @@ edition = "2021"
# See more keys and their definitions at
https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-arrow = { version = "24.0.0", features = ["prettyprint"] }
+arrow = { version = "25.0.0", features = ["prettyprint"] }
env_logger = "0.9.0"
rand = "0.8"
diff --git a/datafusion/core/src/physical_plan/hash_utils.rs
b/datafusion/core/src/physical_plan/hash_utils.rs
index 61e72b07c..899008aa4 100644
--- a/datafusion/core/src/physical_plan/hash_utils.rs
+++ b/datafusion/core/src/physical_plan/hash_utils.rs
@@ -62,29 +62,23 @@ fn hash_decimal128<'a>(
if array.null_count() == 0 {
if mul_col {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
- *hash = combine_hashes(
- random_state.hash_one(&array.value(i).as_i128()),
- *hash,
- );
+ *hash = combine_hashes(random_state.hash_one(&array.value(i)),
*hash);
}
} else {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
- *hash = random_state.hash_one(&array.value(i).as_i128());
+ *hash = random_state.hash_one(&array.value(i));
}
}
} else if mul_col {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
if !array.is_null(i) {
- *hash = combine_hashes(
- random_state.hash_one(&array.value(i).as_i128()),
- *hash,
- );
+ *hash = combine_hashes(random_state.hash_one(&array.value(i)),
*hash);
}
}
} else {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
if !array.is_null(i) {
- *hash = random_state.hash_one(&array.value(i).as_i128());
+ *hash = random_state.hash_one(&array.value(i));
}
}
}
diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rs
index bb884e142..d6e2c05fc 100644
--- a/datafusion/core/src/test/mod.rs
+++ b/datafusion/core/src/test/mod.rs
@@ -242,14 +242,17 @@ pub fn table_with_decimal() -> Arc<dyn TableProvider> {
}
fn make_decimal() -> RecordBatch {
- let mut decimal_builder = Decimal128Builder::with_capacity(20, 10, 3);
+ let mut decimal_builder = Decimal128Builder::with_capacity(20);
for i in 110000..110010 {
- decimal_builder.append_value(i as i128).unwrap();
+ decimal_builder.append_value(i as i128);
}
for i in 100000..100010 {
- decimal_builder.append_value(-i as i128).unwrap();
+ decimal_builder.append_value(-i as i128);
}
- let array = decimal_builder.finish();
+ let array = decimal_builder
+ .finish()
+ .with_precision_and_scale(10, 3)
+ .unwrap();
let schema = Schema::new(vec![Field::new("c1", array.data_type().clone(),
true)]);
RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap()
}
diff --git a/datafusion/core/tests/sql/mod.rs b/datafusion/core/tests/sql/mod.rs
index e0167b5c4..9d1c440a1 100644
--- a/datafusion/core/tests/sql/mod.rs
+++ b/datafusion/core/tests/sql/mod.rs
@@ -560,9 +560,9 @@ async fn register_tpch_csv_data(
DataType::Int64 => {
cols.push(Box::new(Int64Builder::with_capacity(records.len())))
}
- DataType::Decimal128(p, s) => cols.push(Box::new(
- Decimal128Builder::with_capacity(records.len(), *p, *s),
- )),
+ DataType::Decimal128(_, _) => {
+
cols.push(Box::new(Decimal128Builder::with_capacity(records.len())))
+ }
_ => {
let msg = format!("Not implemented: {}", field.data_type());
Err(DataFusionError::Plan(msg))?
@@ -600,7 +600,7 @@ async fn register_tpch_csv_data(
.unwrap();
let val = val.trim().replace('.', "");
let value_i128 = val.parse::<i128>().unwrap();
- sb.append_value(value_i128)?;
+ sb.append_value(value_i128);
}
_ => Err(DataFusionError::Plan(format!(
"Not implemented: {}",
@@ -609,7 +609,21 @@ async fn register_tpch_csv_data(
}
}
}
- let cols: Vec<ArrayRef> = cols.iter_mut().map(|it| it.finish()).collect();
+ let cols: Vec<ArrayRef> = cols
+ .iter_mut()
+ .zip(schema.fields())
+ .map(|(it, field)| match field.data_type() {
+ DataType::Decimal128(p, s) => Arc::new(
+ it.as_any_mut()
+ .downcast_mut::<Decimal128Builder>()
+ .unwrap()
+ .finish()
+ .with_precision_and_scale(*p, *s)
+ .unwrap(),
+ ),
+ _ => it.finish(),
+ })
+ .collect();
let batch = RecordBatch::try_new(Arc::clone(&schema), cols)?;
@@ -879,14 +893,17 @@ pub fn table_with_decimal() -> Arc<dyn TableProvider> {
}
fn make_decimal() -> RecordBatch {
- let mut decimal_builder = Decimal128Builder::with_capacity(20, 10, 3);
+ let mut decimal_builder = Decimal128Builder::with_capacity(20);
for i in 110000..110010 {
- decimal_builder.append_value(i as i128).unwrap();
+ decimal_builder.append_value(i as i128);
}
for i in 100000..100010 {
- decimal_builder.append_value(-i as i128).unwrap();
+ decimal_builder.append_value(-i as i128);
}
- let array = decimal_builder.finish();
+ let array = decimal_builder
+ .finish()
+ .with_precision_and_scale(10, 3)
+ .unwrap();
let schema = Schema::new(vec![Field::new("c1", array.data_type().clone(),
true)]);
RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap()
}
diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
index 5d7350f72..d3b348c4b 100644
--- a/datafusion/expr/Cargo.toml
+++ b/datafusion/expr/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
[dependencies]
ahash = { version = "0.8", default-features = false, features =
["runtime-rng"] }
-arrow = { version = "24.0.0", default-features = false }
+arrow = { version = "25.0.0", default-features = false }
datafusion-common = { path = "../common", version = "13.0.0" }
log = "^0.4"
sqlparser = "0.25"
diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml
index 571ddb4bb..1b4dc1059 100644
--- a/datafusion/jit/Cargo.toml
+++ b/datafusion/jit/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []
[dependencies]
-arrow = { version = "24.0.0", default-features = false }
+arrow = { version = "25.0.0", default-features = false }
cranelift = "0.88.0"
cranelift-jit = "0.88.0"
cranelift-module = "0.88.0"
diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
index ee4d4016d..e07d798ca 100644
--- a/datafusion/optimizer/Cargo.toml
+++ b/datafusion/optimizer/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []
[dependencies]
-arrow = { version = "24.0.0", features = ["prettyprint"] }
+arrow = { version = "25.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
chrono = { version = "0.4", default-features = false }
datafusion-common = { path = "../common", version = "13.0.0" }
diff --git a/datafusion/physical-expr/Cargo.toml
b/datafusion/physical-expr/Cargo.toml
index 4707b0e79..ba4dc0b03 100644
--- a/datafusion/physical-expr/Cargo.toml
+++ b/datafusion/physical-expr/Cargo.toml
@@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]
[dependencies]
ahash = { version = "0.8", default-features = false, features =
["runtime-rng"] }
-arrow = { version = "24.0.0", features = ["prettyprint"] }
+arrow = { version = "25.0.0", features = ["prettyprint"] }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
chrono = { version = "0.4", default-features = false }
diff --git a/datafusion/physical-expr/src/aggregate/min_max.rs
b/datafusion/physical-expr/src/aggregate/min_max.rs
index 36d58c780..c415e7db5 100644
--- a/datafusion/physical-expr/src/aggregate/min_max.rs
+++ b/datafusion/physical-expr/src/aggregate/min_max.rs
@@ -177,17 +177,17 @@ macro_rules! typed_min_max_batch_decimal128 {
for i in 1..array.len() {
result = result.$OP(array.value(i));
}
- ScalarValue::Decimal128(Some(result.as_i128()), *$PRECISION,
*$SCALE)
+ ScalarValue::Decimal128(Some(result), *$PRECISION, *$SCALE)
} else {
let mut result = 0_i128;
let mut has_value = false;
for i in 0..array.len() {
if !has_value && array.is_valid(i) {
has_value = true;
- result = array.value(i).as_i128();
+ result = array.value(i);
}
if array.is_valid(i) {
- result = result.$OP(array.value(i).as_i128());
+ result = result.$OP(array.value(i));
}
}
ScalarValue::Decimal128(Some(result), *$PRECISION, *$SCALE)
diff --git a/datafusion/physical-expr/src/aggregate/sum.rs
b/datafusion/physical-expr/src/aggregate/sum.rs
index ca9b4c819..112856988 100644
--- a/datafusion/physical-expr/src/aggregate/sum.rs
+++ b/datafusion/physical-expr/src/aggregate/sum.rs
@@ -169,7 +169,7 @@ fn sum_decimal_batch(values: &ArrayRef, precision: u8,
scale: u8) -> Result<Scal
}
let result = array.into_iter().fold(0_i128, |s, element| match element {
- Some(v) => s + v.as_i128(),
+ Some(v) => s + v,
None => s,
});
diff --git a/datafusion/physical-expr/src/expressions/binary.rs
b/datafusion/physical-expr/src/expressions/binary.rs
index 665ef2d68..6db5b5a78 100644
--- a/datafusion/physical-expr/src/expressions/binary.rs
+++ b/datafusion/physical-expr/src/expressions/binary.rs
@@ -2152,19 +2152,14 @@ mod tests {
precision: u8,
scale: u8,
) -> Decimal128Array {
- let mut decimal_builder =
- Decimal128Builder::with_capacity(array.len(), precision, scale);
- for value in array {
- match value {
- None => {
- decimal_builder.append_null();
- }
- Some(v) => {
- decimal_builder.append_value(*v).expect("valid value");
- }
- }
+ let mut decimal_builder =
Decimal128Builder::with_capacity(array.len());
+ for value in array.iter().copied() {
+ decimal_builder.append_option(value)
}
- decimal_builder.finish()
+ decimal_builder
+ .finish()
+ .with_precision_and_scale(precision, scale)
+ .unwrap()
}
#[test]
diff --git a/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs
b/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs
index 99b8a5a06..d73302d7e 100644
--- a/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs
+++ b/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs
@@ -133,7 +133,7 @@ where
{
Ok(left
.iter()
- .map(|left| left.map(|left| op(left.as_i128(), right)))
+ .map(|left| left.map(|left| op(left, right)))
.collect())
}
@@ -152,7 +152,7 @@ where
.zip(right.iter())
.map(|(left, right)| {
if let (Some(left), Some(right)) = (left, right) {
- Some(op(left.as_i128(), right.as_i128()))
+ Some(op(left, right))
} else {
None
}
@@ -288,7 +288,7 @@ where
.zip(right.iter())
.map(|(left, right)| {
if let (Some(left), Some(right)) = (left, right) {
- Some(op(left.as_i128(), right.as_i128())).transpose()
+ Some(op(left, right)).transpose()
} else {
Ok(None)
}
@@ -307,7 +307,7 @@ where
left.iter()
.map(|left| {
if let Some(left) = left {
- Some(op(left.as_i128(), right)).transpose()
+ Some(op(left, right)).transpose()
} else {
Ok(None)
}
@@ -444,19 +444,15 @@ mod tests {
precision: u8,
scale: u8,
) -> Decimal128Array {
- let mut decimal_builder =
- Decimal128Builder::with_capacity(array.len(), precision, scale);
- for value in array {
- match value {
- None => {
- decimal_builder.append_null();
- }
- Some(v) => {
- decimal_builder.append_value(*v).expect("valid value");
- }
- }
+ let mut decimal_builder =
Decimal128Builder::with_capacity(array.len());
+
+ for value in array.iter().copied() {
+ decimal_builder.append_option(value)
}
- decimal_builder.finish()
+ decimal_builder
+ .finish()
+ .with_precision_and_scale(precision, scale)
+ .unwrap()
}
#[test]
diff --git a/datafusion/physical-expr/src/expressions/cast.rs
b/datafusion/physical-expr/src/expressions/cast.rs
index 5affc857c..a1e87a38c 100644
--- a/datafusion/physical-expr/src/expressions/cast.rs
+++ b/datafusion/physical-expr/src/expressions/cast.rs
@@ -166,7 +166,6 @@ mod tests {
TimestampNanosecondArray, UInt32Array,
},
datatypes::*,
- util::decimal::Decimal128,
};
use datafusion_common::Result;
@@ -286,20 +285,17 @@ mod tests {
.collect::<Decimal128Array>()
.with_precision_and_scale(10, 3)?;
- // closure that converts to i128
- let convert = |v: i128| Decimal128::new(20, 6, &v.to_le_bytes());
-
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal128(10, 3),
Decimal128Array,
DataType::Decimal128(20, 6),
vec![
- Some(convert(1_234_000)),
- Some(convert(2_222_000)),
- Some(convert(3_000)),
- Some(convert(4_000_000)),
- Some(convert(5_000_000)),
+ Some(1_234_000),
+ Some(2_222_000),
+ Some(3_000),
+ Some(4_000_000),
+ Some(5_000_000),
None,
],
DEFAULT_DATAFUSION_CAST_OPTIONS
@@ -310,20 +306,12 @@ mod tests {
.collect::<Decimal128Array>()
.with_precision_and_scale(10, 3)?;
- let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes());
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal128(10, 3),
Decimal128Array,
DataType::Decimal128(10, 2),
- vec![
- Some(convert(123)),
- Some(convert(222)),
- Some(convert(0)),
- Some(convert(400)),
- Some(convert(500)),
- None,
- ],
+ vec![Some(123), Some(222), Some(0), Some(400), Some(500), None,],
DEFAULT_DATAFUSION_CAST_OPTIONS
);
@@ -476,115 +464,72 @@ mod tests {
#[test]
fn test_cast_numeric_to_decimal() -> Result<()> {
// int8
- let convert = |v: i128| Decimal128::new(3, 0, &v.to_le_bytes());
generic_test_cast!(
Int8Array,
DataType::Int8,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(3, 0),
- vec![
- Some(convert(1)),
- Some(convert(2)),
- Some(convert(3)),
- Some(convert(4)),
- Some(convert(5)),
- ],
+ vec![Some(1), Some(2), Some(3), Some(4), Some(5),],
DEFAULT_DATAFUSION_CAST_OPTIONS
);
// int16
- let convert = |v: i128| Decimal128::new(5, 0, &v.to_le_bytes());
generic_test_cast!(
Int16Array,
DataType::Int16,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(5, 0),
- vec![
- Some(convert(1)),
- Some(convert(2)),
- Some(convert(3)),
- Some(convert(4)),
- Some(convert(5)),
- ],
+ vec![Some(1), Some(2), Some(3), Some(4), Some(5),],
DEFAULT_DATAFUSION_CAST_OPTIONS
);
// int32
- let convert = |v: i128| Decimal128::new(10, 0, &v.to_le_bytes());
generic_test_cast!(
Int32Array,
DataType::Int32,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(10, 0),
- vec![
- Some(convert(1)),
- Some(convert(2)),
- Some(convert(3)),
- Some(convert(4)),
- Some(convert(5)),
- ],
+ vec![Some(1), Some(2), Some(3), Some(4), Some(5),],
DEFAULT_DATAFUSION_CAST_OPTIONS
);
// int64
- let convert = |v: i128| Decimal128::new(20, 0, &v.to_le_bytes());
generic_test_cast!(
Int64Array,
DataType::Int64,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(20, 0),
- vec![
- Some(convert(1)),
- Some(convert(2)),
- Some(convert(3)),
- Some(convert(4)),
- Some(convert(5)),
- ],
+ vec![Some(1), Some(2), Some(3), Some(4), Some(5),],
DEFAULT_DATAFUSION_CAST_OPTIONS
);
// int64 to different scale
- let convert = |v: i128| Decimal128::new(20, 2, &v.to_le_bytes());
generic_test_cast!(
Int64Array,
DataType::Int64,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(20, 2),
- vec![
- Some(convert(100)),
- Some(convert(200)),
- Some(convert(300)),
- Some(convert(400)),
- Some(convert(500)),
- ],
+ vec![Some(100), Some(200), Some(300), Some(400), Some(500),],
DEFAULT_DATAFUSION_CAST_OPTIONS
);
// float32
- let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes());
generic_test_cast!(
Float32Array,
DataType::Float32,
vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
Decimal128Array,
DataType::Decimal128(10, 2),
- vec![
- Some(convert(150)),
- Some(convert(250)),
- Some(convert(300)),
- Some(convert(112)),
- Some(convert(550)),
- ],
+ vec![Some(150), Some(250), Some(300), Some(112), Some(550),],
DEFAULT_DATAFUSION_CAST_OPTIONS
);
// float64
- let convert = |v: i128| Decimal128::new(20, 4, &v.to_le_bytes());
generic_test_cast!(
Float64Array,
DataType::Float64,
@@ -592,11 +537,11 @@ mod tests {
Decimal128Array,
DataType::Decimal128(20, 4),
vec![
- Some(convert(15000)),
- Some(convert(25000)),
- Some(convert(30000)),
- Some(convert(11234)),
- Some(convert(55000)),
+ Some(15000),
+ Some(25000),
+ Some(30000),
+ Some(11234),
+ Some(55000),
],
DEFAULT_DATAFUSION_CAST_OPTIONS
);
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs
b/datafusion/physical-expr/src/expressions/in_list.rs
index 1e9fa7e9b..2d3411522 100644
--- a/datafusion/physical-expr/src/expressions/in_list.rs
+++ b/datafusion/physical-expr/src/expressions/in_list.rs
@@ -238,7 +238,7 @@ macro_rules! collection_contains_check_decimal {
if $CONTAINS_NULL {
$ARRAY
.iter()
- .map(|vop| match vop.map(|v|
!$VALUES.contains(&v.as_i128())) {
+ .map(|vop| match vop.map(|v| !$VALUES.contains(&v)) {
Some(true) => None,
x => x,
})
@@ -246,7 +246,7 @@ macro_rules! collection_contains_check_decimal {
} else {
$ARRAY
.iter()
- .map(|vop| vop.map(|v| !$VALUES.contains(&v.as_i128())))
+ .map(|vop| vop.map(|v| !$VALUES.contains(&v)))
.collect::<BooleanArray>()
}
} else {
@@ -254,7 +254,7 @@ macro_rules! collection_contains_check_decimal {
if $CONTAINS_NULL {
$ARRAY
.iter()
- .map(|vop| match vop.map(|v|
$VALUES.contains(&v.as_i128())) {
+ .map(|vop| match vop.map(|v| $VALUES.contains(&v)) {
Some(false) => None,
x => x,
})
@@ -262,7 +262,7 @@ macro_rules! collection_contains_check_decimal {
} else {
$ARRAY
.iter()
- .map(|vop| vop.map(|v| $VALUES.contains(&v.as_i128())))
+ .map(|vop| vop.map(|v| $VALUES.contains(&v)))
.collect::<BooleanArray>()
}
};
diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs
b/datafusion/physical-expr/src/expressions/try_cast.rs
index 154ec85e5..92f345de7 100644
--- a/datafusion/physical-expr/src/expressions/try_cast.rs
+++ b/datafusion/physical-expr/src/expressions/try_cast.rs
@@ -121,7 +121,6 @@ mod tests {
use arrow::array::{
Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray,
};
- use arrow::util::decimal::Decimal128;
use arrow::{
array::{
Array, Float32Array, Float64Array, Int16Array, Int32Array,
Int64Array,
@@ -233,37 +232,28 @@ mod tests {
// try cast one decimal data type to another decimal data type
let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
let decimal_array = create_decimal_array(&array, 10, 3);
- let convert = |v: i128| Decimal128::new(20, 6, &v.to_le_bytes());
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal128(10, 3),
Decimal128Array,
DataType::Decimal128(20, 6),
vec![
- Some(convert(1_234_000)),
- Some(convert(2_222_000)),
- Some(convert(3_000)),
- Some(convert(4_000_000)),
- Some(convert(5_000_000)),
+ Some(1_234_000),
+ Some(2_222_000),
+ Some(3_000),
+ Some(4_000_000),
+ Some(5_000_000),
None,
]
);
let decimal_array = create_decimal_array(&array, 10, 3);
- let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes());
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal128(10, 3),
Decimal128Array,
DataType::Decimal128(10, 2),
- vec![
- Some(convert(123)),
- Some(convert(222)),
- Some(convert(0)),
- Some(convert(400)),
- Some(convert(500)),
- None,
- ]
+ vec![Some(123), Some(222), Some(0), Some(400), Some(500), None,]
);
Ok(())
@@ -382,109 +372,66 @@ mod tests {
#[test]
fn test_try_cast_numeric_to_decimal() -> Result<()> {
// int8
- let convert = |v: i128| Decimal128::new(3, 0, &v.to_le_bytes());
generic_test_cast!(
Int8Array,
DataType::Int8,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(3, 0),
- vec![
- Some(convert(1)),
- Some(convert(2)),
- Some(convert(3)),
- Some(convert(4)),
- Some(convert(5)),
- ]
+ vec![Some(1), Some(2), Some(3), Some(4), Some(5),]
);
// int16
- let convert = |v: i128| Decimal128::new(5, 0, &v.to_le_bytes());
generic_test_cast!(
Int16Array,
DataType::Int16,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(5, 0),
- vec![
- Some(convert(1)),
- Some(convert(2)),
- Some(convert(3)),
- Some(convert(4)),
- Some(convert(5)),
- ]
+ vec![Some(1), Some(2), Some(3), Some(4), Some(5),]
);
// int32
- let convert = |v: i128| Decimal128::new(10, 0, &v.to_le_bytes());
generic_test_cast!(
Int32Array,
DataType::Int32,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(10, 0),
- vec![
- Some(convert(1)),
- Some(convert(2)),
- Some(convert(3)),
- Some(convert(4)),
- Some(convert(5)),
- ]
+ vec![Some(1), Some(2), Some(3), Some(4), Some(5),]
);
// int64
- let convert = |v: i128| Decimal128::new(20, 0, &v.to_le_bytes());
generic_test_cast!(
Int64Array,
DataType::Int64,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(20, 0),
- vec![
- Some(convert(1)),
- Some(convert(2)),
- Some(convert(3)),
- Some(convert(4)),
- Some(convert(5)),
- ]
+ vec![Some(1), Some(2), Some(3), Some(4), Some(5),]
);
// int64 to different scale
- let convert = |v: i128| Decimal128::new(20, 2, &v.to_le_bytes());
generic_test_cast!(
Int64Array,
DataType::Int64,
vec![1, 2, 3, 4, 5],
Decimal128Array,
DataType::Decimal128(20, 2),
- vec![
- Some(convert(100)),
- Some(convert(200)),
- Some(convert(300)),
- Some(convert(400)),
- Some(convert(500)),
- ]
+ vec![Some(100), Some(200), Some(300), Some(400), Some(500),]
);
// float32
- let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes());
generic_test_cast!(
Float32Array,
DataType::Float32,
vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
Decimal128Array,
DataType::Decimal128(10, 2),
- vec![
- Some(convert(150)),
- Some(convert(250)),
- Some(convert(300)),
- Some(convert(112)),
- Some(convert(550)),
- ]
+ vec![Some(150), Some(250), Some(300), Some(112), Some(550),]
);
// float64
- let convert = |v: i128| Decimal128::new(20, 4, &v.to_le_bytes());
generic_test_cast!(
Float64Array,
DataType::Float64,
@@ -492,11 +439,11 @@ mod tests {
Decimal128Array,
DataType::Decimal128(20, 4),
vec![
- Some(convert(15000)),
- Some(convert(25000)),
- Some(convert(30000)),
- Some(convert(11234)),
- Some(convert(55000)),
+ Some(15000),
+ Some(25000),
+ Some(30000),
+ Some(11234),
+ Some(55000),
]
);
Ok(())
@@ -577,12 +524,14 @@ mod tests {
// create decimal array with the specified precision and scale
fn create_decimal_array(array: &[i128], precision: u8, scale: u8) ->
Decimal128Array {
- let mut decimal_builder =
- Decimal128Builder::with_capacity(array.len(), precision, scale);
+ let mut decimal_builder =
Decimal128Builder::with_capacity(array.len());
for value in array {
- decimal_builder.append_value(*value).expect("valid value");
+ decimal_builder.append_value(*value);
}
decimal_builder.append_null();
- decimal_builder.finish()
+ decimal_builder
+ .finish()
+ .with_precision_and_scale(precision, scale)
+ .unwrap()
}
}
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index 569fe5c44..ff2af33b2 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -40,7 +40,7 @@ default = []
json = ["pbjson", "pbjson-build", "serde", "serde_json"]
[dependencies]
-arrow = "24.0.0"
+arrow = "25.0.0"
datafusion = { path = "../core", version = "13.0.0" }
datafusion-common = { path = "../common", version = "13.0.0" }
datafusion-expr = { path = "../expr", version = "13.0.0" }
diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml
index bc32b07d9..52e6e50c7 100644
--- a/datafusion/row/Cargo.toml
+++ b/datafusion/row/Cargo.toml
@@ -37,7 +37,7 @@ path = "src/lib.rs"
jit = ["datafusion-jit"]
[dependencies]
-arrow = "24.0.0"
+arrow = "25.0.0"
datafusion-common = { path = "../common", version = "13.0.0" }
datafusion-jit = { path = "../jit", version = "13.0.0", optional = true }
paste = "^1.0"
diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml
index 4633e225a..4a88a01e3 100644
--- a/datafusion/sql/Cargo.toml
+++ b/datafusion/sql/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []
[dependencies]
-arrow = { version = "24.0.0", default-features = false }
+arrow = { version = "25.0.0", default-features = false }
datafusion-common = { path = "../common", version = "13.0.0" }
datafusion-expr = { path = "../expr", version = "13.0.0" }
sqlparser = "0.25"
