mbutrovich commented on code in PR #345:
URL: https://github.com/apache/datafusion-spark/pull/345#discussion_r4615393024
##########
crates/spark/src/function/type_coercion.rs:
##########
@@ -0,0 +1,427 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Spark's rules for finding a common type of several arguments, from its
`TypeCoercion`
+//! and `AnsiTypeCoercion`, on the Arrow types that Spark's types map to. Also
the checks
+//! and type names that the errors for those arguments use.
+//!
+//! These are for functions that need one type for several arguments, such as
+//! `array_position`. They are not Spark's implicit casts to a function's
declared input
+//! types, which the functions do through their [`datafusion_expr::Signature`].
+
+use std::sync::Arc;
+
+use arrow::datatypes::{DataType, Field, Fields, IntervalUnit};
+
+/// Spark's `findTightestCommonType`. `ansi_mode` picks between
`AnsiTypeCoercion` and
+/// `TypeCoercion`, which differ only in that ANSI widens an integer and a
FLOAT to DOUBLE.
+/// Arrays, maps and structs are widened element by element with this function.
+pub(crate) fn tightest_common_type(
+ left: &DataType,
+ right: &DataType,
+ ansi_mode: bool,
+) -> Option<DataType> {
+ use DataType::*;
+ if left == right {
+ return Some(left.clone());
+ }
+ // Spark has no dictionary types. A dictionary-encoded argument, such as a
+ // Parquet string column, has the type of its values.
+ if let Dictionary(_, value) = left {
+ return tightest_common_type(value, right, ansi_mode);
+ }
+ if let Dictionary(_, value) = right {
+ return tightest_common_type(left, value, ansi_mode);
+ }
+ if let (Some(l), Some(r)) = (numeric_precedence(left),
numeric_precedence(right)) {
+ let wider = if l >= r { left } else { right };
+ // With ANSI mode on, an integer and a FLOAT widen to DOUBLE, which
holds every INT
+ // exactly.
+ return Some(if ansi_mode && *wider == Float32 {
+ Float64
+ } else {
+ wider.clone()
+ });
+ }
+ match (left, right) {
+ (Null, other) | (other, Null) => Some(other.clone()),
+ // Spark has one string type and one binary type.
+ (l, r) if l.is_string() && r.is_string() =>
Some(wider_physical_type(l, r)),
+ (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l,
r)),
+ // The decimal must hold every value of the integral type.
+ (int, Decimal128(p, s)) | (Decimal128(p, s), int) => {
+ let digits = integral_digits(int)?;
+ (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s))
+ }
+ (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => {
+ Some(ts.clone())
+ }
+ // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP.
+ (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp(
+ // Arrow orders the units from Second to Nanosecond.
+ (*lu).max(*ru),
+ ltz.clone().or_else(|| rtz.clone()),
+ )),
+ _ => complex_type(left, right, &|l, r| tightest_common_type(l, r,
ansi_mode)),
+ }
+}
+
+/// Spark's `findTypeForComplex`: widens the elements of arrays, the keys and
values of maps,
+/// and the fields of structs with `find_type`. A nested field is nullable if
it is on either
+/// side. Struct fields must have the same names in the same order. Spark
compares them
+/// ignoring case, but DataFusion cannot cast between structs whose field
names differ only
+/// in case, so this requires an exact match.
+fn complex_type(
+ left: &DataType,
+ right: &DataType,
+ find_type: &dyn Fn(&DataType, &DataType) -> Option<DataType>,
+) -> Option<DataType> {
+ use DataType::*;
+ match (left, right) {
+ (
+ List(lf) | LargeList(lf) | FixedSizeList(lf, _),
+ List(rf) | LargeList(rf) | FixedSizeList(rf, _),
+ ) => {
+ let item = find_type(lf.data_type(), rf.data_type())?;
+ let field = Arc::new(Field::new(
+ lf.name(),
+ item,
+ lf.is_nullable() || rf.is_nullable(),
+ ));
+ Some(match (left, right) {
+ (FixedSizeList(_, ln), FixedSizeList(_, rn)) if ln == rn => {
+ FixedSizeList(field, *ln)
+ }
+ (LargeList(_), _) | (_, LargeList(_)) => LargeList(field),
+ _ => List(field),
+ })
+ }
+ (Struct(lf), Struct(rf)) if lf.len() == rf.len() => {
+ let fields = lf
+ .iter()
+ .zip(rf.iter())
+ .map(|(a, b)| {
+ if a.name() != b.name() {
+ return None;
+ }
+ let data_type = find_type(a.data_type(), b.data_type())?;
+ Some(Field::new(
+ a.name(),
+ data_type,
+ a.is_nullable() || b.is_nullable(),
+ ))
+ })
+ .collect::<Option<Vec<_>>>()?;
+ Some(Struct(Fields::from(fields)))
+ }
+ // The entries are a struct of the key and the value. Spark's
`MapType` has no field
+ // names, so the key and the value are matched by position.
+ (Map(lf, sorted), Map(rf, _)) => {
+ let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type())
else {
+ return None;
+ };
+ let ([lk, lv], [rk, rv]) = (&lkv[..], &rkv[..]) else {
+ return None;
+ };
+ let key = find_type(lk.data_type(), rk.data_type())?;
Review Comment:
> The dictionary checks now come first.
That covers a dictionary at the top level. A dictionary one level down still
reaches `array_position` when both sides have the same type, because the `left
== right` shortcut returns the nested type unchanged. These two queries return
`0`, and Spark 4.2.0 returns `1` for the same values without the `arrow_cast`
calls:
```sql
SELECT array_position(arrow_cast(array(array(CAST('-0.0' AS DOUBLE))),
'List(List(Dictionary(Int32, Float64)))'), arrow_cast(array(CAST(0.0 AS
DOUBLE)), 'List(Dictionary(Int32, Float64))'));
SELECT array_position(arrow_cast(array(named_struct('a', CAST('-0.0' AS
DOUBLE))), 'List(Struct("a": Dictionary(Int32, Float64)))'),
arrow_cast(named_struct('a', CAST(0.0 AS DOUBLE)), 'Struct("a":
Dictionary(Int32, Float64))'));
```
When only one side is a dictionary, `complex_type` unwraps it and the result
is `1`. Could the shortcut skip nested types, so `complex_type` rebuilds them
and unwraps the dictionaries inside?
```rust
// Nested types are widened field by field below, which also unwraps
nested dictionaries.
if left == right && !left.is_nested() {
return Some(left.clone());
}
```
With this, both queries return `1`. Two equal `Union` or list-view types
then have no common type, and Spark has neither. `.slt` records for the two
queries would cover it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]