andygrove commented on code in PR #330:
URL: https://github.com/apache/datafusion-spark/pull/330#discussion_r4603491752


##########
crates/spark/src/function/array/array_position.rs:
##########
@@ -349,7 +352,7 @@ fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit {
 }
 
 /// Spark cannot order maps, so it rejects them at any depth of the element 
type.
-fn contains_map(data_type: &DataType) -> bool {
+pub(super) fn contains_map(data_type: &DataType) -> bool {

Review Comment:
   Done through #345, whose shared `is_orderable` rejects 
`Interval(MonthDayNano)` at any depth for both `array_remove` and 
`array_position`. 552d8d7 adds flat and nested `.slt` error cases, and Spark 
4.2.0 raises the same errors.



##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,502 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use super::array_position::{contains_map, tightest_common_type};
+use super::float_normalize::normalize_array;
+use arrow::array::{
+    Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, 
GenericListArray,
+    OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Removes all elements that are equal to the element from the 
array. The \
+        result is NULL when the array or the element is NULL, while the 
DataFusion built-in \
+        removes the NULL elements when the element is NULL. NULL elements in 
the array are \
+        always kept. Corresponding NULLs nested inside compared arrays or 
structs are equal. \

Review Comment:
   Applied in 552d8d7, and fixed the `.slt` comment the same way. The first 
test now notes that the built-in returns `[1, 3, 2]`.



##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,502 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use super::array_position::{contains_map, tightest_common_type};
+use super::float_normalize::normalize_array;
+use arrow::array::{
+    Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, 
GenericListArray,
+    OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Removes all elements that are equal to the element from the 
array. The \
+        result is NULL when the array or the element is NULL, while the 
DataFusion built-in \
+        removes the NULL elements when the element is NULL. NULL elements in 
the array are \
+        always kept. Corresponding NULLs nested inside compared arrays or 
structs are equal. \
+        For floating point values, also inside nested arrays and structs, NaN 
is equal to NaN \
+        and -0.0 is equal to 0.0; the elements that are kept keep their 
original values. As \
+        in Spark, the array element and the element are cast to their tightest 
common type, \
+        and the result is an array of that type: numbers widen, an integer 
widens to a \
+        decimal only if the decimal can hold all its values, and a date widens 
to a \
+        timestamp. Other combinations are an error, for example a string and a 
number, two \
+        different decimals, or structs whose field names or field types 
differ, as in Spark \
+        with ANSI mode off (with ANSI mode on, Spark widens struct field 
types). Map \
+        elements are also an error.",
+    syntax_example = "array_remove(array, element)",
+    sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+    argument(name = "array", description = "Array expression. NULL returns 
NULL."),
+    argument(
+        name = "element",
+        description = "Element to remove. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+    signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkArrayRemove {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+    fn name(&self) -> &str {
+        "array_remove"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        let [array, _] = take_function_args(self.name(), arg_types)?;
+        Ok(array.clone())
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        let [array, element] = take_function_args(self.name(), arg_types)?;
+        coerce_array_and_element(self.name(), array, element)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let [array, element] = take_function_args(self.name(), &args.args)?;
+        let is_scalar = matches!(
+            (array, element),
+            (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+        );
+        let rows = if is_scalar { 1 } else { args.number_rows };
+        let array = array.to_array(rows)?;
+        // A constant element is compared as a single value against every row.
+        let (element, constant) = match element {
+            ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+            ColumnarValue::Array(element) => (Arc::clone(element), false),
+        };
+
+        let result = match array.data_type() {
+            DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+            DataType::LargeList(_) => remove::<i64>(&array, &element, 
constant)?,
+            other => return exec_err!("array_remove does not support type 
{other}"),
+        };
+
+        if is_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's 
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise. 
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+    array: &ArrayRef,
+    element: &ArrayRef,
+    constant: bool,
+) -> Result<ArrayRef> {
+    let list = array.as_list::<O>();
+    let offsets = list.offsets();
+    // Only the values that this (possibly sliced) list refers to are looked 
at.
+    let base = offsets[0].as_usize();
+    let len = offsets[offsets.len() - 1].as_usize() - base;
+    let values = list.values().slice(base, len);
+
+    // Logical nulls, so that a NULL-typed element, which has no null buffer, 
is NULL.
+    let element_nulls = element.logical_nulls();
+    let row_nulls = if constant {
+        if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+            Some(NullBuffer::new_null(list.len()))
+        } else {
+            list.nulls().cloned()
+        }
+    } else {
+        NullBuffer::union(list.nulls(), element_nulls.as_ref())
+    };
+    let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| 
n.is_valid(row));
+
+    // Equality is computed on copies with canonical NaN and 0.0 at any depth, 
so that it
+    // follows Spark's float equality, while the kept elements are taken from 
the original
+    // values. The comparator treats NULLs nested inside the compared values 
as equal.
+    let mut is_equal: Box<dyn Fn(usize, usize) -> bool> = Box::new(|_, _| 
false);
+    // When every element is NULL there is nothing to compare: NULL elements 
are kept and a
+    // NULL element makes every row NULL. This also covers the NULL type.
+    let any_comparison =
+        values.logical_null_count() < values.len() && 
(0..list.len()).any(row_is_valid);
+    if any_comparison {
+        let normalized = normalize_array(&values)?;
+        let needle = normalize_array(element)?;
+        is_equal = if constant && !normalized.data_type().is_nested() {
+            // Compare every value with the constant at once.
+            let equal = eq(&normalized, &Scalar::new(Arc::clone(&needle)))?;
+            let equal = equal.values().clone();
+            Box::new(move |i, _| equal.value(i))
+        } else {
+            let cmp = make_comparator(
+                normalized.as_ref(),
+                needle.as_ref(),
+                SortOptions::default(),
+            )?;
+            Box::new(move |i, row| cmp(i, if constant { 0 } else { row 
}).is_eq())
+        };
+    }
+
+    let value_nulls = values.logical_nulls();
+    let mut keep = BooleanBufferBuilder::new(len);
+    let mut new_offsets = Vec::with_capacity(list.len() + 1);
+    new_offsets.push(O::usize_as(0));
+    let mut kept = 0usize;
+    for (row, w) in offsets.windows(2).enumerate() {
+        let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base);
+        if row_is_valid(row) {
+            for i in start..end {
+                let keep_it = value_nulls.as_ref().is_some_and(|n| 
n.is_null(i))
+                    || !is_equal(i, row);
+                keep.append(keep_it);
+                kept += usize::from(keep_it);
+            }
+        } else {
+            keep.append_n(end - start, false);
+        }
+        new_offsets.push(O::usize_as(kept));
+    }
+    let keep: BooleanBuffer = keep.finish();

Review Comment:
   Done in 7af017e, for any constant element that isn't nested, not only 
floats. Floats compare in place (`==`, or `is_nan` for a NaN element), other 
types use Arrow's `eq`, and `keep` and the offsets come from bitmap operations 
and set-bit counts. NULL rows are cleared with one per-row pass. The new 
`benches/array_remove.rs` at 8192 rows: a constant Float64 went from 591 us to 
176 us, a NaN constant from 456 us to 130 us, and a constant Int64 from 515 us 
to 167 us. A column element is unchanged. 
`constant_fast_path_matches_comparator_path` checks that both paths agree on 
edge floats, NULL rows and a sliced list.



##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,502 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use super::array_position::{contains_map, tightest_common_type};
+use super::float_normalize::normalize_array;
+use arrow::array::{
+    Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, 
GenericListArray,
+    OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Removes all elements that are equal to the element from the 
array. The \
+        result is NULL when the array or the element is NULL, while the 
DataFusion built-in \
+        removes the NULL elements when the element is NULL. NULL elements in 
the array are \
+        always kept. Corresponding NULLs nested inside compared arrays or 
structs are equal. \
+        For floating point values, also inside nested arrays and structs, NaN 
is equal to NaN \
+        and -0.0 is equal to 0.0; the elements that are kept keep their 
original values. As \
+        in Spark, the array element and the element are cast to their tightest 
common type, \
+        and the result is an array of that type: numbers widen, an integer 
widens to a \
+        decimal only if the decimal can hold all its values, and a date widens 
to a \
+        timestamp. Other combinations are an error, for example a string and a 
number, two \
+        different decimals, or structs whose field names or field types 
differ, as in Spark \
+        with ANSI mode off (with ANSI mode on, Spark widens struct field 
types). Map \
+        elements are also an error.",
+    syntax_example = "array_remove(array, element)",
+    sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+    argument(name = "array", description = "Array expression. NULL returns 
NULL."),
+    argument(
+        name = "element",
+        description = "Element to remove. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+    signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkArrayRemove {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+    fn name(&self) -> &str {
+        "array_remove"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        let [array, _] = take_function_args(self.name(), arg_types)?;
+        Ok(array.clone())
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        let [array, element] = take_function_args(self.name(), arg_types)?;
+        coerce_array_and_element(self.name(), array, element)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let [array, element] = take_function_args(self.name(), &args.args)?;
+        let is_scalar = matches!(
+            (array, element),
+            (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+        );
+        let rows = if is_scalar { 1 } else { args.number_rows };
+        let array = array.to_array(rows)?;
+        // A constant element is compared as a single value against every row.
+        let (element, constant) = match element {
+            ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+            ColumnarValue::Array(element) => (Arc::clone(element), false),
+        };
+
+        let result = match array.data_type() {
+            DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+            DataType::LargeList(_) => remove::<i64>(&array, &element, 
constant)?,
+            other => return exec_err!("array_remove does not support type 
{other}"),
+        };
+
+        if is_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's 
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise. 
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+    array: &ArrayRef,
+    element: &ArrayRef,
+    constant: bool,
+) -> Result<ArrayRef> {
+    let list = array.as_list::<O>();
+    let offsets = list.offsets();
+    // Only the values that this (possibly sliced) list refers to are looked 
at.
+    let base = offsets[0].as_usize();
+    let len = offsets[offsets.len() - 1].as_usize() - base;
+    let values = list.values().slice(base, len);
+
+    // Logical nulls, so that a NULL-typed element, which has no null buffer, 
is NULL.
+    let element_nulls = element.logical_nulls();
+    let row_nulls = if constant {
+        if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+            Some(NullBuffer::new_null(list.len()))
+        } else {
+            list.nulls().cloned()
+        }
+    } else {
+        NullBuffer::union(list.nulls(), element_nulls.as_ref())
+    };
+    let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| 
n.is_valid(row));
+
+    // Equality is computed on copies with canonical NaN and 0.0 at any depth, 
so that it
+    // follows Spark's float equality, while the kept elements are taken from 
the original
+    // values. The comparator treats NULLs nested inside the compared values 
as equal.
+    let mut is_equal: Box<dyn Fn(usize, usize) -> bool> = Box::new(|_, _| 
false);
+    // When every element is NULL there is nothing to compare: NULL elements 
are kept and a
+    // NULL element makes every row NULL. This also covers the NULL type.
+    let any_comparison =
+        values.logical_null_count() < values.len() && 
(0..list.len()).any(row_is_valid);
+    if any_comparison {
+        let normalized = normalize_array(&values)?;
+        let needle = normalize_array(element)?;
+        is_equal = if constant && !normalized.data_type().is_nested() {
+            // Compare every value with the constant at once.
+            let equal = eq(&normalized, &Scalar::new(Arc::clone(&needle)))?;
+            let equal = equal.values().clone();
+            Box::new(move |i, _| equal.value(i))
+        } else {
+            let cmp = make_comparator(
+                normalized.as_ref(),
+                needle.as_ref(),
+                SortOptions::default(),
+            )?;
+            Box::new(move |i, row| cmp(i, if constant { 0 } else { row 
}).is_eq())
+        };
+    }
+
+    let value_nulls = values.logical_nulls();
+    let mut keep = BooleanBufferBuilder::new(len);
+    let mut new_offsets = Vec::with_capacity(list.len() + 1);
+    new_offsets.push(O::usize_as(0));
+    let mut kept = 0usize;
+    for (row, w) in offsets.windows(2).enumerate() {
+        let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base);
+        if row_is_valid(row) {
+            for i in start..end {
+                let keep_it = value_nulls.as_ref().is_some_and(|n| 
n.is_null(i))
+                    || !is_equal(i, row);
+                keep.append(keep_it);
+                kept += usize::from(keep_it);
+            }
+        } else {
+            keep.append_n(end - start, false);
+        }
+        new_offsets.push(O::usize_as(kept));
+    }
+    let keep: BooleanBuffer = keep.finish();
+    let kept_values = if kept == len {
+        values
+    } else {
+        filter(&values, &BooleanArray::new(keep, None))?
+    };
+
+    let (DataType::List(field) | DataType::LargeList(field)) = 
list.data_type() else {
+        unreachable!("remove is only called with lists");
+    };
+    Ok(Arc::new(GenericListArray::<O>::try_new(
+        Arc::clone(field),
+        OffsetBuffer::new(new_offsets.into()),
+        kept_values,
+        row_nulls,
+    )?))
+}
+
+/// Casts the array element and the element to their tightest common type, and 
rejects the
+/// argument types that Spark's `ArrayRemove.checkInputDataTypes` rejects.
+///
+/// Unlike `array_position`, Spark accepts a NULL-typed common element type 
here (for
+/// example `array_remove(array(), NULL)`, which returns NULL), and reports an 
argument that
+/// is not an array, including an untyped NULL, as `ARRAY_FUNCTION_DIFF_TYPES`.
+fn coerce_array_and_element(
+    name: &str,
+    array: &DataType,
+    element: &DataType,
+) -> Result<Vec<DataType>> {
+    let diff_types_err = || {
+        plan_err!(
+            "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` 
should have \
+             been \"ARRAY\" followed by a value with same element type, but 
it's \
+             [{array}, {element}]"
+        )
+    };

Review Comment:
   Done in 552d8d7 with `spark_sql_type` from #345. Both messages now end with 
`. SQLSTATE: 42K09`, and the `.slt` error cases assert the full text.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to