andygrove commented on code in PR #330:
URL: https://github.com/apache/datafusion-spark/pull/330#discussion_r4607531168


##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,611 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use crate::function::float_normalize::normalize_array;
+use crate::function::type_coercion::{
+    is_orderable, spark_sql_type, struct_field_types_match, 
tightest_common_type,
+};
+use arrow::array::{
+    Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, 
BooleanBufferBuilder,
+    GenericListArray, OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field, Float32Type, Float64Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use num_traits::Float;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Removes all elements that are equal to the element from the 
array, while \
+        the DataFusion built-in removes only the first one. The result is NULL 
when the array \
+        or the element is NULL. NULL elements in the array are always kept. 
Corresponding NULLs nested inside compared arrays or structs are equal. \
+        For floating point values, also inside nested arrays and structs, NaN 
is equal to NaN \
+        and -0.0 is equal to 0.0; the elements that are kept keep their 
original values. As \
+        in Spark, the array element and the element are cast to their tightest 
common type, \
+        and the result is an array of that type: numbers widen, an integer 
widens to a \
+        decimal only if the decimal can hold all its values, and a date widens 
to a \
+        timestamp. Other combinations are an error, for example a string and a 
number, two \
+        different decimals, or structs whose field names or field types 
differ, as in Spark \
+        with ANSI mode off (with ANSI mode on, Spark widens struct field 
types). Map and \
+        INTERVAL elements, at any depth, are also an error.",
+    syntax_example = "array_remove(array, element)",
+    sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+    argument(name = "array", description = "Array expression. NULL returns 
NULL."),
+    argument(
+        name = "element",
+        description = "Element to remove. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+    signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkArrayRemove {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+    fn name(&self) -> &str {
+        "array_remove"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        let [array, _] = take_function_args(self.name(), arg_types)?;
+        Ok(array.clone())
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        let [array, element] = take_function_args(self.name(), arg_types)?;
+        coerce_array_and_element(self.name(), array, element)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let [array, element] = take_function_args(self.name(), &args.args)?;
+        let is_scalar = matches!(
+            (array, element),
+            (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+        );
+        let rows = if is_scalar { 1 } else { args.number_rows };
+        let array = array.to_array(rows)?;
+        // A constant element is compared as a single value against every row.
+        let (element, constant) = match element {
+            ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+            ColumnarValue::Array(element) => (Arc::clone(element), false),
+        };
+
+        let result = match array.data_type() {
+            DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+            DataType::LargeList(_) => remove::<i64>(&array, &element, 
constant)?,
+            other => return exec_err!("array_remove does not support type 
{other}"),
+        };
+
+        if is_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's 
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise. 
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+    array: &ArrayRef,
+    element: &ArrayRef,
+    constant: bool,
+) -> Result<ArrayRef> {
+    let list = array.as_list::<O>();
+    let offsets = list.offsets();
+    // Only the values that this (possibly sliced) list refers to are looked 
at.
+    let base = offsets[0].as_usize();
+    let len = offsets[offsets.len() - 1].as_usize() - base;
+    let values = list.values().slice(base, len);
+
+    // Logical nulls, so that a NULL-typed element, which has no null buffer, 
is NULL.
+    let element_nulls = element.logical_nulls();
+    let row_nulls = if constant {
+        if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+            Some(NullBuffer::new_null(list.len()))
+        } else {
+            list.nulls().cloned()
+        }
+    } else {
+        NullBuffer::union(list.nulls(), element_nulls.as_ref())
+    };
+    let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| 
n.is_valid(row));
+
+    // When every element is NULL there is nothing to compare: NULL elements 
are kept and a
+    // NULL element makes every row NULL. This also covers the NULL type.
+    let any_comparison =
+        values.logical_null_count() < values.len() && 
(0..list.len()).any(row_is_valid);
+    let value_nulls = values.logical_nulls();
+    let mut new_offsets = Vec::with_capacity(list.len() + 1);
+    new_offsets.push(O::usize_as(0));
+    let mut kept = 0usize;
+
+    let keep = if any_comparison && constant && 
!values.data_type().is_nested() {

Review Comment:
   Thanks, fixed in 72b5ce0. A constant element of a different type now takes 
the comparator path and gets its error. The new unit test calls `remove` with a 
List(Float64) array and a Float32 or Int64 element, and a List(Int32) array 
with an Int64 element, as both a constant and a column. It panics without the 
fix.



##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,611 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use crate::function::float_normalize::normalize_array;
+use crate::function::type_coercion::{
+    is_orderable, spark_sql_type, struct_field_types_match, 
tightest_common_type,
+};
+use arrow::array::{
+    Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, 
BooleanBufferBuilder,
+    GenericListArray, OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field, Float32Type, Float64Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use num_traits::Float;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Removes all elements that are equal to the element from the 
array, while \
+        the DataFusion built-in removes only the first one. The result is NULL 
when the array \
+        or the element is NULL. NULL elements in the array are always kept. 
Corresponding NULLs nested inside compared arrays or structs are equal. \
+        For floating point values, also inside nested arrays and structs, NaN 
is equal to NaN \
+        and -0.0 is equal to 0.0; the elements that are kept keep their 
original values. As \
+        in Spark, the array element and the element are cast to their tightest 
common type, \
+        and the result is an array of that type: numbers widen, an integer 
widens to a \
+        decimal only if the decimal can hold all its values, and a date widens 
to a \
+        timestamp. Other combinations are an error, for example a string and a 
number, two \
+        different decimals, or structs whose field names or field types 
differ, as in Spark \
+        with ANSI mode off (with ANSI mode on, Spark widens struct field 
types). Map and \
+        INTERVAL elements, at any depth, are also an error.",
+    syntax_example = "array_remove(array, element)",
+    sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+    argument(name = "array", description = "Array expression. NULL returns 
NULL."),
+    argument(
+        name = "element",
+        description = "Element to remove. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+    signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkArrayRemove {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+    fn name(&self) -> &str {
+        "array_remove"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        let [array, _] = take_function_args(self.name(), arg_types)?;
+        Ok(array.clone())
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        let [array, element] = take_function_args(self.name(), arg_types)?;
+        coerce_array_and_element(self.name(), array, element)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let [array, element] = take_function_args(self.name(), &args.args)?;
+        let is_scalar = matches!(
+            (array, element),
+            (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+        );
+        let rows = if is_scalar { 1 } else { args.number_rows };
+        let array = array.to_array(rows)?;
+        // A constant element is compared as a single value against every row.
+        let (element, constant) = match element {
+            ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+            ColumnarValue::Array(element) => (Arc::clone(element), false),
+        };
+
+        let result = match array.data_type() {
+            DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+            DataType::LargeList(_) => remove::<i64>(&array, &element, 
constant)?,
+            other => return exec_err!("array_remove does not support type 
{other}"),
+        };
+
+        if is_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's 
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise. 
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+    array: &ArrayRef,
+    element: &ArrayRef,
+    constant: bool,
+) -> Result<ArrayRef> {
+    let list = array.as_list::<O>();
+    let offsets = list.offsets();
+    // Only the values that this (possibly sliced) list refers to are looked 
at.
+    let base = offsets[0].as_usize();
+    let len = offsets[offsets.len() - 1].as_usize() - base;
+    let values = list.values().slice(base, len);
+
+    // Logical nulls, so that a NULL-typed element, which has no null buffer, 
is NULL.
+    let element_nulls = element.logical_nulls();
+    let row_nulls = if constant {
+        if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+            Some(NullBuffer::new_null(list.len()))
+        } else {
+            list.nulls().cloned()
+        }
+    } else {
+        NullBuffer::union(list.nulls(), element_nulls.as_ref())
+    };
+    let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| 
n.is_valid(row));
+
+    // When every element is NULL there is nothing to compare: NULL elements 
are kept and a
+    // NULL element makes every row NULL. This also covers the NULL type.
+    let any_comparison =
+        values.logical_null_count() < values.len() && 
(0..list.len()).any(row_is_valid);
+    let value_nulls = values.logical_nulls();
+    let mut new_offsets = Vec::with_capacity(list.len() + 1);
+    new_offsets.push(O::usize_as(0));
+    let mut kept = 0usize;
+
+    let keep = if any_comparison && constant && 
!values.data_type().is_nested() {
+        // A constant element that is not nested is compared with every value 
at once, and
+        // the elements to keep are found with bitmap operations.
+        let equal = match values.data_type() {
+            DataType::Float32 => float_equal::<Float32Type>(&values, element),
+            DataType::Float64 => float_equal::<Float64Type>(&values, element),
+            _ => eq(&values, &Scalar::new(Arc::clone(element)))?
+                .values()
+                .clone(),
+        };
+        let mut keep = !&equal;
+        if let Some(nulls) = &value_nulls {
+            keep = &keep | &!nulls.inner();
+        }
+        if row_nulls.as_ref().is_some_and(|n| n.null_count() > 0) {
+            // The elements of a NULL row are dropped.
+            let mut in_valid_row = BooleanBufferBuilder::new(len);
+            for (row, w) in offsets.windows(2).enumerate() {
+                in_valid_row.append_n((w[1] - w[0]).as_usize(), 
row_is_valid(row));
+            }
+            keep = &keep & &in_valid_row.finish();
+        }
+        for w in offsets.windows(2) {
+            let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - 
base);
+            kept += keep.slice(start, end - start).count_set_bits();

Review Comment:
   Thanks, applied in 1ce5ba6.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to