mbutrovich commented on code in PR #330:
URL: https://github.com/apache/datafusion-spark/pull/330#discussion_r4605632755


##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,611 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use crate::function::float_normalize::normalize_array;
+use crate::function::type_coercion::{
+    is_orderable, spark_sql_type, struct_field_types_match, 
tightest_common_type,
+};
+use arrow::array::{
+    Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, 
BooleanBufferBuilder,
+    GenericListArray, OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field, Float32Type, Float64Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use num_traits::Float;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Removes all elements that are equal to the element from the 
array, while \
+        the DataFusion built-in removes only the first one. The result is NULL 
when the array \
+        or the element is NULL. NULL elements in the array are always kept. 
Corresponding NULLs nested inside compared arrays or structs are equal. \
+        For floating point values, also inside nested arrays and structs, NaN 
is equal to NaN \
+        and -0.0 is equal to 0.0; the elements that are kept keep their 
original values. As \
+        in Spark, the array element and the element are cast to their tightest 
common type, \
+        and the result is an array of that type: numbers widen, an integer 
widens to a \
+        decimal only if the decimal can hold all its values, and a date widens 
to a \
+        timestamp. Other combinations are an error, for example a string and a 
number, two \
+        different decimals, or structs whose field names or field types 
differ, as in Spark \
+        with ANSI mode off (with ANSI mode on, Spark widens struct field 
types). Map and \
+        INTERVAL elements, at any depth, are also an error.",
+    syntax_example = "array_remove(array, element)",
+    sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+    argument(name = "array", description = "Array expression. NULL returns 
NULL."),
+    argument(
+        name = "element",
+        description = "Element to remove. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+    signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkArrayRemove {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+    fn name(&self) -> &str {
+        "array_remove"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        let [array, _] = take_function_args(self.name(), arg_types)?;
+        Ok(array.clone())
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        let [array, element] = take_function_args(self.name(), arg_types)?;
+        coerce_array_and_element(self.name(), array, element)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let [array, element] = take_function_args(self.name(), &args.args)?;
+        let is_scalar = matches!(
+            (array, element),
+            (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+        );
+        let rows = if is_scalar { 1 } else { args.number_rows };
+        let array = array.to_array(rows)?;
+        // A constant element is compared as a single value against every row.
+        let (element, constant) = match element {
+            ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+            ColumnarValue::Array(element) => (Arc::clone(element), false),
+        };
+
+        let result = match array.data_type() {
+            DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+            DataType::LargeList(_) => remove::<i64>(&array, &element, 
constant)?,
+            other => return exec_err!("array_remove does not support type 
{other}"),
+        };
+
+        if is_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's 
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise. 
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+    array: &ArrayRef,
+    element: &ArrayRef,
+    constant: bool,
+) -> Result<ArrayRef> {
+    let list = array.as_list::<O>();
+    let offsets = list.offsets();
+    // Only the values that this (possibly sliced) list refers to are looked 
at.
+    let base = offsets[0].as_usize();
+    let len = offsets[offsets.len() - 1].as_usize() - base;
+    let values = list.values().slice(base, len);
+
+    // Logical nulls, so that a NULL-typed element, which has no null buffer, 
is NULL.
+    let element_nulls = element.logical_nulls();
+    let row_nulls = if constant {
+        if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+            Some(NullBuffer::new_null(list.len()))
+        } else {
+            list.nulls().cloned()
+        }
+    } else {
+        NullBuffer::union(list.nulls(), element_nulls.as_ref())
+    };
+    let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| 
n.is_valid(row));
+
+    // When every element is NULL there is nothing to compare: NULL elements 
are kept and a
+    // NULL element makes every row NULL. This also covers the NULL type.
+    let any_comparison =
+        values.logical_null_count() < values.len() && 
(0..list.len()).any(row_is_valid);
+    let value_nulls = values.logical_nulls();
+    let mut new_offsets = Vec::with_capacity(list.len() + 1);
+    new_offsets.push(O::usize_as(0));
+    let mut kept = 0usize;
+
+    let keep = if any_comparison && constant && 
!values.data_type().is_nested() {

Review Comment:
   The float arms of the bitmap path downcast the element with 
`as_primitive::<T>()` ([line 
251](https://github.com/apache/datafusion-spark/blob/7af017e4f6bac29bf9b83571b20c8ea3ff5e9fa9/crates/spark/src/function/array/array_remove.rs#L251)),
 which panics when the element's type differs from the array's element type. 
SQL coerces both to one type, but Comet calls `invoke_with_args()` without 
coercion. Calling it with a `List(Float64)` array and a `Float32` or `Int64` 
scalar element now panics with `primitive array`. Before this commit, the 
comparator path returned `Can't compare arrays of different types` for the same 
call, and it still does for a column element. This is the same concern as the 
[`MutableArrayData` check in 
#70](https://github.com/apache/datafusion-spark/pull/70#discussion_r4602383067).
   
   Could the bitmap path require matching types, so a mismatch falls through to 
the comparator path and its error, with a unit test that calls 
`invoke_with_args` with a `List(Float64)` array and a `Float32` scalar?
   
   ```suggestion
       // A caller that skips coercion, such as Comet, can pass an element of 
another type.
       // The comparator path returns an error for it instead of panicking on a 
downcast.
       let keep = if any_comparison
           && constant
           && !values.data_type().is_nested()
           && element.data_type() == values.data_type()
       {
   ```



##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,611 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use crate::function::float_normalize::normalize_array;
+use crate::function::type_coercion::{
+    is_orderable, spark_sql_type, struct_field_types_match, 
tightest_common_type,
+};
+use arrow::array::{
+    Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, 
BooleanBufferBuilder,
+    GenericListArray, OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field, Float32Type, Float64Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use num_traits::Float;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Removes all elements that are equal to the element from the 
array, while \
+        the DataFusion built-in removes only the first one. The result is NULL 
when the array \
+        or the element is NULL. NULL elements in the array are always kept. 
Corresponding NULLs nested inside compared arrays or structs are equal. \
+        For floating point values, also inside nested arrays and structs, NaN 
is equal to NaN \
+        and -0.0 is equal to 0.0; the elements that are kept keep their 
original values. As \
+        in Spark, the array element and the element are cast to their tightest 
common type, \
+        and the result is an array of that type: numbers widen, an integer 
widens to a \
+        decimal only if the decimal can hold all its values, and a date widens 
to a \
+        timestamp. Other combinations are an error, for example a string and a 
number, two \
+        different decimals, or structs whose field names or field types 
differ, as in Spark \
+        with ANSI mode off (with ANSI mode on, Spark widens struct field 
types). Map and \
+        INTERVAL elements, at any depth, are also an error.",
+    syntax_example = "array_remove(array, element)",
+    sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+    argument(name = "array", description = "Array expression. NULL returns 
NULL."),
+    argument(
+        name = "element",
+        description = "Element to remove. NULL returns NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+    signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkArrayRemove {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+    fn name(&self) -> &str {
+        "array_remove"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        let [array, _] = take_function_args(self.name(), arg_types)?;
+        Ok(array.clone())
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        let [array, element] = take_function_args(self.name(), arg_types)?;
+        coerce_array_and_element(self.name(), array, element)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        let [array, element] = take_function_args(self.name(), &args.args)?;
+        let is_scalar = matches!(
+            (array, element),
+            (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+        );
+        let rows = if is_scalar { 1 } else { args.number_rows };
+        let array = array.to_array(rows)?;
+        // A constant element is compared as a single value against every row.
+        let (element, constant) = match element {
+            ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+            ColumnarValue::Array(element) => (Arc::clone(element), false),
+        };
+
+        let result = match array.data_type() {
+            DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+            DataType::LargeList(_) => remove::<i64>(&array, &element, 
constant)?,
+            other => return exec_err!("array_remove does not support type 
{other}"),
+        };
+
+        if is_scalar {
+            Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+                &result, 0,
+            )?))
+        } else {
+            Ok(ColumnarValue::Array(result))
+        }
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's 
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise. 
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+    array: &ArrayRef,
+    element: &ArrayRef,
+    constant: bool,
+) -> Result<ArrayRef> {
+    let list = array.as_list::<O>();
+    let offsets = list.offsets();
+    // Only the values that this (possibly sliced) list refers to are looked 
at.
+    let base = offsets[0].as_usize();
+    let len = offsets[offsets.len() - 1].as_usize() - base;
+    let values = list.values().slice(base, len);
+
+    // Logical nulls, so that a NULL-typed element, which has no null buffer, 
is NULL.
+    let element_nulls = element.logical_nulls();
+    let row_nulls = if constant {
+        if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+            Some(NullBuffer::new_null(list.len()))
+        } else {
+            list.nulls().cloned()
+        }
+    } else {
+        NullBuffer::union(list.nulls(), element_nulls.as_ref())
+    };
+    let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| 
n.is_valid(row));
+
+    // When every element is NULL there is nothing to compare: NULL elements 
are kept and a
+    // NULL element makes every row NULL. This also covers the NULL type.
+    let any_comparison =
+        values.logical_null_count() < values.len() && 
(0..list.len()).any(row_is_valid);
+    let value_nulls = values.logical_nulls();
+    let mut new_offsets = Vec::with_capacity(list.len() + 1);
+    new_offsets.push(O::usize_as(0));
+    let mut kept = 0usize;
+
+    let keep = if any_comparison && constant && 
!values.data_type().is_nested() {
+        // A constant element that is not nested is compared with every value 
at once, and
+        // the elements to keep are found with bitmap operations.
+        let equal = match values.data_type() {
+            DataType::Float32 => float_equal::<Float32Type>(&values, element),
+            DataType::Float64 => float_equal::<Float64Type>(&values, element),
+            _ => eq(&values, &Scalar::new(Arc::clone(element)))?
+                .values()
+                .clone(),
+        };
+        let mut keep = !&equal;
+        if let Some(nulls) = &value_nulls {
+            keep = &keep | &!nulls.inner();
+        }
+        if row_nulls.as_ref().is_some_and(|n| n.null_count() > 0) {
+            // The elements of a NULL row are dropped.
+            let mut in_valid_row = BooleanBufferBuilder::new(len);
+            for (row, w) in offsets.windows(2).enumerate() {
+                in_valid_row.append_n((w[1] - w[0]).as_usize(), 
row_is_valid(row));
+            }
+            keep = &keep & &in_valid_row.finish();
+        }
+        for w in offsets.windows(2) {
+            let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - 
base);
+            kept += keep.slice(start, end - start).count_set_bits();

Review Comment:
   
[`BooleanBuffer::slice`](https://github.com/apache/arrow-rs/blob/b391e5af7dff9f6909af45c01f7dddb68ba08c0f/arrow-buffer/src/buffer/boolean.rs#L517-L527)
 clones the underlying `Buffer`, an `Arc` increment and decrement, for every 
row. 
[`Buffer::count_set_bits_offset`](https://github.com/apache/arrow-rs/blob/b391e5af7dff9f6909af45c01f7dddb68ba08c0f/arrow-buffer/src/buffer/immutable.rs#L372-L374)
 counts the same bits without the clone.
   
   ```suggestion
               kept += keep
                   .inner()
                   .count_set_bits_offset(keep.offset() + start, end - start);
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to