andygrove commented on code in PR #330:
URL: https://github.com/apache/datafusion-spark/pull/330#discussion_r4607531168
##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,611 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use crate::function::float_normalize::normalize_array;
+use crate::function::type_coercion::{
+ is_orderable, spark_sql_type, struct_field_types_match,
tightest_common_type,
+};
+use arrow::array::{
+ Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray,
BooleanBufferBuilder,
+ GenericListArray, OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field, Float32Type, Float64Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use num_traits::Float;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Removes all elements that are equal to the element from the
array, while \
+ the DataFusion built-in removes only the first one. The result is NULL
when the array \
+ or the element is NULL. NULL elements in the array are always kept.
Corresponding NULLs nested inside compared arrays or structs are equal. \
+ For floating point values, also inside nested arrays and structs, NaN
is equal to NaN \
+ and -0.0 is equal to 0.0; the elements that are kept keep their
original values. As \
+ in Spark, the array element and the element are cast to their tightest
common type, \
+ and the result is an array of that type: numbers widen, an integer
widens to a \
+ decimal only if the decimal can hold all its values, and a date widens
to a \
+ timestamp. Other combinations are an error, for example a string and a
number, two \
+ different decimals, or structs whose field names or field types
differ, as in Spark \
+ with ANSI mode off (with ANSI mode on, Spark widens struct field
types). Map and \
+ INTERVAL elements, at any depth, are also an error.",
+ syntax_example = "array_remove(array, element)",
+ sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL."),
+ argument(
+ name = "element",
+ description = "Element to remove. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+ signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayRemove {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+ fn name(&self) -> &str {
+ "array_remove"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ let [array, _] = take_function_args(self.name(), arg_types)?;
+ Ok(array.clone())
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [array, element] = take_function_args(self.name(), arg_types)?;
+ coerce_array_and_element(self.name(), array, element)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let [array, element] = take_function_args(self.name(), &args.args)?;
+ let is_scalar = matches!(
+ (array, element),
+ (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+ );
+ let rows = if is_scalar { 1 } else { args.number_rows };
+ let array = array.to_array(rows)?;
+ // A constant element is compared as a single value against every row.
+ let (element, constant) = match element {
+ ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+ ColumnarValue::Array(element) => (Arc::clone(element), false),
+ };
+
+ let result = match array.data_type() {
+ DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+ DataType::LargeList(_) => remove::<i64>(&array, &element,
constant)?,
+ other => return exec_err!("array_remove does not support type
{other}"),
+ };
+
+ if is_scalar {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ } else {
+ Ok(ColumnarValue::Array(result))
+ }
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise.
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+ array: &ArrayRef,
+ element: &ArrayRef,
+ constant: bool,
+) -> Result<ArrayRef> {
+ let list = array.as_list::<O>();
+ let offsets = list.offsets();
+ // Only the values that this (possibly sliced) list refers to are looked
at.
+ let base = offsets[0].as_usize();
+ let len = offsets[offsets.len() - 1].as_usize() - base;
+ let values = list.values().slice(base, len);
+
+ // Logical nulls, so that a NULL-typed element, which has no null buffer,
is NULL.
+ let element_nulls = element.logical_nulls();
+ let row_nulls = if constant {
+ if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+ Some(NullBuffer::new_null(list.len()))
+ } else {
+ list.nulls().cloned()
+ }
+ } else {
+ NullBuffer::union(list.nulls(), element_nulls.as_ref())
+ };
+ let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n|
n.is_valid(row));
+
+ // When every element is NULL there is nothing to compare: NULL elements
are kept and a
+ // NULL element makes every row NULL. This also covers the NULL type.
+ let any_comparison =
+ values.logical_null_count() < values.len() &&
(0..list.len()).any(row_is_valid);
+ let value_nulls = values.logical_nulls();
+ let mut new_offsets = Vec::with_capacity(list.len() + 1);
+ new_offsets.push(O::usize_as(0));
+ let mut kept = 0usize;
+
+ let keep = if any_comparison && constant &&
!values.data_type().is_nested() {
Review Comment:
Thanks, fixed in 72b5ce0. A constant element of a different type now takes
the comparator path and gets its error. The new unit test calls `remove` with a
List(Float64) array and a Float32 or Int64 element, and a List(Int32) array
with an Int64 element, as both a constant and a column. It panics without the
fix.
##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,611 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use crate::function::float_normalize::normalize_array;
+use crate::function::type_coercion::{
+ is_orderable, spark_sql_type, struct_field_types_match,
tightest_common_type,
+};
+use arrow::array::{
+ Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray,
BooleanBufferBuilder,
+ GenericListArray, OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field, Float32Type, Float64Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use num_traits::Float;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Removes all elements that are equal to the element from the
array, while \
+ the DataFusion built-in removes only the first one. The result is NULL
when the array \
+ or the element is NULL. NULL elements in the array are always kept.
Corresponding NULLs nested inside compared arrays or structs are equal. \
+ For floating point values, also inside nested arrays and structs, NaN
is equal to NaN \
+ and -0.0 is equal to 0.0; the elements that are kept keep their
original values. As \
+ in Spark, the array element and the element are cast to their tightest
common type, \
+ and the result is an array of that type: numbers widen, an integer
widens to a \
+ decimal only if the decimal can hold all its values, and a date widens
to a \
+ timestamp. Other combinations are an error, for example a string and a
number, two \
+ different decimals, or structs whose field names or field types
differ, as in Spark \
+ with ANSI mode off (with ANSI mode on, Spark widens struct field
types). Map and \
+ INTERVAL elements, at any depth, are also an error.",
+ syntax_example = "array_remove(array, element)",
+ sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL."),
+ argument(
+ name = "element",
+ description = "Element to remove. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+ signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayRemove {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+ fn name(&self) -> &str {
+ "array_remove"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ let [array, _] = take_function_args(self.name(), arg_types)?;
+ Ok(array.clone())
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [array, element] = take_function_args(self.name(), arg_types)?;
+ coerce_array_and_element(self.name(), array, element)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let [array, element] = take_function_args(self.name(), &args.args)?;
+ let is_scalar = matches!(
+ (array, element),
+ (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+ );
+ let rows = if is_scalar { 1 } else { args.number_rows };
+ let array = array.to_array(rows)?;
+ // A constant element is compared as a single value against every row.
+ let (element, constant) = match element {
+ ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+ ColumnarValue::Array(element) => (Arc::clone(element), false),
+ };
+
+ let result = match array.data_type() {
+ DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+ DataType::LargeList(_) => remove::<i64>(&array, &element,
constant)?,
+ other => return exec_err!("array_remove does not support type
{other}"),
+ };
+
+ if is_scalar {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ } else {
+ Ok(ColumnarValue::Array(result))
+ }
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise.
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+ array: &ArrayRef,
+ element: &ArrayRef,
+ constant: bool,
+) -> Result<ArrayRef> {
+ let list = array.as_list::<O>();
+ let offsets = list.offsets();
+ // Only the values that this (possibly sliced) list refers to are looked
at.
+ let base = offsets[0].as_usize();
+ let len = offsets[offsets.len() - 1].as_usize() - base;
+ let values = list.values().slice(base, len);
+
+ // Logical nulls, so that a NULL-typed element, which has no null buffer,
is NULL.
+ let element_nulls = element.logical_nulls();
+ let row_nulls = if constant {
+ if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+ Some(NullBuffer::new_null(list.len()))
+ } else {
+ list.nulls().cloned()
+ }
+ } else {
+ NullBuffer::union(list.nulls(), element_nulls.as_ref())
+ };
+ let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n|
n.is_valid(row));
+
+ // When every element is NULL there is nothing to compare: NULL elements
are kept and a
+ // NULL element makes every row NULL. This also covers the NULL type.
+ let any_comparison =
+ values.logical_null_count() < values.len() &&
(0..list.len()).any(row_is_valid);
+ let value_nulls = values.logical_nulls();
+ let mut new_offsets = Vec::with_capacity(list.len() + 1);
+ new_offsets.push(O::usize_as(0));
+ let mut kept = 0usize;
+
+ let keep = if any_comparison && constant &&
!values.data_type().is_nested() {
+ // A constant element that is not nested is compared with every value
at once, and
+ // the elements to keep are found with bitmap operations.
+ let equal = match values.data_type() {
+ DataType::Float32 => float_equal::<Float32Type>(&values, element),
+ DataType::Float64 => float_equal::<Float64Type>(&values, element),
+ _ => eq(&values, &Scalar::new(Arc::clone(element)))?
+ .values()
+ .clone(),
+ };
+ let mut keep = !&equal;
+ if let Some(nulls) = &value_nulls {
+ keep = &keep | &!nulls.inner();
+ }
+ if row_nulls.as_ref().is_some_and(|n| n.null_count() > 0) {
+ // The elements of a NULL row are dropped.
+ let mut in_valid_row = BooleanBufferBuilder::new(len);
+ for (row, w) in offsets.windows(2).enumerate() {
+ in_valid_row.append_n((w[1] - w[0]).as_usize(),
row_is_valid(row));
+ }
+ keep = &keep & &in_valid_row.finish();
+ }
+ for w in offsets.windows(2) {
+ let (start, end) = (w[0].as_usize() - base, w[1].as_usize() -
base);
+ kept += keep.slice(start, end - start).count_set_bits();
Review Comment:
Thanks, applied in 1ce5ba6.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]