andygrove commented on code in PR #330:
URL: https://github.com/apache/datafusion-spark/pull/330#discussion_r4603491752
##########
crates/spark/src/function/array/array_position.rs:
##########
@@ -349,7 +352,7 @@ fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit {
}
/// Spark cannot order maps, so it rejects them at any depth of the element
type.
-fn contains_map(data_type: &DataType) -> bool {
+pub(super) fn contains_map(data_type: &DataType) -> bool {
Review Comment:
Done through #345, whose shared `is_orderable` rejects
`Interval(MonthDayNano)` at any depth for both `array_remove` and
`array_position`. 552d8d7 adds flat and nested `.slt` error cases, and Spark
4.2.0 raises the same errors.
##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,502 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use super::array_position::{contains_map, tightest_common_type};
+use super::float_normalize::normalize_array;
+use arrow::array::{
+ Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder,
GenericListArray,
+ OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Removes all elements that are equal to the element from the
array. The \
+ result is NULL when the array or the element is NULL, while the
DataFusion built-in \
+ removes the NULL elements when the element is NULL. NULL elements in
the array are \
+ always kept. Corresponding NULLs nested inside compared arrays or
structs are equal. \
Review Comment:
Applied in 552d8d7, and fixed the `.slt` comment the same way. The first
test now notes that the built-in returns `[1, 3, 2]`.
##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,502 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use super::array_position::{contains_map, tightest_common_type};
+use super::float_normalize::normalize_array;
+use arrow::array::{
+ Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder,
GenericListArray,
+ OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Removes all elements that are equal to the element from the
array. The \
+ result is NULL when the array or the element is NULL, while the
DataFusion built-in \
+ removes the NULL elements when the element is NULL. NULL elements in
the array are \
+ always kept. Corresponding NULLs nested inside compared arrays or
structs are equal. \
+ For floating point values, also inside nested arrays and structs, NaN
is equal to NaN \
+ and -0.0 is equal to 0.0; the elements that are kept keep their
original values. As \
+ in Spark, the array element and the element are cast to their tightest
common type, \
+ and the result is an array of that type: numbers widen, an integer
widens to a \
+ decimal only if the decimal can hold all its values, and a date widens
to a \
+ timestamp. Other combinations are an error, for example a string and a
number, two \
+ different decimals, or structs whose field names or field types
differ, as in Spark \
+ with ANSI mode off (with ANSI mode on, Spark widens struct field
types). Map \
+ elements are also an error.",
+ syntax_example = "array_remove(array, element)",
+ sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL."),
+ argument(
+ name = "element",
+ description = "Element to remove. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+ signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayRemove {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+ fn name(&self) -> &str {
+ "array_remove"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ let [array, _] = take_function_args(self.name(), arg_types)?;
+ Ok(array.clone())
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [array, element] = take_function_args(self.name(), arg_types)?;
+ coerce_array_and_element(self.name(), array, element)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let [array, element] = take_function_args(self.name(), &args.args)?;
+ let is_scalar = matches!(
+ (array, element),
+ (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+ );
+ let rows = if is_scalar { 1 } else { args.number_rows };
+ let array = array.to_array(rows)?;
+ // A constant element is compared as a single value against every row.
+ let (element, constant) = match element {
+ ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+ ColumnarValue::Array(element) => (Arc::clone(element), false),
+ };
+
+ let result = match array.data_type() {
+ DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+ DataType::LargeList(_) => remove::<i64>(&array, &element,
constant)?,
+ other => return exec_err!("array_remove does not support type
{other}"),
+ };
+
+ if is_scalar {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ } else {
+ Ok(ColumnarValue::Array(result))
+ }
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise.
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+ array: &ArrayRef,
+ element: &ArrayRef,
+ constant: bool,
+) -> Result<ArrayRef> {
+ let list = array.as_list::<O>();
+ let offsets = list.offsets();
+ // Only the values that this (possibly sliced) list refers to are looked
at.
+ let base = offsets[0].as_usize();
+ let len = offsets[offsets.len() - 1].as_usize() - base;
+ let values = list.values().slice(base, len);
+
+ // Logical nulls, so that a NULL-typed element, which has no null buffer,
is NULL.
+ let element_nulls = element.logical_nulls();
+ let row_nulls = if constant {
+ if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+ Some(NullBuffer::new_null(list.len()))
+ } else {
+ list.nulls().cloned()
+ }
+ } else {
+ NullBuffer::union(list.nulls(), element_nulls.as_ref())
+ };
+ let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n|
n.is_valid(row));
+
+ // Equality is computed on copies with canonical NaN and 0.0 at any depth,
so that it
+ // follows Spark's float equality, while the kept elements are taken from
the original
+ // values. The comparator treats NULLs nested inside the compared values
as equal.
+ let mut is_equal: Box<dyn Fn(usize, usize) -> bool> = Box::new(|_, _|
false);
+ // When every element is NULL there is nothing to compare: NULL elements
are kept and a
+ // NULL element makes every row NULL. This also covers the NULL type.
+ let any_comparison =
+ values.logical_null_count() < values.len() &&
(0..list.len()).any(row_is_valid);
+ if any_comparison {
+ let normalized = normalize_array(&values)?;
+ let needle = normalize_array(element)?;
+ is_equal = if constant && !normalized.data_type().is_nested() {
+ // Compare every value with the constant at once.
+ let equal = eq(&normalized, &Scalar::new(Arc::clone(&needle)))?;
+ let equal = equal.values().clone();
+ Box::new(move |i, _| equal.value(i))
+ } else {
+ let cmp = make_comparator(
+ normalized.as_ref(),
+ needle.as_ref(),
+ SortOptions::default(),
+ )?;
+ Box::new(move |i, row| cmp(i, if constant { 0 } else { row
}).is_eq())
+ };
+ }
+
+ let value_nulls = values.logical_nulls();
+ let mut keep = BooleanBufferBuilder::new(len);
+ let mut new_offsets = Vec::with_capacity(list.len() + 1);
+ new_offsets.push(O::usize_as(0));
+ let mut kept = 0usize;
+ for (row, w) in offsets.windows(2).enumerate() {
+ let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base);
+ if row_is_valid(row) {
+ for i in start..end {
+ let keep_it = value_nulls.as_ref().is_some_and(|n|
n.is_null(i))
+ || !is_equal(i, row);
+ keep.append(keep_it);
+ kept += usize::from(keep_it);
+ }
+ } else {
+ keep.append_n(end - start, false);
+ }
+ new_offsets.push(O::usize_as(kept));
+ }
+ let keep: BooleanBuffer = keep.finish();
Review Comment:
Done in 7af017e, for any constant element that isn't nested, not only
floats. Floats compare in place (`==`, or `is_nan` for a NaN element), other
types use Arrow's `eq`, and `keep` and the offsets come from bitmap operations
and set-bit counts. NULL rows are cleared with one per-row pass. The new
`benches/array_remove.rs` at 8192 rows: a constant Float64 went from 591 us to
176 us, a NaN constant from 456 us to 130 us, and a constant Int64 from 515 us
to 167 us. A column element is unchanged.
`constant_fast_path_matches_comparator_path` checks that both paths agree on
edge floats, NULL rows and a sliced list.
##########
crates/spark/src/function/array/array_remove.rs:
##########
@@ -0,0 +1,502 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use super::array_position::{contains_map, tightest_common_type};
+use super::float_normalize::normalize_array;
+use arrow::array::{
+ Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder,
GenericListArray,
+ OffsetSizeTrait, Scalar, make_comparator,
+};
+use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer};
+use arrow::compute::kernels::cmp::eq;
+use arrow::compute::{SortOptions, filter};
+use arrow::datatypes::{DataType, Field};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+/// Spark-compatible `array_remove` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_remove>
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Removes all elements that are equal to the element from the
array. The \
+ result is NULL when the array or the element is NULL, while the
DataFusion built-in \
+ removes the NULL elements when the element is NULL. NULL elements in
the array are \
+ always kept. Corresponding NULLs nested inside compared arrays or
structs are equal. \
+ For floating point values, also inside nested arrays and structs, NaN
is equal to NaN \
+ and -0.0 is equal to 0.0; the elements that are kept keep their
original values. As \
+ in Spark, the array element and the element are cast to their tightest
common type, \
+ and the result is an array of that type: numbers widen, an integer
widens to a \
+ decimal only if the decimal can hold all its values, and a date widens
to a \
+ timestamp. Other combinations are an error, for example a string and a
number, two \
+ different decimals, or structs whose field names or field types
differ, as in Spark \
+ with ANSI mode off (with ANSI mode on, Spark widens struct field
types). Map \
+ elements are also an error.",
+ syntax_example = "array_remove(array, element)",
+ sql_example = r#"```sql
+> SELECT array_remove(array(1, 2, 3, NULL, 3), 3);
+[1, 2, NULL]
+```"#,
+ argument(name = "array", description = "Array expression. NULL returns
NULL."),
+ argument(
+ name = "element",
+ description = "Element to remove. NULL returns NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayRemove {
+ signature: Signature,
+}
+
+impl Default for SparkArrayRemove {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkArrayRemove {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkArrayRemove {
+ fn name(&self) -> &str {
+ "array_remove"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ let [array, _] = take_function_args(self.name(), arg_types)?;
+ Ok(array.clone())
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [array, element] = take_function_args(self.name(), arg_types)?;
+ coerce_array_and_element(self.name(), array, element)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let [array, element] = take_function_args(self.name(), &args.args)?;
+ let is_scalar = matches!(
+ (array, element),
+ (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_))
+ );
+ let rows = if is_scalar { 1 } else { args.number_rows };
+ let array = array.to_array(rows)?;
+ // A constant element is compared as a single value against every row.
+ let (element, constant) = match element {
+ ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true),
+ ColumnarValue::Array(element) => (Arc::clone(element), false),
+ };
+
+ let result = match array.data_type() {
+ DataType::List(_) => remove::<i32>(&array, &element, constant)?,
+ DataType::LargeList(_) => remove::<i64>(&array, &element,
constant)?,
+ other => return exec_err!("array_remove does not support type
{other}"),
+ };
+
+ if is_scalar {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ } else {
+ Ok(ColumnarValue::Array(result))
+ }
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// Removes from each row of `array` the non-null elements equal to the row's
element, which
+/// is `element[0]` for every row if `constant`, and `element[row]` otherwise.
A row is NULL
+/// when the array or the element is NULL.
+fn remove<O: OffsetSizeTrait>(
+ array: &ArrayRef,
+ element: &ArrayRef,
+ constant: bool,
+) -> Result<ArrayRef> {
+ let list = array.as_list::<O>();
+ let offsets = list.offsets();
+ // Only the values that this (possibly sliced) list refers to are looked
at.
+ let base = offsets[0].as_usize();
+ let len = offsets[offsets.len() - 1].as_usize() - base;
+ let values = list.values().slice(base, len);
+
+ // Logical nulls, so that a NULL-typed element, which has no null buffer,
is NULL.
+ let element_nulls = element.logical_nulls();
+ let row_nulls = if constant {
+ if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) {
+ Some(NullBuffer::new_null(list.len()))
+ } else {
+ list.nulls().cloned()
+ }
+ } else {
+ NullBuffer::union(list.nulls(), element_nulls.as_ref())
+ };
+ let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n|
n.is_valid(row));
+
+ // Equality is computed on copies with canonical NaN and 0.0 at any depth,
so that it
+ // follows Spark's float equality, while the kept elements are taken from
the original
+ // values. The comparator treats NULLs nested inside the compared values
as equal.
+ let mut is_equal: Box<dyn Fn(usize, usize) -> bool> = Box::new(|_, _|
false);
+ // When every element is NULL there is nothing to compare: NULL elements
are kept and a
+ // NULL element makes every row NULL. This also covers the NULL type.
+ let any_comparison =
+ values.logical_null_count() < values.len() &&
(0..list.len()).any(row_is_valid);
+ if any_comparison {
+ let normalized = normalize_array(&values)?;
+ let needle = normalize_array(element)?;
+ is_equal = if constant && !normalized.data_type().is_nested() {
+ // Compare every value with the constant at once.
+ let equal = eq(&normalized, &Scalar::new(Arc::clone(&needle)))?;
+ let equal = equal.values().clone();
+ Box::new(move |i, _| equal.value(i))
+ } else {
+ let cmp = make_comparator(
+ normalized.as_ref(),
+ needle.as_ref(),
+ SortOptions::default(),
+ )?;
+ Box::new(move |i, row| cmp(i, if constant { 0 } else { row
}).is_eq())
+ };
+ }
+
+ let value_nulls = values.logical_nulls();
+ let mut keep = BooleanBufferBuilder::new(len);
+ let mut new_offsets = Vec::with_capacity(list.len() + 1);
+ new_offsets.push(O::usize_as(0));
+ let mut kept = 0usize;
+ for (row, w) in offsets.windows(2).enumerate() {
+ let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base);
+ if row_is_valid(row) {
+ for i in start..end {
+ let keep_it = value_nulls.as_ref().is_some_and(|n|
n.is_null(i))
+ || !is_equal(i, row);
+ keep.append(keep_it);
+ kept += usize::from(keep_it);
+ }
+ } else {
+ keep.append_n(end - start, false);
+ }
+ new_offsets.push(O::usize_as(kept));
+ }
+ let keep: BooleanBuffer = keep.finish();
+ let kept_values = if kept == len {
+ values
+ } else {
+ filter(&values, &BooleanArray::new(keep, None))?
+ };
+
+ let (DataType::List(field) | DataType::LargeList(field)) =
list.data_type() else {
+ unreachable!("remove is only called with lists");
+ };
+ Ok(Arc::new(GenericListArray::<O>::try_new(
+ Arc::clone(field),
+ OffsetBuffer::new(new_offsets.into()),
+ kept_values,
+ row_nulls,
+ )?))
+}
+
+/// Casts the array element and the element to their tightest common type, and
rejects the
+/// argument types that Spark's `ArrayRemove.checkInputDataTypes` rejects.
+///
+/// Unlike `array_position`, Spark accepts a NULL-typed common element type
here (for
+/// example `array_remove(array(), NULL)`, which returns NULL), and reports an
argument that
+/// is not an array, including an untyped NULL, as `ARRAY_FUNCTION_DIFF_TYPES`.
+fn coerce_array_and_element(
+ name: &str,
+ array: &DataType,
+ element: &DataType,
+) -> Result<Vec<DataType>> {
+ let diff_types_err = || {
+ plan_err!(
+ "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}`
should have \
+ been \"ARRAY\" followed by a value with same element type, but
it's \
+ [{array}, {element}]"
+ )
+ };
Review Comment:
Done in 552d8d7 with `spark_sql_type` from #345. Both messages now end with
`. SQLSTATE: 42K09`, and the `.slt` error cases assert the full text.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]