andygrove commented on code in PR #331:
URL: https://github.com/apache/datafusion-spark/pull/331#discussion_r4607590096


##########
crates/spark/src/function/array/array_extrema.rs:
##########
@@ -0,0 +1,780 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! Spark-compatible `array_max` and `array_min`.
+
+use crate::function::float_normalize::normalize_array;
+use crate::function::type_coercion::{is_orderable, spark_sql_type};
+use arrow::array::{
+    Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray, 
OffsetSizeTrait,
+    PrimitiveArray, UInt64Array, make_comparator, new_null_array,
+};
+use arrow::compute::{SortOptions, take};
+use arrow::datatypes::{
+    DataType, Date32Type, Decimal128Type, Field, Float32Type, Float64Type, 
Int8Type,
+    Int16Type, Int32Type, Int64Type, TimeUnit, TimestampMicrosecondType,
+    TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
+};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use std::cmp::Ordering;
+use std::sync::Arc;
+
+/// Spark-compatible `array_max` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_max>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Returns the maximum element of the array, using Spark's 
ordering. NULL \
+        elements are skipped, and an empty array or an array of only NULL 
elements returns \
+        NULL. For floating point values, also inside nested arrays and 
structs, NaN is \
+        greater than every other value, including infinity, and -0.0 is equal 
to 0.0. On a \
+        tie the first of the tied elements is returned as it is, so the sign 
of a zero and the \
+        bits of a NaN are kept. Strings and binary values compare bytewise. 
Arrays compare \
+        element by element and then by length, and structs compare field by 
field; NULLs \
+        nested inside them sort before other values. Map and INTERVAL 
elements, at any \
+        depth, are an error, as in Spark. DataFusion's built-in of the same 
name differs: it \
+        orders -0.0 before 0.0 rather than returning the first of them, sorts 
NULLs nested \
+        inside arrays and structs after other values, rejects a NULL argument 
at planning \
+        instead of returning NULL, and fails at execution on map elements.",
+    syntax_example = "array_max(array)",
+    sql_example = r#"```sql
+> SELECT array_max(array(1, 20, NULL, 3));
+20
+```"#,
+    argument(name = "array", description = "Array expression. NULL returns 
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMax {
+    signature: Signature,
+}
+
+impl Default for SparkArrayMax {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkArrayMax {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkArrayMax {
+    fn name(&self) -> &str {
+        "array_max"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        element_type(self.name(), arg_types)
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        coerce_array(self.name(), arg_types)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        invoke(self.name(), args, Ordering::Greater)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+/// Spark-compatible `array_min` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_min>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Returns the minimum element of the array, using Spark's 
ordering. NULL \
+        elements are skipped, and an empty array or an array of only NULL 
elements returns \
+        NULL. For floating point values, also inside nested arrays and 
structs, NaN is \
+        greater than every other value, including infinity, and -0.0 is equal 
to 0.0. On a \
+        tie the first of the tied elements is returned as it is, so the sign 
of a zero and the \
+        bits of a NaN are kept. Strings and binary values compare bytewise. 
Arrays compare \
+        element by element and then by length, and structs compare field by 
field; NULLs \
+        nested inside them sort before other values. Map and INTERVAL 
elements, at any \
+        depth, are an error, as in Spark. DataFusion's built-in of the same 
name differs: it \
+        orders -0.0 before 0.0 rather than returning the first of them, sorts 
NULLs nested \
+        inside arrays and structs after other values, rejects a NULL argument 
at planning \
+        instead of returning NULL, and fails at execution on map elements.",
+    syntax_example = "array_min(array)",
+    sql_example = r#"```sql
+> SELECT array_min(array(1, 20, NULL, 3));
+1
+```"#,
+    argument(name = "array", description = "Array expression. NULL returns 
NULL.")
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkArrayMin {
+    signature: Signature,
+}
+
+impl Default for SparkArrayMin {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkArrayMin {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkArrayMin {
+    fn name(&self) -> &str {
+        "array_min"
+    }
+
+    fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        element_type(self.name(), arg_types)
+    }
+
+    fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+        coerce_array(self.name(), arg_types)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> 
Result<ColumnarValue> {
+        invoke(self.name(), args, Ordering::Less)
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.doc()
+    }
+}
+
+/// The element type of the array argument. A NULL-typed argument, which Spark 
accepts,
+/// returns the NULL type.
+fn element_type(name: &str, arg_types: &[DataType]) -> Result<DataType> {
+    let [array] = take_function_args(name, arg_types)?;
+    match array {
+        DataType::Null => Ok(DataType::Null),
+        DataType::List(field)
+        | DataType::LargeList(field)
+        | DataType::FixedSizeList(field, _) => Ok(field.data_type().clone()),
+        other => unexpected_input_type(name, other),
+    }
+}
+
+/// Accepts the argument types that Spark's `ArrayMax` and `ArrayMin` accept: 
an array of an
+/// orderable type, or NULL. Fixed-size lists are cast to lists.
+fn coerce_array(name: &str, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+    let [array] = take_function_args(name, arg_types)?;
+    let coerced = match array {
+        DataType::Null | DataType::List(_) | DataType::LargeList(_) => 
array.clone(),
+        DataType::FixedSizeList(field, _) => 
DataType::List(Arc::new(Field::new(
+            field.name(),
+            field.data_type().clone(),
+            true,
+        ))),
+        other => return unexpected_input_type(name, other),
+    };

Review Comment:
   Fixed in 4dcf746. coerce_types now casts dictionary types to their value 
type at any depth of list and struct, so dictionary floats compare the same way 
as plain ones. This follows the rule #345 and #350 use, where a dictionary has 
the type of its values. The nested case will follow whatever #349 settles on. I 
added float cases for List, LargeList, FixedSizeList, a nested list and a 
struct field to array_extrema.slt, plus a dictionary string case.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to