andygrove commented on code in PR #63:
URL: https://github.com/apache/datafusion-spark/pull/63#discussion_r4585464722


##########
crates/sqllogictest/test_files/array/array_position.slt:
##########
@@ -0,0 +1,564 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+# Tests for Spark-compatible array_position.
+# Returns the 1-based position of the first match, 0 if not found, NULL if the 
array or element is NULL.
+# Expected values verified with pyspark==4.2.0.
+
+query I
+SELECT array_position(array(1, 2, 3, 4), 3);
+----
+3
+
+# Not found returns 0 (DataFusion's built-in returns NULL)
+query I
+SELECT array_position(array(1, 2, 3, 4), 5);
+----
+0
+
+query I
+SELECT array_position(array('a', 'b', 'c'), 'b');
+----
+2
+
+# NULL elements in the array do not match and still count towards the position
+query I
+SELECT array_position(array(1, 2, NULL, 3), 3);
+----
+4
+
+query I
+SELECT array_position(array(1, 2, NULL, 3), 5);
+----
+0
+
+# First occurrence wins
+query I
+SELECT array_position(array(1, 2, 1, 3), 1);
+----
+1
+
+query I
+SELECT array_position(array(1, 2, 3), CAST(NULL AS INT));
+----
+NULL
+
+query I
+SELECT array_position(CAST(NULL AS ARRAY<INT>), 1);
+----
+NULL
+
+# The tightest common type of the array element and an untyped NULL element is 
the array
+# element type, so the NULL is cast to it and this returns NULL.
+query I
+SELECT array_position(array(1, 2, 3), NULL);
+----
+NULL
+
+# A NULL element returns NULL even if the array contains NULL. The DataFusion 
built-in returns
+# the position of the first NULL instead.
+query I
+SELECT array_position(array(1, NULL), CAST(NULL AS BIGINT));
+----
+NULL
+
+query I
+SELECT array_position(CAST(NULL AS ARRAY<INT>), CAST(NULL AS INT));
+----
+NULL
+
+# An untyped NULL array is rejected at planning time, as in Spark.
+query error \[DATATYPE_MISMATCH\.NULL_TYPE\] Null typed values cannot be used 
as arguments of `array_position`
+SELECT array_position(NULL, 1);
+
+query error \[DATATYPE_MISMATCH\.NULL_TYPE\]
+SELECT array_position(NULL, NULL);
+
+# The common type of an empty array and an untyped NULL is the NULL type.
+query error \[DATATYPE_MISMATCH\.NULL_TYPE\]
+SELECT array_position(array(), NULL);
+
+query error \[DATATYPE_MISMATCH\.NULL_TYPE\]
+SELECT array_position(array(NULL), NULL);
+
+query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter 
of `array_position` requires the "ARRAY" type
+SELECT array_position(1, 1);
+
+query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\]
+SELECT array_position('abc', 'a');
+
+query T
+SELECT arrow_typeof(array_position(array(1, 2, 3), 2));
+----
+Int64
+
+# Several matches return the first position
+query I
+SELECT array_position(array(312, 773, 708, 708), 708);
+----
+3
+
+query I
+SELECT array_position(array(312, 773, 708, 708), 414);
+----
+0
+
+query I
+SELECT array_position(array(NULL, ''), '');
+----
+2
+
+query I
+SELECT array_position(array(NULL, ''), 'a');
+----
+0
+
+query I
+SELECT array_position(array(NULL), 1L);
+----
+0
+
+query I
+SELECT array_position(CAST(NULL AS ARRAY<STRING>), '');
+----
+NULL
+
+# Spark's `array(1, NULL)[0]` is its first element; arrays in this crate are 
1-based, so the
+# same element is `[1]` here.
+query I
+SELECT array_position(array(1, NULL), array(1, NULL)[1]);
+----
+1
+
+query I
+SELECT array_position(array(), 1);
+----
+0
+
+# NaN equals NaN
+query I
+SELECT array_position(array(CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE)), 
CAST('NaN' AS DOUBLE));
+----
+1
+
+query I
+SELECT array_position(array(CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT)), 
CAST('NaN' AS FLOAT));
+----
+1
+
+query I
+SELECT array_position(array(true, false), false);
+----
+2
+
+query I
+SELECT array_position(array(CAST(1.5 AS DECIMAL(3,1)), CAST(2.5 AS 
DECIMAL(3,1))), CAST(2.5 AS DECIMAL(3,1)));
+----
+2
+
+query I
+SELECT array_position(array(DATE '2020-01-01', DATE '2021-01-01'), DATE 
'2021-01-01');
+----
+2
+
+query I
+SELECT array_position(array(X'01', X'02'), X'02');
+----
+2
+
+# Nested arrays
+query I
+SELECT array_position(array(array(1, 2), array(3)), array(3));
+----
+2
+
+query I
+SELECT array_position(array(array(1, 2), array(3, 4)), array(1, 2));
+----
+1
+
+query I
+SELECT array_position(array(array(5, 6), array(2, 1)), array(1, 2));
+----
+0
+
+###
+### Type coercion
+###
+
+# The array element and the element are both cast to their tightest common type
+# (Spark's findTightestCommonType): here INT and BIGINT become BIGINT.
+query I
+SELECT array_position(array(CAST(1 AS INT), CAST(2 AS INT), CAST(3 AS INT)), 
2L);
+----
+2
+
+query I
+SELECT array_position(array(1, 2, 3), CAST(2.0 AS DOUBLE));
+----
+2
+
+# INT and DOUBLE become DOUBLE, so 1.23 does not match 1
+query I
+SELECT array_position(array(CAST(1 AS INT)), CAST(1.23 AS DOUBLE));

Review Comment:
   Thanks. `array_position(array(1, 2), column2)` and `array_position(column1, 
3)` were already in the column inputs section, and I added 
`array_position(array(1, 2, 3), column2)` next to them in bed1cf0, which 
matches on every non-NULL row and returns NULL for the NULL element. Verified 
with PySpark 4.2.0.



##########
crates/spark/src/function/array/array_position.rs:
##########
@@ -0,0 +1,413 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use super::float_normalize::normalize_array;
+use arrow::array::{
+    Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator,
+};
+use arrow::buffer::{NullBuffer, ScalarBuffer};
+use arrow::compute::SortOptions;
+use arrow::datatypes::{DataType, Field, Fields, TimeUnit};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+/// Spark-compatible `array_position` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_position>
+#[user_doc(
+    doc_section(label = "Array Functions"),
+    description = "Returns the 1-based position of the first occurrence of the 
element in the \
+        array, as a `BIGINT`. The DataFusion built-in returns a `UInt64`, and 
returns NULL when \
+        the element is not found, where this returns 0. The result is NULL 
only when the array \

Review Comment:
   Done in bed1cf0: "Top-level NULL elements in the array never match, while 
corresponding NULLs nested inside compared arrays or structs are equal." Also 
added `.slt` cases for a nested array and a struct with NULLs, both checked 
against PySpark 4.2.0.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to