andygrove commented on code in PR #328:
URL: https://github.com/apache/datafusion-spark/pull/328#discussion_r4603594354


##########
crates/spark/src/function/string/contains.rs:
##########
@@ -0,0 +1,651 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, BooleanArray, Scalar};
+use arrow::compute::kernels::cast::cast;
+use arrow::compute::kernels::comparison::contains as arrow_contains;
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+    Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `contains` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#contains>
+#[user_doc(
+    doc_section(label = "String Functions"),
+    description = "Returns true if `right` is found inside `left`. 
DataFusion's own `contains` takes only \
+        strings; this one also accepts binary. When both arguments are binary 
the match is on bytes. \
+        When one argument is a string and the other is binary, both are 
compared as bytes, which is \
+        the same as Spark casting the binary to a string. Integer, boolean and 
date arguments are \
+        cast to a string first. The match is case-sensitive, and an empty 
`right` is found in every \
+        value, including an empty `left`. Returns NULL if either input is 
NULL.",
+    syntax_example = "contains(left, right)",
+    sql_example = r#"```sql
+> SELECT contains('Spark SQL', 'Spark');
+true
+> SELECT contains('Spark SQL', 'SPARK');
+false
+> SELECT contains(x'537061726b2053514c', x'537061726b');
+true
+```"#,
+    argument(
+        name = "left",
+        description = "String or binary expression to search in. NULL returns 
NULL."
+    ),
+    argument(
+        name = "right",
+        description = "String or binary expression to search for. NULL returns 
NULL."
+    )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkContains {
+    signature: Signature,
+}
+
+impl Default for SparkContains {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkContains {
+    pub fn new() -> Self {
+        Self {
+            signature: Signature::user_defined(Volatility::Immutable),
+        }
+    }
+}
+
+/// How an argument type takes part in `contains`.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum Kind {
+    Null,
+    String,
+    Binary,
+}
+
+/// Classifies an argument type, returning the type it is coerced to before
+/// unification. Types Spark implicitly casts to a string and whose Arrow 
string
+/// form matches Spark's become `Utf8`.
+fn classify(name: &str, dt: &DataType) -> Result<(Kind, DataType)> {
+    match dt {
+        DataType::Null => Ok((Kind::Null, DataType::Null)),
+        DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+            Ok((Kind::String, dt.clone()))
+        }
+        DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
+            Ok((Kind::Binary, dt.clone()))
+        }
+        DataType::FixedSizeBinary(_) => Ok((Kind::Binary, DataType::Binary)),
+        DataType::Dictionary(_, value) => classify(name, value),
+        DataType::Int8
+        | DataType::Int16
+        | DataType::Int32
+        | DataType::Int64
+        | DataType::UInt8
+        | DataType::UInt16
+        | DataType::UInt32
+        | DataType::UInt64
+        | DataType::Boolean
+        | DataType::Date32 => Ok((Kind::String, DataType::Utf8)),
+        other => plan_err!(
+            "Unsupported Data Type: Spark `{name}` function expects STRING or 
BINARY arguments, got {other}"
+        ),

Review Comment:
   Filed #347. It closes once the casts to string from #88 exist, and relates 
to #346, which would let `contains` declare string inputs the way Spark's 
`ImplicitCastInputTypes` does. In dd706b3, the doc now says other types are 
rejected and links the issue. `contains.slt` checks that a DOUBLE is rejected 
and records Spark 4.2.0's results as commented-out queries, including the 
decimal case that depends on ANSI mode.



##########
crates/sqllogictest/test_files/string/contains.slt:
##########
@@ -2,28 +2,145 @@
 # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
 # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
 
-# This file was originally created by a porting script from:
-#   
https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
-# This file is part of the implementation of the datafusion-spark function 
library.
-# For more information, please see:
-#   https://github.com/apache/datafusion/issues/15914
-
-## Original Query: SELECT contains('Spark SQL', 'SPARK');
-## PySpark 3.5.5 Result: {'contains(Spark SQL, SPARK)': False, 
'typeof(contains(Spark SQL, SPARK))': 'boolean', 'typeof(Spark SQL)': 'string', 
'typeof(SPARK)': 'string'}
-#query
-#SELECT contains('Spark SQL'::string, 'SPARK'::string);
-
-## Original Query: SELECT contains('Spark SQL', 'Spark');
-## PySpark 3.5.5 Result: {'contains(Spark SQL, Spark)': True, 
'typeof(contains(Spark SQL, Spark))': 'boolean', 'typeof(Spark SQL)': 'string', 
'typeof(Spark)': 'string'}
-#query
-#SELECT contains('Spark SQL'::string, 'Spark'::string);
-
-## Original Query: SELECT contains('Spark SQL', null);
-## PySpark 3.5.5 Result: {'contains(Spark SQL, NULL)': None, 
'typeof(contains(Spark SQL, NULL))': 'boolean', 'typeof(Spark SQL)': 'string', 
'typeof(NULL)': 'void'}
-#query
-#SELECT contains('Spark SQL'::string, NULL::void);
-
-## Original Query: SELECT contains(x'537061726b2053514c', x'537061726b');
-## PySpark 3.5.5 Result: {"contains(X'537061726B2053514C', X'537061726B')": 
True, "typeof(contains(X'537061726B2053514C', X'537061726B'))": 'boolean', 
"typeof(X'537061726B2053514C')": 'binary', "typeof(X'537061726B')": 'binary'}
-#query
-#SELECT contains(X'537061726B2053514C'::binary, X'537061726B'::binary);
+# Expected values verified against Apache Spark 4.2.0.
+
+## Examples from Spark's documentation
+
+query B
+SELECT contains('Spark SQL'::string, 'Spark'::string);
+----
+true
+
+query B
+SELECT contains('Spark SQL'::string, 'SPARK'::string);
+----
+false
+
+query B
+SELECT contains('Spark SQL'::string, NULL);
+----
+NULL
+
+query B
+SELECT contains(X'537061726B2053514C', X'537061726B');
+----
+true
+
+## NULL handling
+
+query BBB
+SELECT contains(NULL, 'a'::string), contains(NULL, NULL), 
contains(X'537061726B', NULL);
+----
+NULL NULL NULL
+
+## Empty strings: an empty needle is found everywhere
+
+query BBBBB
+SELECT contains(''::string, ''::string), contains('abc'::string, ''::string), 
contains(''::string, 'a'::string), contains(X'', X''), contains(X'01', X'');
+----
+true true false true true
+
+## Case sensitivity, multi-byte characters and needles longer than the haystack
+
+query BBBBB
+SELECT contains('a'::string, 'A'::string), contains('héllo wörld'::string, 
'ö'::string), contains('日本語'::string, '本'::string), contains('abc'::string, 
'abcd'::string), contains('aaa'::string, 'aa'::string);
+----
+false true true false true
+
+## Binary: byte-level match, including bytes that are not valid UTF-8
+
+query BB
+SELECT contains(X'00FF01', X'FF'), contains(X'00FF01', X'FE');
+----
+true false
+
+query B
+SELECT contains(X'C328', X'28');
+----
+true
+
+## Mixed string and binary: Spark casts the binary to a string
+
+query BB
+SELECT contains(X'537061726B', 'Spa'::string), contains('Spark'::string, 
X'537061');
+----
+true true
+
+## Integer, boolean and date arguments are cast to a string
+
+query BBB
+SELECT contains(12345, 23), contains(12345, '34'::string), contains(true, 
'ru'::string);
+----
+true true true
+
+query B
+SELECT contains(DATE '2024-01-15', '-01-'::string);
+----
+true
+
+## Column inputs
+
+query B
+SELECT contains(s, p) FROM VALUES ('hello world', 'world'), ('hello', ''), 
(NULL, 'a'), ('abc', NULL), ('abc', 'd') AS t(s, p);
+----
+true
+true
+NULL
+NULL
+false
+
+# Column haystack, literal needle
+query B
+SELECT contains(s, 'o') FROM VALUES ('hello'), ('abc'), (NULL), ('') AS t(s);
+----
+true
+false
+NULL
+false
+
+# Literal haystack, column needle
+query B
+SELECT contains('hello world'::string, p) FROM VALUES ('hello'), ('xyz'), 
(NULL), ('') AS t(p);
+----
+true
+false
+NULL
+true
+
+# Binary columns
+query B
+SELECT contains(b, X'FF') FROM VALUES (X'00FF01'), (X'0102'), (NULL), (X'') AS 
t(b);
+----
+true
+false
+NULL
+false
+
+## Errors
+
+statement error
+SELECT contains('abc'::string);
+
+statement error
+SELECT contains(array('a'), 'a'::string);

Review Comment:
   Done in dd706b3. Both now assert the message.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to