andygrove commented on code in PR #328:
URL: https://github.com/apache/datafusion-spark/pull/328#discussion_r4607469687
##########
crates/spark/src/function/string/contains.rs:
##########
@@ -0,0 +1,651 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, BooleanArray, Scalar};
+use arrow::compute::kernels::cast::cast;
+use arrow::compute::kernels::comparison::contains as arrow_contains;
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `contains` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#contains>
+#[user_doc(
+ doc_section(label = "String Functions"),
+ description = "Returns true if `right` is found inside `left`.
DataFusion's own `contains` takes only \
+ strings; this one also accepts binary. When both arguments are binary
the match is on bytes. \
+ When one argument is a string and the other is binary, both are
compared as bytes, which is \
+ the same as Spark casting the binary to a string. Integer, boolean and
date arguments are \
+ cast to a string first. The match is case-sensitive, and an empty
`right` is found in every \
+ value, including an empty `left`. Returns NULL if either input is
NULL.",
+ syntax_example = "contains(left, right)",
+ sql_example = r#"```sql
+> SELECT contains('Spark SQL', 'Spark');
+true
+> SELECT contains('Spark SQL', 'SPARK');
+false
+> SELECT contains(x'537061726b2053514c', x'537061726b');
+true
+```"#,
+ argument(
+ name = "left",
+ description = "String or binary expression to search in. NULL returns
NULL."
+ ),
+ argument(
+ name = "right",
+ description = "String or binary expression to search for. NULL returns
NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkContains {
+ signature: Signature,
+}
+
+impl Default for SparkContains {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkContains {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+/// How an argument type takes part in `contains`.
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum Kind {
+ Null,
+ String,
+ Binary,
+}
+
+/// Classifies an argument type, returning the type it is coerced to before
+/// unification. Types Spark implicitly casts to a string and whose Arrow
string
+/// form matches Spark's become `Utf8`.
+fn classify(name: &str, dt: &DataType) -> Result<(Kind, DataType)> {
+ match dt {
+ DataType::Null => Ok((Kind::Null, DataType::Null)),
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => {
+ Ok((Kind::String, dt.clone()))
+ }
+ DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
+ Ok((Kind::Binary, dt.clone()))
+ }
+ DataType::FixedSizeBinary(_) => Ok((Kind::Binary, DataType::Binary)),
+ DataType::Dictionary(_, value) => classify(name, value),
+ DataType::Int8
+ | DataType::Int16
+ | DataType::Int32
+ | DataType::Int64
+ | DataType::UInt8
+ | DataType::UInt16
+ | DataType::UInt32
+ | DataType::UInt64
+ | DataType::Boolean
+ | DataType::Date32 => Ok((Kind::String, DataType::Utf8)),
+ other => plan_err!(
+ "Unsupported Data Type: Spark `{name}` function expects STRING or
BINARY arguments, got {other}"
+ ),
Review Comment:
Fixed in 928fda1. The records use `1.5::double`, and the decimal case is
recorded with ANSI mode off (`true`) and on (`false`), then ANSI mode is turned
back off. I checked the expected values with PySpark 4.2.0 in both modes. With
the records uncommented, each one now fails only with the #347 type error, not
a parse error.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]