This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new fc5888b49b feat(spark): implement Spark `length` function (#17475)
fc5888b49b is described below
commit fc5888b49b9757b20289e479f8b41440f383deb4
Author: Zhen Wang <[email protected]>
AuthorDate: Tue Sep 9 22:26:34 2025 +0800
feat(spark): implement Spark `length` function (#17475)
---
datafusion/spark/src/function/string/length.rs | 282 +++++++++++++++++++++
datafusion/spark/src/function/string/mod.rs | 9 +-
.../test_files/spark/string/char_length.slt | 30 +--
.../test_files/spark/string/character_length.slt | 30 +--
.../sqllogictest/test_files/spark/string/len.slt | 22 +-
.../test_files/spark/string/length.slt | 22 +-
6 files changed, 330 insertions(+), 65 deletions(-)
diff --git a/datafusion/spark/src/function/string/length.rs
b/datafusion/spark/src/function/string/length.rs
new file mode 100644
index 0000000000..1fa54d000e
--- /dev/null
+++ b/datafusion/spark/src/function/string/length.rs
@@ -0,0 +1,282 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow::array::{
+ Array, ArrayRef, AsArray, BinaryArrayType, PrimitiveArray, StringArrayType,
+};
+use arrow::datatypes::{DataType, Int32Type};
+use datafusion_common::exec_err;
+use datafusion_expr::{
+ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
+use datafusion_functions::utils::make_scalar_function;
+use std::sync::Arc;
+
+/// Spark-compatible `length` expression
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#length>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkLengthFunc {
+ signature: Signature,
+ aliases: Vec<String>,
+}
+
+impl Default for SparkLengthFunc {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkLengthFunc {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::uniform(
+ 1,
+ vec![
+ DataType::Utf8View,
+ DataType::Utf8,
+ DataType::LargeUtf8,
+ DataType::Binary,
+ DataType::LargeBinary,
+ DataType::BinaryView,
+ ],
+ Volatility::Immutable,
+ ),
+ aliases: vec![
+ String::from("character_length"),
+ String::from("char_length"),
+ String::from("len"),
+ ],
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkLengthFunc {
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+
+ fn name(&self) -> &str {
+ "length"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _args: &[DataType]) ->
datafusion_common::Result<DataType> {
+ // spark length always returns Int32
+ Ok(DataType::Int32)
+ }
+
+ fn invoke_with_args(
+ &self,
+ args: ScalarFunctionArgs,
+ ) -> datafusion_common::Result<ColumnarValue> {
+ make_scalar_function(spark_length, vec![])(&args.args)
+ }
+
+ fn aliases(&self) -> &[String] {
+ &self.aliases
+ }
+}
+
+fn spark_length(args: &[ArrayRef]) -> datafusion_common::Result<ArrayRef> {
+ match args[0].data_type() {
+ DataType::Utf8 => {
+ let string_array = args[0].as_string::<i32>();
+ character_length::<_>(string_array)
+ }
+ DataType::LargeUtf8 => {
+ let string_array = args[0].as_string::<i64>();
+ character_length::<_>(string_array)
+ }
+ DataType::Utf8View => {
+ let string_array = args[0].as_string_view();
+ character_length::<_>(string_array)
+ }
+ DataType::Binary => {
+ let binary_array = args[0].as_binary::<i32>();
+ byte_length::<_>(binary_array)
+ }
+ DataType::LargeBinary => {
+ let binary_array = args[0].as_binary::<i64>();
+ byte_length::<_>(binary_array)
+ }
+ DataType::BinaryView => {
+ let binary_array = args[0].as_binary_view();
+ byte_length::<_>(binary_array)
+ }
+ other => exec_err!("Unsupported data type {other:?} for function
`length`"),
+ }
+}
+
+fn character_length<'a, V>(array: V) -> datafusion_common::Result<ArrayRef>
+where
+ V: StringArrayType<'a>,
+{
+ // String characters are variable length encoded in UTF-8, counting the
+ // number of chars requires expensive decoding, however checking if the
+ // string is ASCII only is relatively cheap.
+ // If strings are ASCII only, count bytes instead.
+ let is_array_ascii_only = array.is_ascii();
+ let nulls = array.nulls().cloned();
+ let array = {
+ if is_array_ascii_only {
+ let values: Vec<_> = (0..array.len())
+ .map(|i| {
+ // Safety: we are iterating with array.len() so the index
is always valid
+ let value = unsafe { array.value_unchecked(i) };
+ value.len() as i32
+ })
+ .collect();
+ PrimitiveArray::<Int32Type>::new(values.into(), nulls)
+ } else {
+ let values: Vec<_> = (0..array.len())
+ .map(|i| {
+ // Safety: we are iterating with array.len() so the index
is always valid
+ if array.is_null(i) {
+ i32::default()
+ } else {
+ let value = unsafe { array.value_unchecked(i) };
+ if value.is_empty() {
+ i32::default()
+ } else if value.is_ascii() {
+ value.len() as i32
+ } else {
+ value.chars().count() as i32
+ }
+ }
+ })
+ .collect();
+ PrimitiveArray::<Int32Type>::new(values.into(), nulls)
+ }
+ };
+
+ Ok(Arc::new(array))
+}
+
+fn byte_length<'a, V>(array: V) -> datafusion_common::Result<ArrayRef>
+where
+ V: BinaryArrayType<'a>,
+{
+ let nulls = array.nulls().cloned();
+ let values: Vec<_> = (0..array.len())
+ .map(|i| {
+ // Safety: we are iterating with array.len() so the index is
always valid
+ let value = unsafe { array.value_unchecked(i) };
+ value.len() as i32
+ })
+ .collect();
+ Ok(Arc::new(PrimitiveArray::<Int32Type>::new(
+ values.into(),
+ nulls,
+ )))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::function::utils::test::test_scalar_function;
+ use arrow::array::{Array, Int32Array};
+ use arrow::datatypes::DataType::Int32;
+ use datafusion_common::{Result, ScalarValue};
+ use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
+
+ macro_rules! test_spark_length_string {
+ ($INPUT:expr, $EXPECTED:expr) => {
+ test_scalar_function!(
+ SparkLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::Utf8($INPUT))],
+ $EXPECTED,
+ i32,
+ Int32,
+ Int32Array
+ );
+
+ test_scalar_function!(
+ SparkLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::LargeUtf8($INPUT))],
+ $EXPECTED,
+ i32,
+ Int32,
+ Int32Array
+ );
+
+ test_scalar_function!(
+ SparkLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::Utf8View($INPUT))],
+ $EXPECTED,
+ i32,
+ Int32,
+ Int32Array
+ );
+ };
+ }
+
+ macro_rules! test_spark_length_binary {
+ ($INPUT:expr, $EXPECTED:expr) => {
+ test_scalar_function!(
+ SparkLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::Binary($INPUT))],
+ $EXPECTED,
+ i32,
+ Int32,
+ Int32Array
+ );
+
+ test_scalar_function!(
+ SparkLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::LargeBinary($INPUT))],
+ $EXPECTED,
+ i32,
+ Int32,
+ Int32Array
+ );
+
+ test_scalar_function!(
+ SparkLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::BinaryView($INPUT))],
+ $EXPECTED,
+ i32,
+ Int32,
+ Int32Array
+ );
+ };
+ }
+
+ #[test]
+ fn test_functions() -> Result<()> {
+ test_spark_length_string!(Some(String::from("chars")), Ok(Some(5)));
+ test_spark_length_string!(Some(String::from("josé")), Ok(Some(4)));
+ // test long strings (more than 12 bytes for StringView)
+ test_spark_length_string!(Some(String::from("joséjoséjoséjosé")),
Ok(Some(16)));
+ test_spark_length_string!(Some(String::from("")), Ok(Some(0)));
+ test_spark_length_string!(None, Ok(None));
+
+ test_spark_length_binary!(Some(String::from("chars").into_bytes()),
Ok(Some(5)));
+ test_spark_length_binary!(Some(String::from("josé").into_bytes()),
Ok(Some(5)));
+ // test long strings (more than 12 bytes for BinaryView)
+ test_spark_length_binary!(
+ Some(String::from("joséjoséjoséjosé").into_bytes()),
+ Ok(Some(20))
+ );
+ test_spark_length_binary!(Some(String::from("").into_bytes()),
Ok(Some(0)));
+ test_spark_length_binary!(None, Ok(None));
+
+ Ok(())
+ }
+}
diff --git a/datafusion/spark/src/function/string/mod.rs
b/datafusion/spark/src/function/string/mod.rs
index e83b696bc1..48bf77c1ca 100644
--- a/datafusion/spark/src/function/string/mod.rs
+++ b/datafusion/spark/src/function/string/mod.rs
@@ -18,6 +18,7 @@
pub mod ascii;
pub mod char;
pub mod ilike;
+pub mod length;
pub mod like;
pub mod luhn_check;
@@ -28,6 +29,7 @@ use std::sync::Arc;
make_udf_function!(ascii::SparkAscii, ascii);
make_udf_function!(char::CharFunc, char);
make_udf_function!(ilike::SparkILike, ilike);
+make_udf_function!(length::SparkLengthFunc, length);
make_udf_function!(like::SparkLike, like);
make_udf_function!(luhn_check::SparkLuhnCheck, luhn_check);
@@ -49,6 +51,11 @@ pub mod expr_fn {
"Returns true if str matches pattern (case insensitive).",
str pattern
));
+ export_functions!((
+ length,
+ "Returns the character length of string data or number of bytes of
binary data. The length of string data includes the trailing spaces. The length
of binary data includes binary zeros.",
+ arg1
+ ));
export_functions!((
like,
"Returns true if str matches pattern (case sensitive).",
@@ -62,5 +69,5 @@ pub mod expr_fn {
}
pub fn functions() -> Vec<Arc<ScalarUDF>> {
- vec![ascii(), char(), ilike(), like(), luhn_check()]
+ vec![ascii(), char(), ilike(), length(), like(), luhn_check()]
}
diff --git a/datafusion/sqllogictest/test_files/spark/string/char_length.slt
b/datafusion/sqllogictest/test_files/spark/string/char_length.slt
index e13c325b6f..d9f86d45d2 100644
--- a/datafusion/sqllogictest/test_files/spark/string/char_length.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/char_length.slt
@@ -15,23 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-# This file was originally created by a porting script from:
-#
https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
-# This file is part of the implementation of the datafusion-spark function
library.
-# For more information, please see:
-# https://github.com/apache/datafusion/issues/15914
+query I
+SELECT CHAR_LENGTH('Spark SQL ');
+----
+10
-## Original Query: SELECT CHAR_LENGTH('Spark SQL ');
-## PySpark 3.5.5 Result: {'char_length(Spark SQL )': 10,
'typeof(char_length(Spark SQL ))': 'int', 'typeof(Spark SQL )': 'string'}
-#query
-#SELECT CHAR_LENGTH('Spark SQL '::string);
+query I
+SELECT char_length('Spark SQL ');
+----
+10
-## Original Query: SELECT char_length('Spark SQL ');
-## PySpark 3.5.5 Result: {'char_length(Spark SQL )': 10,
'typeof(char_length(Spark SQL ))': 'int', 'typeof(Spark SQL )': 'string'}
-#query
-#SELECT char_length('Spark SQL '::string);
-
-## Original Query: SELECT char_length(x'537061726b2053514c');
-## PySpark 3.5.5 Result: {"char_length(X'537061726B2053514C')": 9,
"typeof(char_length(X'537061726B2053514C'))": 'int',
"typeof(X'537061726B2053514C')": 'binary'}
-#query
-#SELECT char_length(X'537061726B2053514C'::binary);
+query I
+SELECT char_length(x'537061726b2053514c');
+----
+9
diff --git
a/datafusion/sqllogictest/test_files/spark/string/character_length.slt
b/datafusion/sqllogictest/test_files/spark/string/character_length.slt
index 24ca2830fb..644741416e 100644
--- a/datafusion/sqllogictest/test_files/spark/string/character_length.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/character_length.slt
@@ -15,23 +15,17 @@
# specific language governing permissions and limitations
# under the License.
-# This file was originally created by a porting script from:
-#
https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
-# This file is part of the implementation of the datafusion-spark function
library.
-# For more information, please see:
-# https://github.com/apache/datafusion/issues/15914
+query I
+SELECT CHARACTER_LENGTH('Spark SQL ');
+----
+10
-## Original Query: SELECT CHARACTER_LENGTH('Spark SQL ');
-## PySpark 3.5.5 Result: {'character_length(Spark SQL )': 10,
'typeof(character_length(Spark SQL ))': 'int', 'typeof(Spark SQL )': 'string'}
-#query
-#SELECT CHARACTER_LENGTH('Spark SQL '::string);
+query I
+SELECT character_length('Spark SQL ');
+----
+10
-## Original Query: SELECT character_length('Spark SQL ');
-## PySpark 3.5.5 Result: {'character_length(Spark SQL )': 10,
'typeof(character_length(Spark SQL ))': 'int', 'typeof(Spark SQL )': 'string'}
-#query
-#SELECT character_length('Spark SQL '::string);
-
-## Original Query: SELECT character_length(x'537061726b2053514c');
-## PySpark 3.5.5 Result: {"character_length(X'537061726B2053514C')": 9,
"typeof(character_length(X'537061726B2053514C'))": 'int',
"typeof(X'537061726B2053514C')": 'binary'}
-#query
-#SELECT character_length(X'537061726B2053514C'::binary);
+query I
+SELECT character_length(x'537061726b2053514c');
+----
+9
diff --git a/datafusion/sqllogictest/test_files/spark/string/len.slt
b/datafusion/sqllogictest/test_files/spark/string/len.slt
index 8f0be80276..3dd359f4a7 100644
--- a/datafusion/sqllogictest/test_files/spark/string/len.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/len.slt
@@ -15,18 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-# This file was originally created by a porting script from:
-#
https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
-# This file is part of the implementation of the datafusion-spark function
library.
-# For more information, please see:
-# https://github.com/apache/datafusion/issues/15914
+query I
+SELECT len('Spark SQL ');
+----
+10
-## Original Query: SELECT len('Spark SQL ');
-## PySpark 3.5.5 Result: {'len(Spark SQL )': 10, 'typeof(len(Spark SQL ))':
'int', 'typeof(Spark SQL )': 'string'}
-#query
-#SELECT len('Spark SQL '::string);
-
-## Original Query: SELECT len(x'537061726b2053514c');
-## PySpark 3.5.5 Result: {"len(X'537061726B2053514C')": 9,
"typeof(len(X'537061726B2053514C'))": 'int', "typeof(X'537061726B2053514C')":
'binary'}
-#query
-#SELECT len(X'537061726B2053514C'::binary);
+query I
+SELECT len(x'537061726b2053514c');
+----
+9
diff --git a/datafusion/sqllogictest/test_files/spark/string/length.slt
b/datafusion/sqllogictest/test_files/spark/string/length.slt
index 57efc230f0..be453b0c4c 100644
--- a/datafusion/sqllogictest/test_files/spark/string/length.slt
+++ b/datafusion/sqllogictest/test_files/spark/string/length.slt
@@ -15,18 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-# This file was originally created by a porting script from:
-#
https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
-# This file is part of the implementation of the datafusion-spark function
library.
-# For more information, please see:
-# https://github.com/apache/datafusion/issues/15914
+query I
+SELECT length('Spark SQL ');
+----
+10
-## Original Query: SELECT length('Spark SQL ');
-## PySpark 3.5.5 Result: {'length(Spark SQL )': 10, 'typeof(length(Spark SQL
))': 'int', 'typeof(Spark SQL )': 'string'}
-#query
-#SELECT length('Spark SQL '::string);
-
-## Original Query: SELECT length(x'537061726b2053514c');
-## PySpark 3.5.5 Result: {"length(X'537061726B2053514C')": 9,
"typeof(length(X'537061726B2053514C'))": 'int',
"typeof(X'537061726B2053514C')": 'binary'}
-#query
-#SELECT length(X'537061726B2053514C'::binary);
+query I
+SELECT length(x'537061726b2053514c');
+----
+9
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
