This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23930-21ad1897826a2dd00f31dd8bb574735ca70143d9 in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 9912d1e4e94872f849bbe5ab87bf139710ec558a Author: linfeng <[email protected]> AuthorDate: Mon Aug 3 12:39:05 2026 +0800 perf: preserve dictionary encoding for `character_length`, `initcap`, and `reverse` (#23930) ## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Follow-up to #22905 - Related to #19458 and #20935 ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> Previously, coercion materialized dictionary-encoded inputs for `character_length`, `initcap`, and `reverse`. This lost the encoding and evaluated the function for every row instead of once per dictionary value entry. This PR extends dictionary preservation to these functions. `character_length` and `reverse` now use `Coercible` signatures to explicitly model string inputs and binary-to-string coercion while preserving dictionary encoding. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> - Preserve dictionary encoding for `character_length`, `initcap`, and `reverse`. - Evaluate only dictionary values while reusing the original keys. - Migrate `character_length` and `reverse` from `Uniform` to `Coercible`. - Add tests and benchmarks. ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Yes, covered by SQL logic tests. ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> Yes. These functions now preserve dictionary encoding in their output. `character_length` and `reverse` now accept logical string and binary inputs instead of implicitly converting unrelated types to strings. ## Benchmark ``` group branch main ----- ------ ---- dictionary_encoding/string/cardinality_10/character_length 1.00 218.5±32.62ns ? ?/sec 31.04 6.8±0.07µs ? ?/sec dictionary_encoding/string/cardinality_10/initcap 1.00 292.2±4.75ns ? ?/sec 359.88 105.2±0.91µs ? ?/sec dictionary_encoding/string/cardinality_10/reverse 1.00 501.6±15.71ns ? ?/sec 140.60 70.5±1.05µs ? ?/sec dictionary_encoding/string/cardinality_100/character_length 1.00 307.5±24.93ns ? ?/sec 21.85 6.7±0.09µs ? ?/sec dictionary_encoding/string/cardinality_100/initcap 1.00 1486.1±65.36ns ? ?/sec 70.56 104.9±1.57µs ? ?/sec dictionary_encoding/string/cardinality_100/reverse 1.00 1410.9±52.69ns ? ?/sec 49.28 69.5±0.68µs ? ?/sec dictionary_encoding/string/cardinality_1000/character_length 1.00 1007.4±93.42ns ? ?/sec 6.64 6.7±0.07µs ? ?/sec dictionary_encoding/string/cardinality_1000/initcap 1.00 12.8±0.17µs ? ?/sec 8.41 107.5±8.55µs ? ?/sec dictionary_encoding/string/cardinality_1000/reverse 1.00 10.5±0.24µs ? ?/sec 6.56 69.1±0.58µs ? ?/sec dictionary_encoding/string/cardinality_8192/character_length 1.00 6.6±0.09µs ? ?/sec 1.02 6.7±0.18µs ? ?/sec dictionary_encoding/string/cardinality_8192/initcap 1.00 104.2±1.36µs ? ?/sec 1.00 104.2±1.63µs ? ?/sec dictionary_encoding/string/cardinality_8192/reverse 1.24 85.4±2.99µs ? ?/sec 1.00 69.0±0.58µs ? ?/sec ``` --- datafusion/functions/Cargo.toml | 2 +- .../functions/benches/dictionary_encoding.rs | 8 +- .../functions/src/unicode/character_length.rs | 31 ++++-- datafusion/functions/src/unicode/initcap.rs | 115 ++++++++++--------- datafusion/functions/src/unicode/reverse.rs | 35 +++--- datafusion/sqllogictest/test_files/binary.slt | 18 ++- datafusion/sqllogictest/test_files/functions.slt | 123 ++++++++++++++++++++- .../test_files/string/string_literal.slt | 5 + .../test_files/string/string_query.slt.part | 4 +- 9 files changed, 258 insertions(+), 83 deletions(-) diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml index 3c88c29056..a170e9f07c 100644 --- a/datafusion/functions/Cargo.toml +++ b/datafusion/functions/Cargo.toml @@ -407,4 +407,4 @@ required-features = ["math_expressions"] [[bench]] harness = false name = "dictionary_encoding" -required-features = ["string_expressions"] +required-features = ["string_expressions", "unicode_expressions"] diff --git a/datafusion/functions/benches/dictionary_encoding.rs b/datafusion/functions/benches/dictionary_encoding.rs index 3afc1d5eb4..05541fc10e 100644 --- a/datafusion/functions/benches/dictionary_encoding.rs +++ b/datafusion/functions/benches/dictionary_encoding.rs @@ -42,10 +42,16 @@ fn create_string_dictionary(cardinality: usize) -> ArrayRef { } fn benchmark_dictionary_string_udfs(c: &mut Criterion) { - let udfs: [(&str, Arc<ScalarUDF>); 3] = [ + let udfs: [(&str, Arc<ScalarUDF>); 6] = [ ("ascii", datafusion_functions::string::ascii()), ("bit_length", datafusion_functions::string::bit_length()), + ( + "character_length", + datafusion_functions::unicode::character_length(), + ), + ("initcap", datafusion_functions::unicode::initcap()), ("octet_length", datafusion_functions::string::octet_length()), + ("reverse", datafusion_functions::unicode::reverse()), ]; let config_options = Arc::new(ConfigOptions::default()); diff --git a/datafusion/functions/src/unicode/character_length.rs b/datafusion/functions/src/unicode/character_length.rs index 465b15ace1..9f0d952a02 100644 --- a/datafusion/functions/src/unicode/character_length.rs +++ b/datafusion/functions/src/unicode/character_length.rs @@ -15,16 +15,19 @@ // specific language governing permissions and limitations // under the License. -use crate::utils::{make_scalar_function, utf8_to_int_type}; +use crate::utils::{ + make_scalar_function, transform_leaf_type_preserving_encoding, utf8_to_int_type, +}; use arrow::array::{ Array, ArrayRef, ArrowPrimitiveType, AsArray, OffsetSizeTrait, PrimitiveArray, StringArrayType, }; use arrow::datatypes::{ArrowNativeType, DataType, Int32Type, Int64Type}; use datafusion_common::Result; +use datafusion_common::types::{NativeType, logical_string}; use datafusion_expr::{ - ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; use std::sync::Arc; @@ -59,11 +62,16 @@ impl Default for CharacterLengthFunc { impl CharacterLengthFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::uniform( - 1, - vec![Utf8, LargeUtf8, Utf8View], + signature: Signature::coercible( + vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Any], + NativeType::String, + ) + .with_encoding_preservation(EncodingPreservation::dictionary()), + ], Volatility::Immutable, ), aliases: vec![String::from("length"), String::from("char_length")], @@ -81,7 +89,9 @@ impl ScalarUDFImpl for CharacterLengthFunc { } fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> { - utf8_to_int_type(&arg_types[0], "character_length") + transform_leaf_type_preserving_encoding(&arg_types[0], &|data_type| { + utf8_to_int_type(data_type, "character_length") + }) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> { @@ -114,6 +124,11 @@ fn character_length(args: &[ArrayRef]) -> Result<ArrayRef> { let string_array = args[0].as_string_view(); character_length_general::<Int32Type, _>(&string_array) } + DataType::Dictionary(_, _) => { + let dictionary = args[0].as_any_dictionary(); + let converted = character_length(&[Arc::clone(dictionary.values())])?; + Ok(dictionary.with_values(converted)) + } _ => unreachable!("CharacterLengthFunc"), } } diff --git a/datafusion/functions/src/unicode/initcap.rs b/datafusion/functions/src/unicode/initcap.rs index 8981d59aec..0332ab5d44 100644 --- a/datafusion/functions/src/unicode/initcap.rs +++ b/datafusion/functions/src/unicode/initcap.rs @@ -17,18 +17,17 @@ use std::sync::Arc; -use arrow::array::{Array, ArrayRef, GenericStringArray, OffsetSizeTrait}; +use arrow::array::{Array, ArrayRef, AsArray, GenericStringArray, OffsetSizeTrait}; use arrow::buffer::Buffer; use arrow::datatypes::DataType; use crate::strings::{GenericStringArrayBuilder, StringViewArrayBuilder}; -use crate::utils::{make_scalar_function, utf8_to_str_type}; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; use datafusion_common::types::logical_string; use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{ - Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; @@ -64,9 +63,10 @@ impl InitcapFunc { pub fn new() -> Self { Self { signature: Signature::coercible( - vec![Coercion::new_exact(TypeSignatureClass::Native( - logical_string(), - ))], + vec![ + Coercion::new_exact(TypeSignatureClass::Native(logical_string())) + .with_encoding_preservation(EncodingPreservation::dictionary()), + ], Volatility::Immutable, ), } @@ -83,54 +83,16 @@ impl ScalarUDFImpl for InitcapFunc { } fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> { - if let DataType::Utf8View = arg_types[0] { - Ok(DataType::Utf8View) - } else { - utf8_to_str_type(&arg_types[0], "initcap") - } + Ok(arg_types[0].clone()) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> { - let arg = &args.args[0]; - - // Scalar fast path - handle directly without array conversion - if let ColumnarValue::Scalar(scalar) = arg { - return match scalar { - ScalarValue::Utf8(None) - | ScalarValue::LargeUtf8(None) - | ScalarValue::Utf8View(None) => Ok(arg.clone()), - ScalarValue::Utf8(Some(s)) => { - let mut result = String::new(); - initcap_string(s, &mut result); - Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(result)))) - } - ScalarValue::LargeUtf8(Some(s)) => { - let mut result = String::new(); - initcap_string(s, &mut result); - Ok(ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(result)))) - } - ScalarValue::Utf8View(Some(s)) => { - let mut result = String::new(); - initcap_string(s, &mut result); - Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some(result)))) - } - other => { - exec_err!( - "Unsupported data type {:?} for function `initcap`", - other.data_type() - ) - } - }; - } - - // Array path - let args = &args.args; - match args[0].data_type() { - DataType::Utf8 => make_scalar_function(initcap::<i32>, vec![])(args), - DataType::LargeUtf8 => make_scalar_function(initcap::<i64>, vec![])(args), - DataType::Utf8View => make_scalar_function(initcap_utf8view, vec![])(args), - other => { - exec_err!("Unsupported data type {other:?} for function `initcap`") + match &args.args[0] { + ColumnarValue::Scalar(scalar) => { + Ok(ColumnarValue::Scalar(initcap_scalar(scalar)?)) + } + ColumnarValue::Array(array) => { + Ok(ColumnarValue::Array(initcap_array(array)?)) } } } @@ -140,6 +102,55 @@ impl ScalarUDFImpl for InitcapFunc { } } +fn initcap_scalar(scalar: &ScalarValue) -> Result<ScalarValue> { + match scalar { + ScalarValue::Utf8(None) + | ScalarValue::LargeUtf8(None) + | ScalarValue::Utf8View(None) => Ok(scalar.clone()), + ScalarValue::Utf8(Some(s)) => { + let mut result = String::new(); + initcap_string(s, &mut result); + Ok(ScalarValue::Utf8(Some(result))) + } + ScalarValue::LargeUtf8(Some(s)) => { + let mut result = String::new(); + initcap_string(s, &mut result); + Ok(ScalarValue::LargeUtf8(Some(result))) + } + ScalarValue::Utf8View(Some(s)) => { + let mut result = String::new(); + initcap_string(s, &mut result); + Ok(ScalarValue::Utf8View(Some(result))) + } + ScalarValue::Dictionary(key_type, value) => Ok(ScalarValue::Dictionary( + key_type.clone(), + Box::new(initcap_scalar(value)?), + )), + other => { + exec_err!( + "Unsupported data type {:?} for function `initcap`", + other.data_type() + ) + } + } +} + +fn initcap_array(array: &ArrayRef) -> Result<ArrayRef> { + match array.data_type() { + DataType::Utf8 => initcap::<i32>(&[Arc::clone(array)]), + DataType::LargeUtf8 => initcap::<i64>(&[Arc::clone(array)]), + DataType::Utf8View => initcap_utf8view(&[Arc::clone(array)]), + DataType::Dictionary(_, _) => { + let dictionary = array.as_any_dictionary(); + let converted = initcap_array(dictionary.values())?; + Ok(dictionary.with_values(converted)) + } + other => { + exec_err!("Unsupported data type {other:?} for function `initcap`") + } + } +} + /// Converts the first letter of each word to uppercase and the rest to /// lowercase. Words are sequences of alphanumeric characters separated by /// non-alphanumeric characters. diff --git a/datafusion/functions/src/unicode/reverse.rs b/datafusion/functions/src/unicode/reverse.rs index 813dcb5f50..9dfc25fbdf 100644 --- a/datafusion/functions/src/unicode/reverse.rs +++ b/datafusion/functions/src/unicode/reverse.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + use crate::strings::{ BulkNullStringArrayBuilder, GenericStringArrayBuilder, StringViewArrayBuilder, }; @@ -22,10 +24,11 @@ use crate::utils::make_scalar_function; use DataType::{LargeUtf8, Utf8, Utf8View}; use arrow::array::{Array, ArrayRef, AsArray, StringArrayType}; use arrow::datatypes::DataType; -use datafusion_common::{Result, exec_err}; +use datafusion_common::Result; +use datafusion_common::types::{NativeType, logical_string}; use datafusion_expr::{ - ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; @@ -56,11 +59,16 @@ impl Default for ReverseFunc { impl ReverseFunc { pub fn new() -> Self { - use DataType::*; Self { - signature: Signature::uniform( - 1, - vec![Utf8View, Utf8, LargeUtf8], + signature: Signature::coercible( + vec![ + Coercion::new_implicit( + TypeSignatureClass::Native(logical_string()), + vec![TypeSignatureClass::Any], + NativeType::String, + ) + .with_encoding_preservation(EncodingPreservation::dictionary()), + ], Volatility::Immutable, ), } @@ -81,13 +89,7 @@ impl ScalarUDFImpl for ReverseFunc { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> { - let args = &args.args; - match args[0].data_type() { - Utf8 | Utf8View | LargeUtf8 => make_scalar_function(reverse, vec![])(args), - other => { - exec_err!("Unsupported data type {other:?} for function reverse") - } - } + make_scalar_function(reverse, vec![])(&args.args) } fn documentation(&self) -> Option<&Documentation> { @@ -113,6 +115,11 @@ fn reverse(args: &[ArrayRef]) -> Result<ArrayRef> { &args[0].as_string_view(), StringViewArrayBuilder::with_capacity(len), ), + DataType::Dictionary(_, _) => { + let dictionary = args[0].as_any_dictionary(); + let converted = reverse(&[Arc::clone(dictionary.values())])?; + Ok(dictionary.with_values(converted)) + } _ => unreachable!( "Reverse can only be applied to Utf8View, Utf8 and LargeUtf8 types" ), diff --git a/datafusion/sqllogictest/test_files/binary.slt b/datafusion/sqllogictest/test_files/binary.slt index 94c1365cb9..91a9449343 100644 --- a/datafusion/sqllogictest/test_files/binary.slt +++ b/datafusion/sqllogictest/test_files/binary.slt @@ -281,7 +281,7 @@ SELECT cast(binary as varchar) as str, character_length(binary) as binary_len, cast(largebinary as varchar) as large_str, - character_length(binary) as largebinary_len + character_length(largebinary) as largebinary_len from t; ---- Foo 3 Foo 3 @@ -298,6 +298,20 @@ SELECT character_length(X'20'); query error Encountered non UTF\-8 data: invalid utf\-8 sequence of 1 bytes from index 0 SELECT character_length(X'c328'); +# reverse function +query TTTT +SELECT + cast(binary as varchar) as str, + reverse(binary) as binary_reversed, + cast(largebinary as varchar) as large_str, + reverse(largebinary) as largebinary_reversed +from t; +---- +Foo ooF Foo ooF +NULL NULL NULL NULL +Bar raB Bar raB +FooBar raBooF FooBar raBooF + # regexp_replace query TTTT SELECT @@ -363,4 +377,4 @@ hellohello query T SELECT 'hello' || arrow_cast(arrow_cast('hello', 'Binary'), 'BinaryView'); ---- -hellohello \ No newline at end of file +hellohello diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt index 78045936a1..78bdeb3e15 100644 --- a/datafusion/sqllogictest/test_files/functions.slt +++ b/datafusion/sqllogictest/test_files/functions.slt @@ -68,7 +68,7 @@ SELECT length('') ---- 0 -query I +query ? SELECT length(arrow_cast('', 'Dictionary(Int32, Utf8)')) ---- 0 @@ -83,7 +83,7 @@ SELECT length('josé') ---- 4 -query I +query ? SELECT length(arrow_cast('josé', 'Dictionary(Int32, Utf8)')) ---- 4 @@ -507,6 +507,84 @@ SELECT initcap(arrow_cast('foo', 'Dictionary(Int32, Utf8)')) ---- Foo +query TTTT +SELECT initcap(arrow_cast('foo BAR', 'Dictionary(Int32, LargeUtf8)')), + arrow_typeof(initcap(arrow_cast('foo BAR', 'Dictionary(Int32, LargeUtf8)'))), + initcap(arrow_cast( + arrow_cast('foo BAR', 'Dictionary(Int32, Utf8)'), + 'Dictionary(Int32, Utf8View)' + )), + arrow_typeof(initcap(arrow_cast( + arrow_cast('foo BAR', 'Dictionary(Int32, Utf8)'), + 'Dictionary(Int32, Utf8View)' + ))) +---- +Foo Bar Dictionary(Int32, LargeUtf8) Foo Bar Dictionary(Int32, Utf8View) + +query ?T +SELECT initcap(arrow_cast( + arrow_cast('foo BAR', 'Dictionary(UInt32, Utf8)'), + 'Dictionary(Int32, Dictionary(UInt32, Utf8))' + )), + arrow_typeof(initcap(arrow_cast( + arrow_cast('foo BAR', 'Dictionary(UInt32, Utf8)'), + 'Dictionary(Int32, Dictionary(UInt32, Utf8))' + ))) +---- +Foo Bar Dictionary(Int32, Dictionary(UInt32, Utf8)) + +statement ok +CREATE TABLE unicode_dictionary_test AS +SELECT column1 AS id, + arrow_cast(column2, 'Dictionary(Int32, Utf8)') AS dict_col, + arrow_cast( + arrow_cast(column2, 'Dictionary(UInt32, Utf8)'), + 'Dictionary(Int32, Dictionary(UInt32, Utf8))' + ) AS nested_dict_col +FROM (VALUES +(1, 'foo BAR'), +(2, 'éclair CAFÉ'), +(3, NULL)); + +query T?TT +SELECT initcap(dict_col), initcap(nested_dict_col), + arrow_typeof(initcap(dict_col)), + arrow_typeof(initcap(nested_dict_col)) +FROM unicode_dictionary_test +ORDER BY id +---- +Foo Bar Foo Bar Dictionary(Int32, Utf8) Dictionary(Int32, Dictionary(UInt32, Utf8)) +Éclair Café Éclair Café Dictionary(Int32, Utf8) Dictionary(Int32, Dictionary(UInt32, Utf8)) +NULL NULL Dictionary(Int32, Utf8) Dictionary(Int32, Dictionary(UInt32, Utf8)) + +query TTTT +SELECT reverse(arrow_cast('foo BAR', 'Dictionary(Int32, LargeUtf8)')), + arrow_typeof(reverse(arrow_cast('foo BAR', 'Dictionary(Int32, LargeUtf8)'))), + reverse(arrow_cast( + arrow_cast('foo BAR', 'Dictionary(Int32, Utf8)'), + 'Dictionary(Int32, Utf8View)' + )), + arrow_typeof(reverse(arrow_cast( + arrow_cast('foo BAR', 'Dictionary(Int32, Utf8)'), + 'Dictionary(Int32, Utf8View)' + ))) +---- +RAB oof Dictionary(Int32, LargeUtf8) RAB oof Dictionary(Int32, Utf8View) + +query T?TT +SELECT reverse(dict_col), reverse(nested_dict_col), + arrow_typeof(reverse(dict_col)), + arrow_typeof(reverse(nested_dict_col)) +FROM unicode_dictionary_test +ORDER BY id +---- +RAB oof RAB oof Dictionary(Int32, Utf8) Dictionary(Int32, Dictionary(UInt32, Utf8)) +ÉFAC rialcé ÉFAC rialcé Dictionary(Int32, Utf8) Dictionary(Int32, Dictionary(UInt32, Utf8)) +NULL NULL Dictionary(Int32, Utf8) Dictionary(Int32, Dictionary(UInt32, Utf8)) + +statement ok +DROP TABLE unicode_dictionary_test + query ? SELECT ascii(arrow_cast('é', 'Dictionary(Int32, Utf8)')) ---- @@ -688,11 +766,39 @@ SELECT character_length('foo') ---- 3 -query I +query ? SELECT character_length(arrow_cast('foo', 'Dictionary(Int32, Utf8)')) ---- 3 +query ?T +SELECT character_length(arrow_cast( + arrow_cast('é', 'Dictionary(UInt32, Utf8)'), + 'Dictionary(Int32, Dictionary(UInt32, Utf8))' + )), + arrow_typeof(character_length(arrow_cast( + arrow_cast('é', 'Dictionary(UInt32, Utf8)'), + 'Dictionary(Int32, Dictionary(UInt32, Utf8))' + ))) +---- +1 Dictionary(Int32, Dictionary(UInt32, Int32)) + +query ?T?T +SELECT character_length(arrow_cast('foo', 'Dictionary(Int32, LargeUtf8)')), + arrow_typeof(character_length( + arrow_cast('foo', 'Dictionary(Int32, LargeUtf8)') + )), + character_length(arrow_cast( + arrow_cast('foo', 'Dictionary(Int32, Utf8)'), + 'Dictionary(Int32, Utf8View)' + )), + arrow_typeof(character_length(arrow_cast( + arrow_cast('foo', 'Dictionary(Int32, Utf8)'), + 'Dictionary(Int32, Utf8View)' + ))) +---- +3 Dictionary(Int32, Int64) 3 Dictionary(Int32, Int32) + query I SELECT octet_length('foo') ---- @@ -755,6 +861,17 @@ ORDER BY id 2 2 Dictionary(Int32, Int32) Dictionary(Int32, Dictionary(UInt32, Int32)) NULL NULL Dictionary(Int32, Int32) Dictionary(Int32, Dictionary(UInt32, Int32)) +query ??TT +SELECT character_length(dict_col), character_length(nested_dict_col), + arrow_typeof(character_length(dict_col)), + arrow_typeof(character_length(nested_dict_col)) +FROM string_length_dictionary_test +ORDER BY id +---- +3 3 Dictionary(Int32, Int32) Dictionary(Int32, Dictionary(UInt32, Int32)) +1 1 Dictionary(Int32, Int32) Dictionary(Int32, Dictionary(UInt32, Int32)) +NULL NULL Dictionary(Int32, Int32) Dictionary(Int32, Dictionary(UInt32, Int32)) + query ??TT SELECT ascii(dict_col), ascii(nested_dict_col), arrow_typeof(ascii(dict_col)), diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt b/datafusion/sqllogictest/test_files/string/string_literal.slt index d5c4004c95..07aacaad93 100644 --- a/datafusion/sqllogictest/test_files/string/string_literal.slt +++ b/datafusion/sqllogictest/test_files/string/string_literal.slt @@ -544,6 +544,11 @@ SELECT reverse(arrow_cast('abcde', 'Dictionary(Int32, Utf8)')) ---- edcba +query T +SELECT arrow_typeof(reverse(arrow_cast('abcde', 'Dictionary(Int32, Utf8)'))) +---- +Dictionary(Int32, Utf8) + query T SELECT reverse('loẅks') ---- diff --git a/datafusion/sqllogictest/test_files/string/string_query.slt.part b/datafusion/sqllogictest/test_files/string/string_query.slt.part index 096045b4f5..dcddf06b55 100644 --- a/datafusion/sqllogictest/test_files/string/string_query.slt.part +++ b/datafusion/sqllogictest/test_files/string/string_query.slt.part @@ -1329,8 +1329,8 @@ NULL NULL NULL NULL NULL NULL query II SELECT - CHARACTER_LENGTH(ascii_1), - CHARACTER_LENGTH(unicode_1) + arrow_cast(CHARACTER_LENGTH(ascii_1), 'Int64'), + arrow_cast(CHARACTER_LENGTH(unicode_1), 'Int64') FROM test_basic_operator ---- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
