comphead commented on code in PR #73:
URL: https://github.com/apache/datafusion-spark/pull/73#discussion_r4566028163
##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,250 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Int32Array, StringArray,
StringBuilder};
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{Result, exec_datafusion_err, exec_err};
+use datafusion_expr::{
+ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract(str, regexp[, idx])`.
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+///
+/// Returns the text matched by group `idx` (default `1`, `0` is the whole
match)
+/// of the first match of `regexp` in `str`.
+///
+/// - NULL if any argument is NULL.
+/// - An empty string if there is no match, or if the group did not take part
in
+/// the match.
+/// - An error if `idx` is outside `0..=groupCount`
+/// (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`) or `regexp` is not a valid
+/// pattern (`INVALID_PARAMETER_VALUE.PATTERN`).
+///
+/// Patterns use the Rust [`regex`] syntax, not `java.util.regex`. Java-only
+/// features (lookaround, backreferences, possessive quantifiers) are rejected
+/// as invalid patterns here, and shorthands such as `\d` match Unicode digits
+/// where Java's match only ASCII.
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+ signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkRegexpExtract {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+ fn name(&self) -> &str {
+ NAME
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Utf8)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ if !(2..=3).contains(&arg_types.len()) {
+ return Err(invalid_arg_count_exec_err(NAME, (2, 3),
arg_types.len()));
+ }
+ let is_text = |t: &DataType| {
+ matches!(
+ t,
+ DataType::Utf8
+ | DataType::LargeUtf8
+ | DataType::Utf8View
+ | DataType::Null
+ )
+ };
+ if !is_text(&arg_types[0]) || !is_text(&arg_types[1]) {
+ return exec_err!(
+ "{NAME} expects string str and regexp, got {} and {}",
+ arg_types[0],
+ arg_types[1]
+ );
+ }
+ let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+ if let Some(idx) = arg_types.get(2) {
+ if !idx.is_integer() && *idx != DataType::Null {
+ return exec_err!("{NAME} expects an INT idx, got {idx}");
+ }
+ coerced.push(DataType::Int32);
+ }
+ Ok(coerced)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let num_rows = args.number_rows;
+ let arrays = ColumnarValue::values_to_arrays(&args.args)?;
+ let strings = arrays[0].as_string::<i32>();
+ let patterns = arrays[1].as_string::<i32>();
+ let default_idx;
+ let indexes = match arrays.get(2) {
+ Some(idx) => as_int32_array(idx)?,
+ None => {
+ default_idx = Int32Array::from(vec![1; num_rows]);
+ &default_idx
+ }
+ };
+ Ok(ColumnarValue::Array(extract(strings, patterns, indexes)?))
+ }
+}
+
+fn extract(
+ strings: &StringArray,
+ patterns: &StringArray,
+ indexes: &Int32Array,
+) -> Result<ArrayRef> {
+ let mut builder = StringBuilder::with_capacity(strings.len(), 0);
+ // The pattern is almost always a literal, so keep only the last one
compiled,
+ // with a capture buffer reused across rows instead of allocated per match.
+ let mut cached: Option<(&str, Regex, CaptureLocations)> = None;
+
+ for row in 0..strings.len() {
+ if strings.is_null(row) || patterns.is_null(row) ||
indexes.is_null(row) {
+ builder.append_null();
+ continue;
+ }
+ let pattern = patterns.value(row);
+ let (_, regex, locations) = match &mut cached {
+ Some(entry) if entry.0 == pattern => entry,
+ _ => {
+ let regex = compile(pattern)?;
+ let locations = regex.capture_locations();
+ cached.insert((pattern, regex, locations))
+ }
+ };
+ let idx = indexes.value(row);
+ let group_count = regex.captures_len() - 1;
+ let group = usize::try_from(idx).ok().filter(|g| *g <= group_count);
+ let Some(group) = group else {
Review Comment:
Spark checks `idx` only when the pattern matches. In
`RegExpExtract.nullSafeEval` the `checkGroupIndex` call is inside `if
(m.find)`, and the else branch returns an empty string. Here the check runs for
every non-null row, so `regexp_extract('abc', '\d+')` (default idx 1, no
groups) fails with `REGEX_GROUP_INDEX` where Spark returns `''`. On a column,
one non-matching row fails the whole query. The golden-file error cases all use
a subject that matches, which is why they pass. Could we call `captures_read`
first and validate only on `Some`? An `.slt` case with no match and an
out-of-range idx would pin it.
##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,250 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Int32Array, StringArray,
StringBuilder};
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{Result, exec_datafusion_err, exec_err};
+use datafusion_expr::{
+ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract(str, regexp[, idx])`.
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+///
+/// Returns the text matched by group `idx` (default `1`, `0` is the whole
match)
+/// of the first match of `regexp` in `str`.
+///
+/// - NULL if any argument is NULL.
+/// - An empty string if there is no match, or if the group did not take part
in
+/// the match.
+/// - An error if `idx` is outside `0..=groupCount`
+/// (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`) or `regexp` is not a valid
+/// pattern (`INVALID_PARAMETER_VALUE.PATTERN`).
+///
+/// Patterns use the Rust [`regex`] syntax, not `java.util.regex`. Java-only
+/// features (lookaround, backreferences, possessive quantifiers) are rejected
+/// as invalid patterns here, and shorthands such as `\d` match Unicode digits
+/// where Java's match only ASCII.
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+ signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkRegexpExtract {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+ fn name(&self) -> &str {
+ NAME
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Utf8)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ if !(2..=3).contains(&arg_types.len()) {
+ return Err(invalid_arg_count_exec_err(NAME, (2, 3),
arg_types.len()));
+ }
+ let is_text = |t: &DataType| {
+ matches!(
+ t,
+ DataType::Utf8
+ | DataType::LargeUtf8
+ | DataType::Utf8View
+ | DataType::Null
+ )
+ };
+ if !is_text(&arg_types[0]) || !is_text(&arg_types[1]) {
+ return exec_err!(
+ "{NAME} expects string str and regexp, got {} and {}",
+ arg_types[0],
+ arg_types[1]
+ );
+ }
+ let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+ if let Some(idx) = arg_types.get(2) {
+ if !idx.is_integer() && *idx != DataType::Null {
+ return exec_err!("{NAME} expects an INT idx, got {idx}");
+ }
+ coerced.push(DataType::Int32);
+ }
+ Ok(coerced)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let num_rows = args.number_rows;
+ let arrays = ColumnarValue::values_to_arrays(&args.args)?;
+ let strings = arrays[0].as_string::<i32>();
+ let patterns = arrays[1].as_string::<i32>();
+ let default_idx;
+ let indexes = match arrays.get(2) {
+ Some(idx) => as_int32_array(idx)?,
+ None => {
+ default_idx = Int32Array::from(vec![1; num_rows]);
Review Comment:
The usual call has a literal pattern and idx, but `values_to_arrays` expands
both into `num_rows` copies, the default idx allocates `vec![1; num_rows]`, and
every row compares the pattern string against the cached one. Matching on
`ColumnarValue::Scalar` for the pattern and idx would compile once and drop
those allocations, keeping the per-row path for column arguments. Also,
`StringBuilder::with_capacity(strings.len(), 0)` on line 121 reserves no data
bytes. The output is a set of substrings of the subject, so
`strings.value_data().len()` is an upper bound, which Comet uses.
##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,250 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, AsArray, Int32Array, StringArray,
StringBuilder};
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{Result, exec_datafusion_err, exec_err};
+use datafusion_expr::{
+ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
+};
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract(str, regexp[, idx])`.
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+///
+/// Returns the text matched by group `idx` (default `1`, `0` is the whole
match)
+/// of the first match of `regexp` in `str`.
+///
+/// - NULL if any argument is NULL.
+/// - An empty string if there is no match, or if the group did not take part
in
+/// the match.
+/// - An error if `idx` is outside `0..=groupCount`
+/// (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`) or `regexp` is not a valid
+/// pattern (`INVALID_PARAMETER_VALUE.PATTERN`).
+///
+/// Patterns use the Rust [`regex`] syntax, not `java.util.regex`. Java-only
+/// features (lookaround, backreferences, possessive quantifiers) are rejected
+/// as invalid patterns here, and shorthands such as `\d` match Unicode digits
+/// where Java's match only ASCII.
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+ signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkRegexpExtract {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+ fn name(&self) -> &str {
+ NAME
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Utf8)
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ if !(2..=3).contains(&arg_types.len()) {
+ return Err(invalid_arg_count_exec_err(NAME, (2, 3),
arg_types.len()));
+ }
+ let is_text = |t: &DataType| {
+ matches!(
+ t,
+ DataType::Utf8
+ | DataType::LargeUtf8
+ | DataType::Utf8View
+ | DataType::Null
+ )
+ };
+ if !is_text(&arg_types[0]) || !is_text(&arg_types[1]) {
+ return exec_err!(
+ "{NAME} expects string str and regexp, got {} and {}",
+ arg_types[0],
+ arg_types[1]
+ );
+ }
+ let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+ if let Some(idx) = arg_types.get(2) {
+ if !idx.is_integer() && *idx != DataType::Null {
+ return exec_err!("{NAME} expects an INT idx, got {idx}");
+ }
+ coerced.push(DataType::Int32);
+ }
+ Ok(coerced)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let num_rows = args.number_rows;
+ let arrays = ColumnarValue::values_to_arrays(&args.args)?;
+ let strings = arrays[0].as_string::<i32>();
+ let patterns = arrays[1].as_string::<i32>();
+ let default_idx;
+ let indexes = match arrays.get(2) {
+ Some(idx) => as_int32_array(idx)?,
+ None => {
+ default_idx = Int32Array::from(vec![1; num_rows]);
+ &default_idx
+ }
+ };
+ Ok(ColumnarValue::Array(extract(strings, patterns, indexes)?))
+ }
+}
+
+fn extract(
+ strings: &StringArray,
+ patterns: &StringArray,
+ indexes: &Int32Array,
+) -> Result<ArrayRef> {
+ let mut builder = StringBuilder::with_capacity(strings.len(), 0);
+ // The pattern is almost always a literal, so keep only the last one
compiled,
+ // with a capture buffer reused across rows instead of allocated per match.
+ let mut cached: Option<(&str, Regex, CaptureLocations)> = None;
+
+ for row in 0..strings.len() {
+ if strings.is_null(row) || patterns.is_null(row) ||
indexes.is_null(row) {
+ builder.append_null();
+ continue;
+ }
+ let pattern = patterns.value(row);
+ let (_, regex, locations) = match &mut cached {
+ Some(entry) if entry.0 == pattern => entry,
+ _ => {
+ let regex = compile(pattern)?;
+ let locations = regex.capture_locations();
+ cached.insert((pattern, regex, locations))
+ }
+ };
+ let idx = indexes.value(row);
+ let group_count = regex.captures_len() - 1;
+ let group = usize::try_from(idx).ok().filter(|g| *g <= group_count);
+ let Some(group) = group else {
+ return Err(exec_datafusion_err!(
+ "[INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX] The value of
parameter(s) \
+ `idx` in `{NAME}` is invalid: Expects group index between 0
and \
+ {group_count}, but got {idx}."
+ ));
+ };
+ let subject = strings.value(row);
+ let text = regex
+ .captures_read(locations, subject)
+ .and_then(|_| locations.get(group))
+ .map_or("", |(start, end)| &subject[start..end]);
+ builder.append_value(text);
+ }
+ Ok(Arc::new(builder.finish()))
+}
+
+fn compile(pattern: &str) -> Result<Regex> {
+ Regex::new(pattern).map_err(|_| {
+ exec_datafusion_err!(
+ "[INVALID_PARAMETER_VALUE.PATTERN] The value of parameter(s)
`regexp` in \
+ `{NAME}` is invalid: '{pattern}'."
+ )
+ })
+}
+
+#[cfg(test)]
Review Comment:
These tests repeat cases already in `regexp_extract.slt`, or could be
written there. Per-row patterns can be a table with a few rows that alternate
patterns. CI runs only `cargo test -p datafusion-spark-sqllogictest`, not
`cargo test -p datafusion-spark`, so the Rust unit tests here do not run in CI.
Could they move to `.slt`?
##########
crates/spark/src/function/string/mod.rs:
##########
@@ -45,6 +46,7 @@ make_udf_function!(soundex::SparkSoundex, soundex);
make_udf_function!(make_valid_utf8::SparkMakeValidUtf8, make_valid_utf8);
make_udf_function!(is_valid_utf8::SparkIsValidUtf8, is_valid_utf8);
make_udf_function!(quote::SparkQuote, quote);
+make_udf_function!(regexp_extract::SparkRegexpExtract, regexp_extract);
Review Comment:
Every other function in this module also has an `export_functions!` entry in
`expr_fn`. `regexp_extract` does not, so
`datafusion_spark::expr_fn::regexp_extract` is missing for the DataFrame API.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]