kosiew commented on code in PR #73:
URL: https://github.com/apache/datafusion-spark/pull/73#discussion_r4598513102
##########
crates/spark/src/function/string/regexp_extract.rs:
##########
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{
+ Array, ArrayRef, AsArray, Int32Array, StringArray, StringBuilder,
new_null_array,
+};
+use arrow::compute::can_cast_types;
+use arrow::datatypes::DataType;
+use datafusion_common::cast::as_int32_array;
+use datafusion_common::{
+ DataFusionError, Result, ScalarValue, exec_datafusion_err,
internal_datafusion_err,
+ internal_err, plan_err,
+};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use regex::{CaptureLocations, Regex};
+
+use crate::function::error_utils::invalid_arg_count_exec_err;
+
+const NAME: &str = "regexp_extract";
+
+/// Spark-compatible `regexp_extract` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#regexp_extract>
+#[user_doc(
+ doc_section(label = "Regular Expression Functions"),
+ description = "Returns the text matched by group `idx` of the first match
of `regexp` in `str`. \
+ Unlike DataFusion's `regexp_match`, which returns a list of every
capture group, this returns a single string. \
+ Returns an empty string if there is no match, or if the group did not
take part in the match. \
+ `idx` is only checked when the pattern matches: an `idx` outside
`0..=groupCount` is an error \
+ (`INVALID_PARAMETER_VALUE.REGEX_GROUP_INDEX`, as in Spark 3.5 and
later; Spark 3.4 raised \
+ `_LEGACY_ERROR_TEMP_2006`/`2007`) on a row that matches, and an empty
string on a row that does not. \
+ An invalid pattern is an error (`INVALID_PARAMETER_VALUE.PATTERN`).
Returns NULL if any argument is NULL, \
+ and a NULL row is not checked, so an invalid pattern is only reported
on a row it is applied to. \
+ As in Spark, `str` and `regexp` are cast to strings and `idx` to INT. \
+ Known differences from Spark
(https://github.com/apache/datafusion-spark/issues/255): patterns use the Rust \
+ `regex` engine, not Java's `java.util.regex`. Look-around,
backreferences, and patterns that exceed the Rust \
+ engine's compiled-size limit raise an \"unsupported regex pattern\"
error, where Spark evaluates them. \
+ Possessive quantifiers are read as nested greedy repetition (`a++` as
`(a+)+`), so they can match \
+ where Java's would not. \
+ `\\d`, `\\w` and `\\s` match Unicode characters, where Java's match
only ASCII. \
+ `.` matches `\\r`, where Java's does not. `$` matches only at the very
end of the input, where Java's also \
+ matches before a final line terminator. `(?i)` folds case using
Unicode rules (so `k` matches the Kelvin \
+ sign), where Java folds only ASCII by default. Rust's `(?P<name>...)`
named groups are accepted, where Java \
+ rejects them as an invalid pattern.",
+ syntax_example = "regexp_extract(str, regexp[, idx])",
+ sql_example = r#"```sql
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)');
+1
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 0);
+1a
+> SELECT regexp_extract('1a 2b 14m', '(\d+)([a-z]+)', 2);
+a
+> SELECT regexp_extract('abc', '(\d+)', 1);
+-- empty string
+> SELECT regexp_extract('a1', '(\d+)', NULL);
+NULL
+```"#,
+ argument(
+ name = "str",
+ description = "String expression to search. Values of other types,
such as numbers and binary, are cast to a string. NULL returns NULL."
+ ),
+ argument(
+ name = "regexp",
+ description = "String regular expression in Rust `regex` syntax.
Values of other types are cast to a string. NULL returns NULL."
+ ),
+ argument(
+ name = "idx",
+ description = "Optional capture group index, cast to INT (numeric and
string values are accepted). Defaults to 1; 0 is the whole match. NULL returns
NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkRegexpExtract {
+ signature: Signature,
+}
+
+impl Default for SparkRegexpExtract {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkRegexpExtract {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkRegexpExtract {
+ fn name(&self) -> &str {
+ NAME
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ Ok(DataType::Utf8)
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ if !(2..=3).contains(&arg_types.len()) {
+ return Err(invalid_arg_count_exec_err(NAME, (2, 3),
arg_types.len()));
+ }
+ // Like Spark's implicit casts: str and regexp are cast to strings,
idx to INT.
+ let castable_to_string = |t: &DataType| {
+ *t == DataType::Null || (!t.is_nested() && can_cast_types(t,
&DataType::Utf8))
+ };
+ for (name, arg_type) in ["str", "regexp"].iter().zip(arg_types) {
+ if !castable_to_string(arg_type) {
+ return plan_err!("{NAME} expects a string {name}, got
{arg_type}");
+ }
+ }
+ let mut coerced = vec![DataType::Utf8, DataType::Utf8];
+ if let Some(idx) = arg_types.get(2) {
+ let castable_to_int = idx.is_numeric()
+ || matches!(
+ idx,
+ DataType::Null
+ | DataType::Utf8
+ | DataType::LargeUtf8
+ | DataType::Utf8View
+ );
+ if !castable_to_int {
+ return plan_err!("{NAME} expects an INT idx, got {idx}");
+ }
+ coerced.push(DataType::Int32);
+ }
+ Ok(coerced)
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let all_scalar = args
+ .args
+ .iter()
+ .all(|arg| matches!(arg, ColumnarValue::Scalar(_)));
+ let len = if all_scalar { 1 } else { args.number_rows };
+
+ // The usual call has a literal pattern and idx: keep those as scalars
so
+ // the pattern is compiled once and no per-row copies are allocated.
+ let pattern = match &args.args[1] {
+ ColumnarValue::Scalar(scalar) => match scalar_str(scalar)? {
+ Some(pattern) => Pattern::Scalar(pattern),
+ None => return null_result(len, all_scalar),
+ },
+ ColumnarValue::Array(array) =>
Pattern::Array(array.as_string::<i32>()),
+ };
+ let index = match args.args.get(2) {
+ None => Index::Scalar(1),
+ Some(ColumnarValue::Scalar(ScalarValue::Int32(Some(idx)))) => {
+ Index::Scalar(*idx)
+ }
+ Some(ColumnarValue::Scalar(ScalarValue::Int32(None) |
ScalarValue::Null)) => {
+ return null_result(len, all_scalar);
+ }
+ Some(ColumnarValue::Scalar(other)) => {
+ return internal_err!(
+ "{NAME} expects an INT idx, got {}",
+ other.data_type()
+ );
+ }
+ Some(ColumnarValue::Array(array)) =>
Index::Array(as_int32_array(array)?),
+ };
+
+ let strings = args.args[0].to_array(len)?;
+ let result = extract(strings.as_string::<i32>(), pattern, index)?;
+ if all_scalar {
+ Ok(ColumnarValue::Scalar(ScalarValue::try_from_array(
+ &result, 0,
+ )?))
+ } else {
+ Ok(ColumnarValue::Array(result))
+ }
+ }
+}
+
+#[derive(Clone, Copy)]
+enum Pattern<'a> {
+ Scalar(&'a str),
+ Array(&'a StringArray),
+}
+
+#[derive(Clone, Copy)]
+enum Index<'a> {
+ Scalar(i32),
+ Array(&'a Int32Array),
+}
+
+fn scalar_str(scalar: &ScalarValue) -> Result<Option<&str>> {
+ match scalar {
+ ScalarValue::Null => Ok(None),
+ other => other.try_as_str().ok_or_else(|| {
+ internal_datafusion_err!(
+ "{NAME} expects a string regexp, got {}",
+ other.data_type()
+ )
+ }),
+ }
+}
+
+fn null_result(len: usize, all_scalar: bool) -> Result<ColumnarValue> {
+ Ok(if all_scalar {
+ ColumnarValue::Scalar(ScalarValue::Utf8(None))
+ } else {
+ ColumnarValue::Array(new_null_array(&DataType::Utf8, len))
+ })
+}
+
+fn extract(strings: &StringArray, pattern: Pattern, index: Index) ->
Result<ArrayRef> {
+ // Each result is a substring of its input, so the input bytes bound the
output.
+ let mut builder =
+ StringBuilder::with_capacity(strings.len(),
strings.value_data().len());
Review Comment:
`strings.value_data().len()` includes the retained parent buffer for sliced
arrays, so even an empty or tiny slice can allocate a parent-sized output
buffer. A physical-call probe with 65536 padding bytes on each side of `'a1'`
produced output buffers of 131078 bytes for zero rows and 131082 bytes for one
row, risking OOM when outputs accumulate. Please use
`datafusion_common::utils::offset_span_len(strings.offsets())` or equivalent
visible-offset bounds, and add Rust regression tests for empty and
nonzero-offset slices that verify values and bounded allocation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]