andygrove commented on code in PR #302:
URL: https://github.com/apache/datafusion-spark/pull/302#discussion_r4585373272
##########
crates/spark/src/function/math/positive.rs:
##########
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use arrow::compute::CastOptions;
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `positive` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#positive>
+#[user_doc(
+ doc_section(label = "Math Functions"),
+ description = "Returns `expr` unchanged. Accepts integer, floating-point,
decimal and interval values. \
+ A string is cast to DOUBLE: a malformed string returns NULL with
`datafusion.execution.enable_ansi_mode` off \
+ and fails with it on. Returns NULL if the input is NULL.",
+ syntax_example = "positive(expr)",
+ sql_example = r#"```sql
+> SELECT positive(1);
+1
+> SELECT positive('-1.11');
+-1.11
+```"#,
+ argument(
+ name = "expr",
+ description = "Numeric, interval or string expression. NULL returns
NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkPositive {
+ signature: Signature,
+}
+
+impl Default for SparkPositive {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkPositive {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+fn is_string(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+ )
+}
+
+impl ScalarUDFImpl for SparkPositive {
+ fn name(&self) -> &str {
+ "positive"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ if is_string(&arg_types[0]) {
+ Ok(DataType::Float64)
+ } else {
+ Ok(arg_types[0].clone())
+ }
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [data_type] = take_function_args(self.name(), arg_types)?;
+ match data_type {
+ // A NULL literal is treated as DOUBLE, like Spark's implicit cast
+ DataType::Null => Ok(vec![DataType::Float64]),
+ t if t.is_numeric()
+ || is_string(t)
+ || matches!(t, DataType::Interval(_) | DataType::Duration(_))
=>
+ {
+ Ok(vec![t.clone()])
+ }
+ _ => plan_err!(
+ "positive requires a numeric or interval input, got
{data_type}"
+ ),
+ }
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let enable_ansi_mode = args.config_options.execution.enable_ansi_mode;
+ let [arg] = take_function_args(self.name(), args.args)?;
+ if is_string(&arg.data_type()) {
+ // A malformed string is NULL in legacy mode and an error under
ANSI
+ let options = CastOptions {
+ safe: !enable_ansi_mode,
+ ..Default::default()
+ };
+ arg.cast_to(&DataType::Float64, Some(&options))
Review Comment:
String to DOUBLE goes through Arrow's `cast`, which parses with
`lexical_core`. Spark's `Cast` uses `java.lang.Double.parseDouble`, plus its
own special-literal handling. Java accepts a type suffix (`'1.5d'`, `'1.5F'`)
and hex floats (`'0x1p3'`), and lexical rejects both. So `SELECT
positive('1.5d')` is `1.5` in Spark but NULL here with ANSI off, and an error
with ANSI on. Spark's string-to-double semantics should be matched or shared,
or the gap should be recorded as a known divergence with a disabled test.
##########
crates/spark/src/function/math/positive.rs:
##########
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use arrow::compute::CastOptions;
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `positive` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#positive>
+#[user_doc(
+ doc_section(label = "Math Functions"),
+ description = "Returns `expr` unchanged. Accepts integer, floating-point,
decimal and interval values. \
+ A string is cast to DOUBLE: a malformed string returns NULL with
`datafusion.execution.enable_ansi_mode` off \
+ and fails with it on. Returns NULL if the input is NULL.",
+ syntax_example = "positive(expr)",
+ sql_example = r#"```sql
+> SELECT positive(1);
+1
+> SELECT positive('-1.11');
+-1.11
+```"#,
+ argument(
+ name = "expr",
+ description = "Numeric, interval or string expression. NULL returns
NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkPositive {
+ signature: Signature,
+}
+
+impl Default for SparkPositive {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkPositive {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+fn is_string(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+ )
+}
+
+impl ScalarUDFImpl for SparkPositive {
+ fn name(&self) -> &str {
+ "positive"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ if is_string(&arg_types[0]) {
+ Ok(DataType::Float64)
+ } else {
+ Ok(arg_types[0].clone())
+ }
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [data_type] = take_function_args(self.name(), arg_types)?;
+ match data_type {
+ // A NULL literal is treated as DOUBLE, like Spark's implicit cast
+ DataType::Null => Ok(vec![DataType::Float64]),
+ t if t.is_numeric()
+ || is_string(t)
+ || matches!(t, DataType::Interval(_) | DataType::Duration(_))
=>
+ {
+ Ok(vec![t.clone()])
+ }
+ _ => plan_err!(
Review Comment:
`coerce_types` rejects dictionary-encoded inputs, because `Dictionary(_,
Utf8)` and `Dictionary(_, Int32)` are neither `is_numeric()` nor `is_string`. A
Parquet column read as a dictionary fails at plan time with `positive requires
a numeric or interval input, got Dictionary(Int32, Utf8)`. Spark accepts the
same data. Consider unwrapping `Dictionary(_, value)` and coercing to the value
type, so DataFusion inserts the cast.
##########
crates/spark/src/function/math/positive.rs:
##########
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use arrow::compute::CastOptions;
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `positive` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#positive>
+#[user_doc(
+ doc_section(label = "Math Functions"),
+ description = "Returns `expr` unchanged. Accepts integer, floating-point,
decimal and interval values. \
+ A string is cast to DOUBLE: a malformed string returns NULL with
`datafusion.execution.enable_ansi_mode` off \
+ and fails with it on. Returns NULL if the input is NULL.",
+ syntax_example = "positive(expr)",
+ sql_example = r#"```sql
+> SELECT positive(1);
+1
+> SELECT positive('-1.11');
+-1.11
+```"#,
+ argument(
+ name = "expr",
+ description = "Numeric, interval or string expression. NULL returns
NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkPositive {
+ signature: Signature,
+}
+
+impl Default for SparkPositive {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkPositive {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+fn is_string(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+ )
+}
+
+impl ScalarUDFImpl for SparkPositive {
+ fn name(&self) -> &str {
+ "positive"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ if is_string(&arg_types[0]) {
+ Ok(DataType::Float64)
+ } else {
+ Ok(arg_types[0].clone())
+ }
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [data_type] = take_function_args(self.name(), arg_types)?;
+ match data_type {
+ // A NULL literal is treated as DOUBLE, like Spark's implicit cast
+ DataType::Null => Ok(vec![DataType::Float64]),
+ t if t.is_numeric()
+ || is_string(t)
+ || matches!(t, DataType::Interval(_) | DataType::Duration(_))
=>
+ {
+ Ok(vec![t.clone()])
+ }
+ _ => plan_err!(
+ "positive requires a numeric or interval input, got
{data_type}"
+ ),
+ }
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
Review Comment:
Spark's `UnaryPositive` is `RuntimeReplaceable` with `replacement = child`.
Here the identity stays as an opaque UDF node in the plan. That blocks
predicate pushdown and pruning on `positive(col) = 5`, and it costs a dispatch
per batch. Implementing `simplify` would fix this at the root. It returns the
argument unchanged for non-string inputs, and for strings a `Cast` or `TryCast`
to Float64 depending on ANSI mode.
##########
crates/spark/src/function/math/positive.rs:
##########
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use arrow::compute::CastOptions;
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `positive` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#positive>
+#[user_doc(
+ doc_section(label = "Math Functions"),
+ description = "Returns `expr` unchanged. Accepts integer, floating-point,
decimal and interval values. \
+ A string is cast to DOUBLE: a malformed string returns NULL with
`datafusion.execution.enable_ansi_mode` off \
+ and fails with it on. Returns NULL if the input is NULL.",
+ syntax_example = "positive(expr)",
+ sql_example = r#"```sql
+> SELECT positive(1);
+1
+> SELECT positive('-1.11');
+-1.11
+```"#,
+ argument(
+ name = "expr",
+ description = "Numeric, interval or string expression. NULL returns
NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkPositive {
+ signature: Signature,
+}
+
+impl Default for SparkPositive {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkPositive {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+fn is_string(data_type: &DataType) -> bool {
Review Comment:
This local `is_string` helper duplicates `DataType::is_string()`, which
already exists in arrow-schema 60.1 (`matches!(self, Utf8 | LargeUtf8 |
Utf8View)`). Call `data_type.is_string()` instead and drop the helper.
##########
crates/sqllogictest/test_files/math/positive.slt:
##########
@@ -10,5 +10,115 @@
## Original Query: SELECT positive(1);
## PySpark 3.5.5 Result: {'(+ 1)': 1, 'typeof((+ 1))': 'int', 'typeof(1)':
'int'}
-#query
-#SELECT positive(1::int);
+query IT
+SELECT positive(1::int), arrow_typeof(positive(1::int));
+----
+1 Int32
+
+# Integers keep their type and value, including the minimum value
+query IIII
+SELECT positive((-128)::tinyint), positive((-32768)::smallint),
positive((-2147483648)::int), positive(CAST('-9223372036854775808' AS BIGINT));
+----
+-128 -32768 -2147483648 -9223372036854775808
+
+query TTTT
+SELECT arrow_typeof(positive(1::tinyint)) AS a,
arrow_typeof(positive(1::smallint)) AS b, arrow_typeof(positive(1::int)) AS c,
arrow_typeof(positive(1::bigint)) AS d;
+----
+Int8 Int16 Int32 Int64
+
+# Floating point values are returned unchanged
+query RRRR
+SELECT positive(1.5::float), positive(CAST('NaN' AS DOUBLE)),
positive(CAST('Infinity' AS DOUBLE)), positive(CAST('-Infinity' AS DOUBLE));
+----
+1.5 NaN Infinity -Infinity
+
+query T
+SELECT arrow_typeof(positive(1.5::float));
+----
+Float32
+
+# The sign of -0.0 is kept: pow(-0.0, -1.0) is -Infinity
+query R
+SELECT pow(positive(CAST('-0.0' AS DOUBLE)), -1.0);
+----
+-Infinity
+
+# DECIMAL keeps its precision and scale
+query RT
+SELECT positive(CAST(-1.11 AS DECIMAL(3, 2))),
arrow_typeof(positive(CAST(-1.11 AS DECIMAL(3, 2))));
+----
+-1.11 Decimal128(3, 2)
+
+# A STRING is cast to DOUBLE
+query RT
+SELECT positive('-1.11'), arrow_typeof(positive('-1.11'));
Review Comment:
The string tests only use scalar literals (`positive('-1.11')`,
`positive('abc')`), which are Utf8 scalars. Nothing exercises the array path of
the `cast_to` call: a string column holding valid, malformed and NULL rows, in
both ANSI modes. Nothing covers `LargeUtf8` or `Utf8View` either, though
`is_string` explicitly accepts them. Please add a `VALUES` column test with
`arrow_cast(..., 'Utf8View')`/`'LargeUtf8'`.
##########
crates/sqllogictest/test_files/math/positive.slt:
##########
@@ -10,5 +10,115 @@
## Original Query: SELECT positive(1);
## PySpark 3.5.5 Result: {'(+ 1)': 1, 'typeof((+ 1))': 'int', 'typeof(1)':
'int'}
-#query
-#SELECT positive(1::int);
+query IT
+SELECT positive(1::int), arrow_typeof(positive(1::int));
+----
+1 Int32
+
+# Integers keep their type and value, including the minimum value
+query IIII
+SELECT positive((-128)::tinyint), positive((-32768)::smallint),
positive((-2147483648)::int), positive(CAST('-9223372036854775808' AS BIGINT));
+----
+-128 -32768 -2147483648 -9223372036854775808
+
+query TTTT
+SELECT arrow_typeof(positive(1::tinyint)) AS a,
arrow_typeof(positive(1::smallint)) AS b, arrow_typeof(positive(1::int)) AS c,
arrow_typeof(positive(1::bigint)) AS d;
+----
+Int8 Int16 Int32 Int64
+
+# Floating point values are returned unchanged
+query RRRR
+SELECT positive(1.5::float), positive(CAST('NaN' AS DOUBLE)),
positive(CAST('Infinity' AS DOUBLE)), positive(CAST('-Infinity' AS DOUBLE));
+----
+1.5 NaN Infinity -Infinity
+
+query T
+SELECT arrow_typeof(positive(1.5::float));
+----
+Float32
+
+# The sign of -0.0 is kept: pow(-0.0, -1.0) is -Infinity
+query R
+SELECT pow(positive(CAST('-0.0' AS DOUBLE)), -1.0);
+----
+-Infinity
+
+# DECIMAL keeps its precision and scale
+query RT
+SELECT positive(CAST(-1.11 AS DECIMAL(3, 2))),
arrow_typeof(positive(CAST(-1.11 AS DECIMAL(3, 2))));
+----
+-1.11 Decimal128(3, 2)
+
+# A STRING is cast to DOUBLE
+query RT
+SELECT positive('-1.11'), arrow_typeof(positive('-1.11'));
+----
+-1.11 Float64
+
+# A malformed STRING is NULL when ANSI mode is off
+query R
+SELECT positive('abc');
+----
+NULL
+
+# Intervals are returned unchanged
+query BT
+SELECT positive(make_dt_interval(1::int, 2::int)) = make_dt_interval(1::int,
2::int), arrow_typeof(positive(make_dt_interval(1::int, 2::int)));
Review Comment:
The interval tests cover only `Duration` (day-time) and
`Interval(YearMonth)`. `coerce_types` also admits `Interval(MonthDayNano)`
(Spark's CalendarInterval, which `TypeCollection.NumericAndInterval` includes)
and `Interval(DayTime)`. Neither has a test, so a regression in that
pass-through would go unnoticed.
##########
crates/spark/src/function/math/positive.rs:
##########
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use arrow::compute::CastOptions;
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `positive` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#positive>
+#[user_doc(
+ doc_section(label = "Math Functions"),
+ description = "Returns `expr` unchanged. Accepts integer, floating-point,
decimal and interval values. \
+ A string is cast to DOUBLE: a malformed string returns NULL with
`datafusion.execution.enable_ansi_mode` off \
+ and fails with it on. Returns NULL if the input is NULL.",
+ syntax_example = "positive(expr)",
+ sql_example = r#"```sql
+> SELECT positive(1);
+1
+> SELECT positive('-1.11');
+-1.11
+```"#,
+ argument(
+ name = "expr",
+ description = "Numeric, interval or string expression. NULL returns
NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkPositive {
+ signature: Signature,
+}
+
+impl Default for SparkPositive {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkPositive {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+fn is_string(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+ )
+}
+
+impl ScalarUDFImpl for SparkPositive {
+ fn name(&self) -> &str {
+ "positive"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ if is_string(&arg_types[0]) {
+ Ok(DataType::Float64)
+ } else {
+ Ok(arg_types[0].clone())
+ }
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [data_type] = take_function_args(self.name(), arg_types)?;
+ match data_type {
+ // A NULL literal is treated as DOUBLE, like Spark's implicit cast
+ DataType::Null => Ok(vec![DataType::Float64]),
+ t if t.is_numeric()
+ || is_string(t)
+ || matches!(t, DataType::Interval(_) | DataType::Duration(_))
=>
Review Comment:
This is broader than Spark and inconsistent with the sibling `negative`. It
accepts unsigned ints, Float16, and `Duration` in every unit (s, ms, ns), while
`SparkNegative` restricts itself to `Numeric(1)` plus the three `Interval`
units. As a result, `positive(make_dt_interval(1, 2))` plans but
`negative(make_dt_interval(1, 2))` fails with a signature error. The two unary
operators should accept the same types.
##########
crates/spark/src/function/math/positive.rs:
##########
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use arrow::compute::CastOptions;
+use arrow::datatypes::DataType;
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, plan_err};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+
+/// Spark-compatible `positive` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#positive>
+#[user_doc(
+ doc_section(label = "Math Functions"),
+ description = "Returns `expr` unchanged. Accepts integer, floating-point,
decimal and interval values. \
+ A string is cast to DOUBLE: a malformed string returns NULL with
`datafusion.execution.enable_ansi_mode` off \
+ and fails with it on. Returns NULL if the input is NULL.",
+ syntax_example = "positive(expr)",
+ sql_example = r#"```sql
+> SELECT positive(1);
+1
+> SELECT positive('-1.11');
+-1.11
+```"#,
+ argument(
+ name = "expr",
+ description = "Numeric, interval or string expression. NULL returns
NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkPositive {
+ signature: Signature,
+}
+
+impl Default for SparkPositive {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkPositive {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::user_defined(Volatility::Immutable),
+ }
+ }
+}
+
+fn is_string(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
+ )
+}
+
+impl ScalarUDFImpl for SparkPositive {
+ fn name(&self) -> &str {
+ "positive"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ if is_string(&arg_types[0]) {
+ Ok(DataType::Float64)
+ } else {
+ Ok(arg_types[0].clone())
+ }
+ }
+
+ fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
+ let [data_type] = take_function_args(self.name(), arg_types)?;
+ match data_type {
+ // A NULL literal is treated as DOUBLE, like Spark's implicit cast
+ DataType::Null => Ok(vec![DataType::Float64]),
+ t if t.is_numeric()
+ || is_string(t)
+ || matches!(t, DataType::Interval(_) | DataType::Duration(_))
=>
+ {
+ Ok(vec![t.clone()])
+ }
+ _ => plan_err!(
+ "positive requires a numeric or interval input, got
{data_type}"
Review Comment:
The error text says `positive requires a numeric or interval input`, but
string inputs are accepted too (and cast to DOUBLE). Users who hit this error
get a misleading list of allowed types. Suggest `numeric, interval or string
input`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]