andygrove commented on code in PR #344:
URL: https://github.com/apache/datafusion-spark/pull/344#discussion_r4619961151
##########
crates/spark/src/function/math/log1p.rs:
##########
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use crate::function::error_utils::unsupported_data_type_exec_err;
+use arrow::array::{ArrayRef, AsArray, Float64Array};
+use arrow::datatypes::{DataType, Float64Type};
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+/// Spark-compatible `log1p` expression.
+///
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#log1p>
+#[user_doc(
+ doc_section(label = "Math Functions"),
+ description = "Returns `ln(1 + expr)` as a DOUBLE, computed accurately
when `expr` is \
+ close to 0. Values less than or equal to -1 return NULL. NULL input
returns NULL.",
+ syntax_example = "log1p(expr)",
+ sql_example = r#"```sql
+> SELECT log1p(0);
+0
+> SELECT log1p(-1);
+NULL
+```"#,
+ argument(
+ name = "expr",
+ description = "Numeric expression evaluated as DOUBLE. Values <= -1
and NULL return NULL."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkLog1p {
+ signature: Signature,
+}
+
+impl Default for SparkLog1p {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkLog1p {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::exact(vec![DataType::Float64],
Volatility::Immutable),
Review Comment:
Spark's `Log1p` declares `inputTypes = Seq(DoubleType)` with
`ImplicitCastInputTypes`. That means it accepts strings (and errors on a
malformed one under ANSI), and it rejects BOOLEAN, DATE and BINARY with
`UNEXPECTED_INPUT_TYPE`. `Signature::exact(Float64)` rejects strings, so
`log1p('0.5')` in this PR's own test fails.
Since #350 we have `SparkInputTypes` for exactly this case. Could you switch
to `SparkInputTypes::new(vec![SparkInputType::Double])` and add `coerce_types`
and `with_updated_config` the way `hash/sha2.rs` on `main` does? Then the
argument is always Float64, and the two `unsupported_data_type_exec_err` arms
can go.
##########
crates/sqllogictest/test_files/math/log1p.slt:
##########
@@ -2,13 +2,104 @@
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under
one or more contributor license agreements
-# This file was originally created by a porting script from:
-#
https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
-# This file is part of the implementation of the datafusion-spark function
library.
-# For more information, please see:
-# https://github.com/apache/datafusion/issues/15914
-
-## Original Query: SELECT log1p(0);
-## PySpark 3.5.5 Result: {'LOG1P(0)': 0.0, 'typeof(LOG1P(0))': 'double',
'typeof(0)': 'int'}
-#query
-#SELECT log1p(0::int);
+# Basic values and a negative input within the domain
+query R
+SELECT log1p(0.5::DOUBLE);
+----
+0.4054651081081644
+
+query R
+SELECT log1p(1.0::DOUBLE);
+----
+0.6931471805599453
+
+query R
+SELECT log1p(-0.5::DOUBLE);
+----
+-0.6931471805599453
+
+# A value immediately greater than the boundary -1
+query R
+SELECT log1p(-0.9999999999999999::DOUBLE);
+----
+-36.7368005696771
+
+# The boundary and values outside the domain return NULL
+query R
+SELECT log1p(-1.0::DOUBLE);
+----
+NULL
+
+query R
+SELECT log1p(-2.0::DOUBLE);
+----
+NULL
+
+query R
+SELECT log1p(CAST('-Infinity' AS DOUBLE));
+----
+NULL
+
+# NULL input returns NULL
+query R
+SELECT log1p(CAST(NULL AS DOUBLE));
+----
+NULL
+
+# Evaluate multiple values in a column
+query R
+SELECT log1p(a)
+FROM (VALUES
+ (0.0::DOUBLE),
+ (0.5::DOUBLE),
+ (-0.5::DOUBLE),
+ (-1.0::DOUBLE)
+) AS t(a);
+----
+0
+0.4054651081081644
+-0.6931471805599453
+NULL
+
+# Small inputs retain precision near zero
+query T
+SELECT log1p(1e-300::DOUBLE) = 1e-300::DOUBLE;
+----
+true
+
+query T
+SELECT log1p(1e-20::DOUBLE) = 1e-20::DOUBLE;
+----
+true
+
+# NaN and positive infinity are preserved
+query R
+SELECT log1p(CAST('NaN' AS DOUBLE));
+----
+NaN
+
+query R
+SELECT log1p(CAST('Infinity' AS DOUBLE));
+----
+Infinity
+
+# Integer, float, decimal, and string inputs
+query R
+SELECT log1p(CAST(1 AS INT));
+----
+0.6931471805599453
+
+query R
+SELECT log1p(CAST(0.5 AS FLOAT));
+----
+0.4054651081081644
+
+query R
+SELECT log1p(CAST(0.5 AS DECIMAL(10, 2)));
+----
+0.4054651081081644
+
+query R
+SELECT log1p('0.5');
+----
+0.4054651081081644
Review Comment:
Could you add a few cases for the inputs Spark rejects, like `log1p(true)`,
`log1p(DATE '2020-01-01')` and `log1p(X'01')`, as `query error`? A malformed
string like `log1p('abc')` would also be useful. It returns NULL with ANSI off
and raises `CAST_INVALID_INPUT` with ANSI on, so it needs the ANSI wrapper
described in `test_files/README.md`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]