(auron) branch master updated: [AURON #1708] Fix spark_normalize_nan_and_zero None (#1709)

[csy]

This is an automated email from the ASF dual-hosted git repository.

csy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git


The following commit(s) were added to refs/heads/master by this push:
     new 59261a09 [AURON #1708] Fix spark_normalize_nan_and_zero None (#1709)
59261a09 is described below

commit 59261a098b051cf257429cd761d3b21d870cf3e2
Author: cxzl25 <3898450+cxz...@users.noreply.github.com>
AuthorDate: Thu Dec 11 15:39:15 2025 +0800

    [AURON #1708] Fix spark_normalize_nan_and_zero None (#1709)
    
    # Which issue does this PR close?
    
    Closes #1708
    
     # Rationale for this change
    
    # What changes are included in this PR?
    
    # Are there any user-facing changes?
    
    # How was this patch tested?
    1. Add UT
    2. SQL validation in production environment
---
 .../src/spark_normalize_nan_and_zero.rs            | 31 +++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git 
a/native-engine/datafusion-ext-functions/src/spark_normalize_nan_and_zero.rs 
b/native-engine/datafusion-ext-functions/src/spark_normalize_nan_and_zero.rs
index ee94ca69..0fc9c357 100644
--- a/native-engine/datafusion-ext-functions/src/spark_normalize_nan_and_zero.rs
+++ b/native-engine/datafusion-ext-functions/src/spark_normalize_nan_and_zero.rs
@@ -37,6 +37,12 @@ pub fn spark_normalize_nan_and_zero(args: &[ColumnarValue]) 
-> Result<ColumnarVa
         ColumnarValue::Scalar(ScalarValue::Float64(Some(v))) => 
Ok(ColumnarValue::Scalar(
             ScalarValue::Float64(Some(normalize_float(*v))),
         )),
+        ColumnarValue::Scalar(ScalarValue::Float32(None)) => {
+            Ok(ColumnarValue::Scalar(ScalarValue::Float32(None)))
+        }
+        ColumnarValue::Scalar(ScalarValue::Float64(None)) => {
+            Ok(ColumnarValue::Scalar(ScalarValue::Float64(None)))
+        }
         ColumnarValue::Array(array) => match array.as_ref().data_type() {
             DataType::Float32 => 
Ok(ColumnarValue::Array(Arc::new(Float32Array::from_iter(
                 as_float32_array(array)?
@@ -54,7 +60,7 @@ pub fn spark_normalize_nan_and_zero(args: &[ColumnarValue]) 
-> Result<ColumnarVa
             ),
         },
         other_dt => df_execution_err!(
-            "spark_normalize_nan_and_zero only supports non-null 
Float32/Float64 scalars or Float32/Float64 arrays, not: {:?}",
+            "spark_normalize_nan_and_zero only supports Float32/Float64 
scalars or Float32/Float64 arrays, not: {:?}",
             other_dt.data_type()
         ),
     }
@@ -152,4 +158,27 @@ mod test {
         Float32,
         Float32Array
     );
+
+    macro_rules! test_normalize_nan_and_zero_null_scalar {
+        ($test_name:ident, $scalar_variant:ident) => {
+            #[test]
+            fn $test_name() -> Result<(), Box<dyn Error>> {
+                let input_columnar_value =
+                    ColumnarValue::Scalar(ScalarValue::$scalar_variant(None));
+                let result = 
spark_normalize_nan_and_zero(&vec![input_columnar_value])?;
+                match result {
+                    ColumnarValue::Scalar(ScalarValue::$scalar_variant(None)) 
=> Ok(()),
+                    other => panic!(
+                        "Expected null {} scalar, got: {:?}",
+                        stringify!($scalar_variant),
+                        other
+                    ),
+                }
+            }
+        };
+    }
+
+    
test_normalize_nan_and_zero_null_scalar!(test_normalize_nan_and_zero_null_scalar_f64,
 Float64);
+
+    
test_normalize_nan_and_zero_null_scalar!(test_normalize_nan_and_zero_null_scalar_f32,
 Float32);
 }