This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 7b0ed2d616 Refactor Spark `date_add`/`date_sub`/`bitwise_not` to
remove unnecessary scalar arg check (#19473)
7b0ed2d616 is described below
commit 7b0ed2d616a5264cd9b129d89ba7454dca879e7e
Author: Jeffrey Vo <[email protected]>
AuthorDate: Fri Jan 16 11:06:05 2026 +0900
Refactor Spark `date_add`/`date_sub`/`bitwise_not` to remove unnecessary
scalar arg check (#19473)
Same as #19466 but for `date_add`, `date_sub` and `bitwise_not`
> If we have a scalar argument that is null, that means the datatype it
is from is already nullable, so theres no need to check both; we only
need to check the nullability of the datatype
---
.../spark/src/function/bitwise/bitwise_not.rs | 52 +++-------------------
datafusion/spark/src/function/datetime/date_add.rs | 29 +-----------
datafusion/spark/src/function/datetime/date_sub.rs | 26 +----------
3 files changed, 7 insertions(+), 100 deletions(-)
diff --git a/datafusion/spark/src/function/bitwise/bitwise_not.rs
b/datafusion/spark/src/function/bitwise/bitwise_not.rs
index 5f8cf36911..e7285d4804 100644
--- a/datafusion/spark/src/function/bitwise/bitwise_not.rs
+++ b/datafusion/spark/src/function/bitwise/bitwise_not.rs
@@ -73,25 +73,11 @@ impl ScalarUDFImpl for SparkBitwiseNot {
}
fn return_field_from_args(&self, args: ReturnFieldArgs) ->
Result<FieldRef> {
- if args.arg_fields.len() != 1 {
- return plan_err!("bitwise_not expects exactly 1 argument");
- }
-
- let input_field = &args.arg_fields[0];
-
- let out_dt = input_field.data_type().clone();
- let mut out_nullable = input_field.is_nullable();
-
- let scalar_null_present = args
- .scalar_arguments
- .iter()
- .any(|opt_s| opt_s.is_some_and(|sv| sv.is_null()));
-
- if scalar_null_present {
- out_nullable = true;
- }
-
- Ok(Arc::new(Field::new(self.name(), out_dt, out_nullable)))
+ Ok(Arc::new(Field::new(
+ self.name(),
+ args.arg_fields[0].data_type().clone(),
+ args.arg_fields[0].is_nullable(),
+ )))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
@@ -196,32 +182,4 @@ mod tests {
assert!(out_i64_null.is_nullable());
assert_eq!(out_i64_null.data_type(), &DataType::Int64);
}
-
- #[test]
- fn test_bitwise_not_nullability_with_null_scalar() -> Result<()> {
- use arrow::datatypes::{DataType, Field};
- use datafusion_common::ScalarValue;
- use std::sync::Arc;
-
- let func = SparkBitwiseNot::new();
-
- let non_nullable: FieldRef = Arc::new(Field::new("col",
DataType::Int32, false));
-
- let out = func.return_field_from_args(ReturnFieldArgs {
- arg_fields: &[Arc::clone(&non_nullable)],
- scalar_arguments: &[None],
- })?;
- assert!(!out.is_nullable());
- assert_eq!(out.data_type(), &DataType::Int32);
-
- let null_scalar = ScalarValue::Int32(None);
- let out_with_null_scalar = func.return_field_from_args(ReturnFieldArgs
{
- arg_fields: &[Arc::clone(&non_nullable)],
- scalar_arguments: &[Some(&null_scalar)],
- })?;
- assert!(out_with_null_scalar.is_nullable());
- assert_eq!(out_with_null_scalar.data_type(), &DataType::Int32);
-
- Ok(())
- }
}
diff --git a/datafusion/spark/src/function/datetime/date_add.rs
b/datafusion/spark/src/function/datetime/date_add.rs
index 78b9c904ce..3745f77969 100644
--- a/datafusion/spark/src/function/datetime/date_add.rs
+++ b/datafusion/spark/src/function/datetime/date_add.rs
@@ -82,12 +82,7 @@ impl ScalarUDFImpl for SparkDateAdd {
}
fn return_field_from_args(&self, args: ReturnFieldArgs) ->
Result<FieldRef> {
- let nullable = args.arg_fields.iter().any(|f| f.is_nullable())
- || args
- .scalar_arguments
- .iter()
- .any(|arg| matches!(arg, Some(sv) if sv.is_null()));
-
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
Ok(Arc::new(Field::new(
self.name(),
DataType::Date32,
@@ -142,7 +137,6 @@ fn spark_date_add(args: &[ArrayRef]) -> Result<ArrayRef> {
mod tests {
use super::*;
use arrow::datatypes::Field;
- use datafusion_common::ScalarValue;
#[test]
fn test_date_add_non_nullable_inputs() {
@@ -181,25 +175,4 @@ mod tests {
assert_eq!(ret_field.data_type(), &DataType::Date32);
assert!(ret_field.is_nullable());
}
-
- #[test]
- fn test_date_add_null_scalar() {
- let func = SparkDateAdd::new();
- let args = &[
- Arc::new(Field::new("date", DataType::Date32, false)),
- Arc::new(Field::new("num", DataType::Int32, false)),
- ];
-
- let null_scalar = ScalarValue::Int32(None);
-
- let ret_field = func
- .return_field_from_args(ReturnFieldArgs {
- arg_fields: args,
- scalar_arguments: &[None, Some(&null_scalar)],
- })
- .unwrap();
-
- assert_eq!(ret_field.data_type(), &DataType::Date32);
- assert!(ret_field.is_nullable());
- }
}
diff --git a/datafusion/spark/src/function/datetime/date_sub.rs
b/datafusion/spark/src/function/datetime/date_sub.rs
index 34894317f6..af1b8d5a4e 100644
--- a/datafusion/spark/src/function/datetime/date_sub.rs
+++ b/datafusion/spark/src/function/datetime/date_sub.rs
@@ -75,12 +75,7 @@ impl ScalarUDFImpl for SparkDateSub {
}
fn return_field_from_args(&self, args: ReturnFieldArgs) ->
Result<FieldRef> {
- let nullable = args.arg_fields.iter().any(|f| f.is_nullable())
- || args
- .scalar_arguments
- .iter()
- .any(|arg| matches!(arg, Some(sv) if sv.is_null()));
-
+ let nullable = args.arg_fields.iter().any(|f| f.is_nullable());
Ok(Arc::new(Field::new(
self.name(),
DataType::Date32,
@@ -139,7 +134,6 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result<ArrayRef> {
#[cfg(test)]
mod tests {
use super::*;
- use datafusion_common::ScalarValue;
#[test]
fn test_date_sub_nullability_non_nullable_args() {
@@ -174,22 +168,4 @@ mod tests {
assert!(result.is_nullable());
assert_eq!(result.data_type(), &DataType::Date32);
}
-
- #[test]
- fn test_date_sub_nullability_scalar_null_argument() {
- let udf = SparkDateSub::new();
- let date_field = Arc::new(Field::new("d", DataType::Date32, false));
- let days_field = Arc::new(Field::new("n", DataType::Int32, false));
- let null_scalar = ScalarValue::Int32(None);
-
- let result = udf
- .return_field_from_args(ReturnFieldArgs {
- arg_fields: &[date_field, days_field],
- scalar_arguments: &[None, Some(&null_scalar)],
- })
- .unwrap();
-
- assert!(result.is_nullable());
- assert_eq!(result.data_type(), &DataType::Date32);
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
