This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23641-d185ed54cf5c65b0241bb3ef203ba11388ad605f in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 8a37a07d5ef1316663e13daa39f8d2de015f4168 Author: linfeng <[email protected]> AuthorDate: Sat Jul 18 03:24:18 2026 +0800 fix: handle null date and timestamp format arguments (#23641) ## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Closes #23640. ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> The formatted `to_date` and `to_timestamp*` functions can produce internal errors when the input string is NULL or when all format arguments are NULL. A NULL value in a format column is also treated as an empty string instead of being skipped. `to_unixtime` uses the same timestamp parsing path, so it is affected as well. ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> - Return NULL when the scalar input string is NULL. - Return NULL when all format arguments are NULL. - Skip NULL formats in array rows. - Keep returning a parsing error when all non-NULL formats fail. ## Are these changes tested? Yes, SLTs were added for `to_date`, `to_timestamp`, and `to_unixtime`. <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> Yes, only the bug fix described above. No API changes. --- datafusion/functions/src/datetime/common.rs | 34 +++++++++++++++------- .../sqllogictest/test_files/datetime/dates.slt | 21 +++++++++++++ .../test_files/datetime/timestamps.slt | 29 ++++++++++++++++++ 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/datafusion/functions/src/datetime/common.rs b/datafusion/functions/src/datetime/common.rs index 2db64beafa..9a7f94bd59 100644 --- a/datafusion/functions/src/datetime/common.rs +++ b/datafusion/functions/src/datetime/common.rs @@ -32,7 +32,7 @@ use chrono::{DateTime, TimeZone, Utc}; use datafusion_common::cast::as_generic_string_array; use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, - internal_datafusion_err, unwrap_or_internal_err, + internal_datafusion_err, }; use datafusion_expr::ColumnarValue; @@ -353,9 +353,9 @@ where // if the first argument is a scalar utf8 all arguments are expected to be scalar utf8 ColumnarValue::Scalar(scalar) => match scalar.try_as_str() { Some(a) => { - let a = a.as_ref(); - // ASK: Why do we trust `a` to be non-null at this point? - let a = unwrap_or_internal_err!(a); + let Some(a) = a.as_ref() else { + return Ok(ColumnarValue::Scalar(scalar_value(dt, None)?)); + }; let mut ret = None; @@ -384,7 +384,10 @@ where } } - unwrap_or_internal_err!(ret) + match ret { + Some(ret) => ret, + None => Ok(ColumnarValue::Scalar(scalar_value(dt, None)?)), + } } other => { exec_err!("Unsupported data type {other:?} for function {name}") @@ -483,12 +486,21 @@ where if let Some(x) = x { for arg in args { let v = match arg { - ColumnarValue::Array(a) => match a.data_type() { - DataType::Utf8View => Ok(a.as_string_view().value(pos)), - DataType::LargeUtf8 => Ok(a.as_string::<i64>().value(pos)), - DataType::Utf8 => Ok(a.as_string::<i32>().value(pos)), - other => exec_err!("Unexpected type encountered '{other}'"), - }, + ColumnarValue::Array(a) => { + if a.is_null(pos) { + continue; + } + match a.data_type() { + DataType::Utf8View => Ok(a.as_string_view().value(pos)), + DataType::LargeUtf8 => { + Ok(a.as_string::<i64>().value(pos)) + } + DataType::Utf8 => Ok(a.as_string::<i32>().value(pos)), + other => { + exec_err!("Unexpected type encountered '{other}'") + } + } + } ColumnarValue::Scalar(s) => match s.try_as_str() { Some(Some(v)) => Ok(v), Some(None) => continue, // null string diff --git a/datafusion/sqllogictest/test_files/datetime/dates.slt b/datafusion/sqllogictest/test_files/datetime/dates.slt index d2a7360b12..68d87eceed 100644 --- a/datafusion/sqllogictest/test_files/datetime/dates.slt +++ b/datafusion/sqllogictest/test_files/datetime/dates.slt @@ -298,6 +298,27 @@ SELECT to_date('2020-09-08 12/00/00+00:00', '%c', '%+') query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08 12/00/00\+00:00' using format '%q': trailing input SELECT to_date('2020-09-08 12/00/00+00:00', '%q') +# NULL string scalar inputs and all-NULL scalar formats return NULL +query DD +SELECT + to_date(NULL::VARCHAR, '%Y-%m-%d'), + to_date('2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +---- +NULL NULL + +# NULL array formats are skipped for each row; rows with no usable format return NULL +query ID +SELECT id, to_date(value, format1, format2) +FROM ( + VALUES + (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'), + (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +) AS t(id, value, format1, format2) +ORDER BY id +---- +1 2020-09-08 +2 NULL + statement ok create table ts_utf8_data(ts varchar(100), format varchar(100)) as values ('2020-09-08 12/00/00+00:00', '%Y-%m-%d %H/%M/%S%#z'), diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index 8ba095ef93..9ac00e72b4 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -518,6 +518,35 @@ SELECT COUNT(*) FROM ts_data_secs where ts > to_timestamp_seconds('2020-09-08 12 ---- 2 +# NULL string scalar inputs and all-NULL scalar formats return NULL +query PP +SELECT + to_timestamp(NULL::VARCHAR, '%Y-%m-%d'), + to_timestamp('2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +---- +NULL NULL + +# NULL array formats are skipped for each row; rows with no usable format return NULL +query IP +SELECT id, to_timestamp(value, format1, format2) +FROM ( + VALUES + (1, '2020-09-08', NULL::VARCHAR, '%Y-%m-%d'), + (2, '2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +) AS t(id, value, format1, format2) +ORDER BY id +---- +1 2020-09-08T00:00:00 +2 NULL + +# to_unixtime uses the same formatted string parsing path +query II +SELECT + to_unixtime(NULL::VARCHAR, '%Y-%m-%d'), + to_unixtime('2020-09-08', NULL::VARCHAR, NULL::VARCHAR) +---- +NULL NULL + # to_timestamp float inputs query PPP --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
