This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24046-c1cb715f7f5bdd2710417bcddb212f14f8c8c5fd in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit a478fb17b31477c38bb7707bd18d13de0364ea3f Author: Nuno Faria <[email protected]> AuthorDate: Sat Aug 8 12:01:00 2026 +0100 fix: Correctly process numeric literals with underscores (#24046) ## Which issue does this PR close? - Closes #23877. ## Rationale for this change Be able to use numeric literals with underscores as separators, which are valid in some dialects like Postgres. ## What changes are included in this PR? The `parse_sql_number` method now removes underscores from numbers first before passing to the Rust parser. ## Are these changes tested? Yes. ## Are there any user-facing changes? No. --- datafusion/sql/src/expr/mod.rs | 28 ++++++++++++++++++++++++++++ datafusion/sql/src/expr/value.rs | 23 ++++++++++++++++++++--- datafusion/sqllogictest/test_files/expr.slt | 15 +++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index c2e4822f76..b1de4e95fd 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -1593,4 +1593,32 @@ mod tests { assert!(matches!(expr, Expr::Alias(_))); } + + #[test] + fn test_parse_numbers_with_underscores() { + use datafusion_common::ScalarValue::*; + + let context_provider = TestContextProvider::new(); + let sql_to_rel = SqlToRel::new(&context_provider); + + // (input, positive result, negative result) + let test_cases = [ + ("1_000", Int64(Some(1000)), Int64(Some(-1000))), + ("100_000", Int64(Some(100000)), Int64(Some(-100000))), + ("1_2_3_4", Int64(Some(1234)), Int64(Some(-1234))), + ("0_0", Int64(Some(0)), Int64(Some(-0))), + ("1_23.4_56", Float64(Some(123.456)), Float64(Some(-123.456))), + ]; + + for (literal, out_positive, out_negative) in test_cases { + assert_eq!( + sql_to_rel.parse_sql_number(literal, false).unwrap(), + Expr::Literal(out_positive, None) + ); + assert_eq!( + sql_to_rel.parse_sql_number(literal, true).unwrap(), + Expr::Literal(out_negative, None) + ); + } + } } diff --git a/datafusion/sql/src/expr/value.rs b/datafusion/sql/src/expr/value.rs index 13a47f545c..1307e917e4 100644 --- a/datafusion/sql/src/expr/value.rs +++ b/datafusion/sql/src/expr/value.rs @@ -74,10 +74,27 @@ impl<S: ContextProvider> SqlToRel<'_, S> { unsigned_number: &str, negative: bool, ) -> Result<Expr> { - let signed_number: Cow<str> = if negative { - Cow::Owned(format!("-{unsigned_number}")) - } else { + // remove underscores, since the Rust parser used here does not support them + let signed_number = if !negative && !unsigned_number.contains('_') { Cow::Borrowed(unsigned_number) + } else { + let mut signed_number = + String::with_capacity(unsigned_number.len() + usize::from(negative)); + if negative { + signed_number.push('-'); + } + unsigned_number.bytes().for_each(|b| { + if b != b'_' { + signed_number.push(b as char); + } + }); + Cow::Owned(signed_number) + }; + + let unsigned_number = if negative { + &signed_number[1..] + } else { + &signed_number }; // Try to parse as i64 first, then u64 if negative is false, then decimal or f64 diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index ba4e4d03b3..32113890aa 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -2665,3 +2665,18 @@ false statement ok drop table t; + + +# Test numeric literals with underscore separators +# (https://github.com/apache/datafusion/issues/23877) + +statement ok +set datafusion.sql_parser.dialect = 'postgres' + +query IIIRI +select 1_000, 1_2_3_4, -1_2_3_4, 1_2.3_4, 0_0 +---- +1000 1234 -1234 12.34 0 + +statement ok +reset datafusion.sql_parser.dialect --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
