This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 7fa7689c41 Update to `sqlparser-rs` v0.50.0 (#12014)
7fa7689c41 is described below
commit 7fa7689c41903bd7b4b6e049c60890d0042df03f
Author: Samuel Colvin <[email protected]>
AuthorDate: Sat Aug 17 11:46:27 2024 +0100
Update to `sqlparser-rs` v0.50.0 (#12014)
* Support HEAD of sqlparser main
* special case ID as a non-keyword when unparsing
* fix EXTRACT expresssions
* TODO REVERT: comment out failing test
Making this commit just to let tests progress.
* use sqlparser-rs v0.50.0
---
Cargo.toml | 2 +-
datafusion-cli/Cargo.lock | 4 +--
datafusion/sql/src/expr/mod.rs | 2 +-
datafusion/sql/src/planner.rs | 3 +-
datafusion/sql/src/relation/mod.rs | 1 +
datafusion/sql/src/statement.rs | 2 +-
datafusion/sql/src/unparser/ast.rs | 5 ++-
datafusion/sql/src/unparser/dialect.rs | 5 ++-
datafusion/sql/src/unparser/expr.rs | 1 +
datafusion/sql/src/unparser/plan.rs | 2 ++
datafusion/sql/src/utils.rs | 1 +
datafusion/sqllogictest/test_files/expr.slt | 52 +++++++++++++++++++++--------
12 files changed, 59 insertions(+), 21 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index dfb70dd1eb..ae344a46a1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -135,7 +135,7 @@ rand = "0.8"
regex = "1.8"
rstest = "0.22.0"
serde_json = "1"
-sqlparser = { version = "0.49", features = ["visitor"] }
+sqlparser = { version = "0.50.0", features = ["visitor"] }
tempfile = "3"
thiserror = "1.0.44"
tokio = { version = "1.36", features = ["macros", "rt", "sync"] }
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index cda57ba9b4..52e4a00035 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -3542,9 +3542,9 @@ checksum =
"6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "sqlparser"
-version = "0.49.0"
+version = "0.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e"
+checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac"
dependencies = [
"log",
"sqlparser_derive",
diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs
index 7c94e5ead5..035fd3816c 100644
--- a/datafusion/sql/src/expr/mod.rs
+++ b/datafusion/sql/src/expr/mod.rs
@@ -178,7 +178,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
SQLExpr::Value(value) => {
self.parse_value(value,
planner_context.prepare_param_data_types())
}
- SQLExpr::Extract { field, expr } => {
+ SQLExpr::Extract { field, expr, .. } => {
let mut extract_args = vec![
Expr::Literal(ScalarValue::from(format!("{field}"))),
self.sql_expr_to_logical_expr(*expr, schema,
planner_context)?,
diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs
index bf7c3fe0be..9ad515087a 100644
--- a/datafusion/sql/src/planner.rs
+++ b/datafusion/sql/src/planner.rs
@@ -438,7 +438,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
}
SQLDataType::Bytea => Ok(DataType::Binary),
SQLDataType::Interval =>
Ok(DataType::Interval(IntervalUnit::MonthDayNano)),
- SQLDataType::Struct(fields) => {
+ SQLDataType::Struct(fields, _) => {
let fields = fields
.iter()
.enumerate()
@@ -513,6 +513,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
| SQLDataType::Union(_)
| SQLDataType::Nullable(_)
| SQLDataType::LowCardinality(_)
+ | SQLDataType::Trigger
=> not_impl_err!(
"Unsupported SQL type {sql_type:?}"
),
diff --git a/datafusion/sql/src/relation/mod.rs
b/datafusion/sql/src/relation/mod.rs
index 5d7b3d5918..50027a0efd 100644
--- a/datafusion/sql/src/relation/mod.rs
+++ b/datafusion/sql/src/relation/mod.rs
@@ -36,6 +36,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
if let Some(func_args) = args {
let tbl_func_name =
name.0.first().unwrap().value.to_string();
let args = func_args
+ .args
.into_iter()
.flat_map(|arg| {
if let
FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) = arg
diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
index 6d47232ec2..e75a96e78d 100644
--- a/datafusion/sql/src/statement.rs
+++ b/datafusion/sql/src/statement.rs
@@ -198,8 +198,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
match statement {
Statement::ExplainTable {
describe_alias: DescribeAlias::Describe, // only parse
'DESCRIBE table_name' and not 'EXPLAIN table_name'
- hive_format: _,
table_name,
+ ..
} => self.describe_table_to_plan(table_name),
Statement::Explain {
verbose,
diff --git a/datafusion/sql/src/unparser/ast.rs
b/datafusion/sql/src/unparser/ast.rs
index c10db98314..71ff712985 100644
--- a/datafusion/sql/src/unparser/ast.rs
+++ b/datafusion/sql/src/unparser/ast.rs
@@ -428,7 +428,10 @@ impl TableRelationBuilder {
None => return
Err(Into::into(UninitializedFieldError::from("name"))),
},
alias: self.alias.clone(),
- args: self.args.clone(),
+ args: self.args.clone().map(|args| ast::TableFunctionArgs {
+ args,
+ settings: None,
+ }),
with_hints: self.with_hints.clone(),
version: self.version.clone(),
partitions: self.partitions.clone(),
diff --git a/datafusion/sql/src/unparser/dialect.rs
b/datafusion/sql/src/unparser/dialect.rs
index 7eca326386..74f154d787 100644
--- a/datafusion/sql/src/unparser/dialect.rs
+++ b/datafusion/sql/src/unparser/dialect.rs
@@ -131,7 +131,10 @@ pub struct DefaultDialect {}
impl Dialect for DefaultDialect {
fn identifier_quote_style(&self, identifier: &str) -> Option<char> {
let identifier_regex =
Regex::new(r"^[a-zA-Z_][a-zA-Z0-9_]*$").unwrap();
- if ALL_KEYWORDS.contains(&identifier.to_uppercase().as_str())
+ let id_upper = identifier.to_uppercase();
+ // special case ignore "ID", see
https://github.com/sqlparser-rs/sqlparser-rs/issues/1382
+ // ID is a keyword in ClickHouse, but we don't want to quote it when
unparsing SQL here
+ if (id_upper != "ID" && ALL_KEYWORDS.contains(&id_upper.as_str()))
|| !identifier_regex.is_match(identifier)
{
Some('"')
diff --git a/datafusion/sql/src/unparser/expr.rs
b/datafusion/sql/src/unparser/expr.rs
index c941d7098d..9ce627aecc 100644
--- a/datafusion/sql/src/unparser/expr.rs
+++ b/datafusion/sql/src/unparser/expr.rs
@@ -592,6 +592,7 @@ impl Unparser<'_> {
return Some(ast::Expr::Extract {
field,
expr: Box::new(date_expr),
+ syntax: ast::ExtractSyntax::From,
});
}
}
diff --git a/datafusion/sql/src/unparser/plan.rs
b/datafusion/sql/src/unparser/plan.rs
index 024f33fb2c..8b5a5b0942 100644
--- a/datafusion/sql/src/unparser/plan.rs
+++ b/datafusion/sql/src/unparser/plan.rs
@@ -403,6 +403,7 @@ impl Unparser<'_> {
let ast_join = ast::Join {
relation,
+ global: false,
join_operator: self
.join_operator_to_sql(join.join_type, join_constraint),
};
@@ -435,6 +436,7 @@ impl Unparser<'_> {
let ast_join = ast::Join {
relation,
+ global: false,
join_operator: self.join_operator_to_sql(
JoinType::Inner,
ast::JoinConstraint::On(ast::Expr::Value(ast::Value::Boolean(
diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs
index 5cdc546e02..af161bba45 100644
--- a/datafusion/sql/src/utils.rs
+++ b/datafusion/sql/src/utils.rs
@@ -268,6 +268,7 @@ pub(crate) fn value_to_string(value: &Value) ->
Option<String> {
Value::SingleQuotedString(s) => Some(s.to_string()),
Value::DollarQuotedString(s) => Some(s.to_string()),
Value::Number(_, _) | Value::Boolean(_) => Some(value.to_string()),
+ Value::UnicodeStringLiteral(s) => Some(s.to_string()),
Value::DoubleQuotedString(_)
| Value::EscapedStringLiteral(_)
| Value::NationalStringLiteral(_)
diff --git a/datafusion/sqllogictest/test_files/expr.slt
b/datafusion/sqllogictest/test_files/expr.slt
index 3c3b0631e3..81ae60f3ba 100644
--- a/datafusion/sqllogictest/test_files/expr.slt
+++ b/datafusion/sqllogictest/test_files/expr.slt
@@ -848,8 +848,10 @@ SELECT EXTRACT("year" FROM timestamp
'2020-09-08T12:00:00+00:00')
----
2020
-query error
+query R
SELECT EXTRACT('year' FROM timestamp '2020-09-08T12:00:00+00:00')
+----
+2020
query R
SELECT date_part('QUARTER', CAST('2000-01-01' AS DATE))
@@ -866,8 +868,10 @@ SELECT EXTRACT("quarter" FROM
to_timestamp('2020-09-08T12:00:00+00:00'))
----
3
-query error
+query R
SELECT EXTRACT('quarter' FROM to_timestamp('2020-09-08T12:00:00+00:00'))
+----
+3
query R
SELECT date_part('MONTH', CAST('2000-01-01' AS DATE))
@@ -884,8 +888,10 @@ SELECT EXTRACT("month" FROM
to_timestamp('2020-09-08T12:00:00+00:00'))
----
9
-query error
+query R
SELECT EXTRACT('month' FROM to_timestamp('2020-09-08T12:00:00+00:00'))
+----
+9
query R
SELECT date_part('WEEK', CAST('2003-01-01' AS DATE))
@@ -902,8 +908,10 @@ SELECT EXTRACT("WEEK" FROM
to_timestamp('2020-09-08T12:00:00+00:00'))
----
37
-query error
+query R
SELECT EXTRACT('WEEK' FROM to_timestamp('2020-09-08T12:00:00+00:00'))
+----
+37
query R
SELECT date_part('DAY', CAST('2000-01-01' AS DATE))
@@ -920,8 +928,10 @@ SELECT EXTRACT("day" FROM
to_timestamp('2020-09-08T12:00:00+00:00'))
----
8
-query error
+query R
SELECT EXTRACT('day' FROM to_timestamp('2020-09-08T12:00:00+00:00'))
+----
+8
query R
SELECT date_part('DOY', CAST('2000-01-01' AS DATE))
@@ -938,8 +948,10 @@ SELECT EXTRACT("doy" FROM
to_timestamp('2020-09-08T12:00:00+00:00'))
----
252
-query error
+query R
SELECT EXTRACT('doy' FROM to_timestamp('2020-09-08T12:00:00+00:00'))
+----
+252
query R
SELECT date_part('DOW', CAST('2000-01-01' AS DATE))
@@ -956,8 +968,10 @@ SELECT EXTRACT("dow" FROM
to_timestamp('2020-09-08T12:00:00+00:00'))
----
2
-query error
+query R
SELECT EXTRACT('dow' FROM to_timestamp('2020-09-08T12:00:00+00:00'))
+----
+2
query R
SELECT date_part('HOUR', CAST('2000-01-01' AS DATE))
@@ -974,8 +988,10 @@ SELECT EXTRACT("hour" FROM
to_timestamp('2020-09-08T12:03:03+00:00'))
----
12
-query error
+query R
SELECT EXTRACT('hour' FROM to_timestamp('2020-09-08T12:03:03+00:00'))
+----
+12
query R
SELECT EXTRACT(minute FROM to_timestamp('2020-09-08T12:12:00+00:00'))
@@ -987,8 +1003,10 @@ SELECT EXTRACT("minute" FROM
to_timestamp('2020-09-08T12:12:00+00:00'))
----
12
-query error
+query R
SELECT EXTRACT('minute' FROM to_timestamp('2020-09-08T12:12:00+00:00'))
+----
+12
query R
SELECT date_part('minute', to_timestamp('2020-09-08T12:12:00+00:00'))
@@ -1035,17 +1053,25 @@ SELECT EXTRACT("nanosecond" FROM timestamp
'2020-09-08T12:00:12.12345678+00:00')
----
12123456780
-query error
+query R
SELECT EXTRACT('second' FROM timestamp '2020-09-08T12:00:12.12345678+00:00')
+----
+12.12345678
-query error
+query R
SELECT EXTRACT('millisecond' FROM timestamp
'2020-09-08T12:00:12.12345678+00:00')
+----
+12123.45678
-query error
+query R
SELECT EXTRACT('microsecond' FROM timestamp
'2020-09-08T12:00:12.12345678+00:00')
+----
+12123456.78
-query error
+query R
SELECT EXTRACT('nanosecond' FROM timestamp
'2020-09-08T12:00:12.12345678+00:00')
+----
+12123456780
# Keep precision when coercing Utf8 to Timestamp
query R
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
