This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 4d02cc0114 Update to sqlparser `0.42.0` (#9000)
4d02cc0114 is described below
commit 4d02cc0114908d4f805b2323f20751b1f6d9c2f4
Author: Andrew Lamb <[email protected]>
AuthorDate: Thu Jan 25 17:10:12 2024 -0500
Update to sqlparser `0.42.0` (#9000)
* Update to sqlparser `0.42.0`
* Update datafusion Cargo.lock
---
Cargo.toml | 2 +-
datafusion-cli/Cargo.lock | 30 ++++++++++-----------
datafusion/sql/src/parser.rs | 12 ++++-----
datafusion/sql/src/planner.rs | 2 ++
datafusion/sql/src/statement.rs | 59 ++++++++++++++++++++++++++++++++---------
5 files changed, 71 insertions(+), 34 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index cd88e18fe1..3b1362d224 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -70,7 +70,7 @@ parquet = { version = "50.0.0", default-features = false,
features = ["arrow", "
rand = "0.8"
rstest = "0.18.0"
serde_json = "1"
-sqlparser = { version = "0.41.0", features = ["visitor"] }
+sqlparser = { version = "0.43.0", features = ["visitor"] }
tempfile = "3"
thiserror = "1.0.44"
url = "2.2"
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index c90b59b924..a718f7591a 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -867,15 +867,15 @@ checksum =
"baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
-version = "0.4.31"
+version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
+checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb"
dependencies = [
"android-tzdata",
"iana-time-zone",
"num-traits",
"serde",
- "windows-targets 0.48.5",
+ "windows-targets 0.52.0",
]
[[package]]
@@ -2475,18 +2475,18 @@ dependencies = [
[[package]]
name = "pin-project"
-version = "1.1.3"
+version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
+checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
-version = "1.1.3"
+version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
+checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690"
dependencies = [
"proc-macro2",
"quote",
@@ -2579,9 +2579,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.76"
+version = "1.0.78"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
+checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
dependencies = [
"unicode-ident",
]
@@ -2673,9 +2673,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.10.2"
+version = "1.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
dependencies = [
"aho-corasick",
"memchr",
@@ -2685,9 +2685,9 @@ dependencies = [
[[package]]
name = "regex-automata"
-version = "0.4.3"
+version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd"
dependencies = [
"aho-corasick",
"memchr",
@@ -3148,9 +3148,9 @@ checksum =
"6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "sqlparser"
-version = "0.41.0"
+version = "0.43.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964"
+checksum = "a748c164141797ef0a712aaf16aa71df6f23e80ffea446daa2dd30e3325f89f3"
dependencies = [
"log",
"sqlparser_derive",
diff --git a/datafusion/sql/src/parser.rs b/datafusion/sql/src/parser.rs
index dbd72ec5eb..effc1d096c 100644
--- a/datafusion/sql/src/parser.rs
+++ b/datafusion/sql/src/parser.rs
@@ -366,7 +366,7 @@ impl<'a> DFParser<'a> {
CopyToSource::Query(query)
} else {
// parse as table reference
- let table_name = self.parser.parse_object_name()?;
+ let table_name = self.parser.parse_object_name(true)?;
CopyToSource::Relation(table_name)
};
@@ -465,7 +465,7 @@ impl<'a> DFParser<'a> {
loop {
if let Token::Word(_) = self.parser.peek_token().token {
- let identifier = self.parser.parse_identifier()?;
+ let identifier = self.parser.parse_identifier(false)?;
partitions.push(identifier.to_string());
} else {
return self.expected("partition name",
self.parser.peek_token());
@@ -567,17 +567,17 @@ impl<'a> DFParser<'a> {
}
fn parse_column_def(&mut self) -> Result<ColumnDef, ParserError> {
- let name = self.parser.parse_identifier()?;
+ let name = self.parser.parse_identifier(false)?;
let data_type = self.parser.parse_data_type()?;
let collation = if self.parser.parse_keyword(Keyword::COLLATE) {
- Some(self.parser.parse_object_name()?)
+ Some(self.parser.parse_object_name(false)?)
} else {
None
};
let mut options = vec![];
loop {
if self.parser.parse_keyword(Keyword::CONSTRAINT) {
- let name = Some(self.parser.parse_identifier()?);
+ let name = Some(self.parser.parse_identifier(false)?);
if let Some(option) =
self.parser.parse_optional_column_option()? {
options.push(ColumnOptionDef { name, option });
} else {
@@ -608,7 +608,7 @@ impl<'a> DFParser<'a> {
let if_not_exists =
self.parser
.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
- let table_name = self.parser.parse_object_name()?;
+ let table_name = self.parser.parse_object_name(true)?;
let (columns, constraints) = self.parse_columns()?;
#[derive(Default)]
diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs
index d4dd42edcd..f1d4f3ff56 100644
--- a/datafusion/sql/src/planner.rs
+++ b/datafusion/sql/src/planner.rs
@@ -465,6 +465,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
| SQLDataType::Int64
| SQLDataType::Float64
| SQLDataType::Struct(_)
+ | SQLDataType::JSONB
+ | SQLDataType::Unspecified
=> not_impl_err!(
"Unsupported SQL type {sql_type:?}"
),
diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs
index b9fb4c65dc..84a394f324 100644
--- a/datafusion/sql/src/statement.rs
+++ b/datafusion/sql/src/statement.rs
@@ -52,9 +52,10 @@ use datafusion_expr::{
};
use sqlparser::ast;
use sqlparser::ast::{
- Assignment, ColumnDef, Expr as SQLExpr, Expr, Ident, ObjectName,
ObjectType, Query,
- SchemaName, SetExpr, ShowCreateObject, ShowStatementFilter, Statement,
- TableConstraint, TableFactor, TableWithJoins, TransactionMode,
UnaryOperator, Value,
+ Assignment, ColumnDef, CreateTableOptions, Expr as SQLExpr, Expr, Ident,
ObjectName,
+ ObjectType, Query, SchemaName, SetExpr, ShowCreateObject,
ShowStatementFilter,
+ Statement, TableConstraint, TableFactor, TableWithJoins, TransactionMode,
+ UnaryOperator, Value,
};
use sqlparser::parser::ParserError::ParserError;
@@ -90,18 +91,21 @@ fn calc_inline_constraints_from_columns(columns:
&[ColumnDef]) -> Vec<TableConst
for column in columns {
for ast::ColumnOptionDef { name, option } in &column.options {
match option {
- ast::ColumnOption::Unique { is_primary } => {
- constraints.push(ast::TableConstraint::Unique {
- name: name.clone(),
- columns: vec![column.name.clone()],
- is_primary: *is_primary,
- })
- }
+ ast::ColumnOption::Unique {
+ is_primary,
+ characteristics,
+ } => constraints.push(ast::TableConstraint::Unique {
+ name: name.clone(),
+ columns: vec![column.name.clone()],
+ is_primary: *is_primary,
+ characteristics: *characteristics,
+ }),
ast::ColumnOption::ForeignKey {
foreign_table,
referred_columns,
on_delete,
on_update,
+ characteristics,
} => constraints.push(ast::TableConstraint::ForeignKey {
name: name.clone(),
columns: vec![],
@@ -109,6 +113,7 @@ fn calc_inline_constraints_from_columns(columns:
&[ColumnDef]) -> Vec<TableConst
referred_columns: referred_columns.to_vec(),
on_delete: *on_delete,
on_update: *on_update,
+ characteristics: *characteristics,
}),
ast::ColumnOption::Check(expr) => {
constraints.push(ast::TableConstraint::Check {
@@ -124,6 +129,7 @@ fn calc_inline_constraints_from_columns(columns:
&[ColumnDef]) -> Vec<TableConst
| ast::ColumnOption::CharacterSet(_)
| ast::ColumnOption::Generated { .. }
| ast::ColumnOption::Comment(_)
+ | ast::ColumnOption::Options(_)
| ast::ColumnOption::OnUpdate(_) => {}
}
}
@@ -292,9 +298,22 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
name,
columns,
query,
- with_options,
+ options: CreateTableOptions::None,
..
- } if with_options.is_empty() => {
+ } => {
+ let columns = columns
+ .into_iter()
+ .map(|view_column_def| {
+ if let Some(options) = view_column_def.options {
+ plan_err!(
+ "Options not supported for view columns:
{options:?}"
+ )
+ } else {
+ Ok(view_column_def.name)
+ }
+ })
+ .collect::<Result<Vec<_>>>()?;
+
let mut plan = self.query_to_plan(*query, &mut
PlannerContext::new())?;
plan = self.apply_expr_alias(plan, columns)?;
@@ -440,6 +459,9 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
on,
returning,
ignore,
+ table_alias,
+ replace_into,
+ priority,
} => {
if or.is_some() {
plan_err!("Inserts with or clauses not supported")?;
@@ -465,6 +487,19 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
let Some(source) = source else {
plan_err!("Inserts without a source not supported")?
};
+ if let Some(table_alias) = table_alias {
+ plan_err!(
+ "Inserts with a table alias not supported:
{table_alias:?}"
+ )?
+ };
+ if replace_into {
+ plan_err!("Inserts with a `REPLACE INTO` clause not
supported")?
+ };
+ if let Some(priority) = priority {
+ plan_err!(
+ "Inserts with a `PRIORITY` clause not supported:
{priority:?}"
+ )?
+ };
let _ = into; // optional keyword doesn't change behavior
self.insert_to_plan(table_name, columns, source, overwrite)
}
