This is an automated email from the ASF dual-hosted git repository.
iffyio pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git
The following commit(s) were added to refs/heads/main by this push:
new b1b379e5 Add support of parsing struct field's options in BigQuery
(#1890)
b1b379e5 is described below
commit b1b379e57086ad7fcfa67136c0c64b14d4efab9d
Author: hulk <hulk.webs...@gmail.com>
AuthorDate: Wed Jun 18 13:00:53 2025 +0800
Add support of parsing struct field's options in BigQuery (#1890)
Co-authored-by: Ifeanyi Ubah <ify1...@yahoo.com>
---
src/ast/mod.rs | 12 +++-
src/parser/mod.rs | 3 +
tests/sqlparser_bigquery.rs | 159 +++++++++++++++++++++++++++++-------------
tests/sqlparser_clickhouse.rs | 10 ++-
tests/sqlparser_duckdb.rs | 6 ++
5 files changed, 138 insertions(+), 52 deletions(-)
diff --git a/src/ast/mod.rs b/src/ast/mod.rs
index a1d8ff6f..04401a48 100644
--- a/src/ast/mod.rs
+++ b/src/ast/mod.rs
@@ -428,14 +428,22 @@ impl fmt::Display for Interval {
pub struct StructField {
pub field_name: Option<Ident>,
pub field_type: DataType,
+ /// Struct field options.
+ /// See
[BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#column_name_and_column_schema)
+ pub options: Option<Vec<SqlOption>>,
}
impl fmt::Display for StructField {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if let Some(name) = &self.field_name {
- write!(f, "{name} {}", self.field_type)
+ write!(f, "{name} {}", self.field_type)?;
} else {
- write!(f, "{}", self.field_type)
+ write!(f, "{}", self.field_type)?;
+ }
+ if let Some(options) = &self.options {
+ write!(f, " OPTIONS({})", display_separated(options, ", "))
+ } else {
+ Ok(())
}
}
}
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index 73cc3e0e..19c11d29 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -3076,6 +3076,7 @@ impl<'a> Parser<'a> {
Ok(StructField {
field_name: Some(field_name),
field_type,
+ options: None,
})
});
self.expect_token(&Token::RParen)?;
@@ -3109,10 +3110,12 @@ impl<'a> Parser<'a> {
let (field_type, trailing_bracket) = self.parse_data_type_helper()?;
+ let options = self.maybe_parse_options(Keyword::OPTIONS)?;
Ok((
StructField {
field_name,
field_type,
+ options,
},
trailing_bracket,
))
diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs
index 6a303577..e37e4a44 100644
--- a/tests/sqlparser_bigquery.rs
+++ b/tests/sqlparser_bigquery.rs
@@ -601,11 +601,13 @@ fn parse_nested_data_types() {
field_name: Some("a".into()),
field_type:
DataType::Array(ArrayElemTypeDef::AngleBracket(
Box::new(DataType::Int64,)
- ))
+ )),
+ options: None,
},
StructField {
field_name: Some("b".into()),
- field_type: DataType::Bytes(Some(42))
+ field_type: DataType::Bytes(Some(42)),
+ options: None,
},
],
StructBracketKind::AngleBrackets
@@ -619,6 +621,7 @@ fn parse_nested_data_types() {
vec![StructField {
field_name: None,
field_type: DataType::Int64,
+ options: None,
}],
StructBracketKind::AngleBrackets
),
@@ -771,6 +774,7 @@ fn parse_typed_struct_syntax_bigquery() {
fields: vec![StructField {
field_name: None,
field_type: DataType::Int64,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -799,7 +803,8 @@ fn parse_typed_struct_syntax_bigquery() {
quote_style: None,
span: Span::empty(),
}),
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
},
StructField {
field_name: Some(Ident {
@@ -807,7 +812,8 @@ fn parse_typed_struct_syntax_bigquery() {
quote_style: None,
span: Span::empty(),
}),
- field_type: DataType::String(None)
+ field_type: DataType::String(None),
+ options: None,
},
]
},
@@ -825,17 +831,20 @@ fn parse_typed_struct_syntax_bigquery() {
field_name: Some("arr".into()),
field_type:
DataType::Array(ArrayElemTypeDef::AngleBracket(Box::new(
DataType::Float64
- )))
+ ))),
+ options: None,
},
StructField {
field_name: Some("str".into()),
field_type: DataType::Struct(
vec![StructField {
field_name: None,
- field_type: DataType::Bool
+ field_type: DataType::Bool,
+ options: None,
}],
StructBracketKind::AngleBrackets
- )
+ ),
+ options: None,
},
]
},
@@ -858,13 +867,15 @@ fn parse_typed_struct_syntax_bigquery() {
field_type: DataType::Struct(
Default::default(),
StructBracketKind::AngleBrackets
- )
+ ),
+ options: None,
},
StructField {
field_name: Some("y".into()),
field_type:
DataType::Array(ArrayElemTypeDef::AngleBracket(Box::new(
DataType::Struct(Default::default(),
StructBracketKind::AngleBrackets)
- )))
+ ))),
+ options: None,
},
]
},
@@ -879,7 +890,8 @@ fn parse_typed_struct_syntax_bigquery() {
values: vec![Expr::Value(Value::Boolean(true).with_empty_span())],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Bool
+ field_type: DataType::Bool,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -891,7 +903,8 @@ fn parse_typed_struct_syntax_bigquery() {
)],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Bytes(Some(42))
+ field_type: DataType::Bytes(Some(42)),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -907,7 +920,8 @@ fn parse_typed_struct_syntax_bigquery() {
)],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Date
+ field_type: DataType::Date,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -920,7 +934,8 @@ fn parse_typed_struct_syntax_bigquery() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Datetime(None)
+ field_type: DataType::Datetime(None),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -930,7 +945,8 @@ fn parse_typed_struct_syntax_bigquery() {
values: vec![Expr::value(number("5.0"))],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Float64
+ field_type: DataType::Float64,
+ options: None,
}]
},
expr_from_projection(&select.projection[2])
@@ -940,7 +956,8 @@ fn parse_typed_struct_syntax_bigquery() {
values: vec![Expr::value(number("1"))],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
}]
},
expr_from_projection(&select.projection[3])
@@ -962,7 +979,8 @@ fn parse_typed_struct_syntax_bigquery() {
})],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Interval
+ field_type: DataType::Interval,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -977,7 +995,8 @@ fn parse_typed_struct_syntax_bigquery() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::JSON
+ field_type: DataType::JSON,
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -993,7 +1012,8 @@ fn parse_typed_struct_syntax_bigquery() {
)],
fields: vec![StructField {
field_name: None,
- field_type: DataType::String(Some(42))
+ field_type: DataType::String(Some(42)),
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1006,7 +1026,8 @@ fn parse_typed_struct_syntax_bigquery() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Timestamp(None, TimezoneInfo::None)
+ field_type: DataType::Timestamp(None, TimezoneInfo::None),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1020,7 +1041,8 @@ fn parse_typed_struct_syntax_bigquery() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Time(None, TimezoneInfo::None)
+ field_type: DataType::Time(None, TimezoneInfo::None),
+ options: None,
}]
},
expr_from_projection(&select.projection[2])
@@ -1037,7 +1059,8 @@ fn parse_typed_struct_syntax_bigquery() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Numeric(ExactNumberInfo::None)
+ field_type: DataType::Numeric(ExactNumberInfo::None),
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1050,7 +1073,8 @@ fn parse_typed_struct_syntax_bigquery() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::BigNumeric(ExactNumberInfo::None)
+ field_type: DataType::BigNumeric(ExactNumberInfo::None),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1067,10 +1091,12 @@ fn parse_typed_struct_syntax_bigquery() {
StructField {
field_name: Some("key".into()),
field_type: DataType::Int64,
+ options: None,
},
StructField {
field_name: Some("value".into()),
field_type: DataType::Int64,
+ options: None,
},
]
},
@@ -1092,6 +1118,7 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
fields: vec![StructField {
field_name: None,
field_type: DataType::Int64,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1120,7 +1147,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
quote_style: None,
span: Span::empty(),
}),
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
},
StructField {
field_name: Some(Ident {
@@ -1128,7 +1156,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
quote_style: None,
span: Span::empty(),
}),
- field_type: DataType::String(None)
+ field_type: DataType::String(None),
+ options: None,
},
]
},
@@ -1151,13 +1180,15 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
field_type: DataType::Struct(
Default::default(),
StructBracketKind::AngleBrackets
- )
+ ),
+ options: None,
},
StructField {
field_name: Some("y".into()),
field_type:
DataType::Array(ArrayElemTypeDef::AngleBracket(Box::new(
DataType::Struct(Default::default(),
StructBracketKind::AngleBrackets)
- )))
+ ))),
+ options: None,
},
]
},
@@ -1172,7 +1203,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
values: vec![Expr::Value(Value::Boolean(true).with_empty_span())],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Bool
+ field_type: DataType::Bool,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1184,7 +1216,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
)],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Bytes(Some(42))
+ field_type: DataType::Bytes(Some(42)),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1200,7 +1233,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
)],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Date
+ field_type: DataType::Date,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1213,7 +1247,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Datetime(None)
+ field_type: DataType::Datetime(None),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1223,7 +1258,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
values: vec![Expr::value(number("5.0"))],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Float64
+ field_type: DataType::Float64,
+ options: None,
}]
},
expr_from_projection(&select.projection[2])
@@ -1233,7 +1269,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
values: vec![Expr::value(number("1"))],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
}]
},
expr_from_projection(&select.projection[3])
@@ -1255,7 +1292,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
})],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Interval
+ field_type: DataType::Interval,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1270,7 +1308,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::JSON
+ field_type: DataType::JSON,
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1286,7 +1325,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
)],
fields: vec![StructField {
field_name: None,
- field_type: DataType::String(Some(42))
+ field_type: DataType::String(Some(42)),
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1299,7 +1339,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Timestamp(None, TimezoneInfo::None)
+ field_type: DataType::Timestamp(None, TimezoneInfo::None),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1313,7 +1354,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Time(None, TimezoneInfo::None)
+ field_type: DataType::Time(None, TimezoneInfo::None),
+ options: None,
}]
},
expr_from_projection(&select.projection[2])
@@ -1330,7 +1372,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::Numeric(ExactNumberInfo::None)
+ field_type: DataType::Numeric(ExactNumberInfo::None),
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1343,7 +1386,8 @@ fn parse_typed_struct_syntax_bigquery_and_generic() {
}],
fields: vec![StructField {
field_name: None,
- field_type: DataType::BigNumeric(ExactNumberInfo::None)
+ field_type: DataType::BigNumeric(ExactNumberInfo::None),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1360,7 +1404,8 @@ fn parse_typed_struct_with_field_name_bigquery() {
values: vec![Expr::value(number("5"))],
fields: vec![StructField {
field_name: Some(Ident::from("x")),
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1372,7 +1417,8 @@ fn parse_typed_struct_with_field_name_bigquery() {
)],
fields: vec![StructField {
field_name: Some(Ident::from("y")),
- field_type: DataType::String(None)
+ field_type: DataType::String(None),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1387,11 +1433,13 @@ fn parse_typed_struct_with_field_name_bigquery() {
fields: vec![
StructField {
field_name: Some(Ident::from("x")),
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
},
StructField {
field_name: Some(Ident::from("y")),
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
}
]
},
@@ -1409,7 +1457,8 @@ fn
parse_typed_struct_with_field_name_bigquery_and_generic() {
values: vec![Expr::value(number("5"))],
fields: vec![StructField {
field_name: Some(Ident::from("x")),
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
}]
},
expr_from_projection(&select.projection[0])
@@ -1421,7 +1470,8 @@ fn
parse_typed_struct_with_field_name_bigquery_and_generic() {
)],
fields: vec![StructField {
field_name: Some(Ident::from("y")),
- field_type: DataType::String(None)
+ field_type: DataType::String(None),
+ options: None,
}]
},
expr_from_projection(&select.projection[1])
@@ -1436,11 +1486,13 @@ fn
parse_typed_struct_with_field_name_bigquery_and_generic() {
fields: vec![
StructField {
field_name: Some(Ident::from("x")),
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
},
StructField {
field_name: Some(Ident::from("y")),
- field_type: DataType::Int64
+ field_type: DataType::Int64,
+ options: None,
}
]
},
@@ -2407,3 +2459,16 @@ fn test_any_type() {
fn test_any_type_dont_break_custom_type() {
bigquery_and_generic().verified_stmt("CREATE TABLE foo (x ANY)");
}
+
+#[test]
+fn test_struct_field_options() {
+ bigquery().verified_stmt(concat!(
+ "CREATE TABLE my_table (",
+ "f0 STRUCT<a STRING, b INT64>, ",
+ "f1 STRUCT<",
+ "a STRING OPTIONS(description = 'This is a string', type = 'string'),
",
+ "b INT64",
+ "> OPTIONS(description = 'This is a struct field')",
+ ")",
+ ));
+}
diff --git a/tests/sqlparser_clickhouse.rs b/tests/sqlparser_clickhouse.rs
index d0218b6c..93b4c4f5 100644
--- a/tests/sqlparser_clickhouse.rs
+++ b/tests/sqlparser_clickhouse.rs
@@ -669,11 +669,13 @@ fn parse_create_table_with_nested_data_types() {
DataType::Tuple(vec![
StructField {
field_name: None,
- field_type: DataType::FixedString(128)
+ field_type: DataType::FixedString(128),
+ options: None,
},
StructField {
field_name: None,
- field_type: DataType::Int128
+ field_type: DataType::Int128,
+ options: None,
}
])
))),
@@ -685,12 +687,14 @@ fn parse_create_table_with_nested_data_types() {
StructField {
field_name: Some("a".into()),
field_type: DataType::Datetime64(9, None),
+ options: None,
},
StructField {
field_name: Some("b".into()),
field_type:
DataType::Array(ArrayElemTypeDef::Parenthesis(
Box::new(DataType::Uuid)
- ))
+ )),
+ options: None,
},
]),
options: vec![],
diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs
index 8e498365..f503ed55 100644
--- a/tests/sqlparser_duckdb.rs
+++ b/tests/sqlparser_duckdb.rs
@@ -44,10 +44,12 @@ fn test_struct() {
StructField {
field_name: Some(Ident::new("v")),
field_type: DataType::Varchar(None),
+ options: None,
},
StructField {
field_name: Some(Ident::new("i")),
field_type: DataType::Integer(None),
+ options: None,
},
],
StructBracketKind::Parentheses,
@@ -84,6 +86,7 @@ fn test_struct() {
StructField {
field_name: Some(Ident::new("v")),
field_type: DataType::Varchar(None),
+ options: None,
},
StructField {
field_name: Some(Ident::new("s")),
@@ -92,14 +95,17 @@ fn test_struct() {
StructField {
field_name: Some(Ident::new("a1")),
field_type: DataType::Integer(None),
+ options: None,
},
StructField {
field_name: Some(Ident::new("a2")),
field_type: DataType::Varchar(None),
+ options: None,
},
],
StructBracketKind::Parentheses,
),
+ options: None,
},
],
StructBracketKind::Parentheses,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org
