This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git
The following commit(s) were added to refs/heads/main by this push:
new a134910a Rename `TokenWithLocation` to `TokenWithSpan`, in backwards
compatible way (#1562)
a134910a is described below
commit a134910a362d12acb668ddf63239525073e7340f
Author: Andrew Lamb <[email protected]>
AuthorDate: Sat Nov 30 07:55:21 2024 -0500
Rename `TokenWithLocation` to `TokenWithSpan`, in backwards compatible way
(#1562)
---
src/ast/helpers/attached_token.rs | 10 +++---
src/ast/query.rs | 4 +--
src/parser/mod.rs | 66 +++++++++++++++++++--------------------
src/tokenizer.rs | 50 +++++++++++++++--------------
4 files changed, 67 insertions(+), 63 deletions(-)
diff --git a/src/ast/helpers/attached_token.rs
b/src/ast/helpers/attached_token.rs
index 48696c33..ed340359 100644
--- a/src/ast/helpers/attached_token.rs
+++ b/src/ast/helpers/attached_token.rs
@@ -19,7 +19,7 @@ use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
use core::fmt::{self, Debug, Formatter};
use core::hash::{Hash, Hasher};
-use crate::tokenizer::{Token, TokenWithLocation};
+use crate::tokenizer::{Token, TokenWithSpan};
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
@@ -33,11 +33,11 @@ use sqlparser_derive::{Visit, VisitMut};
#[derive(Clone)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
-pub struct AttachedToken(pub TokenWithLocation);
+pub struct AttachedToken(pub TokenWithSpan);
impl AttachedToken {
pub fn empty() -> Self {
- AttachedToken(TokenWithLocation::wrap(Token::EOF))
+ AttachedToken(TokenWithSpan::wrap(Token::EOF))
}
}
@@ -75,8 +75,8 @@ impl Hash for AttachedToken {
}
}
-impl From<TokenWithLocation> for AttachedToken {
- fn from(value: TokenWithLocation) -> Self {
+impl From<TokenWithSpan> for AttachedToken {
+ fn from(value: TokenWithSpan) -> Self {
AttachedToken(value)
}
}
diff --git a/src/ast/query.rs b/src/ast/query.rs
index 716ffe98..f3a76d89 100644
--- a/src/ast/query.rs
+++ b/src/ast/query.rs
@@ -27,7 +27,7 @@ use sqlparser_derive::{Visit, VisitMut};
use crate::{
ast::*,
- tokenizer::{Token, TokenWithLocation},
+ tokenizer::{Token, TokenWithSpan},
};
/// The most complete variant of a `SELECT` query expression, optionally
@@ -643,7 +643,7 @@ pub struct WildcardAdditionalOptions {
impl Default for WildcardAdditionalOptions {
fn default() -> Self {
Self {
- wildcard_token: TokenWithLocation::wrap(Token::Mul).into(),
+ wildcard_token: TokenWithSpan::wrap(Token::Mul).into(),
opt_ilike: None,
opt_exclude: None,
opt_except: None,
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index fe6fae8b..1f8dc8ba 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -265,7 +265,7 @@ enum ParserState {
}
pub struct Parser<'a> {
- tokens: Vec<TokenWithLocation>,
+ tokens: Vec<TokenWithSpan>,
/// The index of the first unprocessed token in [`Parser::tokens`].
index: usize,
/// The current state of the parser.
@@ -359,7 +359,7 @@ impl<'a> Parser<'a> {
}
/// Reset this parser to parse the specified token stream
- pub fn with_tokens_with_locations(mut self, tokens:
Vec<TokenWithLocation>) -> Self {
+ pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithSpan>) ->
Self {
self.tokens = tokens;
self.index = 0;
self
@@ -368,9 +368,9 @@ impl<'a> Parser<'a> {
/// Reset this parser state to parse the specified tokens
pub fn with_tokens(self, tokens: Vec<Token>) -> Self {
// Put in dummy locations
- let tokens_with_locations: Vec<TokenWithLocation> = tokens
+ let tokens_with_locations: Vec<TokenWithSpan> = tokens
.into_iter()
- .map(|token| TokenWithLocation {
+ .map(|token| TokenWithSpan {
token,
span: Span::empty(),
})
@@ -1147,7 +1147,7 @@ impl<'a> Parser<'a> {
match self.peek_token().token {
Token::LParen | Token::Period => {
let mut id_parts: Vec<Ident> = vec![w.to_ident(w_span)];
- let mut ending_wildcard: Option<TokenWithLocation> = None;
+ let mut ending_wildcard: Option<TokenWithSpan> = None;
while self.consume_token(&Token::Period) {
let next_token = self.next_token();
match next_token.token {
@@ -3273,7 +3273,7 @@ impl<'a> Parser<'a> {
/// Return the first non-whitespace token that has not yet been processed
/// (or None if reached end-of-file)
- pub fn peek_token(&self) -> TokenWithLocation {
+ pub fn peek_token(&self) -> TokenWithSpan {
self.peek_nth_token(0)
}
@@ -3308,19 +3308,19 @@ impl<'a> Parser<'a> {
/// yet been processed.
///
/// See [`Self::peek_token`] for an example.
- pub fn peek_tokens_with_location<const N: usize>(&self) ->
[TokenWithLocation; N] {
+ pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithSpan;
N] {
let mut index = self.index;
core::array::from_fn(|_| loop {
let token = self.tokens.get(index);
index += 1;
- if let Some(TokenWithLocation {
+ if let Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) = token
{
continue;
}
- break token.cloned().unwrap_or(TokenWithLocation {
+ break token.cloned().unwrap_or(TokenWithSpan {
token: Token::EOF,
span: Span::empty(),
});
@@ -3328,18 +3328,18 @@ impl<'a> Parser<'a> {
}
/// Return nth non-whitespace token that has not yet been processed
- pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation {
+ pub fn peek_nth_token(&self, mut n: usize) -> TokenWithSpan {
let mut index = self.index;
loop {
index += 1;
match self.tokens.get(index - 1) {
- Some(TokenWithLocation {
+ Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) => continue,
non_whitespace => {
if n == 0 {
- return
non_whitespace.cloned().unwrap_or(TokenWithLocation {
+ return non_whitespace.cloned().unwrap_or(TokenWithSpan
{
token: Token::EOF,
span: Span::empty(),
});
@@ -3352,16 +3352,16 @@ impl<'a> Parser<'a> {
/// Return the first token, possibly whitespace, that has not yet been
processed
/// (or None if reached end-of-file).
- pub fn peek_token_no_skip(&self) -> TokenWithLocation {
+ pub fn peek_token_no_skip(&self) -> TokenWithSpan {
self.peek_nth_token_no_skip(0)
}
/// Return nth token, possibly whitespace, that has not yet been processed.
- pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithLocation {
+ pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithSpan {
self.tokens
.get(self.index + n)
.cloned()
- .unwrap_or(TokenWithLocation {
+ .unwrap_or(TokenWithSpan {
token: Token::EOF,
span: Span::empty(),
})
@@ -3378,25 +3378,25 @@ impl<'a> Parser<'a> {
/// Return the first non-whitespace token that has not yet been processed
/// (or None if reached end-of-file) and mark it as processed. OK to call
/// repeatedly after reaching EOF.
- pub fn next_token(&mut self) -> TokenWithLocation {
+ pub fn next_token(&mut self) -> TokenWithSpan {
loop {
self.index += 1;
match self.tokens.get(self.index - 1) {
- Some(TokenWithLocation {
+ Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) => continue,
token => {
return token
.cloned()
- .unwrap_or_else(|| TokenWithLocation::wrap(Token::EOF))
+ .unwrap_or_else(|| TokenWithSpan::wrap(Token::EOF))
}
}
}
}
/// Return the first unprocessed token, possibly whitespace.
- pub fn next_token_no_skip(&mut self) -> Option<&TokenWithLocation> {
+ pub fn next_token_no_skip(&mut self) -> Option<&TokenWithSpan> {
self.index += 1;
self.tokens.get(self.index - 1)
}
@@ -3408,7 +3408,7 @@ impl<'a> Parser<'a> {
loop {
assert!(self.index > 0);
self.index -= 1;
- if let Some(TokenWithLocation {
+ if let Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) = self.tokens.get(self.index)
@@ -3420,7 +3420,7 @@ impl<'a> Parser<'a> {
}
/// Report `found` was encountered instead of `expected`
- pub fn expected<T>(&self, expected: &str, found: TokenWithLocation) ->
Result<T, ParserError> {
+ pub fn expected<T>(&self, expected: &str, found: TokenWithSpan) ->
Result<T, ParserError> {
parser_err!(
format!("Expected: {expected}, found: {found}"),
found.span.start
@@ -3435,7 +3435,7 @@ impl<'a> Parser<'a> {
}
#[must_use]
- pub fn parse_keyword_token(&mut self, expected: Keyword) ->
Option<TokenWithLocation> {
+ pub fn parse_keyword_token(&mut self, expected: Keyword) ->
Option<TokenWithSpan> {
match self.peek_token().token {
Token::Word(w) if expected == w.keyword => Some(self.next_token()),
_ => None,
@@ -3524,7 +3524,7 @@ impl<'a> Parser<'a> {
/// If the current token is the `expected` keyword, consume the token.
/// Otherwise, return an error.
- pub fn expect_keyword(&mut self, expected: Keyword) ->
Result<TokenWithLocation, ParserError> {
+ pub fn expect_keyword(&mut self, expected: Keyword) ->
Result<TokenWithSpan, ParserError> {
if let Some(token) = self.parse_keyword_token(expected) {
Ok(token)
} else {
@@ -3568,7 +3568,7 @@ impl<'a> Parser<'a> {
}
/// Bail out if the current token is not an expected keyword, or consume
it if it is
- pub fn expect_token(&mut self, expected: &Token) ->
Result<TokenWithLocation, ParserError> {
+ pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithSpan,
ParserError> {
if self.peek_token() == *expected {
Ok(self.next_token())
} else {
@@ -4107,7 +4107,7 @@ impl<'a> Parser<'a> {
Keyword::ARCHIVE => Ok(Some(CreateFunctionUsing::Archive(uri))),
_ => self.expected(
"JAR, FILE or ARCHIVE, got {:?}",
-
TokenWithLocation::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
+
TokenWithSpan::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
),
}
}
@@ -6832,7 +6832,7 @@ impl<'a> Parser<'a> {
if let Some(name) = name {
return self.expected(
"FULLTEXT or SPATIAL option without constraint name",
- TokenWithLocation {
+ TokenWithSpan {
token: Token::make_keyword(&name.to_string()),
span: next_token.span,
},
@@ -7808,7 +7808,7 @@ impl<'a> Parser<'a> {
Some('\'') => Ok(Value::SingleQuotedString(w.value)),
_ => self.expected(
"A value?",
- TokenWithLocation {
+ TokenWithSpan {
token: Token::Word(w),
span,
},
@@ -7816,7 +7816,7 @@ impl<'a> Parser<'a> {
},
_ => self.expected(
"a concrete value",
- TokenWithLocation {
+ TokenWithSpan {
token: Token::Word(w),
span,
},
@@ -7878,7 +7878,7 @@ impl<'a> Parser<'a> {
}
unexpected => self.expected(
"a value",
- TokenWithLocation {
+ TokenWithSpan {
token: unexpected,
span,
},
@@ -7927,7 +7927,7 @@ impl<'a> Parser<'a> {
Token::HexStringLiteral(ref s) =>
Ok(Value::HexStringLiteral(s.to_string())),
unexpected => self.expected(
"a string value",
- TokenWithLocation {
+ TokenWithSpan {
token: unexpected,
span,
},
@@ -8618,7 +8618,7 @@ impl<'a> Parser<'a> {
let token = self
.next_token_no_skip()
.cloned()
- .unwrap_or(TokenWithLocation::wrap(Token::EOF));
+ .unwrap_or(TokenWithSpan::wrap(Token::EOF));
requires_whitespace = match token.token {
Token::Word(next_word) if
next_word.quote_style.is_none() => {
ident.value.push_str(&next_word.value);
@@ -11683,7 +11683,7 @@ impl<'a> Parser<'a> {
/// If it is not possible to parse it, will return an option.
pub fn parse_wildcard_additional_options(
&mut self,
- wildcard_token: TokenWithLocation,
+ wildcard_token: TokenWithSpan,
) -> Result<WildcardAdditionalOptions, ParserError> {
let opt_ilike = if dialect_of!(self is GenericDialect |
SnowflakeDialect) {
self.parse_optional_select_item_ilike()?
@@ -12708,7 +12708,7 @@ impl<'a> Parser<'a> {
}
/// Consume the parser and return its underlying token buffer
- pub fn into_tokens(self) -> Vec<TokenWithLocation> {
+ pub fn into_tokens(self) -> Vec<TokenWithSpan> {
self.tokens
}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index bed2d9b5..7a79445e 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -521,42 +521,46 @@ impl Span {
}
}
+/// Backwards compatibility struct for [`TokenWithSpan`]
+#[deprecated(since = "0.53.0", note = "please use `TokenWithSpan` instead")]
+pub type TokenWithLocation = TokenWithSpan;
+
/// A [Token] with [Location] attached to it
#[derive(Debug, Clone, Hash, Ord, PartialOrd, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
-pub struct TokenWithLocation {
+pub struct TokenWithSpan {
pub token: Token,
pub span: Span,
}
-impl TokenWithLocation {
- pub fn new(token: Token, span: Span) -> TokenWithLocation {
- TokenWithLocation { token, span }
+impl TokenWithSpan {
+ pub fn new(token: Token, span: Span) -> TokenWithSpan {
+ TokenWithSpan { token, span }
}
- pub fn wrap(token: Token) -> TokenWithLocation {
- TokenWithLocation::new(token, Span::empty())
+ pub fn wrap(token: Token) -> TokenWithSpan {
+ TokenWithSpan::new(token, Span::empty())
}
- pub fn at(token: Token, start: Location, end: Location) ->
TokenWithLocation {
- TokenWithLocation::new(token, Span::new(start, end))
+ pub fn at(token: Token, start: Location, end: Location) -> TokenWithSpan {
+ TokenWithSpan::new(token, Span::new(start, end))
}
}
-impl PartialEq<Token> for TokenWithLocation {
+impl PartialEq<Token> for TokenWithSpan {
fn eq(&self, other: &Token) -> bool {
&self.token == other
}
}
-impl PartialEq<TokenWithLocation> for Token {
- fn eq(&self, other: &TokenWithLocation) -> bool {
+impl PartialEq<TokenWithSpan> for Token {
+ fn eq(&self, other: &TokenWithSpan) -> bool {
self == &other.token
}
}
-impl fmt::Display for TokenWithLocation {
+impl fmt::Display for TokenWithSpan {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.token.fmt(f)
}
@@ -716,8 +720,8 @@ impl<'a> Tokenizer<'a> {
}
/// Tokenize the statement and produce a vector of tokens with location
information
- pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithLocation>,
TokenizerError> {
- let mut tokens: Vec<TokenWithLocation> = vec![];
+ pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithSpan>,
TokenizerError> {
+ let mut tokens: Vec<TokenWithSpan> = vec![];
self.tokenize_with_location_into_buf(&mut tokens)
.map(|_| tokens)
}
@@ -726,7 +730,7 @@ impl<'a> Tokenizer<'a> {
/// If an error is thrown, the buffer will contain all tokens that were
successfully parsed before the error.
pub fn tokenize_with_location_into_buf(
&mut self,
- buf: &mut Vec<TokenWithLocation>,
+ buf: &mut Vec<TokenWithSpan>,
) -> Result<(), TokenizerError> {
let mut state = State {
peekable: self.query.chars().peekable(),
@@ -738,7 +742,7 @@ impl<'a> Tokenizer<'a> {
while let Some(token) = self.next_token(&mut state)? {
let span = location.span_to(state.location());
- buf.push(TokenWithLocation { token, span });
+ buf.push(TokenWithSpan { token, span });
location = state.location();
}
@@ -2751,25 +2755,25 @@ mod tests {
.tokenize_with_location()
.unwrap();
let expected = vec![
- TokenWithLocation::at(Token::make_keyword("SELECT"), (1,
1).into(), (1, 7).into()),
- TokenWithLocation::at(
+ TokenWithSpan::at(Token::make_keyword("SELECT"), (1, 1).into(),
(1, 7).into()),
+ TokenWithSpan::at(
Token::Whitespace(Whitespace::Space),
(1, 7).into(),
(1, 8).into(),
),
- TokenWithLocation::at(Token::make_word("a", None), (1, 8).into(),
(1, 9).into()),
- TokenWithLocation::at(Token::Comma, (1, 9).into(), (1, 10).into()),
- TokenWithLocation::at(
+ TokenWithSpan::at(Token::make_word("a", None), (1, 8).into(), (1,
9).into()),
+ TokenWithSpan::at(Token::Comma, (1, 9).into(), (1, 10).into()),
+ TokenWithSpan::at(
Token::Whitespace(Whitespace::Newline),
(1, 10).into(),
(2, 1).into(),
),
- TokenWithLocation::at(
+ TokenWithSpan::at(
Token::Whitespace(Whitespace::Space),
(2, 1).into(),
(2, 2).into(),
),
- TokenWithLocation::at(Token::make_word("b", None), (2, 2).into(),
(2, 3).into()),
+ TokenWithSpan::at(Token::make_word("b", None), (2, 2).into(), (2,
3).into()),
];
compare(expected, tokens);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
