This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch gh-readonly-queue/main/pr-2161-44c402385e15e0d96c68d151dcb8019fe31687f8 in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git
commit ee3b6223e673b9c0f737d649a7f934de0186bc64 Author: Louis Vialar <[email protected]> AuthorDate: Tue Jan 13 11:18:32 2026 +0100 Tokenize empty line comments correctly (#2161) --- src/tokenizer.rs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index eb935a4f..a9f9fb44 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1387,7 +1387,11 @@ impl<'a> Tokenizer<'a> { Some('-') => { let mut is_comment = true; if self.dialect.requires_single_line_comment_whitespace() { - is_comment = Some(' ') == chars.peekable.clone().nth(1); + is_comment = chars + .peekable + .clone() + .nth(1) + .is_some_and(char::is_whitespace); } if is_comment { @@ -4069,6 +4073,24 @@ mod tests { Token::Minus, ], ); + + all_dialects_where(|d| d.requires_single_line_comment_whitespace()).tokenizes_to( + "--\n-- Table structure for table...\n--\n", + vec![ + Token::Whitespace(Whitespace::SingleLineComment { + prefix: "--".to_string(), + comment: "\n".to_string(), + }), + Token::Whitespace(Whitespace::SingleLineComment { + prefix: "--".to_string(), + comment: " Table structure for table...\n".to_string(), + }), + Token::Whitespace(Whitespace::SingleLineComment { + prefix: "--".to_string(), + comment: "\n".to_string(), + }), + ], + ); } #[test] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
