This is an automated email from the ASF dual-hosted git repository. joshtynjala pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/royale-compiler.git
commit a71bff565c7853ca9a416118202b0856e8b32f45 Author: Josh Tynjala <[email protected]> AuthorDate: Tue Dec 6 09:08:07 2022 -0800 compiler: verbatim string tokens are now of type TOKEN_VERBATIM_STRING so that the formatter can know the difference It still results in the same LiteralNode in the AST. This is purely at the token level. --- .../apache/royale/compiler/parsing/IASToken.java | 3 ++- .../royale/compiler/internal/parsing/as/ASParser.g | 4 +++- .../compiler/internal/parsing/as/ASToken.java | 3 +++ .../compiler/internal/parsing/as/BaseASParser.java | 21 ++++++++------------- .../compiler/internal/parsing/as/RawASTokenizer.lex | 7 ++++++- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/compiler-common/src/main/java/org/apache/royale/compiler/parsing/IASToken.java b/compiler-common/src/main/java/org/apache/royale/compiler/parsing/IASToken.java index d2ba7d317..c41d3ed8c 100644 --- a/compiler-common/src/main/java/org/apache/royale/compiler/parsing/IASToken.java +++ b/compiler-common/src/main/java/org/apache/royale/compiler/parsing/IASToken.java @@ -60,7 +60,8 @@ public interface IASToken extends ICMToken, ISourceLocation COLON, DEFAULT_XML_STATEMENT, UNKNOWN, - INCLUDE + INCLUDE, + VERBATIM_STRING } /** diff --git a/compiler/src/main/antlr/org/apache/royale/compiler/internal/parsing/as/ASParser.g b/compiler/src/main/antlr/org/apache/royale/compiler/internal/parsing/as/ASParser.g index 90432099a..00a2b8084 100644 --- a/compiler/src/main/antlr/org/apache/royale/compiler/internal/parsing/as/ASParser.g +++ b/compiler/src/main/antlr/org/apache/royale/compiler/internal/parsing/as/ASParser.g @@ -2498,8 +2498,10 @@ primaryExpression returns [ExpressionNodeBase n] { n = LanguageIdentifierNode.buildThis(token); } | token=numericLiteral { n = new NumericLiteralNode(token); } + | TOKEN_VERBATIM_STRING + { n = transformRawString(token); } | TOKEN_LITERAL_STRING - { n = transformStringLiteral(token); } + { n = new LiteralNode(token, LiteralType.STRING); } | TOKEN_VOID_0 { n = new LiteralNode(token, LiteralType.OBJECT); } | TOKEN_LITERAL_REGEXP diff --git a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/ASToken.java b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/ASToken.java index de34ff3f8..11df97b41 100644 --- a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/ASToken.java +++ b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/ASToken.java @@ -433,6 +433,7 @@ public class ASToken extends TokenBase implements IASToken, ASTokenTypes case TOKEN_LITERAL_HEX_NUMBER: case TOKEN_LITERAL_REGEXP: case TOKEN_LITERAL_STRING: + case TOKEN_VERBATIM_STRING: case TOKEN_LITERAL_XMLLIST: case TOKEN_E4X_XMLLIST_CLOSE: return true; @@ -937,6 +938,8 @@ public class ASToken extends TokenBase implements IASToken, ASTokenTypes return ASTokenKind.SCOPE_CLOSE; case TOKEN_BLOCK_OPEN: return ASTokenKind.SCOPE_OPEN; + case TOKEN_VERBATIM_STRING: + return ASTokenKind.VERBATIM_STRING; case TOKEN_LITERAL_STRING: return ASTokenKind.STRING_LITERAL; case TOKEN_LITERAL_NUMBER: diff --git a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/BaseASParser.java b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/BaseASParser.java index 9a647b3da..7174b62f4 100644 --- a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/BaseASParser.java +++ b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/as/BaseASParser.java @@ -3112,21 +3112,16 @@ abstract class BaseASParser extends LLkParser implements IProblemReporter return parsingProjectConfigVariables; } - protected final LiteralNode transformStringLiteral(ASToken token) + protected final LiteralNode transformRawString(ASToken token) { String tokenText = token.getText(); - if (tokenText.startsWith("@")) - { - // the only change needed is to remove the @ symbol - // the escape sequences are already handled for us - String newText = tokenText.substring(1); - ASToken newToken = new ASToken(ASTokenTypes.TOKEN_LITERAL_STRING, token.getStart(), token.getEnd(), token.getLine(), token.getColumn(), newText); - newToken.setEndLine(token.getEndLine()); - newToken.setEndColumn(token.getEndColumn()); - return new LiteralNode(newToken, LiteralType.STRING); - } - - return new LiteralNode(token, LiteralType.STRING); + // the only change needed is to remove the @ symbol + // the escape sequences are already handled for us + String newText = tokenText.substring(1); + ASToken newToken = new ASToken(ASTokenTypes.TOKEN_VERBATIM_STRING, token.getStart(), token.getEnd(), token.getLine(), token.getColumn(), newText); + newToken.setEndLine(token.getEndLine()); + newToken.setEndColumn(token.getEndColumn()); + return new LiteralNode(newToken, LiteralType.STRING); } private final ExpressionNodeBase transformNullishCoalescingExpression(ExpressionNodeBase left, ASToken op, ExpressionNodeBase right) diff --git a/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/as/RawASTokenizer.lex b/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/as/RawASTokenizer.lex index e4a0eee83..ff925c878 100644 --- a/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/as/RawASTokenizer.lex +++ b/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/as/RawASTokenizer.lex @@ -324,11 +324,16 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]" <STRINGLITERAL> "\"" { continueAggregate(); - if (stringKind == StringKind.STRING || stringKind == StringKind.RAW) + if (stringKind == StringKind.STRING) { yybegin(YYINITIAL); return buildAggregateToken(TOKEN_LITERAL_STRING); } + else if (stringKind == StringKind.RAW) + { + yybegin(YYINITIAL); + return buildAggregateToken(TOKEN_VERBATIM_STRING); + } } <STRINGLITERAL> "'"
