This is an automated email from the ASF dual-hosted git repository.
joshtynjala pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/royale-compiler.git
The following commit(s) were added to refs/heads/develop by this push:
new fce284723 ASTokenFormatter: fix unicode and ascii escape sequences
getting lost when formatting string literals
fce284723 is described below
commit fce284723bcbe04443901a8f4c006125c31f3947
Author: Josh Tynjala <[email protected]>
AuthorDate: Wed Sep 13 09:45:31 2023 -0700
ASTokenFormatter: fix unicode and ascii escape sequences getting lost when
formatting string literals
Now copies the string from the original source code as-sis instead of
trying to convert back to escaped characters
---
.../apache/royale/formatter/ASTokenFormatter.java | 32 +++--
.../apache/royale/formatter/TestStringLiteral.java | 142 ++++++++++++++++++++-
2 files changed, 163 insertions(+), 11 deletions(-)
diff --git
a/formatter/src/main/java/org/apache/royale/formatter/ASTokenFormatter.java
b/formatter/src/main/java/org/apache/royale/formatter/ASTokenFormatter.java
index 27c5ca2ca..3cddb3214 100644
--- a/formatter/src/main/java/org/apache/royale/formatter/ASTokenFormatter.java
+++ b/formatter/src/main/java/org/apache/royale/formatter/ASTokenFormatter.java
@@ -170,7 +170,7 @@ public class ASTokenFormatter extends BaseTokenFormatter {
List<IASToken> tokens =
insertExtraAS3Tokens(repairedTokensList, text);
try {
- return parseTokens(tokens);
+ return parseTokens(tokens, text);
} catch (Exception e) {
if (problems != null) {
System.err.println(e);
@@ -181,7 +181,7 @@ public class ASTokenFormatter extends BaseTokenFormatter {
}
}
- private String parseTokens(List<IASToken> tokens) throws Exception {
+ private String parseTokens(List<IASToken> tokens, String fileText)
throws Exception {
indent = 0;
inCaseOrDefaultClause = false;
inControlFlowStatement = false;
@@ -513,7 +513,7 @@ public class ASTokenFormatter extends BaseTokenFormatter {
}
// include the token's own text
- builder.append(getTokenText(token, indent,
skipFormatting));
+ builder.append(getTokenText(token, indent,
skipFormatting, fileText));
// characters that must appear after the token
if (token.getType() !=
ASTokenTypes.HIDDEN_TOKEN_SINGLE_LINE_COMMENT
@@ -1109,7 +1109,7 @@ public class ASTokenFormatter extends BaseTokenFormatter {
}
}
- private String getTokenText(IASToken token, int indent, boolean
skipFormatting) {
+ private String getTokenText(IASToken token, int indent, boolean
skipFormatting, String fileText) {
if (token instanceof MetaDataPayloadToken) {
MetaDataPayloadToken metaPlayloadToken =
(MetaDataPayloadToken) token;
return formatMetadataToken(metaPlayloadToken);
@@ -1134,7 +1134,7 @@ public class ASTokenFormatter extends BaseTokenFormatter {
return
formatMultiLineComment(token.getText());
}
case ASTokenTypes.TOKEN_LITERAL_STRING: {
- return formatLiteralString(token);
+ return formatLiteralString(token,
fileText);
}
case ASTokenTypes.TOKEN_SEMICOLON: {
if (skipFormatting) {
@@ -1233,7 +1233,19 @@ public class ASTokenFormatter extends BaseTokenFormatter
{
return comment;
}
- private String formatLiteralString(IASToken token) {
+ private String formatLiteralString(IASToken token, String fileText) {
+ int start = token.getAbsoluteStart();
+ int end = token.getAbsoluteEnd();
+ if (start != -1 && start < end && end < fileText.length()) {
+ // escape sequences are converted to real characters
when the
+ // original source code is converted to to tokens
+ // the user won't be happy if their strings get changed
+ // (and, in some cases, it may become an invalid
string),
+ // so grab the original string from the file
+ return fileText.substring(start, end);
+ }
+ // we should never get here, but this should handle most
strings, if
+ // something unexpected happens
String string = token.getText();
String charsToEscape = "\b\t\n\f\r\\";
String escapeChars = "btnfr\\";
@@ -1395,7 +1407,7 @@ public class ASTokenFormatter extends BaseTokenFormatter {
return numNewLinesInWhitespace;
}
- private List<IASToken> insertExtraAS3Tokens(List<IASToken>
originalTokens, String text) {
+ private List<IASToken> insertExtraAS3Tokens(List<IASToken>
originalTokens, String fileText) {
ArrayList<IASToken> tokens = new ArrayList<IASToken>();
IASToken prevToken = null;
for (IASToken token : originalTokens) {
@@ -1411,7 +1423,7 @@ public class ASTokenFormatter extends BaseTokenFormatter {
int start = prevToken.getAbsoluteEnd();
int end = token.getAbsoluteStart();
if (end > start) {
- String tokenText =
text.substring(start, end);
+ String tokenText =
fileText.substring(start, end);
ASToken extraToken = new
ASToken(TOKEN_TYPE_EXTRA, start, end, prevToken.getEndLine(),
prevToken.getEndColumn(), tokenText);
extraToken.setEndLine(token.getLine());
@@ -1424,9 +1436,9 @@ public class ASTokenFormatter extends BaseTokenFormatter {
}
if (prevToken != null) {
int start = prevToken.getAbsoluteEnd();
- int end = text.length();
+ int end = fileText.length();
if (end > start) {
- String tokenText = text.substring(start, end);
+ String tokenText = fileText.substring(start,
end);
ASToken extraToken = new
ASToken(TOKEN_TYPE_EXTRA, start, end, prevToken.getEndLine(),
prevToken.getEndColumn(),
tokenText);
extraToken.setEndLine(prevToken.getLine());
diff --git
a/formatter/src/test/java/org/apache/royale/formatter/TestStringLiteral.java
b/formatter/src/test/java/org/apache/royale/formatter/TestStringLiteral.java
index 90b64b65c..19235bf28 100644
--- a/formatter/src/test/java/org/apache/royale/formatter/TestStringLiteral.java
+++ b/formatter/src/test/java/org/apache/royale/formatter/TestStringLiteral.java
@@ -45,7 +45,47 @@ public class TestStringLiteral extends BaseFormatterTests {
}
@Test
- public void testWithNewLine() {
+ public void testTabEscape() {
+ FormatterSettings settings = new FormatterSettings();
+ settings.insertSpaceAfterKeywordsInControlFlowStatements = true;
+ settings.placeOpenBraceOnNewLine = true;
+ settings.insertSpaces = false;
+ ASTokenFormatter formatter = new ASTokenFormatter(settings);
+ String result = formatter.format("file.as",
+ // @formatter:off
+ "\"\\t\";",
+ // @formatter:on
+ problems
+ );
+ assertEquals(
+ // @formatter:off
+ "\"\\t\";",
+ // @formatter:on
+ result);
+ }
+
+ @Test
+ public void testCarriageReturnEscape() {
+ FormatterSettings settings = new FormatterSettings();
+ settings.insertSpaceAfterKeywordsInControlFlowStatements = true;
+ settings.placeOpenBraceOnNewLine = true;
+ settings.insertSpaces = false;
+ ASTokenFormatter formatter = new ASTokenFormatter(settings);
+ String result = formatter.format("file.as",
+ // @formatter:off
+ "\"\\r\";",
+ // @formatter:on
+ problems
+ );
+ assertEquals(
+ // @formatter:off
+ "\"\\r\";",
+ // @formatter:on
+ result);
+ }
+
+ @Test
+ public void testNewLineEscape() {
FormatterSettings settings = new FormatterSettings();
settings.insertSpaceAfterKeywordsInControlFlowStatements = true;
settings.placeOpenBraceOnNewLine = true;
@@ -64,6 +104,106 @@ public class TestStringLiteral extends BaseFormatterTests {
result);
}
+ @Test
+ public void testBackslashEscape() {
+ FormatterSettings settings = new FormatterSettings();
+ settings.insertSpaceAfterKeywordsInControlFlowStatements = true;
+ settings.placeOpenBraceOnNewLine = true;
+ settings.insertSpaces = false;
+ ASTokenFormatter formatter = new ASTokenFormatter(settings);
+ String result = formatter.format("file.as",
+ // @formatter:off
+ "\"\\\\\";",
+ // @formatter:on
+ problems
+ );
+ assertEquals(
+ // @formatter:off
+ "\"\\\\\";",
+ // @formatter:on
+ result);
+ }
+
+ @Test
+ public void testBackspaceEscape() {
+ FormatterSettings settings = new FormatterSettings();
+ settings.insertSpaceAfterKeywordsInControlFlowStatements = true;
+ settings.placeOpenBraceOnNewLine = true;
+ settings.insertSpaces = false;
+ ASTokenFormatter formatter = new ASTokenFormatter(settings);
+ String result = formatter.format("file.as",
+ // @formatter:off
+ "\"\\b\";",
+ // @formatter:on
+ problems
+ );
+ assertEquals(
+ // @formatter:off
+ "\"\\b\";",
+ // @formatter:on
+ result);
+ }
+
+ @Test
+ public void testFormFeedEscape() {
+ FormatterSettings settings = new FormatterSettings();
+ settings.insertSpaceAfterKeywordsInControlFlowStatements = true;
+ settings.placeOpenBraceOnNewLine = true;
+ settings.insertSpaces = false;
+ ASTokenFormatter formatter = new ASTokenFormatter(settings);
+ String result = formatter.format("file.as",
+ // @formatter:off
+ "\"\\f\";",
+ // @formatter:on
+ problems
+ );
+ assertEquals(
+ // @formatter:off
+ "\"\\f\";",
+ // @formatter:on
+ result);
+ }
+
+ @Test
+ public void testAsciiEscape() {
+ FormatterSettings settings = new FormatterSettings();
+ settings.insertSpaceAfterKeywordsInControlFlowStatements = true;
+ settings.placeOpenBraceOnNewLine = true;
+ settings.insertSpaces = false;
+ ASTokenFormatter formatter = new ASTokenFormatter(settings);
+ String result = formatter.format("file.as",
+ // @formatter:off
+ "\"\\x21\";",
+ // @formatter:on
+ problems
+ );
+ assertEquals(
+ // @formatter:off
+ "\"\\x21\";",
+ // @formatter:on
+ result);
+ }
+
+ @Test
+ public void testUnicodeEscape() {
+ FormatterSettings settings = new FormatterSettings();
+ settings.insertSpaceAfterKeywordsInControlFlowStatements = true;
+ settings.placeOpenBraceOnNewLine = true;
+ settings.insertSpaces = false;
+ ASTokenFormatter formatter = new ASTokenFormatter(settings);
+ String result = formatter.format("file.as",
+ // @formatter:off
+ "\"\\u263a\";",
+ // @formatter:on
+ problems
+ );
+ assertEquals(
+ // @formatter:off
+ "\"\\u263a\";",
+ // @formatter:on
+ result);
+ }
+
@Test
public void testDoubleQuoteWithEscapedDoubleQuote() {
FormatterSettings settings = new FormatterSettings();
