This is an automated email from the ASF dual-hosted git repository. joshtynjala pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/royale-compiler.git
commit 58fb870f87d2d451a709e41f50cd43d009752d08 Author: Josh Tynjala <[email protected]> AuthorDate: Thu Oct 27 13:41:39 2022 -0700 linter: add formatting/whitespace tokens to TokenQuery to support formatting rules --- .../java/org/apache/royale/linter/ASLinter.java | 52 ++++++- .../java/org/apache/royale/linter/MXMLLinter.java | 37 ++++- .../org/apache/royale/linter/MXMLTokenQuery.java | 117 ++++++++++++-- .../java/org/apache/royale/linter/TokenQuery.java | 173 ++++++++++++++++++--- .../royale/linter/rules/EmptyStatementRule.java | 2 +- .../linter/rules/LineCommentPositionRule.java | 4 +- .../royale/linter/rules/MissingASDocRule.java | 2 +- 7 files changed, 341 insertions(+), 46 deletions(-) diff --git a/linter/src/main/java/org/apache/royale/linter/ASLinter.java b/linter/src/main/java/org/apache/royale/linter/ASLinter.java index 2f1761658..f4e293029 100644 --- a/linter/src/main/java/org/apache/royale/linter/ASLinter.java +++ b/linter/src/main/java/org/apache/royale/linter/ASLinter.java @@ -142,6 +142,8 @@ public class ASLinter extends BaseLinter { // of the repaired tokens, so add them all at the end repairedTokensList.addAll(comments); + repairedTokensList = insertWhitespaceTokens(repairedTokensList, text); + IASToken[] allTokens = repairedTokensList.toArray(new IASToken[0]); TokenQuery tokenQuery = new TokenQuery(allTokens); visitNode(node, tokenQuery, fileProblems); @@ -179,7 +181,7 @@ public class ASLinter extends BaseLinter { private void visitNode(IASNode node, TokenQuery tokenQuery, Collection<ICompilerProblem> problems) { ASTNodeID nodeID = node.getNodeID(); - IASToken prevComment = tokenQuery.getPreviousComment(node); + IASToken prevComment = tokenQuery.getCommentBefore(node); boolean linterOn = true; while (prevComment != null) { String commentText = null; @@ -190,7 +192,7 @@ public class ASLinter extends BaseLinter { commentText = commentText.substring(2, commentText.length() - 2).trim(); } else { // not the type of comment that we care about - prevComment = tokenQuery.getPreviousComment(prevComment); + prevComment = tokenQuery.getCommentBefore(prevComment); continue; } if (LINTER_TAG_ON.equals(commentText)) { @@ -201,7 +203,7 @@ public class ASLinter extends BaseLinter { linterOn = false; break; } - prevComment = tokenQuery.getPreviousComment(prevComment); + prevComment = tokenQuery.getCommentBefore(prevComment); } if (linterOn) { for (LinterRule rule : settings.rules) { @@ -216,4 +218,48 @@ public class ASLinter extends BaseLinter { visitNode(child, tokenQuery, problems); } } + + private List<IASToken> insertWhitespaceTokens(List<IASToken> originalTokens, String text) { + ArrayList<IASToken> tokens = new ArrayList<IASToken>(); + IASToken prevToken = null; + for (IASToken token : originalTokens) { + if (prevToken != null) { + + boolean skipSemicolon = token.getType() == ASTokenTypes.TOKEN_SEMICOLON && token.isImplicit() + && prevToken != null && (prevToken.getType() == ASTokenTypes.HIDDEN_TOKEN_SINGLE_LINE_COMMENT + || prevToken.getType() == ASTokenTypes.TOKEN_BLOCK_OPEN); + if (skipSemicolon) { + continue; + } + + int start = prevToken.getAbsoluteEnd(); + int end = token.getAbsoluteStart(); + if (end > start) { + String tokenText = text.substring(start, end); + ASToken whitespaceToken = new ASToken(TokenQuery.TOKEN_TYPE_WHITESPACE, start, end, + prevToken.getEndLine(), + prevToken.getEndColumn(), tokenText); + whitespaceToken.setEndLine(token.getLine()); + whitespaceToken.setEndLine(token.getColumn()); + tokens.add(whitespaceToken); + } + } + tokens.add(token); + prevToken = token; + } + if (prevToken != null) { + int start = prevToken.getAbsoluteEnd(); + int end = text.length(); + if (end > start) { + String tokenText = text.substring(start, end); + ASToken whitespaceToken = new ASToken(TokenQuery.TOKEN_TYPE_WHITESPACE, start, end, + prevToken.getEndLine(), + prevToken.getEndColumn(), tokenText); + whitespaceToken.setEndLine(prevToken.getLine()); + whitespaceToken.setEndLine(prevToken.getColumn()); + tokens.add(whitespaceToken); + } + } + return tokens; + } } diff --git a/linter/src/main/java/org/apache/royale/linter/MXMLLinter.java b/linter/src/main/java/org/apache/royale/linter/MXMLLinter.java index 5f4784a6b..bd5662755 100644 --- a/linter/src/main/java/org/apache/royale/linter/MXMLLinter.java +++ b/linter/src/main/java/org/apache/royale/linter/MXMLLinter.java @@ -71,8 +71,9 @@ public class MXMLLinter extends BaseLinter { if (!settings.ignoreProblems && hasErrors(fileProblems)) { return; } + boolean skipLinting = false; - IMXMLToken[] allTokens = originalTokens.toArray(new IMXMLToken[0]); + IMXMLToken[] allTokens = insertFormattingTokens(originalTokens, text); MXMLTokenQuery tokenQuery = new MXMLTokenQuery(allTokens); for (LinterRule rule : settings.rules) { Map<MXMLTokenKind, MXMLTokenVisitor> tokenHandlers = rule.getMXMLTokenVisitors(); @@ -179,4 +180,38 @@ public class MXMLLinter extends BaseLinter { current = current.getNextSibling(true); } } + + private IMXMLToken[] insertFormattingTokens(List<MXMLToken> originalTokens, String text) { + ArrayList<IMXMLToken> tokens = new ArrayList<IMXMLToken>(); + IMXMLToken prevToken = null; + for (IMXMLToken token : originalTokens) { + if (prevToken != null) { + int start = prevToken.getEnd(); + int end = token.getStart(); + if (end > start) { + String tokenText = text.substring(start, end); + MXMLToken formattingToken = new MXMLToken(MXMLTokenQuery.TOKEN_TYPE_FORMATTING, start, end, + prevToken.getLine(), prevToken.getColumn() + end - start, tokenText); + formattingToken.setEndLine(token.getLine()); + formattingToken.setEndLine(token.getColumn()); + tokens.add(formattingToken); + } + } + tokens.add(token); + prevToken = token; + } + if (prevToken != null) { + int start = prevToken.getEnd(); + int end = text.length(); + if (end > start) { + String tokenText = text.substring(start, end); + MXMLToken formattingToken = new MXMLToken(MXMLTokenQuery.TOKEN_TYPE_FORMATTING, start, end, + prevToken.getLine(), prevToken.getColumn() + end - start, tokenText); + formattingToken.setEndLine(prevToken.getLine()); + formattingToken.setEndLine(prevToken.getColumn()); + tokens.add(formattingToken); + } + } + return tokens.toArray(new IMXMLToken[0]); + } } diff --git a/linter/src/main/java/org/apache/royale/linter/MXMLTokenQuery.java b/linter/src/main/java/org/apache/royale/linter/MXMLTokenQuery.java index 82fb83043..66eec2b97 100644 --- a/linter/src/main/java/org/apache/royale/linter/MXMLTokenQuery.java +++ b/linter/src/main/java/org/apache/royale/linter/MXMLTokenQuery.java @@ -19,12 +19,12 @@ package org.apache.royale.linter; -import java.util.List; - import org.apache.royale.compiler.common.ISourceLocation; import org.apache.royale.compiler.parsing.IMXMLToken; public class MXMLTokenQuery { + public static final int TOKEN_TYPE_FORMATTING = 999999; + public MXMLTokenQuery(IMXMLToken[] tokens) { allTokens = tokens; } @@ -42,23 +42,26 @@ public class MXMLTokenQuery { * Returns the token immediately before a source location. */ public IMXMLToken getTokenBefore(ISourceLocation sourceLocation) { - return getTokenBefore(sourceLocation, false); + return getTokenBefore(sourceLocation, false, false); } /** * Returns the token immediately before a source location, with the option - * to skip comment tokens. + * to skip comment and formatting tokens. */ - public IMXMLToken getTokenBefore(ISourceLocation sourceLocation, boolean skipComments) { + public IMXMLToken getTokenBefore(ISourceLocation sourceLocation, boolean skipComments, boolean skipFormatting) { IMXMLToken result = null; - for (IMXMLToken otherToken : allTokens) { - if (skipComments && isComment(otherToken)) { + for (IMXMLToken token : allTokens) { + if (skipComments && isComment(token)) { continue; } - if (otherToken.getStart() >= sourceLocation.getAbsoluteStart()) { + if (skipFormatting && isFormatting(token)) { + continue; + } + if (token.getStart() >= sourceLocation.getAbsoluteStart()) { return result; } - result = otherToken; + result = token; } return null; } @@ -67,19 +70,22 @@ public class MXMLTokenQuery { * Returns the token immediately after a source location. */ public IMXMLToken getTokenAfter(ISourceLocation sourceLocation) { - return getTokenAfter(sourceLocation, false); + return getTokenAfter(sourceLocation, false, false); } /** * Returns the token immediately after a source location, with the option to * skip comment tokens. */ - public IMXMLToken getTokenAfter(ISourceLocation sourceLocation, boolean skipComments) { + public IMXMLToken getTokenAfter(ISourceLocation sourceLocation, boolean skipComments, boolean skipFormatting) { for (IMXMLToken token : allTokens) { - if (skipComments && isComment(token)) { - continue; - } if (token.getStart() >= sourceLocation.getAbsoluteEnd()) { + if (skipComments && isComment(token)) { + continue; + } + if (skipFormatting && isFormatting(token)) { + continue; + } return token; } } @@ -92,5 +98,88 @@ public class MXMLTokenQuery { public boolean isComment(IMXMLToken token) { return token.getMXMLTokenKind() == IMXMLToken.MXMLTokenKind.COMMENT; } + + /** + * Returns the first comment that appears before the start of a particular + * source location. + */ + public IMXMLToken getCommentBefore(ISourceLocation before) { + IMXMLToken result = null; + for (IMXMLToken token : allTokens) { + if (token.getStart() >= before.getAbsoluteStart()) { + return result; + } + if (isComment(token)) { + result = token; + } + } + return null; + } + + /** + * Returns the first comment that appears after the end of a particular + * source location. + */ + public IMXMLToken getCommentAfter(ISourceLocation after) { + for (IMXMLToken token : allTokens) { + if (token.getStart() >= after.getAbsoluteEnd() && isComment(token)) { + return token; + } + } + return null; + } + + /** + * Checks if a token is formatting. + */ + public boolean isFormatting(IMXMLToken token) { + return token.getType() == TOKEN_TYPE_FORMATTING; + } + + /** + * Returns the first formatting token that appears before the start of a + * particular source location. + */ + public IMXMLToken getFormattingBefore(ISourceLocation before) { + IMXMLToken result = null; + for (IMXMLToken token : allTokens) { + if (token.getStart() >= before.getAbsoluteStart()) { + return result; + } + if (isFormatting(token)) { + result = token; + } + } + return null; + } + + /** + * Returns the first formatting token that appears after the end of a + * particular source location. + */ + public IMXMLToken getFormattingAfter(ISourceLocation after) { + for (IMXMLToken token : allTokens) { + if (token.getStart() >= after.getAbsoluteEnd() && isFormatting(token)) { + return token; + } + } + return null; + } + + /** + * Returns the first non-comment, non-formatting token that appears before + * the start of a particular source location. + */ + public IMXMLToken getSignificantTokenBefore(ISourceLocation before) { + return getTokenBefore(before, true, true); + } + + /** + * Returns the first non-comment, non-formatting token that appears after + * the end of a particular source location. + */ + public IMXMLToken getSignificantTokenAfter(ISourceLocation after) { + return getTokenAfter(after, true, true); + } } diff --git a/linter/src/main/java/org/apache/royale/linter/TokenQuery.java b/linter/src/main/java/org/apache/royale/linter/TokenQuery.java index 99b69c621..b6c1089f4 100644 --- a/linter/src/main/java/org/apache/royale/linter/TokenQuery.java +++ b/linter/src/main/java/org/apache/royale/linter/TokenQuery.java @@ -28,6 +28,8 @@ import org.apache.royale.compiler.parsing.IASToken; import org.apache.royale.compiler.tree.as.IASNode; public class TokenQuery { + public static final int TOKEN_TYPE_WHITESPACE = 999999; + public TokenQuery(IASToken[] tokens) { allTokens = tokens; } @@ -45,6 +47,14 @@ public class TokenQuery { * Returns all tokens inside of a particular node. */ public IASToken[] getTokens(IASNode node) { + return getTokens(node, false, false); + } + + /** + * Returns all tokens inside of a particular node, with the option to skip + * comment and whitespace tokens. + */ + public IASToken[] getTokens(IASNode node, boolean skipComments, boolean skipWhitespace) { List<IASToken> result = new ArrayList<>(); for (IASToken token : allTokens) { if (token.getAbsoluteStart() < node.getAbsoluteStart()) { @@ -53,53 +63,67 @@ public class TokenQuery { if (token.getAbsoluteStart() >= node.getAbsoluteEnd()) { break; } + if (skipComments && isComment(token)) { + continue; + } + if (skipWhitespace && isWhitespace(token)) { + continue; + } result.add(token); } return result.toArray(new IASToken[0]); } /** - * Returns the token immediately before a source location. + * Returns the token immediately before a source location. Includes comment + * and whitespace tokens. */ public IASToken getTokenBefore(ISourceLocation sourceLocation) { - return getTokenBefore(sourceLocation, false); + return getTokenBefore(sourceLocation, false, false); } /** * Returns the token immediately before a source location, with the option - * to skip comment tokens. + * to skip comment and whitespace tokens. */ - public IASToken getTokenBefore(ISourceLocation sourceLocation, boolean skipComments) { + public IASToken getTokenBefore(ISourceLocation sourceLocation, boolean skipComments, boolean skipWhitespace) { IASToken result = null; - for (IASToken otherToken : allTokens) { - if (skipComments && isComment(otherToken)) { + for (IASToken token : allTokens) { + if (skipComments && isComment(token)) { continue; } - if (otherToken.getAbsoluteStart() >= sourceLocation.getAbsoluteStart()) { + if (skipWhitespace && isWhitespace(token)) { + continue; + } + if (token.getAbsoluteStart() >= sourceLocation.getAbsoluteStart()) { return result; } - result = otherToken; + result = token; } return null; } /** - * Returns the token immediately after a source location. + * Returns the token immediately after a source location. Includes comment + * and whitespace tokens. */ public IASToken getTokenAfter(ISourceLocation sourceLocation) { - return getTokenAfter(sourceLocation, false); + return getTokenAfter(sourceLocation, false, false); } /** * Returns the token immediately after a source location, with the option to - * skip comment tokens. + * skip comment and whitespace tokens. */ - public IASToken getTokenAfter(ISourceLocation sourceLocation, boolean skipComments) { + public IASToken getTokenAfter(ISourceLocation sourceLocation, boolean skipComments, boolean skipWhitespace) { for (IASToken token : allTokens) { - if (skipComments && isComment(token)) { - continue; - } if (token.getAbsoluteStart() >= sourceLocation.getAbsoluteEnd()) { + if (skipComments && isComment(token)) { + continue; + } + if (skipWhitespace && isWhitespace(token)) { + continue; + } return token; } } @@ -107,11 +131,26 @@ public class TokenQuery { } /** - * Returns the first token inside a node. + * Returns the first token inside a node. Includes comment + * and whitespace tokens. */ public IASToken getFirstToken(IASNode node) { + return getFirstToken(node, false, false); + } + + /** + * Returns the first token inside a node, with the option to + * skip comment and whitespace tokens. + */ + public IASToken getFirstToken(IASNode node, boolean skipComments, boolean skipWhitespace) { for (IASToken token : allTokens) { if (token.getAbsoluteStart() >= node.getAbsoluteStart()) { + if (skipComments && isComment(token)) { + continue; + } + if (skipWhitespace && isWhitespace(token)) { + continue; + } return token; } } @@ -119,12 +158,27 @@ public class TokenQuery { } /** - * Returns the last token inside a node. + * Returns the last token inside a node. Includes comment + * and whitespace tokens. */ public IASToken getLastToken(IASNode node) { + return getLastToken(node, false, false); + } + + /** + * Returns the last token inside a node, with the option to + * skip comment and whitespace tokens. + */ + public IASToken getLastToken(IASNode node, boolean skipComments, boolean skipWhitespace) { IASToken result = null; for (IASToken token : allTokens) { if (token.getAbsoluteStart() >= node.getAbsoluteStart()) { + if (skipComments && isComment(token)) { + continue; + } + if (skipWhitespace && isWhitespace(token)) { + continue; + } result = token; } else if (result != null) { break; @@ -133,6 +187,36 @@ public class TokenQuery { return result; } + /** + * Returns the first token of the specified type that appears before the + * start of a particular source location. + */ + public IASToken getPreviousTokenOfType(ISourceLocation before, int type) { + IASToken result = null; + for (IASToken token : allTokens) { + if (token.getAbsoluteStart() >= before.getAbsoluteStart()) { + return result; + } + if (token.getType() == type) { + result = token; + } + } + return null; + } + + /** + * Returns the first token of the specified type that appears after the end + * of a particular source location. + */ + public IASToken getNextTokenOfType(ISourceLocation after, int type) { + for (IASToken token : allTokens) { + if (token.getType() == type && token.getAbsoluteStart() >= after.getAbsoluteEnd()) { + return token; + } + } + return null; + } + /** * Returns all comment tokens inside a node. */ @@ -157,47 +241,86 @@ public class TokenQuery { || token.getType() == ASTokenTypes.TOKEN_ASDOC_COMMENT; } - public IASToken getPreviousTokenOfType(ISourceLocation before, int type) { + /** + * Returns the first comment that appears before the start of a particular + * source location. + */ + public IASToken getCommentBefore(ISourceLocation before) { IASToken result = null; for (IASToken token : allTokens) { if (token.getAbsoluteStart() >= before.getAbsoluteStart()) { return result; } - if (token.getType() == type) { + if (isComment(token)) { result = token; } } return null; } - public IASToken getNextTokenOfType(ISourceLocation after, int type) { + /** + * Returns the first comment that appears after the end of a particular + * source location. + */ + public IASToken getCommentAfter(ISourceLocation after) { for (IASToken token : allTokens) { - if (token.getType() == type && token.getAbsoluteStart() >= after.getAbsoluteEnd()) { + if (token.getAbsoluteStart() >= after.getAbsoluteEnd() && isComment(token)) { return token; } } return null; } - public IASToken getPreviousComment(ISourceLocation before) { + /** + * Checks if a token is whitespace. + */ + public boolean isWhitespace(IASToken token) { + return token.getType() == TOKEN_TYPE_WHITESPACE; + } + + /** + * Returns the first whitespace that appears before the start of a + * particular source location. + */ + public IASToken getWhitespaceBefore(ISourceLocation before) { IASToken result = null; for (IASToken token : allTokens) { if (token.getAbsoluteStart() >= before.getAbsoluteStart()) { return result; } - if (isComment(token)) { + if (isWhitespace(token)) { result = token; } } return null; } - public IASToken getNextComment(ISourceLocation after) { + /** + * Returns the first whitespace that appears after the end of a + * particular source location. + */ + public IASToken getWhitespaceAfter(ISourceLocation after) { for (IASToken token : allTokens) { - if (token.getAbsoluteStart() >= after.getAbsoluteEnd() && isComment(token)) { + if (token.getAbsoluteStart() >= after.getAbsoluteEnd() && isWhitespace(token)) { return token; } } return null; } + + /** + * Returns the first non-comment, non-whitespace token that appears before + * the start of a particular source location. + */ + public IASToken getSignificantTokenBefore(ISourceLocation before) { + return getTokenBefore(before, true, true); + } + + /** + * Returns the first non-comment, non-whitespace token that appears after + * the end of a particular source location. + */ + public IASToken getSignificantTokenAfter(ISourceLocation after) { + return getTokenAfter(after, true, true); + } } diff --git a/linter/src/main/java/org/apache/royale/linter/rules/EmptyStatementRule.java b/linter/src/main/java/org/apache/royale/linter/rules/EmptyStatementRule.java index 62159539c..a6c63b0cc 100644 --- a/linter/src/main/java/org/apache/royale/linter/rules/EmptyStatementRule.java +++ b/linter/src/main/java/org/apache/royale/linter/rules/EmptyStatementRule.java @@ -47,7 +47,7 @@ public class EmptyStatementRule extends LinterRule { } private void checkSemicolon(IASToken semicolon, TokenQuery tokenQuery, Collection<ICompilerProblem> problems) { - IASToken prevToken = tokenQuery.getTokenBefore(semicolon); + IASToken prevToken = tokenQuery.getSignificantTokenBefore(semicolon); if (prevToken == null) { return; } diff --git a/linter/src/main/java/org/apache/royale/linter/rules/LineCommentPositionRule.java b/linter/src/main/java/org/apache/royale/linter/rules/LineCommentPositionRule.java index 6a58c927c..28d5e193b 100644 --- a/linter/src/main/java/org/apache/royale/linter/rules/LineCommentPositionRule.java +++ b/linter/src/main/java/org/apache/royale/linter/rules/LineCommentPositionRule.java @@ -50,17 +50,19 @@ public class LineCommentPositionRule extends LinterRule { public LineCommentPosition position = LineCommentPosition.ABOVE; private void checkSingleLineComment(IASToken comment, TokenQuery tokenQuery, Collection<ICompilerProblem> problems) { - IASToken prevToken = tokenQuery.getTokenBefore(comment); + IASToken prevToken = tokenQuery.getSignificantTokenBefore(comment); if (prevToken == null) { return; } if (LineCommentPosition.ABOVE.equals(position)) { if (prevToken.getLine() == comment.getLine()) { + // is beside the comment problems.add(new LineCommentPositionLinterProblem(comment, position)); } } else if (LineCommentPosition.BESIDE.equals(position)) { if (prevToken.getLine() != comment.getLine()) { + // is not beside the comment problems.add(new LineCommentPositionLinterProblem(comment, position)); } } diff --git a/linter/src/main/java/org/apache/royale/linter/rules/MissingASDocRule.java b/linter/src/main/java/org/apache/royale/linter/rules/MissingASDocRule.java index 5c8e3ba86..e2fc08879 100644 --- a/linter/src/main/java/org/apache/royale/linter/rules/MissingASDocRule.java +++ b/linter/src/main/java/org/apache/royale/linter/rules/MissingASDocRule.java @@ -74,7 +74,7 @@ public class MissingASDocRule extends LinterRule { if (!definitionNode.hasNamespace("public")) { return; } - IASToken token = tokenQuery.getTokenBefore(definitionNode); + IASToken token = tokenQuery.getTokenBefore(definitionNode, false, true); if (token.getType() == ASTokenTypes.TOKEN_ASDOC_COMMENT) { String docComment = token.getText(); if (!isDocCommentEmpty(docComment)) {
