Repository: flex-falcon Updated Branches: refs/heads/develop e28b138d4 -> c040b5c88
FLEX-35183 FLEX-35184 fix unterminated CDATA and Comments in MXML and E4X Project: http://git-wip-us.apache.org/repos/asf/flex-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/flex-falcon/commit/c040b5c8 Tree: http://git-wip-us.apache.org/repos/asf/flex-falcon/tree/c040b5c8 Diff: http://git-wip-us.apache.org/repos/asf/flex-falcon/diff/c040b5c8 Branch: refs/heads/develop Commit: c040b5c88980130cd43fb903d52f9cdef59a7101 Parents: e28b138 Author: Alex Harui <[email protected]> Authored: Wed Nov 23 15:29:10 2016 -0800 Committer: Alex Harui <[email protected]> Committed: Wed Nov 23 15:29:25 2016 -0800 ---------------------------------------------------------------------- .../compiler/internal/caches/MXMLDataCache.java | 2 + .../internal/parsing/as/BaseRawASTokenizer.java | 22 +++++++ .../parsing/mxml/BaseRawMXMLTokenizer.java | 41 +++++++++++++ .../internal/parsing/mxml/MXMLScopeBuilder.java | 1 + .../internal/parsing/mxml/MXMLTokenizer.java | 2 + .../problems/CDataNotClosedProblem.java | 41 +++++++++++++ .../problems/CommentNotClosedProblem.java | 41 +++++++++++++ .../problems/MXMLUnclosedTagProblem.java | 2 +- .../internal/parsing/as/RawASTokenizer.lex | 48 +++------------ .../internal/parsing/mxml/RawMXMLTokenizer.lex | 64 ++++++-------------- 10 files changed, 179 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java ---------------------------------------------------------------------- diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java b/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java index 7f06a2d..af7f4a2 100644 --- a/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java +++ b/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java @@ -86,6 +86,8 @@ public class MXMLDataCache extends ConcurrentCacheStoreBase<MXMLData> // Build tags and attributes from the tokens. final MXMLData mxmlData = new MXMLData(tokens, tokenizer.getPrefixMap(), fileSpec); + if (tokenizer.hasTokenizationProblems()) + mxmlData.getProblems().addAll(tokenizer.getTokenizationProblems()); return mxmlData; } catch (FileNotFoundException e) http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java ---------------------------------------------------------------------- diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java index cf22ce6..3a80b66 100644 --- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java +++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java @@ -21,6 +21,8 @@ package org.apache.flex.compiler.internal.parsing.as; import org.apache.flex.compiler.common.ISourceLocation; import org.apache.flex.compiler.problems.ASDocNotClosedProblem; +import org.apache.flex.compiler.problems.CDataNotClosedProblem; +import org.apache.flex.compiler.problems.CommentNotClosedProblem; import org.apache.flex.compiler.problems.ICompilerProblem; import org.apache.flex.compiler.problems.StringLiteralMustBeTerminatedBeforeLineBreakProblem; import org.apache.flex.compiler.problems.StringLiteralNotClosedProblem; @@ -340,6 +342,26 @@ public abstract class BaseRawASTokenizer extends BaseRawTokenizer<ASToken> } /** + * Report syntax error: input ended before Comment is closed. + */ + protected final void reportUnclosedComment() + { + final ISourceLocation location = getCurrentSourceLocation(0); + final ICompilerProblem problem = new CommentNotClosedProblem(location); + getProblems().add(problem); + } + + /** + * Report syntax error: input ended before CDATA is closed. + */ + protected final void reportUnclosedCDATA() + { + final ISourceLocation location = getCurrentSourceLocation(0); + final ICompilerProblem problem = new CDataNotClosedProblem(location); + getProblems().add(problem); + } + + /** * Convert escaped unicode sequence such as {@code \u00FF} and {@code \xFF} * to unicode code point. * http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java ---------------------------------------------------------------------- diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java index c6709c8..09277cf 100644 --- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java +++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java @@ -24,7 +24,10 @@ import java.util.List; import org.apache.flex.compiler.internal.parsing.as.ASToken; import org.apache.flex.compiler.parsing.ICMToken; +import org.apache.flex.compiler.problems.ASDocNotClosedProblem; +import org.apache.flex.compiler.problems.CommentNotClosedProblem; import org.apache.flex.compiler.problems.ICompilerProblem; +import org.apache.flex.compiler.problems.MXMLUnclosedTagProblem; import antlr.CommonToken; import antlr.Token; @@ -59,6 +62,8 @@ public abstract class BaseRawMXMLTokenizer private Token lastToken = null; + protected String sourcePath = null; + public BaseRawMXMLTokenizer() { // no-arg constructor @@ -106,6 +111,7 @@ public abstract class BaseRawMXMLTokenizer CommonToken token = new CommonToken(type, text); token.setLine(line); token.setColumn(column); + token.setFilename(sourcePath); lastToken = token; return token; } @@ -159,6 +165,11 @@ public abstract class BaseRawMXMLTokenizer return lastToken != null ? lastToken.getText() : ""; } + public void setSourcePath(String sourcePath) + { + this.sourcePath = sourcePath; + } + protected void setLastToken(Token token) { lastToken = token; @@ -241,6 +252,36 @@ public abstract class BaseRawMXMLTokenizer return yytext() + " (" + line + ")"; } + /** + * Report an unclosed entity problem + * + * @param The token + */ + protected void reportUnclosedCDATA(MXMLToken token) + { + getProblems().add(new MXMLUnclosedTagProblem((ASToken)token, "CDATA")); + } + + /** + * Report an unclosed entity problem + * + * @param The token + */ + protected void reportUnclosedComment(MXMLToken token) + { + getProblems().add(new CommentNotClosedProblem((ASToken)token)); + } + + /** + * Report an unclosed entity problem + * + * @param The token + */ + protected void reportUnclosedASDocComment(MXMLToken token) + { + getProblems().add(new ASDocNotClosedProblem((ASToken)token)); + } + protected List<ICompilerProblem> problems = null; /** http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java ---------------------------------------------------------------------- diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java index 79e59c2..bd778e7 100644 --- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java +++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java @@ -106,6 +106,7 @@ public class MXMLScopeBuilder fileScope.addDefinition(packageDefinition); problems = new LinkedList<ICompilerProblem>(); + problems.addAll(mxmlData.getProblems()); includeHandler = new IncludeHandler(fileSpecGetter); includeHandler.setProjectAndCompilationUnit(project, compilationUnit); http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java ---------------------------------------------------------------------- diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java index 49a3fea..51718ca 100644 --- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java +++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java @@ -89,6 +89,7 @@ public class MXMLTokenizer implements IMXMLTokenizer, Closeable public MXMLTokenizer(String path) { tokenizer = new RawMXMLTokenizer(); + tokenizer.setSourcePath(path); problems = new ArrayList<ICompilerProblem>(); rootPrefixMap = new MutablePrefixMap(); this.path = path; @@ -116,6 +117,7 @@ public class MXMLTokenizer implements IMXMLTokenizer, Closeable public void setPath(String path) { this.path = path; + tokenizer.setSourcePath(path); } public void setReader(Reader reader) { http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java ---------------------------------------------------------------------- diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java b/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java new file mode 100644 index 0000000..e542f3d --- /dev/null +++ b/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java @@ -0,0 +1,41 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.flex.compiler.problems; + +import org.apache.flex.compiler.common.ISourceLocation; +import org.apache.flex.compiler.problems.annotations.ProblemClassification; + +/** + * Syntax error: input ended before ASDoc is closed. + */ +@ProblemClassification(CompilerProblemClassification.SYNTAX_ERROR) +public final class CDataNotClosedProblem extends ParserProblem +{ + public static final String DESCRIPTION = + "input ended before CDATA is closed."; + + public static final int errorCode = 1549; + + public CDataNotClosedProblem(ISourceLocation location) + { + super(location); + } +} + http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java ---------------------------------------------------------------------- diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java b/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java new file mode 100644 index 0000000..4c5954c --- /dev/null +++ b/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java @@ -0,0 +1,41 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.flex.compiler.problems; + +import org.apache.flex.compiler.common.ISourceLocation; +import org.apache.flex.compiler.problems.annotations.ProblemClassification; + +/** + * Syntax error: input ended before ASDoc is closed. + */ +@ProblemClassification(CompilerProblemClassification.SYNTAX_ERROR) +public final class CommentNotClosedProblem extends ParserProblem +{ + public static final String DESCRIPTION = + "input ended before comment is closed."; + + public static final int errorCode = 1549; + + public CommentNotClosedProblem(ISourceLocation location) + { + super(location); + } +} + http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java ---------------------------------------------------------------------- diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java b/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java index dab73ad..b668100 100644 --- a/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java +++ b/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java @@ -30,7 +30,7 @@ import org.apache.flex.compiler.problems.annotations.ProblemClassification; public final class MXMLUnclosedTagProblem extends ParserProblem { public static final String DESCRIPTION = - "'${tokenText}' tag (or tag before this tag) is unclosed or tag not allowed here."; + "'${tokenText}' tag (or tag before this tag, or non-tag inside this tag) is unclosed or tag not allowed here."; public static final int errorCode = 1552; http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex ---------------------------------------------------------------------- diff --git a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex index f0fd6b6..5c93547 100644 --- a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex +++ b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex @@ -988,27 +988,12 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]" yybegin(MARKUP_IGNORE); } -<CDATA> ([^\]])* +<CDATA> [^] { continueAggregate(); } -<CDATA> ("]"+[^\]>]) -{ - continueAggregate(); -} - -<CDATA> {WHITE_SPACE_CHAR}+ -{ - continueAggregate(); -} - -<CDATA> . -{ - continueAggregate(); -} - -<CDATA> ("]""]"+">") +<CDATA> ~("]""]"+">") { continueAggregate(); yybegin(E4X); @@ -1017,32 +1002,16 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]" <CDATA><<EOF>> { - continueAggregate(); - yybegin(E4X); - return buildAggregateToken(TOKEN_E4X_CDATA); -} - -<E4XCOMMENT> ([^-])* -{ - continueAggregate(); -} - -<E4XCOMMENT> ("-"+[^->]) -{ - continueAggregate(); + reportUnclosedCDATA(); + return null; } -<E4XCOMMENT> {WHITE_SPACE_CHAR}+ +<E4XCOMMENT> [^] { continueAggregate(); } -<E4XCOMMENT> . -{ - continueAggregate(); -} - -<E4XCOMMENT> ("-""-"+">") +<E4XCOMMENT> ~("-""-"+">") { continueAggregate(); yybegin(E4X); @@ -1051,9 +1020,8 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]" <E4XCOMMENT><<EOF>> { - continueAggregate(); - yybegin(E4X); - return buildAggregateToken(TOKEN_E4X_COMMENT); + reportUnclosedComment(); + return null; } <DIRECTIVE> ([^?])* http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex ---------------------------------------------------------------------- diff --git a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex index 4c3a6f6..4447086 100644 --- a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex +++ b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex @@ -81,7 +81,9 @@ public RawMXMLTokenizer() protected final Token buildToken(int type, int start, int end, int line, int column, String text) { - return new MXMLToken(type, start, end, line, column, text); + MXMLToken token = new MXMLToken(type, start, end, line, column, text); + token.setSourcePath(sourcePath); + return token; } public void reset() @@ -260,52 +262,39 @@ WHITE_SPACE_CHAR=[\r\n\ \t\b\012] // before returning to the initial state. // -<COMMENT,ASDOC_COMMENT> ([^-])* -{ - continueAggregate(); -} - -<COMMENT,ASDOC_COMMENT> ("-"+[^->]) -{ - continueAggregate(); -} - -<COMMENT,ASDOC_COMMENT> {WHITE_SPACE_CHAR}+ +<COMMENT,ASDOC_COMMENT> [^] { continueAggregate(); } -<COMMENT> ("-""-"+">") +<COMMENT> ~("-""-"+">") { continueAggregate(); yybegin(YYINITIAL); return buildAggregateToken(TOKEN_COMMENT); } -<ASDOC_COMMENT> ("-""-"+">") +<ASDOC_COMMENT> ~("-""-"+">") { continueAggregate(); yybegin(YYINITIAL); return buildAggregateToken(TOKEN_ASDOC_COMMENT); } -<COMMENT,ASDOC_COMMENT> . -{ - continueAggregate(); -} - <COMMENT><<EOF>> { - continueAggregate(); - yybegin(YYINITIAL); - return buildAggregateToken(TOKEN_COMMENT); + MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_COMMENT); + if (token != null) + reportUnclosedComment(token); + return token; } <ASDOC_COMMENT><<EOF>> { - continueAggregate(); - yybegin(YYINITIAL); - return buildAggregateToken(TOKEN_ASDOC_COMMENT); + MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_ASDOC_COMMENT); + if (token != null) + reportUnclosedASDocComment(token); + return token; } // @@ -517,38 +506,25 @@ WHITE_SPACE_CHAR=[\r\n\ \t\b\012] // before returning to the initial state. // -<CDATA> ([^\]])* +<CDATA> [^] { continueAggregate(); } -<CDATA> ("]"+[^\]>]) -{ - continueAggregate(); -} - -<CDATA> {WHITE_SPACE_CHAR}+ -{ - continueAggregate(); -} -<CDATA> ("]""]"+">") +<CDATA> ~("]""]"+">") { continueAggregate(); yybegin(YYINITIAL); return buildAggregateToken(TOKEN_CDATA); } -<CDATA> . -{ - continueAggregate(); -} - <CDATA><<EOF>> { - continueAggregate(); - yybegin(YYINITIAL); - return buildAggregateToken(TOKEN_CDATA); + MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_CDATA); + if (token != null) + reportUnclosedCDATA(token); + return token; } //
