http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6017f069/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java index 25ee006..3d816d3 100644 --- a/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java +++ b/src/main/java/org/apache/sysml/parser/pydml/PyDMLParserWrapper.java @@ -39,25 +39,16 @@ import org.apache.commons.logging.LogFactory; import org.apache.sysml.api.DMLScript; import org.apache.sysml.parser.AParserWrapper; import org.apache.sysml.parser.DMLProgram; -import org.apache.sysml.parser.ForStatement; -import org.apache.sysml.parser.ForStatementBlock; import org.apache.sysml.parser.FunctionStatementBlock; -import org.apache.sysml.parser.IfStatement; -import org.apache.sysml.parser.IfStatementBlock; import org.apache.sysml.parser.ImportStatement; import org.apache.sysml.parser.LanguageException; -import org.apache.sysml.parser.ParForStatement; -import org.apache.sysml.parser.ParForStatementBlock; import org.apache.sysml.parser.ParseException; import org.apache.sysml.parser.Statement; -import org.apache.sysml.parser.StatementBlock; -import org.apache.sysml.parser.WhileStatement; -import org.apache.sysml.parser.WhileStatementBlock; +import org.apache.sysml.parser.common.SyntacticErrorListener.CustomErrorListener; import org.apache.sysml.parser.dml.DMLParserWrapper; import org.apache.sysml.parser.pydml.PydmlParser.FunctionStatementContext; -import org.apache.sysml.parser.pydml.PydmlParser.PmlprogramContext; +import org.apache.sysml.parser.pydml.PydmlParser.ProgramrootContext; import org.apache.sysml.parser.pydml.PydmlParser.StatementContext; -import org.apache.sysml.parser.pydml.PydmlSyntacticErrorListener.CustomDmlErrorListener; /** * Logic of this wrapper is similar to DMLParserWrapper. @@ -70,37 +61,6 @@ public class PyDMLParserWrapper extends AParserWrapper private static final Log LOG = LogFactory.getLog(DMLScript.class.getName()); /** - * Custom wrapper to convert statement into statement blocks. Called by doParse and in PydmlSyntacticValidator for for, parfor, while, ... - * @param current a statement - * @return corresponding statement block - */ - public static StatementBlock getStatementBlock(org.apache.sysml.parser.Statement current) { - StatementBlock blk = null; - if(current instanceof ParForStatement) { - blk = new ParForStatementBlock(); - blk.addStatement(current); - } - else if(current instanceof ForStatement) { - blk = new ForStatementBlock(); - blk.addStatement(current); - } - else if(current instanceof IfStatement) { - blk = new IfStatementBlock(); - blk.addStatement(current); - } - else if(current instanceof WhileStatement) { - blk = new WhileStatementBlock(); - blk.addStatement(current); - } - else { - // This includes ImportStatement - blk = new StatementBlock(); - blk.addStatement(current); - } - return blk; - } - - /** * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command) * @param fileName either full path or null --> only used for better error handling * @param dmlScript required @@ -116,16 +76,14 @@ public class PyDMLParserWrapper extends AParserWrapper throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename"); } - // Set the pipeline required for ANTLR parsing - PyDMLParserWrapper parser = new PyDMLParserWrapper(); - prog = parser.doParse(fileName, dmlScript, argVals); + prog = doParse(fileName, dmlScript, argVals); if(prog == null) { - throw new ParseException("One or more errors found during parsing. (could not construct AST for file: " + fileName + "). Cannot proceed ahead."); + throw new ParseException("One or more errors found during parsing (could not construct AST for file: " + fileName + "). Cannot proceed ahead."); } return prog; } - + /** * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import' * @param fileName @@ -137,7 +95,7 @@ public class PyDMLParserWrapper extends AParserWrapper ANTLRInputStream in; try { if(dmlScript == null) { - dmlScript = DMLParserWrapper.readDMLScript(fileName); + dmlScript = DMLParserWrapper.readDMLScript(fileName, LOG); } InputStream stream = new ByteArrayInputStream(dmlScript.getBytes()); @@ -153,8 +111,8 @@ public class PyDMLParserWrapper extends AParserWrapper throw new ParseException("ERROR: " + e.getMessage(), e); } - PmlprogramContext ast = null; - CustomDmlErrorListener errorListener = new CustomDmlErrorListener(); + ProgramrootContext ast = null; + CustomErrorListener errorListener = new CustomErrorListener(); try { PydmlLexer lexer = new PydmlLexer(in); @@ -169,7 +127,7 @@ public class PyDMLParserWrapper extends AParserWrapper antlr4Parser.removeErrorListeners(); antlr4Parser.setErrorHandler(new BailErrorStrategy()); try{ - ast = antlr4Parser.pmlprogram(); + ast = antlr4Parser.programroot(); // If successful, no need to try out full LL(*) ... SLL was enough } catch(ParseCancellationException ex) { @@ -177,26 +135,26 @@ public class PyDMLParserWrapper extends AParserWrapper tokens.reset(); antlr4Parser.reset(); if(fileName != null) { - errorListener.pushCurrentFileName(fileName); + errorListener.setCurrentFileName(fileName); } else { - errorListener.pushCurrentFileName("MAIN_SCRIPT"); + errorListener.setCurrentFileName("MAIN_SCRIPT"); } // Set our custom error listener antlr4Parser.addErrorListener(errorListener); antlr4Parser.setErrorHandler(new DefaultErrorStrategy()); antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL); - ast = antlr4Parser.pmlprogram(); + ast = antlr4Parser.programroot(); } } else { // Set our custom error listener antlr4Parser.removeErrorListeners(); antlr4Parser.addErrorListener(errorListener); - errorListener.pushCurrentFileName(fileName); + errorListener.setCurrentFileName(fileName); // Now do the parsing - ast = antlr4Parser.pmlprogram(); + ast = antlr4Parser.programroot(); } } catch(Exception e) { @@ -210,10 +168,9 @@ public class PyDMLParserWrapper extends AParserWrapper ParseTree tree = ast; // And also do syntactic validation ParseTreeWalker walker = new ParseTreeWalker(); - PydmlSyntacticValidatorHelper helper = new PydmlSyntacticValidatorHelper(errorListener); - PydmlSyntacticValidator validator = new PydmlSyntacticValidator(helper, fileName, argVals); + PydmlSyntacticValidator validator = new PydmlSyntacticValidator(errorListener, argVals); walker.walk(validator, tree); - errorListener.popFileName(); + errorListener.unsetCurrentFileName(); if(errorListener.isAtleastOneError()) { return null; } @@ -227,7 +184,7 @@ public class PyDMLParserWrapper extends AParserWrapper } - private DMLProgram createDMLProgram(PmlprogramContext ast) { + private DMLProgram createDMLProgram(ProgramrootContext ast) { DMLProgram dmlPgm = new DMLProgram();
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6017f069/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4 ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4 b/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4 index 63b9835..10063c0 100644 --- a/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4 +++ b/src/main/java/org/apache/sysml/parser/pydml/Pydml.g4 @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -39,6 +39,8 @@ grammar Pydml; * specific language governing permissions and limitations * under the License. */ + import org.apache.sysml.parser.common.ExpressionInfo; + import org.apache.sysml.parser.common.StatementInfo; } // This antlr grammar is based on Python 3.3 language reference: https://docs.python.org/3.3/reference/grammar.html @@ -148,7 +150,7 @@ tokens { INDENT, DEDENT } // DML Program is a list of expression // For now, we only allow global function definitions (not nested or inside a while block) -pmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* NEWLINE* EOF; +programroot: (blocks+=statement | functionBlocks+=functionStatement)* NEWLINE* EOF; @@ -163,19 +165,19 @@ statement returns [ StatementInfo info ] | 'setwd' OPEN_PAREN pathValue = STRING CLOSE_PAREN NEWLINE # PathStatement // ------------------------------------------ // AssignmentStatement - | targetList+=dataIdentifier '=' 'ifdef' OPEN_PAREN commandLineParam=dataIdentifier ',' source=expression CLOSE_PAREN NEWLINE # IfdefAssignmentStatement + | targetList=dataIdentifier '=' 'ifdef' OPEN_PAREN commandLineParam=dataIdentifier ',' source=expression CLOSE_PAREN NEWLINE # IfdefAssignmentStatement // ------------------------------------------ // Treat function call as AssignmentStatement or MultiAssignmentStatement // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A // Convert FunctionCallIdentifier(paramExprs, ..) -> source | // TODO: Throw an informative error if user doesnot provide the optional assignment - ( targetList+=dataIdentifier '=' )? name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallAssignmentStatement + ( targetList=dataIdentifier '=' )? name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallAssignmentStatement | OPEN_BRACK targetList+=dataIdentifier (',' targetList+=dataIdentifier)* CLOSE_BRACK '=' name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallMultiAssignmentStatement // {notifyErrorListeners("Too many parentheses");} // We don't support block statement // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement // ------------------------------------------ - | targetList+=dataIdentifier '=' source=expression NEWLINE # AssignmentStatement + | targetList=dataIdentifier '=' source=expression NEWLINE # AssignmentStatement // IfStatement // | 'if' OPEN_PAREN predicate=expression CLOSE_PAREN (ifBody+=statement ';'* | NEWLINE INDENT (ifBody+=statement)+ DEDENT ) ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement | 'if' (OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression) ':' NEWLINE INDENT (ifBody+=statement)+ DEDENT ('else' ':' NEWLINE INDENT (elseBody+=statement)+ DEDENT )? # IfStatement http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6017f069/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticErrorListener.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticErrorListener.java deleted file mode 100644 index 8b2cc34..0000000 --- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticErrorListener.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.parser.pydml; - -import java.util.Stack; - -import org.antlr.v4.runtime.BaseErrorListener; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.sysml.api.DMLScript; - -public class PydmlSyntacticErrorListener -{ - private static final Log LOG = LogFactory.getLog(DMLScript.class.getName()); - - public static class CustomDmlErrorListener extends BaseErrorListener { - - private boolean atleastOneError = false; - private Stack<String> currentFileName = new Stack<String>(); - - public void pushCurrentFileName(String currentFilePath) { - currentFileName.push(currentFilePath); - } - - public String peekFileName() { - return currentFileName.peek(); - } - - public String popFileName() { - return currentFileName.pop(); - } - - public void validationError(int line, int charPositionInLine, String msg) { - try { - setAtleastOneError(true); - // Print error messages with file name - if(currentFileName == null || currentFileName.empty()) { - LOG.error("line "+line+":"+charPositionInLine+" "+msg); - } - else { - String fileName = currentFileName.peek(); - LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg); - } - } - catch(Exception e1) { - LOG.error("ERROR: while customizing error message:" + e1); - } - } - - public void validationWarning(int line, int charPositionInLine, String msg) { - try { - //atleastOneError = true; ---> not an error, just warning - // Print error messages with file name - if(currentFileName == null || currentFileName.empty()) - LOG.warn("line "+line+":"+charPositionInLine+" "+msg); - else { - String fileName = currentFileName.peek(); - LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg); - } - } - catch(Exception e1) { - LOG.warn("ERROR: while customizing error message:" + e1); - } - } - - @Override - public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, - int line, int charPositionInLine, - String msg, RecognitionException e) - { - try { - setAtleastOneError(true); - // Print error messages with file name - if(currentFileName == null || currentFileName.empty()) - LOG.error("line "+line+":"+charPositionInLine+" "+msg); - else { - String fileName = currentFileName.peek(); - LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg); - } - } - catch(Exception e1) { - LOG.error("ERROR: while customizing error message:" + e1); - } - } - - public boolean isAtleastOneError() { - return atleastOneError; - } - - public void setAtleastOneError(boolean atleastOneError) { - this.atleastOneError = atleastOneError; - } - } -}
