http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/pydml/StatementInfo.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/StatementInfo.java b/src/main/java/org/apache/sysml/parser/pydml/StatementInfo.java new file mode 100644 index 0000000..c19f1cd --- /dev/null +++ b/src/main/java/org/apache/sysml/parser/pydml/StatementInfo.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.parser.pydml; + +/** + * This class exists solely to prevent compiler warnings. + * + * <p> + * The ExpressionInfo and StatementInfo classes are shared among both parsers + * (R-like and Python-like dialects), and Antlr-generated code assumes that + * these classes are present in the parser's namespace. + */ +class StatementInfo extends org.apache.sysml.parser.dml.StatementInfo { + +}
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java b/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java deleted file mode 100644 index 71962cf..0000000 --- a/src/main/java/org/apache/sysml/parser/python/ExpressionInfo.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.parser.python; - -/** - * This class exists solely to prevent compiler warnings. - * - * <p> - * The ExpressionInfo and StatementInfo classes are shared among both parsers - * (R-like and Python-like dialects), and Antlr-generated code assumes that - * these classes are present in the parser's namespace. - */ -class ExpressionInfo extends org.apache.sysml.parser.antlr4.ExpressionInfo { - -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java b/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java deleted file mode 100644 index e52bb73..0000000 --- a/src/main/java/org/apache/sysml/parser/python/PyDMLParserWrapper.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.parser.python; - -import java.io.ByteArrayInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.BailErrorStrategy; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.DefaultErrorStrategy; -import org.antlr.v4.runtime.atn.PredictionMode; -import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeWalker; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.parser.AParserWrapper; -import org.apache.sysml.parser.DMLProgram; -import org.apache.sysml.parser.ForStatement; -import org.apache.sysml.parser.ForStatementBlock; -import org.apache.sysml.parser.FunctionStatementBlock; -import org.apache.sysml.parser.IfStatement; -import org.apache.sysml.parser.IfStatementBlock; -import org.apache.sysml.parser.ImportStatement; -import org.apache.sysml.parser.LanguageException; -import org.apache.sysml.parser.ParForStatement; -import org.apache.sysml.parser.ParForStatementBlock; -import org.apache.sysml.parser.ParseException; -import org.apache.sysml.parser.Statement; -import org.apache.sysml.parser.StatementBlock; -import org.apache.sysml.parser.WhileStatement; -import org.apache.sysml.parser.WhileStatementBlock; -import org.apache.sysml.parser.antlr4.DMLParserWrapper; -import org.apache.sysml.parser.python.PydmlParser.FunctionStatementContext; -import org.apache.sysml.parser.python.PydmlParser.PmlprogramContext; -import org.apache.sysml.parser.python.PydmlParser.StatementContext; -import org.apache.sysml.parser.python.PydmlSyntacticErrorListener.CustomDmlErrorListener; - -/** - * Logic of this wrapper is similar to DMLParserWrapper. - * - * Note: ExpressionInfo and StatementInfo are simply wrapper objects and are reused in both DML and PyDML parsers. - * - */ -public class PyDMLParserWrapper extends AParserWrapper -{ - private static final Log LOG = LogFactory.getLog(DMLScript.class.getName()); - - /** - * Custom wrapper to convert statement into statement blocks. Called by doParse and in PydmlSyntacticValidator for for, parfor, while, ... - * @param current a statement - * @return corresponding statement block - */ - public static StatementBlock getStatementBlock(org.apache.sysml.parser.Statement current) { - StatementBlock blk = null; - if(current instanceof ParForStatement) { - blk = new ParForStatementBlock(); - blk.addStatement(current); - } - else if(current instanceof ForStatement) { - blk = new ForStatementBlock(); - blk.addStatement(current); - } - else if(current instanceof IfStatement) { - blk = new IfStatementBlock(); - blk.addStatement(current); - } - else if(current instanceof WhileStatement) { - blk = new WhileStatementBlock(); - blk.addStatement(current); - } - else { - // This includes ImportStatement - blk = new StatementBlock(); - blk.addStatement(current); - } - return blk; - } - - /** - * Parses the passed file with command line parameters. You can either pass both (local file) or just dmlScript (hdfs) or just file name (import command) - * @param fileName either full path or null --> only used for better error handling - * @param dmlScript required - * @param argVals - * @return - * @throws ParseException - */ - @Override - public DMLProgram parse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException { - DMLProgram prog = null; - - if(dmlScript == null || dmlScript.trim().isEmpty()) { - throw new ParseException("Incorrect usage of parse. Please pass dmlScript not just filename"); - } - - // Set the pipeline required for ANTLR parsing - PyDMLParserWrapper parser = new PyDMLParserWrapper(); - prog = parser.doParse(fileName, dmlScript, argVals); - - if(prog == null) { - throw new ParseException("One or more errors found during parsing. (could not construct AST for file: " + fileName + "). Cannot proceed ahead."); - } - return prog; - } - - /** - * This function is supposed to be called directly only from PydmlSyntacticValidator when it encounters 'import' - * @param fileName - * @return null if atleast one error - */ - public DMLProgram doParse(String fileName, String dmlScript, HashMap<String,String> argVals) throws ParseException { - DMLProgram dmlPgm = null; - - ANTLRInputStream in; - try { - if(dmlScript == null) { - dmlScript = DMLParserWrapper.readDMLScript(fileName); - } - - InputStream stream = new ByteArrayInputStream(dmlScript.getBytes()); - in = new org.antlr.v4.runtime.ANTLRInputStream(stream); - } - catch (FileNotFoundException e) { - throw new ParseException("ERROR: Cannot find file:" + fileName, e); - } - catch (IOException e) { - throw new ParseException("ERROR: Cannot open file:" + fileName, e); - } - catch (LanguageException e) { - throw new ParseException("ERROR: " + e.getMessage(), e); - } - - PmlprogramContext ast = null; - CustomDmlErrorListener errorListener = new CustomDmlErrorListener(); - - try { - PydmlLexer lexer = new PydmlLexer(in); - CommonTokenStream tokens = new CommonTokenStream(lexer); - PydmlParser antlr4Parser = new PydmlParser(tokens); - - boolean tryOptimizedParsing = false; // For now no optimization, since it is not able to parse integer value. - - if(tryOptimizedParsing) { - // Try faster and simpler SLL - antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.SLL); - antlr4Parser.removeErrorListeners(); - antlr4Parser.setErrorHandler(new BailErrorStrategy()); - try{ - ast = antlr4Parser.pmlprogram(); - // If successful, no need to try out full LL(*) ... SLL was enough - } - catch(ParseCancellationException ex) { - // Error occurred, so now try full LL(*) for better error messages - tokens.reset(); - antlr4Parser.reset(); - if(fileName != null) { - errorListener.pushCurrentFileName(fileName); - } - else { - errorListener.pushCurrentFileName("MAIN_SCRIPT"); - } - // Set our custom error listener - antlr4Parser.addErrorListener(errorListener); - antlr4Parser.setErrorHandler(new DefaultErrorStrategy()); - antlr4Parser.getInterpreter().setPredictionMode(PredictionMode.LL); - ast = antlr4Parser.pmlprogram(); - } - } - else { - // Set our custom error listener - antlr4Parser.removeErrorListeners(); - antlr4Parser.addErrorListener(errorListener); - errorListener.pushCurrentFileName(fileName); - - // Now do the parsing - ast = antlr4Parser.pmlprogram(); - } - } - catch(Exception e) { - throw new ParseException("ERROR: Cannot parse the program:" + fileName, e); - } - - - try { - // Now convert the parse tree into DMLProgram - // Do syntactic validation while converting - ParseTree tree = ast; - // And also do syntactic validation - ParseTreeWalker walker = new ParseTreeWalker(); - PydmlSyntacticValidatorHelper helper = new PydmlSyntacticValidatorHelper(errorListener); - PydmlSyntacticValidator validator = new PydmlSyntacticValidator(helper, fileName, argVals); - walker.walk(validator, tree); - errorListener.popFileName(); - if(errorListener.isAtleastOneError()) { - return null; - } - dmlPgm = createDMLProgram(ast); - } - catch(Exception e) { - throw new ParseException("ERROR: Cannot translate the parse tree into DMLProgram" + e.getMessage(), e); - } - - return dmlPgm; - } - - - private DMLProgram createDMLProgram(PmlprogramContext ast) { - - DMLProgram dmlPgm = new DMLProgram(); - - // First add all the functions - for(FunctionStatementContext fn : ast.functionBlocks) { - FunctionStatementBlock functionStmtBlk = new FunctionStatementBlock(); - functionStmtBlk.addStatement(fn.info.stmt); - try { - // TODO: currently the logic of nested namespace is not clear. - String namespace = DMLProgram.DEFAULT_NAMESPACE; - dmlPgm.addFunctionStatementBlock(namespace, fn.info.functionName, functionStmtBlk); - } catch (LanguageException e) { - LOG.error("line: " + fn.start.getLine() + ":" + fn.start.getCharPositionInLine() + " cannot process the function " + fn.info.functionName); - return null; - } - } - - // Then add all the statements - for(StatementContext stmtCtx : ast.blocks) { - Statement current = stmtCtx.info.stmt; - if(current == null) { - LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the statement"); - return null; - } - - // Ignore Newline logic - if(current.isEmptyNewLineStatement()) { - continue; - } - - if(current instanceof ImportStatement) { - // Handle import statements separately - if(stmtCtx.info.namespaces != null) { - // Add the DMLProgram entries into current program - for(Map.Entry<String, DMLProgram> entry : stmtCtx.info.namespaces.entrySet()) { - dmlPgm.getNamespaces().put(entry.getKey(), entry.getValue()); - } - } - else { - LOG.error("line: " + stmtCtx.start.getLine() + ":" + stmtCtx.start.getCharPositionInLine() + " cannot process the import statement"); - return null; - } - } - - // Now wrap statement into individual statement block - // merge statement will take care of merging these blocks - dmlPgm.addStatementBlock(getStatementBlock(current)); - } - - dmlPgm.mergeStatementBlocks(); - return dmlPgm; - } -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/Pydml.g4 ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/python/Pydml.g4 b/src/main/java/org/apache/sysml/parser/python/Pydml.g4 deleted file mode 100644 index 7fbe42d..0000000 --- a/src/main/java/org/apache/sysml/parser/python/Pydml.g4 +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -grammar Pydml; - -@header -{ - // package org.apache.sysml.python; - //import org.apache.sysml.parser.antlr4.StatementInfo; - //import org.apache.sysml.parser.antlr4.ExpressionInfo; -} - -// This antlr grammar is based on Python 3.3 language reference: https://docs.python.org/3.3/reference/grammar.html - -tokens { INDENT, DEDENT } - -@lexer::members { - private boolean debugIndentRules = false; - - // Indentation level stack - private java.util.Stack<Integer> indents = new java.util.Stack<Integer>(); - - // Extra tokens queue (see the NEWLINE rule). - private java.util.Queue<Token> tokens = new java.util.LinkedList<Token>(); - - // Number of opened braces, brackets and parenthesis. - private int opened = 0; - - // This is only used to set the line number for dedent - private Token lastToken = null; - - - @Override - public void emit(Token t) { - if(debugIndentRules) - System.out.println("Emitted token:" + t); - - super.setToken(t); - tokens.offer(t); - } - - - @Override - public Token nextToken() { - if (_input.LA(1) == EOF && !this.indents.isEmpty()) { - if(debugIndentRules) - System.out.println("EOF reached and expecting some DEDENTS, so emitting them"); - - tokens.poll(); - this.emit(commonToken(PydmlParser.NEWLINE, "\n")); - - // Now emit as much DEDENT tokens as needed. - while (!indents.isEmpty()) { - if(debugIndentRules) - System.out.println("Emitting (inserted) DEDENTS"); - - this.emit(createDedent()); - indents.pop(); - } - // Put the EOF back on the token stream. - this.emit(commonToken(PydmlParser.EOF, "<EOF>")); - } - Token next = super.nextToken(); - if (next.getChannel() == Token.DEFAULT_CHANNEL) { - // Keep track of the last token on the default channel. - this.lastToken = next; - } - Token retVal = tokens.isEmpty() ? next : tokens.poll(); - - if(debugIndentRules) - System.out.println("Returning nextToken: [" + retVal + "]<<" + tokens.isEmpty()); - - return retVal; - } - - private Token createDedent() { - CommonToken dedent = commonToken(PydmlParser.DEDENT, ""); - dedent.setLine(this.lastToken.getLine()); - return dedent; - } - - private CommonToken commonToken(int type, String text) { - // Nike: Main change: This logic was screwed up and was emitting additional 3 characters, so commenting it for now. - // int start = this.getCharIndex(); - // int stop = start + text.length(); - // return new CommonToken(this._tokenFactorySourcePair, type, DEFAULT_TOKEN_CHANNEL, start, stop); - return new CommonToken(type, text); // Main change - } - - // Calculates the indentation level from the spaces: - // "Tabs are replaced (from left to right) by one to eight spaces - // such that the total number of characters up to and including - // the replacement is a multiple of eight [...]" - // https://docs.python.org/3.1/reference/lexical_analysis.html#indentation - static int getIndentationCount(String spaces) { - int count = 0; - for (char ch : spaces.toCharArray()) { - switch (ch) { - case '\t': - count += 8 - (count % 8); - break; - default: - // A normal space char. - count++; - } - } - return count; - } -} - - -// 2. Modify this g4 by comparing it with Java: -// - https://pythonconquerstheuniverse.wordpress.com/2009/10/03/python-java-a-side-by-side-comparison/ -// - http://www.cs.gordon.edu/courses/cps122/handouts-2014/From%20Python%20to%20Java%20Lecture/A%20Comparison%20of%20the%20Syntax%20of%20Python%20and%20Java.pdf -// - http://cs.joensuu.fi/~pviktor/python/slides/cheatsheet.pdf -// - http://www.interfaceware.com/manual/chameleon/scripts/quickreference.pdf - -// DML Program is a list of expression -// For now, we only allow global function definitions (not nested or inside a while block) -pmlprogram: (blocks+=statement | functionBlocks+=functionStatement)* NEWLINE* EOF; - - - -statement returns [ StatementInfo info ] -@init { - // This actions occurs regardless of how many alternatives in this rule - $info = new StatementInfo(); -} : - // ------------------------------------------ - // ImportStatement - 'source' OPEN_PAREN filePath = STRING CLOSE_PAREN 'as' namespace=ID NEWLINE # ImportStatement - | 'setwd' OPEN_PAREN pathValue = STRING CLOSE_PAREN NEWLINE # PathStatement - // ------------------------------------------ - // AssignmentStatement - | targetList+=dataIdentifier '=' 'ifdef' OPEN_PAREN commandLineParam=dataIdentifier ',' source=expression CLOSE_PAREN NEWLINE # IfdefAssignmentStatement - // ------------------------------------------ - // Treat function call as AssignmentStatement or MultiAssignmentStatement - // For backward compatibility and also since the behavior of foo() * A + foo() ... where foo returns A - // Convert FunctionCallIdentifier(paramExprs, ..) -> source - | // TODO: Throw an informative error if user doesnot provide the optional assignment - ( targetList+=dataIdentifier '=' )? name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallAssignmentStatement - | OPEN_BRACK targetList+=dataIdentifier (',' targetList+=dataIdentifier)* CLOSE_BRACK '=' name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN NEWLINE # FunctionCallMultiAssignmentStatement - // {notifyErrorListeners("Too many parentheses");} - // We don't support block statement - // | '{' body+=expression ';'* ( body+=expression ';'* )* '}' # BlockStatement - // ------------------------------------------ - | targetList+=dataIdentifier '=' source=expression NEWLINE # AssignmentStatement - // IfStatement - // | 'if' OPEN_PAREN predicate=expression CLOSE_PAREN (ifBody+=statement ';'* | NEWLINE INDENT (ifBody+=statement)+ DEDENT ) ('else' (elseBody+=statement ';'* | '{' (elseBody+=statement ';'*)* '}'))? # IfStatement - | 'if' (OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression) ':' NEWLINE INDENT (ifBody+=statement)+ DEDENT ('else' ':' NEWLINE INDENT (elseBody+=statement)+ DEDENT )? # IfStatement - // ------------------------------------------ - // ForStatement & ParForStatement - | 'for' (OPEN_PAREN iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* CLOSE_PAREN | iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # ForStatement - // Convert strictParameterizedExpression to HashMap<String, String> for parForParams - | 'parfor' (OPEN_PAREN iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* CLOSE_PAREN | iterVar=ID 'in' iterPred=iterablePredicate (',' parForParams+=strictParameterizedExpression)* ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # ParForStatement - | 'while' ( OPEN_PAREN predicate=expression CLOSE_PAREN | predicate=expression ) ':' NEWLINE INDENT (body+=statement)+ DEDENT # WhileStatement - // ------------------------------------------ - | NEWLINE #IgnoreNewLine -; - -iterablePredicate returns [ ExpressionInfo info ] - @init { - // This actions occurs regardless of how many alternatives in this rule - $info = new ExpressionInfo(); - } : - from=expression ':' to=expression #IterablePredicateColonExpression - | ID OPEN_PAREN from=expression ',' to=expression ',' increment=expression CLOSE_PAREN #IterablePredicateSeqExpression - ; - -functionStatement returns [ StatementInfo info ] -@init { - // This actions occurs regardless of how many alternatives in this rule - $info = new StatementInfo(); -} : - // ------------------------------------------ - // FunctionStatement & ExternalFunctionStatement - // small change: only allow typed arguments here ... instead of data identifier - 'def' name=ID OPEN_PAREN ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? CLOSE_PAREN ( '->' OPEN_PAREN ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? CLOSE_PAREN )? ':' NEWLINE INDENT (body+=statement)+ DEDENT # InternalFunctionDefExpression - | 'defExternal' name=ID OPEN_PAREN ( inputParams+=typedArgNoAssign (',' inputParams+=typedArgNoAssign)* )? CLOSE_PAREN ( '->' OPEN_PAREN ( outputParams+=typedArgNoAssign (',' outputParams+=typedArgNoAssign)* )? CLOSE_PAREN )? 'implemented' 'in' OPEN_PAREN ( otherParams+=strictParameterizedKeyValueString (',' otherParams+=strictParameterizedKeyValueString)* )? CLOSE_PAREN NEWLINE # ExternalFunctionDefExpression - // ------------------------------------------ -; - - -// Other data identifiers are typedArgNoAssign, parameterizedExpression and strictParameterizedExpression -dataIdentifier returns [ ExpressionInfo dataInfo ] -@init { - // This actions occurs regardless of how many alternatives in this rule - $dataInfo = new ExpressionInfo(); - // $dataInfo.expr = new org.apache.sysml.parser.DataIdentifier(); -} : - // ------------------------------------------ - // IndexedIdentifier - name=ID OPEN_BRACK (rowLower=expression (':' rowUpper=expression)?)? ',' (colLower=expression (':' colUpper=expression)?)? CLOSE_BRACK # IndexedExpression - // ------------------------------------------ - | ID # SimpleDataIdentifierExpression - | COMMANDLINE_NAMED_ID # CommandlineParamExpression - | COMMANDLINE_POSITION_ID # CommandlinePositionExpression -; -expression returns [ ExpressionInfo info ] -@init { - // This actions occurs regardless of how many alternatives in this rule - $info = new ExpressionInfo(); - // $info.expr = new org.apache.sysml.parser.BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.INVALID); -} : - // ------------------------------------------ - // BinaryExpression - // power - <assoc=right> left=expression op='**' right=expression # PowerExpression - // unary plus and minus - | op=('-'|'+') left=expression # UnaryExpression - // sequence - since we are only using this into for loop => Array not supported - //| left=expression op=':' right=expression # SequenceExpression - // matrix multiply - // | left=expression op='*' right=expression # MatrixMulExpression - // modulus and integer division - | left=expression op=('//' | '%' ) right=expression # ModIntDivExpression - // arithmetic multiply and divide - | left=expression op=('*'|'/') right=expression # MultDivExpression - // arithmetic addition and subtraction - | left=expression op=('+'|'-') right=expression # AddSubExpression - // ------------------------------------------ - // RelationalExpression - | left=expression op=('>'|'>='|'<'|'<='|'=='|'!=') right=expression # RelationalExpression - // ------------------------------------------ - // BooleanExpression - // boolean not - | op='!' left=expression # BooleanNotExpression - // boolean and - | left=expression op=('&'|'and') right=expression # BooleanAndExpression - // boolean or - | left=expression op=('|'|'or') right=expression # BooleanOrExpression - - // --------------------------------- - // only applicable for builtin function expressions - // Add following additional functions and check number of parameters: - // power, full, matrix, reshape, dot - // Also take care whether there is y.transpose() => which sometinamespace - | name=ID OPEN_PAREN (paramExprs+=parameterizedExpression (',' paramExprs+=parameterizedExpression)* )? CLOSE_PAREN ';'* # BuiltinFunctionExpression - - // 4. Atomic - | OPEN_PAREN left=expression CLOSE_PAREN # AtomicExpression - - // Should you allow indexed expression here ? - // | OPEN_BRACK targetList+=expression (',' targetList+=expression)* CLOSE_BRACK # MultiIdExpression - - // | BOOLEAN # ConstBooleanIdExpression - | 'True' # ConstTrueExpression - | 'False' # ConstFalseExpression - | INT # ConstIntIdExpression - | DOUBLE # ConstDoubleIdExpression - | STRING # ConstStringIdExpression - | dataIdentifier # DataIdExpression - // Special - // | 'NULL' | 'NA' | 'Inf' | 'NaN' -; - -typedArgNoAssign : paramName=ID ':' paramType=ml_type ; -parameterizedExpression : (paramName=ID '=')? paramVal=expression; -strictParameterizedExpression : paramName=ID '=' paramVal=expression ; -strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ; -// sometimes this is matrix object and sometimes its namespace -ID : (ALPHABET (ALPHABET|DIGIT|'_')* '.')? ALPHABET (ALPHABET|DIGIT|'_')* - // Special ID cases: - // | 'matrix' // --> This is a special case which causes lot of headache - // | 'scalar' | 'float' | 'int' | 'bool' // corresponds to as.scalar, as.double, as.integer and as.logical - | 'index.return' -; -// Unfortunately, we have datatype name clashing with builtin function name: matrix :( -// Therefore, ugly work around for checking datatype -ml_type : valueType | dataType OPEN_BRACK valueType CLOSE_BRACK; -// Note to reduce number of keywords, these are case-sensitive, -// To allow case-insenstive, 'int' becomes: ('i' | 'I') ('n' | 'N') ('t' | 'T') -valueType: - ID # ValueDataTypeCheck - // 'int' | 'str' | 'bool' | 'float' -; -dataType: - // 'scalar' # ScalarDataTypeDummyCheck - // | - ID # MatrixDataTypeCheck //{ if($ID.text.compareTo("matrix") != 0) { notifyErrorListeners("incorrect datatype"); } } - //| 'matrix' //---> See ID, this causes lot of headache - ; -INT : DIGIT+ [Ll]?; -// BOOLEAN : 'TRUE' | 'FALSE'; -DOUBLE: DIGIT+ '.' DIGIT* EXP? [Ll]? -| DIGIT+ EXP? [Ll]? -| '.' DIGIT+ EXP? [Ll]? -; -DIGIT: '0'..'9'; -ALPHABET : [a-zA-Z] ; -fragment EXP : ('E' | 'e') ('+' | '-')? INT ; -COMMANDLINE_NAMED_ID: '$' ALPHABET (ALPHABET|DIGIT|'_')*; -COMMANDLINE_POSITION_ID: '$' DIGIT+; - -// supports single and double quoted string with escape characters -STRING: '"' ( ESC | ~[\\"] )*? '"' | '\'' ( ESC | ~[\\'] )*? '\''; -fragment ESC : '\\' [abtnfrv"'\\] ; -// Comments, whitespaces and new line -// LINE_COMMENT : '#' .*? '\r'? '\n' -> skip ; -// MULTILINE_BLOCK_COMMENT : '/*' .*? '*/' -> skip ; -// WHITESPACE : (' ' | '\r' | '\n')+ -> skip ; - -OPEN_BRACK : '[' {opened++;}; -CLOSE_BRACK : ']' {opened--;}; -OPEN_PAREN : '(' {opened++;}; -CLOSE_PAREN : ')' {opened--;}; -// OPEN_BRACE : '{' {opened++;}; -// CLOSE_BRACE : '}' {opened--;}; - -fragment SPACES : [ \t]+ ; -fragment COMMENT : '#' ~[\r\n]* ; -fragment LINE_JOINING : '\\' SPACES? ( '\r'? '\n' | '\r' ) ; - -NEWLINE : ( '\r'? '\n' | '\r' ) SPACES? -{ - String newLine = getText().replaceAll("[^\r\n]+", ""); - String spaces = getText().replaceAll("[\r\n]+", ""); - int next = _input.LA(1); - if (opened > 0 || next == '\r' || next == '\n' || next == '#') { - // If we're inside a list or on a blank line, ignore all indents, - // dedents and line breaks. - skip(); - if(debugIndentRules) { - if(next == '\r' || next == '\n') { - System.out.println("4.1 Skipping (blank lines)"); - } - else if(next == '#') { - System.out.println("4.2 Skipping (comment)"); - } - else { - System.out.println("4.2 Skipping something else"); - } - } - } - else { - emit(commonToken(NEWLINE, newLine)); - - int indent = getIndentationCount(spaces); - int previous = indents.isEmpty() ? 0 : indents.peek(); - if (indent == previous) { - if(debugIndentRules) - System.out.println("3. Skipping identation as of same size:" + next); - - // skip indents of the same size as the present indent-size - skip(); - } - else if (indent > previous) { - if(debugIndentRules) - System.out.println("1. Indent:" + next); - - indents.push(indent); - emit(commonToken(PydmlParser.INDENT, spaces)); - } - else { - // Possibly emit more than 1 DEDENT token. - while(!indents.isEmpty() && indents.peek() > indent) { - if(debugIndentRules) - System.out.println("2. Dedent:" + next); - - this.emit(createDedent()); - indents.pop(); - } - } - } -} -; - -SKIP : ( SPACES | COMMENT | LINE_JOINING ) -> skip ; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c04fc99f/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java b/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java deleted file mode 100644 index 9ae63a9..0000000 --- a/src/main/java/org/apache/sysml/parser/python/PydmlSyntacticErrorListener.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.parser.python; - -import java.util.Stack; - -import org.antlr.v4.runtime.BaseErrorListener; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.sysml.api.DMLScript; - -public class PydmlSyntacticErrorListener -{ - private static final Log LOG = LogFactory.getLog(DMLScript.class.getName()); - - public static class CustomDmlErrorListener extends BaseErrorListener { - - private boolean atleastOneError = false; - private Stack<String> currentFileName = new Stack<String>(); - - public void pushCurrentFileName(String currentFilePath) { - currentFileName.push(currentFilePath); - } - - public String peekFileName() { - return currentFileName.peek(); - } - - public String popFileName() { - return currentFileName.pop(); - } - - public void validationError(int line, int charPositionInLine, String msg) { - try { - setAtleastOneError(true); - // Print error messages with file name - if(currentFileName == null || currentFileName.empty()) { - LOG.error("line "+line+":"+charPositionInLine+" "+msg); - } - else { - String fileName = currentFileName.peek(); - LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg); - } - } - catch(Exception e1) { - LOG.error("ERROR: while customizing error message:" + e1); - } - } - - public void validationWarning(int line, int charPositionInLine, String msg) { - try { - //atleastOneError = true; ---> not an error, just warning - // Print error messages with file name - if(currentFileName == null || currentFileName.empty()) - LOG.warn("line "+line+":"+charPositionInLine+" "+msg); - else { - String fileName = currentFileName.peek(); - LOG.warn(fileName + " line "+line+":"+charPositionInLine+" "+msg); - } - } - catch(Exception e1) { - LOG.warn("ERROR: while customizing error message:" + e1); - } - } - - @Override - public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, - int line, int charPositionInLine, - String msg, RecognitionException e) - { - try { - setAtleastOneError(true); - // Print error messages with file name - if(currentFileName == null || currentFileName.empty()) - LOG.error("line "+line+":"+charPositionInLine+" "+msg); - else { - String fileName = currentFileName.peek(); - LOG.error(fileName + " line "+line+":"+charPositionInLine+" "+msg); - } - } - catch(Exception e1) { - LOG.error("ERROR: while customizing error message:" + e1); - } - } - - public boolean isAtleastOneError() { - return atleastOneError; - } - - public void setAtleastOneError(boolean atleastOneError) { - this.atleastOneError = atleastOneError; - } - } -}
