Repository: systemml Updated Branches: refs/heads/master f1d35b780 -> c495533b6
http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/Statement.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/Statement.java b/src/main/java/org/apache/sysml/parser/Statement.java index 744117d..0b3aac2 100644 --- a/src/main/java/org/apache/sysml/parser/Statement.java +++ b/src/main/java/org/apache/sysml/parser/Statement.java @@ -19,10 +19,12 @@ package org.apache.sysml.parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.misc.Interval; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -public abstract class Statement +public abstract class Statement implements ParseInfo { @@ -87,19 +89,57 @@ public abstract class Statement private String _filename; private int _beginLine, _beginColumn; private int _endLine, _endColumn; + private String _text; public void setFilename(String passed) { _filename = passed; } public void setBeginLine(int passed) { _beginLine = passed; } public void setBeginColumn(int passed) { _beginColumn = passed;} public void setEndLine(int passed) { _endLine = passed; } public void setEndColumn(int passed) { _endColumn = passed; } - - public void setAllPositions(String filename, int blp, int bcp, int elp, int ecp){ - _filename = filename; - _beginLine = blp; - _beginColumn = bcp; - _endLine = elp; - _endColumn = ecp; + + /** + * Set ParserRuleContext values (begin line, begin column, end line, end + * column, and text). + * + * @param ctx + * the antlr ParserRuleContext + */ + public void setCtxValues(ParserRuleContext ctx) { + setBeginLine(ctx.start.getLine()); + setBeginColumn(ctx.start.getCharPositionInLine()); + setEndLine(ctx.stop.getLine()); + setEndColumn(ctx.stop.getCharPositionInLine()); + // preserve whitespace if possible + if ((ctx.start != null) && (ctx.stop != null) && (ctx.start.getStartIndex() != -1) + && (ctx.stop.getStopIndex() != -1) && (ctx.start.getStartIndex() <= ctx.stop.getStopIndex()) + && (ctx.start.getInputStream() != null)) { + String text = ctx.start.getInputStream() + .getText(Interval.of(ctx.start.getStartIndex(), ctx.stop.getStopIndex())); + if (text != null) { + text = text.trim(); + } + setText(text); + } else { + String text = ctx.getText(); + if (text != null) { + text = text.trim(); + } + setText(text); + } + } + + /** + * Set ParserRuleContext values (begin line, begin column, end line, end + * column, and text) and file name. + * + * @param ctx + * the antlr ParserRuleContext + * @param filename + * the filename (if it exists) + */ + public void setCtxValuesAndFilename(ParserRuleContext ctx, String filename) { + setCtxValues(ctx); + setFilename(filename); } public int getBeginLine() { return _beginLine; } @@ -107,37 +147,77 @@ public abstract class Statement public int getEndLine() { return _endLine; } public int getEndColumn() { return _endColumn; } public String getFilename() { return _filename; } - - public String printErrorLocation(){ - return "ERROR: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- "; + + public String printErrorLocation() { + String file = _filename; + if (file == null) { + file = ""; + } else { + file = file + " "; + } + if (getText() != null) { + return "ERROR: " + file + "[line " + _beginLine + ":" + _beginColumn + "] -> " + getText() + " -- "; + } else { + return "ERROR: " + file + "[line " + _beginLine + ":" + _beginColumn + "] -- "; + } } - - public String printWarningLocation(){ - return "WARNING: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- "; + + public String printWarningLocation() { + String file = _filename; + if (file == null) { + file = ""; + } else { + file = file + " "; + } + if (getText() != null) { + return "WARNING: " + file + "[line " + _beginLine + ":" + _beginColumn + "] -> " + getText() + " -- "; + } else { + return "WARNING: " + file + "[line " + _beginLine + ":" + _beginColumn + "] -- "; + } } - public void raiseValidateError( String msg, boolean conditional ) throws LanguageException { + public void raiseValidateError(String msg, boolean conditional) throws LanguageException { raiseValidateError(msg, conditional, null); } - - public void raiseValidateError( String msg, boolean conditional, String errorCode ) - throws LanguageException - { - if( conditional ) //warning if conditional - { + + public void raiseValidateError(String msg, boolean conditional, String errorCode) throws LanguageException { + if (conditional) {// warning if conditional String fullMsg = this.printWarningLocation() + msg; - - LOG.warn( fullMsg ); - } - else //error and exception if unconditional - { + LOG.warn(fullMsg); + } else {// error and exception if unconditional String fullMsg = this.printErrorLocation() + msg; - - //LOG.error( fullMsg ); //no redundant error - if( errorCode != null ) - throw new LanguageException( fullMsg, errorCode ); - else - throw new LanguageException( fullMsg ); + if (errorCode != null) + throw new LanguageException(fullMsg, errorCode); + else + throw new LanguageException(fullMsg); } - } + } + + public String getText() { + return _text; + } + + public void setText(String text) { + this._text = text; + } + + /** + * Set parse information. + * + * @param parseInfo + * parse information, such as beginning line position, beginning + * column position, ending line position, ending column position, + * text, and filename + * @param filename + * the DML/PYDML filename (if it exists) + */ + public void setParseInfo(ParseInfo parseInfo) { + _beginLine = parseInfo.getBeginLine(); + _beginColumn = parseInfo.getBeginColumn(); + _endLine = parseInfo.getEndLine(); + _endColumn = parseInfo.getEndColumn(); + _text = parseInfo.getText(); + _filename = parseInfo.getFilename(); + } + } http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/StatementBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/StatementBlock.java b/src/main/java/org/apache/sysml/parser/StatementBlock.java index d5cf1d7..768f6c0 100644 --- a/src/main/java/org/apache/sysml/parser/StatementBlock.java +++ b/src/main/java/org/apache/sysml/parser/StatementBlock.java @@ -41,7 +41,7 @@ import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence; import org.apache.sysml.utils.MLContextProxy; -public class StatementBlock extends LiveVariableAnalysis +public class StatementBlock extends LiveVariableAnalysis implements ParseInfo { protected static final Log LOG = LogFactory.getLog(StatementBlock.class.getName()); @@ -188,14 +188,21 @@ public class StatementBlock extends LiveVariableAnalysis || (sourceExpr instanceof ParameterizedBuiltinFunctionExpression && ((ParameterizedBuiltinFunctionExpression)sourceExpr).multipleReturns())) return false; - //function calls (only mergable if inlined dml-bodied function) - if (sourceExpr instanceof FunctionCallIdentifier){ + // function calls (only mergable if inlined dml-bodied function) + if (sourceExpr instanceof FunctionCallIdentifier) { FunctionCallIdentifier fcall = (FunctionCallIdentifier) sourceExpr; - FunctionStatementBlock fblock = dmlProg.getFunctionStatementBlock(fcall.getNamespace(), fcall.getName()); - if (fblock == null){ - throw new LanguageException(sourceExpr.printErrorLocation() + "function " + fcall.getName() + " is undefined in namespace " + fcall.getNamespace()); + FunctionStatementBlock fblock = dmlProg.getFunctionStatementBlock(fcall.getNamespace(), + fcall.getName()); + if (fblock == null) { + if (DMLProgram.DEFAULT_NAMESPACE.equals(fcall.getNamespace())) { + throw new LanguageException( + sourceExpr.printErrorLocation() + "Function " + fcall.getName() + "() is undefined."); + } else { + throw new LanguageException(sourceExpr.printErrorLocation() + "Function " + fcall.getName() + + "() is undefined in namespace '" + fcall.getNamespace() + "'."); + } } - if( !rIsInlineableFunction(fblock, dmlProg) ) + if (!rIsInlineableFunction(fblock, dmlProg)) return false; } } @@ -445,15 +452,15 @@ public class StatementBlock extends LiveVariableAnalysis //auto casting of inputs on inlining (if required) ValueType targetVT = newTarget.getValueType(); - if( newTarget.getDataType()==DataType.SCALAR && currCallParam.getOutput() != null - && targetVT != currCallParam.getOutput().getValueType() && targetVT != ValueType.STRING ) - { - currCallParam = new BuiltinFunctionExpression(BuiltinFunctionExpression.getValueTypeCastOperator(targetVT), new Expression[] {currCallParam}, - newTarget.getFilename(), newTarget.getBeginLine(), newTarget.getBeginColumn(), newTarget.getEndLine(), newTarget.getEndColumn()); + if (newTarget.getDataType() == DataType.SCALAR && currCallParam.getOutput() != null + && targetVT != currCallParam.getOutput().getValueType() && targetVT != ValueType.STRING) { + currCallParam = new BuiltinFunctionExpression( + BuiltinFunctionExpression.getValueTypeCastOperator(targetVT), + new Expression[] { currCallParam }, newTarget); } // create the assignment statement to bind the call parameter to formal parameter - AssignmentStatement binding = new AssignmentStatement(newTarget, currCallParam, newTarget.getBeginLine(), newTarget.getBeginColumn(), newTarget.getEndLine(), newTarget.getEndColumn()); + AssignmentStatement binding = new AssignmentStatement(newTarget, currCallParam, newTarget); newStatements.add(binding); } @@ -512,13 +519,14 @@ public class StatementBlock extends LiveVariableAnalysis //auto casting of inputs on inlining (always, redundant cast removed during Hop Rewrites) ValueType sourceVT = newSource.getValueType(); - if( newSource.getDataType()==DataType.SCALAR && sourceVT != ValueType.STRING ){ - newSource = new BuiltinFunctionExpression(BuiltinFunctionExpression.getValueTypeCastOperator(sourceVT), new Expression[] {newSource}, - newTarget.getFilename(), newTarget.getBeginLine(), newTarget.getBeginColumn(), newTarget.getEndLine(), newTarget.getEndColumn()); + if (newSource.getDataType() == DataType.SCALAR && sourceVT != ValueType.STRING) { + newSource = new BuiltinFunctionExpression( + BuiltinFunctionExpression.getValueTypeCastOperator(sourceVT), + new Expression[] { newSource }, newTarget); } // create the assignment statement to bind the call parameter to formal parameter - AssignmentStatement binding = new AssignmentStatement(newTarget, newSource, newTarget.getBeginLine(), newTarget.getBeginColumn(), newTarget.getEndLine(), newTarget.getEndColumn()); + AssignmentStatement binding = new AssignmentStatement(newTarget, newSource, newTarget); newStatements.add(binding); } @@ -618,12 +626,12 @@ public class StatementBlock extends LiveVariableAnalysis bife.raiseValidateError("Undefined Variable (" + id.getName() + ") used in statement", false, LanguageErrorCodes.INVALID_PARAMETERS); } IntIdentifier intid = null; - if (bife.getOpCode() == Expression.BuiltinFunctionOp.NROW){ - intid = new IntIdentifier((currVal instanceof IndexedIdentifier)?((IndexedIdentifier)currVal).getOrigDim1():currVal.getDim1(), - bife.getFilename(), bife.getBeginLine(), bife.getBeginColumn(), bife.getEndLine(), bife.getEndColumn()); + if (bife.getOpCode() == Expression.BuiltinFunctionOp.NROW) { + intid = new IntIdentifier((currVal instanceof IndexedIdentifier) + ? ((IndexedIdentifier) currVal).getOrigDim1() : currVal.getDim1(), bife); } else { - intid = new IntIdentifier((currVal instanceof IndexedIdentifier)?((IndexedIdentifier)currVal).getOrigDim2():currVal.getDim2(), - bife.getFilename(), bife.getBeginLine(), bife.getBeginColumn(), bife.getEndLine(), bife.getEndColumn()); + intid = new IntIdentifier((currVal instanceof IndexedIdentifier) + ? ((IndexedIdentifier) currVal).getOrigDim2() : currVal.getDim2(), bife); } // handle case when nrow / ncol called on variable with size unknown (dims == -1) @@ -766,10 +774,13 @@ public class StatementBlock extends LiveVariableAnalysis for (Expression expression : expressions) { expression.validateExpression(ids.getVariables(), currConstVars, conditional); if (expression.getOutput().getDataType() != Expression.DataType.SCALAR) { - raiseValidateError("print statement can only print scalars", conditional); + if (expression.getOutput().getDataType() == Expression.DataType.MATRIX) { + pstmt.raiseValidateError("Print statements can only print scalars. To print a matrix, please wrap it in a toString() function.", conditional); + } else { + pstmt.raiseValidateError("Print statements can only print scalars.", conditional); + } } } - } // no work to perform for PathStatement or ImportStatement @@ -813,10 +824,8 @@ public class StatementBlock extends LiveVariableAnalysis } } //case of unspecified format parameter, use default - else - { - s.addExprParam(DataExpression.FORMAT_TYPE, new StringIdentifier(FormatType.TEXT.toString(), - s.getFilename(), s.getBeginLine(), s.getBeginColumn(), s.getEndLine(), s.getEndColumn()), true); + else { + s.addExprParam(DataExpression.FORMAT_TYPE, new StringIdentifier(FormatType.TEXT.toString(), s), true); s.getIdentifier().setFormatType(FormatType.TEXT); } } @@ -850,8 +859,8 @@ public class StatementBlock extends LiveVariableAnalysis + " can only be a string with one of following values: binary, text, mm, csv", conditionalValidate, LanguageErrorCodes.INVALID_PARAMETERS); } } else { - dataExpr.addVarParam(DataExpression.FORMAT_TYPE, new StringIdentifier(FormatType.TEXT.toString(), - dataExpr.getFilename(), dataExpr.getBeginLine(), dataExpr.getBeginColumn(), dataExpr.getEndLine(), dataExpr.getEndColumn())); + dataExpr.addVarParam(DataExpression.FORMAT_TYPE, + new StringIdentifier(FormatType.TEXT.toString(), dataExpr)); s.getTarget().setFormatType(FormatType.TEXT); } } @@ -1001,19 +1010,32 @@ public class StatementBlock extends LiveVariableAnalysis private String _filename = "MAIN SCRIPT"; private int _beginLine = 0, _beginColumn = 0; private int _endLine = 0, _endColumn = 0; + private String _text; public void setFilename (String fname) { _filename = fname; } public void setBeginLine(int passed) { _beginLine = passed; } public void setBeginColumn(int passed) { _beginColumn = passed; } public void setEndLine(int passed) { _endLine = passed; } public void setEndColumn(int passed) { _endColumn = passed; } + public void setText(String text) { _text = text; } - public void setAllPositions(String fname, int blp, int bcp, int elp, int ecp){ - _filename = fname; - _beginLine = blp; - _beginColumn = bcp; - _endLine = elp; - _endColumn = ecp; + /** + * Set parse information. + * + * @param parseInfo + * parse information, such as beginning line position, beginning + * column position, ending line position, ending column position, + * text, and filename + * @param filename + * the DML/PYDML filename (if it exists) + */ + public void setParseInfo(ParseInfo parseInfo) { + _beginLine = parseInfo.getBeginLine(); + _beginColumn = parseInfo.getBeginColumn(); + _endLine = parseInfo.getEndLine(); + _endColumn = parseInfo.getEndColumn(); + _text = parseInfo.getText(); + _filename = parseInfo.getFilename(); } public String getFilename() { return _filename; } @@ -1021,6 +1043,7 @@ public class StatementBlock extends LiveVariableAnalysis public int getBeginColumn() { return _beginColumn; } public int getEndLine() { return _endLine; } public int getEndColumn() { return _endColumn; } + public String getText() { return _text; } public String printErrorLocation(){ return "ERROR: " + _filename + " -- line " + _beginLine + ", column " + _beginColumn + " -- "; http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/StringIdentifier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/StringIdentifier.java b/src/main/java/org/apache/sysml/parser/StringIdentifier.java index 6b8ba1e..ed0c1d6 100644 --- a/src/main/java/org/apache/sysml/parser/StringIdentifier.java +++ b/src/main/java/org/apache/sysml/parser/StringIdentifier.java @@ -19,24 +19,34 @@ package org.apache.sysml.parser; - +import org.antlr.v4.runtime.ParserRuleContext; public class StringIdentifier extends ConstIdentifier { private String _val; - - public Expression rewriteExpression(String prefix) throws LanguageException{ + + public Expression rewriteExpression(String prefix) throws LanguageException { return this; } - - public StringIdentifier(String val, String filename, int blp, int bcp, int elp, int ecp){ + + public StringIdentifier(String val, ParseInfo parseInfo) { super(); - _val = val; - setDimensions(0,0); - computeDataType(); - setValueType(ValueType.STRING); - setAllPositions(filename, blp, bcp, elp, ecp); + setInfo(val); + setParseInfo(parseInfo); + } + + public StringIdentifier(ParserRuleContext ctx, String val, String filename) { + super(); + setInfo(val); + setCtxValuesAndFilename(ctx, filename); + } + + private void setInfo(String val) { + _val = val; + setDimensions(0, 0); + computeDataType(); + setValueType(ValueType.STRING); } public String getValue(){ http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java index 3a7a347..bab9cd0 100644 --- a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java @@ -160,27 +160,11 @@ public abstract class CommonSyntacticValidator { } protected void setFileLineColumn(Expression expr, ParserRuleContext ctx) { - String txt = ctx.getText(); - expr.setFilename(currentFile); - expr.setBeginLine(ctx.start.getLine()); - expr.setBeginColumn(ctx.start.getCharPositionInLine()); - expr.setEndLine(ctx.stop.getLine()); - expr.setEndColumn(ctx.stop.getCharPositionInLine()); - if(expr.getBeginColumn() == expr.getEndColumn() && expr.getBeginLine() == expr.getEndLine() && txt.length() > 1) { - expr.setEndColumn(expr.getBeginColumn() + txt.length() - 1); - } + expr.setCtxValuesAndFilename(ctx, currentFile); } protected void setFileLineColumn(Statement stmt, ParserRuleContext ctx) { - String txt = ctx.getText(); - stmt.setFilename(currentFile); - stmt.setBeginLine(ctx.start.getLine()); - stmt.setBeginColumn(ctx.start.getCharPositionInLine()); - stmt.setEndLine(ctx.stop.getLine()); - stmt.setEndColumn(ctx.stop.getCharPositionInLine()); - if(stmt.getBeginColumn() == stmt.getEndColumn() && stmt.getBeginLine() == stmt.getEndLine() && txt.length() > 1) { - stmt.setEndColumn(stmt.getBeginColumn() + txt.length() - 1); - } + stmt.setCtxValuesAndFilename(ctx, currentFile); } // For String literal "True/TRUE" @@ -234,11 +218,6 @@ public abstract class CommonSyntacticValidator { protected void unaryExpressionHelper(ParserRuleContext ctx, ExpressionInfo left, ExpressionInfo me, String op) { if(left.expr != null) { - Token start = ctx.start; - String fileName = currentFile; - int line = start.getLine(); - int col = start.getCharPositionInLine(); - if(left.expr instanceof IntIdentifier) { if(op.equals("-")) { ((IntIdentifier) left.expr).multiplyByMinusOne(); @@ -252,9 +231,9 @@ public abstract class CommonSyntacticValidator { me.expr = left.expr; } else { - Expression right = new IntIdentifier(1, fileName, line, col, line, col); + Expression right = new IntIdentifier(ctx, 1, currentFile); if(op.equals("-")) { - right = new IntIdentifier(-1, fileName, line, col, line, col); + right = new IntIdentifier(ctx, -1, currentFile); } Expression.BinaryOp bop = Expression.getBinaryOp("*"); @@ -281,12 +260,8 @@ public abstract class CommonSyntacticValidator { protected void constDoubleIdExpressionHelper(ParserRuleContext ctx, ExpressionInfo me) { try { - Token start = ctx.start; double val = Double.parseDouble(ctx.getText()); - int linePosition = start.getLine(); - int charPosition = start.getCharPositionInLine(); - me.expr = new DoubleIdentifier(val, currentFile, linePosition, charPosition, linePosition, charPosition); - setFileLineColumn(me.expr, ctx); + me.expr = new DoubleIdentifier(ctx, val, currentFile); } catch(Exception e) { notifyErrorListeners("cannot parse the float value: \'" + ctx.getText() + "\'", ctx.getStart()); @@ -296,12 +271,8 @@ public abstract class CommonSyntacticValidator { protected void constIntIdExpressionHelper(ParserRuleContext ctx, ExpressionInfo me) { try { - Token start = ctx.start; long val = Long.parseLong(ctx.getText()); - int linePosition = start.getLine(); - int charPosition = start.getCharPositionInLine(); - me.expr = new IntIdentifier(val, currentFile, linePosition, charPosition, linePosition, charPosition); - setFileLineColumn(me.expr, ctx); + me.expr = new IntIdentifier(ctx, val, currentFile); } catch(Exception e) { notifyErrorListeners("cannot parse the int value: \'" + ctx.getText() + "\'", ctx.getStart()); @@ -348,16 +319,11 @@ public abstract class CommonSyntacticValidator { return; } - int linePosition = ctx.start.getLine(); - int charPosition = ctx.start.getCharPositionInLine(); - me.expr = new StringIdentifier(val, currentFile, linePosition, charPosition, linePosition, charPosition); - setFileLineColumn(me.expr, ctx); + me.expr = new StringIdentifier(ctx, val, currentFile); } protected void booleanIdentifierHelper(ParserRuleContext ctx, boolean val, ExpressionInfo info) { - int linePosition = ctx.start.getLine(); - int charPosition = ctx.start.getCharPositionInLine(); - info.expr = new BooleanIdentifier(val, currentFile, linePosition, charPosition, linePosition, charPosition); + info.expr = new BooleanIdentifier(ctx, val, currentFile); setFileLineColumn(info.expr, ctx); } @@ -367,32 +333,25 @@ public abstract class CommonSyntacticValidator { // error occurs, then dataInfo.expr is null which would cause a null pointer exception with the following code. // Therefore, check for null so that parsing can continue so all parsing issues can be determined. if (me.expr != null) { - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - me.expr.setAllPositions(currentFile, line, col, line, col); - setFileLineColumn(me.expr, ctx); + me.expr.setCtxValuesAndFilename(ctx, currentFile); } } - protected ConstIdentifier getConstIdFromString(String varValue, Token start) { - - int linePosition = start.getLine(); - int charPosition = start.getCharPositionInLine(); - + protected ConstIdentifier getConstIdFromString(ParserRuleContext ctx, String varValue) { // Compare to "True/TRUE" if(varValue.equals(trueStringLiteral())) - return new BooleanIdentifier(true, currentFile, linePosition, charPosition, linePosition, charPosition); + return new BooleanIdentifier(ctx, true, currentFile); // Compare to "False/FALSE" if(varValue.equals(falseStringLiteral())) - return new BooleanIdentifier(false, currentFile, linePosition, charPosition, linePosition, charPosition); + return new BooleanIdentifier(ctx, false, currentFile); // Check for long literal // NOTE: we use exception handling instead of Longs.tryParse for backwards compatibility with guava <14.1 // Also the alternative of Ints.tryParse and falling back to double would not be lossless in all cases. try { long lval = Long.parseLong(varValue); - return new IntIdentifier(lval, currentFile, linePosition, charPosition, linePosition, charPosition); + return new IntIdentifier(ctx, lval, currentFile); } catch(Exception ex) { //continue @@ -402,7 +361,7 @@ public abstract class CommonSyntacticValidator { // NOTE: we use exception handling instead of Doubles.tryParse for backwards compatibility with guava <14.0 try { double dval = Double.parseDouble(varValue); - return new DoubleIdentifier(dval, currentFile, linePosition, charPosition, linePosition, charPosition); + return new DoubleIdentifier(ctx, dval, currentFile); } catch(Exception ex) { //continue @@ -420,14 +379,14 @@ public abstract class CommonSyntacticValidator { // the commandline parameters can be passed without any quotes val = extractStringInQuotes(text, false); } - return new StringIdentifier(val, currentFile, linePosition, charPosition, linePosition, charPosition); + return new StringIdentifier(ctx, val, currentFile); } - protected void fillExpressionInfoCommandLineParameters(String varName, ExpressionInfo dataInfo, Token start) { + protected void fillExpressionInfoCommandLineParameters(ParserRuleContext ctx, String varName, ExpressionInfo dataInfo) { if(!varName.startsWith("$")) { - notifyErrorListeners("commandline param doesnot start with $", start); + notifyErrorListeners("commandline param does not start with $", ctx.start); return; } @@ -435,7 +394,7 @@ public abstract class CommonSyntacticValidator { for(Map.Entry<String, String> arg : this.argVals.entrySet()) { if(arg.getKey().equals(varName)) { if(varValue != null) { - notifyErrorListeners("multiple values passed for the parameter " + varName + " via commandline", start); + notifyErrorListeners("multiple values passed for the parameter " + varName + " via commandline", ctx.start); return; } else { @@ -453,7 +412,7 @@ public abstract class CommonSyntacticValidator { if(varValue.equals("")) return; - dataInfo.expr = getConstIdFromString(varValue, start); + dataInfo.expr = getConstIdFromString(ctx, varValue); } protected void exitAssignmentStatementHelper(ParserRuleContext ctx, String lhs, ExpressionInfo dataInfo, @@ -467,12 +426,8 @@ public abstract class CommonSyntacticValidator { if(dataInfo.expr instanceof DataIdentifier) { target = (DataIdentifier) dataInfo.expr; Expression source = rhs.expr; - - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); try { - info.stmt = new AssignmentStatement(target, source, line, col, line, col); - setFileLineColumn(info.stmt, ctx); + info.stmt = new AssignmentStatement(ctx, target, source, currentFile); } catch (LanguageException e) { // TODO: extract more meaningful info from this exception. notifyErrorListeners("invalid assignment", lhsStart); @@ -494,11 +449,7 @@ public abstract class CommonSyntacticValidator { ArrayList<ParameterExpression> paramExpression, StatementInfo thisinfo) { if (DMLScript.VALIDATOR_IGNORE_ISSUES == true) { // create dummy print statement try { - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - ArrayList<Expression> expList = new ArrayList<Expression>(); - thisinfo.stmt = new PrintStatement(functionName, expList, line, col, line, col); - setFileLineColumn(thisinfo.stmt, ctx); + thisinfo.stmt = new PrintStatement(ctx, functionName, currentFile); } catch (LanguageException e) { e.printStackTrace(); } @@ -515,12 +466,9 @@ public abstract class CommonSyntacticValidator { return; } try { - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - ArrayList<Expression> expList = new ArrayList<Expression>(); + List<Expression> expList = new ArrayList<Expression>(); expList.add(expr); - thisinfo.stmt = new PrintStatement(functionName, expList, line, col, line, col); - setFileLineColumn(thisinfo.stmt, ctx); + thisinfo.stmt = new PrintStatement(ctx, functionName, expList, currentFile); } catch (LanguageException e) { notifyErrorListeners("cannot process " + functionName + "() function", ctx.start); return; @@ -541,16 +489,12 @@ public abstract class CommonSyntacticValidator { return; } try { - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - List<Expression> expressions = new ArrayList<Expression>(); for (ParameterExpression pe : paramExpression) { Expression expression = pe.getExpr(); expressions.add(expression); } - thisinfo.stmt = new PrintStatement(functionName, expressions, line, col, line, col); - setFileLineColumn(thisinfo.stmt, ctx); + thisinfo.stmt = new PrintStatement(ctx, functionName, expressions, currentFile); } catch (LanguageException e) { notifyErrorListeners("cannot process " + functionName + "() function", ctx.start); return; @@ -565,9 +509,6 @@ public abstract class CommonSyntacticValidator { return; } if(paramExpression.get(0).getExpr() instanceof DataIdentifier) { - String fileName = currentFile; - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); HashMap<String, Expression> varParams = new HashMap<String, Expression>(); varParams.put(DataExpression.IO_FILENAME, paramExpression.get(1).getExpr()); for(int i = 2; i < paramExpression.size(); i++) { @@ -575,9 +516,9 @@ public abstract class CommonSyntacticValidator { varParams.put(paramExpression.get(i).getName(), paramExpression.get(i).getExpr()); } - DataExpression dataExpression = new DataExpression(DataOp.WRITE, varParams, fileName, line, col, line, col); - info.stmt = new OutputStatement((DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, fileName, line, col, line, col); - setFileLineColumn(info.stmt, ctx); + DataExpression dataExpression = new DataExpression(ctx, DataOp.WRITE, varParams, currentFile); + info.stmt = new OutputStatement(ctx, (DataIdentifier) paramExpression.get(0).getExpr(), DataOp.WRITE, + currentFile); ((OutputStatement)info.stmt).setExprParams(dataExpression); } else { @@ -587,8 +528,7 @@ public abstract class CommonSyntacticValidator { protected void setAssignmentStatement(ParserRuleContext ctx, StatementInfo info, DataIdentifier target, Expression expression) { try { - info.stmt = new AssignmentStatement(target, expression, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine()); - setFileLineColumn(info.stmt, ctx); + info.stmt = new AssignmentStatement(ctx, target, expression, currentFile); } catch (LanguageException e) { // TODO: extract more meaningful info from this exception. notifyErrorListeners("invalid function call", ctx.start); @@ -651,9 +591,6 @@ public abstract class CommonSyntacticValidator { protected boolean buildForBuiltInFunction(ParserRuleContext ctx, String functionName, ArrayList<ParameterExpression> paramExpressions, Action f) { // In global namespace, so it can be a builtin function // Double verification: verify passed function name is a (non-parameterized) built-in function. - String fileName = currentFile; - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); try { if (functions.contains(functionName)) { // It is a user function definition (which takes precedence if name same as built-in) @@ -667,14 +604,15 @@ public abstract class CommonSyntacticValidator { return true; } - BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(functionName, paramExpressions, fileName, line, col, line, col); + BuiltinFunctionExpression bife = BuiltinFunctionExpression.getBuiltinFunctionExpression(ctx, functionName, paramExpressions, currentFile); if (bife != null){ // It is a builtin function f.execute(bife); return true; } - ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression.getParamBuiltinFunctionExpression(functionName, paramExpressions, fileName, line, col, line, col); + ParameterizedBuiltinFunctionExpression pbife = ParameterizedBuiltinFunctionExpression + .getParamBuiltinFunctionExpression(ctx, functionName, paramExpressions, currentFile); if (pbife != null){ // It is a parameterized builtin function f.execute(pbife); @@ -682,7 +620,7 @@ public abstract class CommonSyntacticValidator { } // built-in read, rand ... - DataExpression dbife = DataExpression.getDataExpression(functionName, paramExpressions, fileName, line, col, line, col, errorListener); + DataExpression dbife = DataExpression.getDataExpression(ctx, functionName, paramExpressions, currentFile, errorListener); if (dbife != null){ f.execute(dbife); return true; @@ -748,7 +686,7 @@ public abstract class CommonSyntacticValidator { String inferNamespace = (sourceNamespace != null && sourceNamespace.length() > 0 && DMLProgram.DEFAULT_NAMESPACE.equals(namespace)) ? sourceNamespace : namespace; functCall.setFunctionNamespace(inferNamespace); - functCall.setAllPositions(currentFile, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.stop.getLine(), ctx.stop.getCharPositionInLine()); + functCall.setCtxValuesAndFilename(ctx, currentFile); setAssignmentStatement(ctx, info, target, functCall); } @@ -763,8 +701,7 @@ public abstract class CommonSyntacticValidator { protected void setMultiAssignmentStatement(ArrayList<DataIdentifier> target, Expression expression, ParserRuleContext ctx, StatementInfo info) { info.stmt = new MultiAssignmentStatement(target, expression); - info.stmt.setAllPositions(currentFile, ctx.start.getLine(), ctx.start.getCharPositionInLine(), ctx.start.getLine(), ctx.start.getCharPositionInLine()); - setFileLineColumn(info.stmt, ctx); + info.stmt.setCtxValuesAndFilename(ctx, currentFile); } // ----------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java b/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java index 90d0265..6a27acf 100644 --- a/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java +++ b/src/main/java/org/apache/sysml/parser/common/CustomErrorListener.java @@ -29,6 +29,7 @@ import org.antlr.v4.runtime.Recognizer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysml.api.DMLScript; +import org.apache.sysml.parser.ParseInfo; public class CustomErrorListener extends BaseErrorListener { @@ -85,6 +86,25 @@ public class CustomErrorListener extends BaseErrorListener { } } + public void validationError(ParseInfo parseInfo, String msg) { + parseIssues.add(new ParseIssue(parseInfo.getBeginLine(), parseInfo.getBeginColumn(), msg, currentFileName, + ParseIssueType.VALIDATION_ERROR)); + + try { + setAtLeastOneError(true); + if (currentFileName == null) { + log.error("line " + parseInfo.getBeginLine() + ":" + parseInfo.getBeginColumn() + " (" + + parseInfo.getText() + "): " + msg); + } else { + String fileName = currentFileName; + log.error(fileName + " line " + parseInfo.getBeginLine() + ":" + parseInfo.getBeginColumn() + " (" + + parseInfo.getText() + "): " + msg); + } + } catch (Exception e) { + log.error("ERROR: while customizing error message:" + e); + } + } + /** * Validation warning occurred. Add the warning to the list of parse issues. * http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java index bcedc97..62b499f 100644 --- a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java @@ -354,14 +354,14 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D private void handleCommandlineArgumentExpression(DataIdentifierContext ctx) { String varName = ctx.getText().trim(); - fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start); + fillExpressionInfoCommandLineParameters(ctx, varName, ctx.dataInfo); if(ctx.dataInfo.expr == null) { if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) { String msg = "The parameter " + varName + " either needs to be passed " + "through commandline or initialized to default value."; if( ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_UNSPECIFIED_ARGS) ) { - ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start); + ctx.dataInfo.expr = getConstIdFromString(ctx, " "); if (!ConfigurationManager.getCompilerConfigFlag(ConfigType.MLCONTEXT)) { raiseWarning(msg, ctx.start); } @@ -607,10 +607,7 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D IfStatement ifStmt = new IfStatement(); ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr); ifStmt.setConditionalPredicate(predicate); - String fileName = currentFile; - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - ifStmt.setAllPositions(fileName, line, col, line, col); + ifStmt.setCtxValuesAndFilename(ctx, currentFile); if(ctx.ifBody.size() > 0) { for(StatementContext stmtCtx : ctx.ifBody) { @@ -635,9 +632,7 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D WhileStatement whileStmt = new WhileStatement(); ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr); whileStmt.setPredicate(predicate); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - whileStmt.setAllPositions(currentFile, line, col, line, col); + whileStmt.setCtxValuesAndFilename(ctx, currentFile); if(ctx.body.size() > 0) { for(StatementContext stmtCtx : ctx.body) { @@ -653,8 +648,6 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D @Override public void exitForStatement(ForStatementContext ctx) { ForStatement forStmt = new ForStatement(); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText()); HashMap<String, String> parForParamValues = null; @@ -662,7 +655,8 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D if(ctx.iterPred.info.increment != null) { incrementExpr = ctx.iterPred.info.increment; } - IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, currentFile, line, col, line, col); + IterablePredicate predicate = new IterablePredicate(ctx, iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, + incrementExpr, parForParamValues, currentFile); forStmt.setPredicate(predicate); if(ctx.body.size() > 0) { @@ -672,14 +666,11 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D forStmt.mergeStatementBlocks(); } ctx.info.stmt = forStmt; - setFileLineColumn(ctx.info.stmt, ctx); } @Override public void exitParForStatement(ParForStatementContext ctx) { ParForStatement parForStmt = new ParForStatement(); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText()); HashMap<String, String> parForParamValues = new HashMap<String, String>(); @@ -696,7 +687,8 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D if( ctx.iterPred.info.increment != null ) { incrementExpr = ctx.iterPred.info.increment; } - IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, currentFile, line, col, line, col); + IterablePredicate predicate = new IterablePredicate(ctx, iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, + incrementExpr, parForParamValues, currentFile); parForStmt.setPredicate(predicate); if(ctx.body.size() > 0) { for(StatementContext stmtCtx : ctx.body) { @@ -705,7 +697,6 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D parForStmt.mergeStatementBlocks(); } ctx.info.stmt = parForStmt; - setFileLineColumn(ctx.info.stmt, ctx); } private ArrayList<DataIdentifier> getFunctionParameters(List<TypedArgNoAssignContext> ctx) { @@ -914,11 +905,8 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D source = ctx.source.info.expr; } - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); try { - ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col); - setFileLineColumn(ctx.info.stmt, ctx); + ctx.info.stmt = new AssignmentStatement(ctx, target, source, currentFile); } catch (LanguageException e) { notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.start); return; http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java index 4c9b416..8b75309 100644 --- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java @@ -358,21 +358,14 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements // Add expression for nrow(X) for implicit upper bound Expression.BuiltinFunctionOp bop = Expression.BuiltinFunctionOp.NROW; DataIdentifier x = new DataIdentifier(ctx.name.getText()); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - Expression expr = new BuiltinFunctionExpression(bop, new Expression[]{x}, - currentFile, line, col, line, col); - setFileLineColumn(expr, ctx); + Expression expr = new BuiltinFunctionExpression(ctx, bop, new Expression[] { x }, currentFile); rowIndices.add(expr); } } else if(!isRowLower && isRowUpper && isRowSliceImplicit) { // Add expression for `1` for implicit lower bound // Note: We go ahead and increment by 1 to convert from 0-based to 1-based indexing - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - IntIdentifier one = new IntIdentifier(1, currentFile, line, col, line, col); - setFileLineColumn(one, ctx); + IntIdentifier one = new IntIdentifier(ctx, 1, currentFile); rowIndices.add(one); // Add given upper bound @@ -398,21 +391,14 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements // Add expression for ncol(X) for implicit upper bound Expression.BuiltinFunctionOp bop = Expression.BuiltinFunctionOp.NCOL; DataIdentifier x = new DataIdentifier(ctx.name.getText()); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - Expression expr = new BuiltinFunctionExpression(bop, new Expression[]{x}, - currentFile, line, col, line, col); - setFileLineColumn(expr, ctx); + Expression expr = new BuiltinFunctionExpression(ctx, bop, new Expression[] { x }, currentFile); colIndices.add(expr); } } else if(!isColLower && isColUpper && isColSliceImplicit) { // Add expression for `1` for implicit lower bound // Note: We go ahead and increment by 1 to convert from 0-based to 1-based indexing - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - IntIdentifier one = new IntIdentifier(1, currentFile, line, col, line, col); - setFileLineColumn(one, ctx); + IntIdentifier one = new IntIdentifier(ctx, 1, currentFile); colIndices.add(one); // Add given upper bound @@ -445,9 +431,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements Expression.BinaryOp bop = Expression.getBinaryOp("+"); Expression retVal = new BinaryExpression(bop); ((BinaryExpression)retVal).setLeft(expr); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - ((BinaryExpression)retVal).setRight(new DoubleIdentifier(1.0, currentFile, line, col, line, col)); + ((BinaryExpression)retVal).setRight(new DoubleIdentifier(ctx, 1.0, currentFile)); setFileLineColumn(retVal, ctx); return retVal; } @@ -470,14 +454,14 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements private void handleCommandlineArgumentExpression(DataIdentifierContext ctx) { String varName = ctx.getText().trim(); - fillExpressionInfoCommandLineParameters(varName, ctx.dataInfo, ctx.start); + fillExpressionInfoCommandLineParameters(ctx, varName, ctx.dataInfo); if(ctx.dataInfo.expr == null) { if(!(ctx.parent instanceof IfdefAssignmentStatementContext)) { String msg = "The parameter " + varName + " either needs to be passed " + "through commandline or initialized to default value."; if( ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_UNSPECIFIED_ARGS) ) { - ctx.dataInfo.expr = getConstIdFromString(" ", ctx.start); + ctx.dataInfo.expr = getConstIdFromString(ctx, " "); if (!ConfigurationManager.getCompilerConfigFlag(ConfigType.MLCONTEXT)) { raiseWarning(msg, ctx.start); } @@ -664,10 +648,6 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements return new ConvertedDMLSyntax(namespace, functionName, paramExpression); } - String fileName = currentFile; - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("len")) { if(paramExpression.size() != 1) { notifyErrorListeners("The builtin function \'" + functionName + "\' accepts 1 arguments", fnName); @@ -812,7 +792,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements paramExpression.get(0).setName("rows"); paramExpression.get(1).setName("cols"); paramExpression.get(2).setName("sparsity"); - paramExpression.add(new ParameterExpression("pdf", new StringIdentifier("normal", fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression("pdf", new StringIdentifier(ctx, "normal", currentFile))); functionName = "rand"; namespace = DMLProgram.DEFAULT_NAMESPACE; } @@ -826,7 +806,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements paramExpression.get(1).setName("cols"); paramExpression.get(2).setName("sparsity"); paramExpression.get(3).setName("lambda"); - paramExpression.add(new ParameterExpression("pdf", new StringIdentifier("poisson", fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression("pdf", new StringIdentifier(ctx, "poisson", currentFile))); functionName = "rand"; namespace = DMLProgram.DEFAULT_NAMESPACE; } @@ -841,7 +821,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements paramExpression.get(2).setName("sparsity"); paramExpression.get(3).setName("min"); paramExpression.get(4).setName("max"); - paramExpression.add(new ParameterExpression("pdf", new StringIdentifier("uniform", fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression("pdf", new StringIdentifier(ctx, "uniform", currentFile))); functionName = "rand"; namespace = DMLProgram.DEFAULT_NAMESPACE; } @@ -885,9 +865,9 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements initializerString = initializerString.replaceAll("\\[", ""); initializerString = initializerString.replaceAll("\\]", ""); paramExpression = new ArrayList<ParameterExpression>(); - paramExpression.add(new ParameterExpression(null, new StringIdentifier(initializerString, fileName, line, col, line, col))); - paramExpression.add(new ParameterExpression("rows", new IntIdentifier(rows, fileName, line, col, line, col))); - paramExpression.add(new ParameterExpression("cols", new IntIdentifier(cols, fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression(null, new StringIdentifier(ctx, initializerString, currentFile))); + paramExpression.add(new ParameterExpression("rows", new IntIdentifier(ctx, rows, currentFile))); + paramExpression.add(new ParameterExpression("cols", new IntIdentifier(ctx, cols, currentFile))); } else { functionName = "as.matrix"; @@ -955,10 +935,10 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements } StringIdentifier marginVal = null; if(axis == 0) { - marginVal = new StringIdentifier("rows", fileName, line, col, line, col); + marginVal = new StringIdentifier(ctx, "rows", currentFile); } else { - marginVal = new StringIdentifier("cols", fileName, line, col, line, col); + marginVal = new StringIdentifier(ctx, "cols", currentFile); } paramExpression.get(0).setName("target"); paramExpression.get(1).setName("margin"); @@ -997,7 +977,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements paramExpression.get(0).setName("target"); paramExpression.get(1).setName("mean"); paramExpression.get(2).setName("sd"); - paramExpression.add(new ParameterExpression("dist", new StringIdentifier("normal", fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression("dist", new StringIdentifier(ctx, "normal", currentFile))); namespace = DMLProgram.DEFAULT_NAMESPACE; } else if(namespace.equals("expon") && functionName.equals("cdf")) { @@ -1009,7 +989,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements functionName = "cdf"; paramExpression.get(0).setName("target"); paramExpression.get(1).setName("mean"); - paramExpression.add(new ParameterExpression("dist", new StringIdentifier("exp", fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression("dist", new StringIdentifier(ctx, "exp", currentFile))); namespace = DMLProgram.DEFAULT_NAMESPACE; } else if(namespace.equals("chi") && functionName.equals("cdf")) { @@ -1021,7 +1001,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements functionName = "cdf"; paramExpression.get(0).setName("target"); paramExpression.get(1).setName("df"); - paramExpression.add(new ParameterExpression("dist", new StringIdentifier("chisq", fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression("dist", new StringIdentifier(ctx, "chisq", currentFile))); namespace = DMLProgram.DEFAULT_NAMESPACE; } else if(namespace.equals("f") && functionName.equals("cdf")) { @@ -1034,7 +1014,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements paramExpression.get(0).setName("target"); paramExpression.get(1).setName("df1"); paramExpression.get(2).setName("df2"); - paramExpression.add(new ParameterExpression("dist", new StringIdentifier("f", fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression("dist", new StringIdentifier(ctx, "f", currentFile))); namespace = DMLProgram.DEFAULT_NAMESPACE; } else if(namespace.equals("t") && functionName.equals("cdf")) { @@ -1046,7 +1026,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements functionName = "cdf"; paramExpression.get(0).setName("target"); paramExpression.get(1).setName("df"); - paramExpression.add(new ParameterExpression("dist", new StringIdentifier("t", fileName, line, col, line, col))); + paramExpression.add(new ParameterExpression("dist", new StringIdentifier(ctx, "t", currentFile))); namespace = DMLProgram.DEFAULT_NAMESPACE; } else if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("percentile")) { @@ -1245,10 +1225,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements IfStatement ifStmt = new IfStatement(); ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr); ifStmt.setConditionalPredicate(predicate); - String fileName = currentFile; - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - ifStmt.setAllPositions(fileName, line, col, line, col); + ifStmt.setCtxValuesAndFilename(ctx, currentFile); if(ctx.ifBody.size() > 0) { for(StatementContext stmtCtx : ctx.ifBody) { @@ -1282,10 +1259,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements IfStatement elifStmt = new IfStatement(); ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr); elifStmt.setConditionalPredicate(predicate); - String fileName = currentFile; - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - elifStmt.setAllPositions(fileName, line, col, line, col); + elifStmt.setCtxValuesAndFilename(ctx, currentFile); if(ctx.elifBody.size() > 0) { for (StatementContext stmtCtx : ctx.elifBody) { @@ -1303,9 +1277,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements WhileStatement whileStmt = new WhileStatement(); ConditionalPredicate predicate = new ConditionalPredicate(ctx.predicate.info.expr); whileStmt.setPredicate(predicate); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); - whileStmt.setAllPositions(currentFile, line, col, line, col); + whileStmt.setCtxValuesAndFilename(ctx, currentFile); if(ctx.body.size() > 0) { for(StatementContext stmtCtx : ctx.body) { @@ -1321,8 +1293,6 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements @Override public void exitForStatement(ForStatementContext ctx) { ForStatement forStmt = new ForStatement(); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText()); HashMap<String, String> parForParamValues = null; @@ -1330,7 +1300,8 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements if(ctx.iterPred.info.increment != null) { incrementExpr = ctx.iterPred.info.increment; } - IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, currentFile, line, col, line, col); + IterablePredicate predicate = new IterablePredicate(ctx, iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, + incrementExpr, parForParamValues, currentFile); forStmt.setPredicate(predicate); if(ctx.body.size() > 0) { @@ -1346,8 +1317,6 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements @Override public void exitParForStatement(ParForStatementContext ctx) { ParForStatement parForStmt = new ParForStatement(); - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); DataIdentifier iterVar = new DataIdentifier(ctx.iterVar.getText()); HashMap<String, String> parForParamValues = new HashMap<String, String>(); @@ -1364,7 +1333,8 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements if( ctx.iterPred.info.increment != null ) { incrementExpr = ctx.iterPred.info.increment; } - IterablePredicate predicate = new IterablePredicate(iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, incrementExpr, parForParamValues, currentFile, line, col, line, col); + IterablePredicate predicate = new IterablePredicate(ctx, iterVar, ctx.iterPred.info.from, ctx.iterPred.info.to, + incrementExpr, parForParamValues, currentFile); parForStmt.setPredicate(predicate); if(ctx.body.size() > 0) { for(StatementContext stmtCtx : ctx.body) { @@ -1373,7 +1343,6 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements parForStmt.mergeStatementBlocks(); } ctx.info.stmt = parForStmt; - setFileLineColumn(ctx.info.stmt, ctx); } @@ -1576,11 +1545,8 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements source = ctx.source.info.expr; } - int line = ctx.start.getLine(); - int col = ctx.start.getCharPositionInLine(); try { - ctx.info.stmt = new AssignmentStatement(target, source, line, col, line, col); - setFileLineColumn(ctx.info.stmt, ctx); + ctx.info.stmt = new AssignmentStatement(ctx, target, source, currentFile); } catch (LanguageException e) { notifyErrorListeners("invalid assignment for ifdef function", ctx.targetList.start); return; @@ -1621,7 +1587,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements // Introduce empty StatementInfo // This is later ignored by PyDMLParserWrapper try { - ctx.info.stmt = new AssignmentStatement(null, null, 0, 0, 0, 0); + ctx.info.stmt = new AssignmentStatement(ctx, null, null); ctx.info.stmt.setEmptyNewLineStatement(true); } catch (LanguageException e) { e.printStackTrace(); http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java index 43e93f4..f1b3d5d 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java @@ -30,6 +30,7 @@ import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.hops.recompile.Recompiler; import org.apache.sysml.lops.Lop; import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.parser.ParseInfo; import org.apache.sysml.parser.StatementBlock; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.DMLScriptException; @@ -50,7 +51,7 @@ import org.apache.sysml.utils.Statistics; import org.apache.sysml.yarn.DMLAppMasterUtils; -public class ProgramBlock +public class ProgramBlock implements ParseInfo { protected static final Log LOG = LogFactory.getLog(ProgramBlock.class.getName()); private static final boolean CHECK_MATRIX_SPARSITY = false; @@ -412,28 +413,43 @@ public class ProgramBlock public String _filename; public int _beginLine, _beginColumn; public int _endLine, _endColumn; + public String _text; public void setFilename(String passed) { _filename = passed; } public void setBeginLine(int passed) { _beginLine = passed; } public void setBeginColumn(int passed) { _beginColumn = passed; } public void setEndLine(int passed) { _endLine = passed; } public void setEndColumn(int passed) { _endColumn = passed; } - - public void setAllPositions(String filename, int blp, int bcp, int elp, int ecp){ - _filename = filename; - _beginLine = blp; - _beginColumn = bcp; - _endLine = elp; - _endColumn = ecp; - } + public void setText(String text) { _text = text; } public String getFilename() { return _filename; } public int getBeginLine() { return _beginLine; } public int getBeginColumn() { return _beginColumn; } public int getEndLine() { return _endLine; } public int getEndColumn() { return _endColumn; } + public String getText() { return _text; } public String printBlockErrorLocation(){ return "ERROR: Runtime error in program block generated from statement block between lines " + _beginLine + " and " + _endLine + " -- "; } + + /** + * Set parse information. + * + * @param parseInfo + * parse information, such as beginning line position, beginning + * column position, ending line position, ending column position, + * text, and filename + * @param filename + * the DML/PYDML filename (if it exists) + */ + public void setParseInfo(ParseInfo parseInfo) { + _beginLine = parseInfo.getBeginLine(); + _beginColumn = parseInfo.getBeginColumn(); + _endLine = parseInfo.getEndLine(); + _endColumn = parseInfo.getEndColumn(); + _text = parseInfo.getText(); + _filename = parseInfo.getFilename(); + } + } http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java index d786eaa..afdf8be 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java @@ -551,7 +551,7 @@ public class ProgramConverter //create new statement (shallow copy livein/liveout for recompile, line numbers for explain) ret = new StatementBlock(); ret.setDMLProg(sb.getDMLProg()); - ret.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + ret.setParseInfo(sb); ret.setLiveIn( sb.liveIn() ); ret.setLiveOut( sb.liveOut() ); ret.setUpdatedVariables( sb.variablesUpdated() ); @@ -591,7 +591,7 @@ public class ProgramConverter //create new statement (shallow copy livein/liveout for recompile, line numbers for explain) ret = new IfStatementBlock(); ret.setDMLProg(sb.getDMLProg()); - ret.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + ret.setParseInfo(sb); ret.setLiveIn( sb.liveIn() ); ret.setLiveOut( sb.liveOut() ); ret.setUpdatedVariables( sb.variablesUpdated() ); @@ -632,7 +632,7 @@ public class ProgramConverter //create new statement (shallow copy livein/liveout for recompile, line numbers for explain) ret = new WhileStatementBlock(); ret.setDMLProg(sb.getDMLProg()); - ret.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + ret.setParseInfo(sb); ret.setLiveIn( sb.liveIn() ); ret.setLiveOut( sb.liveOut() ); ret.setUpdatedVariables( sb.variablesUpdated() ); @@ -678,7 +678,7 @@ public class ProgramConverter //create new statement (shallow copy livein/liveout for recompile, line numbers for explain) ret.setDMLProg(sb.getDMLProg()); - ret.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + ret.setParseInfo(sb); ret.setLiveIn( sb.liveIn() ); ret.setLiveOut( sb.liveOut() ); ret.setUpdatedVariables( sb.variablesUpdated() ); http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java index 59f2423..2c09850 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/OuterTableExpandTest.java @@ -256,7 +256,7 @@ public class OuterTableExpandTest extends AutomatedTestBase //check compiled/executed jobs if( rtplatform == RUNTIME_PLATFORM.HADOOP ) { - int expectedNumCompiled = 3; //reblock+gmr+gmr if rexpand; otherwise 3/5 + int expectedNumCompiled = 2; //reblock+gmr if rexpand; otherwise 3/5 int expectedNumExecuted = expectedNumCompiled; checkNumCompiledMRJobs(expectedNumCompiled); checkNumExecutedMRJobs(expectedNumExecuted);
