[SYSTEMML-1858] Allow parsed text in validation errors Add parsed text from original script into objects such as Statements and Expressions so that it is available in validation errors. Improve several validation error messages.
Closes #635. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/c495533b Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/c495533b Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/c495533b Branch: refs/heads/master Commit: c495533b6570c27450b34623523c12acb1f86575 Parents: f1d35b7 Author: Deron Eriksson <[email protected]> Authored: Wed Aug 23 16:30:37 2017 -0700 Committer: Deron Eriksson <[email protected]> Committed: Thu Aug 24 16:36:17 2017 -0700 ---------------------------------------------------------------------- .../apache/sysml/api/mlcontext/MLContext.java | 36 +- src/main/java/org/apache/sysml/hops/Hop.java | 35 +- .../sysml/hops/rewrite/HopRewriteUtils.java | 8 +- .../RewriteInjectSparkLoopCheckpointing.java | 2 +- .../RewriteSplitDagDataDependentOperators.java | 2 +- .../rewrite/RewriteSplitDagUnknownCSVRead.java | 2 +- .../sysml/parser/AssignmentStatement.java | 47 ++- .../apache/sysml/parser/BinaryExpression.java | 55 ++- .../apache/sysml/parser/BooleanExpression.java | 43 +- .../apache/sysml/parser/BooleanIdentifier.java | 36 +- .../sysml/parser/BuiltinFunctionExpression.java | 100 ++--- .../sysml/parser/ConditionalPredicate.java | 5 +- .../org/apache/sysml/parser/DMLTranslator.java | 155 ++++--- .../org/apache/sysml/parser/DataExpression.java | 412 ++++++++----------- .../org/apache/sysml/parser/DataIdentifier.java | 6 +- .../apache/sysml/parser/DoubleIdentifier.java | 48 ++- .../org/apache/sysml/parser/Expression.java | 158 ++++--- .../apache/sysml/parser/ForStatementBlock.java | 41 +- .../sysml/parser/FunctionCallIdentifier.java | 6 +- .../sysml/parser/FunctionStatementBlock.java | 5 +- .../apache/sysml/parser/IndexedIdentifier.java | 72 +--- .../org/apache/sysml/parser/IntIdentifier.java | 48 ++- .../apache/sysml/parser/IterablePredicate.java | 22 +- .../sysml/parser/MultiAssignmentStatement.java | 5 +- .../apache/sysml/parser/OutputStatement.java | 24 +- .../ParameterizedBuiltinFunctionExpression.java | 36 +- .../java/org/apache/sysml/parser/ParseInfo.java | 143 +++++++ .../org/apache/sysml/parser/PrintStatement.java | 33 +- .../sysml/parser/RelationalExpression.java | 58 ++- .../java/org/apache/sysml/parser/Statement.java | 146 +++++-- .../org/apache/sysml/parser/StatementBlock.java | 95 +++-- .../apache/sysml/parser/StringIdentifier.java | 30 +- .../parser/common/CommonSyntacticValidator.java | 131 ++---- .../parser/common/CustomErrorListener.java | 20 + .../sysml/parser/dml/DmlSyntacticValidator.java | 30 +- .../parser/pydml/PydmlSyntacticValidator.java | 92 ++--- .../runtime/controlprogram/ProgramBlock.java | 34 +- .../controlprogram/parfor/ProgramConverter.java | 8 +- .../functions/misc/OuterTableExpandTest.java | 2 +- 39 files changed, 1181 insertions(+), 1050 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java index f74d593..83eedb3 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java @@ -534,39 +534,27 @@ public class MLContext { if (datatype.compareToIgnoreCase("frame") != 0) { MatrixObject mo = getMatrixObject(target); if (mo != null) { - int blp = source.getBeginLine(); - int bcp = source.getBeginColumn(); - int elp = source.getEndLine(); - int ecp = source.getEndColumn(); - exp.addVarParam(DataExpression.READROWPARAM, - new IntIdentifier(mo.getNumRows(), source.getFilename(), blp, bcp, elp, ecp)); - exp.addVarParam(DataExpression.READCOLPARAM, - new IntIdentifier(mo.getNumColumns(), source.getFilename(), blp, bcp, elp, ecp)); - exp.addVarParam(DataExpression.READNUMNONZEROPARAM, - new IntIdentifier(mo.getNnz(), source.getFilename(), blp, bcp, elp, ecp)); - exp.addVarParam(DataExpression.DATATYPEPARAM, - new StringIdentifier("matrix", source.getFilename(), blp, bcp, elp, ecp)); - exp.addVarParam(DataExpression.VALUETYPEPARAM, - new StringIdentifier("double", source.getFilename(), blp, bcp, elp, ecp)); + exp.addVarParam(DataExpression.READROWPARAM, new IntIdentifier(mo.getNumRows(), source)); + exp.addVarParam(DataExpression.READCOLPARAM, new IntIdentifier(mo.getNumColumns(), source)); + exp.addVarParam(DataExpression.READNUMNONZEROPARAM, new IntIdentifier(mo.getNnz(), source)); + exp.addVarParam(DataExpression.DATATYPEPARAM, new StringIdentifier("matrix", source)); + exp.addVarParam(DataExpression.VALUETYPEPARAM, new StringIdentifier("double", source)); if (mo.getMetaData() instanceof MatrixFormatMetaData) { MatrixFormatMetaData metaData = (MatrixFormatMetaData) mo.getMetaData(); if (metaData.getOutputInfo() == OutputInfo.CSVOutputInfo) { exp.addVarParam(DataExpression.FORMAT_TYPE, - new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_CSV, source.getFilename(), - blp, bcp, elp, ecp)); + new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_CSV, source)); } else if (metaData.getOutputInfo() == OutputInfo.TextCellOutputInfo) { exp.addVarParam(DataExpression.FORMAT_TYPE, - new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_TEXT, - source.getFilename(), blp, bcp, elp, ecp)); + new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_TEXT, source)); } else if (metaData.getOutputInfo() == OutputInfo.BinaryBlockOutputInfo) { - exp.addVarParam(DataExpression.ROWBLOCKCOUNTPARAM, new IntIdentifier( - mo.getNumRowsPerBlock(), source.getFilename(), blp, bcp, elp, ecp)); - exp.addVarParam(DataExpression.COLUMNBLOCKCOUNTPARAM, new IntIdentifier( - mo.getNumColumnsPerBlock(), source.getFilename(), blp, bcp, elp, ecp)); + exp.addVarParam(DataExpression.ROWBLOCKCOUNTPARAM, + new IntIdentifier(mo.getNumRowsPerBlock(), source)); + exp.addVarParam(DataExpression.COLUMNBLOCKCOUNTPARAM, + new IntIdentifier(mo.getNumColumnsPerBlock(), source)); exp.addVarParam(DataExpression.FORMAT_TYPE, - new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_BINARY, - source.getFilename(), blp, bcp, elp, ecp)); + new StringIdentifier(DataExpression.FORMAT_TYPE_VALUE_BINARY, source)); } else { throw new MLContextException("Unsupported format through MLContext"); } http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 2d9b1bf..b48e227 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -42,6 +42,7 @@ import org.apache.sysml.lops.ReBlock; import org.apache.sysml.lops.UnaryCP; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.parser.ParseInfo; import org.apache.sysml.runtime.controlprogram.LocalVariableMap; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; @@ -52,7 +53,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.util.UtilFunctions; -public abstract class Hop +public abstract class Hop implements ParseInfo { protected static final Log LOG = LogFactory.getLog(Hop.class.getName()); @@ -1820,26 +1821,21 @@ public abstract class Hop public int _beginLine, _beginColumn; public int _endLine, _endColumn; public String _filename; + public String _text; public void setBeginLine(int passed) { _beginLine = passed; } public void setBeginColumn(int passed) { _beginColumn = passed; } public void setEndLine(int passed) { _endLine = passed; } public void setEndColumn(int passed) { _endColumn = passed; } public void setFilename(String passed) { _filename = passed; } - - public void setAllPositions(String filename, int blp, int bcp, int elp, int ecp){ - _filename = filename; - _beginLine = blp; - _beginColumn = bcp; - _endLine = elp; - _endColumn = ecp; - } + public void setText(String text) { _text = text; } public int getBeginLine() { return _beginLine; } public int getBeginColumn() { return _beginColumn; } public int getEndLine() { return _endLine; } public int getEndColumn() { return _endColumn; } public String getFilename() { return _filename; } + public String getText() { return _text; } public String printErrorLocation(){ if(_filename != null) @@ -1857,5 +1853,24 @@ public abstract class Hop { lop.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); } - + + /** + * Set parse information. + * + * @param parseInfo + * parse information, such as beginning line position, beginning + * column position, ending line position, ending column position, + * text, and filename + * @param filename + * the DML/PYDML filename (if it exists) + */ + public void setParseInfo(ParseInfo parseInfo) { + _beginLine = parseInfo.getBeginLine(); + _beginColumn = parseInfo.getBeginColumn(); + _endLine = parseInfo.getEndLine(); + _endColumn = parseInfo.getEndColumn(); + _text = parseInfo.getText(); + _filename = parseInfo.getFilename(); + } + } // end class http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java index 5a3b9aa..914e0cb 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java @@ -656,11 +656,11 @@ public class HopRewriteUtils hnew.setOutputBlocksizes(hold.getRowsInBlock(), hold.getColsInBlock()); hnew.refreshSizeInformation(); } - - public static void copyLineNumbers( Hop src, Hop dest ) { - dest.setAllPositions(src.getFilename(), src.getBeginLine(), src.getBeginColumn(), src.getEndLine(), src.getEndColumn()); + + public static void copyLineNumbers(Hop src, Hop dest) { + dest.setParseInfo(src); } - + public static void updateHopCharacteristics( Hop hop, long brlen, long bclen, Hop src ) { updateHopCharacteristics(hop, brlen, bclen, new MemoTable(), src); http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java index 1c59e6e..876b162 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteInjectSparkLoopCheckpointing.java @@ -89,7 +89,7 @@ public class RewriteInjectSparkLoopCheckpointing extends StatementBlockRewriteRu { StatementBlock sb0 = new StatementBlock(); sb0.setDMLProg(sb.getDMLProg()); - sb0.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + sb0.setParseInfo(sb); ArrayList<Hop> hops = new ArrayList<Hop>(); VariableSet livein = new VariableSet(); VariableSet liveout = new VariableSet(); http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java index 351cf87..1f49500 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java @@ -97,7 +97,7 @@ public class RewriteSplitDagDataDependentOperators extends StatementBlockRewrite //duplicate sb incl live variable sets StatementBlock sb1 = new StatementBlock(); sb1.setDMLProg(sb.getDMLProg()); - sb1.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + sb1.setParseInfo(sb); sb1.setLiveIn(new VariableSet()); sb1.setLiveOut(new VariableSet()); http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java index 6d9942f..ef8d9dc 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java @@ -67,7 +67,7 @@ public class RewriteSplitDagUnknownCSVRead extends StatementBlockRewriteRule //duplicate sb incl live variable sets StatementBlock sb1 = new StatementBlock(); sb1.setDMLProg(sb.getDMLProg()); - sb1.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + sb1.setParseInfo(sb); sb1.setLiveIn(new VariableSet()); sb1.setLiveOut(new VariableSet()); http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/AssignmentStatement.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/AssignmentStatement.java b/src/main/java/org/apache/sysml/parser/AssignmentStatement.java index b59887b..2df4ed0 100644 --- a/src/main/java/org/apache/sysml/parser/AssignmentStatement.java +++ b/src/main/java/org/apache/sysml/parser/AssignmentStatement.java @@ -21,6 +21,7 @@ package org.apache.sysml.parser; import java.util.ArrayList; +import org.antlr.v4.runtime.ParserRuleContext; import org.apache.sysml.api.DMLScript; import org.apache.sysml.debug.DMLBreakpointManager; @@ -32,35 +33,39 @@ public class AssignmentStatement extends Statement private Expression _source; // rewrites statement to support function inlining (creates deep copy) - public Statement rewriteStatement(String prefix) throws LanguageException{ - + public Statement rewriteStatement(String prefix) throws LanguageException { + // rewrite target (deep copy) - DataIdentifier newTarget = (DataIdentifier)_targetList.get(0).rewriteExpression(prefix); - + DataIdentifier newTarget = (DataIdentifier) _targetList.get(0).rewriteExpression(prefix); + // rewrite source (deep copy) Expression newSource = _source.rewriteExpression(prefix); - + // create rewritten assignment statement (deep copy) - AssignmentStatement retVal = new AssignmentStatement(newTarget, newSource,this.getBeginLine(), - this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); - + AssignmentStatement retVal = new AssignmentStatement(newTarget, newSource, this); + return retVal; } - - public AssignmentStatement(DataIdentifier t, Expression s, int beginLine, int beginCol, int endLine, int endCol) - throws LanguageException - { + + public AssignmentStatement(DataIdentifier di, Expression exp, ParseInfo parseInfo) { _targetList = new ArrayList<DataIdentifier>(); - _targetList.add(t); - _source = s; - - setBeginLine(beginLine); - setBeginColumn(beginCol); - setEndLine(endLine); - setEndColumn(endCol); - + _targetList.add(di); + _source = exp; + setParseInfo(parseInfo); } - + + public AssignmentStatement(ParserRuleContext ctx, DataIdentifier di, Expression exp) throws LanguageException { + _targetList = new ArrayList<DataIdentifier>(); + _targetList.add(di); + _source = exp; + setCtxValues(ctx); + } + + public AssignmentStatement(ParserRuleContext ctx, DataIdentifier di, Expression exp, String filename) throws LanguageException { + this(ctx, di, exp); + setFilename(filename); + } + public DataIdentifier getTarget(){ return _targetList.get(0); } http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/BinaryExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/BinaryExpression.java b/src/main/java/org/apache/sysml/parser/BinaryExpression.java index 121e488..68678e7 100644 --- a/src/main/java/org/apache/sysml/parser/BinaryExpression.java +++ b/src/main/java/org/apache/sysml/parser/BinaryExpression.java @@ -29,8 +29,7 @@ public class BinaryExpression extends Expression private BinaryOp _opcode; public Expression rewriteExpression(String prefix) throws LanguageException{ - BinaryExpression newExpr = new BinaryExpression(this._opcode, - this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); + BinaryExpression newExpr = new BinaryExpression(this._opcode, this); newExpr.setLeft(_left.rewriteExpression(prefix)); newExpr.setRight(_right.rewriteExpression(prefix)); return newExpr; @@ -44,18 +43,13 @@ public class BinaryExpression extends Expression setBeginColumn(0); setEndLine(0); setEndColumn(0); + setText(null); } - - public BinaryExpression(BinaryOp bop, String filename, int beginLine, int beginColumn, int endLine, int endColumn) { + + public BinaryExpression(BinaryOp bop, ParseInfo parseInfo) { _opcode = bop; - - setFilename(filename); - setBeginLine(beginLine); - setBeginColumn(beginColumn); - setEndLine(endLine); - setEndColumn(endColumn); + setParseInfo(parseInfo); } - public BinaryOp getOpCode() { return _opcode; @@ -66,9 +60,7 @@ public class BinaryExpression extends Expression // update script location information --> left expression is BEFORE in script if (_left != null){ - setFilename(_left.getFilename()); - setBeginLine(_left.getBeginLine()); - setBeginColumn(_left.getBeginColumn()); + setParseInfo(_left); } } @@ -78,9 +70,7 @@ public class BinaryExpression extends Expression // update script location information --> right expression is AFTER in script if (_right != null){ - setFilename(_right.getFilename()); - setBeginLine(_right.getEndLine()); - setBeginColumn(_right.getEndColumn()); + setParseInfo(_right); } } @@ -103,11 +93,11 @@ public class BinaryExpression extends Expression throws LanguageException { //recursive validate - if (_left instanceof FunctionCallIdentifier || _right instanceof FunctionCallIdentifier){ - raiseValidateError("user-defined function calls not supported in binary expressions", - false, LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION); + if (_left instanceof FunctionCallIdentifier || _right instanceof FunctionCallIdentifier) { + raiseValidateError("User-defined function calls not supported in binary expressions.", false, + LanguageException.LanguageErrorCodes.UNSUPPORTED_EXPRESSION); } - + _left.validateExpression(ids, constVars, conditional); _right.validateExpression(ids, constVars, conditional); @@ -120,7 +110,7 @@ public class BinaryExpression extends Expression String outputName = getTempName(); DataIdentifier output = new DataIdentifier(outputName); - output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); + output.setParseInfo(this); output.setDataType(computeDataType(this.getLeft(), this.getRight(), true)); ValueType resultVT = computeValueType(this.getLeft(), this.getRight(), true); @@ -171,16 +161,17 @@ public class BinaryExpression extends Expression pivot = right; } - if ((pivot != null) && (aux != null)) - { - //check dimensions binary operations (if dims known) - if (isSameDimensionBinaryOp(this.getOpCode()) && pivot.dimsKnown() && aux.dimsKnown() ) - { - if( (pivot.getDim1() != aux.getDim1() && aux.getDim1()>1) //number of rows must always be equivalent if not row vector - || (pivot.getDim2() != aux.getDim2() && aux.getDim2()>1)) //number of cols must be equivalent if not col vector - { - raiseValidateError("Mismatch in dimensions for operation "+ this.toString(), conditional, LanguageException.LanguageErrorCodes.INVALID_PARAMETERS); - } + if ((pivot != null) && (aux != null)) { + // check dimensions binary operations (if dims known) + if (isSameDimensionBinaryOp(this.getOpCode()) && pivot.dimsKnown() && aux.dimsKnown()) { + // number of rows must always be equivalent if not row vector + // number of cols must be equivalent if not col vector + if ((pivot.getDim1() != aux.getDim1() && aux.getDim1() > 1) + || (pivot.getDim2() != aux.getDim2() && aux.getDim2() > 1)) { + raiseValidateError("Mismatch in dimensions for operation '" + this.getText() + "'. " + pivot + + " is " + pivot.getDim1() + "x" + pivot.getDim2() + " and " + aux + " is " + aux.getDim1() + + "x" + aux.getDim2() + ".", conditional); + } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/BooleanExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/BooleanExpression.java b/src/main/java/org/apache/sysml/parser/BooleanExpression.java index 2996da5..2069c06 100644 --- a/src/main/java/org/apache/sysml/parser/BooleanExpression.java +++ b/src/main/java/org/apache/sysml/parser/BooleanExpression.java @@ -37,18 +37,14 @@ public class BooleanExpression extends Expression setBeginColumn(0); setEndLine(0); setEndColumn(0); + setText(null); } - - public BooleanExpression(BooleanOp bop, String filename, int beginLine, int beginColumn, int endLine, int endColumn){ + + public BooleanExpression(BooleanOp bop, ParseInfo parseInfo) { _opcode = bop; - - setFilename(filename); - setBeginLine(beginLine); - setBeginColumn(beginColumn); - setEndLine(endLine); - setEndColumn(endColumn); + setParseInfo(parseInfo); } - + public BooleanOp getOpCode(){ return _opcode; } @@ -58,9 +54,7 @@ public class BooleanExpression extends Expression // update script location information --> left expression is BEFORE in script if (_left != null){ - this.setFilename(_left.getFilename()); - this.setBeginLine(_left.getBeginLine()); - this.setBeginColumn(_left.getBeginColumn()); + this.setParseInfo(_left); } } @@ -69,9 +63,7 @@ public class BooleanExpression extends Expression // update script location information --> right expression is AFTER in script if (_right != null){ - this.setFilename(_right.getFilename()); - this.setBeginLine(_right.getBeginLine()); - this.setBeginColumn(_right.getBeginColumn()); + this.setParseInfo(_right); } } @@ -84,9 +76,7 @@ public class BooleanExpression extends Expression } public Expression rewriteExpression(String prefix) throws LanguageException{ - - - BooleanExpression newExpr = new BooleanExpression(this._opcode, this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); + BooleanExpression newExpr = new BooleanExpression(this._opcode, this); newExpr.setLeft(_left.rewriteExpression(prefix)); if (_right != null) newExpr.setRight(_right.rewriteExpression(prefix)); @@ -118,17 +108,20 @@ public class BooleanExpression extends Expression } String outputName = getTempName(); DataIdentifier output = new DataIdentifier(outputName); - output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); + output.setParseInfo(this); output.setBooleanProperties(); this.setOutput(output); - if ((_opcode == Expression.BooleanOp.CONDITIONALAND) || - (_opcode == Expression.BooleanOp.CONDITIONALOR)) - { //always unconditional (because unsupported operation) - raiseValidateError("Unsupported boolean operation " + _opcode.toString(), false, LanguageException.LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + if ((_opcode == Expression.BooleanOp.CONDITIONALAND) || (_opcode == Expression.BooleanOp.CONDITIONALOR)) { + // always unconditional (because unsupported operation) + if (_opcode == Expression.BooleanOp.CONDITIONALAND) { + raiseValidateError("Conditional AND (&&) not supported.", false); + } else if (_opcode == Expression.BooleanOp.CONDITIONALOR) { + raiseValidateError("Conditional OR (||) not supported.", false); + } } - } - + } + public String toString(){ if (_opcode == BooleanOp.NOT) { return "(" + _opcode.toString() + " " + _left.toString() + ")"; http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java b/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java index 501926c..6ca2b9f 100644 --- a/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java +++ b/src/main/java/org/apache/sysml/parser/BooleanIdentifier.java @@ -19,21 +19,39 @@ package org.apache.sysml.parser; - +import org.antlr.v4.runtime.ParserRuleContext; public class BooleanIdentifier extends ConstIdentifier { private boolean _val; - - public BooleanIdentifier(boolean val, String filename, int blp, int bcp, int elp, int ecp){ + + public BooleanIdentifier(boolean val) { super(); - _val = val; - setDimensions(0,0); - computeDataType(); - setValueType(ValueType.BOOLEAN); - setAllPositions(filename, blp, bcp, elp, ecp); + setInfo(val); + setBeginLine(-1); + setBeginColumn(-1); + setEndLine(-1); + setEndColumn(-1); + setText(null); } - + + public BooleanIdentifier(boolean val, ParseInfo parseInfo) { + this(val); + setParseInfo(parseInfo); + } + + public BooleanIdentifier(ParserRuleContext ctx, boolean val, String filename) { + this(val); + setCtxValuesAndFilename(ctx, filename); + } + + private void setInfo(boolean val) { + _val = val; + setDimensions(0, 0); + computeDataType(); + setValueType(ValueType.BOOLEAN); + } + public Expression rewriteExpression(String prefix) throws LanguageException{ return this; } http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java index 5d3ae93..e03add3 100644 --- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import org.antlr.v4.runtime.ParserRuleContext; import org.apache.sysml.parser.LanguageException.LanguageErrorCodes; import org.apache.sysml.runtime.util.ConvolutionUtils; import org.apache.sysml.runtime.util.UtilFunctions; @@ -32,9 +33,9 @@ public class BuiltinFunctionExpression extends DataIdentifier protected Expression[] _args = null; private BuiltinFunctionOp _opcode; - public BuiltinFunctionExpression(BuiltinFunctionOp bifop, ArrayList<ParameterExpression> args, String fname, int blp, int bcp, int elp, int ecp) { + public BuiltinFunctionExpression(ParserRuleContext ctx, BuiltinFunctionOp bifop, ArrayList<ParameterExpression> args, String fname) { _opcode = bifop; - setAllPositions(fname, blp, bcp, elp, ecp); + setCtxValuesAndFilename(ctx, fname); args = expandConvolutionArguments(args); _args = new Expression[args.size()]; for(int i=0; i < args.size(); i++) { @@ -42,25 +43,31 @@ public class BuiltinFunctionExpression extends DataIdentifier } } - public BuiltinFunctionExpression(BuiltinFunctionOp bifop, Expression[] args, String fname, int blp, int bcp, int elp, int ecp) { + public BuiltinFunctionExpression(BuiltinFunctionOp bifop, Expression[] args, ParseInfo parseInfo) { + _opcode = bifop; + _args = new Expression[args.length]; + for (int i = 0; i < args.length; i++) { + _args[i] = args[i]; + } + setParseInfo(parseInfo); + } + + public BuiltinFunctionExpression(ParserRuleContext ctx, BuiltinFunctionOp bifop, Expression[] args, String fname) { _opcode = bifop; _args = new Expression[args.length]; for(int i=0; i < args.length; i++) { _args[i] = args[i]; } - this.setAllPositions(fname, blp, bcp, elp, ecp); + setCtxValuesAndFilename(ctx, fname); } public Expression rewriteExpression(String prefix) throws LanguageException { - Expression[] newArgs = new Expression[_args.length]; - for(int i=0; i < _args.length; i++) { + for (int i = 0; i < _args.length; i++) { newArgs[i] = _args[i].rewriteExpression(prefix); } - BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(this._opcode, newArgs, - this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); + BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(this._opcode, newArgs, this); return retVal; - } public BuiltinFunctionOp getOpCode() { @@ -108,7 +115,7 @@ public class BuiltinFunctionExpression extends DataIdentifier int count = 0; for (DataIdentifier outParam: stmt.getTargetList()){ DataIdentifier tmp = new DataIdentifier(outParam); - tmp.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); + tmp.setParseInfo(this); _outputs[count++] = tmp; } @@ -149,11 +156,12 @@ public class BuiltinFunctionExpression extends DataIdentifier long inrows = getFirstExpr().getOutput().getDim1(); long incols = getFirstExpr().getOutput().getDim2(); - - if ( inrows != incols ) { - raiseValidateError("LU Decomposition can only be done on a square matrix. Input matrix is rectangular (rows=" + inrows + ", cols="+incols+")", conditional); + + if (inrows != incols) { + raiseValidateError("LU Decomposition requires a square matrix. Matrix " + getFirstExpr() + " is " + + inrows + "x" + incols + ".", conditional); } - + // Output1 - P luOut1.setDataType(DataType.MATRIX); luOut1.setValueType(ValueType.DOUBLE); @@ -318,10 +326,8 @@ public class BuiltinFunctionExpression extends DataIdentifier HashSet<String> expand = new HashSet<String>(); expand.add("input_shape"); expand.add("pool_size"); expand.add("stride"); expand.add("padding"); paramExpression = expandListParams(paramExpression, expand); - paramExpression.add(new ParameterExpression("filter_shape1", - new IntIdentifier(1, getFilename(), getBeginLine(), getBeginColumn(), getEndLine(), getEndColumn()))); - paramExpression.add(new ParameterExpression("filter_shape2", - new IntIdentifier(1, getFilename(), getBeginLine(), getBeginColumn(), getEndLine(), getEndColumn()))); + paramExpression.add(new ParameterExpression("filter_shape1", new IntIdentifier(1, this))); + paramExpression.add(new ParameterExpression("filter_shape2", new IntIdentifier(1, this))); paramExpression = replaceListParams(paramExpression, "pool_size", "filter_shape", 3); if(_opcode == BuiltinFunctionOp.MAX_POOL_BACKWARD) paramExpression = orderConvolutionParams(paramExpression, 2); @@ -357,7 +363,7 @@ public class BuiltinFunctionExpression extends DataIdentifier // checkIdentifierParams(); String outputName = getTempName(); DataIdentifier output = new DataIdentifier(outputName); - output.setAllPositions(this.getFilename(), this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); + output.setParseInfo(this); if (this.getFirstExpr() == null) { raiseValidateError("Function " + this + " has no arguments.", false); @@ -680,11 +686,12 @@ public class BuiltinFunctionExpression extends DataIdentifier // First input: is always of type MATRIX checkMatrixParam(getFirstExpr()); - - if ( getSecondExpr() == null ) - raiseValidateError("Invalid number of arguments to table(): " - + this.toString(), conditional, LanguageErrorCodes.INVALID_PARAMETERS); - + + if (getSecondExpr() == null) + raiseValidateError( + "Invalid number of arguments to table(). The table() function requires 2, 3, 4, or 5 arguments.", + conditional); + // Second input: can be MATRIX or SCALAR // cases: table(A,B) or table(A,1) if ( getSecondExpr().getOutput().getDataType() == DataType.MATRIX) @@ -1021,9 +1028,7 @@ public class BuiltinFunctionExpression extends DataIdentifier // default value: 1 if from <= to; -1 if from > to if(getThirdExpr() == null) { expandArguments(); - _args[2] = new DoubleIdentifier(((from > to) ? -1.0 : 1.0), - this.getFilename(), this.getBeginLine(), this.getBeginColumn(), - this.getEndLine(), this.getEndColumn()); + _args[2] = new DoubleIdentifier(((from > to) ? -1.0 : 1.0), this); } incr = getDoubleValue(getThirdExpr()); @@ -1400,28 +1405,30 @@ public class BuiltinFunctionExpression extends DataIdentifier protected void checkNumParameters(int count) //always unconditional throws LanguageException { - if (getFirstExpr() == null){ - raiseValidateError("Missing parameter for function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS); + if (getFirstExpr() == null) { + raiseValidateError("Missing argument for function " + this.getOpCode(), false, + LanguageErrorCodes.INVALID_PARAMETERS); + } + + if (((count == 1) && (getSecondExpr() != null || getThirdExpr() != null)) + || ((count == 2) && (getThirdExpr() != null))) { + raiseValidateError("Invalid number of arguments for function " + this.getOpCode().toString().toLowerCase() + + "(). This function only takes 1 or 2 arguments.", false); + } else if (((count == 2) && (getSecondExpr() == null)) + || ((count == 3) && (getSecondExpr() == null || getThirdExpr() == null))) { + raiseValidateError("Missing argument for function " + this.getOpCode(), false, + LanguageErrorCodes.INVALID_PARAMETERS); } - - if (((count == 1) && (getSecondExpr()!= null || getThirdExpr() != null)) || - ((count == 2) && (getThirdExpr() != null))){ - raiseValidateError("Invalid number of parameters for function "+ this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS); - } - else if (((count == 2) && (getSecondExpr() == null)) || - ((count == 3) && (getSecondExpr() == null || getThirdExpr() == null))){ - raiseValidateError( "Missing parameter for function "+this.getOpCode(), false, LanguageErrorCodes.INVALID_PARAMETERS); - } } - protected void checkMatrixParam(Expression e) //always unconditional - throws LanguageException - { + protected void checkMatrixParam(Expression e) throws LanguageException { if (e.getOutput().getDataType() != DataType.MATRIX) { - raiseValidateError("Expecting matrix parameter for function "+ this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + raiseValidateError( + "Expecting matrix argument for function " + this.getOpCode().toString().toLowerCase() + "().", + false); } } - + protected void checkMatrixFrameParam(Expression e) //always unconditional throws LanguageException { @@ -1520,9 +1527,9 @@ public class BuiltinFunctionExpression extends DataIdentifier } } - public static BuiltinFunctionExpression getBuiltinFunctionExpression( + public static BuiltinFunctionExpression getBuiltinFunctionExpression(ParserRuleContext ctx, String functionName, ArrayList<ParameterExpression> paramExprsPassed, - String filename, int blp, int bcp, int elp, int ecp) { + String filename) { if (functionName == null || paramExprsPassed == null) return null; @@ -1708,8 +1715,7 @@ public class BuiltinFunctionExpression extends DataIdentifier else return null; - BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(bifop, paramExprsPassed, - filename, blp, bcp, elp, ecp); + BuiltinFunctionExpression retVal = new BuiltinFunctionExpression(ctx, bifop, paramExprsPassed, filename); return retVal; } // end method getBuiltinFunctionExpression http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java index 25f1fa9..1253391 100644 --- a/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java +++ b/src/main/java/org/apache/sysml/parser/ConditionalPredicate.java @@ -19,7 +19,7 @@ package org.apache.sysml.parser; -public class ConditionalPredicate +public class ConditionalPredicate implements ParseInfo { @@ -61,17 +61,20 @@ public class ConditionalPredicate private String _filename; private int _beginLine, _beginColumn; private int _endLine, _endColumn; + private String _text; public void setFilename(String fname) { _filename = fname; } public void setBeginLine(int passed) { _beginLine = passed; } public void setBeginColumn(int passed) { _beginColumn = passed; } public void setEndLine(int passed) { _endLine = passed; } public void setEndColumn(int passed) { _endColumn = passed; } + public void setText(String text) { _text = text; } public String getFilename() { return _filename; } public int getBeginLine() { return _beginLine; } public int getBeginColumn() { return _beginColumn; } public int getEndLine() { return _endLine; } public int getEndColumn() { return _endColumn; } + public String getText() { return _text; } } http://git-wip-us.apache.org/repos/asf/systemml/blob/c495533b/src/main/java/org/apache/sysml/parser/DMLTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java index 28202dc..1dbdfdc 100644 --- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java @@ -548,7 +548,7 @@ public class DMLTranslator retPB.setStatementBlock(sb); // add location information - retPB.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + retPB.setParseInfo(sb); } // process If Statement - add runtime program blocks to program @@ -615,7 +615,7 @@ public class DMLTranslator retPB.setStatementBlock(sb); // add location information - retPB.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + retPB.setParseInfo(sb); } // process For Statement - add runtime program blocks to program @@ -693,7 +693,7 @@ public class DMLTranslator retPB.setStatementBlock(sb); // add location information - retPB.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + retPB.setParseInfo(sb); } // process function statement block - add runtime program blocks to program @@ -775,7 +775,7 @@ public class DMLTranslator retPB = rtpb; // add location information - retPB.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + retPB.setParseInfo(sb); } else { @@ -806,7 +806,7 @@ public class DMLTranslator retPB.setStatementBlock(sb); // add location information - retPB.setAllPositions(sb.getFilename(), sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn()); + retPB.setParseInfo(sb); } return retPB; @@ -1270,7 +1270,7 @@ public class DMLTranslator long actualDim1 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim1() : var.getDim1(); long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2(); DataOp read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock()); - read.setAllPositions(var.getFilename(), var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn()); + read.setParseInfo(var); ids.put(varName, read); } } @@ -1326,8 +1326,7 @@ public class DMLTranslator DataIdentifier target = createTarget(); target.setDataType(DataType.SCALAR); target.setValueType(ValueType.STRING); - target.setAllPositions(current.getFilename(), current.getBeginLine(), target.getBeginColumn(), - current.getEndLine(), current.getEndColumn()); + target.setParseInfo(current); PrintStatement ps = (PrintStatement) current; PRINTTYPE ptype = ps.getType(); @@ -1338,16 +1337,14 @@ public class DMLTranslator Expression source = ps.getExpressions().get(0); Hop ae = processExpression(source, target, ids); Hop printHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae); - printHop.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), - current.getEndColumn()); + printHop.setParseInfo(current); output.add(printHop); } else if (ptype == PRINTTYPE.STOP) { Hop.OpOp1 op = Hop.OpOp1.STOP; Expression source = ps.getExpressions().get(0); Hop ae = processExpression(source, target, ids); Hop stopHop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), op, ae); - stopHop.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), - current.getEndColumn()); + stopHop.setParseInfo(current); output.add(stopHop); sb.setSplitDag(true); //avoid merge } else if (ptype == PRINTTYPE.PRINTF) { @@ -1392,7 +1389,7 @@ public class DMLTranslator if ((statementId != null) && (statementId.intValue() == i)) { DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null); transientwrite.setOutputParams(ae.getDim1(), ae.getDim2(), ae.getNnz(), ae.getUpdateType(), ae.getRowsInBlock(), ae.getColsInBlock()); - transientwrite.setAllPositions(target.getFilename(), target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndLine()); + transientwrite.setParseInfo(target); updatedLiveOut.addVariable(target.getName(), target); output.add(transientwrite); } @@ -1423,7 +1420,7 @@ public class DMLTranslator if ((statementId != null) && (statementId.intValue() == i)) { DataOp transientwrite = new DataOp(target.getName(), target.getDataType(), target.getValueType(), ae, DataOpTypes.TRANSIENTWRITE, null); transientwrite.setOutputParams(origDim1, origDim2, ae.getNnz(), ae.getUpdateType(), ae.getRowsInBlock(), ae.getColsInBlock()); - transientwrite.setAllPositions(target.getFilename(), target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn()); + transientwrite.setParseInfo(target); updatedLiveOut.addVariable(target.getName(), target); output.add(transientwrite); } @@ -1633,7 +1630,7 @@ public class DMLTranslator read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock()); - read.setAllPositions(var.getFilename(), var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn()); + read.setParseInfo(var); } _ids.put(varName, read); } @@ -1641,7 +1638,7 @@ public class DMLTranslator DataIdentifier target = new DataIdentifier(Expression.getTempName()); target.setDataType(DataType.SCALAR); target.setValueType(ValueType.BOOLEAN); - target.setAllPositions(passedSB.getFilename(), passedSB.getBeginLine(), passedSB.getBeginColumn(), passedSB.getEndLine(), passedSB.getEndColumn()); + target.setParseInfo(passedSB); Hop predicateHops = null; Expression predicate = cp.getPredicate(); @@ -1656,30 +1653,21 @@ public class DMLTranslator // handle constant identifier // a) translate 0 --> FALSE; translate 1 --> TRUE // b) disallow string values - if ( (predicate instanceof IntIdentifier && ((IntIdentifier)predicate).getValue() == 0) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier)predicate).getValue() == 0.0)) { - cp.setPredicate(new BooleanIdentifier(false, - predicate.getFilename(), - predicate.getBeginLine(), predicate.getBeginColumn(), - predicate.getEndLine(), predicate.getEndColumn())); - - } - else if ( (predicate instanceof IntIdentifier && ((IntIdentifier)predicate).getValue() == 1) || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier)predicate).getValue() == 1.0)) { - cp.setPredicate(new BooleanIdentifier(true, - predicate.getFilename(), - predicate.getBeginLine(), predicate.getBeginColumn(), - predicate.getEndLine(), predicate.getEndColumn())); - } - else if (predicate instanceof IntIdentifier || predicate instanceof DoubleIdentifier){ - cp.setPredicate(new BooleanIdentifier(true, - predicate.getFilename(), - predicate.getBeginLine(), predicate.getBeginColumn(), - predicate.getEndLine(), predicate.getEndColumn())); - LOG.warn(predicate.printWarningLocation() + "Numerical value '" + predicate.toString() + "' (!= 0/1) is converted to boolean TRUE by DML"); - } - else if (predicate instanceof StringIdentifier) { - LOG.error(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate"); - throw new ParseException(predicate.printErrorLocation() + "String value '" + predicate.toString() + "' is not allowed for iterable predicate"); - + if ((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 0) + || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 0.0)) { + cp.setPredicate(new BooleanIdentifier(false, predicate)); + } else if ((predicate instanceof IntIdentifier && ((IntIdentifier) predicate).getValue() == 1) + || (predicate instanceof DoubleIdentifier && ((DoubleIdentifier) predicate).getValue() == 1.0)) { + cp.setPredicate(new BooleanIdentifier(true, predicate)); + } else if (predicate instanceof IntIdentifier || predicate instanceof DoubleIdentifier) { + cp.setPredicate(new BooleanIdentifier(true, predicate)); + LOG.warn(predicate.printWarningLocation() + "Numerical value '" + predicate.toString() + + "' (!= 0/1) is converted to boolean TRUE by DML"); + } else if (predicate instanceof StringIdentifier) { + LOG.error(predicate.printErrorLocation() + "String value '" + predicate.toString() + + "' is not allowed for iterable predicate"); + throw new ParseException(predicate.printErrorLocation() + "String value '" + predicate.toString() + + "' is not allowed for iterable predicate"); } predicateHops = processExpression(cp.getPredicate(), null, _ids); } @@ -1731,7 +1719,7 @@ public class DMLTranslator long actualDim2 = (var instanceof IndexedIdentifier) ? ((IndexedIdentifier)var).getOrigDim2() : var.getDim2(); read = new DataOp(var.getName(), var.getDataType(), var.getValueType(), DataOpTypes.TRANSIENTREAD, null, actualDim1, actualDim2, var.getNnz(), var.getRowsInBlock(), var.getColumnsInBlock()); - read.setAllPositions(var.getFilename(), var.getBeginLine(), var.getBeginColumn(), var.getEndLine(), var.getEndColumn()); + read.setParseInfo(var); } _ids.put(varName, read); } @@ -1809,28 +1797,28 @@ public class DMLTranslator else if (source instanceof IntIdentifier) { IntIdentifier sourceInt = (IntIdentifier) source; LiteralOp litop = new LiteralOp(sourceInt.getValue()); - litop.setAllPositions(sourceInt.getFilename(), sourceInt.getBeginLine(), sourceInt.getBeginColumn(), sourceInt.getEndLine(), sourceInt.getEndColumn()); + litop.setParseInfo(sourceInt); setIdentifierParams(litop, sourceInt); return litop; } else if (source instanceof DoubleIdentifier) { DoubleIdentifier sourceDouble = (DoubleIdentifier) source; LiteralOp litop = new LiteralOp(sourceDouble.getValue()); - litop.setAllPositions(source.getFilename(), sourceDouble.getBeginLine(), sourceDouble.getBeginColumn(), sourceDouble.getEndLine(), sourceDouble.getEndColumn()); + litop.setParseInfo(sourceDouble); setIdentifierParams(litop, sourceDouble); return litop; } else if (source instanceof BooleanIdentifier) { BooleanIdentifier sourceBoolean = (BooleanIdentifier) source; LiteralOp litop = new LiteralOp(sourceBoolean.getValue()); - litop.setAllPositions(sourceBoolean.getFilename(), sourceBoolean.getBeginLine(), sourceBoolean.getBeginColumn(), sourceBoolean.getEndLine(), sourceBoolean.getEndColumn()); + litop.setParseInfo(sourceBoolean); setIdentifierParams(litop, sourceBoolean); return litop; } else if (source instanceof StringIdentifier) { StringIdentifier sourceString = (StringIdentifier) source; LiteralOp litop = new LiteralOp(sourceString.getValue()); - litop.setAllPositions(sourceString.getFilename(), sourceString.getBeginLine(), sourceString.getBeginColumn(), sourceString.getEndLine(), sourceString.getEndColumn()); + litop.setParseInfo(sourceString); setIdentifierParams(litop, sourceString); return litop; } @@ -1897,7 +1885,7 @@ public class DMLTranslator rowUpperHops = new LiteralOp(target.getOrigDim1()); else { rowUpperHops = new UnaryOp(target.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NROW, hops.get(target.getName())); - rowUpperHops.setAllPositions(target.getFilename(), target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn()); + rowUpperHops.setParseInfo(target); } } if (target.getColLowerBound() != null) @@ -1934,7 +1922,7 @@ public class DMLTranslator setIdentifierParams(leftIndexOp, target); - leftIndexOp.setAllPositions(target.getFilename(), target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn()); + leftIndexOp.setParseInfo(target); leftIndexOp.setDim1(target.getOrigDim1()); leftIndexOp.setDim2(target.getOrigDim2()); @@ -1961,7 +1949,7 @@ public class DMLTranslator rowUpperHops = new LiteralOp(source.getOrigDim1()); else { rowUpperHops = new UnaryOp(source.getName(), DataType.SCALAR, ValueType.INT, Hop.OpOp1.NROW, hops.get(source.getName())); - rowUpperHops.setAllPositions(source.getFilename(), source.getBeginLine(),source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + rowUpperHops.setParseInfo(source); } } if (source.getColLowerBound() != null) @@ -1990,7 +1978,7 @@ public class DMLTranslator hops.get(source.getName()), rowLowerHops, rowUpperHops, colLowerHops, colUpperHops, source.getRowLowerEqualsUpper(), source.getColLowerEqualsUpper()); - indexOp.setAllPositions(target.getFilename(), target.getBeginLine(), target.getBeginColumn(), target.getEndLine(), target.getEndColumn()); + indexOp.setParseInfo(target); setIdentifierParams(indexOp, target); return indexOp; @@ -2048,7 +2036,7 @@ public class DMLTranslator throw new ParseException("Unsupported parsing of binary expression: "+source.getOpCode()); } setIdentifierParams(currBop, source.getOutput()); - currBop.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + currBop.setParseInfo(source); return currBop; } @@ -2092,7 +2080,7 @@ public class DMLTranslator op = OpOp2.NOTEQUAL; } currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), op, left, right); - currBop.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + currBop.setParseInfo(source); return currBop; } @@ -2126,7 +2114,7 @@ public class DMLTranslator if (source.getRight() == null) { Hop currUop = new UnaryOp(target.getName(), target.getDataType(), target.getValueType(), Hop.OpOp1.NOT, left); - currUop.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + currUop.setParseInfo(source); return currUop; } else { @@ -2142,7 +2130,7 @@ public class DMLTranslator throw new RuntimeException(source.printErrorLocation() + "Unknown boolean operation " + source.getOpCode()); } currBop = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), op, left, right); - currBop.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + currBop.setParseInfo(source); // setIdentifierParams(currBop,source.getOutput()); return currBop; } @@ -2221,9 +2209,9 @@ public class DMLTranslator // set properties for created hops based on outputs of source expression for ( int i=0; i < source.getOutputs().length; i++ ) { setIdentifierParams( outputs.get(i), source.getOutputs()[i]); - outputs.get(i).setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + outputs.get(i).setParseInfo(source); } - currBuiltinOp.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + currBuiltinOp.setParseInfo(source); return currBuiltinOp; } @@ -2342,9 +2330,7 @@ public class DMLTranslator } setIdentifierParams(currBuiltinOp, source.getOutput()); - - currBuiltinOp.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); - + currBuiltinOp.setParseInfo(source); return currBuiltinOp; } @@ -2431,7 +2417,7 @@ public class DMLTranslator setIdentifierParams(currBuiltinOp, source.getOutput()); if( source.getOpCode()==DataExpression.DataOp.READ ) ((DataOp)currBuiltinOp).setInputBlockSizes(target.getRowsInBlock(), target.getColumnsInBlock()); - currBuiltinOp.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + currBuiltinOp.setParseInfo(source); return currBuiltinOp; } @@ -2494,9 +2480,9 @@ public class DMLTranslator // set properties for created hops based on outputs of source expression for ( int i=0; i < source.getOutputs().length; i++ ) { setIdentifierParams( outputs.get(i), source.getOutputs()[i]); - outputs.get(i).setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + outputs.get(i).setParseInfo(source); } - currBuiltinOp.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + currBuiltinOp.setParseInfo(source); return currBuiltinOp; } @@ -3169,7 +3155,7 @@ public class DMLTranslator // Since the dimension of output doesnot match that of input variable for these operations setIdentifierParams(currBuiltinOp, source.getOutput()); } - currBuiltinOp.setAllPositions(source.getFilename(), source.getBeginLine(), source.getBeginColumn(), source.getEndLine(), source.getEndColumn()); + currBuiltinOp.setParseInfo(source); return currBuiltinOp; } @@ -3230,20 +3216,17 @@ public class DMLTranslator throw new ParseException("Unsupported skip"); } - for(int i = skip; i < allExpr.length; i++) { - if(i == numImgIndex) { // skip=1 ==> i==5 and skip=2 => i==6 + for (int i = skip; i < allExpr.length; i++) { + if (i == numImgIndex) { // skip=1 ==> i==5 and skip=2 => i==6 Expression numImg = allExpr[numImgIndex]; - Expression numChannels = allExpr[numImgIndex+1]; - BinaryExpression tmp = new BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.MULT, - numImg.getFilename(), numImg.getBeginLine(), numImg.getBeginColumn(), numImg.getEndLine(), numImg.getEndColumn()); + Expression numChannels = allExpr[numImgIndex + 1]; + BinaryExpression tmp = new BinaryExpression(org.apache.sysml.parser.Expression.BinaryOp.MULT, numImg); tmp.setLeft(numImg); tmp.setRight(numChannels); ret.add(processTempIntExpression(tmp, hops)); - ret.add(processExpression(new IntIdentifier(1, numImg.getFilename(), numImg.getBeginLine(), numImg.getBeginColumn(), - numImg.getEndLine(), numImg.getEndColumn()), null, hops)); + ret.add(processExpression(new IntIdentifier(1, numImg), null, hops)); i++; - } - else + } else ret.add(processExpression(allExpr[i], null, hops)); } return ret; @@ -3339,22 +3322,24 @@ public class DMLTranslator && ((AssignmentStatement)s).getSource() instanceof DataExpression ) { DataExpression dexpr = (DataExpression) ((AssignmentStatement)s).getSource(); - if( dexpr.isRead() ){ + if (dexpr.isRead()) { String pfname = dexpr.getVarParam(DataExpression.IO_FILENAME).toString(); - if( pWrites.containsKey(pfname) && !pfname.trim().isEmpty() ) //found read-after-write - { - //update read with essential write meta data + // found read-after-write + if (pWrites.containsKey(pfname) && !pfname.trim().isEmpty()) { + // update read with essential write meta data DataIdentifier di = pWrites.get(pfname); - FormatType ft = (di.getFormatType()!=null) ? di.getFormatType() : FormatType.TEXT; - dexpr.addVarParam(DataExpression.FORMAT_TYPE, new StringIdentifier(ft.toString(),di.getFilename(),di.getBeginLine(),di.getBeginColumn(),di.getEndLine(),di.getEndColumn())); - if( di.getDim1()>=0 ) - dexpr.addVarParam(DataExpression.READROWPARAM, new IntIdentifier(di.getDim1(),di.getFilename(),di.getBeginLine(),di.getBeginColumn(),di.getEndLine(),di.getEndColumn())); - if( di.getDim2()>=0 ) - dexpr.addVarParam(DataExpression.READCOLPARAM, new IntIdentifier(di.getDim2(),di.getFilename(),di.getBeginLine(),di.getBeginColumn(),di.getEndLine(),di.getEndColumn())); - if( di.getValueType()!=ValueType.UNKNOWN ) - dexpr.addVarParam(DataExpression.VALUETYPEPARAM, new StringIdentifier(di.getValueType().toString(),di.getFilename(),di.getBeginLine(),di.getBeginColumn(),di.getEndLine(),di.getEndColumn())); - if( di.getDataType()!=DataType.UNKNOWN ) - dexpr.addVarParam(DataExpression.DATATYPEPARAM, new StringIdentifier(di.getDataType().toString(),di.getFilename(),di.getBeginLine(),di.getBeginColumn(),di.getEndLine(),di.getEndColumn())); + FormatType ft = (di.getFormatType() != null) ? di.getFormatType() : FormatType.TEXT; + dexpr.addVarParam(DataExpression.FORMAT_TYPE, new StringIdentifier(ft.toString(), di)); + if (di.getDim1() >= 0) + dexpr.addVarParam(DataExpression.READROWPARAM, new IntIdentifier(di.getDim1(), di)); + if (di.getDim2() >= 0) + dexpr.addVarParam(DataExpression.READCOLPARAM, new IntIdentifier(di.getDim2(), di)); + if (di.getValueType() != ValueType.UNKNOWN) + dexpr.addVarParam(DataExpression.VALUETYPEPARAM, + new StringIdentifier(di.getValueType().toString(), di)); + if (di.getDataType() != DataType.UNKNOWN) + dexpr.addVarParam(DataExpression.DATATYPEPARAM, + new StringIdentifier(di.getDataType().toString(), di)); ret = true; } }
