[SYSTEMML-1326] Cleanup hop rewrites (removed redundancy, minor fixes) This patch removes redundancy from existing hop rewrites by consolidating common primitives into HopRewriteUtils in order to avoid subtle bugs such as missing size propagation, missing line numbers, and missing blocking configurations.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/1fe1a02d Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/1fe1a02d Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/1fe1a02d Branch: refs/heads/master Commit: 1fe1a02d210356207d75cc3ffc2f246cd4a8d11b Parents: 4316efe Author: Matthias Boehm <[email protected]> Authored: Wed Feb 22 18:23:12 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Fri Feb 24 12:27:26 2017 -0800 ---------------------------------------------------------------------- .../java/org/apache/sysml/hops/AggBinaryOp.java | 68 +-- .../sysml/hops/ParameterizedBuiltinOp.java | 45 +- .../java/org/apache/sysml/hops/ReorgOp.java | 4 +- .../sysml/hops/globalopt/gdfgraph/GDFNode.java | 5 +- .../sysml/hops/rewrite/HopRewriteUtils.java | 179 +++--- .../RewriteAlgebraicSimplificationDynamic.java | 596 ++++++------------- .../RewriteAlgebraicSimplificationStatic.java | 416 ++++--------- .../hops/rewrite/RewriteConstantFolding.java | 13 +- .../rewrite/RewriteForLoopVectorization.java | 45 +- .../rewrite/RewriteIndexingVectorization.java | 18 +- .../RewriteMatrixMultChainOptimization.java | 4 +- .../rewrite/RewriteRemoveReadAfterWrite.java | 7 +- .../rewrite/RewriteRemoveUnnecessaryCasts.java | 7 +- .../RewriteSplitDagDataDependentOperators.java | 19 +- .../rewrite/RewriteSplitDagUnknownCSVRead.java | 4 +- .../org/apache/sysml/parser/DMLTranslator.java | 4 +- .../parfor/opt/OptimizerRuleBased.java | 5 +- 17 files changed, 489 insertions(+), 950 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1fe1a02d/src/main/java/org/apache/sysml/hops/AggBinaryOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java index 73dd8a4..dd9182d 100644 --- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java +++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java @@ -191,7 +191,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop else if( et == ExecType.SPARK ) { //matrix mult operation selection part 3 (SPARK type) - boolean tmmRewrite = input1 instanceof ReorgOp && ((ReorgOp)input1).getOp()==ReOrgOp.TRANSPOSE; + boolean tmmRewrite = HopRewriteUtils.isTransposeOperation(input1); _method = optFindMMultMethodSpark ( input1.getDim1(), input1.getDim2(), input1.getRowsInBlock(), input1.getColsInBlock(), input1.getNnz(), input2.getDim1(), input2.getDim2(), input2.getRowsInBlock(), input2.getColsInBlock(), input2.getNnz(), @@ -459,7 +459,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop { int index = left ? 0 : 1; return !(getInput().get(index) instanceof DataOp && ((DataOp)getInput().get(index)).requiresCheckpoint()) - && !(getInput().get(index) instanceof ReorgOp && ((ReorgOp)getInput().get(index)).getOp()==ReOrgOp.TRANSPOSE) + && !HopRewriteUtils.isTransposeOperation(getInput().get(index)) && getInput().get(index).getParent().size()==1 //bagg is only parent && !getInput().get(index).areDimsBelowThreshold() && getInput().get(index).optFindExecType() == ExecType.SPARK @@ -479,15 +479,13 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop Hop in1 = getInput().get(0); Hop in2 = getInput().get(1); - if( in1 instanceof ReorgOp - && ((ReorgOp)in1).getOp() == ReOrgOp.TRANSPOSE + if( HopRewriteUtils.isTransposeOperation(in1) && in1.getInput().get(0) == in2 ) { ret = MMTSJType.LEFT; } - if( in2 instanceof ReorgOp - && ((ReorgOp)in2).getOp() == ReOrgOp.TRANSPOSE + if( HopRewriteUtils.isTransposeOperation(in2) && in2.getInput().get(0) == in1 ) { ret = MMTSJType.RIGHT; @@ -510,7 +508,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop Hop in2 = getInput().get(1); //check for transpose left input (both chain types) - if( in1 instanceof ReorgOp && ((ReorgOp)in1).getOp() == ReOrgOp.TRANSPOSE ) + if( HopRewriteUtils.isTransposeOperation(in1) ) { Hop X = in1.getInput().get(0); @@ -615,7 +613,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop Hop rightInput = getInput().get(1); Hop nrow = HopRewriteUtils.createValueHop(pmInput, true); //NROW - HopRewriteUtils.setOutputBlocksizes(nrow, 0, 0); + nrow.setOutputBlocksizes(0, 0); nrow.setForcedExecType(ExecType.CP); HopRewriteUtils.copyLineNumbers(this, nrow); Lop lnrow = nrow.constructLops(); @@ -644,7 +642,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop Hop h2 = getInput().get(1); Lop left; Lop right; boolean isLeftTransposed; boolean isRightTransposed; - if( h1 instanceof ReorgOp && ((ReorgOp)h1).getOp()==ReOrgOp.TRANSPOSE ) { + if( HopRewriteUtils.isTransposeOperation(h1) ) { isLeftTransposed = true; left = h1.getInput().get(0).constructLops(); } @@ -652,7 +650,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop isLeftTransposed = false; left = h1.constructLops(); } - if( h2 instanceof ReorgOp && ((ReorgOp)h2).getOp()==ReOrgOp.TRANSPOSE ) { + if( HopRewriteUtils.isTransposeOperation(h2) ) { isRightTransposed = true; right = h2.getInput().get(0).constructLops(); } @@ -872,8 +870,6 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop Hop pmInput = getInput().get(0); Hop rightInput = getInput().get(1); - long brlen = pmInput.getRowsInBlock(); - long bclen = pmInput.getColsInBlock(); Lop lpmInput = pmInput.constructLops(); Hop nrow = null; @@ -887,30 +883,19 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop //v = rowMaxIndex(t(pm)) * rowMax(t(pm)) ReorgOp transpose = HopRewriteUtils.createTranspose(pmInput); transpose.setForcedExecType(ExecType.SPARK); - HopRewriteUtils.copyLineNumbers(this, transpose); - AggUnaryOp agg1 = new AggUnaryOp("tmp2a", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAXINDEX, Direction.Row, transpose); - HopRewriteUtils.setOutputBlocksizes(agg1, brlen, bclen); - agg1.refreshSizeInformation(); + AggUnaryOp agg1 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAXINDEX, Direction.Row); agg1.setForcedExecType(ExecType.SPARK); - HopRewriteUtils.copyLineNumbers(this, agg1); - AggUnaryOp agg2 = new AggUnaryOp("tmp2b", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAX, Direction.Row, transpose); - HopRewriteUtils.setOutputBlocksizes(agg2, brlen, bclen); - agg2.refreshSizeInformation(); + AggUnaryOp agg2 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAX, Direction.Row); agg2.setForcedExecType(ExecType.SPARK); - HopRewriteUtils.copyLineNumbers(this, agg2); - BinaryOp mult = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, agg1, agg2); - HopRewriteUtils.setOutputBlocksizes(mult, brlen, bclen); - mult.refreshSizeInformation(); + BinaryOp mult = HopRewriteUtils.createBinary(agg1, agg2, OpOp2.MULT); mult.setForcedExecType(ExecType.SPARK); - //mult.computeMemEstimate(memo); //select exec type - HopRewriteUtils.copyLineNumbers(this, mult); //compute NROW target via nrow(m) nrow = HopRewriteUtils.createValueHop(pmInput, true); - HopRewriteUtils.setOutputBlocksizes(nrow, 0, 0); + nrow.setOutputBlocksizes(0, 0); nrow.setForcedExecType(ExecType.CP); HopRewriteUtils.copyLineNumbers(this, nrow); @@ -921,7 +906,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop { //compute NROW target via max(v) nrow = HopRewriteUtils.createAggUnaryOp(pmInput, AggOp.MAX, Direction.RowCol); - HopRewriteUtils.setOutputBlocksizes(nrow, 0, 0); + nrow.setOutputBlocksizes(0, 0); nrow.setForcedExecType(etVect); HopRewriteUtils.copyLineNumbers(this, nrow); } @@ -1239,8 +1224,6 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop Hop pmInput = getInput().get(0); Hop rightInput = getInput().get(1); - long brlen = pmInput.getRowsInBlock(); - long bclen = pmInput.getColsInBlock(); Lop lpmInput = pmInput.constructLops(); Hop nrow = null; @@ -1254,29 +1237,19 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop //v = rowMaxIndex(t(pm)) * rowMax(t(pm)) ReorgOp transpose = HopRewriteUtils.createTranspose(pmInput); transpose.setForcedExecType(ExecType.MR); - HopRewriteUtils.copyLineNumbers(this, transpose); - AggUnaryOp agg1 = new AggUnaryOp("tmp2a", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAXINDEX, Direction.Row, transpose); - HopRewriteUtils.setOutputBlocksizes(agg1, brlen, bclen); - agg1.refreshSizeInformation(); + AggUnaryOp agg1 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAXINDEX, Direction.Row); agg1.setForcedExecType(ExecType.MR); - HopRewriteUtils.copyLineNumbers(this, agg1); - AggUnaryOp agg2 = new AggUnaryOp("tmp2b", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAX, Direction.Row, transpose); - HopRewriteUtils.setOutputBlocksizes(agg2, brlen, bclen); - agg2.refreshSizeInformation(); + AggUnaryOp agg2 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAX, Direction.Row); agg2.setForcedExecType(ExecType.MR); - HopRewriteUtils.copyLineNumbers(this, agg2); - BinaryOp mult = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, agg1, agg2); - HopRewriteUtils.setOutputBlocksizes(mult, brlen, bclen); - mult.refreshSizeInformation(); + BinaryOp mult = HopRewriteUtils.createBinary(agg1, agg2, OpOp2.MULT); mult.setForcedExecType(ExecType.MR); - HopRewriteUtils.copyLineNumbers(this, mult); //compute NROW target via nrow(m) nrow = HopRewriteUtils.createValueHop(pmInput, true); - HopRewriteUtils.setOutputBlocksizes(nrow, 0, 0); + nrow.setOutputBlocksizes(0, 0); nrow.setForcedExecType(ExecType.CP); HopRewriteUtils.copyLineNumbers(this, nrow); @@ -1287,7 +1260,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop { //compute NROW target via max(v) nrow = HopRewriteUtils.createAggUnaryOp(pmInput, AggOp.MAX, Direction.RowCol); - HopRewriteUtils.setOutputBlocksizes(nrow, 0, 0); + nrow.setOutputBlocksizes(0, 0); nrow.setForcedExecType(etVect); HopRewriteUtils.copyLineNumbers(this, nrow); } @@ -1345,7 +1318,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop //even a ba in CP does not imply that both transposes can be executed in CP) if( CP ) //in-memory ba { - if( h1 instanceof ReorgOp && ((ReorgOp)h1).getOp()==ReOrgOp.TRANSPOSE ) + if( HopRewriteUtils.isTransposeOperation(h1) ) { long m = h1.getDim1(); long cd = h1.getDim2(); @@ -1861,8 +1834,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop Hop input1 = getInput().get(0); Hop input2 = getInput().get(1); - if( isMatrixMultiply() ) - { + if( isMatrixMultiply() ) { setDim1(input1.getDim1()); setDim2(input2.getDim2()); } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1fe1a02d/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java index e2ec190..72e9115 100644 --- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java +++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java @@ -512,11 +512,11 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop //step1: compute index vectors Hop ppred0 = input; if( !isPPredInput ) { //ppred only if required - ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp(0)); + ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL); HopRewriteUtils.updateHopCharacteristics(ppred0, brlen, bclen, memo, this); } - UnaryOp cumsum = new UnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, ppred0); + UnaryOp cumsum = HopRewriteUtils.createUnary(ppred0, OpOp1.CUMSUM); HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, memo, this); Lop loutput = null; @@ -524,14 +524,14 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop double mbudget = OptimizerUtils.getRemoteMemBudgetMap(true); if( _outputPermutationMatrix && mest < mbudget ) //SPECIAL CASE: SELECTION VECTOR { - BinaryOp sel = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, ppred0, cumsum); + BinaryOp sel = HopRewriteUtils.createBinary(ppred0, cumsum, OpOp2.MULT); HopRewriteUtils.updateHopCharacteristics(sel, brlen, bclen, memo, this); loutput = sel.constructLops(); } else //GENERAL CASE: GENERAL PERMUTATION MATRIX { //max ensures non-zero entries and at least one output row - BinaryOp max = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MAX, cumsum, new LiteralOp(1)); + BinaryOp max = HopRewriteUtils.createBinary(cumsum, new LiteralOp(1), OpOp2.MAX); HopRewriteUtils.updateHopCharacteristics(max, brlen, bclen, memo, this); DataGenOp seq = HopRewriteUtils.createSeqDataGenOp(input); @@ -541,7 +541,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop //step 2: compute removeEmpty(rows) output via table, seq guarantees right column dimension //note: weights always the input (even if isPPredInput) because input also includes 0s TernaryOp table = new TernaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp3.CTABLE, max, seq, input); - HopRewriteUtils.setOutputBlocksizes(table, brlen, bclen); + table.setOutputBlocksizes(brlen, bclen); table.refreshSizeInformation(); table.setForcedExecType(ExecType.MR); //force MR HopRewriteUtils.copyLineNumbers(this, table); @@ -581,23 +581,18 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop if(selectHop == null) { //Step1: compute row/col non-empty indicators - ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp(0)); - HopRewriteUtils.setOutputBlocksizes(ppred0, brlen, bclen); - ppred0.refreshSizeInformation(); + ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL); ppred0.setForcedExecType(ExecType.MR); //always MR - HopRewriteUtils.copyLineNumbers(this, ppred0); emptyInd = ppred0; if( !((rmRows && clen == 1) || (!rmRows && rlen==1)) ){ - emptyInd = new AggUnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAX, rmRows?Direction.Row:Direction.Col, ppred0); - HopRewriteUtils.setOutputBlocksizes(emptyInd, brlen, bclen); - emptyInd.refreshSizeInformation(); + emptyInd = HopRewriteUtils.createAggUnaryOp(ppred0, AggOp.MAX, rmRows?Direction.Row:Direction.Col); emptyInd.setForcedExecType(ExecType.MR); //always MR HopRewriteUtils.copyLineNumbers(this, emptyInd); } } else { emptyInd = selectHop; - HopRewriteUtils.setOutputBlocksizes(emptyInd, brlen, bclen); + emptyInd.setOutputBlocksizes(brlen, bclen); emptyInd.refreshSizeInformation(); emptyInd.setForcedExecType(ExecType.MR); //always MR HopRewriteUtils.copyLineNumbers(this, emptyInd); @@ -610,7 +605,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this); } - UnaryOp cumsum = new UnaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, cumsumInput); + UnaryOp cumsum = HopRewriteUtils.createUnary(cumsumInput, OpOp1.CUMSUM); HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, this); Hop cumsumOutput = cumsum; @@ -619,10 +614,10 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop HopRewriteUtils.updateHopCharacteristics(cumsumOutput, brlen, bclen, this); } - Hop maxDim = new AggUnaryOp("tmp4", DataType.SCALAR, ValueType.DOUBLE, AggOp.MAX, Direction.RowCol, cumsumOutput); //alternative: right indexing + Hop maxDim = HopRewriteUtils.createAggUnaryOp(cumsumOutput, AggOp.MAX, Direction.RowCol); //alternative: right indexing HopRewriteUtils.updateHopCharacteristics(maxDim, brlen, bclen, this); - BinaryOp offsets = new BinaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, cumsumOutput, emptyInd); + BinaryOp offsets = HopRewriteUtils.createBinary(cumsumOutput, emptyInd, OpOp2.MULT); HopRewriteUtils.updateHopCharacteristics(offsets, brlen, bclen, this); //Step 3: gather non-empty rows/cols into final results @@ -713,23 +708,17 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop if(selectHop == null) { //Step1: compute row/col non-empty indicators - ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp(0)); - HopRewriteUtils.setOutputBlocksizes(ppred0, brlen, bclen); - ppred0.refreshSizeInformation(); + ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL); ppred0.setForcedExecType(ExecType.SPARK); //always Spark - HopRewriteUtils.copyLineNumbers(this, ppred0); emptyInd = ppred0; if( !((rmRows && clen == 1) || (!rmRows && rlen==1)) ){ - emptyInd = new AggUnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, AggOp.MAX, rmRows?Direction.Row:Direction.Col, ppred0); - HopRewriteUtils.setOutputBlocksizes(emptyInd, brlen, bclen); - emptyInd.refreshSizeInformation(); + emptyInd = HopRewriteUtils.createAggUnaryOp(ppred0, AggOp.MAX, rmRows?Direction.Row:Direction.Col); emptyInd.setForcedExecType(ExecType.SPARK); //always Spark - HopRewriteUtils.copyLineNumbers(this, emptyInd); } } else { emptyInd = selectHop; - HopRewriteUtils.setOutputBlocksizes(emptyInd, brlen, bclen); + emptyInd.setOutputBlocksizes(brlen, bclen); emptyInd.refreshSizeInformation(); emptyInd.setForcedExecType(ExecType.SPARK); //always Spark HopRewriteUtils.copyLineNumbers(this, emptyInd); @@ -742,7 +731,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this); } - UnaryOp cumsum = new UnaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, cumsumInput); + UnaryOp cumsum = HopRewriteUtils.createUnary(cumsumInput, OpOp1.CUMSUM); HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, this); Hop cumsumOutput = cumsum; @@ -751,10 +740,10 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop HopRewriteUtils.updateHopCharacteristics(cumsumOutput, brlen, bclen, this); } - Hop maxDim = new AggUnaryOp("tmp4", DataType.SCALAR, ValueType.DOUBLE, AggOp.MAX, Direction.RowCol, cumsumOutput); //alternative: right indexing + Hop maxDim = HopRewriteUtils.createAggUnaryOp(cumsumOutput, AggOp.MAX, Direction.RowCol); //alternative: right indexing HopRewriteUtils.updateHopCharacteristics(maxDim, brlen, bclen, this); - BinaryOp offsets = new BinaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, cumsumOutput, emptyInd); + BinaryOp offsets = HopRewriteUtils.createBinary(cumsumOutput, emptyInd, OpOp2.MULT); HopRewriteUtils.updateHopCharacteristics(offsets, brlen, bclen, this); //Step 3: gather non-empty rows/cols into final results http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1fe1a02d/src/main/java/org/apache/sysml/hops/ReorgOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ReorgOp.java b/src/main/java/org/apache/sysml/hops/ReorgOp.java index abe03a8..8d0b4b4 100644 --- a/src/main/java/org/apache/sysml/hops/ReorgOp.java +++ b/src/main/java/org/apache/sysml/hops/ReorgOp.java @@ -256,7 +256,7 @@ public class ReorgOp extends Hop implements MultiThreadedHop vinput = new IndexingOp("tmp1", getDataType(), getValueType(), input, new LiteralOp(1L), HopRewriteUtils.createValueHop(input, true), by, by, false, true); vinput.refreshSizeInformation(); - HopRewriteUtils.setOutputBlocksizes(vinput, getRowsInBlock(), getColsInBlock()); + vinput.setOutputBlocksizes(getRowsInBlock(), getColsInBlock()); HopRewriteUtils.copyLineNumbers(this, vinput); } @@ -314,7 +314,7 @@ public class ReorgOp extends Hop implements MultiThreadedHop //generate table TernaryOp table = new TernaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp3.CTABLE, seq, voutput, new LiteralOp(1L) ); - HopRewriteUtils.setOutputBlocksizes(table, getRowsInBlock(), getColsInBlock()); + table.setOutputBlocksizes(getRowsInBlock(), getColsInBlock()); table.refreshSizeInformation(); table.setForcedExecType(ExecType.MR); //force MR HopRewriteUtils.copyLineNumbers(this, table); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1fe1a02d/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java index e87a911..e385a86 100644 --- a/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java +++ b/src/main/java/org/apache/sysml/hops/globalopt/gdfgraph/GDFNode.java @@ -28,9 +28,8 @@ import org.apache.sysml.hops.Hop.DataGenMethod; import org.apache.sysml.hops.Hop.Direction; import org.apache.sysml.hops.Hop.FileFormatTypes; import org.apache.sysml.hops.Hop.OpOp1; -import org.apache.sysml.hops.Hop.ReOrgOp; -import org.apache.sysml.hops.ReorgOp; import org.apache.sysml.hops.UnaryOp; +import org.apache.sysml.hops.rewrite.HopRewriteUtils; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.runtime.controlprogram.Program; import org.apache.sysml.runtime.controlprogram.ProgramBlock; @@ -149,7 +148,7 @@ public class GDFNode { return ( _hop instanceof UnaryOp && format!=FileFormatTypes.CSV || (_hop instanceof AggUnaryOp && ((AggUnaryOp)_hop).getDirection()==Direction.RowCol && format!=FileFormatTypes.CSV) - || (_hop instanceof ReorgOp && ((ReorgOp)_hop).getOp()==ReOrgOp.TRANSPOSE && format!=FileFormatTypes.CSV) + || (HopRewriteUtils.isTransposeOperation(_hop) && format!=FileFormatTypes.CSV) || format==FileFormatTypes.BINARY ); //any op } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/1fe1a02d/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java index d3be09d..7f65ddd 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java @@ -24,6 +24,7 @@ import java.util.HashMap; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.hops.AggBinaryOp; import org.apache.sysml.hops.AggUnaryOp; import org.apache.sysml.hops.BinaryOp; @@ -222,16 +223,12 @@ public class HopRewriteUtils return childs.indexOf(child); } - public static void removeChildReference( Hop parent, Hop child ) - { - //remove child reference + public static void removeChildReference( Hop parent, Hop child ) { parent.getInput().remove( child ); child.getParent().remove( parent ); } - public static void removeChildReferenceByPos( Hop parent, Hop child, int posChild ) - { - //remove child reference + public static void removeChildReferenceByPos( Hop parent, Hop child, int posChild ) { parent.getInput().remove( posChild ); child.getParent().remove( parent ); } @@ -246,18 +243,35 @@ public class HopRewriteUtils parent.getInput().clear(); } - public static void addChildReference( Hop parent, Hop child ) - { + public static void addChildReference( Hop parent, Hop child ) { parent.getInput().add( child ); child.getParent().add( parent ); } - public static void addChildReference( Hop parent, Hop child, int pos ) - { + public static void addChildReference( Hop parent, Hop child, int pos ){ parent.getInput().add( pos, child ); child.getParent().add( parent ); } + public static void replaceChildReference( Hop parent, Hop inOld, Hop inNew ) { + int pos = getChildReferencePos(parent, inOld); + removeChildReferenceByPos(parent, inOld, pos); + addChildReference(parent, inNew, pos); + parent.refreshSizeInformation(); + } + + public static void replaceChildReference( Hop parent, Hop inOld, Hop inNew, int pos ) { + removeChildReferenceByPos(parent, inOld, pos); + addChildReference(parent, inNew, pos); + parent.refreshSizeInformation(); + } + + public static void cleanupUnreferenced( Hop... inputs ) { + for( Hop input : inputs ) + if( input.getParent().isEmpty() ) + removeAllChildReferences(input); + } + public static Hop createDataGenOp( Hop input, double value ) throws HopsException { @@ -279,8 +293,8 @@ public class HopRewriteUtils //note internal refresh size information Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params); - datagen.setRowsInBlock(input.getRowsInBlock()); - datagen.setColsInBlock(input.getColsInBlock()); + datagen.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock()); + copyLineNumbers(input, datagen); if( value==0 ) datagen.setNnz(0); @@ -335,12 +349,11 @@ public class HopRewriteUtils //note internal refresh size information DataGenOp datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params2); - datagen.setRowsInBlock(inputGen.getRowsInBlock()); - datagen.setColsInBlock(inputGen.getColsInBlock()); + datagen.setOutputBlocksizes(inputGen.getRowsInBlock(), inputGen.getColsInBlock()); + copyLineNumbers(inputGen, datagen); - if( smin==0 && smax==0 ) { + if( smin==0 && smax==0 ) datagen.setNnz(0); - } return datagen; } @@ -366,8 +379,8 @@ public class HopRewriteUtils //note internal refresh size information Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params); - datagen.setRowsInBlock(rowInput.getRowsInBlock()); - datagen.setColsInBlock(colInput.getColsInBlock()); + datagen.setOutputBlocksizes(rowInput.getRowsInBlock(), colInput.getColsInBlock()); + copyLineNumbers(rowInput, datagen); if( value==0 ) datagen.setNnz(0); @@ -399,8 +412,8 @@ public class HopRewriteUtils //note internal refresh size information Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params); - datagen.setRowsInBlock(rowInput.getRowsInBlock()); - datagen.setColsInBlock(colInput.getColsInBlock()); + datagen.setOutputBlocksizes(rowInput.getRowsInBlock(), colInput.getColsInBlock()); + copyLineNumbers(rowInput, datagen); if( value==0 ) datagen.setNnz(0); @@ -425,8 +438,8 @@ public class HopRewriteUtils //note internal refresh size information Hop datagen = new DataGenOp(DataGenMethod.RAND, new DataIdentifier("tmp"), params); - datagen.setRowsInBlock(rowInput.getRowsInBlock()); - datagen.setColsInBlock(colInput.getColsInBlock()); + datagen.setOutputBlocksizes(rowInput.getRowsInBlock(), colInput.getColsInBlock()); + copyLineNumbers(rowInput, datagen); if( value==0 ) datagen.setNnz(0); @@ -441,8 +454,8 @@ public class HopRewriteUtils public static ReorgOp createReorg(Hop input, ReOrgOp rop) { ReorgOp transpose = new ReorgOp(input.getName(), input.getDataType(), input.getValueType(), rop, input); - HopRewriteUtils.setOutputBlocksizes(transpose, input.getRowsInBlock(), input.getColsInBlock()); - HopRewriteUtils.copyLineNumbers(input, transpose); + transpose.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock()); + copyLineNumbers(input, transpose); transpose.refreshSizeInformation(); return transpose; @@ -451,31 +464,35 @@ public class HopRewriteUtils public static UnaryOp createUnary(Hop input, OpOp1 type) throws HopsException { - DataType dt = (type==OpOp1.CAST_AS_SCALAR) ? DataType.SCALAR : input.getDataType(); - UnaryOp unary = new UnaryOp(input.getName(), dt, input.getValueType(), type, input); - HopRewriteUtils.setOutputBlocksizes(unary, input.getRowsInBlock(), input.getColsInBlock()); - HopRewriteUtils.copyLineNumbers(input, unary); + DataType dt = (type==OpOp1.CAST_AS_SCALAR) ? DataType.SCALAR : + (type==OpOp1.CAST_AS_MATRIX) ? DataType.MATRIX : input.getDataType(); + ValueType vt = (type==OpOp1.CAST_AS_MATRIX) ? ValueType.DOUBLE : input.getValueType(); + UnaryOp unary = new UnaryOp(input.getName(), dt, vt, type, input); + unary.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock()); + if( type == OpOp1.CAST_AS_SCALAR || type == OpOp1.CAST_AS_MATRIX ) { + int dim = (type==OpOp1.CAST_AS_SCALAR) ? 0 : 1; + int blksz = (type==OpOp1.CAST_AS_SCALAR) ? 0 : ConfigurationManager.getBlocksize(); + setOutputParameters(unary, dim, dim, blksz, blksz, -1); + } + + copyLineNumbers(input, unary); unary.refreshSizeInformation(); return unary; } - public static BinaryOp createMinus(Hop input) - { - BinaryOp minus = new BinaryOp(input.getName(), input.getDataType(), input.getValueType(), OpOp2.MINUS, new LiteralOp(0), input); - HopRewriteUtils.setOutputBlocksizes(minus, input.getRowsInBlock(), input.getColsInBlock()); - HopRewriteUtils.copyLineNumbers(input, minus); - minus.refreshSizeInformation(); - - return minus; + public static BinaryOp createBinaryMinus(Hop input) { + return createBinary(new LiteralOp(0), input, OpOp2.MINUS); } public static BinaryOp createBinary(Hop input1, Hop input2, OpOp2 op) { - BinaryOp bop = new BinaryOp(input1.getName(), input1.getDataType(), - input1.getValueType(), op, input1, input2); - HopRewriteUtils.setOutputBlocksizes(bop, input1.getRowsInBlock(), input1.getColsInBlock()); - HopRewriteUtils.copyLineNumbers(input1, bop); + Hop mainInput = input1.getDataType().isMatrix() ? input1 : + input2.getDataType().isMatrix() ? input2 : input1; + BinaryOp bop = new BinaryOp(mainInput.getName(), mainInput.getDataType(), + mainInput.getValueType(), op, input1, input2); + bop.setOutputBlocksizes(mainInput.getRowsInBlock(), mainInput.getColsInBlock()); + copyLineNumbers(mainInput, bop); bop.refreshSizeInformation(); return bop; @@ -485,23 +502,20 @@ public class HopRewriteUtils return createAggUnaryOp(input, AggOp.SUM, Direction.RowCol); } - public static AggUnaryOp createAggUnaryOp( Hop input, AggOp op, Direction dir ) - { + public static AggUnaryOp createAggUnaryOp( Hop input, AggOp op, Direction dir ) { DataType dt = (dir==Direction.RowCol) ? DataType.SCALAR : input.getDataType(); - AggUnaryOp auop = new AggUnaryOp(input.getName(), dt, input.getValueType(), op, dir, input); - auop.setRowsInBlock(input.getRowsInBlock()); - auop.setColsInBlock(input.getColsInBlock()); + auop.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock()); + copyLineNumbers(input, auop); auop.refreshSizeInformation(); return auop; } - public static AggBinaryOp createMatrixMultiply(Hop left, Hop right) - { + public static AggBinaryOp createMatrixMultiply(Hop left, Hop right) { AggBinaryOp mmult = new AggBinaryOp(left.getName(), left.getDataType(), left.getValueType(), OpOp2.MULT, AggOp.SUM, left, right); - mmult.setRowsInBlock(left.getRowsInBlock()); - mmult.setColsInBlock(right.getColsInBlock()); + mmult.setOutputBlocksizes(left.getRowsInBlock(), right.getColsInBlock()); + copyLineNumbers(left, mmult); mmult.refreshSizeInformation(); return mmult; @@ -550,55 +564,42 @@ public class HopRewriteUtils //note internal refresh size information DataGenOp datagen = new DataGenOp(DataGenMethod.SEQ, new DataIdentifier("tmp"), params); - datagen.setRowsInBlock(input.getRowsInBlock()); - datagen.setColsInBlock(input.getColsInBlock()); + datagen.setOutputBlocksizes(input.getRowsInBlock(), input.getColsInBlock()); + copyLineNumbers(input, datagen); return datagen; } public static TernaryOp createTernaryOp(Hop mleft, Hop smid, Hop mright, OpOp3 op) { TernaryOp ternOp = new TernaryOp("tmp", DataType.MATRIX, ValueType.DOUBLE, op, mleft, smid, mright); - ternOp.setRowsInBlock(mleft.getRowsInBlock()); - ternOp.setColsInBlock(mleft.getColsInBlock()); + ternOp.setOutputBlocksizes(mleft.getRowsInBlock(), mleft.getColsInBlock()); + copyLineNumbers(mleft, ternOp); ternOp.refreshSizeInformation(); return ternOp; } - public static void setOutputBlocksizes( Hop hop, long brlen, long bclen ) - { - hop.setRowsInBlock( brlen ); - hop.setColsInBlock( bclen ); - } - - public static void setOutputParameters( Hop hop, long rlen, long clen, long brlen, long bclen, long nnz ) - { + public static void setOutputParameters( Hop hop, long rlen, long clen, long brlen, long bclen, long nnz ) { hop.setDim1( rlen ); hop.setDim2( clen ); - hop.setRowsInBlock( brlen ); - hop.setColsInBlock( bclen ); + hop.setOutputBlocksizes(brlen, bclen ); hop.setNnz( nnz ); } - public static void setOutputParametersForScalar( Hop hop ) - { + public static void setOutputParametersForScalar( Hop hop ) { hop.setDim1( 0 ); hop.setDim2( 0 ); - hop.setRowsInBlock( -1 ); - hop.setColsInBlock( -1 ); + hop.setOutputBlocksizes(-1, -1 ); hop.setNnz( -1 ); } - public static void refreshOutputParameters( Hop hnew, Hop hold ) - { + public static void refreshOutputParameters( Hop hnew, Hop hold ) { hnew.setDim1( hold.getDim1() ); hnew.setDim2( hold.getDim2() ); - hnew.setRowsInBlock(hold.getRowsInBlock()); - hnew.setColsInBlock(hold.getColsInBlock()); + hnew.setOutputBlocksizes(hold.getRowsInBlock(), hold.getColsInBlock()); hnew.refreshSizeInformation(); } - public static void copyLineNumbers( Hop src, Hop dest ) - { + public static void copyLineNumbers( Hop src, Hop dest ) { dest.setAllPositions(src.getBeginLine(), src.getBeginColumn(), src.getEndLine(), src.getEndColumn()); } @@ -610,7 +611,7 @@ public class HopRewriteUtils public static void updateHopCharacteristics( Hop hop, long brlen, long bclen, MemoTable memo, Hop src ) { //update block sizes and dimensions - setOutputBlocksizes(hop, brlen, bclen); + hop.setOutputBlocksizes(brlen, bclen); hop.refreshSizeInformation(); //compute memory estimates (for exec type selection) @@ -668,7 +669,7 @@ public class HopRewriteUtils } public static boolean isOuterProductLikeMM( Hop hop ) { - return hop instanceof AggBinaryOp + return isMatrixMultiply(hop) && hop.getInput().get(0).getDim1() > hop.getInput().get(0).getDim2() && hop.getInput().get(1).getDim1() < hop.getInput().get(1).getDim2(); } @@ -715,6 +716,10 @@ public class HopRewriteUtils return (hop instanceof ReorgOp && ((ReorgOp)hop).getOp()==ReOrgOp.TRANSPOSE); } + public static boolean isTransposeOperation(Hop hop, int maxParents) { + return isTransposeOperation(hop) && hop.getParent().size() <= maxParents; + } + public static boolean containsTransposeOperation(ArrayList<Hop> hops) { boolean ret = false; for( Hop hop : hops ) @@ -723,8 +728,16 @@ public class HopRewriteUtils } public static boolean isTransposeOfItself(Hop hop1, Hop hop2) { - return hop1 instanceof ReorgOp && ((ReorgOp)hop1).getOp()==ReOrgOp.TRANSPOSE && hop1.getInput().get(0) == hop2 - || hop2 instanceof ReorgOp && ((ReorgOp)hop2).getOp()==ReOrgOp.TRANSPOSE && hop2.getInput().get(0) == hop1; + return isTransposeOperation(hop1) && hop1.getInput().get(0) == hop2 + || isTransposeOperation(hop2) && hop2.getInput().get(0) == hop1; + } + + public static boolean isBinary(Hop hop, OpOp2 type) { + return hop instanceof BinaryOp && ((BinaryOp)hop).getOp()==type; + } + + public static boolean isBinary(Hop hop, OpOp2 type, int maxParents) { + return isBinary(hop, type) && hop.getParent().size() <= maxParents; } public static boolean isBinaryMatrixScalarOperation(Hop hop) { @@ -733,6 +746,18 @@ public class HopRewriteUtils ||(hop.getInput().get(1).getDataType().isMatrix() && hop.getInput().get(0).getDataType().isScalar())); } + public static boolean isUnary(Hop hop, OpOp1 type) { + return hop instanceof UnaryOp && ((UnaryOp)hop).getOp()==type; + } + + public static boolean isUnary(Hop hop, OpOp1 type, int maxParents) { + return isUnary(hop, type) && hop.getParent().size() <= maxParents; + } + + public static boolean isMatrixMultiply(Hop hop) { + return hop instanceof AggBinaryOp && ((AggBinaryOp)hop).isMatrixMultiply(); + } + public static boolean isNonZeroIndicator(Hop pred, Hop hop ) { if( pred instanceof BinaryOp && ((BinaryOp)pred).getOp()==OpOp2.NOTEQUAL
